iota_source_validation/
toolchain.rs

1// Copyright (c) Mysten Labs, Inc.
2// Modifications Copyright (c) 2024 IOTA Stiftung
3// SPDX-License-Identifier: Apache-2.0
4
5use std::{
6    collections::HashMap,
7    ffi::OsStr,
8    fs::File,
9    io::{self, Seek},
10    path::{Path, PathBuf},
11    process::Command,
12};
13
14use anyhow::{anyhow, bail, ensure};
15use colored::Colorize;
16use iota_sdk_types::Address;
17use move_binary_format::CompiledModule;
18use move_bytecode_source_map::utils::source_map_from_file;
19use move_command_line_common::{
20    env::MOVE_HOME,
21    files::{
22        DEBUG_INFO_EXTENSION, MOVE_COMPILED_EXTENSION, MOVE_EXTENSION, extension_equals,
23        find_filenames,
24    },
25};
26use move_compiler::{
27    compiled_unit::NamedCompiledModule,
28    editions::{Edition, Flavor},
29    shared::{NumericalAddress, files::FileName},
30};
31use move_package::{
32    compilation::{
33        compiled_package::CompiledUnitWithSource, package_layout::CompiledPackageLayout,
34    },
35    lock_file::schema::{Header, ToolchainVersion},
36    source_package::{layout::SourcePackageLayout, parsed_manifest::PackageName},
37};
38use move_symbol_pool::Symbol;
39use tar::Archive;
40use tempfile::TempDir;
41use tracing::{debug, info};
42
43pub(crate) const CURRENT_COMPILER_VERSION: &str = env!("CARGO_PKG_VERSION");
44const LEGACY_COMPILER_VERSION: &str = CURRENT_COMPILER_VERSION; // TODO: update this when Move 2024 is released
45const PRE_TOOLCHAIN_MOVE_LOCK_VERSION: u16 = 0; // Used to detect lockfiles pre-toolchain versioning support
46const CANONICAL_UNIX_BINARY_NAME: &str = "iota";
47const CANONICAL_WIN_BINARY_NAME: &str = "iota.exe";
48
49pub(crate) fn current_toolchain() -> ToolchainVersion {
50    ToolchainVersion {
51        compiler_version: CURRENT_COMPILER_VERSION.into(),
52        edition: Edition::LEGACY, // does not matter, unused for current_toolchain
53        flavor: Flavor::Iota,     // does not matter, unused for current_toolchain
54    }
55}
56
57pub(crate) fn legacy_toolchain() -> ToolchainVersion {
58    ToolchainVersion {
59        compiler_version: LEGACY_COMPILER_VERSION.into(),
60        edition: Edition::LEGACY,
61        flavor: Flavor::Iota,
62    }
63}
64
65/// Ensures `compiled_units` are compiled with the right compiler version, based
66/// on Move.lock contents. This works by detecting if a compiled unit requires a
67/// prior compiler version:
68/// - If so, download the compiler, recompile the unit, and return that unit in
69///   the result.
70/// - If not, simply keep the current compiled unit.
71pub(crate) fn units_for_toolchain(
72    compiled_units: &Vec<(PackageName, CompiledUnitWithSource)>,
73) -> anyhow::Result<Vec<(PackageName, CompiledUnitWithSource)>> {
74    if std::env::var("IOTA_RUN_TOOLCHAIN_BUILD").is_err() {
75        return Ok(compiled_units.clone());
76    }
77    let mut package_version_map: HashMap<Symbol, (ToolchainVersion, Vec<CompiledUnitWithSource>)> =
78        HashMap::new();
79    // First iterate over packages, mapping the required version for each package in
80    // `package_version_map`.
81    for (package, local_unit) in compiled_units {
82        if let Some((_, units)) = package_version_map.get_mut(package) {
83            // We've processed this package's required version.
84            units.push(local_unit.clone());
85            continue;
86        }
87
88        if Address::new(local_unit.unit.address.into_bytes()).is_system_package() {
89            // System packages are always compiled with the current compiler.
90            package_version_map.insert(*package, (current_toolchain(), vec![local_unit.clone()]));
91            continue;
92        }
93
94        let package_root = SourcePackageLayout::try_find_root(&local_unit.source_path)?;
95        let lock_file = package_root.join(SourcePackageLayout::Lock.path());
96        if !lock_file.exists() {
97            // No lock file implies current compiler for this package.
98            package_version_map.insert(*package, (current_toolchain(), vec![local_unit.clone()]));
99            continue;
100        }
101
102        let mut lock_file = File::open(lock_file)?;
103        let lock_version = Header::read(&mut lock_file)?.version;
104        if lock_version == PRE_TOOLCHAIN_MOVE_LOCK_VERSION {
105            // No need to attempt reading lock file toolchain
106            debug!("{package} on legacy compiler",);
107            package_version_map.insert(*package, (legacy_toolchain(), vec![local_unit.clone()]));
108            continue;
109        }
110
111        // Read lock file toolchain info
112        lock_file.rewind()?;
113        let toolchain_version = ToolchainVersion::read(&mut lock_file)?;
114        match toolchain_version {
115            // No ToolchainVersion and new Move.lock version implies current compiler.
116            None => {
117                debug!("{package} on current compiler @ {CURRENT_COMPILER_VERSION}",);
118                package_version_map
119                    .insert(*package, (current_toolchain(), vec![local_unit.clone()]));
120            }
121            // This dependency uses the current compiler.
122            Some(ToolchainVersion {
123                compiler_version, ..
124            }) if compiler_version == CURRENT_COMPILER_VERSION => {
125                debug!("{package} on current compiler @ {CURRENT_COMPILER_VERSION}",);
126                package_version_map
127                    .insert(*package, (current_toolchain(), vec![local_unit.clone()]));
128            }
129            // This dependency needs a prior compiler. Mark it and compile.
130            Some(toolchain_version) => {
131                println!(
132                    "{} {package} compiler @ {}",
133                    "REQUIRE".bold().green(),
134                    toolchain_version.compiler_version.yellow(),
135                );
136                package_version_map.insert(*package, (toolchain_version, vec![local_unit.clone()]));
137            }
138        }
139    }
140
141    let mut units = vec![];
142    // Iterate over compiled units, and check if they need to be recompiled and
143    // replaced by a prior compiler's output.
144    for (package, (toolchain_version, local_units)) in package_version_map {
145        if toolchain_version.compiler_version == CURRENT_COMPILER_VERSION {
146            let local_units: Vec<_> = local_units.iter().map(|u| (package, u.clone())).collect();
147            units.extend(local_units);
148            continue;
149        }
150
151        if local_units.is_empty() {
152            bail!("Expected one or more modules, but none found");
153        }
154        let package_root = SourcePackageLayout::try_find_root(&local_units[0].source_path)?;
155        let install_dir = tempfile::tempdir()?; // place compiled packages in this temp dir, don't pollute this packages build
156        // dir
157        download_and_compile(
158            package_root.clone(),
159            &install_dir,
160            &toolchain_version,
161            &package,
162        )?;
163
164        let compiled_unit_paths = vec![package_root.clone()];
165        let compiled_units = find_filenames(&compiled_unit_paths, |path| {
166            extension_equals(path, MOVE_COMPILED_EXTENSION)
167        })?;
168        let build_path = install_dir
169            .path()
170            .join(CompiledPackageLayout::path(&CompiledPackageLayout::Root))
171            .join(package.as_str());
172        debug!("build path is {}", build_path.display());
173
174        // Add all units compiled with the previous compiler.
175        for bytecode_path in compiled_units {
176            info!("bytecode path {bytecode_path}, {package}");
177            let local_unit = decode_bytecode_file(build_path.clone(), &package, &bytecode_path)?;
178            units.push((package, local_unit))
179        }
180    }
181    Ok(units)
182}
183
184fn download_and_compile(
185    root: PathBuf,
186    install_dir: &TempDir,
187    ToolchainVersion {
188        compiler_version,
189        edition,
190        flavor,
191    }: &ToolchainVersion,
192    dep_name: &Symbol,
193) -> anyhow::Result<()> {
194    let dest_dir = PathBuf::from_iter([&*MOVE_HOME, "binaries"]); // E.g., ~/.move/binaries
195    let dest_version = dest_dir.join(compiler_version);
196    let mut dest_canonical_path = dest_version.clone();
197    dest_canonical_path.extend(["target", "release"]);
198    let mut dest_canonical_binary = dest_canonical_path.clone();
199
200    let platform = detect_platform(&root, compiler_version, &dest_canonical_path)?;
201    if platform == "windows-x86_64" {
202        dest_canonical_binary.push(CANONICAL_WIN_BINARY_NAME);
203    } else {
204        dest_canonical_binary.push(CANONICAL_UNIX_BINARY_NAME);
205    }
206
207    if !dest_canonical_binary.exists() {
208        // Check the platform and proceed if we can download a binary. If not, the user
209        // should follow error instructions to sideload the binary. Download if
210        // binary does not exist.
211        let mainnet_url = format!(
212            "https://github.com/iotaledger/iota/releases/download/mainnet-v{compiler_version}/iota-mainnet-v{compiler_version}-{platform}.tgz",
213        );
214
215        println!(
216            "{} mainnet compiler @ {} (this may take a while)",
217            "DOWNLOADING".bold().green(),
218            compiler_version.yellow()
219        );
220
221        let mut response = match ureq::get(&mainnet_url).call() {
222            Ok(response) => response,
223            Err(ureq::Error::Status(404, _)) => {
224                println!(
225                    "{} iota mainnet compiler {} not available, attempting to download testnet compiler release...",
226                    "WARNING".bold().yellow(),
227                    compiler_version.yellow()
228                );
229                println!(
230                    "{} testnet compiler @ {} (this may take a while)",
231                    "DOWNLOADING".bold().green(),
232                    compiler_version.yellow()
233                );
234                let testnet_url = format!("https://github.com/iotaledger/iota/releases/download/testnet-v{compiler_version}/iota-testnet-v{compiler_version}-{platform}.tgz");
235                ureq::get(&testnet_url).call()?
236            }
237            Err(e) => return Err(e.into()),
238        }.into_reader();
239
240        let dest_tarball = dest_version.join(format!("{compiler_version}.tgz"));
241        debug!("tarball destination: {} ", dest_tarball.display());
242        if let Some(parent) = dest_tarball.parent() {
243            std::fs::create_dir_all(parent)
244                .map_err(|e| anyhow!("failed to create directory for tarball: {e}"))?;
245        }
246        let mut dest_file = File::create(&dest_tarball)?;
247        io::copy(&mut response, &mut dest_file)?;
248
249        // Extract the tarball using the tar crate
250        let tar_gz = File::open(&dest_tarball)?;
251        let tar = flate2::read::GzDecoder::new(tar_gz);
252        let mut archive = Archive::new(tar);
253        archive
254            .unpack(&dest_version)
255            .map_err(|e| anyhow!("failed to untar compiler binary: {e}"))?;
256
257        let mut dest_binary = dest_version.clone();
258        dest_binary.extend(["target", "release"]);
259        if platform == "windows-x86_64" {
260            dest_binary.push(format!("iota-{platform}.exe"));
261        } else {
262            dest_binary.push(format!("iota-{platform}"));
263        }
264        let dest_binary_os = OsStr::new(dest_binary.as_path());
265        set_executable_permission(dest_binary_os)?;
266        std::fs::rename(dest_binary_os, dest_canonical_binary.clone())?;
267    }
268
269    debug!(
270        "{} move build --default-move-edition {} --default-move-flavor {} -p {} --install-dir {}",
271        dest_canonical_binary.display(),
272        edition.to_string().as_str(),
273        flavor.to_string().as_str(),
274        root.display(),
275        install_dir.path().display(),
276    );
277    info!(
278        "{} {} (compiler @ {})",
279        "BUILDING".bold().green(),
280        dep_name.as_str(),
281        compiler_version.yellow()
282    );
283    Command::new(dest_canonical_binary)
284        .args([
285            OsStr::new("move"),
286            OsStr::new("build"),
287            OsStr::new("--default-move-edition"),
288            OsStr::new(edition.to_string().as_str()),
289            OsStr::new("--default-move-flavor"),
290            OsStr::new(flavor.to_string().as_str()),
291            OsStr::new("-p"),
292            OsStr::new(root.as_path()),
293            OsStr::new("--install-dir"),
294            OsStr::new(install_dir.path()),
295        ])
296        .output()
297        .map_err(|e| {
298            anyhow!("failed to build package from compiler binary {compiler_version}: {e}",)
299        })?;
300    Ok(())
301}
302
303fn detect_platform(
304    package_path: &Path,
305    compiler_version: &String,
306    dest_dir: &Path,
307) -> anyhow::Result<String> {
308    let s = match (std::env::consts::OS, std::env::consts::ARCH) {
309        ("macos", "aarch64") => "macos-arm64",
310        ("macos", "x86_64") => "macos-x86_64",
311        ("linux", "x86_64") => "ubuntu-x86_64",
312        ("windows", "x86_64") => "windows-x86_64",
313        (os, arch) => {
314            let mut binary_name = CANONICAL_UNIX_BINARY_NAME;
315            if os == "windows" {
316                binary_name = CANONICAL_WIN_BINARY_NAME;
317            };
318            bail!(
319                "The package {} needs to be built with iota compiler version {compiler_version} but there \
320                 is no binary release available to download for your platform:\n\
321                 Operating System: {os}\n\
322                 Architecture: {arch}\n\
323                 You can manually put a {binary_name} binary for your platform in {} and rerun your command to continue.",
324                package_path.display(),
325                dest_dir.display(),
326            )
327        }
328    };
329    Ok(s.into())
330}
331
332#[cfg(unix)]
333fn set_executable_permission(path: &OsStr) -> anyhow::Result<()> {
334    use std::{fs, os::unix::prelude::PermissionsExt};
335    let mut perms = fs::metadata(path)?.permissions();
336    perms.set_mode(0o755);
337    fs::set_permissions(path, perms)?;
338    Ok(())
339}
340
341#[cfg(not(unix))]
342fn set_executable_permission(path: &OsStr) -> anyhow::Result<()> {
343    Command::new("icacls")
344        .args([path, OsStr::new("/grant"), OsStr::new("Everyone:(RX)")])
345        .status()?;
346    Ok(())
347}
348
349fn decode_bytecode_file(
350    root_path: PathBuf,
351    package_name: &Symbol,
352    bytecode_path_str: &str,
353) -> anyhow::Result<CompiledUnitWithSource> {
354    let package_name_opt = Some(*package_name);
355    let bytecode_path = Path::new(bytecode_path_str);
356    let path_to_file = CompiledPackageLayout::path_to_file_after_category(bytecode_path);
357    let bytecode_bytes = std::fs::read(bytecode_path)?;
358    let source_map = source_map_from_file(
359        &root_path
360            .join(CompiledPackageLayout::DebugInfo.path())
361            .join(&path_to_file)
362            .with_extension(DEBUG_INFO_EXTENSION),
363    )?;
364    let source_path = &root_path
365        .join(CompiledPackageLayout::Sources.path())
366        .join(path_to_file)
367        .with_extension(MOVE_EXTENSION);
368    ensure!(
369        source_path.is_file(),
370        "Error decoding package: Unable to find corresponding source file for '{bytecode_path_str}' in package {package_name}"
371    );
372    let module = CompiledModule::deserialize_with_defaults(&bytecode_bytes)?;
373    let (address_bytes, module_name) = {
374        let id = module.self_id();
375        let parsed_addr = NumericalAddress::new(
376            id.address().into_bytes(),
377            move_compiler::shared::NumberFormat::Hex,
378        );
379        let module_name = FileName::from(id.name().as_str());
380        (parsed_addr, module_name)
381    };
382    let unit = NamedCompiledModule {
383        package_name: package_name_opt,
384        address: address_bytes,
385        name: module_name,
386        module,
387        source_map,
388        address_name: None,
389    };
390    Ok(CompiledUnitWithSource {
391        unit,
392        source_path: source_path.clone(),
393    })
394}
iota_source_validation/toolchain.rs

iota_source_validation/
toolchain.rs
