iota_source_validation/
toolchain.rs

1// Copyright (c) Mysten Labs, Inc.
2// Modifications Copyright (c) 2024 IOTA Stiftung
3// SPDX-License-Identifier: Apache-2.0
4
5use std::{
6    collections::HashMap,
7    ffi::OsStr,
8    fs::File,
9    io::{self, Seek},
10    path::{Path, PathBuf},
11    process::Command,
12};
13
14use anyhow::{anyhow, bail, ensure};
15use colored::Colorize;
16use move_binary_format::CompiledModule;
17use move_bytecode_source_map::utils::source_map_from_file;
18use move_command_line_common::{
19    env::MOVE_HOME,
20    files::{
21        MOVE_COMPILED_EXTENSION, MOVE_EXTENSION, SOURCE_MAP_EXTENSION, extension_equals,
22        find_filenames,
23    },
24};
25use move_compiler::{
26    compiled_unit::NamedCompiledModule,
27    editions::{Edition, Flavor},
28    shared::{NumericalAddress, files::FileName},
29};
30use move_package::{
31    compilation::{
32        compiled_package::CompiledUnitWithSource, package_layout::CompiledPackageLayout,
33    },
34    lock_file::schema::{Header, ToolchainVersion},
35    source_package::{layout::SourcePackageLayout, parsed_manifest::PackageName},
36};
37use move_symbol_pool::Symbol;
38use tar::Archive;
39use tempfile::TempDir;
40use tracing::{debug, info};
41
42pub(crate) const CURRENT_COMPILER_VERSION: &str = env!("CARGO_PKG_VERSION");
43const LEGACY_COMPILER_VERSION: &str = CURRENT_COMPILER_VERSION; // TODO: update this when Move 2024 is released
44const PRE_TOOLCHAIN_MOVE_LOCK_VERSION: u16 = 0; // Used to detect lockfiles pre-toolchain versioning support
45const CANONICAL_UNIX_BINARY_NAME: &str = "iota";
46const CANONICAL_WIN_BINARY_NAME: &str = "iota.exe";
47
48pub(crate) fn current_toolchain() -> ToolchainVersion {
49    ToolchainVersion {
50        compiler_version: CURRENT_COMPILER_VERSION.into(),
51        edition: Edition::LEGACY, // does not matter, unused for current_toolchain
52        flavor: Flavor::Iota,     // does not matter, unused for current_toolchain
53    }
54}
55
56pub(crate) fn legacy_toolchain() -> ToolchainVersion {
57    ToolchainVersion {
58        compiler_version: LEGACY_COMPILER_VERSION.into(),
59        edition: Edition::LEGACY,
60        flavor: Flavor::Iota,
61    }
62}
63
64/// Ensures `compiled_units` are compiled with the right compiler version, based
65/// on Move.lock contents. This works by detecting if a compiled unit requires a
66/// prior compiler version:
67/// - If so, download the compiler, recompile the unit, and return that unit in
68///   the result.
69/// - If not, simply keep the current compiled unit.
70pub(crate) fn units_for_toolchain(
71    compiled_units: &Vec<(PackageName, CompiledUnitWithSource)>,
72) -> anyhow::Result<Vec<(PackageName, CompiledUnitWithSource)>> {
73    if std::env::var("IOTA_RUN_TOOLCHAIN_BUILD").is_err() {
74        return Ok(compiled_units.clone());
75    }
76    let mut package_version_map: HashMap<Symbol, (ToolchainVersion, Vec<CompiledUnitWithSource>)> =
77        HashMap::new();
78    // First iterate over packages, mapping the required version for each package in
79    // `package_version_map`.
80    for (package, local_unit) in compiled_units {
81        if let Some((_, units)) = package_version_map.get_mut(package) {
82            // We've processed this package's required version.
83            units.push(local_unit.clone());
84            continue;
85        }
86
87        if iota_types::is_system_package(local_unit.unit.address.into_inner()) {
88            // System packages are always compiled with the current compiler.
89            package_version_map.insert(*package, (current_toolchain(), vec![local_unit.clone()]));
90            continue;
91        }
92
93        let package_root = SourcePackageLayout::try_find_root(&local_unit.source_path)?;
94        let lock_file = package_root.join(SourcePackageLayout::Lock.path());
95        if !lock_file.exists() {
96            // No lock file implies current compiler for this package.
97            package_version_map.insert(*package, (current_toolchain(), vec![local_unit.clone()]));
98            continue;
99        }
100
101        let mut lock_file = File::open(lock_file)?;
102        let lock_version = Header::read(&mut lock_file)?.version;
103        if lock_version == PRE_TOOLCHAIN_MOVE_LOCK_VERSION {
104            // No need to attempt reading lock file toolchain
105            debug!("{package} on legacy compiler",);
106            package_version_map.insert(*package, (legacy_toolchain(), vec![local_unit.clone()]));
107            continue;
108        }
109
110        // Read lock file toolchain info
111        lock_file.rewind()?;
112        let toolchain_version = ToolchainVersion::read(&mut lock_file)?;
113        match toolchain_version {
114            // No ToolchainVersion and new Move.lock version implies current compiler.
115            None => {
116                debug!("{package} on current compiler @ {CURRENT_COMPILER_VERSION}",);
117                package_version_map
118                    .insert(*package, (current_toolchain(), vec![local_unit.clone()]));
119            }
120            // This dependency uses the current compiler.
121            Some(ToolchainVersion {
122                compiler_version, ..
123            }) if compiler_version == CURRENT_COMPILER_VERSION => {
124                debug!("{package} on current compiler @ {CURRENT_COMPILER_VERSION}",);
125                package_version_map
126                    .insert(*package, (current_toolchain(), vec![local_unit.clone()]));
127            }
128            // This dependency needs a prior compiler. Mark it and compile.
129            Some(toolchain_version) => {
130                println!(
131                    "{} {package} compiler @ {}",
132                    "REQUIRE".bold().green(),
133                    toolchain_version.compiler_version.yellow(),
134                );
135                package_version_map.insert(*package, (toolchain_version, vec![local_unit.clone()]));
136            }
137        }
138    }
139
140    let mut units = vec![];
141    // Iterate over compiled units, and check if they need to be recompiled and
142    // replaced by a prior compiler's output.
143    for (package, (toolchain_version, local_units)) in package_version_map {
144        if toolchain_version.compiler_version == CURRENT_COMPILER_VERSION {
145            let local_units: Vec<_> = local_units.iter().map(|u| (package, u.clone())).collect();
146            units.extend(local_units);
147            continue;
148        }
149
150        if local_units.is_empty() {
151            bail!("Expected one or more modules, but none found");
152        }
153        let package_root = SourcePackageLayout::try_find_root(&local_units[0].source_path)?;
154        let install_dir = tempfile::tempdir()?; // place compiled packages in this temp dir, don't pollute this packages build
155        // dir
156        download_and_compile(
157            package_root.clone(),
158            &install_dir,
159            &toolchain_version,
160            &package,
161        )?;
162
163        let compiled_unit_paths = vec![package_root.clone()];
164        let compiled_units = find_filenames(&compiled_unit_paths, |path| {
165            extension_equals(path, MOVE_COMPILED_EXTENSION)
166        })?;
167        let build_path = install_dir
168            .path()
169            .join(CompiledPackageLayout::path(&CompiledPackageLayout::Root))
170            .join(package.as_str());
171        debug!("build path is {}", build_path.display());
172
173        // Add all units compiled with the previous compiler.
174        for bytecode_path in compiled_units {
175            info!("bytecode path {bytecode_path}, {package}");
176            let local_unit = decode_bytecode_file(build_path.clone(), &package, &bytecode_path)?;
177            units.push((package, local_unit))
178        }
179    }
180    Ok(units)
181}
182
183fn download_and_compile(
184    root: PathBuf,
185    install_dir: &TempDir,
186    ToolchainVersion {
187        compiler_version,
188        edition,
189        flavor,
190    }: &ToolchainVersion,
191    dep_name: &Symbol,
192) -> anyhow::Result<()> {
193    let dest_dir = PathBuf::from_iter([&*MOVE_HOME, "binaries"]); // E.g., ~/.move/binaries
194    let dest_version = dest_dir.join(compiler_version);
195    let mut dest_canonical_path = dest_version.clone();
196    dest_canonical_path.extend(["target", "release"]);
197    let mut dest_canonical_binary = dest_canonical_path.clone();
198
199    let platform = detect_platform(&root, compiler_version, &dest_canonical_path)?;
200    if platform == "windows-x86_64" {
201        dest_canonical_binary.push(CANONICAL_WIN_BINARY_NAME);
202    } else {
203        dest_canonical_binary.push(CANONICAL_UNIX_BINARY_NAME);
204    }
205
206    if !dest_canonical_binary.exists() {
207        // Check the platform and proceed if we can download a binary. If not, the user
208        // should follow error instructions to sideload the binary. Download if
209        // binary does not exist.
210        let mainnet_url = format!(
211            "https://github.com/iotaledger/iota/releases/download/mainnet-v{compiler_version}/iota-mainnet-v{compiler_version}-{platform}.tgz",
212        );
213
214        println!(
215            "{} mainnet compiler @ {} (this may take a while)",
216            "DOWNLOADING".bold().green(),
217            compiler_version.yellow()
218        );
219
220        let mut response = match ureq::get(&mainnet_url).call() {
221            Ok(response) => response,
222            Err(ureq::Error::Status(404, _)) => {
223                println!(
224                    "{} iota mainnet compiler {} not available, attempting to download testnet compiler release...",
225                    "WARNING".bold().yellow(),
226                    compiler_version.yellow()
227                );
228                println!(
229                    "{} testnet compiler @ {} (this may take a while)",
230                    "DOWNLOADING".bold().green(),
231                    compiler_version.yellow()
232                );
233                let testnet_url = format!("https://github.com/iotaledger/iota/releases/download/testnet-v{compiler_version}/iota-testnet-v{compiler_version}-{platform}.tgz");
234                ureq::get(&testnet_url).call()?
235            }
236            Err(e) => return Err(e.into()),
237        }.into_reader();
238
239        let dest_tarball = dest_version.join(format!("{}.tgz", compiler_version));
240        debug!("tarball destination: {} ", dest_tarball.display());
241        if let Some(parent) = dest_tarball.parent() {
242            std::fs::create_dir_all(parent)
243                .map_err(|e| anyhow!("failed to create directory for tarball: {e}"))?;
244        }
245        let mut dest_file = File::create(&dest_tarball)?;
246        io::copy(&mut response, &mut dest_file)?;
247
248        // Extract the tarball using the tar crate
249        let tar_gz = File::open(&dest_tarball)?;
250        let tar = flate2::read::GzDecoder::new(tar_gz);
251        let mut archive = Archive::new(tar);
252        archive
253            .unpack(&dest_version)
254            .map_err(|e| anyhow!("failed to untar compiler binary: {e}"))?;
255
256        let mut dest_binary = dest_version.clone();
257        dest_binary.extend(["target", "release"]);
258        if platform == "windows-x86_64" {
259            dest_binary.push(format!("iota-{platform}.exe"));
260        } else {
261            dest_binary.push(format!("iota-{platform}"));
262        }
263        let dest_binary_os = OsStr::new(dest_binary.as_path());
264        set_executable_permission(dest_binary_os)?;
265        std::fs::rename(dest_binary_os, dest_canonical_binary.clone())?;
266    }
267
268    debug!(
269        "{} move build --default-move-edition {} --default-move-flavor {} -p {} --install-dir {}",
270        dest_canonical_binary.display(),
271        edition.to_string().as_str(),
272        flavor.to_string().as_str(),
273        root.display(),
274        install_dir.path().display(),
275    );
276    info!(
277        "{} {} (compiler @ {})",
278        "BUILDING".bold().green(),
279        dep_name.as_str(),
280        compiler_version.yellow()
281    );
282    Command::new(dest_canonical_binary)
283        .args([
284            OsStr::new("move"),
285            OsStr::new("build"),
286            OsStr::new("--default-move-edition"),
287            OsStr::new(edition.to_string().as_str()),
288            OsStr::new("--default-move-flavor"),
289            OsStr::new(flavor.to_string().as_str()),
290            OsStr::new("-p"),
291            OsStr::new(root.as_path()),
292            OsStr::new("--install-dir"),
293            OsStr::new(install_dir.path()),
294        ])
295        .output()
296        .map_err(|e| {
297            anyhow!("failed to build package from compiler binary {compiler_version}: {e}",)
298        })?;
299    Ok(())
300}
301
302fn detect_platform(
303    package_path: &Path,
304    compiler_version: &String,
305    dest_dir: &Path,
306) -> anyhow::Result<String> {
307    let s = match (std::env::consts::OS, std::env::consts::ARCH) {
308        ("macos", "aarch64") => "macos-arm64",
309        ("macos", "x86_64") => "macos-x86_64",
310        ("linux", "x86_64") => "ubuntu-x86_64",
311        ("windows", "x86_64") => "windows-x86_64",
312        (os, arch) => {
313            let mut binary_name = CANONICAL_UNIX_BINARY_NAME;
314            if os == "windows" {
315                binary_name = CANONICAL_WIN_BINARY_NAME;
316            };
317            bail!(
318                "The package {} needs to be built with iota compiler version {compiler_version} but there \
319                 is no binary release available to download for your platform:\n\
320                 Operating System: {os}\n\
321                 Architecture: {arch}\n\
322                 You can manually put a {binary_name} binary for your platform in {} and rerun your command to continue.",
323                package_path.display(),
324                dest_dir.display(),
325            )
326        }
327    };
328    Ok(s.into())
329}
330
331#[cfg(unix)]
332fn set_executable_permission(path: &OsStr) -> anyhow::Result<()> {
333    use std::{fs, os::unix::prelude::PermissionsExt};
334    let mut perms = fs::metadata(path)?.permissions();
335    perms.set_mode(0o755);
336    fs::set_permissions(path, perms)?;
337    Ok(())
338}
339
340#[cfg(not(unix))]
341fn set_executable_permission(path: &OsStr) -> anyhow::Result<()> {
342    Command::new("icacls")
343        .args([path, OsStr::new("/grant"), OsStr::new("Everyone:(RX)")])
344        .status()?;
345    Ok(())
346}
347
348fn decode_bytecode_file(
349    root_path: PathBuf,
350    package_name: &Symbol,
351    bytecode_path_str: &str,
352) -> anyhow::Result<CompiledUnitWithSource> {
353    let package_name_opt = Some(*package_name);
354    let bytecode_path = Path::new(bytecode_path_str);
355    let path_to_file = CompiledPackageLayout::path_to_file_after_category(bytecode_path);
356    let bytecode_bytes = std::fs::read(bytecode_path)?;
357    let source_map = source_map_from_file(
358        &root_path
359            .join(CompiledPackageLayout::SourceMaps.path())
360            .join(&path_to_file)
361            .with_extension(SOURCE_MAP_EXTENSION),
362    )?;
363    let source_path = &root_path
364        .join(CompiledPackageLayout::Sources.path())
365        .join(path_to_file)
366        .with_extension(MOVE_EXTENSION);
367    ensure!(
368        source_path.is_file(),
369        "Error decoding package: Unable to find corresponding source file for '{bytecode_path_str}' in package {package_name}"
370    );
371    let module = CompiledModule::deserialize_with_defaults(&bytecode_bytes)?;
372    let (address_bytes, module_name) = {
373        let id = module.self_id();
374        let parsed_addr = NumericalAddress::new(
375            id.address().into_bytes(),
376            move_compiler::shared::NumberFormat::Hex,
377        );
378        let module_name = FileName::from(id.name().as_str());
379        (parsed_addr, module_name)
380    };
381    let unit = NamedCompiledModule {
382        package_name: package_name_opt,
383        address: address_bytes,
384        name: module_name,
385        module,
386        source_map,
387        address_name: None,
388    };
389    Ok(CompiledUnitWithSource {
390        unit,
391        source_path: source_path.clone(),
392    })
393}
iota_source_validation/toolchain.rs

iota_source_validation/
toolchain.rs
