iota_package_dump/
lib.rs

1// Copyright (c) Mysten Labs, Inc.
2// Modifications Copyright (c) 2024 IOTA Stiftung
3// SPDX-License-Identifier: Apache-2.0
4
5use std::{
6    collections::BTreeMap,
7    fs,
8    path::{Path, PathBuf},
9};
10
11use anyhow::{Context, Result, bail, ensure};
12use client::Client;
13use fastcrypto::encoding::{Base64, Encoding};
14use iota_types::object::Object;
15use query::{IotaAddress, UInt53, limits, packages};
16use tracing::info;
17
18mod client;
19mod query;
20
21/// Ensure all packages created before `before_checkpoint` are written to the
22/// `output_dir`ectory, from the GraphQL service at `rpc_url`.
23///
24/// `output_dir` can be a path to a non-existent directory, an existing empty
25/// directory, or an existing directory written to in the past. If the path is
26/// non-existent, the invocation creates it. If the path exists but is empty,
27/// the invocation writes to the directory. If the directory has been written to
28/// in the past, the invocation picks back up where the previous invocation left
29/// off.
30pub async fn dump(
31    rpc_url: String,
32    output_dir: PathBuf,
33    before_checkpoint: Option<u64>,
34) -> Result<()> {
35    ensure_output_directory(&output_dir)?;
36
37    let client = Client::new(rpc_url)?;
38    let after_checkpoint = read_last_checkpoint(&output_dir)?;
39    let limit = max_page_size(&client).await?;
40    let (last_checkpoint, packages) =
41        fetch_packages(&client, limit, after_checkpoint, before_checkpoint).await?;
42
43    for package in &packages {
44        let IotaAddress(address) = &package.address;
45        dump_package(&output_dir, package)
46            .with_context(|| format!("Failed to dump package {address}"))?;
47    }
48
49    if let Some(last_checkpoint) = last_checkpoint {
50        write_last_checkpoint(&output_dir, last_checkpoint)?;
51    }
52
53    Ok(())
54}
55
56/// Ensure the output directory exists, either because it already exists as a
57/// writable directory, or by creating a new directory.
58fn ensure_output_directory(path: impl Into<PathBuf>) -> Result<()> {
59    let path: PathBuf = path.into();
60    if !path.exists() {
61        fs::create_dir_all(&path).context("Making output directory")?;
62        return Ok(());
63    }
64
65    ensure!(
66        path.is_dir(),
67        "Output path is not a directory: {}",
68        path.display()
69    );
70
71    let metadata = fs::metadata(&path).context("Getting metadata for output path")?;
72
73    ensure!(
74        !metadata.permissions().readonly(),
75        "Output directory is not writable: {}",
76        path.display()
77    );
78
79    Ok(())
80}
81
82/// Load the last checkpoint that was loaded by a previous run of the tool, if
83/// there is a previous run.
84fn read_last_checkpoint(output: &Path) -> Result<Option<u64>> {
85    let path = output.join("last-checkpoint");
86    if !path.exists() {
87        return Ok(None);
88    }
89
90    let content = fs::read_to_string(&path).context("Failed to read last checkpoint")?;
91    let checkpoint: u64 =
92        serde_json::from_str(&content).context("Failed to parse last checkpoint")?;
93
94    info!("Resuming download after checkpoint {checkpoint}");
95
96    Ok(Some(checkpoint))
97}
98
99/// Write the max checkpoint that we have seen a package from back to the output
100/// directory.
101fn write_last_checkpoint(output: &Path, checkpoint: u64) -> Result<()> {
102    let path = output.join("last-checkpoint");
103    let content =
104        serde_json::to_string(&checkpoint).context("Failed to serialize last checkpoint")?;
105
106    fs::write(path, content).context("Failed to write last checkpoint")?;
107    Ok(())
108}
109
110/// Read the max page size supported by the GraphQL service.
111async fn max_page_size(client: &Client) -> Result<i32> {
112    Ok(client
113        .query(limits::build())
114        .await
115        .context("Failed to fetch max page size")?
116        .service_config
117        .max_page_size)
118}
119
120/// Read all the packages between `after_checkpoint` and `before_checkpoint`, in
121/// batches of `page_size` from the `client` connected to a GraphQL service.
122///
123/// If `after_checkpoint` is not provided, packages are read from genesis. If
124/// `before_checkpoint` is not provided, packages are read until the latest
125/// checkpoint.
126///
127/// Returns the latest checkpoint that was read from in this fetch, and a list
128/// of all the packages that were read.
129async fn fetch_packages(
130    client: &Client,
131    page_size: i32,
132    after_checkpoint: Option<u64>,
133    before_checkpoint: Option<u64>,
134) -> Result<(Option<u64>, Vec<packages::MovePackage>)> {
135    let packages::Query {
136        checkpoint: checkpoint_viewed_at,
137        packages:
138            packages::MovePackageConnection {
139                mut page_info,
140                mut nodes,
141            },
142    } = client
143        .query(packages::build(
144            page_size,
145            None,
146            after_checkpoint.map(UInt53),
147            before_checkpoint.map(UInt53),
148        ))
149        .await
150        .with_context(|| "Failed to fetch page 1 of packages.")?;
151
152    for i in 2.. {
153        if !page_info.has_next_page {
154            break;
155        }
156
157        let packages = client
158            .query(packages::build(
159                page_size,
160                page_info.end_cursor,
161                after_checkpoint.map(UInt53),
162                before_checkpoint.map(UInt53),
163            ))
164            .await
165            .with_context(|| format!("Failed to fetch page {i} of packages."))?
166            .packages;
167
168        nodes.extend(packages.nodes);
169        page_info = packages.page_info;
170
171        info!(
172            "Fetched page {i} ({} package{} so far).",
173            nodes.len(),
174            if nodes.len() == 1 { "" } else { "s" },
175        );
176    }
177
178    use packages::Checkpoint as C;
179    let last_checkpoint = match (checkpoint_viewed_at, before_checkpoint) {
180        (
181            Some(C {
182                sequence_number: UInt53(v),
183            }),
184            Some(b),
185        ) if b > 0 => Some(v.min(b - 1)),
186        (
187            Some(C {
188                sequence_number: UInt53(c),
189            }),
190            _,
191        )
192        | (_, Some(c)) => Some(c),
193        _ => None,
194    };
195
196    Ok((last_checkpoint, nodes))
197}
198
199/// Write out `pkg` to the `output_dir`ectory, using the package's address and
200/// name as the directory name. The following files are written for each
201/// directory:
202///
203/// - `object.bcs` -- the BCS serialized form of the `Object` type containing
204///   the package.
205///
206/// - `linkage.json` -- a JSON serialization of the package's linkage table,
207///   mapping dependency original IDs to the version of the dependency being
208///   depended on and the ID of the object on chain that contains that version.
209///
210/// - `origins.json` -- a JSON serialization of the type origin table, mapping
211///   type names contained in this package to the version of the package that
212///   first introduced that type.
213///
214/// - `*.mv` -- a BCS serialization of each compiled module in the package.
215fn dump_package(output_dir: &Path, pkg: &packages::MovePackage) -> Result<()> {
216    let Some(query::Base64(bcs)) = &pkg.bcs else {
217        bail!("Missing BCS");
218    };
219
220    let bytes = Base64::decode(bcs).context("Failed to decode BCS")?;
221
222    let object = bcs::from_bytes::<Object>(&bytes).context("Failed to deserialize")?;
223    let id = object.id();
224    let Some(package) = object.data.try_as_package() else {
225        bail!("Not a package");
226    };
227
228    let origins: BTreeMap<_, _> = package
229        .type_origin_table()
230        .iter()
231        .map(|o| {
232            (
233                format!("{}::{}", o.module_name, o.datatype_name),
234                o.package.to_string(),
235            )
236        })
237        .collect();
238
239    let package_dir = output_dir.join(format!("{}.{}", id, package.version().value()));
240    fs::create_dir(&package_dir).context("Failed to make output directory")?;
241
242    let linkage_json = serde_json::to_string_pretty(package.linkage_table())
243        .context("Failed to serialize linkage")?;
244    let origins_json =
245        serde_json::to_string_pretty(&origins).context("Failed to serialize type origins")?;
246
247    fs::write(package_dir.join("object.bcs"), bytes).context("Failed to write object BCS")?;
248    fs::write(package_dir.join("linkage.json"), linkage_json).context("Failed to write linkage")?;
249    fs::write(package_dir.join("origins.json"), origins_json)
250        .context("Failed to write type origins")?;
251
252    for (module_name, module_bytes) in package.serialized_module_map() {
253        let module_path = package_dir.join(format!("{module_name}.mv"));
254        fs::write(module_path, module_bytes)
255            .with_context(|| format!("Failed to write module: {module_name}"))?
256    }
257
258    Ok(())
259}
iota_package_dump/lib.rs

iota_package_dump/
lib.rs
