iota_tool/
commands.rs

1// Copyright (c) Mysten Labs, Inc.
2// Modifications Copyright (c) 2024 IOTA Stiftung
3// SPDX-License-Identifier: Apache-2.0
4
5use std::{collections::BTreeMap, env, path::PathBuf, sync::Arc};
6
7use anyhow::Result;
8use clap::*;
9use fastcrypto::encoding::Encoding;
10use futures::{StreamExt, future::join_all};
11use iota_archival::{read_manifest_as_json, write_manifest_from_json};
12use iota_config::{
13    Config,
14    genesis::Genesis,
15    object_storage_config::{ObjectStoreConfig, ObjectStoreType},
16};
17use iota_core::{authority_aggregator::AuthorityAggregatorBuilder, authority_client::AuthorityAPI};
18use iota_protocol_config::Chain;
19use iota_replay::{ReplayToolCommand, execute_replay_command};
20use iota_sdk::{IotaClient, IotaClientBuilder, rpc_types::IotaTransactionBlockResponseOptions};
21use iota_types::{
22    base_types::*,
23    crypto::AuthorityPublicKeyBytes,
24    messages_checkpoint::{CheckpointRequest, CheckpointResponse, CheckpointSequenceNumber},
25    messages_grpc::TransactionInfoRequest,
26    transaction::{SenderSignedData, Transaction},
27};
28use telemetry_subscribers::TracingHandle;
29
30use crate::{
31    ConciseObjectOutput, GroupedObjectOutput, SnapshotVerifyMode, VerboseObjectOutput,
32    check_completed_snapshot,
33    db_tool::{DbToolCommand, execute_db_tool_command, print_db_all_tables},
34    download_db_snapshot, download_formal_snapshot, dump_checkpoints_from_archive,
35    get_latest_available_epoch, get_object, get_transaction_block, make_clients,
36    restore_from_db_checkpoint, verify_archive, verify_archive_by_checksum,
37};
38
39#[derive(Parser, Clone, ValueEnum)]
40pub enum Verbosity {
41    Grouped,
42    Concise,
43    Verbose,
44}
45
46#[derive(Parser)]
47pub enum ToolCommand {
48    /// Inspect if a specific object is or all gas objects owned by an address
49    /// are locked by validators
50    LockedObject {
51        /// Either id or address must be provided
52        /// The object to check
53        #[arg(long, help = "The object ID to fetch")]
54        id: Option<ObjectID>,
55        /// Either id or address must be provided
56        /// If provided, check all gas objects owned by this account
57        #[arg(long)]
58        address: Option<IotaAddress>,
59        /// RPC address to provide the up-to-date committee info
60        #[arg(long)]
61        fullnode_rpc_url: String,
62        /// Should attempt to rescue the object if it's locked but not fully
63        /// locked
64        #[arg(long)]
65        rescue: bool,
66    },
67
68    /// Fetch the same object from all validators
69    FetchObject {
70        #[arg(long, help = "The object ID to fetch")]
71        id: ObjectID,
72
73        #[arg(long, help = "Fetch object at a specific sequence")]
74        version: Option<u64>,
75
76        #[arg(
77            long,
78            help = "Validator to fetch from - if not specified, all validators are queried"
79        )]
80        validator: Option<AuthorityName>,
81
82        // RPC address to provide the up-to-date committee info
83        #[arg(long)]
84        fullnode_rpc_url: String,
85
86        /// Concise mode groups responses by results.
87        /// prints tabular output suitable for processing with unix tools. For
88        /// instance, to quickly check that all validators agree on the history
89        /// of an object: ```text
90        /// $ iota-tool fetch-object --id
91        /// 0x260efde76ebccf57f4c5e951157f5c361cde822c \      --genesis
92        /// $HOME/.iota/iota_config/genesis.blob \      --verbosity
93        /// concise --concise-no-header ```
94        #[arg(value_enum, long, default_value = "grouped", ignore_case = true)]
95        verbosity: Verbosity,
96
97        #[arg(long, help = "don't show header in concise output")]
98        concise_no_header: bool,
99    },
100
101    /// Fetch the effects association with transaction `digest`
102    FetchTransaction {
103        // RPC address to provide the up-to-date committee info
104        #[arg(long)]
105        fullnode_rpc_url: String,
106
107        #[arg(long, help = "The transaction ID to fetch")]
108        digest: TransactionDigest,
109
110        /// If true, show the input transaction as well as the effects
111        #[arg(long = "show-tx")]
112        show_input_tx: bool,
113    },
114
115    /// Tool to read validator & node db.
116    DbTool {
117        /// Path of the DB to read
118        #[arg(long)]
119        db_path: String,
120        #[command(subcommand)]
121        cmd: Option<DbToolCommand>,
122    },
123
124    /// Tool to verify the archive store
125    VerifyArchive {
126        #[arg(long)]
127        genesis: PathBuf,
128        #[command(flatten)]
129        object_store_config: ObjectStoreConfig,
130        #[arg(default_value_t = 5)]
131        download_concurrency: usize,
132    },
133
134    /// Tool to print the archive manifest
135    PrintArchiveManifest {
136        #[command(flatten)]
137        object_store_config: ObjectStoreConfig,
138    },
139    /// Tool to update the archive manifest
140    UpdateArchiveManifest {
141        #[command(flatten)]
142        object_store_config: ObjectStoreConfig,
143        #[arg(long = "archive-path")]
144        archive_json_path: PathBuf,
145    },
146    /// Tool to verify the archive store by comparing file checksums
147    #[command(name = "verify-archive-from-checksums")]
148    VerifyArchiveByChecksum {
149        #[command(flatten)]
150        object_store_config: ObjectStoreConfig,
151        #[arg(default_value_t = 5)]
152        download_concurrency: usize,
153    },
154
155    /// Tool to print archive contents in checkpoint range
156    #[command(name = "dump-archive")]
157    DumpArchiveByChecksum {
158        #[command(flatten)]
159        object_store_config: ObjectStoreConfig,
160        #[arg(default_value_t = 0)]
161        start: u64,
162        end: u64,
163        #[arg(default_value_t = 80)]
164        max_content_length: usize,
165    },
166
167    /// Download all packages to the local filesystem from a GraphQL service.
168    /// Each package gets its own sub-directory, named for its ID on chain
169    /// and version containing two metadata files (linkage.json and
170    /// origins.json), a file containing the overall object and a file for every
171    /// module it contains. Each module file is named for its module name, with
172    /// a .mv suffix, and contains Move bytecode (suitable for passing into
173    /// a disassembler).
174    DumpPackages {
175        /// Connection information for a GraphQL service.
176        #[arg(long, short)]
177        rpc_url: String,
178
179        /// Path to a non-existent directory that can be created and filled with
180        /// package information.
181        #[arg(long, short)]
182        output_dir: PathBuf,
183
184        /// Only fetch packages that were created before this checkpoint (given
185        /// by its sequence number).
186        #[arg(long)]
187        before_checkpoint: Option<u64>,
188
189        /// If false (default), log level will be overridden to "off", and
190        /// output will be reduced to necessary status information.
191        #[arg(short, long)]
192        verbose: bool,
193    },
194
195    DumpValidators {
196        #[arg(long)]
197        genesis: PathBuf,
198
199        #[arg(
200            long,
201            help = "show concise output - name, authority key and network address"
202        )]
203        concise: bool,
204    },
205
206    DumpGenesis {
207        #[arg(long)]
208        genesis: PathBuf,
209    },
210
211    /// Fetch authenticated checkpoint information at a specific sequence
212    /// number. If sequence number is not specified, get the latest
213    /// authenticated checkpoint.
214    FetchCheckpoint {
215        // RPC address to provide the up-to-date committee info
216        #[arg(long)]
217        fullnode_rpc_url: String,
218
219        #[arg(long, help = "Fetch checkpoint at a specific sequence number")]
220        sequence_number: Option<CheckpointSequenceNumber>,
221    },
222
223    Anemo {
224        #[command(next_help_heading = "foo", flatten)]
225        args: anemo_cli::Args,
226    },
227
228    #[command(name = "restore-db")]
229    RestoreFromDBCheckpoint {
230        #[arg(long)]
231        config_path: PathBuf,
232        #[arg(long)]
233        db_checkpoint_path: PathBuf,
234    },
235
236    #[command(
237        name = "download-db-snapshot",
238        about = "Downloads the legacy database snapshot via cloud object store, outputs to local disk"
239    )]
240    DownloadDBSnapshot {
241        #[arg(long, conflicts_with = "latest")]
242        epoch: Option<u64>,
243        #[arg(long, help = "the path to write the downloaded snapshot files")]
244        path: PathBuf,
245        /// skip downloading indexes dir
246        #[arg(long)]
247        skip_indexes: bool,
248        /// Number of parallel downloads to perform. Defaults to a reasonable
249        /// value based on number of available logical cores.
250        #[arg(long)]
251        num_parallel_downloads: Option<usize>,
252        /// Network to download snapshot for. Defaults to "mainnet".
253        /// If `--snapshot-bucket` or `--archive-bucket` is not specified,
254        /// the value of this flag is used to construct default bucket names.
255        #[arg(long, default_value = "mainnet")]
256        network: Chain,
257        /// Snapshot bucket name. If not specified, defaults are
258        /// based on value of `--network` flag.
259        #[arg(long, conflicts_with = "no_sign_request")]
260        snapshot_bucket: Option<String>,
261        /// Snapshot bucket type
262        #[arg(
263            long,
264            conflicts_with = "no_sign_request",
265            help = "Required if --no-sign-request is not set"
266        )]
267        snapshot_bucket_type: Option<ObjectStoreType>,
268        /// Path to snapshot directory on local filesystem.
269        /// Only applicable if `--snapshot-bucket-type` is "file".
270        #[arg(long, help = "only used for testing, when --snapshot-bucket-type=FILE")]
271        snapshot_path: Option<PathBuf>,
272        /// If true, no authentication is needed for snapshot restores
273        #[arg(
274            long,
275            conflicts_with_all = &["snapshot_bucket", "snapshot_bucket_type"],
276            help = "if set, no authentication is needed for snapshot restore"
277        )]
278        no_sign_request: bool,
279        /// Download snapshot of the latest available epoch.
280        /// If `--epoch` is specified, then this flag gets ignored.
281        #[arg(
282            long,
283            conflicts_with = "epoch",
284            help = "defaults to latest available snapshot in chosen bucket"
285        )]
286        latest: bool,
287        /// If false (default), log level will be overridden to "off",
288        /// and output will be reduced to necessary status information.
289        #[arg(long)]
290        verbose: bool,
291    },
292
293    // Restore from formal (slim, DB agnostic) snapshot.
294    #[command(
295        about = "Downloads formal database snapshot via cloud object store, outputs to local disk"
296    )]
297    DownloadFormalSnapshot {
298        #[arg(long, conflicts_with = "latest")]
299        epoch: Option<u64>,
300        #[arg(long)]
301        genesis: PathBuf,
302        #[arg(long)]
303        path: PathBuf,
304        /// Number of parallel downloads to perform. Defaults to a reasonable
305        /// value based on number of available logical cores.
306        #[arg(long)]
307        num_parallel_downloads: Option<usize>,
308        /// Verification mode to employ.
309        #[arg(long, default_value = "normal")]
310        verify: Option<SnapshotVerifyMode>,
311        /// Network to download snapshot for. Defaults to "mainnet".
312        /// If `--snapshot-bucket` or `--archive-bucket` is not specified,
313        /// the value of this flag is used to construct default bucket names.
314        #[arg(long, default_value = "mainnet")]
315        network: Chain,
316        /// Snapshot bucket name. If not specified, defaults are
317        /// based on value of `--network` flag.
318        #[arg(long, conflicts_with = "no_sign_request")]
319        snapshot_bucket: Option<String>,
320        /// Snapshot bucket type
321        #[arg(
322            long,
323            conflicts_with = "no_sign_request",
324            help = "Required if --no-sign-request is not set"
325        )]
326        snapshot_bucket_type: Option<ObjectStoreType>,
327        /// Path to snapshot directory on local filesystem.
328        /// Only applicable if `--snapshot-bucket-type` is "file".
329        #[arg(long)]
330        snapshot_path: Option<PathBuf>,
331        /// If true, no authentication is needed for snapshot restores
332        #[arg(
333            long,
334            conflicts_with_all = &["snapshot_bucket", "snapshot_bucket_type"],
335            help = "if set, no authentication is needed for snapshot restore"
336        )]
337        no_sign_request: bool,
338        /// Download snapshot of the latest available epoch.
339        /// If `--epoch` is specified, then this flag gets ignored.
340        #[arg(
341            long,
342            conflicts_with = "epoch",
343            help = "defaults to latest available snapshot in chosen bucket"
344        )]
345        latest: bool,
346        /// If false (default), log level will be overridden to "off",
347        /// and output will be reduced to necessary status information.
348        #[arg(long)]
349        verbose: bool,
350
351        /// If provided, all checkpoint summaries from genesis to the end of the
352        /// target epoch will be downloaded and (if --verify is
353        /// provided) full checkpoint chain verification
354        /// will be performed. If omitted, only end of epoch checkpoint
355        /// summaries will be downloaded, and (if --verify is provided)
356        /// will be verified via committee signature.
357        #[arg(long)]
358        all_checkpoints: bool,
359    },
360
361    Replay {
362        #[arg(long = "rpc")]
363        rpc_url: Option<String>,
364        #[arg(long)]
365        safety_checks: bool,
366        #[arg(long = "authority")]
367        use_authority: bool,
368        #[arg(
369            long,
370            short,
371            help = "Path to the network config file. This should be specified when rpc_url is not present. \
372            If not specified we will use the default network config file at ~/.iota-replay/network-config.yaml"
373        )]
374        cfg_path: Option<PathBuf>,
375        #[arg(
376            long,
377            help = "The name of the chain to replay from, could be one of: mainnet, testnet, devnet.\
378            When rpc_url is not specified, this is used to load the corresponding config from the network config file.\
379            If not specified, mainnet will be used by default"
380        )]
381        chain: Option<String>,
382        #[command(subcommand)]
383        cmd: ReplayToolCommand,
384    },
385
386    /// Ask all validators to sign a transaction through AuthorityAggregator.
387    SignTransaction {
388        #[arg(long)]
389        genesis: PathBuf,
390
391        #[arg(
392            long,
393            help = "The Base64-encoding of the bcs bytes of SenderSignedData"
394        )]
395        sender_signed_data: String,
396    },
397
398    /// Create an IOTA Genesis Ceremony with multiple remote validators.
399    GenesisCeremony(crate::genesis_ceremony::Ceremony),
400    /// Tool for Fire Drill
401    FireDrill {
402        #[command(subcommand)]
403        fire_drill: crate::fire_drill::FireDrill,
404    },
405}
406
407async fn check_locked_object(
408    iota_client: &Arc<IotaClient>,
409    committee: Arc<BTreeMap<AuthorityPublicKeyBytes, u64>>,
410    id: ObjectID,
411    rescue: bool,
412) -> anyhow::Result<()> {
413    let clients = Arc::new(make_clients(iota_client).await?);
414    let output = get_object(id, None, None, clients.clone()).await?;
415    let output = GroupedObjectOutput::new(output, committee);
416    if output.fully_locked {
417        println!("Object {id} is fully locked.");
418        return Ok(());
419    }
420    let top_record = output.voting_power.first().unwrap();
421    let top_record_stake = top_record.1;
422    let top_record = top_record.0.unwrap();
423    if top_record.4.is_none() {
424        println!(
425            "Object {id} does not seem to be locked by majority of validators (unlocked stake: {top_record_stake})"
426        );
427        return Ok(());
428    }
429
430    let tx_digest = top_record.2;
431    if !rescue {
432        println!("Object {id} is rescueable, top tx: {tx_digest:?}");
433        return Ok(());
434    }
435    println!("Object {id} is rescueable, trying tx {tx_digest}");
436    let validator = output
437        .grouped_results
438        .get(&Some(top_record))
439        .unwrap()
440        .first()
441        .unwrap();
442    let client = &clients.get(validator).unwrap().1;
443    let tx = client
444        .handle_transaction_info_request(TransactionInfoRequest {
445            transaction_digest: tx_digest,
446        })
447        .await?
448        .transaction;
449    let res = iota_client
450        .quorum_driver_api()
451        .execute_transaction_block(
452            Transaction::new(tx),
453            IotaTransactionBlockResponseOptions::full_content(),
454            None,
455        )
456        .await;
457    match res {
458        Ok(_) => {
459            println!("Transaction executed successfully ({tx_digest:?})");
460        }
461        Err(e) => {
462            println!("Failed to execute transaction ({tx_digest:?}): {e:?}");
463        }
464    }
465    Ok(())
466}
467
468impl ToolCommand {
469    pub async fn execute(self, tracing_handle: TracingHandle) -> Result<(), anyhow::Error> {
470        match self {
471            ToolCommand::LockedObject {
472                id,
473                fullnode_rpc_url,
474                rescue,
475                address,
476            } => {
477                let iota_client =
478                    Arc::new(IotaClientBuilder::default().build(fullnode_rpc_url).await?);
479                let committee = Arc::new(
480                    iota_client
481                        .governance_api()
482                        .get_committee_info(None)
483                        .await?
484                        .validators
485                        .into_iter()
486                        .collect::<BTreeMap<_, _>>(),
487                );
488                let object_ids = match id {
489                    Some(id) => vec![id],
490                    None => {
491                        let address = address.expect("Either id or address must be provided");
492                        iota_client
493                            .coin_read_api()
494                            .get_coins_stream(address, None)
495                            .map(|c| c.coin_object_id)
496                            .collect()
497                            .await
498                    }
499                };
500                for ids in object_ids.chunks(30) {
501                    let mut tasks = vec![];
502                    for id in ids {
503                        tasks.push(check_locked_object(
504                            &iota_client,
505                            committee.clone(),
506                            *id,
507                            rescue,
508                        ))
509                    }
510                    join_all(tasks)
511                        .await
512                        .into_iter()
513                        .collect::<Result<Vec<_>, _>>()?;
514                }
515            }
516            ToolCommand::FetchObject {
517                id,
518                validator,
519                version,
520                fullnode_rpc_url,
521                verbosity,
522                concise_no_header,
523            } => {
524                let iota_client =
525                    Arc::new(IotaClientBuilder::default().build(fullnode_rpc_url).await?);
526                let clients = Arc::new(make_clients(&iota_client).await?);
527                let output = get_object(id, version, validator, clients).await?;
528
529                match verbosity {
530                    Verbosity::Grouped => {
531                        let committee = Arc::new(
532                            iota_client
533                                .governance_api()
534                                .get_committee_info(None)
535                                .await?
536                                .validators
537                                .into_iter()
538                                .collect::<BTreeMap<_, _>>(),
539                        );
540                        println!("{}", GroupedObjectOutput::new(output, committee));
541                    }
542                    Verbosity::Verbose => {
543                        println!("{}", VerboseObjectOutput(output));
544                    }
545                    Verbosity::Concise => {
546                        if !concise_no_header {
547                            println!("{}", ConciseObjectOutput::header());
548                        }
549                        println!("{}", ConciseObjectOutput(output));
550                    }
551                }
552            }
553            ToolCommand::FetchTransaction {
554                digest,
555                show_input_tx,
556                fullnode_rpc_url,
557            } => {
558                print!(
559                    "{}",
560                    get_transaction_block(digest, show_input_tx, fullnode_rpc_url).await?
561                );
562            }
563            ToolCommand::DbTool { db_path, cmd } => {
564                let path = PathBuf::from(db_path);
565                match cmd {
566                    Some(c) => execute_db_tool_command(path, c).await?,
567                    None => print_db_all_tables(path)?,
568                }
569            }
570            ToolCommand::DumpPackages {
571                rpc_url,
572                output_dir,
573                before_checkpoint,
574                verbose,
575            } => {
576                if !verbose {
577                    tracing_handle
578                        .update_log("off")
579                        .expect("Failed to update log level");
580                }
581
582                iota_package_dump::dump(rpc_url, output_dir, before_checkpoint).await?;
583            }
584            ToolCommand::DumpValidators { genesis, concise } => {
585                let genesis = Genesis::load(genesis).unwrap();
586                if !concise {
587                    println!("{:#?}", genesis.validator_set_for_tooling());
588                } else {
589                    for (i, val_info) in genesis.validator_set_for_tooling().iter().enumerate() {
590                        let metadata = val_info.verified_metadata();
591                        println!(
592                            "#{:<2} {:<20} {:?} {:?} {}",
593                            i,
594                            metadata.name,
595                            metadata.iota_pubkey_bytes().concise(),
596                            metadata.net_address,
597                            anemo::PeerId(metadata.network_pubkey.0.to_bytes()),
598                        )
599                    }
600                }
601            }
602            ToolCommand::DumpGenesis { genesis } => {
603                let genesis = Genesis::load(genesis)?;
604                println!("{genesis:#?}");
605            }
606            ToolCommand::FetchCheckpoint {
607                sequence_number,
608                fullnode_rpc_url,
609            } => {
610                let iota_client =
611                    Arc::new(IotaClientBuilder::default().build(fullnode_rpc_url).await?);
612                let clients = make_clients(&iota_client).await?;
613
614                for (name, (_, client)) in clients {
615                    let resp = client
616                        .handle_checkpoint(CheckpointRequest {
617                            sequence_number,
618                            request_content: true,
619                            certified: true,
620                        })
621                        .await
622                        .unwrap();
623                    let CheckpointResponse {
624                        checkpoint,
625                        contents,
626                    } = resp;
627                    println!("Validator: {:?}\n", name.concise());
628                    println!("Checkpoint: {checkpoint:?}\n");
629                    println!("Content: {contents:?}\n");
630                }
631            }
632            ToolCommand::Anemo { args } => {
633                let config = crate::make_anemo_config();
634                anemo_cli::run(config, args).await
635            }
636            ToolCommand::RestoreFromDBCheckpoint {
637                config_path,
638                db_checkpoint_path,
639            } => {
640                let config = iota_config::NodeConfig::load(config_path)?;
641                restore_from_db_checkpoint(&config, &db_checkpoint_path).await?;
642            }
643            ToolCommand::DownloadFormalSnapshot {
644                epoch,
645                genesis,
646                path,
647                num_parallel_downloads,
648                verify,
649                network,
650                snapshot_bucket,
651                snapshot_bucket_type,
652                snapshot_path,
653                no_sign_request,
654                latest,
655                verbose,
656                all_checkpoints,
657            } => {
658                if !verbose {
659                    tracing_handle
660                        .update_log("off")
661                        .expect("Failed to update log level");
662                }
663                let num_parallel_downloads = num_parallel_downloads.unwrap_or_else(|| {
664                    num_cpus::get()
665                        .checked_sub(1)
666                        .expect("Failed to get number of CPUs")
667                });
668                let snapshot_bucket =
669                    snapshot_bucket.or_else(|| match (network, no_sign_request) {
670                        (Chain::Mainnet, false) => Some(
671                            env::var("MAINNET_FORMAL_SIGNED_BUCKET")
672                                .unwrap_or("iota-mainnet-formal".to_string()),
673                        ),
674                        (Chain::Mainnet, true) => env::var("MAINNET_FORMAL_UNSIGNED_BUCKET").ok(),
675                        (Chain::Testnet, true) => env::var("TESTNET_FORMAL_UNSIGNED_BUCKET").ok(),
676                        (Chain::Testnet, _) => Some(
677                            env::var("TESTNET_FORMAL_SIGNED_BUCKET")
678                                .unwrap_or("iota-testnet-formal".to_string()),
679                        ),
680                        (Chain::Unknown, _) => {
681                            panic!("Cannot generate default snapshot bucket for unknown network");
682                        }
683                    });
684
685                let aws_endpoint = env::var("AWS_SNAPSHOT_ENDPOINT").ok().or_else(|| {
686                    if no_sign_request {
687                        if network == Chain::Mainnet {
688                            Some("https://formal-snapshot.mainnet.iota.cafe".to_string())
689                        } else if network == Chain::Testnet {
690                            Some("https://formal-snapshot.testnet.iota.cafe".to_string())
691                        } else {
692                            None
693                        }
694                    } else {
695                        None
696                    }
697                });
698
699                let snapshot_bucket_type = if no_sign_request {
700                    ObjectStoreType::S3
701                } else {
702                    snapshot_bucket_type
703                        .expect("You must set either --snapshot-bucket-type or --no-sign-request")
704                };
705                let snapshot_store_config = match snapshot_bucket_type {
706                    ObjectStoreType::S3 => ObjectStoreConfig {
707                        object_store: Some(ObjectStoreType::S3),
708                        bucket: snapshot_bucket.filter(|s| !s.is_empty()),
709                        aws_access_key_id: env::var("AWS_SNAPSHOT_ACCESS_KEY_ID").ok(),
710                        aws_secret_access_key: env::var("AWS_SNAPSHOT_SECRET_ACCESS_KEY").ok(),
711                        aws_region: env::var("AWS_SNAPSHOT_REGION").ok(),
712                        aws_endpoint: aws_endpoint.filter(|s| !s.is_empty()),
713                        aws_virtual_hosted_style_request: env::var(
714                            "AWS_SNAPSHOT_VIRTUAL_HOSTED_REQUESTS",
715                        )
716                        .ok()
717                        .and_then(|b| b.parse().ok())
718                        .unwrap_or(no_sign_request),
719                        object_store_connection_limit: 200,
720                        no_sign_request,
721                        ..Default::default()
722                    },
723                    ObjectStoreType::GCS => ObjectStoreConfig {
724                        object_store: Some(ObjectStoreType::GCS),
725                        bucket: snapshot_bucket,
726                        google_service_account: env::var("GCS_SNAPSHOT_SERVICE_ACCOUNT_FILE_PATH")
727                            .ok(),
728                        object_store_connection_limit: 200,
729                        no_sign_request,
730                        ..Default::default()
731                    },
732                    ObjectStoreType::Azure => ObjectStoreConfig {
733                        object_store: Some(ObjectStoreType::Azure),
734                        bucket: snapshot_bucket,
735                        azure_storage_account: env::var("AZURE_SNAPSHOT_STORAGE_ACCOUNT").ok(),
736                        azure_storage_access_key: env::var("AZURE_SNAPSHOT_STORAGE_ACCESS_KEY")
737                            .ok(),
738                        object_store_connection_limit: 200,
739                        no_sign_request,
740                        ..Default::default()
741                    },
742                    ObjectStoreType::File => {
743                        if snapshot_path.is_some() {
744                            ObjectStoreConfig {
745                                object_store: Some(ObjectStoreType::File),
746                                directory: snapshot_path,
747                                ..Default::default()
748                            }
749                        } else {
750                            panic!(
751                                "--snapshot-path must be specified for --snapshot-bucket-type=file"
752                            );
753                        }
754                    }
755                };
756
757                let archive_bucket = Some(
758                    env::var("FORMAL_SNAPSHOT_ARCHIVE_BUCKET").unwrap_or_else(|_| match network {
759                        Chain::Mainnet => "iota-mainnet-archive".to_string(),
760                        Chain::Testnet => "iota-testnet-archive".to_string(),
761                        Chain::Unknown => {
762                            panic!("Cannot generate default archive bucket for unknown network");
763                        }
764                    }),
765                );
766
767                let mut custom_archive_enabled = false;
768                if let Ok(custom_archive_check) = env::var("CUSTOM_ARCHIVE_BUCKET") {
769                    if custom_archive_check == "true" {
770                        custom_archive_enabled = true;
771                    }
772                }
773                let archive_store_config = if custom_archive_enabled {
774                    let aws_region = Some(
775                        env::var("FORMAL_SNAPSHOT_ARCHIVE_REGION")
776                            .unwrap_or("us-west-2".to_string()),
777                    );
778
779                    let archive_bucket_type = env::var("FORMAL_SNAPSHOT_ARCHIVE_BUCKET_TYPE").expect("If setting `CUSTOM_ARCHIVE_BUCKET=true` Must set FORMAL_SNAPSHOT_ARCHIVE_BUCKET_TYPE, and credentials");
780                    match archive_bucket_type.to_ascii_lowercase().as_str() {
781                        "s3" => ObjectStoreConfig {
782                            object_store: Some(ObjectStoreType::S3),
783                            bucket: archive_bucket.filter(|s| !s.is_empty()),
784                            aws_access_key_id: env::var("AWS_ARCHIVE_ACCESS_KEY_ID").ok(),
785                            aws_secret_access_key: env::var("AWS_ARCHIVE_SECRET_ACCESS_KEY").ok(),
786                            aws_region,
787                            aws_endpoint: env::var("AWS_ARCHIVE_ENDPOINT").ok(),
788                            aws_virtual_hosted_style_request: env::var(
789                                "AWS_ARCHIVE_VIRTUAL_HOSTED_REQUESTS",
790                            )
791                            .ok()
792                            .and_then(|b| b.parse().ok())
793                            .unwrap_or(false),
794                            object_store_connection_limit: 50,
795                            no_sign_request: false,
796                            ..Default::default()
797                        },
798                        "gcs" => ObjectStoreConfig {
799                            object_store: Some(ObjectStoreType::GCS),
800                            bucket: archive_bucket,
801                            google_service_account: env::var(
802                                "GCS_ARCHIVE_SERVICE_ACCOUNT_FILE_PATH",
803                            )
804                            .ok(),
805                            object_store_connection_limit: 50,
806                            no_sign_request: false,
807                            ..Default::default()
808                        },
809                        "azure" => ObjectStoreConfig {
810                            object_store: Some(ObjectStoreType::Azure),
811                            bucket: archive_bucket,
812                            azure_storage_account: env::var("AZURE_ARCHIVE_STORAGE_ACCOUNT").ok(),
813                            azure_storage_access_key: env::var("AZURE_ARCHIVE_STORAGE_ACCESS_KEY")
814                                .ok(),
815                            object_store_connection_limit: 50,
816                            no_sign_request: false,
817                            ..Default::default()
818                        },
819                        _ => panic!(
820                            "If setting `CUSTOM_ARCHIVE_BUCKET=true` must set FORMAL_SNAPSHOT_ARCHIVE_BUCKET_TYPE to one of 'gcs', 'azure', or 's3' "
821                        ),
822                    }
823                } else {
824                    // if not explicitly overridden, just default to the permissionless archive
825                    // store
826                    let aws_endpoint = env::var("AWS_ARCHIVE_ENDPOINT").ok().or_else(|| {
827                        if network == Chain::Mainnet {
828                            Some("https://archive.mainnet.iota.cafe".to_string())
829                        } else if network == Chain::Testnet {
830                            Some("https://archive.testnet.iota.cafe".to_string())
831                        } else {
832                            None
833                        }
834                    });
835
836                    let aws_virtual_hosted_style_request =
837                        env::var("AWS_ARCHIVE_VIRTUAL_HOSTED_REQUESTS")
838                            .ok()
839                            .and_then(|b| b.parse().ok())
840                            .unwrap_or(matches!(network, Chain::Mainnet | Chain::Testnet));
841
842                    ObjectStoreConfig {
843                        object_store: Some(ObjectStoreType::S3),
844                        bucket: archive_bucket.filter(|s| !s.is_empty()),
845                        aws_region: Some("us-west-2".to_string()),
846                        aws_endpoint,
847                        aws_virtual_hosted_style_request,
848                        object_store_connection_limit: 200,
849                        no_sign_request: true,
850                        ..Default::default()
851                    }
852                };
853                let latest_available_epoch =
854                    latest.then_some(get_latest_available_epoch(&snapshot_store_config).await?);
855                let epoch_to_download = epoch.or(latest_available_epoch).expect(
856                    "Either pass epoch with --epoch <epoch_num> or use latest with --latest",
857                );
858
859                if let Err(e) =
860                    check_completed_snapshot(&snapshot_store_config, epoch_to_download).await
861                {
862                    panic!("Aborting snapshot restore: {e}, snapshot may not be uploaded yet");
863                }
864
865                let verify = verify.unwrap_or_default();
866                download_formal_snapshot(
867                    &path,
868                    epoch_to_download,
869                    &genesis,
870                    snapshot_store_config,
871                    archive_store_config,
872                    num_parallel_downloads,
873                    network,
874                    verify,
875                    all_checkpoints,
876                )
877                .await?;
878            }
879            ToolCommand::DownloadDBSnapshot {
880                epoch,
881                path,
882                skip_indexes,
883                num_parallel_downloads,
884                network,
885                snapshot_bucket,
886                snapshot_bucket_type,
887                snapshot_path,
888                no_sign_request,
889                latest,
890                verbose,
891            } => {
892                if !verbose {
893                    tracing_handle
894                        .update_log("off")
895                        .expect("Failed to update log level");
896                }
897                let num_parallel_downloads = num_parallel_downloads.unwrap_or_else(|| {
898                    num_cpus::get()
899                        .checked_sub(1)
900                        .expect("Failed to get number of CPUs")
901                });
902                let snapshot_bucket =
903                    snapshot_bucket.or_else(|| match (network, no_sign_request) {
904                        (Chain::Mainnet, false) => Some(
905                            env::var("MAINNET_DB_SIGNED_BUCKET")
906                                .unwrap_or("iota-mainnet-snapshots".to_string()),
907                        ),
908                        (Chain::Mainnet, true) => env::var("MAINNET_DB_UNSIGNED_BUCKET").ok(),
909                        (Chain::Testnet, true) => env::var("TESTNET_DB_UNSIGNED_BUCKET").ok(),
910                        (Chain::Testnet, _) => Some(
911                            env::var("TESTNET_DB_SIGNED_BUCKET")
912                                .unwrap_or("iota-testnet-snapshots".to_string()),
913                        ),
914                        (Chain::Unknown, _) => {
915                            panic!("Cannot generate default snapshot bucket for unknown network");
916                        }
917                    });
918
919                let aws_endpoint = env::var("AWS_SNAPSHOT_ENDPOINT").ok();
920                let snapshot_bucket_type = if no_sign_request {
921                    ObjectStoreType::S3
922                } else {
923                    snapshot_bucket_type
924                        .expect("You must set either --snapshot-bucket-type or --no-sign-request")
925                };
926                let snapshot_store_config = if no_sign_request {
927                    let aws_endpoint = env::var("AWS_SNAPSHOT_ENDPOINT").ok().or_else(|| {
928                        if network == Chain::Mainnet {
929                            Some("https://rocksdb-snapshot.mainnet.iota.cafe".to_string())
930                        } else if network == Chain::Testnet {
931                            Some("https://rocksdb-snapshot.testnet.iota.cafe".to_string())
932                        } else {
933                            None
934                        }
935                    });
936                    ObjectStoreConfig {
937                        object_store: Some(ObjectStoreType::S3),
938                        aws_endpoint: aws_endpoint.filter(|s| !s.is_empty()),
939                        aws_virtual_hosted_style_request: env::var(
940                            "AWS_SNAPSHOT_VIRTUAL_HOSTED_REQUESTS",
941                        )
942                        .ok()
943                        .and_then(|b| b.parse().ok())
944                        .unwrap_or(no_sign_request),
945                        object_store_connection_limit: 200,
946                        no_sign_request,
947                        ..Default::default()
948                    }
949                } else {
950                    match snapshot_bucket_type {
951                        ObjectStoreType::S3 => ObjectStoreConfig {
952                            object_store: Some(ObjectStoreType::S3),
953                            bucket: snapshot_bucket.filter(|s| !s.is_empty()),
954                            aws_access_key_id: env::var("AWS_SNAPSHOT_ACCESS_KEY_ID").ok(),
955                            aws_secret_access_key: env::var("AWS_SNAPSHOT_SECRET_ACCESS_KEY").ok(),
956                            aws_region: env::var("AWS_SNAPSHOT_REGION").ok(),
957                            aws_endpoint: aws_endpoint.filter(|s| !s.is_empty()),
958                            aws_virtual_hosted_style_request: env::var(
959                                "AWS_SNAPSHOT_VIRTUAL_HOSTED_REQUESTS",
960                            )
961                            .ok()
962                            .and_then(|b| b.parse().ok())
963                            .unwrap_or(no_sign_request),
964                            object_store_connection_limit: 200,
965                            no_sign_request,
966                            ..Default::default()
967                        },
968                        ObjectStoreType::GCS => ObjectStoreConfig {
969                            object_store: Some(ObjectStoreType::GCS),
970                            bucket: snapshot_bucket,
971                            google_service_account: env::var(
972                                "GCS_SNAPSHOT_SERVICE_ACCOUNT_FILE_PATH",
973                            )
974                            .ok(),
975                            google_project_id: env::var("GCS_SNAPSHOT_SERVICE_ACCOUNT_PROJECT_ID")
976                                .ok(),
977                            object_store_connection_limit: 200,
978                            no_sign_request,
979                            ..Default::default()
980                        },
981                        ObjectStoreType::Azure => ObjectStoreConfig {
982                            object_store: Some(ObjectStoreType::Azure),
983                            bucket: snapshot_bucket,
984                            azure_storage_account: env::var("AZURE_SNAPSHOT_STORAGE_ACCOUNT").ok(),
985                            azure_storage_access_key: env::var("AZURE_SNAPSHOT_STORAGE_ACCESS_KEY")
986                                .ok(),
987                            object_store_connection_limit: 200,
988                            no_sign_request,
989                            ..Default::default()
990                        },
991                        ObjectStoreType::File => {
992                            if snapshot_path.is_some() {
993                                ObjectStoreConfig {
994                                    object_store: Some(ObjectStoreType::File),
995                                    directory: snapshot_path,
996                                    ..Default::default()
997                                }
998                            } else {
999                                panic!(
1000                                    "--snapshot-path must be specified for --snapshot-bucket-type=file"
1001                                );
1002                            }
1003                        }
1004                    }
1005                };
1006
1007                let latest_available_epoch =
1008                    latest.then_some(get_latest_available_epoch(&snapshot_store_config).await?);
1009                let epoch_to_download = epoch.or(latest_available_epoch).expect(
1010                    "Either pass epoch with --epoch <epoch_num> or use latest with --latest",
1011                );
1012
1013                if let Err(e) =
1014                    check_completed_snapshot(&snapshot_store_config, epoch_to_download).await
1015                {
1016                    panic!("Aborting snapshot restore: {e}, snapshot may not be uploaded yet");
1017                }
1018                download_db_snapshot(
1019                    &path,
1020                    epoch_to_download,
1021                    snapshot_store_config,
1022                    skip_indexes,
1023                    num_parallel_downloads,
1024                )
1025                .await?;
1026            }
1027            ToolCommand::Replay {
1028                rpc_url,
1029                safety_checks,
1030                cmd,
1031                use_authority,
1032                cfg_path,
1033                chain,
1034            } => {
1035                execute_replay_command(rpc_url, safety_checks, use_authority, cfg_path, chain, cmd)
1036                    .await?;
1037            }
1038            ToolCommand::VerifyArchive {
1039                genesis,
1040                object_store_config,
1041                download_concurrency,
1042            } => {
1043                verify_archive(&genesis, object_store_config, download_concurrency, true).await?;
1044            }
1045            ToolCommand::PrintArchiveManifest {
1046                object_store_config,
1047            } => {
1048                println!("{}", read_manifest_as_json(object_store_config).await?);
1049            }
1050            ToolCommand::UpdateArchiveManifest {
1051                object_store_config,
1052                archive_json_path,
1053            } => {
1054                write_manifest_from_json(object_store_config, archive_json_path).await?;
1055            }
1056            ToolCommand::VerifyArchiveByChecksum {
1057                object_store_config,
1058                download_concurrency,
1059            } => {
1060                verify_archive_by_checksum(object_store_config, download_concurrency).await?;
1061            }
1062            ToolCommand::DumpArchiveByChecksum {
1063                object_store_config,
1064                start,
1065                end,
1066                max_content_length,
1067            } => {
1068                dump_checkpoints_from_archive(object_store_config, start, end, max_content_length)
1069                    .await?;
1070            }
1071            ToolCommand::SignTransaction {
1072                genesis,
1073                sender_signed_data,
1074            } => {
1075                let genesis = Genesis::load(genesis)?;
1076                let sender_signed_data = bcs::from_bytes::<SenderSignedData>(
1077                    &fastcrypto::encoding::Base64::decode(sender_signed_data.as_str()).unwrap(),
1078                )
1079                .unwrap();
1080                let transaction = Transaction::new(sender_signed_data);
1081                let (agg, _) =
1082                    AuthorityAggregatorBuilder::from_genesis(&genesis).build_network_clients();
1083                let result = agg.process_transaction(transaction, None).await;
1084                println!("{result:?}");
1085            }
1086            ToolCommand::GenesisCeremony(cmd) => {
1087                crate::genesis_ceremony::run(cmd).await?;
1088            }
1089            ToolCommand::FireDrill { fire_drill } => {
1090                crate::fire_drill::run_fire_drill(fire_drill).await?;
1091            }
1092        };
1093        Ok(())
1094    }
1095}
iota_tool/commands.rs

iota_tool/
commands.rs
