iota_tool/db_tool/
mod.rs

1// Copyright (c) Mysten Labs, Inc.
2// Modifications Copyright (c) 2024 IOTA Stiftung
3// SPDX-License-Identifier: Apache-2.0
4
5use std::path::{Path, PathBuf};
6
7use anyhow::{anyhow, bail};
8use clap::Parser;
9use iota_core::{
10    authority::{
11        authority_per_epoch_store::AuthorityEpochTables,
12        authority_store_tables::AuthorityPerpetualTables,
13    },
14    checkpoints::CheckpointStore,
15};
16use iota_sdk_types::{CheckpointContentsDigest, CheckpointDigest, ObjectId, TransactionDigest};
17use iota_types::{
18    base_types::EpochId, effects::TransactionEffectsAPI,
19    messages_checkpoint::CheckpointSequenceNumber, storage::ObjectStore,
20};
21use typed_store::rocks::MetricConf;
22
23use self::{
24    db_dump::{StoreName, dump_table, duplicate_objects_summary, list_tables, table_summary},
25    index_search::{SearchRange, search_index},
26};
27use crate::db_tool::db_dump::{compact, print_table_metadata, prune_checkpoints, prune_objects};
28pub mod db_dump;
29mod index_search;
30
31#[derive(Parser)]
32pub enum DbToolCommand {
33    ListTables,
34    Dump(Options),
35    IndexSearchKeyRange(IndexSearchKeyRangeOptions),
36    IndexSearchCount(IndexSearchCountOptions),
37    TableSummary(Options),
38    DuplicatesSummary,
39    ListDBMetadata(Options),
40    PrintLastConsensusIndex,
41    PrintTransaction(PrintTransactionOptions),
42    PrintObject(PrintObjectOptions),
43    PrintCheckpoint(PrintCheckpointOptions),
44    PrintCheckpointContent(PrintCheckpointContentOptions),
45    RewindCheckpointExecution(RewindCheckpointExecutionOptions),
46    Compact,
47    PruneObjects,
48    PruneCheckpoints,
49    SetCheckpointWatermark(SetCheckpointWatermarkOptions),
50}
51
52#[derive(Parser)]
53pub struct IndexSearchKeyRangeOptions {
54    #[arg(long, short = 't')]
55    table_name: String,
56    #[arg(long, short = 's')]
57    start: String,
58    #[arg(long = "end", short = 'e')]
59    end_key: String,
60}
61
62#[derive(Parser)]
63pub struct IndexSearchCountOptions {
64    #[arg(long, short = 't')]
65    table_name: String,
66    #[arg(long, short = 's')]
67    start: String,
68    #[arg(long, short = 'c')]
69    count: u64,
70}
71
72#[derive(Parser)]
73pub struct Options {
74    /// The type of store to dump
75    #[arg(long = "store", short = 's', value_enum)]
76    store_name: StoreName,
77    /// The name of the table to dump
78    #[arg(long, short = 't')]
79    table_name: String,
80    /// The size of page to dump. This is a u16
81    #[arg(long, short = 'p')]
82    page_size: u16,
83    /// The page number to dump
84    #[arg(long = "page-num", short = 'n')]
85    page_number: usize,
86
87    // TODO: We should load this automatically from the system object in AuthorityPerpetualTables.
88    // This is very difficult to do right now because you can't share code between
89    // AuthorityPerpetualTables and AuthorityEpochTablesReadonly.
90    /// The epoch to use when loading AuthorityEpochTables.
91    #[arg(long, short = 'e')]
92    epoch: Option<EpochId>,
93}
94
95#[derive(Parser)]
96pub struct PrintTransactionOptions {
97    #[arg(long, help = "The transaction digest to print")]
98    digest: TransactionDigest,
99}
100
101#[derive(Parser)]
102pub struct PrintObjectOptions {
103    #[arg(long, help = "The object id to print")]
104    id: ObjectId,
105    #[arg(long, help = "The object version to print")]
106    version: Option<u64>,
107}
108
109#[derive(Parser)]
110pub struct PrintCheckpointOptions {
111    #[arg(long, help = "The checkpoint digest to print")]
112    digest: CheckpointDigest,
113}
114
115#[derive(Parser)]
116pub struct PrintCheckpointContentOptions {
117    #[arg(
118        long,
119        help = "The checkpoint content digest (NOT the checkpoint digest)"
120    )]
121    digest: CheckpointContentsDigest,
122}
123
124#[derive(Parser)]
125pub struct RemoveTransactionOptions {
126    #[arg(long, help = "The transaction digest to remove")]
127    digest: TransactionDigest,
128
129    #[arg(long)]
130    confirm: bool,
131
132    /// The epoch to use when loading AuthorityEpochTables.
133    /// Defaults to the current epoch.
134    #[arg(long, short = 'e')]
135    epoch: Option<EpochId>,
136}
137
138#[derive(Parser)]
139pub struct RemoveObjectLockOptions {
140    #[arg(long, help = "The object ID to remove")]
141    id: ObjectId,
142
143    #[arg(long, help = "The object version to remove")]
144    version: u64,
145
146    #[arg(long)]
147    confirm: bool,
148}
149
150#[derive(Parser)]
151pub struct RewindCheckpointExecutionOptions {
152    #[arg(long)]
153    epoch: EpochId,
154
155    #[arg(long)]
156    checkpoint_sequence_number: u64,
157}
158
159#[derive(Parser)]
160pub struct SetCheckpointWatermarkOptions {
161    #[arg(long)]
162    highest_verified: Option<CheckpointSequenceNumber>,
163
164    #[arg(long)]
165    highest_synced: Option<CheckpointSequenceNumber>,
166}
167
168pub async fn execute_db_tool_command(db_path: PathBuf, cmd: DbToolCommand) -> anyhow::Result<()> {
169    match cmd {
170        DbToolCommand::ListTables => print_db_all_tables(db_path),
171        DbToolCommand::Dump(d) => print_all_entries(
172            d.store_name,
173            d.epoch,
174            db_path,
175            &d.table_name,
176            d.page_size,
177            d.page_number,
178        ),
179        DbToolCommand::TableSummary(d) => {
180            print_db_table_summary(d.store_name, d.epoch, db_path, &d.table_name)
181        }
182        DbToolCommand::DuplicatesSummary => print_db_duplicates_summary(db_path),
183        DbToolCommand::ListDBMetadata(d) => {
184            print_table_metadata(d.store_name, d.epoch, db_path, &d.table_name)
185        }
186        DbToolCommand::PrintLastConsensusIndex => print_last_consensus_index(&db_path),
187        DbToolCommand::PrintTransaction(d) => print_transaction(&db_path, d),
188        DbToolCommand::PrintObject(o) => print_object(&db_path, o),
189        DbToolCommand::PrintCheckpoint(d) => print_checkpoint(&db_path, d),
190        DbToolCommand::PrintCheckpointContent(d) => print_checkpoint_content(&db_path, d),
191        DbToolCommand::RewindCheckpointExecution(d) => {
192            rewind_checkpoint_execution(&db_path, d.epoch, d.checkpoint_sequence_number)
193        }
194        DbToolCommand::Compact => compact(db_path),
195        DbToolCommand::PruneObjects => prune_objects(db_path).await,
196        DbToolCommand::PruneCheckpoints => prune_checkpoints(db_path).await,
197        DbToolCommand::IndexSearchKeyRange(rg) => {
198            let res = search_index(
199                db_path,
200                rg.table_name,
201                rg.start,
202                SearchRange::ExclusiveLastKey(rg.end_key),
203            )?;
204            for (k, v) in res {
205                println!("{k}: {v}");
206            }
207            Ok(())
208        }
209        DbToolCommand::IndexSearchCount(sc) => {
210            let res = search_index(
211                db_path,
212                sc.table_name,
213                sc.start,
214                SearchRange::Count(sc.count),
215            )?;
216            for (k, v) in res {
217                println!("{k}: {v}");
218            }
219            Ok(())
220        }
221        DbToolCommand::SetCheckpointWatermark(d) => set_checkpoint_watermark(&db_path, d),
222    }
223}
224
225pub fn print_db_all_tables(db_path: PathBuf) -> anyhow::Result<()> {
226    list_tables(db_path)?.iter().for_each(|t| println!("{t}"));
227    Ok(())
228}
229
230pub fn print_db_duplicates_summary(db_path: PathBuf) -> anyhow::Result<()> {
231    let (total_count, duplicate_count, total_bytes, duplicated_bytes) =
232        duplicate_objects_summary(db_path)?;
233    println!(
234        "Total objects = {total_count}, duplicated objects = {duplicate_count}, total bytes = {total_bytes}, duplicated bytes = {duplicated_bytes}"
235    );
236    Ok(())
237}
238
239pub fn print_last_consensus_index(path: &Path) -> anyhow::Result<()> {
240    let epoch_tables = AuthorityEpochTables::open_tables_read_write(
241        path.to_path_buf(),
242        MetricConf::default(),
243        None,
244        None,
245    );
246    let last_index = epoch_tables.get_last_consensus_index()?;
247    println!("Last consensus index is {last_index:?}");
248    Ok(())
249}
250
251pub fn print_transaction(path: &Path, opt: PrintTransactionOptions) -> anyhow::Result<()> {
252    let perpetual_db = AuthorityPerpetualTables::open(&path.join("store"), None);
253    if let Some((epoch, checkpoint_seq_num)) =
254        perpetual_db.get_checkpoint_sequence_number(&opt.digest)?
255    {
256        println!(
257            "Transaction {:?} executed in epoch {} checkpoint {}",
258            opt.digest, epoch, checkpoint_seq_num
259        );
260    };
261    if let Some(effects) = perpetual_db.get_effects(&opt.digest)? {
262        println!(
263            "Transaction {:?} dependencies: {:#?}",
264            opt.digest,
265            effects.dependencies(),
266        );
267    };
268    Ok(())
269}
270
271pub fn print_object(path: &Path, opt: PrintObjectOptions) -> anyhow::Result<()> {
272    let perpetual_db = AuthorityPerpetualTables::open(&path.join("store"), None);
273
274    let obj = if let Some(version) = opt.version {
275        perpetual_db.try_get_object_by_key(&opt.id, version.into())?
276    } else {
277        perpetual_db.try_get_object(&opt.id)?
278    };
279
280    if let Some(obj) = obj {
281        println!("Object {:?}:\n{:#?}", opt.id, obj);
282    } else {
283        println!("Object {:?} not found", opt.id);
284    }
285
286    Ok(())
287}
288
289pub fn print_checkpoint(path: &Path, opt: PrintCheckpointOptions) -> anyhow::Result<()> {
290    let checkpoint_store = CheckpointStore::new(&path.join("checkpoints"));
291    let checkpoint = checkpoint_store
292        .get_checkpoint_by_digest(&opt.digest)?
293        .ok_or(anyhow!(
294            "Checkpoint digest {:?} not found in checkpoint store",
295            opt.digest
296        ))?;
297    println!("Checkpoint: {checkpoint:?}");
298    drop(checkpoint_store);
299    print_checkpoint_content(
300        path,
301        PrintCheckpointContentOptions {
302            digest: checkpoint.contents_digest,
303        },
304    )
305}
306
307pub fn print_checkpoint_content(
308    path: &Path,
309    opt: PrintCheckpointContentOptions,
310) -> anyhow::Result<()> {
311    let checkpoint_store = CheckpointStore::new(&path.join("checkpoints"));
312    let contents = checkpoint_store
313        .get_checkpoint_contents(&opt.digest)?
314        .ok_or(anyhow!(
315            "Checkpoint content digest {:?} not found in checkpoint store",
316            opt.digest
317        ))?;
318    println!("Checkpoint content: {contents:?}");
319    Ok(())
320}
321
322/// Force sets the highest executed checkpoint.
323/// NOTE: Does not force re-execution of transactions.
324/// Run with: cargo run --package iota-tool -- db-tool --db-path
325/// /opt/iota/db/authorities_db/live rewind-checkpoint-execution --epoch 3
326/// --checkpoint-sequence-number 300000
327pub fn rewind_checkpoint_execution(
328    path: &Path,
329    epoch: EpochId,
330    checkpoint_sequence_number: u64,
331) -> anyhow::Result<()> {
332    let checkpoint_db = CheckpointStore::new(&path.join("checkpoints"));
333    let Some(checkpoint) =
334        checkpoint_db.get_checkpoint_by_sequence_number(checkpoint_sequence_number)?
335    else {
336        bail!("Checkpoint {checkpoint_sequence_number} not found!");
337    };
338    if epoch != checkpoint.epoch() {
339        bail!(
340            "Checkpoint {checkpoint_sequence_number} is in epoch {} not {epoch}!",
341            checkpoint.epoch()
342        );
343    }
344
345    let highest_executed_sequence_number = checkpoint_db
346        .get_highest_executed_checkpoint_seq_number()?
347        .unwrap_or_default();
348    if checkpoint_sequence_number > highest_executed_sequence_number {
349        bail!(
350            "Must rewind checkpoint execution to be not later than highest executed ({checkpoint_sequence_number} > {highest_executed_sequence_number})!"
351        );
352    }
353    checkpoint_db.set_highest_executed_checkpoint_subtle(&checkpoint)?;
354    Ok(())
355}
356
357pub fn print_db_table_summary(
358    store: StoreName,
359    epoch: Option<EpochId>,
360    path: PathBuf,
361    table_name: &str,
362) -> anyhow::Result<()> {
363    let summary = table_summary(store, epoch, path, table_name)?;
364    let quantiles = [25, 50, 75, 90, 99];
365    println!(
366        "Total num keys = {}, total key bytes = {}, total value bytes = {}",
367        summary.num_keys, summary.key_bytes_total, summary.value_bytes_total
368    );
369    println!("Key size distribution:\n");
370    quantiles.iter().for_each(|q| {
371        println!(
372            "p{:?} -> {:?} bytes\n",
373            q,
374            summary.key_hist.value_at_quantile(*q as f64 / 100.0)
375        );
376    });
377    println!("Value size distribution:\n");
378    quantiles.iter().for_each(|q| {
379        println!(
380            "p{:?} -> {:?} bytes\n",
381            q,
382            summary.value_hist.value_at_quantile(*q as f64 / 100.0)
383        );
384    });
385    Ok(())
386}
387
388pub fn print_all_entries(
389    store: StoreName,
390    epoch: Option<EpochId>,
391    path: PathBuf,
392    table_name: &str,
393    page_size: u16,
394    page_number: usize,
395) -> anyhow::Result<()> {
396    for (k, v) in dump_table(store, epoch, path, table_name, page_size, page_number)? {
397        println!("{k:>100?}: {v:?}");
398    }
399    Ok(())
400}
401
402/// Force sets state sync checkpoint watermarks.
403/// Run with (for example):
404/// cargo run --package iota-tool -- db-tool --db-path
405/// /opt/iota/db/authorities_db/live set_checkpoint_watermark --highest-synced
406/// 300000
407pub fn set_checkpoint_watermark(
408    path: &Path,
409    options: SetCheckpointWatermarkOptions,
410) -> anyhow::Result<()> {
411    let checkpoint_db = CheckpointStore::new(&path.join("checkpoints"));
412
413    if let Some(highest_verified) = options.highest_verified {
414        let Some(checkpoint) = checkpoint_db.get_checkpoint_by_sequence_number(highest_verified)?
415        else {
416            bail!("Checkpoint {highest_verified} not found");
417        };
418        checkpoint_db.set_highest_verified_checkpoint_subtle(&checkpoint)?;
419    }
420    if let Some(highest_synced) = options.highest_synced {
421        let Some(checkpoint) = checkpoint_db.get_checkpoint_by_sequence_number(highest_synced)?
422        else {
423            bail!("Checkpoint {highest_synced} not found");
424        };
425        // Verify that checkpoint contents exist, since the checkpoint executor
426        // will panic if it tries to execute a checkpoint whose contents are
427        // missing from the store.
428        if checkpoint_db
429            .get_checkpoint_contents(&checkpoint.contents_digest)?
430            .is_none()
431        {
432            bail!(
433                "Checkpoint contents not found for checkpoint {highest_synced}. \
434                 Setting highest_synced without contents would cause the executor to panic."
435            );
436        }
437        checkpoint_db.set_highest_synced_checkpoint_subtle(&checkpoint)?;
438    }
439    Ok(())
440}
iota_tool/db_tool/mod.rs

iota_tool/db_tool/
mod.rs
