iota_data_ingestion_core/history/mod.rs
1// Copyright (c) Mysten Labs, Inc.
2// Modifications Copyright (c) 2024 IOTA Stiftung
3// SPDX-License-Identifier: Apache-2.0
4//! Handle historical checkpoint data.
5//!
6//! Full checkpoint data for epochs starting from genesis are persisted in
7//! batches as blob files in a remote store.
8//!
9//! Files are optionally compressed with the zstd
10//! compression format. Filenames follow the format <checkpoint_seq_num>.chk
11//! where `checkpoint_seq_num` is the first checkpoint present in that
12//! file. MANIFEST is the index and source of truth for all files present in the
13//! ingestion source history.
14//!
15//! Ingestion Source History Directory Layout
16//! ```text
17//! - ingestion/
18//! - historical/
19//! - MANIFEST
20//! - 0.chk
21//! - 1000.chk
22//! - 3000.chk
23//! - ...
24//! - 100000.chk
25//!
26//! Blob File Disk Format
27//! ┌──────────────────────────────┐
28//! │ magic <4 byte> │
29//! ├──────────────────────────────┤
30//! │ storage format <1 byte> │
31//! ├──────────────────────────────┤
32//! │ file compression <1 byte> │
33//! ├──────────────────────────────┤
34//! │ ┌──────────────────────────┐ │
35//! │ │ Blob 1 │ │
36//! │ ├──────────────────────────┤ │
37//! │ │ ... │ │
38//! │ ├──────────────────────────┤ │
39//! │ │ Blob N │ │
40//! │ └──────────────────────────┘ │
41//! └──────────────────────────────┘
42//! Blob
43//! ┌───────────────┬───────────────────┬──────────────┐
44//! │ len <uvarint> │ encoding <1 byte> │ data <bytes> │
45//! └───────────────┴───────────────────┴──────────────┘
46//!
47//! MANIFEST File Disk Format
48//! ┌──────────────────────────────┐
49//! │ magic<4 byte> │
50//! ├──────────────────────────────┤
51//! │ serialized manifest │
52//! ├──────────────────────────────┤
53//! │ sha3 <32 bytes> │
54//! └──────────────────────────────┘
55//! ```
56
57pub mod manifest;
58pub mod reader;
59
60pub const CHECKPOINT_FILE_MAGIC: u32 = 0x0000BEEF;
61pub const CHECKPOINT_FILE_SUFFIX: &str = "chk";
62const HISTORICAL_DIR_NAME: &str = "historical";
63const INGESTION_DIR_NAME: &str = "ingestion";
64pub const MAGIC_BYTES: usize = 4;
65pub const MANIFEST_FILE_MAGIC: u32 = 0x0000FACE;
66pub const MANIFEST_FILENAME: &str = "MANIFEST";