Skip to main content

iota_data_ingestion_core/
executor.rs

1// Copyright (c) Mysten Labs, Inc.
2// Modifications Copyright (c) 2024 IOTA Stiftung
3// SPDX-License-Identifier: Apache-2.0
4
5use std::{pin::Pin, sync::Arc};
6
7use futures::Future;
8use iota_metrics::spawn_monitored_task;
9use iota_types::{
10    committee::EpochId, full_checkpoint_content::CheckpointData,
11    messages_checkpoint::CheckpointSequenceNumber,
12};
13use prometheus_filtered::Registry;
14use tokio::{sync::mpsc, task::JoinHandle};
15use tokio_util::sync::CancellationToken;
16use tracing::info;
17
18use crate::{
19    DataIngestionMetrics, IngestionError, IngestionResult, ReaderOptions, Worker,
20    config::CheckpointReaderConfigExt,
21    progress_store::{ExecutorProgress, ProgressStore, ProgressStoreWrapper, ShimProgressStore},
22    reader::v2::{CheckpointReader, CheckpointReaderConfig, RemoteUrl},
23    worker_pool::{WorkerPool, WorkerPoolStatus},
24};
25
26pub const MAX_CHECKPOINTS_IN_PROGRESS: usize = 10000;
27
28/// Callback function invoked for each incoming checkpoint to determine the
29/// shutdown action if it exceeds the ingestion limit.
30type ShutdownCallback = Box<dyn Fn(&CheckpointData) -> ShutdownAction + Send>;
31
32/// Determines the shutdown action when a checkpoint reaches the ingestion
33/// limit.
34#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
35pub enum ShutdownAction {
36    /// Include the current checkpoint in the ingestion process, then initiate
37    /// the graceful shutdown process.
38    IncludeAndShutdown,
39    /// Exclude the current checkpoint from ingestion and immediately initiate
40    /// the graceful shutdown process.
41    ExcludeAndShutdown,
42    /// Continue processing the current checkpoint without shutting down.
43    Continue,
44}
45
46/// Common policies for upper limit checkpoint ingestion by the framework.
47///
48/// Once the limit is reached, the framework will start the graceful
49/// shutdown process.
50#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
51#[non_exhaustive]
52pub enum IngestionLimit {
53    /// Last checkpoint sequence number to process.
54    ///
55    /// After processing this checkpoint the framework will start the graceful
56    /// shutdown process.
57    MaxCheckpoint(CheckpointSequenceNumber),
58    /// Last checkpoint to process based on the given epoch.
59    ///
60    /// After processing this checkpoint the framework will start the graceful
61    /// shutdown process.
62    EndOfEpoch(EpochId),
63}
64
65impl IngestionLimit {
66    /// Evaluates whether the given checkpoint triggers a shutdown action based
67    /// on the ingestion limit.
68    fn matches(&self, checkpoint: &CheckpointData) -> ShutdownAction {
69        match self {
70            IngestionLimit::MaxCheckpoint(max) => {
71                if &checkpoint.checkpoint_summary.sequence_number > max {
72                    return ShutdownAction::ExcludeAndShutdown;
73                }
74                ShutdownAction::Continue
75            }
76            IngestionLimit::EndOfEpoch(max) => {
77                if &checkpoint.checkpoint_summary.epoch > max {
78                    return ShutdownAction::ExcludeAndShutdown;
79                }
80                ShutdownAction::Continue
81            }
82        }
83    }
84}
85
86/// The Executor of the main ingestion pipeline process.
87///
88/// This struct orchestrates the execution of multiple worker pools, handling
89/// checkpoint distribution, progress tracking, and shutdown. It utilizes
90/// [`ProgressStore`] for persisting checkpoint progress and provides metrics
91/// for monitoring the indexing process.
92///
93/// # Example
94/// ```rust,no_run
95/// use async_trait::async_trait;
96/// use iota_data_ingestion_core::{
97///     DataIngestionMetrics, FileProgressStore, IndexerExecutor, IngestionError, ReaderOptions,
98///     Worker, WorkerPool,
99///     reader::v2::CheckpointReaderConfig
100/// };
101/// use iota_types::full_checkpoint_content::CheckpointData;
102/// use prometheus_filtered::Registry;
103/// use tokio_util::sync::CancellationToken;
104/// use std::{path::PathBuf, sync::Arc};
105///
106/// struct CustomWorker;
107///
108/// #[async_trait]
109/// impl Worker for CustomWorker {
110///     type Message = ();
111///     type Error = IngestionError;
112///
113///     async fn process_checkpoint(
114///         &self,
115///         checkpoint: Arc<CheckpointData>,
116///     ) -> Result<Self::Message, Self::Error> {
117///         // custom processing logic.
118///         println!(
119///             "Processing Local checkpoint: {:?}",
120///             checkpoint.checkpoint_summary
121///         );
122///         Ok(())
123///     }
124/// }
125///
126/// #[tokio::main]
127/// async fn main() {
128///     let concurrency = 5;
129///     let progress_store = FileProgressStore::new("progress.json").await.unwrap();
130///     let mut executor = IndexerExecutor::new(
131///         progress_store,
132///         1, // number of registered WorkerPools.
133///         DataIngestionMetrics::new(&Registry::new()),
134///         CancellationToken::new(),
135///     );
136///     // register a worker pool with 5 workers to process checkpoints in parallel
137///     let worker_pool = WorkerPool::new(CustomWorker, "local_reader".to_string(), concurrency, Default::default());
138///     // register the worker pool to the executor.
139///     executor.register(worker_pool).await.unwrap();
140///     // run the ingestion pipeline.
141///     executor
142///         .run_with_config(
143///             CheckpointReaderConfig {
144///                 reader_options: ReaderOptions::default(),
145///                 ingestion_path: Some(PathBuf::from("./chk".to_string())), // path to a local directory where checkpoints are stored.
146///                 remote_store_url: None,
147///             }
148///         )
149///         .await
150///         .unwrap();
151/// }
152/// ```
153pub struct IndexerExecutor<P> {
154    pools: Vec<Pin<Box<dyn Future<Output = ()> + Send>>>,
155    pool_senders: Vec<mpsc::Sender<Arc<CheckpointData>>>,
156    progress_store: ProgressStoreWrapper<P>,
157    pool_status_sender: mpsc::Sender<WorkerPoolStatus>,
158    pool_status_receiver: mpsc::Receiver<WorkerPoolStatus>,
159    metrics: DataIngestionMetrics,
160    token: CancellationToken,
161    shutdown_callback: Option<ShutdownCallback>,
162}
163
164impl<P: ProgressStore> IndexerExecutor<P> {
165    pub fn new(
166        progress_store: P,
167        number_of_jobs: usize,
168        metrics: DataIngestionMetrics,
169        token: CancellationToken,
170    ) -> Self {
171        let (pool_status_sender, pool_status_receiver) =
172            mpsc::channel(number_of_jobs * MAX_CHECKPOINTS_IN_PROGRESS);
173        Self {
174            pools: vec![],
175            pool_senders: vec![],
176            progress_store: ProgressStoreWrapper::new(progress_store),
177            pool_status_sender,
178            pool_status_receiver,
179            metrics,
180            token,
181            shutdown_callback: None,
182        }
183    }
184
185    /// Registers new worker pool in executor.
186    pub async fn register<W: Worker + 'static>(
187        &mut self,
188        pool: WorkerPool<W>,
189    ) -> IngestionResult<()> {
190        let checkpoint_number = self.progress_store.load(pool.task_name.clone()).await?;
191        let (sender, receiver) = mpsc::channel(MAX_CHECKPOINTS_IN_PROGRESS);
192        self.pools.push(Box::pin(pool.run(
193            checkpoint_number,
194            receiver,
195            self.pool_status_sender.clone(),
196            self.token.child_token(),
197        )));
198        self.pool_senders.push(sender);
199        Ok(())
200    }
201
202    /// Registers a predicate callback that determines when the ingestion
203    /// process should stop.
204    ///
205    /// This function `f` will be called for every **incoming checkpoint**
206    /// before it’s sent to the worker pool.
207    ///
208    /// Based on the returned [`ShutdownAction`] the executor will evaluate
209    /// whether to continue or stop the ingestion process by initiating the
210    /// graceful shutdown process.
211    ///
212    /// Once a shutdown action is triggered, the executor will stop sending new
213    /// checkpoints and will wait for all previously sent checkpoints to be
214    /// processed by workers before initiating graceful shutdown process.
215    ///
216    /// Note:
217    ///
218    /// Calling this method after
219    /// [`with_ingestion_limit`](Self::with_ingestion_limit) replaces the
220    /// earlier predicate, and vice versa. They are not cumulative.
221    pub fn shutdown_when<F>(&mut self, f: F)
222    where
223        F: Fn(&CheckpointData) -> ShutdownAction + Send + 'static,
224    {
225        self.shutdown_callback = Some(Box::new(f));
226    }
227
228    /// Adds an upper‑limit policy that determines when the ingestion
229    /// process should stop.
230    ///
231    /// This is a convenience method, it internally uses
232    /// [`shutdown_when`](Self::shutdown_when) by registering a predicate
233    /// derived from the provided [`IngestionLimit`].
234    ///
235    /// Note:
236    ///
237    /// Calling this method after [`shutdown_when`](Self::shutdown_when)
238    /// replaces the earlier predicate, and vice versa. They are not cumulative.
239    pub fn with_ingestion_limit(&mut self, limit: IngestionLimit) {
240        self.shutdown_when(move |checkpoint| limit.matches(checkpoint));
241    }
242
243    pub async fn update_watermark(
244        &mut self,
245        task_name: String,
246        watermark: CheckpointSequenceNumber,
247    ) -> IngestionResult<()> {
248        self.progress_store.save(task_name, watermark).await
249    }
250
251    pub async fn read_watermark(
252        &mut self,
253        task_name: String,
254    ) -> IngestionResult<CheckpointSequenceNumber> {
255        self.progress_store.load(task_name).await
256    }
257
258    /// Main executor loop.
259    ///
260    /// # Error
261    ///
262    /// Returns an [`IngestionError::EmptyWorkerPool`] if no worker pool was
263    /// registered.
264    pub async fn run_with_config(
265        mut self,
266        config: impl Into<CheckpointReaderConfigExt>,
267    ) -> IngestionResult<ExecutorProgress> {
268        let reader_checkpoint_number = self.progress_store.min_watermark()?;
269        let checkpoint_reader =
270            CheckpointReader::new(reader_checkpoint_number, config.into()).await?;
271
272        self.run_executor_loop(reader_checkpoint_number, checkpoint_reader)
273            .await
274    }
275
276    /// Common execution logic
277    async fn run_executor_loop(
278        &mut self,
279        mut reader_checkpoint_number: u64,
280        mut checkpoint_reader: CheckpointReader,
281    ) -> IngestionResult<ExecutorProgress> {
282        let worker_pools = std::mem::take(&mut self.pools)
283            .into_iter()
284            .map(|pool| spawn_monitored_task!(pool))
285            .collect::<Vec<JoinHandle<()>>>();
286
287        let mut worker_pools_shutdown_signals = vec![];
288        let mut checkpoint_limit_reached = None;
289        let mut reader_error = None;
290
291        loop {
292            // the min watermark represents the lowest watermark that
293            // has been processed by any worker pool. This guarantees that
294            // all worker pools have processed the checkpoint before the
295            // shutdown process starts.
296            if checkpoint_limit_reached.is_some_and(|ch_seq_num| {
297                self.progress_store
298                    .min_watermark()
299                    .map(|watermark| watermark > ch_seq_num)
300                    .unwrap_or_default()
301            }) {
302                info!(
303                    "checkpoint upper limit reached: last checkpoint was processed, shutdown process started"
304                );
305                self.token.cancel();
306            }
307
308            tokio::select! {
309                Some(worker_pool_progress_msg) = self.pool_status_receiver.recv() => {
310                    match worker_pool_progress_msg {
311                        WorkerPoolStatus::Running((task_name, watermark)) => {
312                            self.progress_store.save(task_name.clone(), watermark).await
313                                .map_err(|err| IngestionError::ProgressStore(err.to_string()))?;
314                            let seq_number = self.progress_store.min_watermark()?;
315                            if seq_number > reader_checkpoint_number {
316                                checkpoint_reader.send_gc_signal(seq_number)?;
317                                reader_checkpoint_number = seq_number;
318                            }
319                            self.metrics.data_ingestion_checkpoint
320                                .with_label_values(&[&task_name])
321                                .set(watermark as i64);
322                        }
323                        WorkerPoolStatus::Shutdown(worker_pool_name) => {
324                            worker_pools_shutdown_signals.push(worker_pool_name);
325                        }
326                    }
327                }
328                checkpoint = checkpoint_reader.checkpoint(), if !self.token.is_cancelled() => {
329                    let Some(checkpoint) = checkpoint else {
330                        reader_error = Some(IngestionError::Channel("checkpoint reader stopped".into()));
331                        // start garceful shutdown sequence.
332                        self.token.cancel();
333                        continue;
334                    };
335                    // once upper limit reached skip sending new checkpoints to workers.
336                    if self.should_shutdown(&checkpoint, &mut checkpoint_limit_reached) {
337                        continue;
338                    }
339
340                    for sender in &self.pool_senders {
341                        sender.send(checkpoint.clone()).await.map_err(|_| {
342                            IngestionError::Channel(
343                                "unable to send new checkpoint to worker pool, receiver half closed".to_owned(),
344                            )
345                        })?;
346                    }
347                }
348            }
349
350            if worker_pools_shutdown_signals.len() == self.pool_senders.len() {
351                // Shutdown worker pools
352                for worker_pool in worker_pools {
353                    worker_pool.await.map_err(|err| IngestionError::Shutdown {
354                        component: "worker pool".into(),
355                        msg: err.to_string(),
356                    })?;
357                }
358                // Shutdown checkpoint reader
359                checkpoint_reader.shutdown().await?;
360                break;
361            }
362        }
363
364        match reader_error {
365            Some(err) => Err(err),
366            None => Ok(self.progress_store.stats()),
367        }
368    }
369
370    /// Check if the current ingestion limit has been reached.
371    ///
372    /// Returns `true` if the ingestion limit has been reached.
373    /// If no ingestion limit is present or it has not been reached yet, the
374    /// function returns `false`.
375    fn should_shutdown(
376        &mut self,
377        checkpoint: &CheckpointData,
378        checkpoint_limit_reached: &mut Option<CheckpointSequenceNumber>,
379    ) -> bool {
380        if checkpoint_limit_reached.is_some() {
381            return true;
382        }
383
384        let Some(shutdown_action) = self
385            .shutdown_callback
386            .as_ref()
387            .map(|matches| matches(checkpoint))
388        else {
389            return false;
390        };
391
392        match shutdown_action {
393            ShutdownAction::IncludeAndShutdown => {
394                checkpoint_limit_reached
395                    .get_or_insert(checkpoint.checkpoint_summary.sequence_number);
396                false
397            }
398            ShutdownAction::ExcludeAndShutdown => {
399                checkpoint_limit_reached.get_or_insert(
400                    checkpoint
401                        .checkpoint_summary
402                        .sequence_number
403                        .saturating_sub(1),
404                );
405                true
406            }
407            ShutdownAction::Continue => false,
408        }
409    }
410}
411
412/// Sets up a single workflow for data ingestion.
413///
414/// This function initializes an [`IndexerExecutor`] with a single worker pool,
415/// using a [`ShimProgressStore`] initialized with the provided
416/// `initial_checkpoint_number`. It then returns a future that runs the executor
417/// and a [`CancellationToken`] for graceful shutdown.
418///
419/// # Docs
420/// For more info please check the [custom indexer docs](https://docs.iota.org/developer/advanced/custom-indexer).
421///
422/// # Example
423/// ```rust,no_run
424/// use std::sync::Arc;
425///
426/// use async_trait::async_trait;
427/// use iota_data_ingestion_core::{
428///     IngestionError, Worker, reader::v2::RemoteUrl, setup_single_workflow,
429/// };
430/// use iota_types::full_checkpoint_content::CheckpointData;
431///
432/// struct CustomWorker;
433///
434/// #[async_trait]
435/// impl Worker for CustomWorker {
436///     type Message = ();
437///     type Error = IngestionError;
438///
439///     async fn process_checkpoint(
440///         &self,
441///         checkpoint: Arc<CheckpointData>,
442///     ) -> Result<Self::Message, Self::Error> {
443///         // custom processing logic.
444///         println!("Processing checkpoint: {:?}", checkpoint.checkpoint_summary);
445///         Ok(())
446///     }
447/// }
448///
449/// #[tokio::main]
450/// async fn main() {
451///     let (executor, _) = setup_single_workflow(
452///         CustomWorker,
453///         RemoteUrl::Fullnode("http://127.0.0.1:50051".into()), // fullnode gRPC API.
454///         0,                                                    // initial checkpoint number.
455///         5,                                                    // concurrency.
456///         None,                                                 // extra reader options.
457///     )
458///     .await
459///     .unwrap();
460///     executor.await.unwrap();
461/// }
462/// ```
463pub async fn setup_single_workflow<W: Worker + 'static>(
464    worker: W,
465    remote_store_url: RemoteUrl,
466    initial_checkpoint_number: CheckpointSequenceNumber,
467    concurrency: usize,
468    reader_options: Option<ReaderOptions>,
469) -> IngestionResult<(
470    impl Future<Output = IngestionResult<ExecutorProgress>>,
471    CancellationToken,
472)> {
473    let metrics = DataIngestionMetrics::new(&Registry::new());
474    let progress_store = ShimProgressStore(initial_checkpoint_number);
475    let token = CancellationToken::new();
476    let mut executor = IndexerExecutor::new(progress_store, 1, metrics, token.child_token());
477    let worker_pool = WorkerPool::new(
478        worker,
479        "workflow".to_string(),
480        concurrency,
481        Default::default(),
482    );
483    executor.register(worker_pool).await?;
484    Ok((
485        executor.run_with_config(CheckpointReaderConfig {
486            reader_options: reader_options.unwrap_or_default(),
487            ingestion_path: None,
488            remote_store_url: Some(remote_store_url),
489        }),
490        token,
491    ))
492}