iota_data_ingestion_core/executor.rs
1// Copyright (c) Mysten Labs, Inc.
2// Modifications Copyright (c) 2024 IOTA Stiftung
3// SPDX-License-Identifier: Apache-2.0
4
5use std::{pin::Pin, sync::Arc};
6
7use futures::Future;
8use iota_metrics::spawn_monitored_task;
9use iota_types::{
10 committee::EpochId, full_checkpoint_content::CheckpointData,
11 messages_checkpoint::CheckpointSequenceNumber,
12};
13use prometheus_filtered::Registry;
14use tokio::{sync::mpsc, task::JoinHandle};
15use tokio_util::sync::CancellationToken;
16use tracing::info;
17
18use crate::{
19 DataIngestionMetrics, IngestionError, IngestionResult, ReaderOptions, Worker,
20 config::CheckpointReaderConfigExt,
21 progress_store::{ExecutorProgress, ProgressStore, ProgressStoreWrapper, ShimProgressStore},
22 reader::v2::{CheckpointReader, CheckpointReaderConfig, RemoteUrl},
23 worker_pool::{WorkerPool, WorkerPoolStatus},
24};
25
26pub const MAX_CHECKPOINTS_IN_PROGRESS: usize = 10000;
27
28/// Callback function invoked for each incoming checkpoint to determine the
29/// shutdown action if it exceeds the ingestion limit.
30type ShutdownCallback = Box<dyn Fn(&CheckpointData) -> ShutdownAction + Send>;
31
32/// Determines the shutdown action when a checkpoint reaches the ingestion
33/// limit.
34#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
35pub enum ShutdownAction {
36 /// Include the current checkpoint in the ingestion process, then initiate
37 /// the graceful shutdown process.
38 IncludeAndShutdown,
39 /// Exclude the current checkpoint from ingestion and immediately initiate
40 /// the graceful shutdown process.
41 ExcludeAndShutdown,
42 /// Continue processing the current checkpoint without shutting down.
43 Continue,
44}
45
46/// Common policies for upper limit checkpoint ingestion by the framework.
47///
48/// Once the limit is reached, the framework will start the graceful
49/// shutdown process.
50#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
51#[non_exhaustive]
52pub enum IngestionLimit {
53 /// Last checkpoint sequence number to process.
54 ///
55 /// After processing this checkpoint the framework will start the graceful
56 /// shutdown process.
57 MaxCheckpoint(CheckpointSequenceNumber),
58 /// Last checkpoint to process based on the given epoch.
59 ///
60 /// After processing this checkpoint the framework will start the graceful
61 /// shutdown process.
62 EndOfEpoch(EpochId),
63}
64
65impl IngestionLimit {
66 /// Evaluates whether the given checkpoint triggers a shutdown action based
67 /// on the ingestion limit.
68 fn matches(&self, checkpoint: &CheckpointData) -> ShutdownAction {
69 match self {
70 IngestionLimit::MaxCheckpoint(max) => {
71 if &checkpoint.checkpoint_summary.sequence_number > max {
72 return ShutdownAction::ExcludeAndShutdown;
73 }
74 ShutdownAction::Continue
75 }
76 IngestionLimit::EndOfEpoch(max) => {
77 if &checkpoint.checkpoint_summary.epoch > max {
78 return ShutdownAction::ExcludeAndShutdown;
79 }
80 ShutdownAction::Continue
81 }
82 }
83 }
84}
85
86/// The Executor of the main ingestion pipeline process.
87///
88/// This struct orchestrates the execution of multiple worker pools, handling
89/// checkpoint distribution, progress tracking, and shutdown. It utilizes
90/// [`ProgressStore`] for persisting checkpoint progress and provides metrics
91/// for monitoring the indexing process.
92///
93/// # Example
94/// ```rust,no_run
95/// use async_trait::async_trait;
96/// use iota_data_ingestion_core::{
97/// DataIngestionMetrics, FileProgressStore, IndexerExecutor, IngestionError, ReaderOptions,
98/// Worker, WorkerPool,
99/// reader::v2::CheckpointReaderConfig
100/// };
101/// use iota_types::full_checkpoint_content::CheckpointData;
102/// use prometheus_filtered::Registry;
103/// use tokio_util::sync::CancellationToken;
104/// use std::{path::PathBuf, sync::Arc};
105///
106/// struct CustomWorker;
107///
108/// #[async_trait]
109/// impl Worker for CustomWorker {
110/// type Message = ();
111/// type Error = IngestionError;
112///
113/// async fn process_checkpoint(
114/// &self,
115/// checkpoint: Arc<CheckpointData>,
116/// ) -> Result<Self::Message, Self::Error> {
117/// // custom processing logic.
118/// println!(
119/// "Processing Local checkpoint: {:?}",
120/// checkpoint.checkpoint_summary
121/// );
122/// Ok(())
123/// }
124/// }
125///
126/// #[tokio::main]
127/// async fn main() {
128/// let concurrency = 5;
129/// let progress_store = FileProgressStore::new("progress.json").await.unwrap();
130/// let mut executor = IndexerExecutor::new(
131/// progress_store,
132/// 1, // number of registered WorkerPools.
133/// DataIngestionMetrics::new(&Registry::new()),
134/// CancellationToken::new(),
135/// );
136/// // register a worker pool with 5 workers to process checkpoints in parallel
137/// let worker_pool = WorkerPool::new(CustomWorker, "local_reader".to_string(), concurrency, Default::default());
138/// // register the worker pool to the executor.
139/// executor.register(worker_pool).await.unwrap();
140/// // run the ingestion pipeline.
141/// executor
142/// .run_with_config(
143/// CheckpointReaderConfig {
144/// reader_options: ReaderOptions::default(),
145/// ingestion_path: Some(PathBuf::from("./chk".to_string())), // path to a local directory where checkpoints are stored.
146/// remote_store_url: None,
147/// }
148/// )
149/// .await
150/// .unwrap();
151/// }
152/// ```
153pub struct IndexerExecutor<P> {
154 pools: Vec<Pin<Box<dyn Future<Output = ()> + Send>>>,
155 pool_senders: Vec<mpsc::Sender<Arc<CheckpointData>>>,
156 progress_store: ProgressStoreWrapper<P>,
157 pool_status_sender: mpsc::Sender<WorkerPoolStatus>,
158 pool_status_receiver: mpsc::Receiver<WorkerPoolStatus>,
159 metrics: DataIngestionMetrics,
160 token: CancellationToken,
161 shutdown_callback: Option<ShutdownCallback>,
162}
163
164impl<P: ProgressStore> IndexerExecutor<P> {
165 pub fn new(
166 progress_store: P,
167 number_of_jobs: usize,
168 metrics: DataIngestionMetrics,
169 token: CancellationToken,
170 ) -> Self {
171 let (pool_status_sender, pool_status_receiver) =
172 mpsc::channel(number_of_jobs * MAX_CHECKPOINTS_IN_PROGRESS);
173 Self {
174 pools: vec![],
175 pool_senders: vec![],
176 progress_store: ProgressStoreWrapper::new(progress_store),
177 pool_status_sender,
178 pool_status_receiver,
179 metrics,
180 token,
181 shutdown_callback: None,
182 }
183 }
184
185 /// Registers new worker pool in executor.
186 pub async fn register<W: Worker + 'static>(
187 &mut self,
188 pool: WorkerPool<W>,
189 ) -> IngestionResult<()> {
190 let checkpoint_number = self.progress_store.load(pool.task_name.clone()).await?;
191 let (sender, receiver) = mpsc::channel(MAX_CHECKPOINTS_IN_PROGRESS);
192 self.pools.push(Box::pin(pool.run(
193 checkpoint_number,
194 receiver,
195 self.pool_status_sender.clone(),
196 self.token.child_token(),
197 )));
198 self.pool_senders.push(sender);
199 Ok(())
200 }
201
202 /// Registers a predicate callback that determines when the ingestion
203 /// process should stop.
204 ///
205 /// This function `f` will be called for every **incoming checkpoint**
206 /// before it’s sent to the worker pool.
207 ///
208 /// Based on the returned [`ShutdownAction`] the executor will evaluate
209 /// whether to continue or stop the ingestion process by initiating the
210 /// graceful shutdown process.
211 ///
212 /// Once a shutdown action is triggered, the executor will stop sending new
213 /// checkpoints and will wait for all previously sent checkpoints to be
214 /// processed by workers before initiating graceful shutdown process.
215 ///
216 /// Note:
217 ///
218 /// Calling this method after
219 /// [`with_ingestion_limit`](Self::with_ingestion_limit) replaces the
220 /// earlier predicate, and vice versa. They are not cumulative.
221 pub fn shutdown_when<F>(&mut self, f: F)
222 where
223 F: Fn(&CheckpointData) -> ShutdownAction + Send + 'static,
224 {
225 self.shutdown_callback = Some(Box::new(f));
226 }
227
228 /// Adds an upper‑limit policy that determines when the ingestion
229 /// process should stop.
230 ///
231 /// This is a convenience method, it internally uses
232 /// [`shutdown_when`](Self::shutdown_when) by registering a predicate
233 /// derived from the provided [`IngestionLimit`].
234 ///
235 /// Note:
236 ///
237 /// Calling this method after [`shutdown_when`](Self::shutdown_when)
238 /// replaces the earlier predicate, and vice versa. They are not cumulative.
239 pub fn with_ingestion_limit(&mut self, limit: IngestionLimit) {
240 self.shutdown_when(move |checkpoint| limit.matches(checkpoint));
241 }
242
243 pub async fn update_watermark(
244 &mut self,
245 task_name: String,
246 watermark: CheckpointSequenceNumber,
247 ) -> IngestionResult<()> {
248 self.progress_store.save(task_name, watermark).await
249 }
250
251 pub async fn read_watermark(
252 &mut self,
253 task_name: String,
254 ) -> IngestionResult<CheckpointSequenceNumber> {
255 self.progress_store.load(task_name).await
256 }
257
258 /// Main executor loop.
259 ///
260 /// # Error
261 ///
262 /// Returns an [`IngestionError::EmptyWorkerPool`] if no worker pool was
263 /// registered.
264 pub async fn run_with_config(
265 mut self,
266 config: impl Into<CheckpointReaderConfigExt>,
267 ) -> IngestionResult<ExecutorProgress> {
268 let reader_checkpoint_number = self.progress_store.min_watermark()?;
269 let checkpoint_reader =
270 CheckpointReader::new(reader_checkpoint_number, config.into()).await?;
271
272 self.run_executor_loop(reader_checkpoint_number, checkpoint_reader)
273 .await
274 }
275
276 /// Common execution logic
277 async fn run_executor_loop(
278 &mut self,
279 mut reader_checkpoint_number: u64,
280 mut checkpoint_reader: CheckpointReader,
281 ) -> IngestionResult<ExecutorProgress> {
282 let worker_pools = std::mem::take(&mut self.pools)
283 .into_iter()
284 .map(|pool| spawn_monitored_task!(pool))
285 .collect::<Vec<JoinHandle<()>>>();
286
287 let mut worker_pools_shutdown_signals = vec![];
288 let mut checkpoint_limit_reached = None;
289 let mut reader_error = None;
290
291 loop {
292 // the min watermark represents the lowest watermark that
293 // has been processed by any worker pool. This guarantees that
294 // all worker pools have processed the checkpoint before the
295 // shutdown process starts.
296 if checkpoint_limit_reached.is_some_and(|ch_seq_num| {
297 self.progress_store
298 .min_watermark()
299 .map(|watermark| watermark > ch_seq_num)
300 .unwrap_or_default()
301 }) {
302 info!(
303 "checkpoint upper limit reached: last checkpoint was processed, shutdown process started"
304 );
305 self.token.cancel();
306 }
307
308 tokio::select! {
309 Some(worker_pool_progress_msg) = self.pool_status_receiver.recv() => {
310 match worker_pool_progress_msg {
311 WorkerPoolStatus::Running((task_name, watermark)) => {
312 self.progress_store.save(task_name.clone(), watermark).await
313 .map_err(|err| IngestionError::ProgressStore(err.to_string()))?;
314 let seq_number = self.progress_store.min_watermark()?;
315 if seq_number > reader_checkpoint_number {
316 checkpoint_reader.send_gc_signal(seq_number)?;
317 reader_checkpoint_number = seq_number;
318 }
319 self.metrics.data_ingestion_checkpoint
320 .with_label_values(&[&task_name])
321 .set(watermark as i64);
322 }
323 WorkerPoolStatus::Shutdown(worker_pool_name) => {
324 worker_pools_shutdown_signals.push(worker_pool_name);
325 }
326 }
327 }
328 checkpoint = checkpoint_reader.checkpoint(), if !self.token.is_cancelled() => {
329 let Some(checkpoint) = checkpoint else {
330 reader_error = Some(IngestionError::Channel("checkpoint reader stopped".into()));
331 // start garceful shutdown sequence.
332 self.token.cancel();
333 continue;
334 };
335 // once upper limit reached skip sending new checkpoints to workers.
336 if self.should_shutdown(&checkpoint, &mut checkpoint_limit_reached) {
337 continue;
338 }
339
340 for sender in &self.pool_senders {
341 sender.send(checkpoint.clone()).await.map_err(|_| {
342 IngestionError::Channel(
343 "unable to send new checkpoint to worker pool, receiver half closed".to_owned(),
344 )
345 })?;
346 }
347 }
348 }
349
350 if worker_pools_shutdown_signals.len() == self.pool_senders.len() {
351 // Shutdown worker pools
352 for worker_pool in worker_pools {
353 worker_pool.await.map_err(|err| IngestionError::Shutdown {
354 component: "worker pool".into(),
355 msg: err.to_string(),
356 })?;
357 }
358 // Shutdown checkpoint reader
359 checkpoint_reader.shutdown().await?;
360 break;
361 }
362 }
363
364 match reader_error {
365 Some(err) => Err(err),
366 None => Ok(self.progress_store.stats()),
367 }
368 }
369
370 /// Check if the current ingestion limit has been reached.
371 ///
372 /// Returns `true` if the ingestion limit has been reached.
373 /// If no ingestion limit is present or it has not been reached yet, the
374 /// function returns `false`.
375 fn should_shutdown(
376 &mut self,
377 checkpoint: &CheckpointData,
378 checkpoint_limit_reached: &mut Option<CheckpointSequenceNumber>,
379 ) -> bool {
380 if checkpoint_limit_reached.is_some() {
381 return true;
382 }
383
384 let Some(shutdown_action) = self
385 .shutdown_callback
386 .as_ref()
387 .map(|matches| matches(checkpoint))
388 else {
389 return false;
390 };
391
392 match shutdown_action {
393 ShutdownAction::IncludeAndShutdown => {
394 checkpoint_limit_reached
395 .get_or_insert(checkpoint.checkpoint_summary.sequence_number);
396 false
397 }
398 ShutdownAction::ExcludeAndShutdown => {
399 checkpoint_limit_reached.get_or_insert(
400 checkpoint
401 .checkpoint_summary
402 .sequence_number
403 .saturating_sub(1),
404 );
405 true
406 }
407 ShutdownAction::Continue => false,
408 }
409 }
410}
411
412/// Sets up a single workflow for data ingestion.
413///
414/// This function initializes an [`IndexerExecutor`] with a single worker pool,
415/// using a [`ShimProgressStore`] initialized with the provided
416/// `initial_checkpoint_number`. It then returns a future that runs the executor
417/// and a [`CancellationToken`] for graceful shutdown.
418///
419/// # Docs
420/// For more info please check the [custom indexer docs](https://docs.iota.org/developer/advanced/custom-indexer).
421///
422/// # Example
423/// ```rust,no_run
424/// use std::sync::Arc;
425///
426/// use async_trait::async_trait;
427/// use iota_data_ingestion_core::{
428/// IngestionError, Worker, reader::v2::RemoteUrl, setup_single_workflow,
429/// };
430/// use iota_types::full_checkpoint_content::CheckpointData;
431///
432/// struct CustomWorker;
433///
434/// #[async_trait]
435/// impl Worker for CustomWorker {
436/// type Message = ();
437/// type Error = IngestionError;
438///
439/// async fn process_checkpoint(
440/// &self,
441/// checkpoint: Arc<CheckpointData>,
442/// ) -> Result<Self::Message, Self::Error> {
443/// // custom processing logic.
444/// println!("Processing checkpoint: {:?}", checkpoint.checkpoint_summary);
445/// Ok(())
446/// }
447/// }
448///
449/// #[tokio::main]
450/// async fn main() {
451/// let (executor, _) = setup_single_workflow(
452/// CustomWorker,
453/// RemoteUrl::Fullnode("http://127.0.0.1:50051".into()), // fullnode gRPC API.
454/// 0, // initial checkpoint number.
455/// 5, // concurrency.
456/// None, // extra reader options.
457/// )
458/// .await
459/// .unwrap();
460/// executor.await.unwrap();
461/// }
462/// ```
463pub async fn setup_single_workflow<W: Worker + 'static>(
464 worker: W,
465 remote_store_url: RemoteUrl,
466 initial_checkpoint_number: CheckpointSequenceNumber,
467 concurrency: usize,
468 reader_options: Option<ReaderOptions>,
469) -> IngestionResult<(
470 impl Future<Output = IngestionResult<ExecutorProgress>>,
471 CancellationToken,
472)> {
473 let metrics = DataIngestionMetrics::new(&Registry::new());
474 let progress_store = ShimProgressStore(initial_checkpoint_number);
475 let token = CancellationToken::new();
476 let mut executor = IndexerExecutor::new(progress_store, 1, metrics, token.child_token());
477 let worker_pool = WorkerPool::new(
478 worker,
479 "workflow".to_string(),
480 concurrency,
481 Default::default(),
482 );
483 executor.register(worker_pool).await?;
484 Ok((
485 executor.run_with_config(CheckpointReaderConfig {
486 reader_options: reader_options.unwrap_or_default(),
487 ingestion_path: None,
488 remote_store_url: Some(remote_store_url),
489 }),
490 token,
491 ))
492}