iota_indexer/
metrics.rs

1// Copyright (c) Mysten Labs, Inc.
2// Modifications Copyright (c) 2024 IOTA Stiftung
3// SPDX-License-Identifier: Apache-2.0
4
5use std::{collections::HashMap, net::SocketAddr};
6
7use axum::{Router, extract::Extension, http::StatusCode, routing::get};
8use iota_metrics::RegistryService;
9use prometheus::{
10    Histogram, IntCounter, IntGauge, Registry, TextEncoder, register_histogram_with_registry,
11    register_int_counter_with_registry, register_int_gauge_with_registry,
12};
13use regex::Regex;
14use tracing::{info, warn};
15
16const METRICS_ROUTE: &str = "/metrics";
17
18pub fn start_prometheus_server(
19    addr: SocketAddr,
20    fn_url: &str,
21) -> Result<(RegistryService, Registry), anyhow::Error> {
22    let converted_fn_url = convert_url(fn_url);
23    if converted_fn_url.is_none() {
24        warn!(
25            "Failed to convert full node url {} to a shorter version",
26            fn_url
27        );
28    }
29    let fn_url_str = converted_fn_url.unwrap_or_else(|| "unknown_url".to_string());
30
31    let labels = HashMap::from([("indexer_fullnode".to_string(), fn_url_str)]);
32    info!("Starting prometheus server with labels: {:?}", labels);
33    let registry = Registry::new_custom(Some("indexer".to_string()), Some(labels))?;
34    let registry_service = RegistryService::new(registry.clone());
35
36    let app = Router::new()
37        .route(METRICS_ROUTE, get(metrics))
38        .layer(Extension(registry_service.clone()));
39
40    tokio::spawn(async move {
41        let listener = tokio::net::TcpListener::bind(addr).await.unwrap();
42        axum::serve(listener, app.into_make_service())
43            .await
44            .unwrap();
45    });
46    Ok((registry_service, registry))
47}
48
49async fn metrics(Extension(registry_service): Extension<RegistryService>) -> (StatusCode, String) {
50    let metrics_families = registry_service.gather_all();
51    match TextEncoder.encode_to_string(&metrics_families) {
52        Ok(metrics) => (StatusCode::OK, metrics),
53        Err(error) => (
54            StatusCode::INTERNAL_SERVER_ERROR,
55            format!("unable to encode metrics: {error}"),
56        ),
57    }
58}
59
60fn convert_url(url_str: &str) -> Option<String> {
61    // NOTE: unwrap here is safe because the regex is a constant.
62    let re = Regex::new(r"https?://([a-z0-9-]+\.[a-z0-9-]+\.[a-z]+)").unwrap();
63    let captures = re.captures(url_str)?;
64
65    captures.get(1).map(|m| m.as_str().to_string())
66}
67
68/// NOTE: for various data ingestion steps, which are expected to be within
69/// [0.001, 100] seconds, and high double digits usually means something is
70/// broken.
71const DATA_INGESTION_LATENCY_SEC_BUCKETS: &[f64] = &[
72    0.001, 0.002, 0.005, 0.01, 0.02, 0.05, 0.1, 0.2, 0.5, 1.0, 2.0, 5.0, 10.0, 20.0, 50.0, 100.0,
73];
74/// NOTE: for objects_snapshot update and advance_epoch, which are expected to
75/// be within [0.1, 100] seconds, and can go up to high hundreds of seconds when
76/// things go wrong.
77const DB_UPDATE_QUERY_LATENCY_SEC_BUCKETS: &[f64] = &[
78    0.1, 0.2, 0.5, 1.0, 2.0, 5.0, 10.0, 20.0, 50.0, 100.0, 200.0, 500.0, 1000.0, 2000.0, 5000.0,
79    10000.0,
80];
81/// NOTE: for json_rpc calls, which are expected to be within [0.01, 100]
82/// seconds, high hundreds of seconds usually means something is broken.
83const JSON_RPC_LATENCY_SEC_BUCKETS: &[f64] = &[
84    0.01, 0.02, 0.05, 0.1, 0.2, 0.5, 1.0, 2.0, 5.0, 10.0, 20.0, 50.0, 100.0, 200.0, 500.0, 1000.0,
85];
86
87// TODO: handle unused metrics
88#[derive(Clone)]
89pub struct IndexerMetrics {
90    pub total_checkpoint_received: IntCounter, // not used
91    pub total_tx_checkpoint_committed: IntCounter,
92    pub total_object_checkpoint_committed: IntCounter, // not used
93    pub total_transaction_committed: IntCounter,
94    pub total_object_change_committed: IntCounter, // not used
95    pub total_transaction_chunk_committed: IntCounter, // not used
96    pub total_object_change_chunk_committed: IntCounter, // not used
97    pub total_epoch_committed: IntCounter,
98    pub latest_fullnode_checkpoint_sequence_number: IntGauge,
99    pub latest_tx_checkpoint_sequence_number: IntGauge,
100    pub latest_indexer_object_checkpoint_sequence_number: IntGauge, // not used
101    pub latest_object_snapshot_sequence_number: IntGauge,
102    // max checkpoint sequence numbers on various stages of indexer data ingestion
103    pub max_downloaded_checkpoint_sequence_number: IntGauge,
104    pub max_indexed_checkpoint_sequence_number: IntGauge,
105    pub max_committed_checkpoint_sequence_number: IntGauge,
106    // the related timestamps of max checkpoint ^ on various stages
107    pub downloaded_checkpoint_timestamp_ms: IntGauge,
108    pub indexed_checkpoint_timestamp_ms: IntGauge,
109    pub committed_checkpoint_timestamp_ms: IntGauge,
110    // lag starting from the timestamp of the latest checkpoint to the current time
111    pub download_lag_ms: IntGauge,
112    pub index_lag_ms: IntGauge,
113    pub db_commit_lag_ms: IntGauge, // not used
114    // latencies of various steps of data ingestion.
115    // checkpoint E2E latency is: fullnode_download_latency + checkpoint_index_latency +
116    // db_commit_latency
117    // Analytical
118    pub latest_move_call_metrics_tx_seq: IntGauge,
119    pub latest_address_metrics_tx_seq: IntGauge,
120    pub latest_network_metrics_cp_seq: IntGauge,
121    pub checkpoint_download_bytes_size: IntGauge, // not used
122    pub tokio_blocking_task_wait_latency: Histogram,
123    pub fullnode_checkpoint_data_download_latency: Histogram, // not used
124    pub fullnode_checkpoint_wait_and_download_latency: Histogram, // not used
125    pub fullnode_transaction_download_latency: Histogram,     // not used
126    pub fullnode_object_download_latency: Histogram,          // not used
127    pub checkpoint_index_latency: Histogram,                  // not used
128    pub indexing_batch_size: IntGauge,                        // not used
129    pub indexing_tx_object_changes_latency: Histogram,
130    pub indexing_objects_latency: Histogram,
131    pub indexing_get_object_in_mem_hit: IntCounter,
132    pub indexing_get_object_db_hit: IntCounter, // not used
133    pub indexing_module_resolver_in_mem_hit: IntCounter, // not used
134    pub indexing_package_resolver_in_mem_hit: IntCounter, // not used
135    pub indexing_packages_latency: Histogram,
136    pub checkpoint_objects_index_latency: Histogram, // not used
137    pub checkpoint_db_commit_latency: Histogram,
138    pub checkpoint_db_commit_latency_step_1: Histogram,
139    pub checkpoint_db_commit_latency_transactions: Histogram,
140    pub checkpoint_db_commit_latency_transactions_chunks: Histogram,
141    pub checkpoint_db_commit_latency_transactions_chunks_transformation: Histogram,
142    pub checkpoint_db_commit_latency_tx_insertion_order: Histogram,
143    pub checkpoint_db_commit_latency_tx_insertion_order_chunks: Histogram,
144    pub checkpoint_db_commit_latency_objects: Histogram,
145    pub checkpoint_db_commit_latency_objects_snapshot: Histogram,
146    pub checkpoint_db_commit_latency_objects_history: Histogram,
147    pub checkpoint_db_commit_latency_objects_chunks: Histogram,
148    pub checkpoint_db_commit_latency_objects_snapshot_chunks: Histogram,
149    pub checkpoint_db_commit_latency_objects_history_chunks: Histogram,
150    pub checkpoint_db_commit_latency_events: Histogram,
151    pub checkpoint_db_commit_latency_events_chunks: Histogram,
152    pub checkpoint_db_commit_latency_event_indices: Histogram,
153    pub checkpoint_db_commit_latency_event_indices_chunks: Histogram,
154    pub checkpoint_db_commit_latency_packages: Histogram,
155    pub checkpoint_db_commit_latency_tx_indices: Histogram,
156    pub checkpoint_db_commit_latency_tx_indices_chunks: Histogram,
157    pub checkpoint_db_commit_latency_checkpoints: Histogram,
158    pub checkpoint_db_commit_latency_epoch: Histogram,
159    pub thousand_transaction_avg_db_commit_latency: Histogram,
160    pub object_db_commit_latency: Histogram, // not used
161    pub object_mutation_db_commit_latency: Histogram, // not used
162    pub object_deletion_db_commit_latency: Histogram, // not used
163    pub epoch_db_commit_latency: Histogram,  // not used
164    // latencies of slow DB update queries, now only advance epoch and objects_snapshot update
165    pub advance_epoch_latency: Histogram,
166    // latencies of RPC endpoints in read.rs
167    pub get_transaction_block_latency: Histogram, // not used
168    pub multi_get_transaction_blocks_latency: Histogram,
169    pub get_object_latency: Histogram,                   // not used
170    pub multi_get_objects_latency: Histogram,            // not used
171    pub try_get_past_object_latency: Histogram,          // not used
172    pub try_multi_get_past_objects_latency: Histogram,   // not used
173    pub get_checkpoint_latency: Histogram,               // not used
174    pub get_checkpoints_latency: Histogram,              // not used
175    pub get_events_latency: Histogram,                   // not used
176    pub get_loaded_child_objects_latency: Histogram,     // not used
177    pub get_total_transaction_blocks_latency: Histogram, // not used
178    pub get_latest_checkpoint_sequence_number_latency: Histogram, // not used
179    // latencies of RPC endpoints in indexer.rs
180    pub get_owned_objects_latency: Histogram, // not used
181    pub query_transaction_blocks_latency: Histogram, // not used
182    pub query_events_latency: Histogram,      // not used
183    pub get_dynamic_fields_latency: Histogram, // not used
184    pub get_dynamic_field_object_latency: Histogram, // not used
185    pub get_protocol_config_latency: Histogram, // not used
186    // latency of event websocket subscription
187    pub subscription_process_latency: Histogram, // not used
188    pub transaction_per_checkpoint: Histogram,
189    // indexer state metrics
190    pub db_conn_pool_size: IntGauge,
191    pub idle_db_conn: IntGauge,
192    pub address_processor_failure: IntCounter, // not used
193    pub checkpoint_metrics_processor_failure: IntCounter, // not used
194    // pruner metrics
195    pub last_pruned_epoch: IntGauge,
196    pub last_pruned_checkpoint: IntGauge,
197    pub last_pruned_transaction: IntGauge,
198    pub epoch_pruning_latency: Histogram, // not used
199}
200
201impl IndexerMetrics {
202    pub fn new(registry: &Registry) -> Self {
203        Self {
204            total_checkpoint_received: register_int_counter_with_registry!(
205                "total_checkpoint_received",
206                "Total number of checkpoint received",
207                registry,
208            )
209            .unwrap(),
210            total_tx_checkpoint_committed: register_int_counter_with_registry!(
211                "total_checkpoint_committed",
212                "Total number of checkpoint committed",
213                registry,
214            )
215            .unwrap(),
216            total_object_checkpoint_committed: register_int_counter_with_registry!(
217                "total_object_checkpoint_committed",
218                "Total number of object checkpoint committed",
219                registry,
220            )
221            .unwrap(),
222            total_transaction_committed: register_int_counter_with_registry!(
223                "total_transaction_committed",
224                "Total number of transaction committed",
225                registry,
226            )
227            .unwrap(),
228            total_object_change_committed: register_int_counter_with_registry!(
229                "total_object_change_committed",
230                "Total number of object change committed",
231                registry,
232            )
233            .unwrap(),
234            total_transaction_chunk_committed: register_int_counter_with_registry!(
235                "total_transaction_chunk_committed",
236                "Total number of transaction chunk committed",
237                registry,
238            )
239            .unwrap(),
240            total_object_change_chunk_committed: register_int_counter_with_registry!(
241                "total_object_change_chunk_committed",
242                "Total number of object change chunk committed",
243                registry,
244            )
245            .unwrap(),
246            total_epoch_committed: register_int_counter_with_registry!(
247                "total_epoch_committed",
248                "Total number of epoch committed",
249                registry,
250            )
251            .unwrap(),
252            latest_fullnode_checkpoint_sequence_number: register_int_gauge_with_registry!(
253                "latest_fullnode_checkpoint_sequence_number",
254                "Latest checkpoint sequence number from the Full Node",
255                registry,
256            )
257            .unwrap(),
258            latest_tx_checkpoint_sequence_number: register_int_gauge_with_registry!(
259                "latest_indexer_checkpoint_sequence_number",
260                "Latest checkpoint sequence number from the Indexer",
261                registry,
262            )
263            .unwrap(),
264            latest_indexer_object_checkpoint_sequence_number: register_int_gauge_with_registry!(
265                "latest_indexer_object_checkpoint_sequence_number",
266                "Latest object checkpoint sequence number from the Indexer",
267                registry,
268            )
269            .unwrap(),
270            latest_object_snapshot_sequence_number: register_int_gauge_with_registry!(
271                "latest_object_snapshot_sequence_number",
272                "Latest object snapshot sequence number from the Indexer",
273                registry,
274            ).unwrap(),
275            max_downloaded_checkpoint_sequence_number: register_int_gauge_with_registry!(
276                "max_downloaded_checkpoint_sequence_number",
277                "Max downloaded checkpoint sequence number",
278                registry,
279            ).unwrap(),
280            max_indexed_checkpoint_sequence_number: register_int_gauge_with_registry!(
281                "max_indexed_checkpoint_sequence_number",
282                "Max indexed checkpoint sequence number",
283                registry,
284            ).unwrap(),
285            max_committed_checkpoint_sequence_number: register_int_gauge_with_registry!(
286                "max_committed_checkpoint_sequence_number",
287                "Max committed checkpoint sequence number",
288                registry,
289            ).unwrap(),
290            downloaded_checkpoint_timestamp_ms: register_int_gauge_with_registry!(
291                "downloaded_checkpoint_timestamp_ms",
292                "Timestamp of the downloaded checkpoint",
293                registry,
294            ).unwrap(),
295            indexed_checkpoint_timestamp_ms: register_int_gauge_with_registry!(
296                "indexed_checkpoint_timestamp_ms",
297                "Timestamp of the indexed checkpoint",
298                registry,
299            ).unwrap(),
300            committed_checkpoint_timestamp_ms: register_int_gauge_with_registry!(
301                "committed_checkpoint_timestamp_ms",
302                "Timestamp of the committed checkpoint",
303                registry,
304            ).unwrap(),
305            download_lag_ms: register_int_gauge_with_registry!(
306                "download_lag_ms",
307                "Lag of the latest checkpoint in milliseconds",
308                registry,
309            ).unwrap(),
310            index_lag_ms: register_int_gauge_with_registry!(
311                "index_lag_ms",
312                "Lag of the latest checkpoint in milliseconds",
313                registry,
314            ).unwrap(),
315            db_commit_lag_ms: register_int_gauge_with_registry!(
316                "db_commit_lag_ms",
317                "Lag of the latest checkpoint in milliseconds",
318                registry,
319            ).unwrap(),
320            latest_move_call_metrics_tx_seq: register_int_gauge_with_registry!(
321                "latest_move_call_metrics_tx_seq",
322                "Latest move call metrics tx seq",
323                registry,
324            ).unwrap(),
325            latest_address_metrics_tx_seq: register_int_gauge_with_registry!(
326                "latest_address_metrics_tx_seq",
327                "Latest address metrics tx seq",
328                registry,
329            ).unwrap(),
330            latest_network_metrics_cp_seq: register_int_gauge_with_registry!(
331                "latest_network_metrics_cp_seq",
332                "Latest network metrics cp seq",
333                registry,
334            ).unwrap(),
335            checkpoint_download_bytes_size: register_int_gauge_with_registry!(
336                "checkpoint_download_bytes_size",
337                "Size of the downloaded checkpoint in bytes",
338                registry,
339            ).unwrap(),
340            fullnode_checkpoint_data_download_latency: register_histogram_with_registry!(
341                "fullnode_checkpoint_data_download_latency",
342                "Time spent in downloading checkpoint and transaction for a new checkpoint from the Full Node",
343                DATA_INGESTION_LATENCY_SEC_BUCKETS.to_vec(),
344                registry,
345            )
346            .unwrap(),
347            fullnode_checkpoint_wait_and_download_latency: register_histogram_with_registry!(
348                "fullnode_checkpoint_wait_and_download_latency",
349                "Time spent in waiting for a new checkpoint from the Full Node",
350                DATA_INGESTION_LATENCY_SEC_BUCKETS.to_vec(),
351                registry,
352            )
353            .unwrap(),
354
355            fullnode_transaction_download_latency: register_histogram_with_registry!(
356                "fullnode_transaction_download_latency",
357                "Time spent in waiting for a new transaction from the Full Node",
358                DATA_INGESTION_LATENCY_SEC_BUCKETS.to_vec(),
359                registry,
360            )
361            .unwrap(),
362            fullnode_object_download_latency: register_histogram_with_registry!(
363                "fullnode_object_download_latency",
364                "Time spent in waiting for a new epoch from the Full Node",
365                DATA_INGESTION_LATENCY_SEC_BUCKETS.to_vec(),
366                registry,
367            )
368            .unwrap(),
369            checkpoint_index_latency: register_histogram_with_registry!(
370                "checkpoint_index_latency",
371                "Time spent in indexing a checkpoint",
372                DATA_INGESTION_LATENCY_SEC_BUCKETS.to_vec(),
373                registry,
374            )
375            .unwrap(),
376            indexing_batch_size: register_int_gauge_with_registry!(
377                "indexing_batch_size",
378                "Size of the indexing batch",
379                registry,
380            ).unwrap(),
381            indexing_tx_object_changes_latency: register_histogram_with_registry!(
382                "indexing_tx_object_changes_latency",
383                "Time spent in indexing object changes for a transaction",
384                DATA_INGESTION_LATENCY_SEC_BUCKETS.to_vec(),
385                registry,
386            )
387            .unwrap(),
388            indexing_objects_latency: register_histogram_with_registry!(
389                "indexing_objects_latency",
390                "Time spent in indexing objects",
391                DATA_INGESTION_LATENCY_SEC_BUCKETS.to_vec(),
392                registry,
393            )
394            .unwrap(),
395            indexing_packages_latency: register_histogram_with_registry!(
396                "indexing_packages_latency",
397                "Time spent in indexing packages",
398                DATA_INGESTION_LATENCY_SEC_BUCKETS.to_vec(),
399                registry,
400            )
401            .unwrap(),
402            indexing_get_object_in_mem_hit: register_int_counter_with_registry!(
403                "indexing_get_object_in_mem_hit",
404                "Total number get object hit in mem",
405                registry,
406            )
407            .unwrap(),
408            indexing_get_object_db_hit: register_int_counter_with_registry!(
409                "indexing_get_object_db_hit",
410                "Total number get object hit in db",
411                registry,
412            )
413            .unwrap(),
414            indexing_module_resolver_in_mem_hit: register_int_counter_with_registry!(
415                "indexing_module_resolver_in_mem_hit",
416                "Total number module resolver hit in mem",
417                registry,
418            )
419            .unwrap(),
420            indexing_package_resolver_in_mem_hit: register_int_counter_with_registry!(
421                "indexing_package_resolver_in_mem_hit",
422                "Total number package resolver hit in mem",
423                registry,
424            )
425            .unwrap(),
426            checkpoint_objects_index_latency: register_histogram_with_registry!(
427                "checkpoint_object_index_latency",
428                "Time spent in indexing a checkpoint objects",
429                DATA_INGESTION_LATENCY_SEC_BUCKETS.to_vec(),
430                registry,
431            )
432            .unwrap(),
433            checkpoint_db_commit_latency: register_histogram_with_registry!(
434                "checkpoint_db_commit_latency",
435                "Time spent committing a checkpoint to the db",
436                DATA_INGESTION_LATENCY_SEC_BUCKETS.to_vec(),
437                registry,
438            )
439            .unwrap(),
440
441            checkpoint_db_commit_latency_step_1: register_histogram_with_registry!(
442                "checkpoint_db_commit_latency_step_1",
443                "Time spent committing a checkpoint to the db, step 1",
444                DATA_INGESTION_LATENCY_SEC_BUCKETS.to_vec(),
445                registry,
446            )
447            .unwrap(),
448            checkpoint_db_commit_latency_transactions: register_histogram_with_registry!(
449                "checkpoint_db_commit_latency_transactions",
450                "Time spent committing transactions",
451                DATA_INGESTION_LATENCY_SEC_BUCKETS.to_vec(),
452                registry,
453            )
454            .unwrap(),
455            checkpoint_db_commit_latency_transactions_chunks: register_histogram_with_registry!(
456                "checkpoint_db_commit_latency_transactions_chunks",
457                "Time spent committing transactions chunks",
458                DATA_INGESTION_LATENCY_SEC_BUCKETS.to_vec(),
459                registry,
460            )
461            .unwrap(),
462            checkpoint_db_commit_latency_transactions_chunks_transformation: register_histogram_with_registry!(
463                "checkpoint_db_commit_latency_transactions_transformation",
464                "Time spent in transactions chunks transformation prior to commit",
465                DATA_INGESTION_LATENCY_SEC_BUCKETS.to_vec(),
466                registry,
467            )
468            .unwrap(),
469            checkpoint_db_commit_latency_tx_insertion_order: register_histogram_with_registry!(
470                "checkpoint_db_commit_latency_tx_insertion_order",
471                "Time spent committing txs insertion order",
472                DATA_INGESTION_LATENCY_SEC_BUCKETS.to_vec(),
473                registry,
474            )
475                .unwrap(),
476            checkpoint_db_commit_latency_tx_insertion_order_chunks: register_histogram_with_registry!(
477                "checkpoint_db_commit_latency_tx_insertion_order_chunks",
478                "Time spent committing txs insertion order chunks",
479                DATA_INGESTION_LATENCY_SEC_BUCKETS.to_vec(),
480                registry,
481            )
482                .unwrap(),
483            checkpoint_db_commit_latency_objects: register_histogram_with_registry!(
484                "checkpoint_db_commit_latency_objects",
485                "Time spent committing objects",
486                DATA_INGESTION_LATENCY_SEC_BUCKETS.to_vec(),
487                registry,
488            )
489            .unwrap(),
490            checkpoint_db_commit_latency_objects_snapshot: register_histogram_with_registry!(
491                "checkpoint_db_commit_latency_objects_snapshot",
492                "Time spent committing objects snapshots",
493                DATA_INGESTION_LATENCY_SEC_BUCKETS.to_vec(),
494                registry,
495            )
496            .unwrap(),
497            checkpoint_db_commit_latency_objects_history: register_histogram_with_registry!(
498                "checkpoint_db_commit_latency_objects_history",
499                "Time spent committing objects history",
500                DATA_INGESTION_LATENCY_SEC_BUCKETS.to_vec(),
501                registry,
502            ).unwrap(),
503            checkpoint_db_commit_latency_objects_chunks: register_histogram_with_registry!(
504                "checkpoint_db_commit_latency_objects_chunks",
505                "Time spent committing objects chunks",
506                DATA_INGESTION_LATENCY_SEC_BUCKETS.to_vec(),
507                registry,
508            )
509            .unwrap(),
510            checkpoint_db_commit_latency_objects_snapshot_chunks: register_histogram_with_registry!(
511                "checkpoint_db_commit_latency_objects_snapshot_chunks",
512                "Time spent committing objects snapshot chunks",
513                DATA_INGESTION_LATENCY_SEC_BUCKETS.to_vec(),
514                registry,
515            )
516            .unwrap(),
517            checkpoint_db_commit_latency_objects_history_chunks: register_histogram_with_registry!(
518                "checkpoint_db_commit_latency_objects_history_chunks",
519                "Time spent committing objects history chunks",
520                DATA_INGESTION_LATENCY_SEC_BUCKETS.to_vec(),
521                registry,
522            ).unwrap(),
523            checkpoint_db_commit_latency_events: register_histogram_with_registry!(
524                "checkpoint_db_commit_latency_events",
525                "Time spent committing events",
526                DATA_INGESTION_LATENCY_SEC_BUCKETS.to_vec(),
527                registry,
528            )
529            .unwrap(),
530            checkpoint_db_commit_latency_events_chunks: register_histogram_with_registry!(
531                "checkpoint_db_commit_latency_events_chunks",
532                "Time spent committing events chunks",
533                DATA_INGESTION_LATENCY_SEC_BUCKETS.to_vec(),
534                registry,
535            )
536            .unwrap(),
537            checkpoint_db_commit_latency_event_indices: register_histogram_with_registry!(
538                "checkpoint_db_commit_latency_event_indices",
539                "Time spent committing event indices",
540                DATA_INGESTION_LATENCY_SEC_BUCKETS.to_vec(),
541                registry,
542            )
543            .unwrap(),
544            checkpoint_db_commit_latency_event_indices_chunks: register_histogram_with_registry!(
545                "checkpoint_db_commit_latency_event_indices_chunks",
546                "Time spent committing event indices chunks",
547                DATA_INGESTION_LATENCY_SEC_BUCKETS.to_vec(),
548                registry,
549            )
550            .unwrap(),
551            checkpoint_db_commit_latency_packages: register_histogram_with_registry!(
552                "checkpoint_db_commit_latency_packages",
553                "Time spent committing packages",
554                DATA_INGESTION_LATENCY_SEC_BUCKETS.to_vec(),
555                registry,
556            )
557            .unwrap(),
558            checkpoint_db_commit_latency_tx_indices: register_histogram_with_registry!(
559                "checkpoint_db_commit_latency_tx_indices",
560                "Time spent committing tx indices",
561                DATA_INGESTION_LATENCY_SEC_BUCKETS.to_vec(),
562                registry,
563            )
564            .unwrap(),
565            checkpoint_db_commit_latency_tx_indices_chunks: register_histogram_with_registry!(
566                "checkpoint_db_commit_latency_tx_indices_chunks",
567                "Time spent committing tx_indices chunks",
568                DATA_INGESTION_LATENCY_SEC_BUCKETS.to_vec(),
569                registry,
570            )
571            .unwrap(),
572            checkpoint_db_commit_latency_checkpoints: register_histogram_with_registry!(
573                "checkpoint_db_commit_latency_checkpoints",
574                "Time spent committing checkpoints",
575                DATA_INGESTION_LATENCY_SEC_BUCKETS.to_vec(),
576                registry,
577            )
578            .unwrap(),
579            checkpoint_db_commit_latency_epoch: register_histogram_with_registry!(
580                "checkpoint_db_commit_latency_epochs",
581                "Time spent committing epochs",
582                DATA_INGESTION_LATENCY_SEC_BUCKETS.to_vec(),
583                registry,
584            )
585            .unwrap(),
586            tokio_blocking_task_wait_latency: register_histogram_with_registry!(
587                "tokio_blocking_task_wait_latency",
588                "Time spent to wait for tokio blocking task pool",
589                DATA_INGESTION_LATENCY_SEC_BUCKETS.to_vec(),
590                registry,
591            ).unwrap(),
592            thousand_transaction_avg_db_commit_latency: register_histogram_with_registry!(
593                "transaction_db_commit_latency",
594                "Average time spent committing 1000 transactions to the db",
595                DATA_INGESTION_LATENCY_SEC_BUCKETS.to_vec(),
596                registry,
597            )
598            .unwrap(),
599            object_db_commit_latency: register_histogram_with_registry!(
600                "object_db_commit_latency",
601                "Time spent committing a object to the db",
602                DATA_INGESTION_LATENCY_SEC_BUCKETS.to_vec(),
603                registry,
604            )
605            .unwrap(),
606            object_mutation_db_commit_latency: register_histogram_with_registry!(
607                "object_mutation_db_commit_latency",
608                "Time spent committing a object mutation to the db",
609                DATA_INGESTION_LATENCY_SEC_BUCKETS.to_vec(),
610                registry,
611            )
612            .unwrap(),
613            object_deletion_db_commit_latency: register_histogram_with_registry!(
614                "object_deletion_db_commit_latency",
615                "Time spent committing a object deletion to the db",
616                DATA_INGESTION_LATENCY_SEC_BUCKETS.to_vec(),
617                registry,
618            )
619            .unwrap(),
620            epoch_db_commit_latency: register_histogram_with_registry!(
621                "epoch_db_commit_latency",
622                "Time spent committing a epoch to the db",
623                DATA_INGESTION_LATENCY_SEC_BUCKETS.to_vec(),
624                registry,
625            )
626            .unwrap(),
627            advance_epoch_latency: register_histogram_with_registry!(
628                "advance_epoch_latency",
629                "Time spent in advancing epoch",
630                DB_UPDATE_QUERY_LATENCY_SEC_BUCKETS.to_vec(),
631                registry,
632            ).unwrap(),
633            subscription_process_latency: register_histogram_with_registry!(
634                "subscription_process_latency",
635                "Time spent in process Websocket subscription",
636                JSON_RPC_LATENCY_SEC_BUCKETS.to_vec(),
637                registry,
638            )
639            .unwrap(),
640            transaction_per_checkpoint: register_histogram_with_registry!(
641                "transaction_per_checkpoint",
642                "Number of transactions per checkpoint",
643                vec![1.0, 2.0, 5.0, 10.0, 20.0, 50.0, 100.0, 200.0, 500.0, 1000.0, 2000.0, 5000.0],
644                registry,
645            )
646            .unwrap(),
647            get_transaction_block_latency: register_histogram_with_registry!(
648                "get_transaction_block_latency",
649                "Time spent in get_transaction_block on the fullnode behind.",
650                JSON_RPC_LATENCY_SEC_BUCKETS.to_vec(),
651                registry
652            )
653            .unwrap(),
654            multi_get_transaction_blocks_latency: register_histogram_with_registry!(
655                "multi_get_transaction_blocks_latency",
656                "Time spent in multi_get_transaction_blocks on the fullnode behind.",
657                JSON_RPC_LATENCY_SEC_BUCKETS.to_vec(),
658                registry
659            )
660            .unwrap(),
661            get_object_latency: register_histogram_with_registry!(
662                "get_object_latency",
663                "Time spent in get_object on the fullnode behind.",
664                JSON_RPC_LATENCY_SEC_BUCKETS.to_vec(),
665                registry
666            )
667            .unwrap(),
668            multi_get_objects_latency: register_histogram_with_registry!(
669                "multi_get_objects_latency",
670                "Time spent in multi_get_objects on the fullnode behind.",
671                JSON_RPC_LATENCY_SEC_BUCKETS.to_vec(),
672                registry
673            )
674            .unwrap(),
675            try_get_past_object_latency: register_histogram_with_registry!(
676                "try_get_past_object_latency",
677                "Time spent in try_get_past_object on the fullnode behind.",
678                JSON_RPC_LATENCY_SEC_BUCKETS.to_vec(),
679                registry
680            )
681            .unwrap(),
682            try_multi_get_past_objects_latency: register_histogram_with_registry!(
683                "try_multi_get_past_objects_latency",
684                "Time spent in try_multi_get_past_objects on the fullnode behind.",
685                JSON_RPC_LATENCY_SEC_BUCKETS.to_vec(),
686                registry
687            )
688            .unwrap(),
689            get_checkpoint_latency: register_histogram_with_registry!(
690                "get_checkpoint_latency",
691                "Time spent in get_checkpoint on the fullnode behind.",
692                JSON_RPC_LATENCY_SEC_BUCKETS.to_vec(),
693                registry
694            )
695            .unwrap(),
696            get_checkpoints_latency: register_histogram_with_registry!(
697                "get_checkpoints_latency",
698                "Time spent in get_checkpoints on the fullnode behind.",
699                JSON_RPC_LATENCY_SEC_BUCKETS.to_vec(),
700                registry
701            )
702            .unwrap(),
703            get_events_latency: register_histogram_with_registry!(
704                "get_events_latency",
705                "Time spent in get_events on the fullnode behind.",
706                JSON_RPC_LATENCY_SEC_BUCKETS.to_vec(),
707                registry
708            )
709            .unwrap(),
710            get_total_transaction_blocks_latency: register_histogram_with_registry!(
711                "get_total_transaction_blocks_latency",
712                "Time spent in get_total_transaction_blocks on the fullnode behind.",
713                JSON_RPC_LATENCY_SEC_BUCKETS.to_vec(),
714                registry
715            )
716            .unwrap(),
717            get_latest_checkpoint_sequence_number_latency: register_histogram_with_registry!(
718                "get_latest_checkpoint_sequence_number_latency",
719                "Time spent in get_latest_checkpoint_sequence_number on the fullnode behind.",
720                JSON_RPC_LATENCY_SEC_BUCKETS.to_vec(),
721                registry
722            )
723            .unwrap(),
724            get_owned_objects_latency: register_histogram_with_registry!(
725                "get_owned_objects_latency",
726                "Time spent in get_owned_objects on the fullnode behind.",
727                JSON_RPC_LATENCY_SEC_BUCKETS.to_vec(),
728                registry
729            )
730            .unwrap(),
731            query_transaction_blocks_latency: register_histogram_with_registry!(
732                "query_transaction_blocks_latency",
733                "Time spent in query_transaction_blocks on the fullnode behind.",
734                JSON_RPC_LATENCY_SEC_BUCKETS.to_vec(),
735                registry
736            )
737            .unwrap(),
738            query_events_latency: register_histogram_with_registry!(
739                "query_events_latency",
740                "Time spent in query_events on the fullnode behind.",
741                JSON_RPC_LATENCY_SEC_BUCKETS.to_vec(),
742                registry
743            )
744            .unwrap(),
745            get_dynamic_fields_latency: register_histogram_with_registry!(
746                "get_dynamic_fields_latency",
747                "Time spent in get_dynamic_fields on the fullnode behind.",
748                JSON_RPC_LATENCY_SEC_BUCKETS.to_vec(),
749                registry
750            )
751            .unwrap(),
752            get_dynamic_field_object_latency: register_histogram_with_registry!(
753                "get_dynamic_field_object_latency",
754                "Time spent in get_dynamic_field_object on the fullnode behind.",
755                JSON_RPC_LATENCY_SEC_BUCKETS.to_vec(),
756                registry
757            )
758            .unwrap(),
759            get_loaded_child_objects_latency: register_histogram_with_registry!(
760                "get_loaded_child_objects_latency",
761                "Time spent in get_loaded_child_objects_latency on the fullnode behind.",
762                JSON_RPC_LATENCY_SEC_BUCKETS.to_vec(),
763                registry
764            )
765            .unwrap(),
766            get_protocol_config_latency: register_histogram_with_registry!(
767                "get_protocol_config_latency",
768                "Time spent in get_protocol_config_latency on the fullnode behind.",
769                JSON_RPC_LATENCY_SEC_BUCKETS.to_vec(),
770                registry
771            )
772            .unwrap(),
773            db_conn_pool_size: register_int_gauge_with_registry!(
774                "db_conn_pool_size",
775                "Size of the database connection pool",
776                registry
777            ).unwrap(),
778            idle_db_conn: register_int_gauge_with_registry!(
779                "idle_db_conn",
780                "Number of idle database connections",
781                registry
782            ).unwrap(),
783            address_processor_failure: register_int_counter_with_registry!(
784                "address_processor_failure",
785                "Total number of address processor failure",
786                registry,
787            )
788            .unwrap(),
789            checkpoint_metrics_processor_failure: register_int_counter_with_registry!(
790                "checkpoint_metrics_processor_failure",
791                "Total number of checkpoint metrics processor failure",
792                registry,
793            )
794            .unwrap(),
795            last_pruned_epoch: register_int_gauge_with_registry!(
796                "last_pruned_epoch",
797                "Last pruned epoch number",
798                registry,
799            )
800            .unwrap(),
801            last_pruned_checkpoint: register_int_gauge_with_registry!(
802                "last_pruned_checkpoint",
803                "Last pruned checkpoint sequence number",
804                registry,
805            )
806            .unwrap(),
807            last_pruned_transaction: register_int_gauge_with_registry!(
808                "last_pruned_transaction",
809                "Last pruned transaction sequence number",
810                registry,
811            ).unwrap(),
812            epoch_pruning_latency: register_histogram_with_registry!(
813                "epoch_pruning_latency",
814                "Time spent in pruning one epoch",
815                DB_UPDATE_QUERY_LATENCY_SEC_BUCKETS.to_vec(),
816                registry
817            ).unwrap(),
818        }
819    }
820}
iota_indexer/metrics.rs

iota_indexer/
metrics.rs
