1use std::sync::Arc;
6
7use prometheus::{
8 Histogram, HistogramVec, IntCounter, IntCounterVec, IntGauge, IntGaugeVec, Registry,
9 exponential_buckets, register_histogram_vec_with_registry, register_histogram_with_registry,
10 register_int_counter_vec_with_registry, register_int_counter_with_registry,
11 register_int_gauge_vec_with_registry, register_int_gauge_with_registry,
12};
13
14use crate::network::metrics::NetworkMetrics;
15
16const FINE_GRAINED_LATENCY_SEC_BUCKETS: &[f64] = &[
18 0.000_001, 0.000_050, 0.000_100, 0.000_500, 0.001, 0.005, 0.01, 0.05, 0.1, 0.15, 0.2, 0.25,
19 0.3, 0.35, 0.4, 0.45, 0.5, 0.6, 0.7, 0.8, 0.9, 1.0, 1.2, 1.4, 1.6, 1.8, 2.0, 2.5, 3.0, 3.5,
20 4.0, 4.5, 5.0, 5.5, 6.0, 6.5, 7.0, 7.5, 8.0, 8.5, 9.0, 9.5, 10., 20., 30., 60., 120.,
21];
22
23const NUM_BUCKETS: &[f64] = &[
24 1.0,
25 2.0,
26 4.0,
27 8.0,
28 10.0,
29 20.0,
30 40.0,
31 80.0,
32 100.0,
33 150.0,
34 200.0,
35 400.0,
36 800.0,
37 1000.0,
38 2000.0,
39 3000.0,
40 5000.0,
41 10000.0,
42 20000.0,
43 30000.0,
44 50000.0,
45 100_000.0,
46 200_000.0,
47 300_000.0,
48 500_000.0,
49 1_000_000.0,
50];
51
52const LATENCY_SEC_BUCKETS: &[f64] = &[
53 0.001, 0.005, 0.01, 0.05, 0.1, 0.15, 0.2, 0.25, 0.3, 0.35, 0.4, 0.45, 0.5, 0.6, 0.7, 0.8, 0.9,
54 1.0, 1.2, 1.4, 1.6, 1.8, 2.0, 2.5, 3.0, 3.5, 4.0, 4.5, 5.0, 5.5, 6.0, 6.5, 7.0, 7.5, 8.0, 8.5,
55 9.0, 9.5, 10., 12.5, 15., 17.5, 20., 25., 30., 60., 90., 120., 180., 300.,
56];
57
58const SIZE_BUCKETS: &[f64] = &[
59 100.,
60 400.,
61 800.,
62 1_000.,
63 2_000.,
64 5_000.,
65 10_000.,
66 20_000.,
67 50_000.,
68 100_000.,
69 200_000.0,
70 300_000.0,
71 400_000.0,
72 500_000.0,
73 1_000_000.0,
74 2_000_000.0,
75 3_000_000.0,
76 5_000_000.0,
77 10_000_000.0,
78]; pub(crate) struct Metrics {
81 pub(crate) node_metrics: NodeMetrics,
82 pub(crate) network_metrics: NetworkMetrics,
83}
84
85pub(crate) fn initialise_metrics(registry: Registry) -> Arc<Metrics> {
86 let node_metrics = NodeMetrics::new(®istry);
87 let network_metrics = NetworkMetrics::new(®istry);
88
89 Arc::new(Metrics {
90 node_metrics,
91 network_metrics,
92 })
93}
94
95#[cfg(test)]
96pub(crate) fn test_metrics() -> Arc<Metrics> {
97 initialise_metrics(Registry::new())
98}
99
100pub(crate) struct NodeMetrics {
101 pub(crate) block_commit_latency: Histogram,
102 pub(crate) proposed_blocks: IntCounterVec,
103 pub(crate) proposed_block_size: Histogram,
104 pub(crate) proposed_block_transactions: Histogram,
105 pub(crate) proposed_block_ancestors: Histogram,
106 pub(crate) proposed_block_ancestors_depth: HistogramVec,
107 pub(crate) highest_verified_authority_round: IntGaugeVec,
108 pub(crate) lowest_verified_authority_round: IntGaugeVec,
109 pub(crate) block_proposal_interval: Histogram,
110 pub(crate) block_proposal_leader_wait_ms: IntCounterVec,
111 pub(crate) block_proposal_leader_wait_count: IntCounterVec,
112 pub(crate) block_timestamp_drift_wait_ms: IntCounterVec,
113 pub(crate) blocks_per_commit_count: Histogram,
114 pub(crate) blocks_pruned_on_commit: IntCounterVec,
115 pub(crate) broadcaster_rtt_estimate_ms: IntGaugeVec,
116 pub(crate) core_add_blocks_batch_size: Histogram,
117 pub(crate) core_check_block_refs_batch_size: Histogram,
118 pub(crate) core_lock_dequeued: IntCounter,
119 pub(crate) core_lock_enqueued: IntCounter,
120 pub(crate) core_skipped_proposals: IntCounterVec,
121 pub(crate) highest_accepted_authority_round: IntGaugeVec,
122 pub(crate) highest_accepted_round: IntGauge,
123 pub(crate) accepted_blocks: IntCounterVec,
124 pub(crate) dag_state_recent_blocks: IntGauge,
125 pub(crate) dag_state_recent_refs: IntGauge,
126 pub(crate) dag_state_store_read_count: IntCounterVec,
127 pub(crate) dag_state_store_write_count: IntCounter,
128 pub(crate) fetch_blocks_scheduler_inflight: IntGauge,
129 pub(crate) fetch_blocks_scheduler_skipped: IntCounterVec,
130 pub(crate) synchronizer_fetched_blocks_by_peer: IntCounterVec,
131 pub(crate) synchronizer_missing_blocks_by_authority: IntCounterVec,
132 pub(crate) synchronizer_current_missing_blocks_by_authority: IntGaugeVec,
133 pub(crate) synchronizer_fetched_blocks_by_authority: IntCounterVec,
134 pub(crate) network_received_excluded_ancestors_from_authority: IntCounterVec,
135 pub(crate) network_excluded_ancestors_sent_to_fetch: IntCounterVec,
136 pub(crate) network_excluded_ancestors_count_by_authority: IntCounterVec,
137 pub(crate) invalid_blocks: IntCounterVec,
138 pub(crate) rejected_blocks: IntCounterVec,
139 pub(crate) rejected_future_blocks: IntCounterVec,
140 pub(crate) subscribed_blocks: IntCounterVec,
141 pub(crate) verified_blocks: IntCounterVec,
142 pub(crate) committed_leaders_total: IntCounterVec,
143 pub(crate) last_committed_authority_round: IntGaugeVec,
144 pub(crate) last_committed_leader_round: IntGauge,
145 pub(crate) last_commit_index: IntGauge,
146 pub(crate) last_known_own_block_round: IntGauge,
147 pub(crate) sync_last_known_own_block_retries: IntCounter,
148 pub(crate) commit_round_advancement_interval: Histogram,
149 pub(crate) last_decided_leader_round: IntGauge,
150 pub(crate) leader_timeout_total: IntCounterVec,
151 pub(crate) smart_selection_wait: IntCounter,
152 pub(crate) ancestor_state_change_by_authority: IntCounterVec,
153 pub(crate) excluded_proposal_ancestors_count_by_authority: IntCounterVec,
154 pub(crate) included_excluded_proposal_ancestors_count_by_authority: IntCounterVec,
155 pub(crate) missing_blocks_total: IntCounter,
156 pub(crate) missing_blocks_after_fetch_total: IntCounter,
157 pub(crate) num_of_bad_nodes: IntGauge,
158 pub(crate) quorum_receive_latency: Histogram,
159 pub(crate) reputation_scores: IntGaugeVec,
160 pub(crate) scope_processing_time: HistogramVec,
161 pub(crate) sub_dags_per_commit_count: Histogram,
162 pub(crate) block_suspensions: IntCounterVec,
163 pub(crate) block_unsuspensions: IntCounterVec,
164 pub(crate) suspended_block_time: HistogramVec,
165 pub(crate) block_manager_suspended_blocks: IntGauge,
166 pub(crate) block_manager_missing_ancestors: IntGauge,
167 pub(crate) block_manager_missing_blocks: IntGauge,
168 pub(crate) block_manager_missing_blocks_by_authority: IntCounterVec,
169 pub(crate) block_manager_missing_ancestors_by_authority: IntCounterVec,
170 pub(crate) block_manager_gced_blocks: IntCounterVec,
171 pub(crate) block_manager_gc_unsuspended_blocks: IntCounterVec,
172 pub(crate) block_manager_skipped_blocks: IntCounterVec,
173 pub(crate) threshold_clock_round: IntGauge,
174 pub(crate) subscriber_connection_attempts: IntCounterVec,
175 pub(crate) subscribed_to: IntGaugeVec,
176 pub(crate) subscribed_by: IntGaugeVec,
177 pub(crate) commit_sync_inflight_fetches: IntGauge,
178 pub(crate) commit_sync_pending_fetches: IntGauge,
179 pub(crate) commit_sync_fetch_commits_handler_uncertified_skipped: IntCounter,
180 pub(crate) commit_sync_fetched_commits: IntCounter,
181 pub(crate) commit_sync_fetched_blocks: IntCounter,
182 pub(crate) commit_sync_total_fetched_blocks_size: IntCounter,
183 pub(crate) commit_sync_quorum_index: IntGauge,
184 pub(crate) commit_sync_highest_synced_index: IntGauge,
185 pub(crate) commit_sync_highest_fetched_index: IntGauge,
186 pub(crate) commit_sync_local_index: IntGauge,
187 pub(crate) commit_sync_gap_on_processing: IntCounter,
188 pub(crate) commit_sync_fetch_loop_latency: Histogram,
189 pub(crate) commit_sync_fetch_once_latency: Histogram,
190 pub(crate) commit_sync_fetch_once_errors: IntCounterVec,
191 pub(crate) commit_sync_fetch_missing_blocks: IntCounterVec,
192 pub(crate) round_prober_received_quorum_round_gaps: IntGaugeVec,
193 pub(crate) round_prober_accepted_quorum_round_gaps: IntGaugeVec,
194 pub(crate) round_prober_low_received_quorum_round: IntGaugeVec,
195 pub(crate) round_prober_low_accepted_quorum_round: IntGaugeVec,
196 pub(crate) round_prober_current_received_round_gaps: IntGaugeVec,
197 pub(crate) round_prober_current_accepted_round_gaps: IntGaugeVec,
198 pub(crate) round_prober_propagation_delays: Histogram,
199 pub(crate) round_prober_last_propagation_delay: IntGauge,
200 pub(crate) round_prober_request_errors: IntCounterVec,
201 pub(crate) uptime: Histogram,
202}
203
204impl NodeMetrics {
205 pub(crate) fn new(registry: &Registry) -> Self {
206 Self {
207 block_commit_latency: register_histogram_with_registry!(
208 "block_commit_latency",
209 "The time taken between block creation and block commit.",
210 LATENCY_SEC_BUCKETS.to_vec(),
211 registry,
212 ).unwrap(),
213 proposed_blocks: register_int_counter_vec_with_registry!(
214 "proposed_blocks",
215 "Total number of proposed blocks. If force is true then this block has been created forcefully via a leader timeout event.",
216 &["force"],
217 registry,
218 ).unwrap(),
219 proposed_block_size: register_histogram_with_registry!(
220 "proposed_block_size",
221 "The size (in bytes) of proposed blocks",
222 SIZE_BUCKETS.to_vec(),
223 registry
224 ).unwrap(),
225 proposed_block_transactions: register_histogram_with_registry!(
226 "proposed_block_transactions",
227 "# of transactions contained in proposed blocks",
228 NUM_BUCKETS.to_vec(),
229 registry
230 ).unwrap(),
231 proposed_block_ancestors: register_histogram_with_registry!(
232 "proposed_block_ancestors",
233 "Number of ancestors in proposed blocks",
234 exponential_buckets(1.0, 1.4, 20).unwrap(),
235 registry,
236 ).unwrap(),
237 proposed_block_ancestors_depth: register_histogram_vec_with_registry!(
238 "proposed_block_ancestors_depth",
239 "The depth in rounds of ancestors included in newly proposed blocks",
240 &["authority"],
241 exponential_buckets(1.0, 2.0, 14).unwrap(),
242 registry,
243 ).unwrap(),
244 highest_verified_authority_round: register_int_gauge_vec_with_registry!(
245 "highest_verified_authority_round",
246 "The highest round of verified block for the corresponding authority",
247 &["authority"],
248 registry,
249 ).unwrap(),
250 lowest_verified_authority_round: register_int_gauge_vec_with_registry!(
251 "lowest_verified_authority_round",
252 "The lowest round of verified block for the corresponding authority",
253 &["authority"],
254 registry,
255 ).unwrap(),
256 block_proposal_interval: register_histogram_with_registry!(
257 "block_proposal_interval",
258 "Intervals (in secs) between block proposals.",
259 FINE_GRAINED_LATENCY_SEC_BUCKETS.to_vec(),
260 registry,
261 ).unwrap(),
262 block_proposal_leader_wait_ms: register_int_counter_vec_with_registry!(
263 "block_proposal_leader_wait_ms",
264 "Total time in ms spent waiting for a leader when proposing blocks.",
265 &["authority"],
266 registry,
267 ).unwrap(),
268 block_proposal_leader_wait_count: register_int_counter_vec_with_registry!(
269 "block_proposal_leader_wait_count",
270 "Total times waiting for a leader when proposing blocks.",
271 &["authority"],
272 registry,
273 ).unwrap(),
274 block_timestamp_drift_wait_ms: register_int_counter_vec_with_registry!(
275 "block_timestamp_drift_wait_ms",
276 "Total time in ms spent waiting, when a received block has timestamp in future.",
277 &["authority", "source"],
278 registry,
279 ).unwrap(),
280 blocks_per_commit_count: register_histogram_with_registry!(
281 "blocks_per_commit_count",
282 "The number of blocks per commit.",
283 NUM_BUCKETS.to_vec(),
284 registry,
285 ).unwrap(),
286 blocks_pruned_on_commit: register_int_counter_vec_with_registry!(
287 "blocks_pruned_on_commit",
288 "Number of blocks that got pruned due to garbage collection during a commit. This is not an accurate metric and measures the pruned blocks on the edge of the commit.",
289 &["authority", "commit_status"],
290 registry,
291 ).unwrap(),
292 broadcaster_rtt_estimate_ms: register_int_gauge_vec_with_registry!(
293 "broadcaster_rtt_estimate_ms",
294 "Estimated RTT latency per peer authority, for block sending in Broadcaster",
295 &["peer"],
296 registry,
297 ).unwrap(),
298 core_add_blocks_batch_size: register_histogram_with_registry!(
299 "core_add_blocks_batch_size",
300 "The number of blocks received from Core for processing on a single batch",
301 NUM_BUCKETS.to_vec(),
302 registry,
303 ).unwrap(),
304 core_check_block_refs_batch_size: register_histogram_with_registry!(
305 "core_check_block_refs_batch_size",
306 "The number of excluded blocks received from Core for search on a single batch",
307 NUM_BUCKETS.to_vec(),
308 registry,
309 ).unwrap(),
310 core_lock_dequeued: register_int_counter_with_registry!(
311 "core_lock_dequeued",
312 "Number of dequeued core requests",
313 registry,
314 ).unwrap(),
315 core_lock_enqueued: register_int_counter_with_registry!(
316 "core_lock_enqueued",
317 "Number of enqueued core requests",
318 registry,
319 ).unwrap(),
320 core_skipped_proposals: register_int_counter_vec_with_registry!(
321 "core_skipped_proposals",
322 "Number of proposals skipped in the Core, per reason",
323 &["reason"],
324 registry,
325 ).unwrap(),
326 highest_accepted_authority_round: register_int_gauge_vec_with_registry!(
327 "highest_accepted_authority_round",
328 "The highest round where a block has been accepted per authority. Resets on restart.",
329 &["authority"],
330 registry,
331 ).unwrap(),
332 highest_accepted_round: register_int_gauge_with_registry!(
333 "highest_accepted_round",
334 "The highest round where a block has been accepted. Resets on restart.",
335 registry,
336 ).unwrap(),
337 accepted_blocks: register_int_counter_vec_with_registry!(
338 "accepted_blocks",
339 "Number of accepted blocks by source (own, others)",
340 &["source"],
341 registry,
342 ).unwrap(),
343 dag_state_recent_blocks: register_int_gauge_with_registry!(
344 "dag_state_recent_blocks",
345 "Number of recent blocks cached in the DagState",
346 registry,
347 ).unwrap(),
348 dag_state_recent_refs: register_int_gauge_with_registry!(
349 "dag_state_recent_refs",
350 "Number of recent refs cached in the DagState",
351 registry,
352 ).unwrap(),
353 dag_state_store_read_count: register_int_counter_vec_with_registry!(
354 "dag_state_store_read_count",
355 "Number of times DagState needs to read from store per operation type",
356 &["type"],
357 registry,
358 ).unwrap(),
359 dag_state_store_write_count: register_int_counter_with_registry!(
360 "dag_state_store_write_count",
361 "Number of times DagState needs to write to store",
362 registry,
363 ).unwrap(),
364 fetch_blocks_scheduler_inflight: register_int_gauge_with_registry!(
365 "fetch_blocks_scheduler_inflight",
366 "Designates whether the synchronizer scheduler task to fetch blocks is currently running",
367 registry,
368 ).unwrap(),
369 fetch_blocks_scheduler_skipped: register_int_counter_vec_with_registry!(
370 "fetch_blocks_scheduler_skipped",
371 "Number of times the scheduler skipped fetching blocks",
372 &["reason"],
373 registry
374 ).unwrap(),
375 synchronizer_fetched_blocks_by_peer: register_int_counter_vec_with_registry!(
376 "synchronizer_fetched_blocks_by_peer",
377 "Number of fetched blocks per peer authority via the synchronizer and also by block authority",
378 &["peer", "type"],
379 registry,
380 ).unwrap(),
381 synchronizer_missing_blocks_by_authority: register_int_counter_vec_with_registry!(
382 "synchronizer_missing_blocks_by_authority",
383 "Number of missing blocks per block author, as observed by the synchronizer during periodic sync.",
384 &["authority"],
385 registry,
386 ).unwrap(),
387 synchronizer_current_missing_blocks_by_authority: register_int_gauge_vec_with_registry!(
388 "synchronizer_current_missing_blocks_by_authority",
389 "Current number of missing blocks per block author, as observed by the synchronizer during periodic sync.",
390 &["authority"],
391 registry,
392 ).unwrap(),
393 synchronizer_fetched_blocks_by_authority: register_int_counter_vec_with_registry!(
394 "synchronizer_fetched_blocks_by_authority",
395 "Number of fetched blocks per block author via the synchronizer",
396 &["authority", "type"],
397 registry,
398 ).unwrap(),
399 network_received_excluded_ancestors_from_authority: register_int_counter_vec_with_registry!(
400 "network_received_excluded_ancestors_from_authority",
401 "Number of excluded ancestors received from each authority.",
402 &["authority"],
403 registry,
404 ).unwrap(),
405 network_excluded_ancestors_count_by_authority: register_int_counter_vec_with_registry!(
406 "network_excluded_ancestors_count_by_authority",
407 "Total number of excluded ancestors per authority.",
408 &["authority"],
409 registry,
410 ).unwrap(),
411 network_excluded_ancestors_sent_to_fetch: register_int_counter_vec_with_registry!(
412 "network_excluded_ancestors_sent_to_fetch",
413 "Number of excluded ancestors sent to fetch.",
414 &["authority"],
415 registry,
416 ).unwrap(),
417 last_known_own_block_round: register_int_gauge_with_registry!(
418 "last_known_own_block_round",
419 "The highest round of our own block as this has been synced from peers during an amnesia recovery",
420 registry,
421 ).unwrap(),
422 sync_last_known_own_block_retries: register_int_counter_with_registry!(
423 "sync_last_known_own_block_retries",
424 "Number of times this node tried to fetch the last own block from peers",
425 registry,
426 ).unwrap(),
427 invalid_blocks: register_int_counter_vec_with_registry!(
429 "invalid_blocks",
430 "Number of invalid blocks per peer authority",
431 &["authority", "source", "error"],
432 registry,
433 ).unwrap(),
434 rejected_blocks: register_int_counter_vec_with_registry!(
435 "rejected_blocks",
436 "Number of blocks rejected before verifications",
437 &["reason"],
438 registry,
439 ).unwrap(),
440 rejected_future_blocks: register_int_counter_vec_with_registry!(
441 "rejected_future_blocks",
442 "Number of blocks rejected because their timestamp is too far in the future",
443 &["authority"],
444 registry,
445 ).unwrap(),
446 subscribed_blocks: register_int_counter_vec_with_registry!(
447 "subscribed_blocks",
448 "Number of blocks received from each peer before verification",
449 &["authority"],
450 registry,
451 ).unwrap(),
452 verified_blocks: register_int_counter_vec_with_registry!(
453 "verified_blocks",
454 "Number of blocks received from each peer that are verified",
455 &["authority"],
456 registry,
457 ).unwrap(),
458 committed_leaders_total: register_int_counter_vec_with_registry!(
459 "committed_leaders_total",
460 "Total number of (direct or indirect) committed leaders per authority",
461 &["authority", "commit_type"],
462 registry,
463 ).unwrap(),
464 last_committed_authority_round: register_int_gauge_vec_with_registry!(
465 "last_committed_authority_round",
466 "The last round committed by authority.",
467 &["authority"],
468 registry,
469 ).unwrap(),
470 last_committed_leader_round: register_int_gauge_with_registry!(
471 "last_committed_leader_round",
472 "The last round where a leader was committed to store and sent to commit consumer.",
473 registry,
474 ).unwrap(),
475 last_commit_index: register_int_gauge_with_registry!(
476 "last_commit_index",
477 "Index of the last commit.",
478 registry,
479 ).unwrap(),
480 commit_round_advancement_interval: register_histogram_with_registry!(
481 "commit_round_advancement_interval",
482 "Intervals (in secs) between commit round advancements.",
483 FINE_GRAINED_LATENCY_SEC_BUCKETS.to_vec(),
484 registry,
485 ).unwrap(),
486 last_decided_leader_round: register_int_gauge_with_registry!(
487 "last_decided_leader_round",
488 "The last round where a commit decision was made.",
489 registry,
490 ).unwrap(),
491 leader_timeout_total: register_int_counter_vec_with_registry!(
492 "leader_timeout_total",
493 "Total number of leader timeouts, either when the min round time has passed, or max leader timeout",
494 &["timeout_type"],
495 registry,
496 ).unwrap(),
497 smart_selection_wait: register_int_counter_with_registry!(
498 "smart_selection_wait",
499 "Number of times we waited for smart ancestor selection.",
500 registry,
501 ).unwrap(),
502 ancestor_state_change_by_authority: register_int_counter_vec_with_registry!(
503 "ancestor_state_change_by_authority",
504 "The total number of times an ancestor state changed to EXCLUDE or INCLUDE.",
505 &["authority", "state"],
506 registry,
507 ).unwrap(),
508 excluded_proposal_ancestors_count_by_authority: register_int_counter_vec_with_registry!(
509 "excluded_proposal_ancestors_count_by_authority",
510 "Total number of excluded ancestors per authority during proposal.",
511 &["authority"],
512 registry,
513 ).unwrap(),
514 included_excluded_proposal_ancestors_count_by_authority: register_int_counter_vec_with_registry!(
515 "included_excluded_proposal_ancestors_count_by_authority",
516 "Total number of ancestors per authority with 'excluded' status that got included in proposal. Either weak or strong type.",
517 &["authority", "type"],
518 registry,
519 ).unwrap(),
520 missing_blocks_total: register_int_counter_with_registry!(
521 "missing_blocks_total",
522 "Total cumulative number of missing blocks",
523 registry,
524 ).unwrap(),
525 missing_blocks_after_fetch_total: register_int_counter_with_registry!(
526 "missing_blocks_after_fetch_total",
527 "Total number of missing blocks after fetching blocks from peer",
528 registry,
529 ).unwrap(),
530 num_of_bad_nodes: register_int_gauge_with_registry!(
531 "num_of_bad_nodes",
532 "The number of bad nodes in the new leader schedule",
533 registry
534 ).unwrap(),
535 quorum_receive_latency: register_histogram_with_registry!(
536 "quorum_receive_latency",
537 "The time it took to receive a new round quorum of blocks",
538 registry
539 ).unwrap(),
540 reputation_scores: register_int_gauge_vec_with_registry!(
541 "reputation_scores",
542 "Reputation scores for each authority",
543 &["authority"],
544 registry,
545 ).unwrap(),
546 scope_processing_time: register_histogram_vec_with_registry!(
547 "scope_processing_time",
548 "The processing time of a specific code scope",
549 &["scope"],
550 FINE_GRAINED_LATENCY_SEC_BUCKETS.to_vec(),
551 registry
552 ).unwrap(),
553 sub_dags_per_commit_count: register_histogram_with_registry!(
554 "sub_dags_per_commit_count",
555 "The number of subdags per commit.",
556 registry,
557 ).unwrap(),
558 block_suspensions: register_int_counter_vec_with_registry!(
559 "block_suspensions",
560 "The number block suspensions. The counter is reported uniquely, so if a block is sent for reprocessing while already suspended then is not double counted",
561 &["authority"],
562 registry,
563 ).unwrap(),
564 block_unsuspensions: register_int_counter_vec_with_registry!(
565 "block_unsuspensions",
566 "The number of block unsuspensions.",
567 &["authority"],
568 registry,
569 ).unwrap(),
570 suspended_block_time: register_histogram_vec_with_registry!(
571 "suspended_block_time",
572 "The time for which a block remains suspended",
573 &["authority"],
574 registry,
575 ).unwrap(),
576 block_manager_suspended_blocks: register_int_gauge_with_registry!(
577 "block_manager_suspended_blocks",
578 "The number of blocks currently suspended in the block manager",
579 registry,
580 ).unwrap(),
581 block_manager_missing_ancestors: register_int_gauge_with_registry!(
582 "block_manager_missing_ancestors",
583 "The number of missing ancestors tracked in the block manager",
584 registry,
585 ).unwrap(),
586 block_manager_missing_blocks: register_int_gauge_with_registry!(
587 "block_manager_missing_blocks",
588 "The number of blocks missing content tracked in the block manager",
589 registry,
590 ).unwrap(),
591 block_manager_missing_blocks_by_authority: register_int_counter_vec_with_registry!(
592 "block_manager_missing_blocks_by_authority",
593 "The number of new missing blocks by block authority",
594 &["authority"],
595 registry,
596 ).unwrap(),
597 block_manager_missing_ancestors_by_authority: register_int_counter_vec_with_registry!(
598 "block_manager_missing_ancestors_by_authority",
599 "The number of missing ancestors by ancestor authority across received blocks",
600 &["authority"],
601 registry,
602 ).unwrap(),
603 block_manager_gced_blocks: register_int_counter_vec_with_registry!(
604 "block_manager_gced_blocks",
605 "The number of blocks that garbage collected and did not get accepted, counted by block's source authority",
606 &["authority"],
607 registry,
608 ).unwrap(),
609 block_manager_gc_unsuspended_blocks: register_int_counter_vec_with_registry!(
610 "block_manager_gc_unsuspended_blocks",
611 "The number of blocks unsuspended because their missing ancestors are garbage collected by the block manager, counted by block's source authority",
612 &["authority"],
613 registry,
614 ).unwrap(),
615 block_manager_skipped_blocks: register_int_counter_vec_with_registry!(
616 "block_manager_skipped_blocks",
617 "The number of blocks skipped by the block manager due to block round being <= gc_round",
618 &["authority"],
619 registry,
620 ).unwrap(),
621 threshold_clock_round: register_int_gauge_with_registry!(
622 "threshold_clock_round",
623 "The current threshold clock round. We only advance to a new round when a quorum of parents have been synced.",
624 registry,
625 ).unwrap(),
626 subscriber_connection_attempts: register_int_counter_vec_with_registry!(
627 "subscriber_connection_attempts",
628 "The number of connection attempts per peer",
629 &["authority", "status"],
630 registry,
631 ).unwrap(),
632 subscribed_to: register_int_gauge_vec_with_registry!(
633 "subscribed_to",
634 "Peers that this authority subscribed to for block streams.",
635 &["authority"],
636 registry,
637 ).unwrap(),
638 subscribed_by: register_int_gauge_vec_with_registry!(
639 "subscribed_by",
640 "Peers subscribing for block streams from this authority.",
641 &["authority"],
642 registry,
643 ).unwrap(),
644 commit_sync_inflight_fetches: register_int_gauge_with_registry!(
645 "commit_sync_inflight_fetches",
646 "The number of inflight fetches in commit syncer",
647 registry,
648 ).unwrap(),
649 commit_sync_pending_fetches: register_int_gauge_with_registry!(
650 "commit_sync_pending_fetches",
651 "The number of pending fetches in commit syncer",
652 registry,
653 ).unwrap(),
654 commit_sync_fetched_commits: register_int_counter_with_registry!(
655 "commit_sync_fetched_commits",
656 "The number of commits fetched via commit syncer",
657 registry,
658 ).unwrap(),
659 commit_sync_fetched_blocks: register_int_counter_with_registry!(
660 "commit_sync_fetched_blocks",
661 "The number of blocks fetched via commit syncer",
662 registry,
663 ).unwrap(),
664 commit_sync_total_fetched_blocks_size: register_int_counter_with_registry!(
665 "commit_sync_total_fetched_blocks_size",
666 "The total size in bytes of blocks fetched via commit syncer",
667 registry,
668 ).unwrap(),
669 commit_sync_quorum_index: register_int_gauge_with_registry!(
670 "commit_sync_quorum_index",
671 "The maximum commit index voted by a quorum of authorities",
672 registry,
673 ).unwrap(),
674 commit_sync_highest_synced_index: register_int_gauge_with_registry!(
675 "commit_sync_fetched_index",
676 "The max commit index among local and fetched commits",
677 registry,
678 ).unwrap(),
679 commit_sync_highest_fetched_index: register_int_gauge_with_registry!(
680 "commit_sync_highest_fetched_index",
681 "The max commit index that has been fetched via network",
682 registry,
683 ).unwrap(),
684 commit_sync_local_index: register_int_gauge_with_registry!(
685 "commit_sync_local_index",
686 "The local commit index",
687 registry,
688 ).unwrap(),
689 commit_sync_gap_on_processing: register_int_counter_with_registry!(
690 "commit_sync_gap_on_processing",
691 "Number of instances where a gap was found in fetched commit processing",
692 registry,
693 ).unwrap(),
694 commit_sync_fetch_loop_latency: register_histogram_with_registry!(
695 "commit_sync_fetch_loop_latency",
696 "The time taken to finish fetching commits and blocks from a given range",
697 LATENCY_SEC_BUCKETS.to_vec(),
698 registry,
699 ).unwrap(),
700 commit_sync_fetch_once_latency: register_histogram_with_registry!(
701 "commit_sync_fetch_once_latency",
702 "The time taken to fetch commits and blocks once",
703 LATENCY_SEC_BUCKETS.to_vec(),
704 registry,
705 ).unwrap(),
706 commit_sync_fetch_once_errors: register_int_counter_vec_with_registry!(
707 "commit_sync_fetch_once_errors",
708 "Number of errors when attempting to fetch commits and blocks from single authority during commit sync.",
709 &["authority", "error"],
710 registry
711 ).unwrap(),
712 commit_sync_fetch_commits_handler_uncertified_skipped: register_int_counter_with_registry!(
713 "commit_sync_fetch_commits_handler_uncertified_skipped",
714 "Number of uncertified commits that got skipped when fetching commits due to lack of votes",
715 registry,
716 ).unwrap(),
717 commit_sync_fetch_missing_blocks: register_int_counter_vec_with_registry!(
718 "commit_sync_fetch_missing_blocks",
719 "Number of ancestor blocks that are missing when processing blocks via commit sync.",
720 &["authority"],
721 registry
722 ).unwrap(),
723 round_prober_received_quorum_round_gaps: register_int_gauge_vec_with_registry!(
724 "round_prober_received_quorum_round_gaps",
725 "Received round gaps among peers for blocks proposed from each authority",
726 &["authority"],
727 registry
728 ).unwrap(),
729 round_prober_accepted_quorum_round_gaps: register_int_gauge_vec_with_registry!(
730 "round_prober_accepted_quorum_round_gaps",
731 "Accepted round gaps among peers for blocks proposed & accepted from each authority",
732 &["authority"],
733 registry
734 ).unwrap(),
735 round_prober_low_received_quorum_round: register_int_gauge_vec_with_registry!(
736 "round_prober_low_received_quorum_round",
737 "Low quorum round among peers for blocks proposed from each authority",
738 &["authority"],
739 registry
740 ).unwrap(),
741 round_prober_low_accepted_quorum_round: register_int_gauge_vec_with_registry!(
742 "round_prober_low_accepted_quorum_round",
743 "Low quorum round among peers for blocks proposed & accepted from each authority",
744 &["authority"],
745 registry
746 ).unwrap(),
747 round_prober_current_received_round_gaps: register_int_gauge_vec_with_registry!(
748 "round_prober_current_received_round_gaps",
749 "Received round gaps from local last proposed round to the low received quorum round of each peer. Can be negative.",
750 &["authority"],
751 registry
752 ).unwrap(),
753 round_prober_current_accepted_round_gaps: register_int_gauge_vec_with_registry!(
754 "round_prober_current_accepted_round_gaps",
755 "Accepted round gaps from local last proposed & accepted round to the low accepted quorum round of each peer. Can be negative.",
756 &["authority"],
757 registry
758 ).unwrap(),
759 round_prober_propagation_delays: register_histogram_with_registry!(
760 "round_prober_propagation_delays",
761 "Round gaps between the last proposed block round and the lower bound of own quorum round",
762 NUM_BUCKETS.to_vec(),
763 registry
764 ).unwrap(),
765 round_prober_last_propagation_delay: register_int_gauge_with_registry!(
766 "round_prober_last_propagation_delay",
767 "Most recent propagation delay observed by RoundProber",
768 registry
769 ).unwrap(),
770 round_prober_request_errors: register_int_counter_vec_with_registry!(
771 "round_prober_request_errors",
772 "Number of errors when probing against peers per error type",
773 &["error_type"],
774 registry
775 ).unwrap(),
776 uptime: register_histogram_with_registry!(
777 "uptime",
778 "Total node uptime",
779 LATENCY_SEC_BUCKETS.to_vec(),
780 registry,
781 ).unwrap(),
782 }
783 }
784}