iota_metrics/
hardware_metrics.rs

1// Copyright (c) 2025 IOTA Stiftung
2// SPDX-License-Identifier: Apache-2.0
3
4use std::{
5    collections::HashMap,
6    path::{Path, PathBuf},
7    sync::{Arc, Mutex},
8};
9
10use prometheus_filtered::{
11    IntGauge, MetricLevel, Opts,
12    core::{Collector, Desc, Number},
13    proto::{LabelPair, Metric, MetricFamily, MetricType},
14};
15use sysinfo::{CpuRefreshKind, Disk, Disks, MemoryRefreshKind, RefreshKind, System};
16
17use crate::RegistryService;
18
19#[derive(thiserror::Error, Debug)]
20pub enum HardwareMetricsErr {
21    #[error("Failed creating metric: {0}")]
22    ErrCreateMetric(prometheus_filtered::Error),
23    #[error("Failed registering hardware metrics onto RegistryService: {0}")]
24    ErrRegisterHardwareMetrics(prometheus_filtered::Error),
25}
26
27/// Register all hardware metrics: CPU specs, Memory specs/usage, Disk
28/// specs/usage
29/// These metrics are all named with a prefix "hw_"
30/// They are both pushed to iota-proxy and exposed on the /metrics endpoint.
31/// The whole group shares one level (`off` hides the collector, any
32/// other level exposes it), since it is a single collector.
33pub fn register_hardware_metrics(
34    registry_service: &RegistryService,
35    db_path: &Path,
36) -> Result<(), HardwareMetricsErr> {
37    // In the simulator these metrics would describe the host, not the simulated
38    // node, and sysinfo's refreshes run on the test thread where the intercepted
39    // clock and rng make them a source of non-determinism. Skip them entirely.
40    #[cfg(msim)]
41    {
42        let _ = (registry_service, db_path);
43        return Ok(());
44    }
45    #[cfg(not(msim))]
46    {
47        let registry = registry_service
48            .new_registry_custom(Some("hw".to_string()), None)
49            .map_err(HardwareMetricsErr::ErrRegisterHardwareMetrics)?;
50        registry
51            .register_filtered(
52                // Bookkeeping key for this collector, does not change the metric name:
53                // the exposed names come from the collector and the "hw" prefix.
54                "metrics",
55                module_path!(),
56                MetricLevel::Warn,
57                HardwareMetrics::new(db_path)?,
58            )
59            .map_err(HardwareMetricsErr::ErrRegisterHardwareMetrics)?;
60        registry_service.add(registry);
61        Ok(())
62    }
63}
64
65#[derive(Clone)]
66pub struct HardwareMetrics {
67    system: Arc<Mutex<System>>,
68    disks: Arc<Mutex<Disks>>,
69    // Descriptions for the static metrics
70    pub static_descriptions: Vec<Desc>,
71    // Static metrics contain metrics that are not expected to change during runtime
72    // e.g. CPU model, memory total, disk total, etc.
73    pub static_metric_families: Vec<MetricFamily>,
74    pub memory_available_collector: IntGauge,
75    // Path where the database is mounted (to identify which disk contains the DB)
76    pub db_path: PathBuf,
77}
78
79impl HardwareMetrics {
80    pub fn new(db_path: &Path) -> Result<Self, HardwareMetricsErr> {
81        let mut system = System::new_with_specifics(
82            RefreshKind::nothing()
83                .with_cpu(CpuRefreshKind::nothing())
84                .with_memory(MemoryRefreshKind::nothing().with_ram()),
85        );
86        system.refresh_all();
87
88        let disks = Disks::new_with_refreshed_list();
89
90        Ok(Self {
91            static_descriptions: Self::static_descriptions(&system, &disks, db_path)?,
92            static_metric_families: Self::static_metric_families(&system, &disks, db_path)?,
93            memory_available_collector: Self::memory_available_collector()?,
94            system: Arc::new(Mutex::new(system)),
95            disks: Arc::new(Mutex::new(disks)),
96            db_path: PathBuf::from(db_path),
97        })
98    }
99
100    pub fn static_descriptions(
101        system: &System,
102        disks: &Disks,
103        db_path: &Path,
104    ) -> Result<Vec<Desc>, HardwareMetricsErr> {
105        let mut descs: Vec<Desc> = Vec::new();
106        for mf in Self::static_metric_families(system, disks, db_path)? {
107            descs.push(Self::metric_family_desc(&mf)?);
108        }
109        Ok(descs)
110    }
111
112    pub fn static_metric_families(
113        system: &System,
114        disks: &Disks,
115        db_path: &Path,
116    ) -> Result<Vec<MetricFamily>, HardwareMetricsErr> {
117        let mut mfs = Vec::new();
118        mfs.push(Self::collect_cpu_specs(system));
119        mfs.extend(Self::memory_total_collector(system)?.collect());
120        for mf in Self::collect_disks_total_bytes(disks, db_path) {
121            mfs.push(mf);
122        }
123        Ok(mfs)
124    }
125
126    fn label(name: &str, value: impl ToString) -> LabelPair {
127        let mut label = LabelPair::new();
128        label.set_name(name.to_string());
129        label.set_value(value.to_string());
130        label
131    }
132
133    fn uint_gauge(
134        name: &str,
135        help: &str,
136        value: u64,
137        labels: &[Option<LabelPair>],
138    ) -> MetricFamily {
139        let mut g = prometheus_filtered::proto::Gauge::default();
140        let mut m = Metric::default();
141        let mut mf = MetricFamily::new();
142
143        g.set_value(value.into_f64());
144        m.set_gauge(g);
145        m.set_label(
146            labels
147                .iter()
148                .filter_map(|opt| opt.as_ref())
149                .cloned()
150                .collect::<Vec<_>>(),
151        );
152
153        mf.mut_metric().push(m);
154        mf.set_name(name.to_string());
155        mf.set_help(help.to_string());
156        mf.set_field_type(MetricType::GAUGE);
157        mf
158    }
159
160    fn metric_family_desc(fam: &MetricFamily) -> Result<Desc, HardwareMetricsErr> {
161        Desc::new(
162            fam.name().to_string(),
163            fam.help().to_string(),
164            vec![],
165            HashMap::new(),
166        )
167        .map_err(HardwareMetricsErr::ErrCreateMetric)
168    }
169
170    fn cpu_vendor_id(system: &System) -> String {
171        let vendor_id = system
172            .cpus()
173            .first()
174            .map_or("cpu_vendor_id_unavailable", |cpu| cpu.vendor_id());
175        match vendor_id {
176            "" => "cpu_vendor_id_unavailable",
177            _ => vendor_id,
178        }
179        .to_string()
180    }
181
182    fn cpu_model(system: &System) -> String {
183        let brand = system
184            .cpus()
185            .first()
186            .map_or("cpu_model_unavailable", |cpu| cpu.brand());
187        match brand {
188            "" => "cpu_model_unavailable",
189            _ => brand,
190        }
191        .to_string()
192    }
193
194    fn collect_cpu_specs(system: &System) -> MetricFamily {
195        Self::uint_gauge(
196            "cpu_core_count",
197            "CPU core count (and labels: model,vendor_id,arch)",
198            System::physical_core_count().unwrap_or_default() as u64,
199            &[
200                Some(Self::label("model", Self::cpu_model(system))),
201                Some(Self::label("vendor_id", Self::cpu_vendor_id(system))),
202                Some(Self::label("arch", System::cpu_arch())),
203            ],
204        )
205    }
206
207    // we deactivated collecting CPU usage per core to avoid performance impact
208    // fn collect_cpu_usage(system: &System) -> Result<Vec<MetricFamily>,
209    // HardwareMetricsErr> { let cpu_usage_per_core: Vec<MetricFamily> =
210    // system.cpus()         .iter()
211    //         .map(|core| {
212    //             let core_name = core.name();
213    //             Self::f64gauge(
214    //                 format!("cpu_{core_name}_usage"),
215    //                 format!("CPU core {core_name} usage in percent"),
216    //                 core.cpu_usage() as f64,
217    //             )
218    //         })
219    //         .collect();
220    //     Ok(cpu_usage_per_core)
221    // }
222
223    fn memory_total_collector(system: &System) -> Result<IntGauge, HardwareMetricsErr> {
224        let mem_total_bytes = system.total_memory();
225        let memory_total_collector =
226            IntGauge::with_opts(Opts::new("memory_total_bytes", "Memory total (bytes)"))
227                .map_err(HardwareMetricsErr::ErrCreateMetric)?;
228        memory_total_collector.set(mem_total_bytes as i64);
229        Ok(memory_total_collector)
230    }
231
232    fn memory_available_collector() -> Result<IntGauge, HardwareMetricsErr> {
233        IntGauge::with_opts(Opts::new(
234            "memory_available_bytes",
235            "Memory available (bytes)",
236        ))
237        .map_err(HardwareMetricsErr::ErrCreateMetric)
238    }
239
240    fn collect_memory_available(&self, system: &System) -> Option<Vec<MetricFamily>> {
241        let memory_available_bytes = match i64::try_from(system.available_memory()) {
242            Ok(bytes) => bytes,
243            Err(e) => {
244                tracing::error!("Failed converting memory_available_bytes to i64: {e}");
245                return None;
246            }
247        };
248        self.memory_available_collector.set(memory_available_bytes);
249        Some(self.memory_available_collector.collect())
250    }
251
252    fn disk_has_db(disk: &Disk, db_path: &Path) -> bool {
253        db_path.starts_with(disk.mount_point())
254    }
255
256    fn collect_disk_available(&self, disks: &Disks) -> Vec<MetricFamily> {
257        let space_available_per_disk: Vec<MetricFamily> = disks
258            .iter()
259            .enumerate()
260            .map(|(idx, disk)| {
261                let disk_name = disk.name().to_string_lossy();
262                let disk_num = idx + 1;
263                Self::uint_gauge(
264                    &format!("disk_{disk_num}_available_bytes",),
265                    &format!("Disk space available (bytes), for disk {disk_num}",),
266                    disk.available_space(),
267                    &[
268                        Some(Self::label("disk_name", disk_name.to_string())),
269                        if Self::disk_has_db(disk, &self.db_path) {
270                            Some(Self::label("is_database_disk", true))
271                        } else {
272                            None
273                        },
274                    ],
275                )
276            })
277            .collect();
278
279        space_available_per_disk
280    }
281
282    fn collect_disks_total_bytes(disks: &Disks, db_path: &Path) -> Vec<MetricFamily> {
283        let total_bytes_per_disk: Vec<MetricFamily> = disks
284            .iter()
285            .enumerate()
286            .map(|(idx, disk)| {
287                let disk_name = disk.name().to_string_lossy();
288                let disk_num = idx + 1;
289                Self::uint_gauge(
290                    &format!("disk_{disk_num}_total_bytes",),
291                    &format!("Disk space total (bytes), for disk {disk_num}",),
292                    disk.total_space(),
293                    &[
294                        Some(Self::label("disk_name", disk_name.to_string())),
295                        if Self::disk_has_db(disk, db_path) {
296                            Some(Self::label("is_database_disk", true))
297                        } else {
298                            None
299                        },
300                    ],
301                )
302            })
303            .collect();
304
305        total_bytes_per_disk
306    }
307}
308
309impl Collector for HardwareMetrics {
310    fn desc(&self) -> Vec<&Desc> {
311        self.static_descriptions.iter().collect()
312    }
313
314    fn collect(&self) -> Vec<MetricFamily> {
315        let mut system = match self.system.lock() {
316            Ok(lock) => lock,
317            Err(e) => {
318                tracing::error!("Failed acquiring lock on System: Lock is poisoned: {e}");
319                return Vec::new();
320            }
321        };
322        system.refresh_memory();
323
324        let mut disks = match self.disks.lock() {
325            Ok(lock) => lock,
326            Err(e) => {
327                tracing::error!("Failed acquiring lock on Disks: Lock is poisoned: {e}");
328                return Vec::new();
329            }
330        };
331        disks.refresh(true);
332
333        let mut mfs = self.static_metric_families.clone();
334        if let Some(families) = self.collect_memory_available(&system) {
335            mfs.extend(families);
336        };
337
338        mfs.extend(self.collect_disk_available(&disks));
339
340        mfs
341    }
342}
343
344#[cfg(test)]
345mod tests {
346    use std::{
347        net::SocketAddrV4,
348        path::PathBuf,
349        sync::LazyLock,
350        time::{SystemTime, UNIX_EPOCH},
351    };
352
353    use super::*;
354
355    static DB_PATH: LazyLock<PathBuf> = LazyLock::new(|| PathBuf::from("/opt/iota/db"));
356
357    #[tokio::test]
358    async fn test_collect_hardware_specs() -> Result<(), String> {
359        let prom_server_addr: SocketAddrV4 = "0.0.0.0:9194".parse().unwrap();
360
361        let registry_svc = crate::start_prometheus_server(prom_server_addr.into());
362
363        register_hardware_metrics(&registry_svc, &DB_PATH)
364            .expect("Failed registering hardware metrics");
365
366        let now = SystemTime::now()
367            .duration_since(UNIX_EPOCH)
368            .unwrap()
369            .as_millis() as i64;
370
371        let mut metric_families = registry_svc.gather_all();
372        for mf in metric_families.iter_mut() {
373            for m in mf.mut_metric() {
374                m.set_timestamp_ms(now);
375            }
376        }
377
378        let find_metric = |family_name: &str| -> Result<&Metric, String> {
379            let fname_namespaced = format!("hw_{}", family_name.trim_start_matches("hw_"));
380            let metric = metric_families
381                .iter()
382                .find(|mf| mf.name() == fname_namespaced)
383                .ok_or_else(|| format!("Metric family not found: {fname_namespaced}"))?
384                .get_metric()
385                .first()
386                .ok_or_else(|| format!("No metrics in family {fname_namespaced}"))?;
387            Ok(metric)
388        };
389        let find_metric_label = |family_name: &str, label_name: &str| -> Result<String, String> {
390            let metric = find_metric(family_name)?;
391            Ok(metric
392                .get_label()
393                .iter()
394                .find(|l| l.name() == label_name)
395                .ok_or_else(|| format!("Label not found: {label_name}"))?
396                .value()
397                .to_string())
398        };
399
400        let cpu_core_count = find_metric("cpu_core_count")?;
401        let core_count: usize = cpu_core_count.get_gauge().value() as usize;
402        assert!(core_count > 0 && core_count < 513);
403
404        // we only check specs are present in labels
405        let _ = find_metric_label("cpu_core_count", "model")?;
406        let _ = find_metric_label("cpu_core_count", "vendor_id")?;
407        let _ = find_metric_label("cpu_core_count", "arch")?;
408
409        let mem_total_bytes = find_metric("memory_total_bytes")?.get_gauge().value();
410        assert!(mem_total_bytes > 0.0);
411        let mem_available_bytes = find_metric("memory_available_bytes")?.get_gauge().value();
412        assert!(mem_available_bytes > 0.0);
413
414        let disk_1_total_bytes = find_metric("disk_1_total_bytes")?;
415        assert!(disk_1_total_bytes.get_gauge().value() > 0.0);
416        let disk_available = find_metric("disk_1_available_bytes")?;
417        assert!(disk_available.get_gauge().value() > 0.0);
418
419        Ok(())
420    }
421}
iota_metrics/hardware_metrics.rs

iota_metrics/
hardware_metrics.rs
