iota_metrics/
hardware_metrics.rs

1// Copyright (c) 2025 IOTA Stiftung
2// SPDX-License-Identifier: Apache-2.0
3
4use std::{
5    collections::HashMap,
6    path::{Path, PathBuf},
7    sync::{Arc, Mutex},
8};
9
10use prometheus_filtered::{
11    Filter, IntGauge, MetricLevel, Opts,
12    core::{Collector, Desc, Number},
13    proto::{LabelPair, Metric, MetricFamily, MetricType},
14};
15use sysinfo::{CpuRefreshKind, Disk, Disks, MemoryRefreshKind, RefreshKind, System};
16
17use crate::RegistryService;
18
19#[derive(thiserror::Error, Debug)]
20pub enum HardwareMetricsErr {
21    #[error("Failed creating metric: {0}")]
22    ErrCreateMetric(prometheus_filtered::Error),
23    #[error("Failed registering hardware metrics onto RegistryService: {0}")]
24    ErrRegisterHardwareMetrics(prometheus_filtered::Error),
25}
26
27/// Returns whether the hardware metrics should be registered under `filter`:
28/// any effective level for this module except `off` registers them.
29pub fn hardware_metrics_enabled(filter: &Filter) -> bool {
30    filter.is_exposed("hw", module_path!(), MetricLevel::Warn)
31}
32
33/// Register all hardware metrics: CPU specs, Memory specs/usage, Disk
34/// specs/usage
35/// These metrics are all named with a prefix "hw_"
36/// They are both pushed to iota-proxy and exposed on the /metrics endpoint.
37pub fn register_hardware_metrics(
38    registry_service: &RegistryService,
39    db_path: &Path,
40) -> Result<(), HardwareMetricsErr> {
41    // In the simulator these metrics would describe the host, not the simulated
42    // node, and sysinfo's refreshes run on the test thread where the intercepted
43    // clock and rng make them a source of non-determinism. Skip them entirely.
44    #[cfg(msim)]
45    {
46        let _ = (registry_service, db_path);
47        return Ok(());
48    }
49    #[cfg(not(msim))]
50    {
51        let registry = registry_service
52            .new_registry_custom(Some("hw".to_string()), None)
53            .map_err(HardwareMetricsErr::ErrRegisterHardwareMetrics)?;
54        registry
55            .register(Box::new(HardwareMetrics::new(db_path)?))
56            .map_err(HardwareMetricsErr::ErrRegisterHardwareMetrics)?;
57        registry_service.add(registry);
58        Ok(())
59    }
60}
61
62pub struct HardwareMetrics {
63    system: Arc<Mutex<System>>,
64    disks: Arc<Mutex<Disks>>,
65    // Descriptions for the static metrics
66    pub static_descriptions: Vec<Desc>,
67    // Static metrics contain metrics that are not expected to change during runtime
68    // e.g. CPU model, memory total, disk total, etc.
69    pub static_metric_families: Vec<MetricFamily>,
70    pub memory_available_collector: IntGauge,
71    // Path where the database is mounted (to identify which disk contains the DB)
72    pub db_path: PathBuf,
73}
74
75impl HardwareMetrics {
76    pub fn new(db_path: &Path) -> Result<Self, HardwareMetricsErr> {
77        let mut system = System::new_with_specifics(
78            RefreshKind::nothing()
79                .with_cpu(CpuRefreshKind::nothing())
80                .with_memory(MemoryRefreshKind::nothing().with_ram()),
81        );
82        system.refresh_all();
83
84        let disks = Disks::new_with_refreshed_list();
85
86        Ok(Self {
87            static_descriptions: Self::static_descriptions(&system, &disks, db_path)?,
88            static_metric_families: Self::static_metric_families(&system, &disks, db_path)?,
89            memory_available_collector: Self::memory_available_collector()?,
90            system: Arc::new(Mutex::new(system)),
91            disks: Arc::new(Mutex::new(disks)),
92            db_path: PathBuf::from(db_path),
93        })
94    }
95
96    pub fn static_descriptions(
97        system: &System,
98        disks: &Disks,
99        db_path: &Path,
100    ) -> Result<Vec<Desc>, HardwareMetricsErr> {
101        let mut descs: Vec<Desc> = Vec::new();
102        for mf in Self::static_metric_families(system, disks, db_path)? {
103            descs.push(Self::metric_family_desc(&mf)?);
104        }
105        Ok(descs)
106    }
107
108    pub fn static_metric_families(
109        system: &System,
110        disks: &Disks,
111        db_path: &Path,
112    ) -> Result<Vec<MetricFamily>, HardwareMetricsErr> {
113        let mut mfs = Vec::new();
114        mfs.push(Self::collect_cpu_specs(system));
115        mfs.extend(Self::memory_total_collector(system)?.collect());
116        for mf in Self::collect_disks_total_bytes(disks, db_path) {
117            mfs.push(mf);
118        }
119        Ok(mfs)
120    }
121
122    fn label(name: &str, value: impl ToString) -> LabelPair {
123        let mut label = LabelPair::new();
124        label.set_name(name.to_string());
125        label.set_value(value.to_string());
126        label
127    }
128
129    fn uint_gauge(
130        name: &str,
131        help: &str,
132        value: u64,
133        labels: &[Option<LabelPair>],
134    ) -> MetricFamily {
135        let mut g = prometheus_filtered::proto::Gauge::default();
136        let mut m = Metric::default();
137        let mut mf = MetricFamily::new();
138
139        g.set_value(value.into_f64());
140        m.set_gauge(g);
141        m.set_label(
142            labels
143                .iter()
144                .filter_map(|opt| opt.as_ref())
145                .cloned()
146                .collect::<Vec<_>>(),
147        );
148
149        mf.mut_metric().push(m);
150        mf.set_name(name.to_string());
151        mf.set_help(help.to_string());
152        mf.set_field_type(MetricType::GAUGE);
153        mf
154    }
155
156    fn metric_family_desc(fam: &MetricFamily) -> Result<Desc, HardwareMetricsErr> {
157        Desc::new(
158            fam.name().to_string(),
159            fam.help().to_string(),
160            vec![],
161            HashMap::new(),
162        )
163        .map_err(HardwareMetricsErr::ErrCreateMetric)
164    }
165
166    fn cpu_vendor_id(system: &System) -> String {
167        let vendor_id = system
168            .cpus()
169            .first()
170            .map_or("cpu_vendor_id_unavailable", |cpu| cpu.vendor_id());
171        match vendor_id {
172            "" => "cpu_vendor_id_unavailable",
173            _ => vendor_id,
174        }
175        .to_string()
176    }
177
178    fn cpu_model(system: &System) -> String {
179        let brand = system
180            .cpus()
181            .first()
182            .map_or("cpu_model_unavailable", |cpu| cpu.brand());
183        match brand {
184            "" => "cpu_model_unavailable",
185            _ => brand,
186        }
187        .to_string()
188    }
189
190    fn collect_cpu_specs(system: &System) -> MetricFamily {
191        Self::uint_gauge(
192            "cpu_core_count",
193            "CPU core count (and labels: model,vendor_id,arch)",
194            System::physical_core_count().unwrap_or_default() as u64,
195            &[
196                Some(Self::label("model", Self::cpu_model(system))),
197                Some(Self::label("vendor_id", Self::cpu_vendor_id(system))),
198                Some(Self::label("arch", System::cpu_arch())),
199            ],
200        )
201    }
202
203    // we deactivated collecting CPU usage per core to avoid performance impact
204    // fn collect_cpu_usage(system: &System) -> Result<Vec<MetricFamily>,
205    // HardwareMetricsErr> { let cpu_usage_per_core: Vec<MetricFamily> =
206    // system.cpus()         .iter()
207    //         .map(|core| {
208    //             let core_name = core.name();
209    //             Self::f64gauge(
210    //                 format!("cpu_{core_name}_usage"),
211    //                 format!("CPU core {core_name} usage in percent"),
212    //                 core.cpu_usage() as f64,
213    //             )
214    //         })
215    //         .collect();
216    //     Ok(cpu_usage_per_core)
217    // }
218
219    fn memory_total_collector(system: &System) -> Result<IntGauge, HardwareMetricsErr> {
220        let mem_total_bytes = system.total_memory();
221        let memory_total_collector =
222            IntGauge::with_opts(Opts::new("memory_total_bytes", "Memory total (bytes)"))
223                .map_err(HardwareMetricsErr::ErrCreateMetric)?;
224        memory_total_collector.set(mem_total_bytes as i64);
225        Ok(memory_total_collector)
226    }
227
228    fn memory_available_collector() -> Result<IntGauge, HardwareMetricsErr> {
229        IntGauge::with_opts(Opts::new(
230            "memory_available_bytes",
231            "Memory available (bytes)",
232        ))
233        .map_err(HardwareMetricsErr::ErrCreateMetric)
234    }
235
236    fn collect_memory_available(&self, system: &System) -> Option<Vec<MetricFamily>> {
237        let memory_available_bytes = match i64::try_from(system.available_memory()) {
238            Ok(bytes) => bytes,
239            Err(e) => {
240                tracing::error!("Failed converting memory_available_bytes to i64: {e}");
241                return None;
242            }
243        };
244        self.memory_available_collector.set(memory_available_bytes);
245        Some(self.memory_available_collector.collect())
246    }
247
248    fn disk_has_db(disk: &Disk, db_path: &Path) -> bool {
249        db_path.starts_with(disk.mount_point())
250    }
251
252    fn collect_disk_available(&self, disks: &Disks) -> Vec<MetricFamily> {
253        let space_available_per_disk: Vec<MetricFamily> = disks
254            .iter()
255            .enumerate()
256            .map(|(idx, disk)| {
257                let disk_name = disk.name().to_string_lossy();
258                let disk_num = idx + 1;
259                Self::uint_gauge(
260                    &format!("disk_{disk_num}_available_bytes",),
261                    &format!("Disk space available (bytes), for disk {disk_num}",),
262                    disk.available_space(),
263                    &[
264                        Some(Self::label("disk_name", disk_name.to_string())),
265                        if Self::disk_has_db(disk, &self.db_path) {
266                            Some(Self::label("is_database_disk", true))
267                        } else {
268                            None
269                        },
270                    ],
271                )
272            })
273            .collect();
274
275        space_available_per_disk
276    }
277
278    fn collect_disks_total_bytes(disks: &Disks, db_path: &Path) -> Vec<MetricFamily> {
279        let total_bytes_per_disk: Vec<MetricFamily> = disks
280            .iter()
281            .enumerate()
282            .map(|(idx, disk)| {
283                let disk_name = disk.name().to_string_lossy();
284                let disk_num = idx + 1;
285                Self::uint_gauge(
286                    &format!("disk_{disk_num}_total_bytes",),
287                    &format!("Disk space total (bytes), for disk {disk_num}",),
288                    disk.total_space(),
289                    &[
290                        Some(Self::label("disk_name", disk_name.to_string())),
291                        if Self::disk_has_db(disk, db_path) {
292                            Some(Self::label("is_database_disk", true))
293                        } else {
294                            None
295                        },
296                    ],
297                )
298            })
299            .collect();
300
301        total_bytes_per_disk
302    }
303}
304
305impl Collector for HardwareMetrics {
306    fn desc(&self) -> Vec<&Desc> {
307        self.static_descriptions.iter().collect()
308    }
309
310    fn collect(&self) -> Vec<MetricFamily> {
311        let mut system = match self.system.lock() {
312            Ok(lock) => lock,
313            Err(e) => {
314                tracing::error!("Failed acquiring lock on System: Lock is poisoned: {e}");
315                return Vec::new();
316            }
317        };
318        system.refresh_memory();
319
320        let mut disks = match self.disks.lock() {
321            Ok(lock) => lock,
322            Err(e) => {
323                tracing::error!("Failed acquiring lock on Disks: Lock is poisoned: {e}");
324                return Vec::new();
325            }
326        };
327        disks.refresh(true);
328
329        let mut mfs = self.static_metric_families.clone();
330        if let Some(families) = self.collect_memory_available(&system) {
331            mfs.extend(families);
332        };
333
334        mfs.extend(self.collect_disk_available(&disks));
335
336        mfs
337    }
338}
339
340#[cfg(test)]
341mod tests {
342    use std::{
343        net::SocketAddrV4,
344        path::PathBuf,
345        sync::LazyLock,
346        time::{SystemTime, UNIX_EPOCH},
347    };
348
349    use super::*;
350
351    static DB_PATH: LazyLock<PathBuf> = LazyLock::new(|| PathBuf::from("/opt/iota/db"));
352
353    #[tokio::test]
354    async fn test_collect_hardware_specs() -> Result<(), String> {
355        let prom_server_addr: SocketAddrV4 = "0.0.0.0:9194".parse().unwrap();
356
357        let registry_svc = crate::start_prometheus_server(prom_server_addr.into());
358
359        register_hardware_metrics(&registry_svc, &DB_PATH)
360            .expect("Failed registering hardware metrics");
361
362        let now = SystemTime::now()
363            .duration_since(UNIX_EPOCH)
364            .unwrap()
365            .as_millis() as i64;
366
367        let mut metric_families = registry_svc.gather_all();
368        for mf in metric_families.iter_mut() {
369            for m in mf.mut_metric() {
370                m.set_timestamp_ms(now);
371            }
372        }
373
374        let find_metric = |family_name: &str| -> Result<&Metric, String> {
375            let fname_namespaced = format!("hw_{}", family_name.trim_start_matches("hw_"));
376            let metric = metric_families
377                .iter()
378                .find(|mf| mf.name() == fname_namespaced)
379                .ok_or_else(|| format!("Metric family not found: {fname_namespaced}"))?
380                .get_metric()
381                .first()
382                .ok_or_else(|| format!("No metrics in family {fname_namespaced}"))?;
383            Ok(metric)
384        };
385        let find_metric_label = |family_name: &str, label_name: &str| -> Result<String, String> {
386            let metric = find_metric(family_name)?;
387            Ok(metric
388                .get_label()
389                .iter()
390                .find(|l| l.name() == label_name)
391                .ok_or_else(|| format!("Label not found: {label_name}"))?
392                .value()
393                .to_string())
394        };
395
396        let cpu_core_count = find_metric("cpu_core_count")?;
397        let core_count: usize = cpu_core_count.get_gauge().value() as usize;
398        assert!(core_count > 0 && core_count < 513);
399
400        // we only check specs are present in labels
401        let _ = find_metric_label("cpu_core_count", "model")?;
402        let _ = find_metric_label("cpu_core_count", "vendor_id")?;
403        let _ = find_metric_label("cpu_core_count", "arch")?;
404
405        let mem_total_bytes = find_metric("memory_total_bytes")?.get_gauge().value();
406        assert!(mem_total_bytes > 0.0);
407        let mem_available_bytes = find_metric("memory_available_bytes")?.get_gauge().value();
408        assert!(mem_available_bytes > 0.0);
409
410        let disk_1_total_bytes = find_metric("disk_1_total_bytes")?;
411        assert!(disk_1_total_bytes.get_gauge().value() > 0.0);
412        let disk_available = find_metric("disk_1_available_bytes")?;
413        assert!(disk_available.get_gauge().value() > 0.0);
414
415        Ok(())
416    }
417}
iota_metrics/hardware_metrics.rs

iota_metrics/
hardware_metrics.rs
