From 92e54418bcdecf9da0a649649544b4d23fd33b03 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jan=20Lo=C5=A1=C5=A5=C3=A1k?= Date: Mon, 24 Aug 2026 19:13:05 +0200 Subject: [PATCH] Align metric names with Prometheus naming conventions Use base units, export cumulative values as _total counters, update docs, dashboards and rules. --- docs/architecture.md | 22 ++- docs/configuration.md | 2 +- docs/metrics.md | 146 ++++++++++++------ examples/pve-exporter.rules.yml | 50 +++--- examples/pve-guest-details.json | 44 +++--- examples/pve-node-details.json | 68 ++++---- examples/pve-overview.json | 38 ++--- metrics/pve_metrics_manager.go | 14 +- metrics/pve_metrics_naming_test.go | 72 +++++++++ metrics/pve_node_container_collector.go | 58 +++---- metrics/pve_node_disk_collector.go | 6 +- metrics/pve_node_status_collector.go | 26 ++-- metrics/pve_node_subscription_collector.go | 8 +- metrics/pve_node_virtual_machine_collector.go | 126 +++++++-------- metrics/pve_node_zfs_collector.go | 24 +-- metrics/pve_ttl_metrics.go | 121 ++++++++++++++- 16 files changed, 541 insertions(+), 284 deletions(-) create mode 100644 metrics/pve_metrics_naming_test.go diff --git a/docs/architecture.md b/docs/architecture.md index 0ada96d..89730b0 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -58,15 +58,18 @@ type PveMetricsCollector interface { ``` Successful execution time is recorded in -`pve_metrics_collection_latency_ms`. A collector error is logged and the +`pve_metrics_collection_duration_seconds`. A collector error is logged and the manager continues with the next collector. ## TTL metrics -Most project metrics use `TTLGaugeVec`, a wrapper around Prometheus `GaugeVec`. -Every label set records its last update. `TTLRegistry` checks registered metrics -every five seconds and removes series that have not been updated for five -minutes. +Gauges use `TTLGaugeVec`, a wrapper around Prometheus `GaugeVec`. Cumulative +values use `TTLCounterVec`, which exports the absolute value reported by the PVE +API as a Prometheus counter (`GaugeVec` cannot be used because a counter name +must carry a `_total` suffix, and `CounterVec` cannot be used because it only +supports `Inc`/`Add`). Every label set records its last update. `TTLRegistry` +checks registered metrics every five seconds and removes series that have not +been updated for five minutes. This prevents deleted guests or storage resources from remaining indefinitely, while allowing short API failures to preserve the last known values. @@ -75,14 +78,19 @@ while allowing short API failures to preserve the last known values. 1. Add typed response models and API methods under `proxmox`. 2. Add a collector implementing `PveMetricsCollector` under `metrics`. -3. Create metrics with `NewTTLGaugeVec`, choose stable labels, and register each - metric with the shared `TTLRegistry`. +3. Create metrics with `NewTTLGaugeVec` or, for cumulative values, with + `NewTTLCounterVec`, choose stable labels, and register each metric with the + shared `TTLRegistry`. 4. Add a boolean switch to `PveMetricsConfiguration` and `config.example.yaml`. 5. Register the collector in `NewPveMetricsManager`. 6. Add parsing, state-mapping, and topology tests as appropriate. 7. Document the switch, labels, units, and state values in [configuration.md](configuration.md) and [metrics.md](metrics.md). +Metric names must follow the Prometheus naming conventions; they are verified by +`TestPveMetricNamesFollowPrometheusConventions`, which lints every registered +metric with `promlint`. + Avoid labels containing changing messages, timestamps, or other unbounded values. For recursive resources such as ZFS, include a stable path so repeated component names remain distinguishable. diff --git a/docs/configuration.md b/docs/configuration.md index ea14377..8a0bc69 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -101,7 +101,7 @@ All switches are booleans. Omitted switches default to `false`. | `subscription` | Node subscription information | `pve_node_subscription_` | | `sdn` | Software-defined network state | `pve_sdn_` | -The exporter-level `pve_metrics_collection_latency_ms` summary is always +The exporter-level `pve_metrics_collection_duration_seconds` summary is always registered. ## Validation notes diff --git a/docs/metrics.md b/docs/metrics.md index be4462e..14bd478 100644 --- a/docs/metrics.md +++ b/docs/metrics.md @@ -3,6 +3,12 @@ All exporter-specific metric names start with `pve_`. The Go Prometheus client also exposes its standard `go_*`, `process_*`, and HTTP handler metrics. +Metric names follow the [Prometheus naming conventions](https://prometheus.io/docs/practices/naming/): +base units only (seconds, bytes, ratios) and a `_total` suffix on every metric +exported as a counter. Counters hold the absolute value reported by the PVE API +and reset when the guest or node restarts, so query them with `rate()` or +`increase()`. All remaining metrics are gauges. + ## Common labels and lifecycle Most resource metrics use these labels: @@ -22,7 +28,7 @@ remain visible for up to five minutes after it stops. | Metric | Labels | Description | | --- | --- | --- | -| `pve_metrics_collection_latency_ms` | `collector` | Summary of successful collector execution time in milliseconds. | +| `pve_metrics_collection_duration_seconds` | `collector` | Summary of successful collector execution time in seconds. | ## Cluster state @@ -42,9 +48,9 @@ and `node`. | Metric | Description | | --- | --- | | `pve_node_state` | Node online state: `1` online, `0` offline. | -| `pve_node_uptime` | Node uptime in seconds. | -| `pve_node_cpu_count` | Number of logical CPUs. | -| `pve_node_cpu_usage` | CPU usage reported by PVE. | +| `pve_node_uptime_seconds` | Node uptime in seconds. | +| `pve_node_cpus` | Number of logical CPUs. | +| `pve_node_cpu_usage_ratio` | CPU usage reported by PVE as a ratio between `0` and `1`. | | `pve_node_memory_total_bytes` | Total memory in bytes. | | `pve_node_memory_used_bytes` | Used memory in bytes. | | `pve_node_memory_free_bytes` | Free memory in bytes. | @@ -57,10 +63,10 @@ and `node`. | `pve_node_rootfs_used_bytes` | Root filesystem used bytes. | | `pve_node_rootfs_total_bytes` | Root filesystem total bytes. | | `pve_node_rootfs_avail_bytes` | Root filesystem bytes available to unprivileged processes. | -| `pve_node_time` | Node UTC Unix timestamp. | -| `pve_node_localtime` | Node local Unix timestamp. | -| `pve_node_cpuinfo` | Constant `1` carrying `flags`, `cores`, `model`, `sockets`, `cpus`, and `hvm` labels. | -| `pve_node_systeminfo` | Constant `1` carrying `kversion`, `pveversion`, `machine`, `sysname`, and `release` labels. | +| `pve_node_time_seconds` | Node UTC Unix timestamp. | +| `pve_node_localtime_seconds` | Node local Unix timestamp. | +| `pve_node_cpu_info` | Constant `1` carrying `flags`, `cores`, `model`, `sockets`, `cpus`, and `hvm` labels. | +| `pve_node_system_info` | Constant `1` carrying `kversion`, `pveversion`, `machine`, `sysname`, and `release` labels. | ## Physical disks @@ -70,7 +76,7 @@ Enabled with `metrics.disk`. Labels are `cluster`, `node`, `wwn`, `type`, | Metric | Description | | --- | --- | | `pve_node_disk_healthy` | SMART health state: `1` for `OK` or `PASSED`, otherwise `0`. | -| `pve_node_disk_wearout` | Device wearout percentage when supplied by the PVE API. | +| `pve_node_disk_wearout_percent` | Device wearout percentage when supplied by the PVE API. | | `pve_node_disk_size_bytes` | Physical disk size in bytes. | ## ZFS @@ -86,9 +92,9 @@ hierarchy, and `leaf` is `true` for a device entry. | Metric | Description | | --- | --- | | `pve_node_zfs_state` | Numeric component state: `0=UNKNOWN`, `1=ONLINE`, `2=DEGRADED`, `3=FAULTED`, `4=OFFLINE`, `5=REMOVED`, `6=UNAVAIL`. | -| `pve_node_zfs_read_errors` | ZFS read error count reported for the component. | -| `pve_node_zfs_write_errors` | ZFS write error count reported for the component. | -| `pve_node_zfs_checksum_errors` | ZFS checksum error count reported for the component. | +| `pve_node_zfs_read_errors_total` | ZFS read error count reported for the component. | +| `pve_node_zfs_write_errors_total` | ZFS write error count reported for the component. | +| `pve_node_zfs_checksum_errors_total` | ZFS checksum error count reported for the component. | ZFS section entries that do not contain a counter do not produce a false zero series. A counter explicitly returned as zero is exported normally. @@ -100,9 +106,9 @@ pve_node_zfs_state != 1 ``` ```promql -(pve_node_zfs_read_errors > 0) -or (pve_node_zfs_write_errors > 0) -or (pve_node_zfs_checksum_errors > 0) +(pve_node_zfs_read_errors_total > 0) +or (pve_node_zfs_write_errors_total > 0) +or (pve_node_zfs_checksum_errors_total > 0) ``` ## PVE storage @@ -126,8 +132,8 @@ Enabled with `metrics.subscription`. The common labels are `cluster` and | --- | --- | | `pve_node_subscription_info` | Constant `1` carrying subscription product and server identifiers. | | `pve_node_subscription_status` | `0` for API status `notfound`, `1` for `active`, `2` for expired or otherwise unusable states such as `invalid`, `suspended`, and `new`. | -| `pve_node_subscription_regdate` | Subscription registration date as a Unix timestamp, when available. | -| `pve_node_subscription_nextduedate` | Next due date as a Unix timestamp, when available. | +| `pve_node_subscription_registration_timestamp_seconds` | Subscription registration date as a Unix timestamp, when available. | +| `pve_node_subscription_next_due_timestamp_seconds` | Next due date as a Unix timestamp, when available. | | `pve_node_subscription_sockets` | Number of covered sockets reported by PVE. | The PVE API cannot distinguish a node that never had a subscription from one @@ -155,16 +161,16 @@ Enabled with `metrics.lxc`. Templates are skipped. Labels are `cluster`, | Metric | Description | | --- | --- | | `pve_ct_state` | `1` running, `0` stopped. | -| `pve_ct_uptime` | Uptime in seconds; updated only while running. | -| `pve_ct_cpu_count` | Configured CPU count. | -| `pve_ct_cpu_usage` | CPU usage reported by PVE; updated only while running. | -| `pve_ct_mem_total_bytes` | Configured memory limit in bytes. | -| `pve_ct_mem_used_bytes` | Used memory in bytes; updated only while running. | -| `pve_ct_network_in_bytes` | Received bytes since container start; updated only while running. | -| `pve_ct_network_out_bytes` | Transmitted bytes since container start; updated only while running. | -| `pve_ct_disk_rd_bytes` | Disk bytes read; updated only while running. | -| `pve_ct_disk_wr_bytes` | Disk bytes written; updated only while running. | -| `pve_ct_disk_usage_bytes` | Used root disk bytes; updated only while running. | +| `pve_ct_uptime_seconds` | Uptime in seconds; updated only while running. | +| `pve_ct_cpus` | Configured CPU count. | +| `pve_ct_cpu_usage_ratio` | CPU usage reported by PVE as a ratio between `0` and `1`; updated only while running. | +| `pve_ct_memory_total_bytes` | Configured memory limit in bytes. | +| `pve_ct_memory_used_bytes` | Used memory in bytes; updated only while running. | +| `pve_ct_network_receive_bytes_total` | Received bytes since container start; updated only while running. | +| `pve_ct_network_transmit_bytes_total` | Transmitted bytes since container start; updated only while running. | +| `pve_ct_disk_read_bytes_total` | Disk bytes read; updated only while running. | +| `pve_ct_disk_write_bytes_total` | Disk bytes written; updated only while running. | +| `pve_ct_disk_used_bytes` | Used root disk bytes; updated only while running. | | `pve_ct_disk_size_bytes` | Configured root disk size in bytes. | | `pve_ct_swap_used_bytes` | Used swap in bytes; updated only while running. | @@ -176,23 +182,77 @@ Enabled with `metrics.qemu`. Templates are skipped. Base labels are `cluster`, | Metric | Extra label | Description | | --- | --- | --- | | `pve_vm_state` | none | `1` running, `0` stopped. | -| `pve_vm_uptime` | none | Uptime in seconds; updated only while running. | +| `pve_vm_uptime_seconds` | none | Uptime in seconds; updated only while running. | | `pve_vm_agent` | none | QEMU guest agent state reported by PVE; updated only while running. | -| `pve_vm_cpu_count` | none | Configured CPU count. | -| `pve_vm_cpu_usage` | none | CPU usage reported by PVE; updated only while running. | -| `pve_vm_mem_total_bytes` | none | Configured maximum memory in bytes. | -| `pve_vm_mem_used_bytes` | none | Used memory in bytes; updated only while running. | -| `pve_vm_disk_usage_bytes` | none | Root disk usage reported by PVE. | +| `pve_vm_cpus` | none | Configured CPU count. | +| `pve_vm_cpu_usage_ratio` | none | CPU usage reported by PVE as a ratio between `0` and `1`; updated only while running. | +| `pve_vm_memory_total_bytes` | none | Configured maximum memory in bytes. | +| `pve_vm_memory_used_bytes` | none | Used memory in bytes; updated only while running. | +| `pve_vm_disk_used_bytes` | none | Root disk usage reported by PVE. | | `pve_vm_disk_size_bytes` | none | Configured maximum root disk size in bytes. | -| `pve_vm_network_in_bytes` | `interface` | Bytes received by a virtual interface. | -| `pve_vm_network_out_bytes` | `interface` | Bytes transmitted by a virtual interface. | -| `pve_vm_disk_rd_operations` | `device` | Successful read operations. | -| `pve_vm_disk_wr_operations` | `device` | Successful write operations. | -| `pve_vm_disk_rd_bytes` | `device` | Bytes read from the block device. | -| `pve_vm_disk_wr_bytes` | `device` | Bytes written to the block device. | -| `pve_vm_disk_failed_rd_ops` | `device` | Failed read operations. | -| `pve_vm_disk_failed_wr_ops` | `device` | Failed write operations. | -| `pve_vm_disk_rd_time_total_ns` | `device` | Total block-device read time in nanoseconds. | -| `pve_vm_disk_wr_time_total_ns` | `device` | Total block-device write time in nanoseconds. | +| `pve_vm_network_receive_bytes_total` | `interface` | Bytes received by a virtual interface. | +| `pve_vm_network_transmit_bytes_total` | `interface` | Bytes transmitted by a virtual interface. | +| `pve_vm_disk_read_operations_total` | `device` | Successful read operations. | +| `pve_vm_disk_write_operations_total` | `device` | Successful write operations. | +| `pve_vm_disk_read_bytes_total` | `device` | Bytes read from the block device. | +| `pve_vm_disk_write_bytes_total` | `device` | Bytes written to the block device. | +| `pve_vm_disk_failed_read_operations_total` | `device` | Failed read operations. | +| `pve_vm_disk_failed_write_operations_total` | `device` | Failed write operations. | +| `pve_vm_disk_read_time_seconds_total` | `device` | Total block-device read time in seconds. | +| `pve_vm_disk_write_time_seconds_total` | `device` | Total block-device write time in seconds. | Interface and block-device metrics are collected only for running VMs. + +## Migration from earlier releases + +All metric names were aligned with the Prometheus naming conventions. The table +below maps the previous names to the current ones. Dashboards, recording rules, +and alerting rules built against the old names have to be updated; the example +dashboards and rules in `examples/` already use the new names. + +| Previous name | Current name | Note | +| --- | --- | --- | +| `pve_ct_cpu_usage` | `pve_ct_cpu_usage_ratio` | Values unchanged, they are already a ratio between `0` and `1`. | +| `pve_ct_cpu_count` | `pve_ct_cpus` | Renamed only. | +| `pve_ct_disk_rd_bytes` | `pve_ct_disk_read_bytes_total` | Type changed to counter. | +| `pve_ct_disk_usage_bytes` | `pve_ct_disk_used_bytes` | Renamed only. | +| `pve_ct_disk_wr_bytes` | `pve_ct_disk_write_bytes_total` | Type changed to counter. | +| `pve_ct_mem_total_bytes` | `pve_ct_memory_total_bytes` | Renamed only. | +| `pve_ct_mem_used_bytes` | `pve_ct_memory_used_bytes` | Renamed only. | +| `pve_ct_network_in_bytes` | `pve_ct_network_receive_bytes_total` | Type changed to counter. | +| `pve_ct_network_out_bytes` | `pve_ct_network_transmit_bytes_total` | Type changed to counter. | +| `pve_ct_uptime` | `pve_ct_uptime_seconds` | Renamed only. | +| `pve_metrics_collection_latency_ms` | `pve_metrics_collection_duration_seconds` | Values converted from milliseconds to seconds. | +| `pve_node_cpuinfo` | `pve_node_cpu_info` | Renamed only. | +| `pve_node_cpu_usage` | `pve_node_cpu_usage_ratio` | Values unchanged, they are already a ratio between `0` and `1`. | +| `pve_node_cpu_count` | `pve_node_cpus` | Renamed only. | +| `pve_node_disk_wearout` | `pve_node_disk_wearout_percent` | Renamed only. | +| `pve_node_localtime` | `pve_node_localtime_seconds` | Renamed only. | +| `pve_node_subscription_nextduedate` | `pve_node_subscription_next_due_timestamp_seconds` | Renamed only. | +| `pve_node_subscription_regdate` | `pve_node_subscription_registration_timestamp_seconds` | Renamed only. | +| `pve_node_systeminfo` | `pve_node_system_info` | Renamed only. | +| `pve_node_time` | `pve_node_time_seconds` | Renamed only. | +| `pve_node_uptime` | `pve_node_uptime_seconds` | Renamed only. | +| `pve_node_zfs_checksum_errors` | `pve_node_zfs_checksum_errors_total` | Type changed to counter. | +| `pve_node_zfs_read_errors` | `pve_node_zfs_read_errors_total` | Type changed to counter. | +| `pve_node_zfs_write_errors` | `pve_node_zfs_write_errors_total` | Type changed to counter. | +| `pve_vm_cpu_usage` | `pve_vm_cpu_usage_ratio` | Values unchanged, they are already a ratio between `0` and `1`. | +| `pve_vm_cpu_count` | `pve_vm_cpus` | Renamed only. | +| `pve_vm_disk_failed_rd_ops` | `pve_vm_disk_failed_read_operations_total` | Type changed to counter. | +| `pve_vm_disk_failed_wr_ops` | `pve_vm_disk_failed_write_operations_total` | Type changed to counter. | +| `pve_vm_disk_rd_bytes` | `pve_vm_disk_read_bytes_total` | Type changed to counter. | +| `pve_vm_disk_rd_operations` | `pve_vm_disk_read_operations_total` | Type changed to counter. | +| `pve_vm_disk_rd_time_total_ns` | `pve_vm_disk_read_time_seconds_total` | Values converted from nanoseconds to seconds, type changed to counter. | +| `pve_vm_disk_usage_bytes` | `pve_vm_disk_used_bytes` | Renamed only. | +| `pve_vm_disk_wr_bytes` | `pve_vm_disk_write_bytes_total` | Type changed to counter. | +| `pve_vm_disk_wr_operations` | `pve_vm_disk_write_operations_total` | Type changed to counter. | +| `pve_vm_disk_wr_time_total_ns` | `pve_vm_disk_write_time_seconds_total` | Values converted from nanoseconds to seconds, type changed to counter. | +| `pve_vm_mem_total_bytes` | `pve_vm_memory_total_bytes` | Renamed only. | +| `pve_vm_mem_used_bytes` | `pve_vm_memory_used_bytes` | Renamed only. | +| `pve_vm_network_in_bytes` | `pve_vm_network_receive_bytes_total` | Type changed to counter. | +| `pve_vm_network_out_bytes` | `pve_vm_network_transmit_bytes_total` | Type changed to counter. | +| `pve_vm_uptime` | `pve_vm_uptime_seconds` | Renamed only. | + +Cumulative metrics are now exported as Prometheus counters instead of gauges, +which is what `rate()` and `increase()` expect and what removes the +"metric might not be a counter" hint in Grafana. diff --git a/examples/pve-exporter.rules.yml b/examples/pve-exporter.rules.yml index 16a473c..2912f1b 100644 --- a/examples/pve-exporter.rules.yml +++ b/examples/pve-exporter.rules.yml @@ -20,7 +20,7 @@ groups: - alert: PveGuestCpuSaturated expr: |- - pve_vm_cpu_usage > 0.95 or pve_ct_cpu_usage > 0.95 + pve_vm_cpu_usage_ratio > 0.95 or pve_ct_cpu_usage_ratio > 0.95 for: 1h labels: severity: warning @@ -74,7 +74,7 @@ groups: - alert: PveNodeCpuSaturated expr: |- - pve_node_cpu_usage > 0.95 + pve_node_cpu_usage_ratio > 0.95 for: 1h labels: severity: warning @@ -99,8 +99,8 @@ groups: - alert: PveClusterClockSkew expr: |- - max by (cluster) (pve_node_localtime) - - min by (cluster) (pve_node_localtime) > 15 + max by (cluster) (pve_node_localtime_seconds) + - min by (cluster) (pve_node_localtime_seconds) > 15 for: 1m labels: severity: critical @@ -113,9 +113,9 @@ groups: - alert: PveNodeSubscriptionExpiringSoon expr: |- ( - pve_node_subscription_nextduedate > time() + pve_node_subscription_next_due_timestamp_seconds > time() and - pve_node_subscription_nextduedate < time() + (30 * 24 * 60 * 60) + pve_node_subscription_next_due_timestamp_seconds < time() + (30 * 24 * 60 * 60) ) and on (cluster, node) (pve_node_subscription_status == 1) @@ -147,7 +147,7 @@ groups: - alert: PveNodeDiskWearout expr: |- - pve_node_disk_wearout < 5 + pve_node_disk_wearout_percent < 5 for: 1m labels: severity: critical @@ -242,7 +242,7 @@ groups: - alert: PveNodeZfsReadErrors expr: |- - pve_node_zfs_read_errors > 0 + pve_node_zfs_read_errors_total > 0 for: 5m labels: severity: warning @@ -255,7 +255,7 @@ groups: - alert: PveNodeZfsWriteErrors expr: |- - pve_node_zfs_write_errors > 0 + pve_node_zfs_write_errors_total > 0 for: 5m labels: severity: warning @@ -268,7 +268,7 @@ groups: - alert: PveNodeZfsChecksumErrors expr: |- - pve_node_zfs_checksum_errors > 0 + pve_node_zfs_checksum_errors_total > 0 for: 5m labels: severity: warning @@ -284,12 +284,12 @@ groups: ( sum by (cluster, node) ( ( - pve_vm_mem_total_bytes + pve_vm_memory_total_bytes and on (cluster, node, vmid, name) (pve_vm_state == 1) ) or ( - pve_ct_mem_total_bytes + pve_ct_memory_total_bytes and on (cluster, node, vmid, name) (pve_ct_state == 1) ) ) @@ -400,9 +400,9 @@ groups: - alert: PveNodeDiskWearoutWarning expr: |- - (pve_node_disk_wearout < 20) + (pve_node_disk_wearout_percent < 20) and - (pve_node_disk_wearout >= 5) + (pve_node_disk_wearout_percent >= 5) for: 15m labels: severity: warning @@ -417,9 +417,9 @@ groups: - alert: PveGuestMemoryHigh expr: |- - pve_vm_mem_used_bytes / pve_vm_mem_total_bytes > 0.90 + pve_vm_memory_used_bytes / pve_vm_memory_total_bytes > 0.90 or - pve_ct_mem_used_bytes / pve_ct_mem_total_bytes > 0.90 + pve_ct_memory_used_bytes / pve_ct_memory_total_bytes > 0.90 for: 15m labels: severity: warning @@ -433,9 +433,9 @@ groups: - alert: PveGuestDiskFillingUp expr: |- - pve_vm_disk_usage_bytes / pve_vm_disk_size_bytes > 0.90 + pve_vm_disk_used_bytes / pve_vm_disk_size_bytes > 0.90 or - pve_ct_disk_usage_bytes / pve_ct_disk_size_bytes > 0.90 + pve_ct_disk_used_bytes / pve_ct_disk_size_bytes > 0.90 for: 15m labels: severity: warning @@ -461,9 +461,9 @@ groups: - alert: PveVmDiskErrorsIncreasing expr: |- - delta(pve_vm_disk_failed_rd_ops[10m]) > 0 + increase(pve_vm_disk_failed_read_operations_total[10m]) > 0 or - delta(pve_vm_disk_failed_wr_ops[10m]) > 0 + increase(pve_vm_disk_failed_write_operations_total[10m]) > 0 for: 1m labels: severity: critical @@ -476,7 +476,7 @@ groups: - alert: PveNodeClockDrift expr: |- - abs(pve_node_time - time()) > 60 + abs(pve_node_time_seconds - time()) > 60 for: 5m labels: severity: warning @@ -489,7 +489,7 @@ groups: - alert: PveNodeRestarted expr: |- - resets(pve_node_uptime[15m]) > 0 + resets(pve_node_uptime_seconds[15m]) > 0 for: 0m labels: severity: info @@ -501,8 +501,8 @@ groups: - alert: PveCollectorSlow expr: |- - rate(pve_metrics_collection_latency_ms_sum[5m]) - / rate(pve_metrics_collection_latency_ms_count[5m]) > 5000 + rate(pve_metrics_collection_duration_seconds_sum[5m]) + / rate(pve_metrics_collection_duration_seconds_count[5m]) > 5 for: 10m labels: severity: warning @@ -510,7 +510,7 @@ groups: summary: PVE exporter collector is slow. description: >- Collector {{ $labels.collector }} has taken an average of - {{ printf "%.0f" $value }} ms per successful run for at least ten + {{ printf "%.1f" $value }} s per successful run for at least ten minutes. - alert: PveExporterDown diff --git a/examples/pve-guest-details.json b/examples/pve-guest-details.json index 4d29536..512fde4 100644 --- a/examples/pve-guest-details.json +++ b/examples/pve-guest-details.json @@ -43,7 +43,7 @@ "spec": { "editorMode": "code", "exemplar": false, - "expr": "pve_vm_uptime{cluster=\"$cluster\", name=\"$guest\"} OR pve_ct_uptime{cluster=\"$cluster\", name=\"$guest\"}", + "expr": "pve_vm_uptime_seconds{cluster=\"$cluster\", name=\"$guest\"} OR pve_ct_uptime_seconds{cluster=\"$cluster\", name=\"$guest\"}", "instant": true, "legendFormat": "__auto", "range": false @@ -133,7 +133,7 @@ "kind": "DataQuery", "spec": { "editorMode": "code", - "expr": "rate(pve_vm_network_out_bytes{cluster=\"$cluster\", name=\"$guest\"}[$__rate_interval]) or rate(pve_ct_network_out_bytes{cluster=\"$cluster\", name=\"$guest\"}[$__rate_interval])", + "expr": "rate(pve_vm_network_transmit_bytes_total{cluster=\"$cluster\", name=\"$guest\"}[$__rate_interval]) or rate(pve_ct_network_transmit_bytes_total{cluster=\"$cluster\", name=\"$guest\"}[$__rate_interval])", "instant": false, "legendFormat": "{{interface}}", "range": true @@ -262,7 +262,7 @@ "kind": "DataQuery", "spec": { "editorMode": "code", - "expr": "rate(pve_vm_disk_rd_bytes{cluster=\"$cluster\", name=\"$guest\"}[$__rate_interval]) OR \r\nrate(pve_ct_disk_rd_bytes{cluster=\"$cluster\", name=\"$guest\"}[$__rate_interval])", + "expr": "rate(pve_vm_disk_read_bytes_total{cluster=\"$cluster\", name=\"$guest\"}[$__rate_interval]) OR \r\nrate(pve_ct_disk_read_bytes_total{cluster=\"$cluster\", name=\"$guest\"}[$__rate_interval])", "instant": false, "legendFormat": "{{device}}", "range": true @@ -391,7 +391,7 @@ "kind": "DataQuery", "spec": { "editorMode": "code", - "expr": "rate(pve_vm_disk_wr_bytes{cluster=\"$cluster\", name=\"$guest\"}[$__rate_interval]) OR \r\nrate(pve_ct_disk_wr_bytes{cluster=\"$cluster\", name=\"$guest\"}[$__rate_interval])", + "expr": "rate(pve_vm_disk_write_bytes_total{cluster=\"$cluster\", name=\"$guest\"}[$__rate_interval]) OR \r\nrate(pve_ct_disk_write_bytes_total{cluster=\"$cluster\", name=\"$guest\"}[$__rate_interval])", "instant": false, "legendFormat": "{{device}}", "range": true @@ -520,7 +520,7 @@ "kind": "DataQuery", "spec": { "editorMode": "code", - "expr": "rate(pve_vm_disk_rd_operations{cluster=\"$cluster\", name=\"$guest\"}[$__rate_interval])", + "expr": "rate(pve_vm_disk_read_operations_total{cluster=\"$cluster\", name=\"$guest\"}[$__rate_interval])", "instant": false, "legendFormat": "{{device}}", "range": true @@ -649,7 +649,7 @@ "kind": "DataQuery", "spec": { "editorMode": "code", - "expr": "rate(pve_vm_disk_wr_operations{cluster=\"$cluster\", name=\"$guest\"}[$__rate_interval])", + "expr": "rate(pve_vm_disk_write_operations_total{cluster=\"$cluster\", name=\"$guest\"}[$__rate_interval])", "instant": false, "legendFormat": "{{device}}", "range": true @@ -778,7 +778,7 @@ "kind": "DataQuery", "spec": { "editorMode": "code", - "expr": "pve_vm_cpu_count{cluster=~\"$cluster\", name=\"$guest\"} OR\r\npve_ct_cpu_count{cluster=~\"$cluster\", name=\"$guest\"}", + "expr": "pve_vm_cpus{cluster=~\"$cluster\", name=\"$guest\"} OR\r\npve_ct_cpu_count{cluster=~\"$cluster\", name=\"$guest\"}", "instant": false, "legendFormat": "Total", "range": true @@ -801,7 +801,7 @@ "spec": { "editorMode": "code", "exemplar": false, - "expr": "(avg_over_time(pve_vm_cpu_usage{cluster=~\"$cluster\", name=\"$guest\"}[$__rate_interval]) * pve_vm_cpu_count{cluster=~\"$cluster\", name=\"$guest\"} and pve_vm_state{cluster=~\"$cluster\", name=\"$guest\"} == 1 or \r\navg_over_time(pve_ct_cpu_usage{cluster=~\"$cluster\", name=\"$guest\"}[$__rate_interval]) * pve_ct_cpu_count{cluster=~\"$cluster\", name=\"$guest\"} and pve_ct_state{cluster=~\"$cluster\", name=\"$guest\"} == 1) ", + "expr": "(avg_over_time(pve_vm_cpu_usage_ratio{cluster=~\"$cluster\", name=\"$guest\"}[$__rate_interval]) * pve_vm_cpus{cluster=~\"$cluster\", name=\"$guest\"} and pve_vm_state{cluster=~\"$cluster\", name=\"$guest\"} == 1 or \r\navg_over_time(pve_ct_cpu_usage_ratio{cluster=~\"$cluster\", name=\"$guest\"}[$__rate_interval]) * pve_ct_cpus{cluster=~\"$cluster\", name=\"$guest\"} and pve_ct_state{cluster=~\"$cluster\", name=\"$guest\"} == 1) ", "format": "time_series", "instant": false, "legendFormat": "Used", @@ -972,7 +972,7 @@ "kind": "DataQuery", "spec": { "editorMode": "code", - "expr": "pve_vm_mem_total_bytes{cluster=~\"$cluster\", name=\"$guest\"} OR\r\npve_ct_mem_total_bytes{cluster=~\"$cluster\", name=\"$guest\"}", + "expr": "pve_vm_memory_total_bytes{cluster=~\"$cluster\", name=\"$guest\"} OR\r\npve_ct_mem_total_bytes{cluster=~\"$cluster\", name=\"$guest\"}", "instant": false, "legendFormat": "Total", "range": true @@ -995,7 +995,7 @@ "spec": { "editorMode": "code", "exemplar": false, - "expr": "pve_vm_mem_used_bytes{cluster=~\"$cluster\", name=\"$guest\"} OR\r\npve_ct_mem_used_bytes{cluster=~\"$cluster\", name=\"$guest\"}", + "expr": "pve_vm_memory_used_bytes{cluster=~\"$cluster\", name=\"$guest\"} OR\r\npve_ct_mem_used_bytes{cluster=~\"$cluster\", name=\"$guest\"}", "format": "time_series", "instant": false, "legendFormat": "Used", @@ -1166,7 +1166,7 @@ "kind": "DataQuery", "spec": { "editorMode": "code", - "expr": "rate(pve_vm_disk_rd_time_total_ns{cluster=\"$cluster\", name=\"$guest\"}[$__rate_interval])", + "expr": "rate(pve_vm_disk_read_time_seconds_total{cluster=\"$cluster\", name=\"$guest\"}[$__rate_interval])", "instant": false, "legendFormat": "{{device}}", "range": true @@ -1243,7 +1243,7 @@ } ] }, - "unit": "ns" + "unit": "s" }, "overrides": [] }, @@ -1295,7 +1295,7 @@ "kind": "DataQuery", "spec": { "editorMode": "code", - "expr": "rate(pve_vm_disk_wr_time_total_ns{cluster=\"$cluster\", name=\"$guest\"}[$__rate_interval])", + "expr": "rate(pve_vm_disk_write_time_seconds_total{cluster=\"$cluster\", name=\"$guest\"}[$__rate_interval])", "instant": false, "legendFormat": "{{device}}", "range": true @@ -1372,7 +1372,7 @@ } ] }, - "unit": "ns" + "unit": "s" }, "overrides": [] }, @@ -1424,7 +1424,7 @@ "kind": "DataQuery", "spec": { "editorMode": "code", - "expr": "rate(pve_vm_disk_failed_rd_ops{cluster=\"$cluster\", name=\"$guest\"}[$__rate_interval])", + "expr": "rate(pve_vm_disk_failed_read_operations_total{cluster=\"$cluster\", name=\"$guest\"}[$__rate_interval])", "instant": false, "legendFormat": "{{device}}", "range": true @@ -1553,7 +1553,7 @@ "kind": "DataQuery", "spec": { "editorMode": "code", - "expr": "rate(pve_vm_disk_failed_wr_ops{cluster=\"$cluster\", name=\"$guest\"}[$__rate_interval])", + "expr": "rate(pve_vm_disk_failed_write_operations_total{cluster=\"$cluster\", name=\"$guest\"}[$__rate_interval])", "instant": false, "legendFormat": "{{device}}", "range": true @@ -1881,7 +1881,7 @@ "spec": { "editorMode": "code", "exemplar": false, - "expr": "pve_vm_cpu_usage{cluster=\"$cluster\", name=\"$guest\"} OR pve_ct_cpu_usage{cluster=\"$cluster\", name=\"$guest\"}", + "expr": "pve_vm_cpu_usage_ratio{cluster=\"$cluster\", name=\"$guest\"} OR pve_ct_cpu_usage_ratio{cluster=\"$cluster\", name=\"$guest\"}", "instant": true, "legendFormat": "__auto", "range": false @@ -1986,7 +1986,7 @@ "spec": { "editorMode": "code", "exemplar": false, - "expr": "pve_vm_mem_used_bytes{cluster=\"$cluster\", name=\"$guest\"} / pve_vm_mem_total_bytes{cluster=\"$cluster\", name=\"$guest\"} OR \r\npve_ct_mem_used_bytes{cluster=\"$cluster\", name=\"$guest\"} / pve_ct_mem_total_bytes{cluster=\"$cluster\", name=\"$guest\"}", + "expr": "pve_vm_memory_used_bytes{cluster=\"$cluster\", name=\"$guest\"} / pve_vm_memory_total_bytes{cluster=\"$cluster\", name=\"$guest\"} OR \r\npve_ct_mem_used_bytes{cluster=\"$cluster\", name=\"$guest\"} / pve_ct_memory_total_bytes{cluster=\"$cluster\", name=\"$guest\"}", "instant": true, "legendFormat": "__auto", "range": false @@ -2091,7 +2091,7 @@ "spec": { "editorMode": "code", "exemplar": false, - "expr": "pve_vm_mem_total_bytes{cluster=\"$cluster\", name=\"$guest\"} OR \r\npve_ct_mem_total_bytes{cluster=\"$cluster\", name=\"$guest\"}", + "expr": "pve_vm_memory_total_bytes{cluster=\"$cluster\", name=\"$guest\"} OR \r\npve_ct_mem_total_bytes{cluster=\"$cluster\", name=\"$guest\"}", "instant": true, "legendFormat": "__auto", "range": false @@ -2188,7 +2188,7 @@ "spec": { "editorMode": "code", "exemplar": false, - "expr": "pve_vm_mem_used_bytes{cluster=\"$cluster\", name=\"$guest\"} OR \r\npve_ct_mem_used_bytes{cluster=\"$cluster\", name=\"$guest\"}", + "expr": "pve_vm_memory_used_bytes{cluster=\"$cluster\", name=\"$guest\"} OR \r\npve_ct_mem_used_bytes{cluster=\"$cluster\", name=\"$guest\"}", "instant": true, "legendFormat": "__auto", "range": false @@ -2285,7 +2285,7 @@ "spec": { "editorMode": "code", "exemplar": false, - "expr": "pve_vm_cpu_count{cluster=\"$cluster\", name=\"$guest\"} OR \r\npve_ct_cpu_count{cluster=\"$cluster\", name=\"$guest\"}", + "expr": "pve_vm_cpus{cluster=\"$cluster\", name=\"$guest\"} OR \r\npve_ct_cpu_count{cluster=\"$cluster\", name=\"$guest\"}", "instant": true, "legendFormat": "__auto", "range": false @@ -2369,7 +2369,7 @@ "kind": "DataQuery", "spec": { "editorMode": "code", - "expr": "rate(pve_vm_network_in_bytes{cluster=\"$cluster\", name=\"$guest\"}[$__rate_interval]) or rate(pve_ct_network_in_bytes{cluster=\"$cluster\", name=\"$guest\"}[$__rate_interval])", + "expr": "rate(pve_vm_network_receive_bytes_total{cluster=\"$cluster\", name=\"$guest\"}[$__rate_interval]) or rate(pve_ct_network_receive_bytes_total{cluster=\"$cluster\", name=\"$guest\"}[$__rate_interval])", "instant": false, "legendFormat": "{{interface}}", "range": true diff --git a/examples/pve-node-details.json b/examples/pve-node-details.json index b858bfa..be994ee 100644 --- a/examples/pve-node-details.json +++ b/examples/pve-node-details.json @@ -43,7 +43,7 @@ "spec": { "editorMode": "code", "exemplar": false, - "expr": "clamp_max(sum(pve_node_load1{cluster=\"$cluster\", node=\"$node\"}) / sum(pve_node_cpu_count{cluster=\"$cluster\", node=\"$node\"} * 2), 1)", + "expr": "clamp_max(sum(pve_node_load1{cluster=\"$cluster\", node=\"$node\"}) / sum(pve_node_cpus{cluster=\"$cluster\", node=\"$node\"} * 2), 1)", "instant": true, "legendFormat": "__auto", "range": false @@ -350,7 +350,7 @@ "spec": { "editorMode": "code", "exemplar": false, - "expr": "sum(pve_node_uptime{cluster=\"$cluster\", node=\"$node\"})", + "expr": "sum(pve_node_uptime_seconds{cluster=\"$cluster\", node=\"$node\"})", "instant": true, "legendFormat": "__auto", "range": false @@ -443,7 +443,7 @@ "spec": { "editorMode": "code", "exemplar": false, - "expr": "sum(pve_node_time{cluster=\"$cluster\", node=\"$node\"}) * 1000", + "expr": "sum(pve_node_time_seconds{cluster=\"$cluster\", node=\"$node\"}) * 1000", "instant": true, "legendFormat": "__auto", "range": false @@ -612,7 +612,7 @@ "spec": { "editorMode": "code", "exemplar": false, - "expr": "topk(20, pve_vm_mem_used_bytes{cluster=~\"$cluster\", node=\"$node\"})", + "expr": "topk(20, pve_vm_memory_used_bytes{cluster=~\"$cluster\", node=\"$node\"})", "format": "table", "instant": true, "legendFormat": "__auto", @@ -796,7 +796,7 @@ "spec": { "editorMode": "code", "exemplar": false, - "expr": "topk(20, \r\n (avg_over_time(pve_vm_cpu_usage{cluster=~\"$cluster\", node=\"$node\"}[1d]) * pve_vm_cpu_count{cluster=~\"$cluster\", node=\"$node\"} and pve_vm_state{cluster=~\"$cluster\", node=\"$node\"} == 1 or \r\n avg_over_time(pve_ct_cpu_usage{cluster=~\"$cluster\", node=\"$node\"}[1d]) * pve_ct_cpu_count{cluster=~\"$cluster\", node=\"$node\"} and pve_ct_state{cluster=~\"$cluster\", node=\"$node\"} == 1) \r\n / on (node) group_left(cluster) pve_node_cpu_count{cluster=~\"$cluster\", node=\"$node\"})", + "expr": "topk(20, \r\n (avg_over_time(pve_vm_cpu_usage_ratio{cluster=~\"$cluster\", node=\"$node\"}[1d]) * pve_vm_cpus{cluster=~\"$cluster\", node=\"$node\"} and pve_vm_state{cluster=~\"$cluster\", node=\"$node\"} == 1 or \r\n avg_over_time(pve_ct_cpu_usage_ratio{cluster=~\"$cluster\", node=\"$node\"}[1d]) * pve_ct_cpus{cluster=~\"$cluster\", node=\"$node\"} and pve_ct_state{cluster=~\"$cluster\", node=\"$node\"} == 1) \r\n / on (node) group_left(cluster) pve_node_cpus{cluster=~\"$cluster\", node=\"$node\"})", "format": "table", "instant": true, "legendFormat": "__auto", @@ -1090,7 +1090,7 @@ "spec": { "editorMode": "code", "exemplar": false, - "expr": "(avg_over_time(pve_vm_cpu_usage{cluster=~\"$cluster\", node=\"$node\"}[$__rate_interval]) * pve_vm_cpu_count{cluster=~\"$cluster\", node=\"$node\"} and pve_vm_state{cluster=~\"$cluster\", node=\"$node\"} == 1 or \r\navg_over_time(pve_ct_cpu_usage{cluster=~\"$cluster\", node=\"$node\"}[$__rate_interval]) * pve_ct_cpu_count{cluster=~\"$cluster\", node=\"$node\"} and pve_ct_state{cluster=~\"$cluster\", node=\"$node\"} == 1) ", + "expr": "(avg_over_time(pve_vm_cpu_usage_ratio{cluster=~\"$cluster\", node=\"$node\"}[$__rate_interval]) * pve_vm_cpus{cluster=~\"$cluster\", node=\"$node\"} and pve_vm_state{cluster=~\"$cluster\", node=\"$node\"} == 1 or \r\navg_over_time(pve_ct_cpu_usage_ratio{cluster=~\"$cluster\", node=\"$node\"}[$__rate_interval]) * pve_ct_cpus{cluster=~\"$cluster\", node=\"$node\"} and pve_ct_state{cluster=~\"$cluster\", node=\"$node\"} == 1) ", "format": "time_series", "instant": false, "legendFormat": "{{name}}", @@ -1221,7 +1221,7 @@ "spec": { "editorMode": "code", "exemplar": false, - "expr": "pve_vm_mem_used_bytes{cluster=~\"$cluster\", node=\"$node\"} OR\r\npve_ct_mem_used_bytes{cluster=~\"$cluster\", node=\"$node\"}", + "expr": "pve_vm_memory_used_bytes{cluster=~\"$cluster\", node=\"$node\"} OR\r\npve_ct_mem_used_bytes{cluster=~\"$cluster\", node=\"$node\"}", "format": "time_series", "instant": false, "legendFormat": "{{name}}", @@ -1377,7 +1377,7 @@ "spec": { "editorMode": "code", "exemplar": false, - "expr": "pve_node_disk_wearout{cluster=\"$cluster\", node=\"$node\"}", + "expr": "pve_node_disk_wearout_percent{cluster=\"$cluster\", node=\"$node\"}", "format": "table", "instant": true, "legendFormat": "__auto", @@ -1734,7 +1734,7 @@ "kind": "DataQuery", "spec": { "editorMode": "code", - "expr": "sum by (name) (rate(pve_vm_network_in_bytes{cluster=\"$cluster\", node=\"$node\"}[$__rate_interval]) or rate(pve_ct_network_in_bytes{cluster=\"$cluster\", node=\"$node\"}[$__rate_interval]))", + "expr": "sum by (name) (rate(pve_vm_network_receive_bytes_total{cluster=\"$cluster\", node=\"$node\"}[$__rate_interval]) or rate(pve_ct_network_receive_bytes_total{cluster=\"$cluster\", node=\"$node\"}[$__rate_interval]))", "instant": false, "legendFormat": "{{name}}", "range": true @@ -1863,7 +1863,7 @@ "kind": "DataQuery", "spec": { "editorMode": "code", - "expr": "sum by (name) (rate(pve_vm_network_out_bytes{cluster=\"$cluster\", node=\"$node\"}[$__rate_interval]) or rate(pve_ct_network_out_bytes{cluster=\"$cluster\", node=\"$node\"}[$__rate_interval]))", + "expr": "sum by (name) (rate(pve_vm_network_transmit_bytes_total{cluster=\"$cluster\", node=\"$node\"}[$__rate_interval]) or rate(pve_ct_network_transmit_bytes_total{cluster=\"$cluster\", node=\"$node\"}[$__rate_interval]))", "instant": false, "legendFormat": "{{name}}", "range": true @@ -1993,7 +1993,7 @@ "spec": { "editorMode": "code", "exemplar": false, - "expr": "sum by (name) (pve_vm_network_in_bytes{cluster=\"$cluster\", node=\"$node\"}) or \r\nsum by (name) (pve_ct_network_in_bytes{cluster=\"$cluster\", node=\"$node\"})", + "expr": "sum by (name) (pve_vm_network_receive_bytes_total{cluster=\"$cluster\", node=\"$node\"}) or \r\nsum by (name) (pve_ct_network_receive_bytes_total{cluster=\"$cluster\", node=\"$node\"})", "format": "table", "instant": true, "legendFormat": "{{name}}", @@ -2017,7 +2017,7 @@ "spec": { "editorMode": "code", "exemplar": false, - "expr": "sum by (name) (pve_vm_network_out_bytes{cluster=\"$cluster\", node=\"$node\"}) or \r\nsum by (name) (pve_ct_network_out_bytes{cluster=\"$cluster\", node=\"$node\"})", + "expr": "sum by (name) (pve_vm_network_transmit_bytes_total{cluster=\"$cluster\", node=\"$node\"}) or \r\nsum by (name) (pve_ct_network_transmit_bytes_total{cluster=\"$cluster\", node=\"$node\"})", "format": "table", "instant": true, "legendFormat": "__auto", @@ -2041,7 +2041,7 @@ "spec": { "editorMode": "code", "exemplar": false, - "expr": "(sum by (name) (pve_vm_network_in_bytes{cluster=\"$cluster\", node=\"$node\"}) or \r\n sum by (name) (pve_ct_network_in_bytes{cluster=\"$cluster\", node=\"$node\"})) \r\n + \r\n(sum by (name) (pve_vm_network_out_bytes{cluster=\"$cluster\", node=\"$node\"}) or \r\n sum by (name) (pve_ct_network_out_bytes{cluster=\"$cluster\", node=\"$node\"}))", + "expr": "(sum by (name) (pve_vm_network_receive_bytes_total{cluster=\"$cluster\", node=\"$node\"}) or \r\n sum by (name) (pve_ct_network_receive_bytes_total{cluster=\"$cluster\", node=\"$node\"})) \r\n + \r\n(sum by (name) (pve_vm_network_transmit_bytes_total{cluster=\"$cluster\", node=\"$node\"}) or \r\n sum by (name) (pve_ct_network_transmit_bytes_total{cluster=\"$cluster\", node=\"$node\"}))", "format": "table", "instant": true, "legendFormat": "__auto", @@ -2065,7 +2065,7 @@ "spec": { "editorMode": "code", "exemplar": false, - "expr": "sum by (name) (delta(pve_vm_network_in_bytes{cluster=\"$cluster\", node=\"$node\"}[$__range]) + \r\n delta(pve_vm_network_out_bytes{cluster=\"$cluster\", node=\"$node\"}[$__range])) or\r\nsum by (name) (delta(pve_ct_network_in_bytes{cluster=\"$cluster\", node=\"$node\"}[$__range]) + \r\n delta(pve_ct_network_out_bytes{cluster=\"$cluster\", node=\"$node\"}[$__range]))", + "expr": "sum by (name) (delta(pve_vm_network_receive_bytes_total{cluster=\"$cluster\", node=\"$node\"}[$__range]) + \r\n delta(pve_vm_network_transmit_bytes_total{cluster=\"$cluster\", node=\"$node\"}[$__range])) or\r\nsum by (name) (delta(pve_ct_network_receive_bytes_total{cluster=\"$cluster\", node=\"$node\"}[$__range]) + \r\n delta(pve_ct_network_transmit_bytes_total{cluster=\"$cluster\", node=\"$node\"}[$__range]))", "format": "table", "instant": true, "legendFormat": "__auto", @@ -2312,7 +2312,7 @@ "kind": "DataQuery", "spec": { "editorMode": "code", - "expr": "sum by (name) (rate(pve_vm_disk_rd_bytes{cluster=\"$cluster\", node=\"$node\"}[$__rate_interval]) or rate(pve_ct_disk_rd_bytes{cluster=\"$cluster\", node=\"$node\"}[$__rate_interval]))", + "expr": "sum by (name) (rate(pve_vm_disk_read_bytes_total{cluster=\"$cluster\", node=\"$node\"}[$__rate_interval]) or rate(pve_ct_disk_read_bytes_total{cluster=\"$cluster\", node=\"$node\"}[$__rate_interval]))", "instant": false, "legendFormat": "{{name}}", "range": true @@ -2441,7 +2441,7 @@ "kind": "DataQuery", "spec": { "editorMode": "code", - "expr": "sum by (name) (rate(pve_vm_disk_wr_bytes{cluster=\"$cluster\", node=\"$node\"}[$__rate_interval]) or rate(pve_ct_disk_wr_bytes{cluster=\"$cluster\", node=\"$node\"}[$__rate_interval]))", + "expr": "sum by (name) (rate(pve_vm_disk_write_bytes_total{cluster=\"$cluster\", node=\"$node\"}[$__rate_interval]) or rate(pve_ct_disk_write_bytes_total{cluster=\"$cluster\", node=\"$node\"}[$__rate_interval]))", "instant": false, "legendFormat": "{{name}}", "range": true @@ -2571,7 +2571,7 @@ "spec": { "editorMode": "code", "exemplar": false, - "expr": "sum by (name) (pve_vm_disk_rd_bytes{cluster=\"$cluster\", node=\"$node\"}) or \r\nsum by (name) (pve_ct_disk_rd_bytes{cluster=\"$cluster\", node=\"$node\"})", + "expr": "sum by (name) (pve_vm_disk_read_bytes_total{cluster=\"$cluster\", node=\"$node\"}) or \r\nsum by (name) (pve_ct_disk_read_bytes_total{cluster=\"$cluster\", node=\"$node\"})", "format": "table", "instant": true, "legendFormat": "{{name}}", @@ -2595,7 +2595,7 @@ "spec": { "editorMode": "code", "exemplar": false, - "expr": "sum by (name) (pve_vm_disk_wr_bytes{cluster=\"$cluster\", node=\"$node\"}) or \r\nsum by (name) (pve_ct_disk_wr_bytes{cluster=\"$cluster\", node=\"$node\"})", + "expr": "sum by (name) (pve_vm_disk_write_bytes_total{cluster=\"$cluster\", node=\"$node\"}) or \r\nsum by (name) (pve_ct_disk_write_bytes_total{cluster=\"$cluster\", node=\"$node\"})", "format": "table", "instant": true, "legendFormat": "__auto", @@ -2619,7 +2619,7 @@ "spec": { "editorMode": "code", "exemplar": false, - "expr": "(sum by (name) (pve_vm_disk_rd_bytes{cluster=\"$cluster\", node=\"$node\"}) or \r\n sum by (name) (pve_ct_disk_rd_bytes{cluster=\"$cluster\", node=\"$node\"})) \r\n + \r\n(sum by (name) (pve_vm_disk_wr_bytes{cluster=\"$cluster\", node=\"$node\"}) or \r\n sum by (name) (pve_ct_disk_wr_bytes{cluster=\"$cluster\", node=\"$node\"}))", + "expr": "(sum by (name) (pve_vm_disk_read_bytes_total{cluster=\"$cluster\", node=\"$node\"}) or \r\n sum by (name) (pve_ct_disk_read_bytes_total{cluster=\"$cluster\", node=\"$node\"})) \r\n + \r\n(sum by (name) (pve_vm_disk_write_bytes_total{cluster=\"$cluster\", node=\"$node\"}) or \r\n sum by (name) (pve_ct_disk_write_bytes_total{cluster=\"$cluster\", node=\"$node\"}))", "format": "table", "instant": true, "legendFormat": "__auto", @@ -2643,7 +2643,7 @@ "spec": { "editorMode": "code", "exemplar": false, - "expr": "sum by (name) (delta(pve_vm_disk_rd_bytes{cluster=\"$cluster\", node=\"$node\"}[$__range]) + \r\n delta(pve_vm_disk_wr_bytes{cluster=\"$cluster\", node=\"$node\"}[$__range])) or\r\nsum by (name) (delta(pve_ct_disk_rd_bytes{cluster=\"$cluster\", node=\"$node\"}[$__range]) + \r\n delta(pve_ct_disk_wr_bytes{cluster=\"$cluster\", node=\"$node\"}[$__range]))", + "expr": "sum by (name) (delta(pve_vm_disk_read_bytes_total{cluster=\"$cluster\", node=\"$node\"}[$__range]) + \r\n delta(pve_vm_disk_write_bytes_total{cluster=\"$cluster\", node=\"$node\"}[$__range])) or\r\nsum by (name) (delta(pve_ct_disk_read_bytes_total{cluster=\"$cluster\", node=\"$node\"}[$__range]) + \r\n delta(pve_ct_disk_write_bytes_total{cluster=\"$cluster\", node=\"$node\"}[$__range]))", "format": "table", "instant": true, "legendFormat": "__auto", @@ -2793,7 +2793,7 @@ "kind": "DataQuery", "spec": { "editorMode": "code", - "expr": "sum by (name) (rate(pve_vm_disk_rd_operations{cluster=\"$cluster\", node=\"$node\"}[$__rate_interval]))", + "expr": "sum by (name) (rate(pve_vm_disk_read_operations_total{cluster=\"$cluster\", node=\"$node\"}[$__rate_interval]))", "instant": false, "legendFormat": "{{name}}", "range": true @@ -2922,7 +2922,7 @@ "kind": "DataQuery", "spec": { "editorMode": "code", - "expr": "sum by (name) (rate(pve_vm_disk_wr_operations{cluster=\"$cluster\", node=\"$node\"}[$__rate_interval]))", + "expr": "sum by (name) (rate(pve_vm_disk_write_operations_total{cluster=\"$cluster\", node=\"$node\"}[$__rate_interval]))", "instant": false, "legendFormat": "{{name}}", "range": true @@ -3240,7 +3240,7 @@ "sockets": "CPU Sockets", "storage": "Storage", "type": "Type", - "{__name__=\"pve_node_cpu_usage\", cluster=\"IMTM\", endpoint=\"9090\", instance=\"10.42.3.132:9090\", job=\"pve-exporter\", node=\"pve-hp1\", prometheus=\"monitoring/monitoring-prometheus\", service=\"pve-exporter\"}": "CPU Usage %" + "{__name__=\"pve_node_cpu_usage_ratio\", cluster=\"IMTM\", endpoint=\"9090\", instance=\"10.42.3.132:9090\", job=\"pve-exporter\", node=\"pve-hp1\", prometheus=\"monitoring/monitoring-prometheus\", service=\"pve-exporter\"}": "CPU Usage %" } } } @@ -3865,7 +3865,7 @@ "spec": { "editorMode": "code", "exemplar": false, - "expr": "pve_node_subscription_nextduedate{cluster=\"$cluster\", node=\"$node\"} * 1000", + "expr": "pve_node_subscription_next_due_timestamp_seconds{cluster=\"$cluster\", node=\"$node\"} * 1000", "format": "time_series", "instant": true, "legendFormat": "__auto", @@ -3951,7 +3951,7 @@ "spec": { "editorMode": "code", "exemplar": false, - "expr": "sum(pve_node_cpu_count{cluster=\"$cluster\", node=\"$node\"})", + "expr": "sum(pve_node_cpus{cluster=\"$cluster\", node=\"$node\"})", "instant": true, "legendFormat": "__auto", "range": false @@ -4036,7 +4036,7 @@ "spec": { "editorMode": "code", "exemplar": false, - "expr": "pve_node_subscription_regdate{cluster=\"$cluster\", node=\"$node\"} * 1000", + "expr": "pve_node_subscription_registration_timestamp_seconds{cluster=\"$cluster\", node=\"$node\"} * 1000", "format": "time_series", "instant": true, "legendFormat": "__auto", @@ -4122,7 +4122,7 @@ "spec": { "editorMode": "code", "exemplar": false, - "expr": "pve_node_systeminfo{cluster=\"$cluster\", node=\"$node\"}", + "expr": "pve_node_system_info{cluster=\"$cluster\", node=\"$node\"}", "format": "time_series", "instant": true, "legendFormat": "{{pveversion}}", @@ -4211,7 +4211,7 @@ "spec": { "editorMode": "code", "exemplar": false, - "expr": "pve_node_systeminfo{cluster=\"$cluster\", node=\"$node\"}", + "expr": "pve_node_system_info{cluster=\"$cluster\", node=\"$node\"}", "format": "time_series", "instant": true, "legendFormat": "{{kversion}}", @@ -4300,7 +4300,7 @@ "spec": { "editorMode": "code", "exemplar": false, - "expr": "pve_node_cpuinfo{cluster=\"$cluster\", node=\"$node\"}", + "expr": "pve_node_cpu_info{cluster=\"$cluster\", node=\"$node\"}", "format": "time_series", "instant": true, "legendFormat": "{{model}}", @@ -4389,7 +4389,7 @@ "spec": { "editorMode": "code", "exemplar": false, - "expr": "pve_node_cpuinfo{cluster=\"$cluster\", node=\"$node\"}", + "expr": "pve_node_cpu_info{cluster=\"$cluster\", node=\"$node\"}", "format": "time_series", "instant": true, "legendFormat": "{{cores}}", @@ -4562,7 +4562,7 @@ "spec": { "editorMode": "code", "exemplar": false, - "expr": "sum(rate(pve_vm_network_in_bytes{cluster=\"$cluster\", node=\"$node\"}[$__rate_interval]) or vector(0)) + sum(rate(pve_ct_network_in_bytes{cluster=\"$cluster\", node=\"$node\"}[$__rate_interval]) or vector(0))", + "expr": "sum(rate(pve_vm_network_receive_bytes_total{cluster=\"$cluster\", node=\"$node\"}[$__rate_interval]) or vector(0)) + sum(rate(pve_ct_network_receive_bytes_total{cluster=\"$cluster\", node=\"$node\"}[$__rate_interval]) or vector(0))", "instant": false, "legendFormat": "__auto", "range": true @@ -4667,7 +4667,7 @@ "spec": { "editorMode": "code", "exemplar": false, - "expr": "sum(rate(pve_vm_network_out_bytes{cluster=\"$cluster\", node=\"$node\"}[$__rate_interval]) or vector(0)) + sum(rate(pve_ct_network_out_bytes{cluster=\"$cluster\", node=\"$node\"}[$__rate_interval]) or vector(0))", + "expr": "sum(rate(pve_vm_network_transmit_bytes_total{cluster=\"$cluster\", node=\"$node\"}[$__rate_interval]) or vector(0)) + sum(rate(pve_ct_network_transmit_bytes_total{cluster=\"$cluster\", node=\"$node\"}[$__rate_interval]) or vector(0))", "instant": false, "legendFormat": "__auto", "range": true @@ -4784,7 +4784,7 @@ "spec": { "editorMode": "code", "exemplar": false, - "expr": "sum(rate(pve_vm_disk_rd_bytes{cluster=\"$cluster\", node=\"$node\"}[$__rate_interval]) or vector(0)) + sum(rate(pve_ct_disk_rd_bytes{cluster=\"$cluster\", node=\"$node\"}[$__rate_interval]) or vector(0))", + "expr": "sum(rate(pve_vm_disk_read_bytes_total{cluster=\"$cluster\", node=\"$node\"}[$__rate_interval]) or vector(0)) + sum(rate(pve_ct_disk_read_bytes_total{cluster=\"$cluster\", node=\"$node\"}[$__rate_interval]) or vector(0))", "instant": false, "legendFormat": "__auto", "range": true @@ -4889,7 +4889,7 @@ "spec": { "editorMode": "code", "exemplar": false, - "expr": "sum(rate(pve_vm_disk_wr_bytes{cluster=\"$cluster\", node=\"$node\"}[$__rate_interval]) or vector(0)) + sum(rate(pve_ct_disk_wr_bytes{cluster=\"$cluster\", node=\"$node\"}[$__rate_interval]) or vector(0))", + "expr": "sum(rate(pve_vm_disk_write_bytes_total{cluster=\"$cluster\", node=\"$node\"}[$__rate_interval]) or vector(0)) + sum(rate(pve_ct_disk_write_bytes_total{cluster=\"$cluster\", node=\"$node\"}[$__rate_interval]) or vector(0))", "instant": false, "legendFormat": "__auto", "range": true diff --git a/examples/pve-overview.json b/examples/pve-overview.json index 55169f4..7fb5a96 100644 --- a/examples/pve-overview.json +++ b/examples/pve-overview.json @@ -266,7 +266,7 @@ "spec": { "editorMode": "code", "exemplar": false, - "expr": "sum(pve_node_cpu_count{cluster=\"$cluster\"})", + "expr": "sum(pve_node_cpus{cluster=\"$cluster\"})", "instant": true, "legendFormat": "__auto", "range": false @@ -456,7 +456,7 @@ "spec": { "editorMode": "code", "exemplar": false, - "expr": "sum(pve_node_load1{cluster=\"$cluster\"}) / (sum(pve_node_cpu_count{cluster=\"$cluster\"}) * 2)", + "expr": "sum(pve_node_load1{cluster=\"$cluster\"}) / (sum(pve_node_cpus{cluster=\"$cluster\"}) * 2)", "instant": true, "legendFormat": "__auto", "range": false @@ -654,7 +654,7 @@ "spec": { "editorMode": "code", "exemplar": false, - "expr": "pve_node_cpuinfo{cluster=~\"$cluster\"}", + "expr": "pve_node_cpu_info{cluster=~\"$cluster\"}", "format": "table", "instant": true, "legendFormat": "__auto", @@ -678,7 +678,7 @@ "spec": { "editorMode": "code", "exemplar": false, - "expr": "clamp_max(pve_node_load5{cluster=~\"$cluster\"} / pve_node_cpu_count{cluster=~\"$cluster\"} / 2, 1)", + "expr": "clamp_max(pve_node_load5{cluster=~\"$cluster\"} / pve_node_cpus{cluster=~\"$cluster\"} / 2, 1)", "format": "table", "instant": false, "legendFormat": "__auto", @@ -750,7 +750,7 @@ "spec": { "editorMode": "code", "exemplar": false, - "expr": "sum by (cluster,node) ((pve_ct_mem_total_bytes and pve_ct_state == 1) or (pve_vm_mem_total_bytes and pve_vm_state == 1)) / sum by (cluster,node) (pve_node_memory_total_bytes)", + "expr": "sum by (cluster,node) ((pve_ct_memory_total_bytes and pve_ct_state == 1) or (pve_vm_memory_total_bytes and pve_vm_state == 1)) / sum by (cluster,node) (pve_node_memory_total_bytes)", "format": "table", "instant": true, "legendFormat": "__auto", @@ -888,7 +888,7 @@ "node": "Node", "pveversion": "PVE Version", "sockets": "CPU Sockets", - "{__name__=\"pve_node_cpu_usage\", cluster=\"IMTM\", endpoint=\"9090\", instance=\"10.42.3.132:9090\", job=\"pve-exporter\", node=\"pve-hp1\", prometheus=\"monitoring/monitoring-prometheus\", service=\"pve-exporter\"}": "CPU Usage %" + "{__name__=\"pve_node_cpu_usage_ratio\", cluster=\"IMTM\", endpoint=\"9090\", instance=\"10.42.3.132:9090\", job=\"pve-exporter\", node=\"pve-hp1\", prometheus=\"monitoring/monitoring-prometheus\", service=\"pve-exporter\"}": "CPU Usage %" } } } @@ -1205,7 +1205,7 @@ "spec": { "editorMode": "code", "exemplar": false, - "expr": "sum(rate(pve_vm_network_in_bytes{cluster=\"$cluster\"}[$__rate_interval]) or vector(0)) + sum(rate(pve_ct_network_in_bytes{cluster=\"$cluster\"}[$__rate_interval]) or vector(0))", + "expr": "sum(rate(pve_vm_network_receive_bytes_total{cluster=\"$cluster\"}[$__rate_interval]) or vector(0)) + sum(rate(pve_ct_network_receive_bytes_total{cluster=\"$cluster\"}[$__rate_interval]) or vector(0))", "instant": false, "legendFormat": "__auto", "range": true @@ -1310,7 +1310,7 @@ "spec": { "editorMode": "code", "exemplar": false, - "expr": "sum(rate(pve_vm_network_out_bytes{cluster=\"$cluster\"}[$__rate_interval]) or vector(0)) + sum(rate(pve_ct_network_out_bytes{cluster=\"$cluster\"}[$__rate_interval]) or vector(0))", + "expr": "sum(rate(pve_vm_network_transmit_bytes_total{cluster=\"$cluster\"}[$__rate_interval]) or vector(0)) + sum(rate(pve_ct_network_transmit_bytes_total{cluster=\"$cluster\"}[$__rate_interval]) or vector(0))", "instant": false, "legendFormat": "__auto", "range": true @@ -1427,7 +1427,7 @@ "spec": { "editorMode": "code", "exemplar": false, - "expr": "sum(rate(pve_vm_disk_rd_bytes{cluster=\"$cluster\"}[$__rate_interval]) or vector(0)) + sum(rate(pve_ct_disk_rd_bytes{cluster=\"$cluster\"}[$__rate_interval]) or vector(0))", + "expr": "sum(rate(pve_vm_disk_read_bytes_total{cluster=\"$cluster\"}[$__rate_interval]) or vector(0)) + sum(rate(pve_ct_disk_read_bytes_total{cluster=\"$cluster\"}[$__rate_interval]) or vector(0))", "instant": false, "legendFormat": "__auto", "range": true @@ -1532,7 +1532,7 @@ "spec": { "editorMode": "code", "exemplar": false, - "expr": "sum(rate(pve_vm_disk_wr_bytes{cluster=\"$cluster\"}[$__rate_interval]) or vector(0)) + sum(rate(pve_ct_disk_wr_bytes{cluster=\"$cluster\"}[$__rate_interval]) or vector(0))", + "expr": "sum(rate(pve_vm_disk_write_bytes_total{cluster=\"$cluster\"}[$__rate_interval]) or vector(0)) + sum(rate(pve_ct_disk_write_bytes_total{cluster=\"$cluster\"}[$__rate_interval]) or vector(0))", "instant": false, "legendFormat": "__auto", "range": true @@ -1794,7 +1794,7 @@ "kind": "DataQuery", "spec": { "editorMode": "code", - "expr": "pve_node_load5{cluster=~\"$cluster\"} / pve_node_cpu_count{cluster=~\"$cluster\"}", + "expr": "pve_node_load5{cluster=~\"$cluster\"} / pve_node_cpus{cluster=~\"$cluster\"}", "instant": false, "legendFormat": "{{node}}", "range": true @@ -2052,7 +2052,7 @@ "kind": "DataQuery", "spec": { "editorMode": "code", - "expr": "sum(rate(pve_vm_network_in_bytes{cluster=\"$cluster\"}[$__rate_interval]) or vector(0)) + sum(rate(pve_ct_network_in_bytes{cluster=\"$cluster\"}[$__rate_interval]) or vector(0))", + "expr": "sum(rate(pve_vm_network_receive_bytes_total{cluster=\"$cluster\"}[$__rate_interval]) or vector(0)) + sum(rate(pve_ct_network_receive_bytes_total{cluster=\"$cluster\"}[$__rate_interval]) or vector(0))", "instant": false, "legendFormat": "Receive", "range": true @@ -2074,7 +2074,7 @@ "kind": "DataQuery", "spec": { "editorMode": "code", - "expr": "sum(rate(pve_vm_network_out_bytes{cluster=\"$cluster\"}[$__rate_interval]) or vector(0)) + sum(rate(pve_ct_network_out_bytes{cluster=\"$cluster\"}[$__rate_interval]) or vector(0))", + "expr": "sum(rate(pve_vm_network_transmit_bytes_total{cluster=\"$cluster\"}[$__rate_interval]) or vector(0)) + sum(rate(pve_ct_network_transmit_bytes_total{cluster=\"$cluster\"}[$__rate_interval]) or vector(0))", "instant": false, "legendFormat": "Transmit", "range": true @@ -2216,7 +2216,7 @@ "kind": "DataQuery", "spec": { "editorMode": "code", - "expr": "sum(rate(pve_vm_disk_rd_bytes{cluster=\"$cluster\"}[$__rate_interval]) or vector(0)) + sum(rate(pve_ct_disk_rd_bytes{cluster=\"$cluster\"}[$__rate_interval]) or vector(0))", + "expr": "sum(rate(pve_vm_disk_read_bytes_total{cluster=\"$cluster\"}[$__rate_interval]) or vector(0)) + sum(rate(pve_ct_disk_read_bytes_total{cluster=\"$cluster\"}[$__rate_interval]) or vector(0))", "instant": false, "legendFormat": "Read", "range": true @@ -2238,7 +2238,7 @@ "kind": "DataQuery", "spec": { "editorMode": "code", - "expr": "sum(rate(pve_vm_disk_wr_bytes{cluster=\"$cluster\"}[$__rate_interval]) or vector(0)) + sum(rate(pve_ct_disk_wr_bytes{cluster=\"$cluster\"}[$__rate_interval]) or vector(0))", + "expr": "sum(rate(pve_vm_disk_write_bytes_total{cluster=\"$cluster\"}[$__rate_interval]) or vector(0)) + sum(rate(pve_ct_disk_write_bytes_total{cluster=\"$cluster\"}[$__rate_interval]) or vector(0))", "instant": false, "legendFormat": "Write", "range": true @@ -2381,7 +2381,7 @@ "spec": { "editorMode": "code", "exemplar": false, - "expr": "topk(20, pve_vm_mem_used_bytes{cluster=~\"$cluster\"})", + "expr": "topk(20, pve_vm_memory_used_bytes{cluster=~\"$cluster\"})", "format": "table", "instant": true, "legendFormat": "__auto", @@ -2582,7 +2582,7 @@ "spec": { "editorMode": "code", "exemplar": false, - "expr": "topk(20, \r\n (avg_over_time(pve_vm_cpu_usage{cluster=~\"$cluster\"}[1d]) * pve_vm_cpu_count{cluster=~\"$cluster\"} and pve_vm_state{cluster=~\"$cluster\"} == 1 or \r\n avg_over_time(pve_ct_cpu_usage{cluster=~\"$cluster\"}[1d]) * pve_ct_cpu_count{cluster=~\"$cluster\"} and pve_ct_state{cluster=~\"$cluster\"} == 1) \r\n / on (node) group_left(cluster) pve_node_cpu_count{cluster=~\"$cluster\"})", + "expr": "topk(20, \r\n (avg_over_time(pve_vm_cpu_usage_ratio{cluster=~\"$cluster\"}[1d]) * pve_vm_cpus{cluster=~\"$cluster\"} and pve_vm_state{cluster=~\"$cluster\"} == 1 or \r\n avg_over_time(pve_ct_cpu_usage_ratio{cluster=~\"$cluster\"}[1d]) * pve_ct_cpus{cluster=~\"$cluster\"} and pve_ct_state{cluster=~\"$cluster\"} == 1) \r\n / on (node) group_left(cluster) pve_node_cpus{cluster=~\"$cluster\"})", "format": "table", "instant": true, "legendFormat": "__auto", @@ -2972,7 +2972,7 @@ "node": "Node", "pveversion": "PVE Version", "sockets": "CPU Sockets", - "{__name__=\"pve_node_cpu_usage\", cluster=\"IMTM\", endpoint=\"9090\", instance=\"10.42.3.132:9090\", job=\"pve-exporter\", node=\"pve-hp1\", prometheus=\"monitoring/monitoring-prometheus\", service=\"pve-exporter\"}": "CPU Usage %" + "{__name__=\"pve_node_cpu_usage_ratio\", cluster=\"IMTM\", endpoint=\"9090\", instance=\"10.42.3.132:9090\", job=\"pve-exporter\", node=\"pve-hp1\", prometheus=\"monitoring/monitoring-prometheus\", service=\"pve-exporter\"}": "CPU Usage %" } } } @@ -3389,7 +3389,7 @@ "sockets": "CPU Sockets", "storage": "Storage", "type": "Type", - "{__name__=\"pve_node_cpu_usage\", cluster=\"IMTM\", endpoint=\"9090\", instance=\"10.42.3.132:9090\", job=\"pve-exporter\", node=\"pve-hp1\", prometheus=\"monitoring/monitoring-prometheus\", service=\"pve-exporter\"}": "CPU Usage %" + "{__name__=\"pve_node_cpu_usage_ratio\", cluster=\"IMTM\", endpoint=\"9090\", instance=\"10.42.3.132:9090\", job=\"pve-exporter\", node=\"pve-hp1\", prometheus=\"monitoring/monitoring-prometheus\", service=\"pve-exporter\"}": "CPU Usage %" } } } diff --git a/metrics/pve_metrics_manager.go b/metrics/pve_metrics_manager.go index 5bc89fa..e5e410f 100644 --- a/metrics/pve_metrics_manager.go +++ b/metrics/pve_metrics_manager.go @@ -23,8 +23,8 @@ type PveMetricsManager struct { collectors []PveMetricsCollector // Metrics collector instances. registry *TTLRegistry // Registry which handles autoamtic dangling metrics deletion. - latencySummary *prometheus.SummaryVec // Collection latency summary. - interval int // Collection interval. + durationSummary *prometheus.SummaryVec // Collection duration summary. + interval int // Collection interval. stop chan struct{} // Stop channel which is used in ticker. } @@ -80,10 +80,10 @@ func NewPveMetricsManager(apiClient *proxmox.PveApiClient, conf *configuration.P c.RegisterCollector(NewPveVirtualMachineCollector(apiClient, c.registry)) } - // Metrics collection latency summary. - c.latencySummary = promauto.NewSummaryVec(prometheus.SummaryOpts{ - Name: "pve_metrics_collection_latency_ms", - Help: "Summary of metrics collection latency milliseconds from PVE API.", + // Metrics collection duration summary. + c.durationSummary = promauto.NewSummaryVec(prometheus.SummaryOpts{ + Name: "pve_metrics_collection_duration_seconds", + Help: "Summary of the PVE API metrics collection duration in seconds.", }, []string{"collector"}) c.registry.StartCleanupLoop(5 * time.Second) @@ -102,7 +102,7 @@ func (c *PveMetricsManager) collectMetrics() { } else { latency := time.Since(start) log.Tracef("Finished collecting '%s' metrics after %s.", collector.GetName(), utils.HumanDuration(latency)) - c.latencySummary.With(prometheus.Labels{"collector": collector.GetName()}).Observe(float64(latency.Milliseconds())) + c.durationSummary.With(prometheus.Labels{"collector": collector.GetName()}).Observe(latency.Seconds()) } } } diff --git a/metrics/pve_metrics_naming_test.go b/metrics/pve_metrics_naming_test.go new file mode 100644 index 0000000..9c4aca3 --- /dev/null +++ b/metrics/pve_metrics_naming_test.go @@ -0,0 +1,72 @@ +package metrics + +import ( + "testing" + + "github.com/prometheus/client_golang/prometheus" + "github.com/prometheus/client_golang/prometheus/testutil/promlint" +) + +// Verifies that all exported PVE metrics follow the Prometheus naming conventions +// (base units, '_total' suffix on counters, '_info' suffix on info metrics, ...) +// so that tools such as promtool or Grafana do not report naming problems. +func TestPveMetricNamesFollowPrometheusConventions(t *testing.T) { + registry := prometheus.NewRegistry() + + // Collectors register their metrics on the default registerer, so it is + // temporarily replaced by a dedicated registry. + defaultRegisterer := prometheus.DefaultRegisterer + prometheus.DefaultRegisterer = registry + defer func() { prometheus.DefaultRegisterer = defaultRegisterer }() + + // Collector constructors only create metrics, the API client is used during + // collection only, so a nil client is enough here. + ttlRegistry := NewTTLRegistry() + NewPveClusterStateCollector(nil, ttlRegistry) + NewPveNodeStatusCollector(nil, ttlRegistry) + NewPveSubscriptionCollector(nil, ttlRegistry) + NewPveNodeDiskCollector(nil, ttlRegistry) + NewPveNodeZfsCollector(nil, ttlRegistry) + NewPveSdnCollector(nil, ttlRegistry) + NewPveStorageCollector(nil, ttlRegistry) + NewPveContainerCollector(nil, ttlRegistry) + NewPveVirtualMachineCollector(nil, ttlRegistry) + + // Metric vectors are exported only once they hold a label set. + for _, metric := range ttlRegistry.metrics { + switch m := metric.(type) { + case *TTLGaugeVec: + m.With(emptyLabels(m.labelNames)).Set(0) + case *TTLCounterVec: + m.With(emptyLabels(m.labelNames)).Set(0) + default: + t.Fatalf("Unknown TTL metric type %T.", metric) + } + } + + families, err := registry.Gather() + if err != nil { + t.Fatalf("Unable to gather metrics. Error: %s.", err) + } + if len(families) != len(ttlRegistry.metrics) { + t.Fatalf("Gathered %d metric families but %d metrics are registered.", len(families), len(ttlRegistry.metrics)) + } + + problems, err := promlint.NewWithMetricFamilies(families).Lint() + if err != nil { + t.Fatalf("Unable to lint metrics. Error: %s.", err) + } + + for _, problem := range problems { + t.Errorf("Metric '%s' violates the Prometheus naming conventions: %s.", problem.Metric, problem.Text) + } +} + +// emptyLabels builds a label set with all given label names set to an empty value. +func emptyLabels(labelNames []string) prometheus.Labels { + labels := prometheus.Labels{} + for _, name := range labelNames { + labels[name] = "" + } + return labels +} diff --git a/metrics/pve_node_container_collector.go b/metrics/pve_node_container_collector.go index e0fcce8..9218dad 100644 --- a/metrics/pve_node_container_collector.go +++ b/metrics/pve_node_container_collector.go @@ -21,11 +21,11 @@ type PveContainerCollector struct { memBytes *TTLGaugeVec // Container memory in bytes prometheus gauge. memBytesUsed *TTLGaugeVec // Container memory usage in bytes prometheus gauge. - netReceive *TTLGaugeVec // Container network RX in bytes prometheus gauge. - netTransmit *TTLGaugeVec // Container network TX in bytes prometheus gauge. + netReceive *TTLCounterVec // Container received network traffic in bytes prometheus counter. + netTransmit *TTLCounterVec // Container transmitted network traffic in bytes prometheus counter. - diskWrite *TTLGaugeVec // Container disk written in bytes prometheus gauge. - diskRead *TTLGaugeVec // Container disk read in bytes prometheus gauge. + diskWrite *TTLCounterVec // Container disk written in bytes prometheus counter. + diskRead *TTLCounterVec // Container disk read in bytes prometheus counter. disk *TTLGaugeVec // Container disk space usage in bytes prometheus gauge. diskMax *TTLGaugeVec // Container disk size in bytes prometheus gauge. @@ -51,8 +51,8 @@ func NewPveContainerCollector(apiClient *proxmox.PveApiClient, registry *TTLRegi // Container uptime. c.uptime = NewTTLGaugeVec( prometheus.GaugeOpts{ - Name: "pve_ct_uptime", - Help: "Container uptime.", + Name: "pve_ct_uptime_seconds", + Help: "Container uptime in seconds.", }, []string{"cluster", "node", "vmid", "name"}, 5*time.Minute, @@ -62,7 +62,7 @@ func NewPveContainerCollector(apiClient *proxmox.PveApiClient, registry *TTLRegi // Container CPU count. c.cpu = NewTTLGaugeVec( prometheus.GaugeOpts{ - Name: "pve_ct_cpu_count", + Name: "pve_ct_cpus", Help: "Container CPU count.", }, []string{"cluster", "node", "vmid", "name"}, @@ -73,8 +73,8 @@ func NewPveContainerCollector(apiClient *proxmox.PveApiClient, registry *TTLRegi // Container CPU usage. c.cpuUsage = NewTTLGaugeVec( prometheus.GaugeOpts{ - Name: "pve_ct_cpu_usage", - Help: "Container CPU usage.", + Name: "pve_ct_cpu_usage_ratio", + Help: "Container CPU usage ratio (0-1).", }, []string{"cluster", "node", "vmid", "name"}, 5*time.Minute, @@ -84,7 +84,7 @@ func NewPveContainerCollector(apiClient *proxmox.PveApiClient, registry *TTLRegi // Container memory total. c.memBytes = NewTTLGaugeVec( prometheus.GaugeOpts{ - Name: "pve_ct_mem_total_bytes", + Name: "pve_ct_memory_total_bytes", Help: "Container total memory in bytes.", }, []string{"cluster", "node", "vmid", "name"}, @@ -95,7 +95,7 @@ func NewPveContainerCollector(apiClient *proxmox.PveApiClient, registry *TTLRegi // Container memory usage. c.memBytesUsed = NewTTLGaugeVec( prometheus.GaugeOpts{ - Name: "pve_ct_mem_used_bytes", + Name: "pve_ct_memory_used_bytes", Help: "Container used memory in bytes.", }, []string{"cluster", "node", "vmid", "name"}, @@ -104,10 +104,10 @@ func NewPveContainerCollector(apiClient *proxmox.PveApiClient, registry *TTLRegi c.registry.Register(c.memBytesUsed) // Container network RX. - c.netReceive = NewTTLGaugeVec( - prometheus.GaugeOpts{ - Name: "pve_ct_network_in_bytes", - Help: "Container network RX bytes.", + c.netReceive = NewTTLCounterVec( + prometheus.CounterOpts{ + Name: "pve_ct_network_receive_bytes_total", + Help: "Container received network traffic in bytes.", }, []string{"cluster", "node", "vmid", "name"}, 5*time.Minute, @@ -115,10 +115,10 @@ func NewPveContainerCollector(apiClient *proxmox.PveApiClient, registry *TTLRegi c.registry.Register(c.netReceive) // Container network TX. - c.netTransmit = NewTTLGaugeVec( - prometheus.GaugeOpts{ - Name: "pve_ct_network_out_bytes", - Help: "Container network TX bytes.", + c.netTransmit = NewTTLCounterVec( + prometheus.CounterOpts{ + Name: "pve_ct_network_transmit_bytes_total", + Help: "Container transmitted network traffic in bytes.", }, []string{"cluster", "node", "vmid", "name"}, 5*time.Minute, @@ -126,10 +126,10 @@ func NewPveContainerCollector(apiClient *proxmox.PveApiClient, registry *TTLRegi c.registry.Register(c.netTransmit) // Container disk written. - c.diskWrite = NewTTLGaugeVec( - prometheus.GaugeOpts{ - Name: "pve_ct_disk_wr_bytes", - Help: "Container disk written bytes.", + c.diskWrite = NewTTLCounterVec( + prometheus.CounterOpts{ + Name: "pve_ct_disk_write_bytes_total", + Help: "Container written bytes to disk.", }, []string{"cluster", "node", "vmid", "name"}, 5*time.Minute, @@ -137,10 +137,10 @@ func NewPveContainerCollector(apiClient *proxmox.PveApiClient, registry *TTLRegi c.registry.Register(c.diskWrite) // Container disk read. - c.diskRead = NewTTLGaugeVec( - prometheus.GaugeOpts{ - Name: "pve_ct_disk_rd_bytes", - Help: "Container disk read bytes.", + c.diskRead = NewTTLCounterVec( + prometheus.CounterOpts{ + Name: "pve_ct_disk_read_bytes_total", + Help: "Container read bytes from disk.", }, []string{"cluster", "node", "vmid", "name"}, 5*time.Minute, @@ -150,8 +150,8 @@ func NewPveContainerCollector(apiClient *proxmox.PveApiClient, registry *TTLRegi // Container disk size. c.disk = NewTTLGaugeVec( prometheus.GaugeOpts{ - Name: "pve_ct_disk_usage_bytes", - Help: "Container disk read bytes.", + Name: "pve_ct_disk_used_bytes", + Help: "Container used disk space in bytes.", }, []string{"cluster", "node", "vmid", "name"}, 5*time.Minute, diff --git a/metrics/pve_node_disk_collector.go b/metrics/pve_node_disk_collector.go index 9011d2f..5cd6d9e 100644 --- a/metrics/pve_node_disk_collector.go +++ b/metrics/pve_node_disk_collector.go @@ -38,13 +38,13 @@ func NewPveNodeDiskCollector(apiClient *proxmox.PveApiClient, registry *TTLRegis // Node disk wearout. c.wearout = NewTTLGaugeVec( prometheus.GaugeOpts{ - Name: "pve_node_disk_wearout", - Help: "Node disk wearout percent.", + Name: "pve_node_disk_wearout_percent", + Help: "Node disk wearout in percent (0-100).", }, []string{"cluster", "node", "wwn", "type", "model", "serial", "vendor", "used", "osd_id"}, 5*time.Minute, ) - c.registry.Register(c.healthy) + c.registry.Register(c.wearout) // Node disk size in bytes. c.sizeBytes = NewTTLGaugeVec( diff --git a/metrics/pve_node_status_collector.go b/metrics/pve_node_status_collector.go index 11b77e2..9ca7eb1 100644 --- a/metrics/pve_node_status_collector.go +++ b/metrics/pve_node_status_collector.go @@ -55,8 +55,8 @@ func NewPveNodeStatusCollector(apiClient *proxmox.PveApiClient, registry *TTLReg // Node uptime. c.uptime = NewTTLGaugeVec( prometheus.GaugeOpts{ - Name: "pve_node_uptime", - Help: "Node uptime.", + Name: "pve_node_uptime_seconds", + Help: "Node uptime in seconds.", }, []string{"cluster", "node"}, 5*time.Minute, @@ -66,7 +66,7 @@ func NewPveNodeStatusCollector(apiClient *proxmox.PveApiClient, registry *TTLReg // Node cpu count. c.cpus = NewTTLGaugeVec( prometheus.GaugeOpts{ - Name: "pve_node_cpu_count", + Name: "pve_node_cpus", Help: "Node CPU count.", }, []string{"cluster", "node"}, @@ -77,8 +77,8 @@ func NewPveNodeStatusCollector(apiClient *proxmox.PveApiClient, registry *TTLReg // Node CPU usage. c.cpuUsage = NewTTLGaugeVec( prometheus.GaugeOpts{ - Name: "pve_node_cpu_usage", - Help: "Cluster node CPU usage %.", + Name: "pve_node_cpu_usage_ratio", + Help: "Node CPU usage ratio (0-1).", }, []string{"cluster", "node"}, 5*time.Minute, @@ -220,8 +220,8 @@ func NewPveNodeStatusCollector(apiClient *proxmox.PveApiClient, registry *TTLReg // Node CPU info. c.cpuInfo = NewTTLGaugeVec( prometheus.GaugeOpts{ - Name: "pve_node_cpuinfo", - Help: "Node CPU info.", + Name: "pve_node_cpu_info", + Help: "Node CPU information.", }, []string{"cluster", "node", "flags", "cores", "model", "sockets", "cpus", "hvm"}, 5*time.Minute, @@ -231,8 +231,8 @@ func NewPveNodeStatusCollector(apiClient *proxmox.PveApiClient, registry *TTLReg // Node system info metrics. c.systemInfo = NewTTLGaugeVec( prometheus.GaugeOpts{ - Name: "pve_node_systeminfo", - Help: "Node system info.", + Name: "pve_node_system_info", + Help: "Node system information.", }, []string{"cluster", "node", "kversion", "pveversion", "machine", "sysname", "release"}, 5*time.Minute, @@ -242,8 +242,8 @@ func NewPveNodeStatusCollector(apiClient *proxmox.PveApiClient, registry *TTLReg // Node time info. c.time = NewTTLGaugeVec( prometheus.GaugeOpts{ - Name: "pve_node_time", - Help: "Node time.", + Name: "pve_node_time_seconds", + Help: "Node UTC time as a unix timestamp in seconds.", }, []string{"cluster", "node"}, 5*time.Minute, @@ -253,8 +253,8 @@ func NewPveNodeStatusCollector(apiClient *proxmox.PveApiClient, registry *TTLReg // Node localtime info. c.localTime = NewTTLGaugeVec( prometheus.GaugeOpts{ - Name: "pve_node_localtime", - Help: "Node localtime.", + Name: "pve_node_localtime_seconds", + Help: "Node local time as a unix timestamp in seconds.", }, []string{"cluster", "node"}, 5*time.Minute, diff --git a/metrics/pve_node_subscription_collector.go b/metrics/pve_node_subscription_collector.go index e6f9dfe..fba6f1f 100644 --- a/metrics/pve_node_subscription_collector.go +++ b/metrics/pve_node_subscription_collector.go @@ -49,8 +49,8 @@ func NewPveSubscriptionCollector(apiClient *proxmox.PveApiClient, registry *TTLR // Node subscription registration date. c.regDate = NewTTLGaugeVec( prometheus.GaugeOpts{ - Name: "pve_node_subscription_regdate", - Help: "Node subscription registration date.", + Name: "pve_node_subscription_registration_timestamp_seconds", + Help: "Node subscription registration date as a unix timestamp in seconds.", }, []string{"cluster", "node"}, 5*time.Minute, @@ -60,8 +60,8 @@ func NewPveSubscriptionCollector(apiClient *proxmox.PveApiClient, registry *TTLR // Node subscription next due date. c.nextDueDate = NewTTLGaugeVec( prometheus.GaugeOpts{ - Name: "pve_node_subscription_nextduedate", - Help: "Node subscription next due date.", + Name: "pve_node_subscription_next_due_timestamp_seconds", + Help: "Node subscription next due date as a unix timestamp in seconds.", }, []string{"cluster", "node"}, 5*time.Minute, diff --git a/metrics/pve_node_virtual_machine_collector.go b/metrics/pve_node_virtual_machine_collector.go index 90c34e4..31c6368 100644 --- a/metrics/pve_node_virtual_machine_collector.go +++ b/metrics/pve_node_virtual_machine_collector.go @@ -26,20 +26,20 @@ type PveVirtualMachineCollector struct { diskMax *TTLGaugeVec // Virtual machine disk size in bytes prometheus gauge. swap *TTLGaugeVec // Virtual machine swap usage in bytes prometheus gauge. - netReceive *TTLGaugeVec // Virtual machine network receive in bytes prometheus gauge. - netTransmit *TTLGaugeVec // Virtual machine network transmit in bytes prometheus gauge. + netReceive *TTLCounterVec // Virtual machine received network traffic in bytes prometheus counter. + netTransmit *TTLCounterVec // Virtual machine transmitted network traffic in bytes prometheus counter. - diskReadOps *TTLGaugeVec // Virtual machine disk read ops prometheus gauge. - diskWriteOps *TTLGaugeVec // Virtual machine disk write ops prometheus gauge. + diskReadOps *TTLCounterVec // Virtual machine disk read operations prometheus counter. + diskWriteOps *TTLCounterVec // Virtual machine disk write operations prometheus counter. - diskReadBytes *TTLGaugeVec // Virtual machine disk read bytes prometheus gauge. - diskWriteBytes *TTLGaugeVec // Virtual machine disk write bytes prometheus gauge. + diskReadBytes *TTLCounterVec // Virtual machine disk read bytes prometheus counter. + diskWriteBytes *TTLCounterVec // Virtual machine disk written bytes prometheus counter. - diskReadTimeNs *TTLGaugeVec // Virtual machine disk read time total prometheus gauge. - diskWriteTimeNs *TTLGaugeVec // Virtual machine disk write time total prometheus gauge. + diskReadTime *TTLCounterVec // Virtual machine disk read time total in seconds prometheus counter. + diskWriteTime *TTLCounterVec // Virtual machine disk write time total in seconds prometheus counter. - diskFailedReadOps *TTLGaugeVec // Virtual machine disk failed read ops prometheus gauge. - diskFailedWriteOps *TTLGaugeVec // Virtual machine disk failed write ops prometheus gauge. + diskFailedReadOps *TTLCounterVec // Virtual machine failed disk read operations prometheus counter. + diskFailedWriteOps *TTLCounterVec // Virtual machine failed disk write operations prometheus counter. agent *TTLGaugeVec // Virtual machine agent enabled prometheus gauge. } @@ -63,8 +63,8 @@ func NewPveVirtualMachineCollector(apiClient *proxmox.PveApiClient, registry *TT // Virtual machine uptime. c.uptime = NewTTLGaugeVec( prometheus.GaugeOpts{ - Name: "pve_vm_uptime", - Help: "Virtual machine uptime.", + Name: "pve_vm_uptime_seconds", + Help: "Virtual machine uptime in seconds.", }, []string{"cluster", "node", "vmid", "name"}, 5*time.Minute, @@ -85,7 +85,7 @@ func NewPveVirtualMachineCollector(apiClient *proxmox.PveApiClient, registry *TT // Virtual machine CPU count. c.cpu = NewTTLGaugeVec( prometheus.GaugeOpts{ - Name: "pve_vm_cpu_count", + Name: "pve_vm_cpus", Help: "Virtual machine CPU count.", }, []string{"cluster", "node", "vmid", "name"}, @@ -96,8 +96,8 @@ func NewPveVirtualMachineCollector(apiClient *proxmox.PveApiClient, registry *TT // Virtual machine CPU usage. c.cpuUsage = NewTTLGaugeVec( prometheus.GaugeOpts{ - Name: "pve_vm_cpu_usage", - Help: "Virtual machine CPU usage.", + Name: "pve_vm_cpu_usage_ratio", + Help: "Virtual machine CPU usage ratio (0-1).", }, []string{"cluster", "node", "vmid", "name"}, 5*time.Minute, @@ -107,7 +107,7 @@ func NewPveVirtualMachineCollector(apiClient *proxmox.PveApiClient, registry *TT // Virtual machine memory total. c.memBytes = NewTTLGaugeVec( prometheus.GaugeOpts{ - Name: "pve_vm_mem_total_bytes", + Name: "pve_vm_memory_total_bytes", Help: "Virtual machine total memory in bytes.", }, []string{"cluster", "node", "vmid", "name"}, @@ -118,7 +118,7 @@ func NewPveVirtualMachineCollector(apiClient *proxmox.PveApiClient, registry *TT // Virtual machine memory usage. c.memBytesUsed = NewTTLGaugeVec( prometheus.GaugeOpts{ - Name: "pve_vm_mem_used_bytes", + Name: "pve_vm_memory_used_bytes", Help: "Virtual machine used memory in bytes.", }, []string{"cluster", "node", "vmid", "name"}, @@ -129,8 +129,8 @@ func NewPveVirtualMachineCollector(apiClient *proxmox.PveApiClient, registry *TT // Virtual machine disk size. c.disk = NewTTLGaugeVec( prometheus.GaugeOpts{ - Name: "pve_vm_disk_usage_bytes", - Help: "Virtual machine disk read bytes.", + Name: "pve_vm_disk_used_bytes", + Help: "Virtual machine used disk space in bytes.", }, []string{"cluster", "node", "vmid", "name"}, 5*time.Minute, @@ -149,10 +149,10 @@ func NewPveVirtualMachineCollector(apiClient *proxmox.PveApiClient, registry *TT c.registry.Register(c.diskMax) // Virtual machine network receive bytes. - c.netReceive = NewTTLGaugeVec( - prometheus.GaugeOpts{ - Name: "pve_vm_network_in_bytes", - Help: "Virtual machine network receive in bytes.", + c.netReceive = NewTTLCounterVec( + prometheus.CounterOpts{ + Name: "pve_vm_network_receive_bytes_total", + Help: "Virtual machine received network traffic in bytes.", }, []string{"cluster", "node", "vmid", "name", "interface"}, 5*time.Minute, @@ -160,10 +160,10 @@ func NewPveVirtualMachineCollector(apiClient *proxmox.PveApiClient, registry *TT c.registry.Register(c.netReceive) // Virtual machine network transmit bytes. - c.netTransmit = NewTTLGaugeVec( - prometheus.GaugeOpts{ - Name: "pve_vm_network_out_bytes", - Help: "Virtual machine network transmit in bytes.", + c.netTransmit = NewTTLCounterVec( + prometheus.CounterOpts{ + Name: "pve_vm_network_transmit_bytes_total", + Help: "Virtual machine transmitted network traffic in bytes.", }, []string{"cluster", "node", "vmid", "name", "interface"}, 5*time.Minute, @@ -171,10 +171,10 @@ func NewPveVirtualMachineCollector(apiClient *proxmox.PveApiClient, registry *TT c.registry.Register(c.netTransmit) // Virtual machine disk read ops. - c.diskReadOps = NewTTLGaugeVec( - prometheus.GaugeOpts{ - Name: "pve_vm_disk_rd_operations", - Help: "Virtual machine disk read ops.", + c.diskReadOps = NewTTLCounterVec( + prometheus.CounterOpts{ + Name: "pve_vm_disk_read_operations_total", + Help: "Virtual machine disk read operations.", }, []string{"cluster", "node", "vmid", "name", "device"}, 5*time.Minute, @@ -182,10 +182,10 @@ func NewPveVirtualMachineCollector(apiClient *proxmox.PveApiClient, registry *TT c.registry.Register(c.diskReadOps) // Virtual machine disk write ops. - c.diskWriteOps = NewTTLGaugeVec( - prometheus.GaugeOpts{ - Name: "pve_vm_disk_wr_operations", - Help: "Virtual machine disk write ops.", + c.diskWriteOps = NewTTLCounterVec( + prometheus.CounterOpts{ + Name: "pve_vm_disk_write_operations_total", + Help: "Virtual machine disk write operations.", }, []string{"cluster", "node", "vmid", "name", "device"}, 5*time.Minute, @@ -193,9 +193,9 @@ func NewPveVirtualMachineCollector(apiClient *proxmox.PveApiClient, registry *TT c.registry.Register(c.diskWriteOps) // Virtual machine disk read bytes. - c.diskReadBytes = NewTTLGaugeVec( - prometheus.GaugeOpts{ - Name: "pve_vm_disk_rd_bytes", + c.diskReadBytes = NewTTLCounterVec( + prometheus.CounterOpts{ + Name: "pve_vm_disk_read_bytes_total", Help: "Virtual machine disk read bytes.", }, []string{"cluster", "node", "vmid", "name", "device"}, @@ -204,9 +204,9 @@ func NewPveVirtualMachineCollector(apiClient *proxmox.PveApiClient, registry *TT c.registry.Register(c.diskReadBytes) // Virtual machine disk write bytes. - c.diskWriteBytes = NewTTLGaugeVec( - prometheus.GaugeOpts{ - Name: "pve_vm_disk_wr_bytes", + c.diskWriteBytes = NewTTLCounterVec( + prometheus.CounterOpts{ + Name: "pve_vm_disk_write_bytes_total", Help: "Virtual machine disk write bytes.", }, []string{"cluster", "node", "vmid", "name", "device"}, @@ -215,10 +215,10 @@ func NewPveVirtualMachineCollector(apiClient *proxmox.PveApiClient, registry *TT c.registry.Register(c.diskWriteBytes) // Virtual machine failed disk read ops. - c.diskFailedReadOps = NewTTLGaugeVec( - prometheus.GaugeOpts{ - Name: "pve_vm_disk_failed_rd_ops", - Help: "Virtual machine failed disk read ops.", + c.diskFailedReadOps = NewTTLCounterVec( + prometheus.CounterOpts{ + Name: "pve_vm_disk_failed_read_operations_total", + Help: "Virtual machine failed disk read operations.", }, []string{"cluster", "node", "vmid", "name", "device"}, 5*time.Minute, @@ -226,37 +226,37 @@ func NewPveVirtualMachineCollector(apiClient *proxmox.PveApiClient, registry *TT c.registry.Register(c.diskFailedReadOps) // Virtual machine failed disk write ops. - c.diskFailedWriteOps = NewTTLGaugeVec( - prometheus.GaugeOpts{ - Name: "pve_vm_disk_failed_wr_ops", - Help: "Virtual machine failed disk write ops.", + c.diskFailedWriteOps = NewTTLCounterVec( + prometheus.CounterOpts{ + Name: "pve_vm_disk_failed_write_operations_total", + Help: "Virtual machine failed disk write operations.", }, []string{"cluster", "node", "vmid", "name", "device"}, 5*time.Minute, ) c.registry.Register(c.diskFailedWriteOps) - // Virtual machine disk read time total nanoseconds. - c.diskReadTimeNs = NewTTLGaugeVec( - prometheus.GaugeOpts{ - Name: "pve_vm_disk_rd_time_total_ns", - Help: "Virtual machine disk read time total in nanoseconds.", + // Virtual machine disk read time total seconds. + c.diskReadTime = NewTTLCounterVec( + prometheus.CounterOpts{ + Name: "pve_vm_disk_read_time_seconds_total", + Help: "Virtual machine disk read time total in seconds.", }, []string{"cluster", "node", "vmid", "name", "device"}, 5*time.Minute, ) - c.registry.Register(c.diskReadTimeNs) + c.registry.Register(c.diskReadTime) - // Virtual machine disk write time total nanoseconds. - c.diskWriteTimeNs = NewTTLGaugeVec( - prometheus.GaugeOpts{ - Name: "pve_vm_disk_wr_time_total_ns", - Help: "Virtual machine disk write time total in nanoseconds.", + // Virtual machine disk write time total seconds. + c.diskWriteTime = NewTTLCounterVec( + prometheus.CounterOpts{ + Name: "pve_vm_disk_write_time_seconds_total", + Help: "Virtual machine disk write time total in seconds.", }, []string{"cluster", "node", "vmid", "name", "device"}, 5*time.Minute, ) - c.registry.Register(c.diskWriteTimeNs) + c.registry.Register(c.diskWriteTime) return &c } @@ -336,8 +336,8 @@ func (c *PveVirtualMachineCollector) CollectMetrics() error { c.diskFailedReadOps.With(labels).Set(float64(value.FailedRdOperations)) c.diskFailedWriteOps.With(labels).Set(float64(value.FailedWrOperations)) - c.diskReadTimeNs.With(labels).Set(float64(value.RdTotalTimeNs)) - c.diskWriteTimeNs.With(labels).Set(float64(value.WrTotalTimeNs)) + c.diskReadTime.With(labels).Set(float64(value.RdTotalTimeNs) / float64(time.Second)) + c.diskWriteTime.With(labels).Set(float64(value.WrTotalTimeNs) / float64(time.Second)) } } } diff --git a/metrics/pve_node_zfs_collector.go b/metrics/pve_node_zfs_collector.go index b18e969..84aa9f0 100644 --- a/metrics/pve_node_zfs_collector.go +++ b/metrics/pve_node_zfs_collector.go @@ -15,9 +15,9 @@ type PveNodeZfsCollector struct { registry *TTLRegistry state *TTLGaugeVec - readErrors *TTLGaugeVec - writeErrors *TTLGaugeVec - checksumErrors *TTLGaugeVec + readErrors *TTLCounterVec + writeErrors *TTLCounterVec + checksumErrors *TTLCounterVec } // zfsMetricComponent is a flattened entry from the recursive ZFS topology. @@ -50,9 +50,9 @@ func NewPveNodeZfsCollector(apiClient *proxmox.PveApiClient, registry *TTLRegist ) c.registry.Register(c.state) - c.readErrors = NewTTLGaugeVec( - prometheus.GaugeOpts{ - Name: "pve_node_zfs_read_errors", + c.readErrors = NewTTLCounterVec( + prometheus.CounterOpts{ + Name: "pve_node_zfs_read_errors_total", Help: "ZFS pool component read error count.", }, componentLabelNames, @@ -60,9 +60,9 @@ func NewPveNodeZfsCollector(apiClient *proxmox.PveApiClient, registry *TTLRegist ) c.registry.Register(c.readErrors) - c.writeErrors = NewTTLGaugeVec( - prometheus.GaugeOpts{ - Name: "pve_node_zfs_write_errors", + c.writeErrors = NewTTLCounterVec( + prometheus.CounterOpts{ + Name: "pve_node_zfs_write_errors_total", Help: "ZFS pool component write error count.", }, componentLabelNames, @@ -70,9 +70,9 @@ func NewPveNodeZfsCollector(apiClient *proxmox.PveApiClient, registry *TTLRegist ) c.registry.Register(c.writeErrors) - c.checksumErrors = NewTTLGaugeVec( - prometheus.GaugeOpts{ - Name: "pve_node_zfs_checksum_errors", + c.checksumErrors = NewTTLCounterVec( + prometheus.CounterOpts{ + Name: "pve_node_zfs_checksum_errors_total", Help: "ZFS pool component checksum error count.", }, componentLabelNames, diff --git a/metrics/pve_ttl_metrics.go b/metrics/pve_ttl_metrics.go index 03177a2..189fe0a 100644 --- a/metrics/pve_ttl_metrics.go +++ b/metrics/pve_ttl_metrics.go @@ -23,6 +23,7 @@ type TTLMetric interface { // it is automatically removed from the underlying GaugeVec. type TTLGaugeVec struct { gaugeVec *prometheus.GaugeVec // Underlying Prometheus GaugeVec. + labelNames []string // Label names of the underlying GaugeVec. ttl time.Duration // Duration after which an unused label set is considered stale. lastUpdate sync.Map // Map storing last update time for each label set (key is a sorted labels string). } @@ -31,8 +32,9 @@ type TTLGaugeVec struct { // The underlying GaugeVec is registered using promauto. func NewTTLGaugeVec(opts prometheus.GaugeOpts, labelNames []string, ttl time.Duration) *TTLGaugeVec { return &TTLGaugeVec{ - gaugeVec: promauto.NewGaugeVec(opts, labelNames), - ttl: ttl, + gaugeVec: promauto.NewGaugeVec(opts, labelNames), + labelNames: labelNames, + ttl: ttl, } } @@ -137,3 +139,118 @@ func (r *TTLRegistry) StartCleanupLoop(interval time.Duration) { } }() } + +// TTLCounterVec exposes cumulative values as Prometheus counters while keeping +// the same "set an absolute value" usage as TTLGaugeVec. The PVE API reports +// counters as absolute totals, which prometheus.CounterVec cannot express +// (it only supports Inc/Add), so the metrics are emitted as constant metrics +// of type counter. Label sets not updated within the TTL are dropped. +type TTLCounterVec struct { + desc *prometheus.Desc // Metric descriptor. + labelNames []string // Label names in exposition order. + ttl time.Duration // Duration after which an unused label set is considered stale. + mu sync.RWMutex // Guards values. + values map[string]*ttlCounterEntry // Current value per label set. +} + +// ttlCounterEntry holds the current value of a single label set. +type ttlCounterEntry struct { + labelValues []string // Label values in exposition order. + value float64 // Current counter value. + lastUpdate time.Time // Time of the last update. +} + +// TTLCounter is a handle to a single label set of a TTLCounterVec. +type TTLCounter struct { + vec *TTLCounterVec + entry *ttlCounterEntry +} + +// Set stores the current absolute value of the counter. +func (c *TTLCounter) Set(value float64) { + c.vec.mu.Lock() + defer c.vec.mu.Unlock() + c.entry.value = value +} + +// NewTTLCounterVec creates a new TTLCounterVec and registers it using promauto. +func NewTTLCounterVec(opts prometheus.CounterOpts, labelNames []string, ttl time.Duration) *TTLCounterVec { + c := &TTLCounterVec{ + desc: prometheus.NewDesc( + prometheus.BuildFQName(opts.Namespace, opts.Subsystem, opts.Name), + opts.Help, + labelNames, + opts.ConstLabels, + ), + labelNames: labelNames, + ttl: ttl, + values: make(map[string]*ttlCounterEntry), + } + prometheus.MustRegister(c) + return c +} + +// With returns the counter handle for the given label set and records the current +// time as the last update for those labels. +func (t *TTLCounterVec) With(labels prometheus.Labels) *TTLCounter { + key := labelsKey(labels) + + t.mu.Lock() + defer t.mu.Unlock() + + entry, ok := t.values[key] + if !ok { + labelValues := make([]string, len(t.labelNames)) + for i, name := range t.labelNames { + labelValues[i] = labels[name] + } + entry = &ttlCounterEntry{labelValues: labelValues} + t.values[key] = entry + } + entry.lastUpdate = time.Now() + + return &TTLCounter{vec: t, entry: entry} +} + +// Delete removes the metric associated with the given label set. +func (t *TTLCounterVec) Delete(labels prometheus.Labels) bool { + key := labelsKey(labels) + + t.mu.Lock() + defer t.mu.Unlock() + + if _, ok := t.values[key]; !ok { + return false + } + delete(t.values, key) + return true +} + +// Cleanup deletes all label sets that have not been updated within the TTL duration. +func (t *TTLCounterVec) Cleanup() { + now := time.Now() + + t.mu.Lock() + defer t.mu.Unlock() + + for key, entry := range t.values { + if now.Sub(entry.lastUpdate) > t.ttl { + delete(t.values, key) + } + } +} + +// Describe implements prometheus.Collector. +func (t *TTLCounterVec) Describe(ch chan<- *prometheus.Desc) { + ch <- t.desc +} + +// Collect implements prometheus.Collector. +func (t *TTLCounterVec) Collect(ch chan<- prometheus.Metric) { + t.mu.RLock() + defer t.mu.RUnlock() + + for _, entry := range t.values { + ch <- prometheus.MustNewConstMetric(t.desc, prometheus.CounterValue, entry.value, entry.labelValues...) + } +}