From 832549d6677a0563454685aeadb0af5102022f6a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jan=20Lo=C5=A1=C5=A5=C3=A1k?= Date: Tue, 18 Aug 2026 01:44:45 +0200 Subject: [PATCH] Added example alerting config --- README.md | 7 + docs/README.md | 2 + docs/alerting.md | 70 +++++ docs/operations.md | 5 + examples/pve-exporter.rules.yml | 526 ++++++++++++++++++++++++++++++++ 5 files changed, 610 insertions(+) create mode 100644 docs/alerting.md create mode 100644 examples/pve-exporter.rules.yml diff --git a/README.md b/README.md index e46cf7d..7512ae3 100644 --- a/README.md +++ b/README.md @@ -77,6 +77,7 @@ http://localhost:9090/metrics - [Documentation index](docs/README.md) - [Configuration reference](docs/configuration.md) - [Metrics reference](docs/metrics.md) +- [Prometheus alerting rules](docs/alerting.md) - [Operations and troubleshooting](docs/operations.md) - [Architecture and adding collectors](docs/architecture.md) @@ -93,6 +94,12 @@ Metrics are collected in the background according to `proxmox.interval`. Prometheus scrapes the most recently collected values and does not trigger a new request to Proxmox VE. +An example alert group for node, guest, storage, subscription, disk, ZFS, SDN, +and capacity conditions is available in +[`examples/pve-exporter.rules.yml`](examples/pve-exporter.rules.yml). See the +[alerting guide](docs/alerting.md) for the `rule_files` configuration and +collector requirements. + ## Docker Pull the latest published image from the Gitea container registry: diff --git a/docs/README.md b/docs/README.md index ec65da0..01aa83f 100644 --- a/docs/README.md +++ b/docs/README.md @@ -9,6 +9,8 @@ and stores the latest values in the Prometheus registry exposed at `/metrics`. API token permissions, and collector switch. - [Metrics reference](metrics.md) lists exported metrics, labels, units, and numeric state mappings. +- [Prometheus alerting](alerting.md) provides a ready-to-use rule group, + installation example, and collector requirements. - [Operations and troubleshooting](operations.md) covers native and container deployment, endpoint failover, logging, security, and common errors. - [Architecture and adding collectors](architecture.md) explains the collection diff --git a/docs/alerting.md b/docs/alerting.md new file mode 100644 index 0000000..62b6808 --- /dev/null +++ b/docs/alerting.md @@ -0,0 +1,70 @@ +# Prometheus Alerting + +The repository includes a ready-to-use example rule group in +[`examples/pve-exporter.rules.yml`](../examples/pve-exporter.rules.yml). It +covers node, guest, storage, subscription, physical disk, ZFS, SDN, clock-skew, +and memory-overcommit conditions. + +## Enable the rule file + +Make the rule file available to Prometheus and reference it from +`prometheus.yml`: + +```yaml +rule_files: + - /etc/prometheus/rules/pve-exporter.rules.yml + +scrape_configs: + - job_name: pve-exporter + static_configs: + - targets: [pve-exporter:9090] +``` + +For a containerized Prometheus deployment, mount the example at the configured +path: + +```yaml +services: + prometheus: + image: prom/prometheus:latest + volumes: + - ./prometheus.yml:/etc/prometheus/prometheus.yml:ro + - ./examples/pve-exporter.rules.yml:/etc/prometheus/rules/pve-exporter.rules.yml:ro +``` + +Validate the file before reloading Prometheus: + +```sh +promtool check rules examples/pve-exporter.rules.yml +``` + +## Collector requirements + +Prometheus evaluates a rule only when its referenced time series exist. Enable +the matching exporter collectors: + +| Alerts | Required collector setting | +| --- | --- | +| Node filesystem, state, CPU, clock skew, memory overcommit | `metrics.nodeStatus` | +| Guest CPU and memory overcommit | `metrics.qemu` and/or `metrics.lxc` | +| Storage state and capacity | `metrics.storage` | +| Subscription expiration | `metrics.subscription` | +| Physical disk health and wear | `metrics.disk` | +| ZFS state and errors | `metrics.zfs` | +| SDN state | `metrics.sdn` | + +The default thresholds and `for` durations are operational examples. Review +them against the size, workload, redundancy, and maintenance policy of your +environment before enabling notifications. + +Three rules require deployment-specific review: + +- `PveUnexpectedStandaloneMode` should be removed or disabled when standalone + PVE operation is expected. +- `PveNodeSubscriptionMissing` is informational and should be removed when + unsubscribed nodes are intentional. +- `PveExporterDown` expects the Prometheus scrape job to be named + `pve-exporter`; adjust its `job` matcher if a different name is used. + +Every rule uses the `cluster` label. Standalone installations are supported and +use the label value `Standalone node - `. diff --git a/docs/operations.md b/docs/operations.md index 5b439aa..f8636b6 100644 --- a/docs/operations.md +++ b/docs/operations.md @@ -48,6 +48,11 @@ scrape_configs: The Prometheus scrape interval and `proxmox.interval` are independent. A scrape does not contact PVE; it reads the latest values collected in the background. +For alerting, load +[`examples/pve-exporter.rules.yml`](../examples/pve-exporter.rules.yml) through +Prometheus `rule_files`. The [alerting guide](alerting.md) lists the required +collectors and a container mount example. + ## API endpoint availability At startup and every five seconds, the exporter requests `api2/json/` from each diff --git a/examples/pve-exporter.rules.yml b/examples/pve-exporter.rules.yml new file mode 100644 index 0000000..16a473c --- /dev/null +++ b/examples/pve-exporter.rules.yml @@ -0,0 +1,526 @@ +groups: + - name: proxmox.rules + rules: + - alert: PveNodeFilesystemFillingUp + expr: |- + ( + pve_node_rootfs_avail_bytes / pve_node_rootfs_total_bytes * 100 < 15 + and + predict_linear(pve_node_rootfs_avail_bytes[6h], 24 * 60 * 60) < 0 + ) + for: 1h + labels: + severity: critical + annotations: + summary: Proxmox root filesystem is predicted to run out of space within 24 hours. + description: >- + Filesystem on node {{ $labels.node }} in the {{ $labels.cluster }} + cluster has only {{ printf "%.2f" $value }}% available space left + and is filling up. + + - alert: PveGuestCpuSaturated + expr: |- + pve_vm_cpu_usage > 0.95 or pve_ct_cpu_usage > 0.95 + for: 1h + labels: + severity: warning + annotations: + summary: Proxmox guest CPU has been saturated for more than an hour. + description: >- + Guest {{ $labels.name }} with ID {{ $labels.vmid }} on node + {{ $labels.node }} in the {{ $labels.cluster }} cluster has sustained + CPU usage above 95%. + + - alert: PveStorageDown + expr: |- + pve_storage_up == 0 + for: 1m + labels: + severity: critical + annotations: + summary: Proxmox storage is down. + description: >- + Storage {{ $labels.storage }} on node {{ $labels.node }} in the + {{ $labels.cluster }} cluster is down. This can severely impact + running guests. + + - alert: PveStorageFillingUp + expr: |- + ( + pve_storage_avail_bytes / pve_storage_total_bytes * 100 < 15 + and + predict_linear(pve_storage_avail_bytes[6h], 24 * 60 * 60) < 0 + ) + for: 1h + labels: + severity: critical + annotations: + summary: Proxmox storage is predicted to run out of space within 24 hours. + description: >- + Storage {{ $labels.storage }} on node {{ $labels.node }} in the + {{ $labels.cluster }} cluster has only {{ printf "%.2f" $value }}% + available space left and is filling up. + + - alert: PveNodeDown + expr: |- + pve_node_state == 0 + for: 1m + labels: + severity: critical + annotations: + summary: Proxmox node is down. + description: >- + Node {{ $labels.node }} in the {{ $labels.cluster }} cluster is down. + + - alert: PveNodeCpuSaturated + expr: |- + pve_node_cpu_usage > 0.95 + for: 1h + labels: + severity: warning + annotations: + summary: Proxmox node CPU has been saturated for more than an hour. + description: >- + CPU usage on node {{ $labels.node }} in the {{ $labels.cluster }} + cluster has remained above 95%. This can affect guests running on + the node. + + - alert: PveSdnDown + expr: |- + pve_sdn_state == 0 + for: 1m + labels: + severity: critical + annotations: + summary: Proxmox software-defined network is down. + description: >- + SDN resource {{ $labels.sdn }} on node {{ $labels.node }} in the + {{ $labels.cluster }} cluster is down. + + - alert: PveClusterClockSkew + expr: |- + max by (cluster) (pve_node_localtime) + - min by (cluster) (pve_node_localtime) > 15 + for: 1m + labels: + severity: critical + annotations: + summary: Proxmox cluster clock skew is too high. + description: >- + Clock skew between nodes in the {{ $labels.cluster }} cluster is + greater than 15 seconds. Cluster functions may be affected. + + - alert: PveNodeSubscriptionExpiringSoon + expr: |- + ( + pve_node_subscription_nextduedate > time() + and + pve_node_subscription_nextduedate < time() + (30 * 24 * 60 * 60) + ) + and on (cluster, node) + (pve_node_subscription_status == 1) + for: 15m + labels: + severity: warning + annotations: + summary: Proxmox node subscription expires in less than 30 days. + description: >- + Subscription for node {{ $labels.node }} in the + {{ $labels.cluster }} cluster expires at + {{ $value | humanizeTimestamp }}. + + - alert: PveNodeDiskFailed + expr: |- + pve_node_disk_healthy == 0 + for: 1m + labels: + severity: critical + annotations: + summary: Proxmox node disk failed. + description: |- + A disk on node {{ $labels.node }} in the {{ $labels.cluster }} cluster is in a failed state. + WWN: {{ $labels.wwn }} + Serial: {{ $labels.serial }} + Model: {{ $labels.model }} + + Immediate device replacement is required to ensure safe operation of the node. + + - alert: PveNodeDiskWearout + expr: |- + pve_node_disk_wearout < 5 + for: 1m + labels: + severity: critical + annotations: + summary: Proxmox node disk has less than 5% wear life remaining. + description: |- + A disk on node {{ $labels.node }} in the {{ $labels.cluster }} cluster is showing significant wear and may fail soon. + WWN: {{ $labels.wwn }} + Serial: {{ $labels.serial }} + Model: {{ $labels.model }} + Current wear level: {{ $value }}% + + Replace the disk to reduce the risk of data loss. + + - alert: PveNodeZfsPoolUnknown + expr: |- + pve_node_zfs_state == 0 + for: 5m + labels: + severity: warning + annotations: + summary: Proxmox ZFS pool component state is unknown. + description: >- + ZFS component {{ $labels.path }} in pool {{ $labels.pool }} on node + {{ $labels.node }} in the {{ $labels.cluster }} cluster has been in + the UNKNOWN state for at least five minutes. + + - alert: PveNodeZfsPoolDegraded + expr: |- + pve_node_zfs_state == 2 + for: 5m + labels: + severity: warning + annotations: + summary: Proxmox ZFS pool component is degraded. + description: >- + ZFS component {{ $labels.path }} in pool {{ $labels.pool }} on node + {{ $labels.node }} in the {{ $labels.cluster }} cluster has been + DEGRADED for at least five minutes. + + - alert: PveNodeZfsPoolFaulted + expr: |- + pve_node_zfs_state == 3 + for: 5m + labels: + severity: critical + annotations: + summary: Proxmox ZFS pool component has faulted. + description: >- + ZFS component {{ $labels.path }} in pool {{ $labels.pool }} on node + {{ $labels.node }} in the {{ $labels.cluster }} cluster has been + FAULTED for at least five minutes. + + - alert: PveNodeZfsPoolOffline + expr: |- + pve_node_zfs_state == 4 + for: 5m + labels: + severity: critical + annotations: + summary: Proxmox ZFS pool component is offline. + description: >- + ZFS component {{ $labels.path }} in pool {{ $labels.pool }} on node + {{ $labels.node }} in the {{ $labels.cluster }} cluster has been + OFFLINE for at least five minutes. + + - alert: PveNodeZfsPoolRemoved + expr: |- + pve_node_zfs_state == 5 + for: 5m + labels: + severity: critical + annotations: + summary: Proxmox ZFS pool component was removed. + description: >- + ZFS component {{ $labels.path }} in pool {{ $labels.pool }} on node + {{ $labels.node }} in the {{ $labels.cluster }} cluster has been + REMOVED for at least five minutes. + + - alert: PveNodeZfsPoolUnavailable + expr: |- + pve_node_zfs_state == 6 + for: 5m + labels: + severity: critical + annotations: + summary: Proxmox ZFS pool component is unavailable. + description: >- + ZFS component {{ $labels.path }} in pool {{ $labels.pool }} on node + {{ $labels.node }} in the {{ $labels.cluster }} cluster has been + UNAVAIL for at least five minutes. + + - alert: PveNodeZfsReadErrors + expr: |- + pve_node_zfs_read_errors > 0 + for: 5m + labels: + severity: warning + annotations: + summary: Proxmox ZFS component reports read errors. + description: >- + ZFS component {{ $labels.path }} in pool {{ $labels.pool }} on node + {{ $labels.node }} in the {{ $labels.cluster }} cluster reports + {{ $value | humanize }} read errors. + + - alert: PveNodeZfsWriteErrors + expr: |- + pve_node_zfs_write_errors > 0 + for: 5m + labels: + severity: warning + annotations: + summary: Proxmox ZFS component reports write errors. + description: >- + ZFS component {{ $labels.path }} in pool {{ $labels.pool }} on node + {{ $labels.node }} in the {{ $labels.cluster }} cluster reports + {{ $value | humanize }} write errors. + + - alert: PveNodeZfsChecksumErrors + expr: |- + pve_node_zfs_checksum_errors > 0 + for: 5m + labels: + severity: warning + annotations: + summary: Proxmox ZFS component reports checksum errors. + description: >- + ZFS component {{ $labels.path }} in pool {{ $labels.pool }} on node + {{ $labels.node }} in the {{ $labels.cluster }} cluster reports + {{ $value | humanize }} checksum errors. + + - alert: PveNodeMemoryOvercommit + expr: |- + ( + sum by (cluster, node) ( + ( + pve_vm_mem_total_bytes + and on (cluster, node, vmid, name) (pve_vm_state == 1) + ) + or + ( + pve_ct_mem_total_bytes + and on (cluster, node, vmid, name) (pve_ct_state == 1) + ) + ) + / pve_node_memory_total_bytes * 100 - 100 + ) > 20 + for: 1m + labels: + severity: critical + annotations: + summary: Proxmox node memory is overcommitted by more than 20%. + description: >- + Node {{ $labels.node }} in the {{ $labels.cluster }} cluster has + {{ printf "%.2f" $value }}% more memory allocated to running guests + than is physically available. Concurrent guest memory demand can + exhaust node memory and disrupt services. + + - alert: PveClusterQuorumLost + expr: |- + (pve_cluster_mode == 1) + and on (cluster) + (pve_cluster_quorate == 0) + for: 1m + labels: + severity: critical + annotations: + summary: Proxmox cluster has lost quorum. + description: >- + The {{ $labels.cluster }} cluster has not been quorate for at least + one minute. Cluster operations and guest availability may be + affected. + + - alert: PveUnexpectedStandaloneMode + expr: |- + pve_cluster_mode == 0 + for: 5m + labels: + severity: warning + annotations: + summary: Proxmox is running in standalone mode. + description: >- + {{ $labels.cluster }} has reported standalone mode for at least five + minutes. Disable this rule when standalone operation is expected. + + - alert: PveNodeMemoryHigh + expr: |- + pve_node_memory_used_bytes / pve_node_memory_total_bytes > 0.90 + for: 15m + labels: + severity: warning + annotations: + summary: Proxmox node memory usage is high. + description: >- + Node {{ $labels.node }} in the {{ $labels.cluster }} cluster has + used {{ $value | humanizePercentage }} of its memory for at least + 15 minutes. + + - alert: PveNodeFilesystemLowSpace + expr: |- + pve_node_rootfs_avail_bytes / pve_node_rootfs_total_bytes < 0.10 + for: 5m + labels: + severity: critical + annotations: + summary: Proxmox root filesystem has less than 10% space available. + description: >- + The root filesystem on node {{ $labels.node }} in the + {{ $labels.cluster }} cluster has only + {{ $value | humanizePercentage }} available space left. + + - alert: PveStorageLowSpace + expr: |- + pve_storage_avail_bytes / pve_storage_total_bytes < 0.10 + for: 5m + labels: + severity: critical + annotations: + summary: Proxmox storage has less than 10% space available. + description: >- + Storage {{ $labels.storage }} on node {{ $labels.node }} in the + {{ $labels.cluster }} cluster has only + {{ $value | humanizePercentage }} available space left. + + - alert: PveNodeSubscriptionExpired + expr: |- + pve_node_subscription_status == 2 + for: 15m + labels: + severity: warning + annotations: + summary: Proxmox node subscription has expired or is unusable. + description: >- + Subscription for node {{ $labels.node }} in the + {{ $labels.cluster }} cluster is expired, invalid, suspended, or + otherwise unusable. + + - alert: PveNodeSubscriptionMissing + expr: |- + pve_node_subscription_status == 0 + for: 15m + labels: + severity: info + annotations: + summary: Proxmox node has no subscription. + description: >- + Node {{ $labels.node }} in the {{ $labels.cluster }} cluster reports + no subscription. Disable this rule when an unsubscribed node is + expected. + + - alert: PveNodeDiskWearoutWarning + expr: |- + (pve_node_disk_wearout < 20) + and + (pve_node_disk_wearout >= 5) + for: 15m + labels: + severity: warning + annotations: + summary: Proxmox node disk has less than 20% wear life remaining. + description: |- + A disk on node {{ $labels.node }} in the {{ $labels.cluster }} cluster is approaching its wear limit. + WWN: {{ $labels.wwn }} + Serial: {{ $labels.serial }} + Model: {{ $labels.model }} + Current wear level: {{ $value }}% + + - alert: PveGuestMemoryHigh + expr: |- + pve_vm_mem_used_bytes / pve_vm_mem_total_bytes > 0.90 + or + pve_ct_mem_used_bytes / pve_ct_mem_total_bytes > 0.90 + for: 15m + labels: + severity: warning + annotations: + summary: Proxmox guest memory usage is high. + description: >- + Guest {{ $labels.name }} with ID {{ $labels.vmid }} on node + {{ $labels.node }} in the {{ $labels.cluster }} cluster has used + {{ $value | humanizePercentage }} of its configured memory for at + least 15 minutes. + + - alert: PveGuestDiskFillingUp + expr: |- + pve_vm_disk_usage_bytes / pve_vm_disk_size_bytes > 0.90 + or + pve_ct_disk_usage_bytes / pve_ct_disk_size_bytes > 0.90 + for: 15m + labels: + severity: warning + annotations: + summary: Proxmox guest root disk usage is high. + description: >- + Guest {{ $labels.name }} with ID {{ $labels.vmid }} on node + {{ $labels.node }} in the {{ $labels.cluster }} cluster has used + {{ $value | humanizePercentage }} of its configured root disk. + + - alert: PveContainerSwapUsageHigh + expr: |- + pve_ct_swap_used_bytes > 256 * 1024 * 1024 + for: 15m + labels: + severity: warning + annotations: + summary: Proxmox container swap usage is high. + description: >- + Container {{ $labels.name }} with ID {{ $labels.vmid }} on node + {{ $labels.node }} in the {{ $labels.cluster }} cluster has used + {{ $value | humanize1024 }}B of swap for at least 15 minutes. + + - alert: PveVmDiskErrorsIncreasing + expr: |- + delta(pve_vm_disk_failed_rd_ops[10m]) > 0 + or + delta(pve_vm_disk_failed_wr_ops[10m]) > 0 + for: 1m + labels: + severity: critical + annotations: + summary: Proxmox virtual machine reports new disk I/O failures. + description: >- + VM {{ $labels.name }} with ID {{ $labels.vmid }} on node + {{ $labels.node }} in the {{ $labels.cluster }} cluster reports new + failed I/O operations on device {{ $labels.device }}. + + - alert: PveNodeClockDrift + expr: |- + abs(pve_node_time - time()) > 60 + for: 5m + labels: + severity: warning + annotations: + summary: Proxmox node clock differs from Prometheus by more than one minute. + description: >- + The clock on node {{ $labels.node }} in the {{ $labels.cluster }} + cluster differs from the Prometheus server by + {{ $value | humanizeDuration }}. + + - alert: PveNodeRestarted + expr: |- + resets(pve_node_uptime[15m]) > 0 + for: 0m + labels: + severity: info + annotations: + summary: Proxmox node restarted recently. + description: >- + Uptime for node {{ $labels.node }} in the {{ $labels.cluster }} + cluster was reset during the last 15 minutes. + + - alert: PveCollectorSlow + expr: |- + rate(pve_metrics_collection_latency_ms_sum[5m]) + / rate(pve_metrics_collection_latency_ms_count[5m]) > 5000 + for: 10m + labels: + severity: warning + annotations: + summary: PVE exporter collector is slow. + description: >- + Collector {{ $labels.collector }} has taken an average of + {{ printf "%.0f" $value }} ms per successful run for at least ten + minutes. + + - alert: PveExporterDown + expr: |- + up{job="pve-exporter"} == 0 + for: 2m + labels: + severity: critical + annotations: + summary: PVE Exporter is unreachable. + description: >- + Prometheus has been unable to scrape PVE Exporter instance + {{ $labels.instance }} for at least two minutes.