lostakj
92e54418bc
Use base units, export cumulative values as _total counters, update docs, dashboards and rules.
527 lines
18 KiB
YAML
527 lines
18 KiB
YAML
groups:
|
|
- name: proxmox.rules
|
|
rules:
|
|
- alert: PveNodeFilesystemFillingUp
|
|
expr: |-
|
|
(
|
|
pve_node_rootfs_avail_bytes / pve_node_rootfs_total_bytes * 100 < 15
|
|
and
|
|
predict_linear(pve_node_rootfs_avail_bytes[6h], 24 * 60 * 60) < 0
|
|
)
|
|
for: 1h
|
|
labels:
|
|
severity: critical
|
|
annotations:
|
|
summary: Proxmox root filesystem is predicted to run out of space within 24 hours.
|
|
description: >-
|
|
Filesystem on node {{ $labels.node }} in the {{ $labels.cluster }}
|
|
cluster has only {{ printf "%.2f" $value }}% available space left
|
|
and is filling up.
|
|
|
|
- alert: PveGuestCpuSaturated
|
|
expr: |-
|
|
pve_vm_cpu_usage_ratio > 0.95 or pve_ct_cpu_usage_ratio > 0.95
|
|
for: 1h
|
|
labels:
|
|
severity: warning
|
|
annotations:
|
|
summary: Proxmox guest CPU has been saturated for more than an hour.
|
|
description: >-
|
|
Guest {{ $labels.name }} with ID {{ $labels.vmid }} on node
|
|
{{ $labels.node }} in the {{ $labels.cluster }} cluster has sustained
|
|
CPU usage above 95%.
|
|
|
|
- alert: PveStorageDown
|
|
expr: |-
|
|
pve_storage_up == 0
|
|
for: 1m
|
|
labels:
|
|
severity: critical
|
|
annotations:
|
|
summary: Proxmox storage is down.
|
|
description: >-
|
|
Storage {{ $labels.storage }} on node {{ $labels.node }} in the
|
|
{{ $labels.cluster }} cluster is down. This can severely impact
|
|
running guests.
|
|
|
|
- alert: PveStorageFillingUp
|
|
expr: |-
|
|
(
|
|
pve_storage_avail_bytes / pve_storage_total_bytes * 100 < 15
|
|
and
|
|
predict_linear(pve_storage_avail_bytes[6h], 24 * 60 * 60) < 0
|
|
)
|
|
for: 1h
|
|
labels:
|
|
severity: critical
|
|
annotations:
|
|
summary: Proxmox storage is predicted to run out of space within 24 hours.
|
|
description: >-
|
|
Storage {{ $labels.storage }} on node {{ $labels.node }} in the
|
|
{{ $labels.cluster }} cluster has only {{ printf "%.2f" $value }}%
|
|
available space left and is filling up.
|
|
|
|
- alert: PveNodeDown
|
|
expr: |-
|
|
pve_node_state == 0
|
|
for: 1m
|
|
labels:
|
|
severity: critical
|
|
annotations:
|
|
summary: Proxmox node is down.
|
|
description: >-
|
|
Node {{ $labels.node }} in the {{ $labels.cluster }} cluster is down.
|
|
|
|
- alert: PveNodeCpuSaturated
|
|
expr: |-
|
|
pve_node_cpu_usage_ratio > 0.95
|
|
for: 1h
|
|
labels:
|
|
severity: warning
|
|
annotations:
|
|
summary: Proxmox node CPU has been saturated for more than an hour.
|
|
description: >-
|
|
CPU usage on node {{ $labels.node }} in the {{ $labels.cluster }}
|
|
cluster has remained above 95%. This can affect guests running on
|
|
the node.
|
|
|
|
- alert: PveSdnDown
|
|
expr: |-
|
|
pve_sdn_state == 0
|
|
for: 1m
|
|
labels:
|
|
severity: critical
|
|
annotations:
|
|
summary: Proxmox software-defined network is down.
|
|
description: >-
|
|
SDN resource {{ $labels.sdn }} on node {{ $labels.node }} in the
|
|
{{ $labels.cluster }} cluster is down.
|
|
|
|
- alert: PveClusterClockSkew
|
|
expr: |-
|
|
max by (cluster) (pve_node_localtime_seconds)
|
|
- min by (cluster) (pve_node_localtime_seconds) > 15
|
|
for: 1m
|
|
labels:
|
|
severity: critical
|
|
annotations:
|
|
summary: Proxmox cluster clock skew is too high.
|
|
description: >-
|
|
Clock skew between nodes in the {{ $labels.cluster }} cluster is
|
|
greater than 15 seconds. Cluster functions may be affected.
|
|
|
|
- alert: PveNodeSubscriptionExpiringSoon
|
|
expr: |-
|
|
(
|
|
pve_node_subscription_next_due_timestamp_seconds > time()
|
|
and
|
|
pve_node_subscription_next_due_timestamp_seconds < time() + (30 * 24 * 60 * 60)
|
|
)
|
|
and on (cluster, node)
|
|
(pve_node_subscription_status == 1)
|
|
for: 15m
|
|
labels:
|
|
severity: warning
|
|
annotations:
|
|
summary: Proxmox node subscription expires in less than 30 days.
|
|
description: >-
|
|
Subscription for node {{ $labels.node }} in the
|
|
{{ $labels.cluster }} cluster expires at
|
|
{{ $value | humanizeTimestamp }}.
|
|
|
|
- alert: PveNodeDiskFailed
|
|
expr: |-
|
|
pve_node_disk_healthy == 0
|
|
for: 1m
|
|
labels:
|
|
severity: critical
|
|
annotations:
|
|
summary: Proxmox node disk failed.
|
|
description: |-
|
|
A disk on node {{ $labels.node }} in the {{ $labels.cluster }} cluster is in a failed state.
|
|
WWN: {{ $labels.wwn }}
|
|
Serial: {{ $labels.serial }}
|
|
Model: {{ $labels.model }}
|
|
|
|
Immediate device replacement is required to ensure safe operation of the node.
|
|
|
|
- alert: PveNodeDiskWearout
|
|
expr: |-
|
|
pve_node_disk_wearout_percent < 5
|
|
for: 1m
|
|
labels:
|
|
severity: critical
|
|
annotations:
|
|
summary: Proxmox node disk has less than 5% wear life remaining.
|
|
description: |-
|
|
A disk on node {{ $labels.node }} in the {{ $labels.cluster }} cluster is showing significant wear and may fail soon.
|
|
WWN: {{ $labels.wwn }}
|
|
Serial: {{ $labels.serial }}
|
|
Model: {{ $labels.model }}
|
|
Current wear level: {{ $value }}%
|
|
|
|
Replace the disk to reduce the risk of data loss.
|
|
|
|
- alert: PveNodeZfsPoolUnknown
|
|
expr: |-
|
|
pve_node_zfs_state == 0
|
|
for: 5m
|
|
labels:
|
|
severity: warning
|
|
annotations:
|
|
summary: Proxmox ZFS pool component state is unknown.
|
|
description: >-
|
|
ZFS component {{ $labels.path }} in pool {{ $labels.pool }} on node
|
|
{{ $labels.node }} in the {{ $labels.cluster }} cluster has been in
|
|
the UNKNOWN state for at least five minutes.
|
|
|
|
- alert: PveNodeZfsPoolDegraded
|
|
expr: |-
|
|
pve_node_zfs_state == 2
|
|
for: 5m
|
|
labels:
|
|
severity: warning
|
|
annotations:
|
|
summary: Proxmox ZFS pool component is degraded.
|
|
description: >-
|
|
ZFS component {{ $labels.path }} in pool {{ $labels.pool }} on node
|
|
{{ $labels.node }} in the {{ $labels.cluster }} cluster has been
|
|
DEGRADED for at least five minutes.
|
|
|
|
- alert: PveNodeZfsPoolFaulted
|
|
expr: |-
|
|
pve_node_zfs_state == 3
|
|
for: 5m
|
|
labels:
|
|
severity: critical
|
|
annotations:
|
|
summary: Proxmox ZFS pool component has faulted.
|
|
description: >-
|
|
ZFS component {{ $labels.path }} in pool {{ $labels.pool }} on node
|
|
{{ $labels.node }} in the {{ $labels.cluster }} cluster has been
|
|
FAULTED for at least five minutes.
|
|
|
|
- alert: PveNodeZfsPoolOffline
|
|
expr: |-
|
|
pve_node_zfs_state == 4
|
|
for: 5m
|
|
labels:
|
|
severity: critical
|
|
annotations:
|
|
summary: Proxmox ZFS pool component is offline.
|
|
description: >-
|
|
ZFS component {{ $labels.path }} in pool {{ $labels.pool }} on node
|
|
{{ $labels.node }} in the {{ $labels.cluster }} cluster has been
|
|
OFFLINE for at least five minutes.
|
|
|
|
- alert: PveNodeZfsPoolRemoved
|
|
expr: |-
|
|
pve_node_zfs_state == 5
|
|
for: 5m
|
|
labels:
|
|
severity: critical
|
|
annotations:
|
|
summary: Proxmox ZFS pool component was removed.
|
|
description: >-
|
|
ZFS component {{ $labels.path }} in pool {{ $labels.pool }} on node
|
|
{{ $labels.node }} in the {{ $labels.cluster }} cluster has been
|
|
REMOVED for at least five minutes.
|
|
|
|
- alert: PveNodeZfsPoolUnavailable
|
|
expr: |-
|
|
pve_node_zfs_state == 6
|
|
for: 5m
|
|
labels:
|
|
severity: critical
|
|
annotations:
|
|
summary: Proxmox ZFS pool component is unavailable.
|
|
description: >-
|
|
ZFS component {{ $labels.path }} in pool {{ $labels.pool }} on node
|
|
{{ $labels.node }} in the {{ $labels.cluster }} cluster has been
|
|
UNAVAIL for at least five minutes.
|
|
|
|
- alert: PveNodeZfsReadErrors
|
|
expr: |-
|
|
pve_node_zfs_read_errors_total > 0
|
|
for: 5m
|
|
labels:
|
|
severity: warning
|
|
annotations:
|
|
summary: Proxmox ZFS component reports read errors.
|
|
description: >-
|
|
ZFS component {{ $labels.path }} in pool {{ $labels.pool }} on node
|
|
{{ $labels.node }} in the {{ $labels.cluster }} cluster reports
|
|
{{ $value | humanize }} read errors.
|
|
|
|
- alert: PveNodeZfsWriteErrors
|
|
expr: |-
|
|
pve_node_zfs_write_errors_total > 0
|
|
for: 5m
|
|
labels:
|
|
severity: warning
|
|
annotations:
|
|
summary: Proxmox ZFS component reports write errors.
|
|
description: >-
|
|
ZFS component {{ $labels.path }} in pool {{ $labels.pool }} on node
|
|
{{ $labels.node }} in the {{ $labels.cluster }} cluster reports
|
|
{{ $value | humanize }} write errors.
|
|
|
|
- alert: PveNodeZfsChecksumErrors
|
|
expr: |-
|
|
pve_node_zfs_checksum_errors_total > 0
|
|
for: 5m
|
|
labels:
|
|
severity: warning
|
|
annotations:
|
|
summary: Proxmox ZFS component reports checksum errors.
|
|
description: >-
|
|
ZFS component {{ $labels.path }} in pool {{ $labels.pool }} on node
|
|
{{ $labels.node }} in the {{ $labels.cluster }} cluster reports
|
|
{{ $value | humanize }} checksum errors.
|
|
|
|
- alert: PveNodeMemoryOvercommit
|
|
expr: |-
|
|
(
|
|
sum by (cluster, node) (
|
|
(
|
|
pve_vm_memory_total_bytes
|
|
and on (cluster, node, vmid, name) (pve_vm_state == 1)
|
|
)
|
|
or
|
|
(
|
|
pve_ct_memory_total_bytes
|
|
and on (cluster, node, vmid, name) (pve_ct_state == 1)
|
|
)
|
|
)
|
|
/ pve_node_memory_total_bytes * 100 - 100
|
|
) > 20
|
|
for: 1m
|
|
labels:
|
|
severity: critical
|
|
annotations:
|
|
summary: Proxmox node memory is overcommitted by more than 20%.
|
|
description: >-
|
|
Node {{ $labels.node }} in the {{ $labels.cluster }} cluster has
|
|
{{ printf "%.2f" $value }}% more memory allocated to running guests
|
|
than is physically available. Concurrent guest memory demand can
|
|
exhaust node memory and disrupt services.
|
|
|
|
- alert: PveClusterQuorumLost
|
|
expr: |-
|
|
(pve_cluster_mode == 1)
|
|
and on (cluster)
|
|
(pve_cluster_quorate == 0)
|
|
for: 1m
|
|
labels:
|
|
severity: critical
|
|
annotations:
|
|
summary: Proxmox cluster has lost quorum.
|
|
description: >-
|
|
The {{ $labels.cluster }} cluster has not been quorate for at least
|
|
one minute. Cluster operations and guest availability may be
|
|
affected.
|
|
|
|
- alert: PveUnexpectedStandaloneMode
|
|
expr: |-
|
|
pve_cluster_mode == 0
|
|
for: 5m
|
|
labels:
|
|
severity: warning
|
|
annotations:
|
|
summary: Proxmox is running in standalone mode.
|
|
description: >-
|
|
{{ $labels.cluster }} has reported standalone mode for at least five
|
|
minutes. Disable this rule when standalone operation is expected.
|
|
|
|
- alert: PveNodeMemoryHigh
|
|
expr: |-
|
|
pve_node_memory_used_bytes / pve_node_memory_total_bytes > 0.90
|
|
for: 15m
|
|
labels:
|
|
severity: warning
|
|
annotations:
|
|
summary: Proxmox node memory usage is high.
|
|
description: >-
|
|
Node {{ $labels.node }} in the {{ $labels.cluster }} cluster has
|
|
used {{ $value | humanizePercentage }} of its memory for at least
|
|
15 minutes.
|
|
|
|
- alert: PveNodeFilesystemLowSpace
|
|
expr: |-
|
|
pve_node_rootfs_avail_bytes / pve_node_rootfs_total_bytes < 0.10
|
|
for: 5m
|
|
labels:
|
|
severity: critical
|
|
annotations:
|
|
summary: Proxmox root filesystem has less than 10% space available.
|
|
description: >-
|
|
The root filesystem on node {{ $labels.node }} in the
|
|
{{ $labels.cluster }} cluster has only
|
|
{{ $value | humanizePercentage }} available space left.
|
|
|
|
- alert: PveStorageLowSpace
|
|
expr: |-
|
|
pve_storage_avail_bytes / pve_storage_total_bytes < 0.10
|
|
for: 5m
|
|
labels:
|
|
severity: critical
|
|
annotations:
|
|
summary: Proxmox storage has less than 10% space available.
|
|
description: >-
|
|
Storage {{ $labels.storage }} on node {{ $labels.node }} in the
|
|
{{ $labels.cluster }} cluster has only
|
|
{{ $value | humanizePercentage }} available space left.
|
|
|
|
- alert: PveNodeSubscriptionExpired
|
|
expr: |-
|
|
pve_node_subscription_status == 2
|
|
for: 15m
|
|
labels:
|
|
severity: warning
|
|
annotations:
|
|
summary: Proxmox node subscription has expired or is unusable.
|
|
description: >-
|
|
Subscription for node {{ $labels.node }} in the
|
|
{{ $labels.cluster }} cluster is expired, invalid, suspended, or
|
|
otherwise unusable.
|
|
|
|
- alert: PveNodeSubscriptionMissing
|
|
expr: |-
|
|
pve_node_subscription_status == 0
|
|
for: 15m
|
|
labels:
|
|
severity: info
|
|
annotations:
|
|
summary: Proxmox node has no subscription.
|
|
description: >-
|
|
Node {{ $labels.node }} in the {{ $labels.cluster }} cluster reports
|
|
no subscription. Disable this rule when an unsubscribed node is
|
|
expected.
|
|
|
|
- alert: PveNodeDiskWearoutWarning
|
|
expr: |-
|
|
(pve_node_disk_wearout_percent < 20)
|
|
and
|
|
(pve_node_disk_wearout_percent >= 5)
|
|
for: 15m
|
|
labels:
|
|
severity: warning
|
|
annotations:
|
|
summary: Proxmox node disk has less than 20% wear life remaining.
|
|
description: |-
|
|
A disk on node {{ $labels.node }} in the {{ $labels.cluster }} cluster is approaching its wear limit.
|
|
WWN: {{ $labels.wwn }}
|
|
Serial: {{ $labels.serial }}
|
|
Model: {{ $labels.model }}
|
|
Current wear level: {{ $value }}%
|
|
|
|
- alert: PveGuestMemoryHigh
|
|
expr: |-
|
|
pve_vm_memory_used_bytes / pve_vm_memory_total_bytes > 0.90
|
|
or
|
|
pve_ct_memory_used_bytes / pve_ct_memory_total_bytes > 0.90
|
|
for: 15m
|
|
labels:
|
|
severity: warning
|
|
annotations:
|
|
summary: Proxmox guest memory usage is high.
|
|
description: >-
|
|
Guest {{ $labels.name }} with ID {{ $labels.vmid }} on node
|
|
{{ $labels.node }} in the {{ $labels.cluster }} cluster has used
|
|
{{ $value | humanizePercentage }} of its configured memory for at
|
|
least 15 minutes.
|
|
|
|
- alert: PveGuestDiskFillingUp
|
|
expr: |-
|
|
pve_vm_disk_used_bytes / pve_vm_disk_size_bytes > 0.90
|
|
or
|
|
pve_ct_disk_used_bytes / pve_ct_disk_size_bytes > 0.90
|
|
for: 15m
|
|
labels:
|
|
severity: warning
|
|
annotations:
|
|
summary: Proxmox guest root disk usage is high.
|
|
description: >-
|
|
Guest {{ $labels.name }} with ID {{ $labels.vmid }} on node
|
|
{{ $labels.node }} in the {{ $labels.cluster }} cluster has used
|
|
{{ $value | humanizePercentage }} of its configured root disk.
|
|
|
|
- alert: PveContainerSwapUsageHigh
|
|
expr: |-
|
|
pve_ct_swap_used_bytes > 256 * 1024 * 1024
|
|
for: 15m
|
|
labels:
|
|
severity: warning
|
|
annotations:
|
|
summary: Proxmox container swap usage is high.
|
|
description: >-
|
|
Container {{ $labels.name }} with ID {{ $labels.vmid }} on node
|
|
{{ $labels.node }} in the {{ $labels.cluster }} cluster has used
|
|
{{ $value | humanize1024 }}B of swap for at least 15 minutes.
|
|
|
|
- alert: PveVmDiskErrorsIncreasing
|
|
expr: |-
|
|
increase(pve_vm_disk_failed_read_operations_total[10m]) > 0
|
|
or
|
|
increase(pve_vm_disk_failed_write_operations_total[10m]) > 0
|
|
for: 1m
|
|
labels:
|
|
severity: critical
|
|
annotations:
|
|
summary: Proxmox virtual machine reports new disk I/O failures.
|
|
description: >-
|
|
VM {{ $labels.name }} with ID {{ $labels.vmid }} on node
|
|
{{ $labels.node }} in the {{ $labels.cluster }} cluster reports new
|
|
failed I/O operations on device {{ $labels.device }}.
|
|
|
|
- alert: PveNodeClockDrift
|
|
expr: |-
|
|
abs(pve_node_time_seconds - time()) > 60
|
|
for: 5m
|
|
labels:
|
|
severity: warning
|
|
annotations:
|
|
summary: Proxmox node clock differs from Prometheus by more than one minute.
|
|
description: >-
|
|
The clock on node {{ $labels.node }} in the {{ $labels.cluster }}
|
|
cluster differs from the Prometheus server by
|
|
{{ $value | humanizeDuration }}.
|
|
|
|
- alert: PveNodeRestarted
|
|
expr: |-
|
|
resets(pve_node_uptime_seconds[15m]) > 0
|
|
for: 0m
|
|
labels:
|
|
severity: info
|
|
annotations:
|
|
summary: Proxmox node restarted recently.
|
|
description: >-
|
|
Uptime for node {{ $labels.node }} in the {{ $labels.cluster }}
|
|
cluster was reset during the last 15 minutes.
|
|
|
|
- alert: PveCollectorSlow
|
|
expr: |-
|
|
rate(pve_metrics_collection_duration_seconds_sum[5m])
|
|
/ rate(pve_metrics_collection_duration_seconds_count[5m]) > 5
|
|
for: 10m
|
|
labels:
|
|
severity: warning
|
|
annotations:
|
|
summary: PVE exporter collector is slow.
|
|
description: >-
|
|
Collector {{ $labels.collector }} has taken an average of
|
|
{{ printf "%.1f" $value }} s per successful run for at least ten
|
|
minutes.
|
|
|
|
- alert: PveExporterDown
|
|
expr: |-
|
|
up{job="pve-exporter"} == 0
|
|
for: 2m
|
|
labels:
|
|
severity: critical
|
|
annotations:
|
|
summary: PVE Exporter is unreachable.
|
|
description: >-
|
|
Prometheus has been unable to scrape PVE Exporter instance
|
|
{{ $labels.instance }} for at least two minutes.
|