groups:
  - name: proxmox.rules
    rules:
      - alert: PveNodeFilesystemFillingUp
        expr: |-
          (
            pve_node_rootfs_avail_bytes / pve_node_rootfs_total_bytes * 100 < 15
          and
            predict_linear(pve_node_rootfs_avail_bytes[6h], 24 * 60 * 60) < 0
          )
        for: 1h
        labels:
          severity: critical
        annotations:
          summary: Proxmox root filesystem is predicted to run out of space within 24 hours.
          description: >-
            Filesystem on node {{ $labels.node }} in the {{ $labels.cluster }}
            cluster has only {{ printf "%.2f" $value }}% available space left
            and is filling up.

      - alert: PveGuestCpuSaturated
        expr: |-
          pve_vm_cpu_usage_ratio > 0.95 or pve_ct_cpu_usage_ratio > 0.95
        for: 1h
        labels:
          severity: warning
        annotations:
          summary: Proxmox guest CPU has been saturated for more than an hour.
          description: >-
            Guest {{ $labels.name }} with ID {{ $labels.vmid }} on node
            {{ $labels.node }} in the {{ $labels.cluster }} cluster has sustained
            CPU usage above 95%.

      - alert: PveStorageDown
        expr: |-
          pve_storage_up == 0
        for: 1m
        labels:
          severity: critical
        annotations:
          summary: Proxmox storage is down.
          description: >-
            Storage {{ $labels.storage }} on node {{ $labels.node }} in the
            {{ $labels.cluster }} cluster is down. This can severely impact
            running guests.

      - alert: PveStorageFillingUp
        expr: |-
          (
            pve_storage_avail_bytes / pve_storage_total_bytes * 100 < 15
          and
            predict_linear(pve_storage_avail_bytes[6h], 24 * 60 * 60) < 0
          )
        for: 1h
        labels:
          severity: critical
        annotations:
          summary: Proxmox storage is predicted to run out of space within 24 hours.
          description: >-
            Storage {{ $labels.storage }} on node {{ $labels.node }} in the
            {{ $labels.cluster }} cluster has only {{ printf "%.2f" $value }}%
            available space left and is filling up.

      - alert: PveNodeDown
        expr: |-
          pve_node_state == 0
        for: 1m
        labels:
          severity: critical
        annotations:
          summary: Proxmox node is down.
          description: >-
            Node {{ $labels.node }} in the {{ $labels.cluster }} cluster is down.

      - alert: PveNodeCpuSaturated
        expr: |-
          pve_node_cpu_usage_ratio > 0.95
        for: 1h
        labels:
          severity: warning
        annotations:
          summary: Proxmox node CPU has been saturated for more than an hour.
          description: >-
            CPU usage on node {{ $labels.node }} in the {{ $labels.cluster }}
            cluster has remained above 95%. This can affect guests running on
            the node.

      - alert: PveSdnDown
        expr: |-
          pve_sdn_state == 0
        for: 1m
        labels:
          severity: critical
        annotations:
          summary: Proxmox software-defined network is down.
          description: >-
            SDN resource {{ $labels.sdn }} on node {{ $labels.node }} in the
            {{ $labels.cluster }} cluster is down.

      - alert: PveClusterClockSkew
        expr: |-
          max by (cluster) (pve_node_localtime_seconds)
          - min by (cluster) (pve_node_localtime_seconds) > 15
        for: 1m
        labels:
          severity: critical
        annotations:
          summary: Proxmox cluster clock skew is too high.
          description: >-
            Clock skew between nodes in the {{ $labels.cluster }} cluster is
            greater than 15 seconds. Cluster functions may be affected.

      - alert: PveNodeSubscriptionExpiringSoon
        expr: |-
          (
            pve_node_subscription_next_due_timestamp_seconds > time()
          and
            pve_node_subscription_next_due_timestamp_seconds < time() + (30 * 24 * 60 * 60)
          )
          and on (cluster, node)
            (pve_node_subscription_status == 1)
        for: 15m
        labels:
          severity: warning
        annotations:
          summary: Proxmox node subscription expires in less than 30 days.
          description: >-
            Subscription for node {{ $labels.node }} in the
            {{ $labels.cluster }} cluster expires at
            {{ $value | humanizeTimestamp }}.

      - alert: PveNodeDiskFailed
        expr: |-
          pve_node_disk_healthy == 0
        for: 1m
        labels:
          severity: critical
        annotations:
          summary: Proxmox node disk failed.
          description: |-
            A disk on node {{ $labels.node }} in the {{ $labels.cluster }} cluster is in a failed state.
            WWN: {{ $labels.wwn }}
            Serial: {{ $labels.serial }}
            Model: {{ $labels.model }}

            Immediate device replacement is required to ensure safe operation of the node.

      - alert: PveNodeDiskWearout
        expr: |-
          pve_node_disk_wearout_percent < 5
        for: 1m
        labels:
          severity: critical
        annotations:
          summary: Proxmox node disk has less than 5% wear life remaining.
          description: |-
            A disk on node {{ $labels.node }} in the {{ $labels.cluster }} cluster is showing significant wear and may fail soon.
            WWN: {{ $labels.wwn }}
            Serial: {{ $labels.serial }}
            Model: {{ $labels.model }}
            Current wear level: {{ $value }}%

            Replace the disk to reduce the risk of data loss.

      - alert: PveNodeZfsPoolUnknown
        expr: |-
          pve_node_zfs_state == 0
        for: 5m
        labels:
          severity: warning
        annotations:
          summary: Proxmox ZFS pool component state is unknown.
          description: >-
            ZFS component {{ $labels.path }} in pool {{ $labels.pool }} on node
            {{ $labels.node }} in the {{ $labels.cluster }} cluster has been in
            the UNKNOWN state for at least five minutes.

      - alert: PveNodeZfsPoolDegraded
        expr: |-
          pve_node_zfs_state == 2
        for: 5m
        labels:
          severity: warning
        annotations:
          summary: Proxmox ZFS pool component is degraded.
          description: >-
            ZFS component {{ $labels.path }} in pool {{ $labels.pool }} on node
            {{ $labels.node }} in the {{ $labels.cluster }} cluster has been
            DEGRADED for at least five minutes.

      - alert: PveNodeZfsPoolFaulted
        expr: |-
          pve_node_zfs_state == 3
        for: 5m
        labels:
          severity: critical
        annotations:
          summary: Proxmox ZFS pool component has faulted.
          description: >-
            ZFS component {{ $labels.path }} in pool {{ $labels.pool }} on node
            {{ $labels.node }} in the {{ $labels.cluster }} cluster has been
            FAULTED for at least five minutes.

      - alert: PveNodeZfsPoolOffline
        expr: |-
          pve_node_zfs_state == 4
        for: 5m
        labels:
          severity: critical
        annotations:
          summary: Proxmox ZFS pool component is offline.
          description: >-
            ZFS component {{ $labels.path }} in pool {{ $labels.pool }} on node
            {{ $labels.node }} in the {{ $labels.cluster }} cluster has been
            OFFLINE for at least five minutes.

      - alert: PveNodeZfsPoolRemoved
        expr: |-
          pve_node_zfs_state == 5
        for: 5m
        labels:
          severity: critical
        annotations:
          summary: Proxmox ZFS pool component was removed.
          description: >-
            ZFS component {{ $labels.path }} in pool {{ $labels.pool }} on node
            {{ $labels.node }} in the {{ $labels.cluster }} cluster has been
            REMOVED for at least five minutes.

      - alert: PveNodeZfsPoolUnavailable
        expr: |-
          pve_node_zfs_state == 6
        for: 5m
        labels:
          severity: critical
        annotations:
          summary: Proxmox ZFS pool component is unavailable.
          description: >-
            ZFS component {{ $labels.path }} in pool {{ $labels.pool }} on node
            {{ $labels.node }} in the {{ $labels.cluster }} cluster has been
            UNAVAIL for at least five minutes.

      - alert: PveNodeZfsReadErrors
        expr: |-
          pve_node_zfs_read_errors_total > 0
        for: 5m
        labels:
          severity: warning
        annotations:
          summary: Proxmox ZFS component reports read errors.
          description: >-
            ZFS component {{ $labels.path }} in pool {{ $labels.pool }} on node
            {{ $labels.node }} in the {{ $labels.cluster }} cluster reports
            {{ $value | humanize }} read errors.

      - alert: PveNodeZfsWriteErrors
        expr: |-
          pve_node_zfs_write_errors_total > 0
        for: 5m
        labels:
          severity: warning
        annotations:
          summary: Proxmox ZFS component reports write errors.
          description: >-
            ZFS component {{ $labels.path }} in pool {{ $labels.pool }} on node
            {{ $labels.node }} in the {{ $labels.cluster }} cluster reports
            {{ $value | humanize }} write errors.

      - alert: PveNodeZfsChecksumErrors
        expr: |-
          pve_node_zfs_checksum_errors_total > 0
        for: 5m
        labels:
          severity: warning
        annotations:
          summary: Proxmox ZFS component reports checksum errors.
          description: >-
            ZFS component {{ $labels.path }} in pool {{ $labels.pool }} on node
            {{ $labels.node }} in the {{ $labels.cluster }} cluster reports
            {{ $value | humanize }} checksum errors.

      - alert: PveNodeMemoryOvercommit
        expr: |-
          (
            sum by (cluster, node) (
              (
                pve_vm_memory_total_bytes
                and on (cluster, node, vmid, name) (pve_vm_state == 1)
              )
              or
              (
                pve_ct_memory_total_bytes
                and on (cluster, node, vmid, name) (pve_ct_state == 1)
              )
            )
            / pve_node_memory_total_bytes * 100 - 100
          ) > 20
        for: 1m
        labels:
          severity: critical
        annotations:
          summary: Proxmox node memory is overcommitted by more than 20%.
          description: >-
            Node {{ $labels.node }} in the {{ $labels.cluster }} cluster has
            {{ printf "%.2f" $value }}% more memory allocated to running guests
            than is physically available. Concurrent guest memory demand can
            exhaust node memory and disrupt services.

      - alert: PveClusterQuorumLost
        expr: |-
          (pve_cluster_mode == 1)
          and on (cluster)
          (pve_cluster_quorate == 0)
        for: 1m
        labels:
          severity: critical
        annotations:
          summary: Proxmox cluster has lost quorum.
          description: >-
            The {{ $labels.cluster }} cluster has not been quorate for at least
            one minute. Cluster operations and guest availability may be
            affected.

      - alert: PveUnexpectedStandaloneMode
        expr: |-
          pve_cluster_mode == 0
        for: 5m
        labels:
          severity: warning
        annotations:
          summary: Proxmox is running in standalone mode.
          description: >-
            {{ $labels.cluster }} has reported standalone mode for at least five
            minutes. Disable this rule when standalone operation is expected.

      - alert: PveNodeMemoryHigh
        expr: |-
          pve_node_memory_used_bytes / pve_node_memory_total_bytes > 0.90
        for: 15m
        labels:
          severity: warning
        annotations:
          summary: Proxmox node memory usage is high.
          description: >-
            Node {{ $labels.node }} in the {{ $labels.cluster }} cluster has
            used {{ $value | humanizePercentage }} of its memory for at least
            15 minutes.

      - alert: PveNodeFilesystemLowSpace
        expr: |-
          pve_node_rootfs_avail_bytes / pve_node_rootfs_total_bytes < 0.10
        for: 5m
        labels:
          severity: critical
        annotations:
          summary: Proxmox root filesystem has less than 10% space available.
          description: >-
            The root filesystem on node {{ $labels.node }} in the
            {{ $labels.cluster }} cluster has only
            {{ $value | humanizePercentage }} available space left.

      - alert: PveStorageLowSpace
        expr: |-
          pve_storage_avail_bytes / pve_storage_total_bytes < 0.10
        for: 5m
        labels:
          severity: critical
        annotations:
          summary: Proxmox storage has less than 10% space available.
          description: >-
            Storage {{ $labels.storage }} on node {{ $labels.node }} in the
            {{ $labels.cluster }} cluster has only
            {{ $value | humanizePercentage }} available space left.

      - alert: PveNodeSubscriptionExpired
        expr: |-
          pve_node_subscription_status == 2
        for: 15m
        labels:
          severity: warning
        annotations:
          summary: Proxmox node subscription has expired or is unusable.
          description: >-
            Subscription for node {{ $labels.node }} in the
            {{ $labels.cluster }} cluster is expired, invalid, suspended, or
            otherwise unusable.

      - alert: PveNodeSubscriptionMissing
        expr: |-
          pve_node_subscription_status == 0
        for: 15m
        labels:
          severity: info
        annotations:
          summary: Proxmox node has no subscription.
          description: >-
            Node {{ $labels.node }} in the {{ $labels.cluster }} cluster reports
            no subscription. Disable this rule when an unsubscribed node is
            expected.

      - alert: PveNodeDiskWearoutWarning
        expr: |-
          (pve_node_disk_wearout_percent < 20)
          and
          (pve_node_disk_wearout_percent >= 5)
        for: 15m
        labels:
          severity: warning
        annotations:
          summary: Proxmox node disk has less than 20% wear life remaining.
          description: |-
            A disk on node {{ $labels.node }} in the {{ $labels.cluster }} cluster is approaching its wear limit.
            WWN: {{ $labels.wwn }}
            Serial: {{ $labels.serial }}
            Model: {{ $labels.model }}
            Current wear level: {{ $value }}%

      - alert: PveGuestMemoryHigh
        expr: |-
          pve_vm_memory_used_bytes / pve_vm_memory_total_bytes > 0.90
          or
          pve_ct_memory_used_bytes / pve_ct_memory_total_bytes > 0.90
        for: 15m
        labels:
          severity: warning
        annotations:
          summary: Proxmox guest memory usage is high.
          description: >-
            Guest {{ $labels.name }} with ID {{ $labels.vmid }} on node
            {{ $labels.node }} in the {{ $labels.cluster }} cluster has used
            {{ $value | humanizePercentage }} of its configured memory for at
            least 15 minutes.

      - alert: PveGuestDiskFillingUp
        expr: |-
          pve_vm_disk_used_bytes / pve_vm_disk_size_bytes > 0.90
          or
          pve_ct_disk_used_bytes / pve_ct_disk_size_bytes > 0.90
        for: 15m
        labels:
          severity: warning
        annotations:
          summary: Proxmox guest root disk usage is high.
          description: >-
            Guest {{ $labels.name }} with ID {{ $labels.vmid }} on node
            {{ $labels.node }} in the {{ $labels.cluster }} cluster has used
            {{ $value | humanizePercentage }} of its configured root disk.

      - alert: PveContainerSwapUsageHigh
        expr: |-
          pve_ct_swap_used_bytes > 256 * 1024 * 1024
        for: 15m
        labels:
          severity: warning
        annotations:
          summary: Proxmox container swap usage is high.
          description: >-
            Container {{ $labels.name }} with ID {{ $labels.vmid }} on node
            {{ $labels.node }} in the {{ $labels.cluster }} cluster has used
            {{ $value | humanize1024 }}B of swap for at least 15 minutes.

      - alert: PveVmDiskErrorsIncreasing
        expr: |-
          increase(pve_vm_disk_failed_read_operations_total[10m]) > 0
          or
          increase(pve_vm_disk_failed_write_operations_total[10m]) > 0
        for: 1m
        labels:
          severity: critical
        annotations:
          summary: Proxmox virtual machine reports new disk I/O failures.
          description: >-
            VM {{ $labels.name }} with ID {{ $labels.vmid }} on node
            {{ $labels.node }} in the {{ $labels.cluster }} cluster reports new
            failed I/O operations on device {{ $labels.device }}.

      - alert: PveNodeClockDrift
        expr: |-
          abs(pve_node_time_seconds - time()) > 60
        for: 5m
        labels:
          severity: warning
        annotations:
          summary: Proxmox node clock differs from Prometheus by more than one minute.
          description: >-
            The clock on node {{ $labels.node }} in the {{ $labels.cluster }}
            cluster differs from the Prometheus server by
            {{ $value | humanizeDuration }}.

      - alert: PveNodeRestarted
        expr: |-
          resets(pve_node_uptime_seconds[15m]) > 0
        for: 0m
        labels:
          severity: info
        annotations:
          summary: Proxmox node restarted recently.
          description: >-
            Uptime for node {{ $labels.node }} in the {{ $labels.cluster }}
            cluster was reset during the last 15 minutes.

      - alert: PveCollectorSlow
        expr: |-
          rate(pve_metrics_collection_duration_seconds_sum[5m])
          / rate(pve_metrics_collection_duration_seconds_count[5m]) > 5
        for: 10m
        labels:
          severity: warning
        annotations:
          summary: PVE exporter collector is slow.
          description: >-
            Collector {{ $labels.collector }} has taken an average of
            {{ printf "%.1f" $value }} s per successful run for at least ten
            minutes.

      - alert: PveExporterDown
        expr: |-
          up{job="pve-exporter"} == 0
        for: 2m
        labels:
          severity: critical
        annotations:
          summary: PVE Exporter is unreachable.
          description: >-
            Prometheus has been unable to scrape PVE Exporter instance
            {{ $labels.instance }} for at least two minutes.