Extended ZFS states

README.md

@@ -4,7 +4,7 @@ Proxmox Virtual Environment Prometheus metrics exporter.
 
 ## Overview
 
-PVE Exporter is a tool that collects metrics from a Proxmox Virtual Environment cluster and exposes them for Prometheus to scrape. This exporter supports gathering metrics for cluster state, LXC containers, QEMU virtual machines, physical disks, node storage, node status, node subscription details, and software-defined networking (SDN).
+PVE Exporter is a tool that collects metrics from a Proxmox Virtual Environment cluster and exposes them for Prometheus to scrape. This exporter supports gathering metrics for cluster state, LXC containers, QEMU virtual machines, physical disks, ZFS pools, node storage, node status, node subscription details, and software-defined networking (SDN).
 
 ## Features
 
@@ -52,18 +52,35 @@ metrics:
   ltc: true            # Enable collection of LXC container metrics.
   qemu: true           # Enable collection of QEMU virtual machine metrics.
   disk: true           # Enable collection of physical disk metrics.
+  zfs: true            # Enable collection of ZFS pool metrics.
   storage: true        # Enable collection of node storage metrics.
   nodeStatus: true     # Enable collection of node status metrics.
   subscription: true   # Enable collection of node subscription details.
   sdn: true            # Enable collection of software-defined network (SDN) metrics.
 ```
 
+The cluster state collector exposes `pve_cluster_mode`, where `1` means that
+Proxmox VE is configured as a cluster and `0` means that it is a standalone node.
+
+The `pve_node_subscription_status` metric uses `0` when no subscription is
+configured, `1` for an active subscription, and `2` for an expired or otherwise
+unusable subscription.
+
+The ZFS collector discovers all pools on every node and exports their recursive
+topology with `cluster`, `node`, `pool`, `component`, `path`, and `leaf` labels:
+
+- `pve_node_zfs_state` (`0=UNKNOWN`, `1=ONLINE`, `2=DEGRADED`, `3=FAULTED`,
+  `4=OFFLINE`, `5=REMOVED`, `6=UNAVAIL`)
+- `pve_node_zfs_read_errors`
+- `pve_node_zfs_write_errors`
+- `pve_node_zfs_checksum_errors`
+
 ## Build
 
 To build the Docker image for PVE Exporter, use the following command:
 
 ```sh
-docker build --rm -t harbor.imtm.cz/private/pve-exporter:version .
+docker build --rm -t gitea.lostak.dev/lostakj/pve-exporter:version .
 ```
 
 Replace `version` with the appropriate version tag you want to use.
@@ -73,7 +90,7 @@ Replace `version` with the appropriate version tag you want to use.
 To run the Docker image, use the following command:
 
 ```sh
-docker run --rm -d -p 9090:9090 --name pve-exporter harbor.imtm.cz/private/pve-exporter:version
+docker run --rm -d -p 9090:9090 --name pve-exporter gitea.lostak.dev/lostakj/pve-exporter:version
 ```
 
 Replace `version` with the appropriate version tag you used during the build.
@@ -101,4 +118,4 @@ Contributions are welcome! Please open an issue or submit a pull request if you
 
 ## Support
 
-If you encounter any issues or have questions, please open an issue on the project's GitHub repository.
+If you encounter any issues or have questions, please open an issue on the project's GitHub repository.

config.example.yaml

@@ -35,6 +35,8 @@ proxmox:
     qemu: true
     # Enable collection of physical disk metrics.
     disk: true
+    # Enable collection of ZFS pool metrics.
+    zfs: true
     # Enable collection of node storage metrics.
     storage: true
     # Enable collection of node status metrics.
@@ -42,4 +44,4 @@ proxmox:
     # Enable collection of node subscription details.
     subscription: true
     # Enable collection of software-defined network (SDN) metrics.
-    sdn: true
+    sdn: true

configuration/configuration.go

@@ -27,6 +27,7 @@ type PveMetricsConfiguration struct {
 	LXC          bool `yaml:"lxc"`          // Enables LXC container metrics collection.
 	QEMU         bool `yaml:"qemu"`         // Enable QEMU virtual machine metrics collection.
 	Disk         bool `yaml:"disk"`         // Enable physical disk metrics collection.
+	ZFS          bool `yaml:"zfs"`          // Enable node ZFS pool metrics collection.
 	Storage      bool `yaml:"storage"`      // Enable node storage metrics collection.
 	NodeStatus   bool `yaml:"nodeStatus"`   // Enable node status metrics collection.
 	Subscription bool `yaml:"subscription"` // Enable node subscription detail collection.

metrics/pve_cluster_state_collector.go

@@ -12,8 +12,9 @@ type PveClusterStateCollector struct {
 	apiClient *proxmox.PveApiClient // PVE API client instance.
 	registry  *TTLRegistry          // TTL metrics registry.
 
-	nodes   *TTLGaugeVec // Count of nodes prometheus gauge.
-	quorate *TTLGaugeVec // Cluster quorum state prometheus gauge.
+	nodes       *TTLGaugeVec // Count of nodes prometheus gauge.
+	quorate     *TTLGaugeVec // Cluster quorum state prometheus gauge.
+	clusterMode *TTLGaugeVec // PVE cluster mode prometheus gauge.
 }
 
 // Create new instance of PVE cluster state collector.
@@ -42,6 +43,17 @@ func NewPveClusterStateCollector(apiClient *proxmox.PveApiClient, registry *TTLR
 		5*time.Minute,
 	)
 	c.registry.Register(c.quorate)
+
+	// PVE cluster mode gauge.
+	c.clusterMode = NewTTLGaugeVec(
+		prometheus.GaugeOpts{
+			Name: "pve_cluster_mode",
+			Help: "PVE cluster mode (1 = cluster, 0 = standalone node).",
+		},
+		[]string{"cluster"},
+		5*time.Minute,
+	)
+	c.registry.Register(c.clusterMode)
 	c.registry.StartCleanupLoop(5 * time.Second)
 
 	return &c
@@ -54,9 +66,10 @@ func (c *PveClusterStateCollector) CollectMetrics() error {
 		return err
 	}
 
-	l := prometheus.Labels{"cluster": cluster.Name}
+	l := prometheus.Labels{"cluster": cluster.GetClusterName()}
 	c.nodes.With(l).Set(float64(cluster.Nodes))
 	c.quorate.With(l).Set(float64(cluster.Quorate))
+	c.clusterMode.With(l).Set(cluster.GetClusterModeNumeric())
 
 	return nil
 }

metrics/pve_metrics_manager.go

@@ -55,6 +55,11 @@ func NewPveMetricsManager(apiClient *proxmox.PveApiClient, conf *configuration.P
 		c.RegisterCollector(NewPveNodeDiskCollector(apiClient, c.registry))
 	}
 
+	// Node ZFS pool collector.
+	if metricsCf.ZFS {
+		c.RegisterCollector(NewPveNodeZfsCollector(apiClient, c.registry))
+	}
+
 	// Node SDN collector.
 	if metricsCf.SDN {
 		c.RegisterCollector(NewPveSdnCollector(apiClient, c.registry))

metrics/pve_node_subscription_collector.go

@@ -39,7 +39,7 @@ func NewPveSubscriptionCollector(apiClient *proxmox.PveApiClient, registry *TTLR
 	c.status = NewTTLGaugeVec(
 		prometheus.GaugeOpts{
 			Name: "pve_node_subscription_status",
-			Help: "Node subscription status.",
+			Help: "Node subscription status (0 = not found, 1 = active, 2 = expired or otherwise unusable).",
 		},
 		[]string{"cluster", "node"},
 		5*time.Minute,
@@ -111,7 +111,7 @@ func (c *PveSubscriptionCollector) CollectMetrics() error {
 			c.info.With(subsLabels).Set(1)
 
 			// Subscription state.
-			c.status.With(labels).Set(subscription.GetActiveNumeric())
+			c.status.With(labels).Set(subscription.GetStatusNumeric())
 
 			// Subscription sockets count.
 			c.sockets.With(labels).Set(float64(subscription.Sockets))

metrics/pve_node_zfs_collector.go

@@ -0,0 +1,162 @@
+package metrics
+
+import (
+	"strconv"
+	"strings"
+	"time"
+
+	"github.com/prometheus/client_golang/prometheus"
+	"lostak.dev/pve-exporter/proxmox"
+)
+
+// PveNodeZfsCollector collects ZFS pool and component health metrics.
+type PveNodeZfsCollector struct {
+	apiClient *proxmox.PveApiClient
+	registry  *TTLRegistry
+
+	state          *TTLGaugeVec
+	readErrors     *TTLGaugeVec
+	writeErrors    *TTLGaugeVec
+	checksumErrors *TTLGaugeVec
+}
+
+// zfsMetricComponent is a flattened entry from the recursive ZFS topology.
+type zfsMetricComponent struct {
+	proxmox.PveZfsComponent
+	Path string
+}
+
+var zfsStateValues = map[string]float64{
+	"ONLINE":   1,
+	"DEGRADED": 2,
+	"FAULTED":  3,
+	"OFFLINE":  4,
+	"REMOVED":  5,
+	"UNAVAIL":  6,
+}
+
+// NewPveNodeZfsCollector creates a ZFS metrics collector.
+func NewPveNodeZfsCollector(apiClient *proxmox.PveApiClient, registry *TTLRegistry) *PveNodeZfsCollector {
+	c := PveNodeZfsCollector{apiClient: apiClient, registry: registry}
+	componentLabelNames := []string{"cluster", "node", "pool", "component", "path", "leaf"}
+
+	c.state = NewTTLGaugeVec(
+		prometheus.GaugeOpts{
+			Name: "pve_node_zfs_state",
+			Help: "ZFS pool component state (0 = UNKNOWN, 1 = ONLINE, 2 = DEGRADED, 3 = FAULTED, 4 = OFFLINE, 5 = REMOVED, 6 = UNAVAIL).",
+		},
+		componentLabelNames,
+		5*time.Minute,
+	)
+	c.registry.Register(c.state)
+
+	c.readErrors = NewTTLGaugeVec(
+		prometheus.GaugeOpts{
+			Name: "pve_node_zfs_read_errors",
+			Help: "ZFS pool component read error count.",
+		},
+		componentLabelNames,
+		5*time.Minute,
+	)
+	c.registry.Register(c.readErrors)
+
+	c.writeErrors = NewTTLGaugeVec(
+		prometheus.GaugeOpts{
+			Name: "pve_node_zfs_write_errors",
+			Help: "ZFS pool component write error count.",
+		},
+		componentLabelNames,
+		5*time.Minute,
+	)
+	c.registry.Register(c.writeErrors)
+
+	c.checksumErrors = NewTTLGaugeVec(
+		prometheus.GaugeOpts{
+			Name: "pve_node_zfs_checksum_errors",
+			Help: "ZFS pool component checksum error count.",
+		},
+		componentLabelNames,
+		5*time.Minute,
+	)
+	c.registry.Register(c.checksumErrors)
+
+	return &c
+}
+
+// CollectMetrics implements PveMetricsCollector.
+func (c *PveNodeZfsCollector) CollectMetrics() error {
+	cluster, err := c.apiClient.GetClusterStatus()
+	if err != nil {
+		return err
+	}
+
+	for _, node := range cluster.NodeStatuses {
+		pools, err := c.apiClient.GetNodeZfsPools(node.Name)
+		if err != nil {
+			return err
+		}
+
+		for _, pool := range *pools {
+			status, err := c.apiClient.GetNodeZfsPoolStatus(node.Name, pool.Name)
+			if err != nil {
+				return err
+			}
+
+			for _, component := range flattenZfsComponents(status) {
+				labels := prometheus.Labels{
+					"cluster":   cluster.GetClusterName(),
+					"node":      node.Name,
+					"pool":      pool.Name,
+					"component": component.Name,
+					"path":      component.Path,
+					"leaf":      strconv.FormatBool(component.Leaf == 1),
+				}
+
+				if component.State != "" {
+					c.state.With(labels).Set(zfsStateNumeric(component.State))
+				}
+				if component.Read != nil {
+					c.readErrors.With(labels).Set(float64(*component.Read))
+				}
+				if component.Write != nil {
+					c.writeErrors.With(labels).Set(float64(*component.Write))
+				}
+				if component.Checksum != nil {
+					c.checksumErrors.With(labels).Set(float64(*component.Checksum))
+				}
+			}
+		}
+	}
+
+	return nil
+}
+
+// GetName implements PveMetricsCollector.
+func (c *PveNodeZfsCollector) GetName() string {
+	return "Node ZFS"
+}
+
+func flattenZfsComponents(status *proxmox.PveZfsPoolStatus) []zfsMetricComponent {
+	components := make([]zfsMetricComponent, 0)
+	appendZfsComponent(&components, status.PveZfsComponent, nil)
+	return components
+}
+
+func appendZfsComponent(components *[]zfsMetricComponent, component proxmox.PveZfsComponent, parents []string) {
+	pathParts := append(append([]string{}, parents...), component.Name)
+	*components = append(*components, zfsMetricComponent{
+		PveZfsComponent: component,
+		Path:            strings.Join(pathParts, " > "),
+	})
+
+	for _, child := range component.Children {
+		appendZfsComponent(components, child, pathParts)
+	}
+}
+
+func zfsStateNumeric(state string) float64 {
+	if value, ok := zfsStateValues[strings.ToUpper(state)]; ok {
+		return value
+	}
+	return 0
+}

metrics/pve_node_zfs_collector_test.go

@@ -0,0 +1,78 @@
+package metrics
+
+import (
+	"testing"
+
+	"lostak.dev/pve-exporter/proxmox"
+)
+
+func TestFlattenZfsComponents(t *testing.T) {
+	zero := uint64(0)
+	status := &proxmox.PveZfsPoolStatus{
+		PveZfsComponent: proxmox.PveZfsComponent{
+			Name:  "hdd",
+			State: "ONLINE",
+			Children: []proxmox.PveZfsComponent{
+				{
+					Name: "hdd",
+					Children: []proxmox.PveZfsComponent{
+						{
+							Name:     "raidz1-0",
+							State:    "ONLINE",
+							Read:     &zero,
+							Write:    &zero,
+							Checksum: &zero,
+							Children: []proxmox.PveZfsComponent{
+								{
+									Name:     "/dev/disk/by-id/disk-1-part1",
+									State:    "ONLINE",
+									Leaf:     1,
+									Read:     &zero,
+									Write:    &zero,
+									Checksum: &zero,
+								},
+							},
+						},
+					},
+				},
+			},
+		},
+	}
+
+	components := flattenZfsComponents(status)
+	if len(components) != 4 {
+		t.Fatalf("flattenZfsComponents() returned %d components, want 4", len(components))
+	}
+
+	leaf := components[3]
+	wantPath := "hdd > hdd > raidz1-0 > /dev/disk/by-id/disk-1-part1"
+	if leaf.Path != wantPath {
+		t.Fatalf("leaf path = %q, want %q", leaf.Path, wantPath)
+	}
+	if leaf.Leaf != 1 || leaf.Read == nil || leaf.Write == nil || leaf.Checksum == nil {
+		t.Fatal("leaf component lost its type or error counters")
+	}
+}
+
+func TestZfsStateNumeric(t *testing.T) {
+	tests := []struct {
+		state string
+		want  float64
+	}{
+		{state: "UNKNOWN", want: 0},
+		{state: "ONLINE", want: 1},
+		{state: "DEGRADED", want: 2},
+		{state: "FAULTED", want: 3},
+		{state: "OFFLINE", want: 4},
+		{state: "REMOVED", want: 5},
+		{state: "UNAVAIL", want: 6},
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.state, func(t *testing.T) {
+			if got := zfsStateNumeric(tt.state); got != tt.want {
+				t.Fatalf("zfsStateNumeric(%q) = %v, want %v", tt.state, got, tt.want)
+			}
+		})
+	}
+}

proxmox/model.go

@@ -223,6 +223,40 @@ type PveDisk struct {
 	Used      string      `json:"used,omitempty"`    // How the drive is used (optional field).
 }
 
+// PveZfsPool represents summary information about a ZFS pool.
+type PveZfsPool struct {
+	Name   string  `json:"name"`   // Pool name.
+	Size   uint64  `json:"size"`   // Total pool size in bytes.
+	Alloc  uint64  `json:"alloc"`  // Allocated pool space in bytes.
+	Free   uint64  `json:"free"`   // Free pool space in bytes.
+	Frag   uint64  `json:"frag"`   // Pool fragmentation percentage.
+	Dedup  float64 `json:"dedup"`  // Pool deduplication ratio.
+	Health string  `json:"health"` // Pool health state.
+}
+
+// PveZfsComponent represents a component in the recursive ZFS pool topology.
+// Error counters are pointers because section nodes such as "cache" do not
+// always contain read, write, or checksum values in the PVE API response.
+type PveZfsComponent struct {
+	Name     string            `json:"name"`     // Component, vdev, or device name.
+	State    string            `json:"state"`    // Component state, for example ONLINE.
+	Leaf     uint8             `json:"leaf"`     // Whether this component is a leaf device.
+	Read     *uint64           `json:"read"`     // Read error count when available.
+	Write    *uint64           `json:"write"`    // Write error count when available.
+	Checksum *uint64           `json:"cksum"`    // Checksum error count when available.
+	Message  string            `json:"msg"`      // Optional component status message.
+	Children []PveZfsComponent `json:"children"` // Nested ZFS components.
+}
+
+// PveZfsPoolStatus represents the detailed status and topology of a ZFS pool.
+type PveZfsPoolStatus struct {
+	PveZfsComponent
+	Status string `json:"status"` // Human-readable pool status.
+	Action string `json:"action"` // Recommended recovery action.
+	Scan   string `json:"scan"`   // Last or current scrub information.
+	Errors string `json:"errors"` // Human-readable pool errors.
+}
+
 // PVE node time.
 type PveNodeTime struct {
 	Time      uint64 `json:"time"`      // Unix timestamp in UTC.
@@ -430,13 +464,18 @@ func (r *PveSdnResource) GetStatusNumeric() float64 {
 	return 0
 }
 
-// GetActiveNumeric returns the numeric state of a subscription.
-// Returns 1 if the subscription status is "active", otherwise returns 0.
-func (r *PveSubscription) GetActiveNumeric() float64 {
-	if r.Status == "active" {
+// GetStatusNumeric returns the numeric state of a subscription.
+// A missing subscription is 0, an active subscription is 1, and an expired
+// or otherwise unusable subscription is 2.
+func (r *PveSubscription) GetStatusNumeric() float64 {
+	switch r.Status {
+	case "notfound":
+		return 0
+	case "active":
 		return 1
+	default:
+		return 2
 	}
-	return 0
 }
 
 // GetSmartPassedState returns the numeric health state of a disk.
@@ -486,6 +525,15 @@ func (r *PveQemuStatus) IsRunning() bool {
 	return false
 }
 
+// GetClusterModeNumeric returns whether the PVE installation is configured as a cluster.
+// The cluster status API includes an object with type "cluster" only in cluster mode.
+func (r *PveClusterStatus) GetClusterModeNumeric() float64 {
+	if r.Type == "cluster" {
+		return 1
+	}
+	return 0
+}
+
 // GetClusterName returns name of cluster for labeling purposes.
 // If there is no cluster then return standalone node with node name. Otherwise, it returns cluster name.
 func (r *PveClusterStatus) GetClusterName() string {

proxmox/model_test.go

@@ -0,0 +1,114 @@
+package proxmox
+
+import (
+	"encoding/json"
+	"testing"
+)
+
+func TestPveClusterStatusGetClusterModeNumeric(t *testing.T) {
+	tests := []struct {
+		name   string
+		status PveClusterStatus
+		want   float64
+	}{
+		{
+			name:   "cluster mode",
+			status: PveClusterStatus{Type: "cluster"},
+			want:   1,
+		},
+		{
+			name:   "standalone node",
+			status: PveClusterStatus{},
+			want:   0,
+		},
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			if got := tt.status.GetClusterModeNumeric(); got != tt.want {
+				t.Fatalf("GetClusterModeNumeric() = %v, want %v", got, tt.want)
+			}
+		})
+	}
+}
+
+func TestPveSubscriptionGetStatusNumeric(t *testing.T) {
+	tests := []struct {
+		status string
+		want   float64
+	}{
+		{status: "notfound", want: 0},
+		{status: "active", want: 1},
+		{status: "expired", want: 2},
+		{status: "invalid", want: 2},
+		{status: "suspended", want: 2},
+		{status: "new", want: 2},
+		{status: "unknown", want: 2},
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.status, func(t *testing.T) {
+			subscription := PveSubscription{Status: tt.status}
+			if got := subscription.GetStatusNumeric(); got != tt.want {
+				t.Fatalf("GetStatusNumeric() = %v, want %v", got, tt.want)
+			}
+		})
+	}
+}
+
+func TestPveZfsPoolStatusUnmarshal(t *testing.T) {
+	payload := []byte(`{
+		"errors": "No known data errors",
+		"state": "ONLINE",
+		"leaf": 0,
+		"name": "hdd",
+		"children": [
+			{
+				"name": "hdd",
+				"state": "ONLINE",
+				"read": 0,
+				"write": 0,
+				"cksum": 0,
+				"children": [
+					{
+						"name": "/dev/disk/by-id/disk-1-part1",
+						"state": "ONLINE",
+						"leaf": 1,
+						"read": 0,
+						"write": 0,
+						"cksum": 0
+					}
+				]
+			},
+			{
+				"name": "cache",
+				"leaf": 0
+			}
+		]
+	}`)
+
+	var status PveZfsPoolStatus
+	if err := json.Unmarshal(payload, &status); err != nil {
+		t.Fatalf("unable to unmarshal ZFS pool status: %v", err)
+	}
+
+	if status.Name != "hdd" || status.State != "ONLINE" || status.Errors != "No known data errors" {
+		t.Fatalf("unexpected pool status: %+v", status)
+	}
+	if len(status.Children) != 2 || len(status.Children[0].Children) != 1 {
+		t.Fatalf("unexpected ZFS topology: %+v", status.Children)
+	}
+
+	leaf := status.Children[0].Children[0]
+	if leaf.Leaf != 1 || leaf.Read == nil || leaf.Write == nil || leaf.Checksum == nil {
+		t.Fatal("leaf component is missing its type or error counters")
+	}
+	if *leaf.Read != 0 || *leaf.Write != 0 || *leaf.Checksum != 0 {
+		t.Fatalf("unexpected leaf error counters: read=%d write=%d checksum=%d", *leaf.Read, *leaf.Write, *leaf.Checksum)
+	}
+
+	cache := status.Children[1]
+	if cache.Read != nil || cache.Write != nil || cache.Checksum != nil {
+		t.Fatal("missing cache counters must stay absent instead of becoming zero metrics")
+	}
+}

proxmox/pve_api_client.go

@@ -4,6 +4,7 @@ import (
 	"encoding/json"
 	"errors"
 	"fmt"
+	"net/url"
 	"time"
 
 	"github.com/mitchellh/mapstructure"
@@ -212,6 +213,37 @@ func (instance *PveApiClient) GetNodeDisksList(node string) (*[]PveDisk, error)
 	return &disks, nil
 }
 
+// Get ZFS pools configured on a PVE node.
+func (instance *PveApiClient) GetNodeZfsPools(node string) (*[]PveZfsPool, error) {
+	res, err := instance.apiClient.PerformRequest("GET", "/nodes/"+node+"/disks/zfs", nil)
+	if err != nil {
+		return nil, err
+	}
+
+	var pools []PveZfsPool
+	if err := json.Unmarshal(res.Data, &pools); err != nil {
+		return nil, err
+	}
+
+	return &pools, nil
+}
+
+// Get detailed status and topology of a ZFS pool on a PVE node.
+func (instance *PveApiClient) GetNodeZfsPoolStatus(node string, pool string) (*PveZfsPoolStatus, error) {
+	path := "/nodes/" + node + "/disks/zfs/" + url.PathEscape(pool)
+	res, err := instance.apiClient.PerformRequest("GET", path, nil)
+	if err != nil {
+		return nil, err
+	}
+
+	var status PveZfsPoolStatus
+	if err := json.Unmarshal(res.Data, &status); err != nil {
+		return nil, err
+	}
+
+	return &status, nil
+}
+
 // Get PVE node time info.
 func (instance *PveApiClient) GetNodeTime(node string) (*PveNodeTime, error) {
 	res, err := instance.apiClient.PerformRequest("GET", "/nodes/"+node+"/time", nil)