Align metric names with Prometheus naming conventions

Use base units, export cumulative values as _total counters, update docs, dashboards and rules.

metrics/pve_metrics_manager.go

@@ -23,8 +23,8 @@ type PveMetricsManager struct {
 	collectors []PveMetricsCollector // Metrics collector instances.
 	registry   *TTLRegistry          // Registry which handles autoamtic dangling metrics deletion.
 
-	latencySummary *prometheus.SummaryVec // Collection latency summary.
-	interval       int                    // Collection interval.
+	durationSummary *prometheus.SummaryVec // Collection duration summary.
+	interval        int                    // Collection interval.
 
 	stop chan struct{} // Stop channel which is used in ticker.
 }
@@ -80,10 +80,10 @@ func NewPveMetricsManager(apiClient *proxmox.PveApiClient, conf *configuration.P
 		c.RegisterCollector(NewPveVirtualMachineCollector(apiClient, c.registry))
 	}
 
-	// Metrics collection latency summary.
-	c.latencySummary = promauto.NewSummaryVec(prometheus.SummaryOpts{
-		Name: "pve_metrics_collection_latency_ms",
-		Help: "Summary of metrics collection latency milliseconds from PVE API.",
+	// Metrics collection duration summary.
+	c.durationSummary = promauto.NewSummaryVec(prometheus.SummaryOpts{
+		Name: "pve_metrics_collection_duration_seconds",
+		Help: "Summary of the PVE API metrics collection duration in seconds.",
 	}, []string{"collector"})
 
 	c.registry.StartCleanupLoop(5 * time.Second)
@@ -102,7 +102,7 @@ func (c *PveMetricsManager) collectMetrics() {
 		} else {
 			latency := time.Since(start)
 			log.Tracef("Finished collecting '%s' metrics after %s.", collector.GetName(), utils.HumanDuration(latency))
-			c.latencySummary.With(prometheus.Labels{"collector": collector.GetName()}).Observe(float64(latency.Milliseconds()))
+			c.durationSummary.With(prometheus.Labels{"collector": collector.GetName()}).Observe(latency.Seconds())
 		}
 	}
 }

metrics/pve_metrics_naming_test.go

@@ -0,0 +1,72 @@
+package metrics
+
+import (
+	"testing"
+
+	"github.com/prometheus/client_golang/prometheus"
+	"github.com/prometheus/client_golang/prometheus/testutil/promlint"
+)
+
+// Verifies that all exported PVE metrics follow the Prometheus naming conventions
+// (base units, '_total' suffix on counters, '_info' suffix on info metrics, ...)
+// so that tools such as promtool or Grafana do not report naming problems.
+func TestPveMetricNamesFollowPrometheusConventions(t *testing.T) {
+	registry := prometheus.NewRegistry()
+
+	// Collectors register their metrics on the default registerer, so it is
+	// temporarily replaced by a dedicated registry.
+	defaultRegisterer := prometheus.DefaultRegisterer
+	prometheus.DefaultRegisterer = registry
+	defer func() { prometheus.DefaultRegisterer = defaultRegisterer }()
+
+	// Collector constructors only create metrics, the API client is used during
+	// collection only, so a nil client is enough here.
+	ttlRegistry := NewTTLRegistry()
+	NewPveClusterStateCollector(nil, ttlRegistry)
+	NewPveNodeStatusCollector(nil, ttlRegistry)
+	NewPveSubscriptionCollector(nil, ttlRegistry)
+	NewPveNodeDiskCollector(nil, ttlRegistry)
+	NewPveNodeZfsCollector(nil, ttlRegistry)
+	NewPveSdnCollector(nil, ttlRegistry)
+	NewPveStorageCollector(nil, ttlRegistry)
+	NewPveContainerCollector(nil, ttlRegistry)
+	NewPveVirtualMachineCollector(nil, ttlRegistry)
+
+	// Metric vectors are exported only once they hold a label set.
+	for _, metric := range ttlRegistry.metrics {
+		switch m := metric.(type) {
+		case *TTLGaugeVec:
+			m.With(emptyLabels(m.labelNames)).Set(0)
+		case *TTLCounterVec:
+			m.With(emptyLabels(m.labelNames)).Set(0)
+		default:
+			t.Fatalf("Unknown TTL metric type %T.", metric)
+		}
+	}
+
+	families, err := registry.Gather()
+	if err != nil {
+		t.Fatalf("Unable to gather metrics. Error: %s.", err)
+	}
+	if len(families) != len(ttlRegistry.metrics) {
+		t.Fatalf("Gathered %d metric families but %d metrics are registered.", len(families), len(ttlRegistry.metrics))
+	}
+
+	problems, err := promlint.NewWithMetricFamilies(families).Lint()
+	if err != nil {
+		t.Fatalf("Unable to lint metrics. Error: %s.", err)
+	}
+
+	for _, problem := range problems {
+		t.Errorf("Metric '%s' violates the Prometheus naming conventions: %s.", problem.Metric, problem.Text)
+	}
+}
+
+// emptyLabels builds a label set with all given label names set to an empty value.
+func emptyLabels(labelNames []string) prometheus.Labels {
+	labels := prometheus.Labels{}
+	for _, name := range labelNames {
+		labels[name] = ""
+	}
+	return labels
+}

metrics/pve_node_container_collector.go

@@ -21,11 +21,11 @@ type PveContainerCollector struct {
 	memBytes     *TTLGaugeVec // Container memory in bytes prometheus gauge.
 	memBytesUsed *TTLGaugeVec // Container memory usage in bytes prometheus gauge.
 
-	netReceive  *TTLGaugeVec // Container network RX in bytes prometheus gauge.
-	netTransmit *TTLGaugeVec // Container network TX in bytes prometheus gauge.
+	netReceive  *TTLCounterVec // Container received network traffic in bytes prometheus counter.
+	netTransmit *TTLCounterVec // Container transmitted network traffic in bytes prometheus counter.
 
-	diskWrite *TTLGaugeVec // Container disk written in bytes prometheus gauge.
-	diskRead  *TTLGaugeVec // Container disk read in bytes prometheus gauge.
+	diskWrite *TTLCounterVec // Container disk written in bytes prometheus counter.
+	diskRead  *TTLCounterVec // Container disk read in bytes prometheus counter.
 
 	disk    *TTLGaugeVec // Container disk space usage in bytes prometheus gauge.
 	diskMax *TTLGaugeVec // Container disk size in bytes prometheus gauge.
@@ -51,8 +51,8 @@ func NewPveContainerCollector(apiClient *proxmox.PveApiClient, registry *TTLRegi
 	// Container uptime.
 	c.uptime = NewTTLGaugeVec(
 		prometheus.GaugeOpts{
-			Name: "pve_ct_uptime",
-			Help: "Container uptime.",
+			Name: "pve_ct_uptime_seconds",
+			Help: "Container uptime in seconds.",
 		},
 		[]string{"cluster", "node", "vmid", "name"},
 		5*time.Minute,
@@ -62,7 +62,7 @@ func NewPveContainerCollector(apiClient *proxmox.PveApiClient, registry *TTLRegi
 	// Container CPU count.
 	c.cpu = NewTTLGaugeVec(
 		prometheus.GaugeOpts{
-			Name: "pve_ct_cpu_count",
+			Name: "pve_ct_cpus",
 			Help: "Container CPU count.",
 		},
 		[]string{"cluster", "node", "vmid", "name"},
@@ -73,8 +73,8 @@ func NewPveContainerCollector(apiClient *proxmox.PveApiClient, registry *TTLRegi
 	// Container CPU usage.
 	c.cpuUsage = NewTTLGaugeVec(
 		prometheus.GaugeOpts{
-			Name: "pve_ct_cpu_usage",
-			Help: "Container CPU usage.",
+			Name: "pve_ct_cpu_usage_ratio",
+			Help: "Container CPU usage ratio (0-1).",
 		},
 		[]string{"cluster", "node", "vmid", "name"},
 		5*time.Minute,
@@ -84,7 +84,7 @@ func NewPveContainerCollector(apiClient *proxmox.PveApiClient, registry *TTLRegi
 	// Container memory total.
 	c.memBytes = NewTTLGaugeVec(
 		prometheus.GaugeOpts{
-			Name: "pve_ct_mem_total_bytes",
+			Name: "pve_ct_memory_total_bytes",
 			Help: "Container total memory in bytes.",
 		},
 		[]string{"cluster", "node", "vmid", "name"},
@@ -95,7 +95,7 @@ func NewPveContainerCollector(apiClient *proxmox.PveApiClient, registry *TTLRegi
 	// Container memory usage.
 	c.memBytesUsed = NewTTLGaugeVec(
 		prometheus.GaugeOpts{
-			Name: "pve_ct_mem_used_bytes",
+			Name: "pve_ct_memory_used_bytes",
 			Help: "Container used memory in bytes.",
 		},
 		[]string{"cluster", "node", "vmid", "name"},
@@ -104,10 +104,10 @@ func NewPveContainerCollector(apiClient *proxmox.PveApiClient, registry *TTLRegi
 	c.registry.Register(c.memBytesUsed)
 
 	// Container network RX.
-	c.netReceive = NewTTLGaugeVec(
-		prometheus.GaugeOpts{
-			Name: "pve_ct_network_in_bytes",
-			Help: "Container network RX bytes.",
+	c.netReceive = NewTTLCounterVec(
+		prometheus.CounterOpts{
+			Name: "pve_ct_network_receive_bytes_total",
+			Help: "Container received network traffic in bytes.",
 		},
 		[]string{"cluster", "node", "vmid", "name"},
 		5*time.Minute,
@@ -115,10 +115,10 @@ func NewPveContainerCollector(apiClient *proxmox.PveApiClient, registry *TTLRegi
 	c.registry.Register(c.netReceive)
 
 	// Container network TX.
-	c.netTransmit = NewTTLGaugeVec(
-		prometheus.GaugeOpts{
-			Name: "pve_ct_network_out_bytes",
-			Help: "Container network TX bytes.",
+	c.netTransmit = NewTTLCounterVec(
+		prometheus.CounterOpts{
+			Name: "pve_ct_network_transmit_bytes_total",
+			Help: "Container transmitted network traffic in bytes.",
 		},
 		[]string{"cluster", "node", "vmid", "name"},
 		5*time.Minute,
@@ -126,10 +126,10 @@ func NewPveContainerCollector(apiClient *proxmox.PveApiClient, registry *TTLRegi
 	c.registry.Register(c.netTransmit)
 
 	// Container disk written.
-	c.diskWrite = NewTTLGaugeVec(
-		prometheus.GaugeOpts{
-			Name: "pve_ct_disk_wr_bytes",
-			Help: "Container disk written bytes.",
+	c.diskWrite = NewTTLCounterVec(
+		prometheus.CounterOpts{
+			Name: "pve_ct_disk_write_bytes_total",
+			Help: "Container written bytes to disk.",
 		},
 		[]string{"cluster", "node", "vmid", "name"},
 		5*time.Minute,
@@ -137,10 +137,10 @@ func NewPveContainerCollector(apiClient *proxmox.PveApiClient, registry *TTLRegi
 	c.registry.Register(c.diskWrite)
 
 	// Container disk read.
-	c.diskRead = NewTTLGaugeVec(
-		prometheus.GaugeOpts{
-			Name: "pve_ct_disk_rd_bytes",
-			Help: "Container disk read bytes.",
+	c.diskRead = NewTTLCounterVec(
+		prometheus.CounterOpts{
+			Name: "pve_ct_disk_read_bytes_total",
+			Help: "Container read bytes from disk.",
 		},
 		[]string{"cluster", "node", "vmid", "name"},
 		5*time.Minute,
@@ -150,8 +150,8 @@ func NewPveContainerCollector(apiClient *proxmox.PveApiClient, registry *TTLRegi
 	// Container disk size.
 	c.disk = NewTTLGaugeVec(
 		prometheus.GaugeOpts{
-			Name: "pve_ct_disk_usage_bytes",
-			Help: "Container disk read bytes.",
+			Name: "pve_ct_disk_used_bytes",
+			Help: "Container used disk space in bytes.",
 		},
 		[]string{"cluster", "node", "vmid", "name"},
 		5*time.Minute,

metrics/pve_node_disk_collector.go

@@ -38,13 +38,13 @@ func NewPveNodeDiskCollector(apiClient *proxmox.PveApiClient, registry *TTLRegis
 	// Node disk wearout.
 	c.wearout = NewTTLGaugeVec(
 		prometheus.GaugeOpts{
-			Name: "pve_node_disk_wearout",
-			Help: "Node disk wearout percent.",
+			Name: "pve_node_disk_wearout_percent",
+			Help: "Node disk wearout in percent (0-100).",
 		},
 		[]string{"cluster", "node", "wwn", "type", "model", "serial", "vendor", "used", "osd_id"},
 		5*time.Minute,
 	)
-	c.registry.Register(c.healthy)
+	c.registry.Register(c.wearout)
 
 	// Node disk size in bytes.
 	c.sizeBytes = NewTTLGaugeVec(

metrics/pve_node_status_collector.go

@@ -55,8 +55,8 @@ func NewPveNodeStatusCollector(apiClient *proxmox.PveApiClient, registry *TTLReg
 	// Node uptime.
 	c.uptime = NewTTLGaugeVec(
 		prometheus.GaugeOpts{
-			Name: "pve_node_uptime",
-			Help: "Node uptime.",
+			Name: "pve_node_uptime_seconds",
+			Help: "Node uptime in seconds.",
 		},
 		[]string{"cluster", "node"},
 		5*time.Minute,
@@ -66,7 +66,7 @@ func NewPveNodeStatusCollector(apiClient *proxmox.PveApiClient, registry *TTLReg
 	// Node cpu count.
 	c.cpus = NewTTLGaugeVec(
 		prometheus.GaugeOpts{
-			Name: "pve_node_cpu_count",
+			Name: "pve_node_cpus",
 			Help: "Node CPU count.",
 		},
 		[]string{"cluster", "node"},
@@ -77,8 +77,8 @@ func NewPveNodeStatusCollector(apiClient *proxmox.PveApiClient, registry *TTLReg
 	// Node CPU usage.
 	c.cpuUsage = NewTTLGaugeVec(
 		prometheus.GaugeOpts{
-			Name: "pve_node_cpu_usage",
-			Help: "Cluster node CPU usage %.",
+			Name: "pve_node_cpu_usage_ratio",
+			Help: "Node CPU usage ratio (0-1).",
 		},
 		[]string{"cluster", "node"},
 		5*time.Minute,
@@ -220,8 +220,8 @@ func NewPveNodeStatusCollector(apiClient *proxmox.PveApiClient, registry *TTLReg
 	// Node CPU info.
 	c.cpuInfo = NewTTLGaugeVec(
 		prometheus.GaugeOpts{
-			Name: "pve_node_cpuinfo",
-			Help: "Node CPU info.",
+			Name: "pve_node_cpu_info",
+			Help: "Node CPU information.",
 		},
 		[]string{"cluster", "node", "flags", "cores", "model", "sockets", "cpus", "hvm"},
 		5*time.Minute,
@@ -231,8 +231,8 @@ func NewPveNodeStatusCollector(apiClient *proxmox.PveApiClient, registry *TTLReg
 	// Node system info metrics.
 	c.systemInfo = NewTTLGaugeVec(
 		prometheus.GaugeOpts{
-			Name: "pve_node_systeminfo",
-			Help: "Node system info.",
+			Name: "pve_node_system_info",
+			Help: "Node system information.",
 		},
 		[]string{"cluster", "node", "kversion", "pveversion", "machine", "sysname", "release"},
 		5*time.Minute,
@@ -242,8 +242,8 @@ func NewPveNodeStatusCollector(apiClient *proxmox.PveApiClient, registry *TTLReg
 	// Node time info.
 	c.time = NewTTLGaugeVec(
 		prometheus.GaugeOpts{
-			Name: "pve_node_time",
-			Help: "Node time.",
+			Name: "pve_node_time_seconds",
+			Help: "Node UTC time as a unix timestamp in seconds.",
 		},
 		[]string{"cluster", "node"},
 		5*time.Minute,
@@ -253,8 +253,8 @@ func NewPveNodeStatusCollector(apiClient *proxmox.PveApiClient, registry *TTLReg
 	// Node localtime info.
 	c.localTime = NewTTLGaugeVec(
 		prometheus.GaugeOpts{
-			Name: "pve_node_localtime",
-			Help: "Node localtime.",
+			Name: "pve_node_localtime_seconds",
+			Help: "Node local time as a unix timestamp in seconds.",
 		},
 		[]string{"cluster", "node"},
 		5*time.Minute,

metrics/pve_node_subscription_collector.go

@@ -49,8 +49,8 @@ func NewPveSubscriptionCollector(apiClient *proxmox.PveApiClient, registry *TTLR
 	// Node subscription registration date.
 	c.regDate = NewTTLGaugeVec(
 		prometheus.GaugeOpts{
-			Name: "pve_node_subscription_regdate",
-			Help: "Node subscription registration date.",
+			Name: "pve_node_subscription_registration_timestamp_seconds",
+			Help: "Node subscription registration date as a unix timestamp in seconds.",
 		},
 		[]string{"cluster", "node"},
 		5*time.Minute,
@@ -60,8 +60,8 @@ func NewPveSubscriptionCollector(apiClient *proxmox.PveApiClient, registry *TTLR
 	// Node subscription next due date.
 	c.nextDueDate = NewTTLGaugeVec(
 		prometheus.GaugeOpts{
-			Name: "pve_node_subscription_nextduedate",
-			Help: "Node subscription next due date.",
+			Name: "pve_node_subscription_next_due_timestamp_seconds",
+			Help: "Node subscription next due date as a unix timestamp in seconds.",
 		},
 		[]string{"cluster", "node"},
 		5*time.Minute,

metrics/pve_node_virtual_machine_collector.go

@@ -26,20 +26,20 @@ type PveVirtualMachineCollector struct {
 	diskMax *TTLGaugeVec // Virtual machine disk size in bytes prometheus gauge.
 	swap    *TTLGaugeVec // Virtual machine swap usage in bytes prometheus gauge.
 
-	netReceive  *TTLGaugeVec // Virtual machine network receive in bytes prometheus gauge.
-	netTransmit *TTLGaugeVec // Virtual machine network transmit in bytes prometheus gauge.
+	netReceive  *TTLCounterVec // Virtual machine received network traffic in bytes prometheus counter.
+	netTransmit *TTLCounterVec // Virtual machine transmitted network traffic in bytes prometheus counter.
 
-	diskReadOps  *TTLGaugeVec // Virtual machine disk read ops prometheus gauge.
-	diskWriteOps *TTLGaugeVec // Virtual machine disk write ops prometheus gauge.
+	diskReadOps  *TTLCounterVec // Virtual machine disk read operations prometheus counter.
+	diskWriteOps *TTLCounterVec // Virtual machine disk write operations prometheus counter.
 
-	diskReadBytes  *TTLGaugeVec // Virtual machine disk read bytes prometheus gauge.
-	diskWriteBytes *TTLGaugeVec // Virtual machine disk write bytes prometheus gauge.
+	diskReadBytes  *TTLCounterVec // Virtual machine disk read bytes prometheus counter.
+	diskWriteBytes *TTLCounterVec // Virtual machine disk written bytes prometheus counter.
 
-	diskReadTimeNs  *TTLGaugeVec // Virtual machine disk read time total prometheus gauge.
-	diskWriteTimeNs *TTLGaugeVec // Virtual machine disk write time total prometheus gauge.
+	diskReadTime  *TTLCounterVec // Virtual machine disk read time total in seconds prometheus counter.
+	diskWriteTime *TTLCounterVec // Virtual machine disk write time total in seconds prometheus counter.
 
-	diskFailedReadOps  *TTLGaugeVec // Virtual machine disk failed read ops prometheus gauge.
-	diskFailedWriteOps *TTLGaugeVec // Virtual machine disk failed write ops prometheus gauge.
+	diskFailedReadOps  *TTLCounterVec // Virtual machine failed disk read operations prometheus counter.
+	diskFailedWriteOps *TTLCounterVec // Virtual machine failed disk write operations prometheus counter.
 
 	agent *TTLGaugeVec // Virtual machine agent enabled prometheus gauge.
 }
@@ -63,8 +63,8 @@ func NewPveVirtualMachineCollector(apiClient *proxmox.PveApiClient, registry *TT
 	// Virtual machine uptime.
 	c.uptime = NewTTLGaugeVec(
 		prometheus.GaugeOpts{
-			Name: "pve_vm_uptime",
-			Help: "Virtual machine uptime.",
+			Name: "pve_vm_uptime_seconds",
+			Help: "Virtual machine uptime in seconds.",
 		},
 		[]string{"cluster", "node", "vmid", "name"},
 		5*time.Minute,
@@ -85,7 +85,7 @@ func NewPveVirtualMachineCollector(apiClient *proxmox.PveApiClient, registry *TT
 	// Virtual machine CPU count.
 	c.cpu = NewTTLGaugeVec(
 		prometheus.GaugeOpts{
-			Name: "pve_vm_cpu_count",
+			Name: "pve_vm_cpus",
 			Help: "Virtual machine CPU count.",
 		},
 		[]string{"cluster", "node", "vmid", "name"},
@@ -96,8 +96,8 @@ func NewPveVirtualMachineCollector(apiClient *proxmox.PveApiClient, registry *TT
 	// Virtual machine CPU usage.
 	c.cpuUsage = NewTTLGaugeVec(
 		prometheus.GaugeOpts{
-			Name: "pve_vm_cpu_usage",
-			Help: "Virtual machine CPU usage.",
+			Name: "pve_vm_cpu_usage_ratio",
+			Help: "Virtual machine CPU usage ratio (0-1).",
 		},
 		[]string{"cluster", "node", "vmid", "name"},
 		5*time.Minute,
@@ -107,7 +107,7 @@ func NewPveVirtualMachineCollector(apiClient *proxmox.PveApiClient, registry *TT
 	// Virtual machine memory total.
 	c.memBytes = NewTTLGaugeVec(
 		prometheus.GaugeOpts{
-			Name: "pve_vm_mem_total_bytes",
+			Name: "pve_vm_memory_total_bytes",
 			Help: "Virtual machine total memory in bytes.",
 		},
 		[]string{"cluster", "node", "vmid", "name"},
@@ -118,7 +118,7 @@ func NewPveVirtualMachineCollector(apiClient *proxmox.PveApiClient, registry *TT
 	// Virtual machine memory usage.
 	c.memBytesUsed = NewTTLGaugeVec(
 		prometheus.GaugeOpts{
-			Name: "pve_vm_mem_used_bytes",
+			Name: "pve_vm_memory_used_bytes",
 			Help: "Virtual machine used memory in bytes.",
 		},
 		[]string{"cluster", "node", "vmid", "name"},
@@ -129,8 +129,8 @@ func NewPveVirtualMachineCollector(apiClient *proxmox.PveApiClient, registry *TT
 	// Virtual machine disk size.
 	c.disk = NewTTLGaugeVec(
 		prometheus.GaugeOpts{
-			Name: "pve_vm_disk_usage_bytes",
-			Help: "Virtual machine disk read bytes.",
+			Name: "pve_vm_disk_used_bytes",
+			Help: "Virtual machine used disk space in bytes.",
 		},
 		[]string{"cluster", "node", "vmid", "name"},
 		5*time.Minute,
@@ -149,10 +149,10 @@ func NewPveVirtualMachineCollector(apiClient *proxmox.PveApiClient, registry *TT
 	c.registry.Register(c.diskMax)
 
 	// Virtual machine network receive bytes.
-	c.netReceive = NewTTLGaugeVec(
-		prometheus.GaugeOpts{
-			Name: "pve_vm_network_in_bytes",
-			Help: "Virtual machine network receive in bytes.",
+	c.netReceive = NewTTLCounterVec(
+		prometheus.CounterOpts{
+			Name: "pve_vm_network_receive_bytes_total",
+			Help: "Virtual machine received network traffic in bytes.",
 		},
 		[]string{"cluster", "node", "vmid", "name", "interface"},
 		5*time.Minute,
@@ -160,10 +160,10 @@ func NewPveVirtualMachineCollector(apiClient *proxmox.PveApiClient, registry *TT
 	c.registry.Register(c.netReceive)
 
 	// Virtual machine network transmit bytes.
-	c.netTransmit = NewTTLGaugeVec(
-		prometheus.GaugeOpts{
-			Name: "pve_vm_network_out_bytes",
-			Help: "Virtual machine network transmit in bytes.",
+	c.netTransmit = NewTTLCounterVec(
+		prometheus.CounterOpts{
+			Name: "pve_vm_network_transmit_bytes_total",
+			Help: "Virtual machine transmitted network traffic in bytes.",
 		},
 		[]string{"cluster", "node", "vmid", "name", "interface"},
 		5*time.Minute,
@@ -171,10 +171,10 @@ func NewPveVirtualMachineCollector(apiClient *proxmox.PveApiClient, registry *TT
 	c.registry.Register(c.netTransmit)
 
 	// Virtual machine disk read ops.
-	c.diskReadOps = NewTTLGaugeVec(
-		prometheus.GaugeOpts{
-			Name: "pve_vm_disk_rd_operations",
-			Help: "Virtual machine disk read ops.",
+	c.diskReadOps = NewTTLCounterVec(
+		prometheus.CounterOpts{
+			Name: "pve_vm_disk_read_operations_total",
+			Help: "Virtual machine disk read operations.",
 		},
 		[]string{"cluster", "node", "vmid", "name", "device"},
 		5*time.Minute,
@@ -182,10 +182,10 @@ func NewPveVirtualMachineCollector(apiClient *proxmox.PveApiClient, registry *TT
 	c.registry.Register(c.diskReadOps)
 
 	// Virtual machine disk write ops.
-	c.diskWriteOps = NewTTLGaugeVec(
-		prometheus.GaugeOpts{
-			Name: "pve_vm_disk_wr_operations",
-			Help: "Virtual machine disk write ops.",
+	c.diskWriteOps = NewTTLCounterVec(
+		prometheus.CounterOpts{
+			Name: "pve_vm_disk_write_operations_total",
+			Help: "Virtual machine disk write operations.",
 		},
 		[]string{"cluster", "node", "vmid", "name", "device"},
 		5*time.Minute,
@@ -193,9 +193,9 @@ func NewPveVirtualMachineCollector(apiClient *proxmox.PveApiClient, registry *TT
 	c.registry.Register(c.diskWriteOps)
 
 	// Virtual machine disk read bytes.
-	c.diskReadBytes = NewTTLGaugeVec(
-		prometheus.GaugeOpts{
-			Name: "pve_vm_disk_rd_bytes",
+	c.diskReadBytes = NewTTLCounterVec(
+		prometheus.CounterOpts{
+			Name: "pve_vm_disk_read_bytes_total",
 			Help: "Virtual machine disk read bytes.",
 		},
 		[]string{"cluster", "node", "vmid", "name", "device"},
@@ -204,9 +204,9 @@ func NewPveVirtualMachineCollector(apiClient *proxmox.PveApiClient, registry *TT
 	c.registry.Register(c.diskReadBytes)
 
 	// Virtual machine disk write bytes.
-	c.diskWriteBytes = NewTTLGaugeVec(
-		prometheus.GaugeOpts{
-			Name: "pve_vm_disk_wr_bytes",
+	c.diskWriteBytes = NewTTLCounterVec(
+		prometheus.CounterOpts{
+			Name: "pve_vm_disk_write_bytes_total",
 			Help: "Virtual machine disk write bytes.",
 		},
 		[]string{"cluster", "node", "vmid", "name", "device"},
@@ -215,10 +215,10 @@ func NewPveVirtualMachineCollector(apiClient *proxmox.PveApiClient, registry *TT
 	c.registry.Register(c.diskWriteBytes)
 
 	// Virtual machine failed disk read ops.
-	c.diskFailedReadOps = NewTTLGaugeVec(
-		prometheus.GaugeOpts{
-			Name: "pve_vm_disk_failed_rd_ops",
-			Help: "Virtual machine failed disk read ops.",
+	c.diskFailedReadOps = NewTTLCounterVec(
+		prometheus.CounterOpts{
+			Name: "pve_vm_disk_failed_read_operations_total",
+			Help: "Virtual machine failed disk read operations.",
 		},
 		[]string{"cluster", "node", "vmid", "name", "device"},
 		5*time.Minute,
@@ -226,37 +226,37 @@ func NewPveVirtualMachineCollector(apiClient *proxmox.PveApiClient, registry *TT
 	c.registry.Register(c.diskFailedReadOps)
 
 	// Virtual machine failed disk write ops.
-	c.diskFailedWriteOps = NewTTLGaugeVec(
-		prometheus.GaugeOpts{
-			Name: "pve_vm_disk_failed_wr_ops",
-			Help: "Virtual machine failed disk write ops.",
+	c.diskFailedWriteOps = NewTTLCounterVec(
+		prometheus.CounterOpts{
+			Name: "pve_vm_disk_failed_write_operations_total",
+			Help: "Virtual machine failed disk write operations.",
 		},
 		[]string{"cluster", "node", "vmid", "name", "device"},
 		5*time.Minute,
 	)
 	c.registry.Register(c.diskFailedWriteOps)
 
-	// Virtual machine disk read time total nanoseconds.
-	c.diskReadTimeNs = NewTTLGaugeVec(
-		prometheus.GaugeOpts{
-			Name: "pve_vm_disk_rd_time_total_ns",
-			Help: "Virtual machine disk read time total in nanoseconds.",
+	// Virtual machine disk read time total seconds.
+	c.diskReadTime = NewTTLCounterVec(
+		prometheus.CounterOpts{
+			Name: "pve_vm_disk_read_time_seconds_total",
+			Help: "Virtual machine disk read time total in seconds.",
 		},
 		[]string{"cluster", "node", "vmid", "name", "device"},
 		5*time.Minute,
 	)
-	c.registry.Register(c.diskReadTimeNs)
+	c.registry.Register(c.diskReadTime)
 
-	// Virtual machine disk write time total nanoseconds.
-	c.diskWriteTimeNs = NewTTLGaugeVec(
-		prometheus.GaugeOpts{
-			Name: "pve_vm_disk_wr_time_total_ns",
-			Help: "Virtual machine disk write time total in nanoseconds.",
+	// Virtual machine disk write time total seconds.
+	c.diskWriteTime = NewTTLCounterVec(
+		prometheus.CounterOpts{
+			Name: "pve_vm_disk_write_time_seconds_total",
+			Help: "Virtual machine disk write time total in seconds.",
 		},
 		[]string{"cluster", "node", "vmid", "name", "device"},
 		5*time.Minute,
 	)
-	c.registry.Register(c.diskWriteTimeNs)
+	c.registry.Register(c.diskWriteTime)
 
 	return &c
 }
@@ -336,8 +336,8 @@ func (c *PveVirtualMachineCollector) CollectMetrics() error {
 					c.diskFailedReadOps.With(labels).Set(float64(value.FailedRdOperations))
 					c.diskFailedWriteOps.With(labels).Set(float64(value.FailedWrOperations))
 
-					c.diskReadTimeNs.With(labels).Set(float64(value.RdTotalTimeNs))
-					c.diskWriteTimeNs.With(labels).Set(float64(value.WrTotalTimeNs))
+					c.diskReadTime.With(labels).Set(float64(value.RdTotalTimeNs) / float64(time.Second))
+					c.diskWriteTime.With(labels).Set(float64(value.WrTotalTimeNs) / float64(time.Second))
 				}
 			}
 		}

metrics/pve_node_zfs_collector.go

@@ -15,9 +15,9 @@ type PveNodeZfsCollector struct {
 	registry  *TTLRegistry
 
 	state          *TTLGaugeVec
-	readErrors     *TTLGaugeVec
-	writeErrors    *TTLGaugeVec
-	checksumErrors *TTLGaugeVec
+	readErrors     *TTLCounterVec
+	writeErrors    *TTLCounterVec
+	checksumErrors *TTLCounterVec
 }
 
 // zfsMetricComponent is a flattened entry from the recursive ZFS topology.
@@ -50,9 +50,9 @@ func NewPveNodeZfsCollector(apiClient *proxmox.PveApiClient, registry *TTLRegist
 	)
 	c.registry.Register(c.state)
 
-	c.readErrors = NewTTLGaugeVec(
-		prometheus.GaugeOpts{
-			Name: "pve_node_zfs_read_errors",
+	c.readErrors = NewTTLCounterVec(
+		prometheus.CounterOpts{
+			Name: "pve_node_zfs_read_errors_total",
 			Help: "ZFS pool component read error count.",
 		},
 		componentLabelNames,
@@ -60,9 +60,9 @@ func NewPveNodeZfsCollector(apiClient *proxmox.PveApiClient, registry *TTLRegist
 	)
 	c.registry.Register(c.readErrors)
 
-	c.writeErrors = NewTTLGaugeVec(
-		prometheus.GaugeOpts{
-			Name: "pve_node_zfs_write_errors",
+	c.writeErrors = NewTTLCounterVec(
+		prometheus.CounterOpts{
+			Name: "pve_node_zfs_write_errors_total",
 			Help: "ZFS pool component write error count.",
 		},
 		componentLabelNames,
@@ -70,9 +70,9 @@ func NewPveNodeZfsCollector(apiClient *proxmox.PveApiClient, registry *TTLRegist
 	)
 	c.registry.Register(c.writeErrors)
 
-	c.checksumErrors = NewTTLGaugeVec(
-		prometheus.GaugeOpts{
-			Name: "pve_node_zfs_checksum_errors",
+	c.checksumErrors = NewTTLCounterVec(
+		prometheus.CounterOpts{
+			Name: "pve_node_zfs_checksum_errors_total",
 			Help: "ZFS pool component checksum error count.",
 		},
 		componentLabelNames,

metrics/pve_ttl_metrics.go

@@ -23,6 +23,7 @@ type TTLMetric interface {
 // it is automatically removed from the underlying GaugeVec.
 type TTLGaugeVec struct {
 	gaugeVec   *prometheus.GaugeVec // Underlying Prometheus GaugeVec.
+	labelNames []string             // Label names of the underlying GaugeVec.
 	ttl        time.Duration        // Duration after which an unused label set is considered stale.
 	lastUpdate sync.Map             // Map storing last update time for each label set (key is a sorted labels string).
 }
@@ -31,8 +32,9 @@ type TTLGaugeVec struct {
 // The underlying GaugeVec is registered using promauto.
 func NewTTLGaugeVec(opts prometheus.GaugeOpts, labelNames []string, ttl time.Duration) *TTLGaugeVec {
 	return &TTLGaugeVec{
-		gaugeVec: promauto.NewGaugeVec(opts, labelNames),
-		ttl:      ttl,
+		gaugeVec:   promauto.NewGaugeVec(opts, labelNames),
+		labelNames: labelNames,
+		ttl:        ttl,
 	}
 }
 
@@ -137,3 +139,118 @@ func (r *TTLRegistry) StartCleanupLoop(interval time.Duration) {
 		}
 	}()
 }
+
+// TTLCounterVec exposes cumulative values as Prometheus counters while keeping
+// the same "set an absolute value" usage as TTLGaugeVec. The PVE API reports
+// counters as absolute totals, which prometheus.CounterVec cannot express
+// (it only supports Inc/Add), so the metrics are emitted as constant metrics
+// of type counter. Label sets not updated within the TTL are dropped.
+type TTLCounterVec struct {
+	desc       *prometheus.Desc            // Metric descriptor.
+	labelNames []string                    // Label names in exposition order.
+	ttl        time.Duration               // Duration after which an unused label set is considered stale.
+	mu         sync.RWMutex                // Guards values.
+	values     map[string]*ttlCounterEntry // Current value per label set.
+}
+
+// ttlCounterEntry holds the current value of a single label set.
+type ttlCounterEntry struct {
+	labelValues []string  // Label values in exposition order.
+	value       float64   // Current counter value.
+	lastUpdate  time.Time // Time of the last update.
+}
+
+// TTLCounter is a handle to a single label set of a TTLCounterVec.
+type TTLCounter struct {
+	vec   *TTLCounterVec
+	entry *ttlCounterEntry
+}
+
+// Set stores the current absolute value of the counter.
+func (c *TTLCounter) Set(value float64) {
+	c.vec.mu.Lock()
+	defer c.vec.mu.Unlock()
+	c.entry.value = value
+}
+
+// NewTTLCounterVec creates a new TTLCounterVec and registers it using promauto.
+func NewTTLCounterVec(opts prometheus.CounterOpts, labelNames []string, ttl time.Duration) *TTLCounterVec {
+	c := &TTLCounterVec{
+		desc: prometheus.NewDesc(
+			prometheus.BuildFQName(opts.Namespace, opts.Subsystem, opts.Name),
+			opts.Help,
+			labelNames,
+			opts.ConstLabels,
+		),
+		labelNames: labelNames,
+		ttl:        ttl,
+		values:     make(map[string]*ttlCounterEntry),
+	}
+	prometheus.MustRegister(c)
+	return c
+}
+
+// With returns the counter handle for the given label set and records the current
+// time as the last update for those labels.
+func (t *TTLCounterVec) With(labels prometheus.Labels) *TTLCounter {
+	key := labelsKey(labels)
+
+	t.mu.Lock()
+	defer t.mu.Unlock()
+
+	entry, ok := t.values[key]
+	if !ok {
+		labelValues := make([]string, len(t.labelNames))
+		for i, name := range t.labelNames {
+			labelValues[i] = labels[name]
+		}
+		entry = &ttlCounterEntry{labelValues: labelValues}
+		t.values[key] = entry
+	}
+	entry.lastUpdate = time.Now()
+
+	return &TTLCounter{vec: t, entry: entry}
+}
+
+// Delete removes the metric associated with the given label set.
+func (t *TTLCounterVec) Delete(labels prometheus.Labels) bool {
+	key := labelsKey(labels)
+
+	t.mu.Lock()
+	defer t.mu.Unlock()
+
+	if _, ok := t.values[key]; !ok {
+		return false
+	}
+	delete(t.values, key)
+	return true
+}
+
+// Cleanup deletes all label sets that have not been updated within the TTL duration.
+func (t *TTLCounterVec) Cleanup() {
+	now := time.Now()
+
+	t.mu.Lock()
+	defer t.mu.Unlock()
+
+	for key, entry := range t.values {
+		if now.Sub(entry.lastUpdate) > t.ttl {
+			delete(t.values, key)
+		}
+	}
+}
+
+// Describe implements prometheus.Collector.
+func (t *TTLCounterVec) Describe(ch chan<- *prometheus.Desc) {
+	ch <- t.desc
+}
+
+// Collect implements prometheus.Collector.
+func (t *TTLCounterVec) Collect(ch chan<- prometheus.Metric) {
+	t.mu.RLock()
+	defer t.mu.RUnlock()
+
+	for _, entry := range t.values {
+		ch <- prometheus.MustNewConstMetric(t.desc, prometheus.CounterValue, entry.value, entry.labelValues...)
+	}
+}