pve-exporter/metrics/pve_node_virtual_machine_collector.go

package metrics

import (
	"strconv"
	"time"

	"github.com/prometheus/client_golang/prometheus"
	"lostak.dev/pve-exporter/proxmox"
)

// PVE virtual machine collector.
type PveVirtualMachineCollector struct {
	apiClient *proxmox.PveApiClient // PVE API client instance.
	registry  *TTLRegistry          // TTL metrics registry.

	state  *TTLGaugeVec // Virtual machine state prometheus gauge.
	uptime *TTLGaugeVec // Virtual machine uptime prometheus gauge.

	cpu      *TTLGaugeVec // Virtual machine count of CPUs prometheus gauge.
	cpuUsage *TTLGaugeVec // Virtual machine CPU usage % prometheus gauge.

	memBytes     *TTLGaugeVec // Virtual machine memory in bytes prometheus gauge.
	memBytesUsed *TTLGaugeVec // Virtual machine memory usage in bytes prometheus gauge.

	disk    *TTLGaugeVec // Virtual machine disk space usage in bytes prometheus gauge.
	diskMax *TTLGaugeVec // Virtual machine disk size in bytes prometheus gauge.
	swap    *TTLGaugeVec // Virtual machine swap usage in bytes prometheus gauge.

	netReceive  *TTLCounterVec // Virtual machine received network traffic in bytes prometheus counter.
	netTransmit *TTLCounterVec // Virtual machine transmitted network traffic in bytes prometheus counter.

	diskReadOps  *TTLCounterVec // Virtual machine disk read operations prometheus counter.
	diskWriteOps *TTLCounterVec // Virtual machine disk write operations prometheus counter.

	diskReadBytes  *TTLCounterVec // Virtual machine disk read bytes prometheus counter.
	diskWriteBytes *TTLCounterVec // Virtual machine disk written bytes prometheus counter.

	diskReadTime  *TTLCounterVec // Virtual machine disk read time total in seconds prometheus counter.
	diskWriteTime *TTLCounterVec // Virtual machine disk write time total in seconds prometheus counter.

	diskFailedReadOps  *TTLCounterVec // Virtual machine failed disk read operations prometheus counter.
	diskFailedWriteOps *TTLCounterVec // Virtual machine failed disk write operations prometheus counter.

	agent *TTLGaugeVec // Virtual machine agent enabled prometheus gauge.
}

// Create new instance of PVE virtual machine collector.
func NewPveVirtualMachineCollector(apiClient *proxmox.PveApiClient, registry *TTLRegistry) *PveVirtualMachineCollector {
	c := PveVirtualMachineCollector{apiClient: apiClient}
	c.registry = registry

	// Virtual machine state.
	c.state = NewTTLGaugeVec(
		prometheus.GaugeOpts{
			Name: "pve_vm_state",
			Help: "Virtual machine state.",
		},
		[]string{"cluster", "node", "vmid", "name"},
		5*time.Minute,
	)
	c.registry.Register(c.state)

	// Virtual machine uptime.
	c.uptime = NewTTLGaugeVec(
		prometheus.GaugeOpts{
			Name: "pve_vm_uptime_seconds",
			Help: "Virtual machine uptime in seconds.",
		},
		[]string{"cluster", "node", "vmid", "name"},
		5*time.Minute,
	)
	c.registry.Register(c.uptime)

	// Virtual machine agent state.
	c.agent = NewTTLGaugeVec(
		prometheus.GaugeOpts{
			Name: "pve_vm_agent",
			Help: "Virtual machine agent state.",
		},
		[]string{"cluster", "node", "vmid", "name"},
		5*time.Minute,
	)
	c.registry.Register(c.agent)

	// Virtual machine CPU count.
	c.cpu = NewTTLGaugeVec(
		prometheus.GaugeOpts{
			Name: "pve_vm_cpus",
			Help: "Virtual machine CPU count.",
		},
		[]string{"cluster", "node", "vmid", "name"},
		5*time.Minute,
	)
	c.registry.Register(c.cpu)

	// Virtual machine CPU usage.
	c.cpuUsage = NewTTLGaugeVec(
		prometheus.GaugeOpts{
			Name: "pve_vm_cpu_usage_ratio",
			Help: "Virtual machine CPU usage ratio (0-1).",
		},
		[]string{"cluster", "node", "vmid", "name"},
		5*time.Minute,
	)
	c.registry.Register(c.cpuUsage)

	// Virtual machine memory total.
	c.memBytes = NewTTLGaugeVec(
		prometheus.GaugeOpts{
			Name: "pve_vm_memory_total_bytes",
			Help: "Virtual machine total memory in bytes.",
		},
		[]string{"cluster", "node", "vmid", "name"},
		5*time.Minute,
	)
	c.registry.Register(c.memBytes)

	// Virtual machine memory usage.
	c.memBytesUsed = NewTTLGaugeVec(
		prometheus.GaugeOpts{
			Name: "pve_vm_memory_used_bytes",
			Help: "Virtual machine used memory in bytes.",
		},
		[]string{"cluster", "node", "vmid", "name"},
		5*time.Minute,
	)
	c.registry.Register(c.memBytesUsed)

	// Virtual machine disk size.
	c.disk = NewTTLGaugeVec(
		prometheus.GaugeOpts{
			Name: "pve_vm_disk_used_bytes",
			Help: "Virtual machine used disk space in bytes.",
		},
		[]string{"cluster", "node", "vmid", "name"},
		5*time.Minute,
	)
	c.registry.Register(c.disk)

	// Virtual machine disk size.
	c.diskMax = NewTTLGaugeVec(
		prometheus.GaugeOpts{
			Name: "pve_vm_disk_size_bytes",
			Help: "Virtual machine disk size bytes.",
		},
		[]string{"cluster", "node", "vmid", "name"},
		5*time.Minute,
	)
	c.registry.Register(c.diskMax)

	// Virtual machine network receive bytes.
	c.netReceive = NewTTLCounterVec(
		prometheus.CounterOpts{
			Name: "pve_vm_network_receive_bytes_total",
			Help: "Virtual machine received network traffic in bytes.",
		},
		[]string{"cluster", "node", "vmid", "name", "interface"},
		5*time.Minute,
	)
	c.registry.Register(c.netReceive)

	// Virtual machine network transmit bytes.
	c.netTransmit = NewTTLCounterVec(
		prometheus.CounterOpts{
			Name: "pve_vm_network_transmit_bytes_total",
			Help: "Virtual machine transmitted network traffic in bytes.",
		},
		[]string{"cluster", "node", "vmid", "name", "interface"},
		5*time.Minute,
	)
	c.registry.Register(c.netTransmit)

	// Virtual machine disk read ops.
	c.diskReadOps = NewTTLCounterVec(
		prometheus.CounterOpts{
			Name: "pve_vm_disk_read_operations_total",
			Help: "Virtual machine disk read operations.",
		},
		[]string{"cluster", "node", "vmid", "name", "device"},
		5*time.Minute,
	)
	c.registry.Register(c.diskReadOps)

	// Virtual machine disk write ops.
	c.diskWriteOps = NewTTLCounterVec(
		prometheus.CounterOpts{
			Name: "pve_vm_disk_write_operations_total",
			Help: "Virtual machine disk write operations.",
		},
		[]string{"cluster", "node", "vmid", "name", "device"},
		5*time.Minute,
	)
	c.registry.Register(c.diskWriteOps)

	// Virtual machine disk read bytes.
	c.diskReadBytes = NewTTLCounterVec(
		prometheus.CounterOpts{
			Name: "pve_vm_disk_read_bytes_total",
			Help: "Virtual machine disk read bytes.",
		},
		[]string{"cluster", "node", "vmid", "name", "device"},
		5*time.Minute,
	)
	c.registry.Register(c.diskReadBytes)

	// Virtual machine disk write bytes.
	c.diskWriteBytes = NewTTLCounterVec(
		prometheus.CounterOpts{
			Name: "pve_vm_disk_write_bytes_total",
			Help: "Virtual machine disk write bytes.",
		},
		[]string{"cluster", "node", "vmid", "name", "device"},
		5*time.Minute,
	)
	c.registry.Register(c.diskWriteBytes)

	// Virtual machine failed disk read ops.
	c.diskFailedReadOps = NewTTLCounterVec(
		prometheus.CounterOpts{
			Name: "pve_vm_disk_failed_read_operations_total",
			Help: "Virtual machine failed disk read operations.",
		},
		[]string{"cluster", "node", "vmid", "name", "device"},
		5*time.Minute,
	)
	c.registry.Register(c.diskFailedReadOps)

	// Virtual machine failed disk write ops.
	c.diskFailedWriteOps = NewTTLCounterVec(
		prometheus.CounterOpts{
			Name: "pve_vm_disk_failed_write_operations_total",
			Help: "Virtual machine failed disk write operations.",
		},
		[]string{"cluster", "node", "vmid", "name", "device"},
		5*time.Minute,
	)
	c.registry.Register(c.diskFailedWriteOps)

	// Virtual machine disk read time total seconds.
	c.diskReadTime = NewTTLCounterVec(
		prometheus.CounterOpts{
			Name: "pve_vm_disk_read_time_seconds_total",
			Help: "Virtual machine disk read time total in seconds.",
		},
		[]string{"cluster", "node", "vmid", "name", "device"},
		5*time.Minute,
	)
	c.registry.Register(c.diskReadTime)

	// Virtual machine disk write time total seconds.
	c.diskWriteTime = NewTTLCounterVec(
		prometheus.CounterOpts{
			Name: "pve_vm_disk_write_time_seconds_total",
			Help: "Virtual machine disk write time total in seconds.",
		},
		[]string{"cluster", "node", "vmid", "name", "device"},
		5*time.Minute,
	)
	c.registry.Register(c.diskWriteTime)

	return &c
}

// PveMetricsCollector interface implementation.
func (c *PveVirtualMachineCollector) CollectMetrics() error {
	cluster, err := c.apiClient.GetClusterStatus()
	if err != nil {
		return err
	}

	for _, node := range cluster.NodeStatuses {
		qemus, err := c.apiClient.GetNodeQemuList(node.Name)
		if err != nil {
			return err
		}

		for _, qemu := range *qemus {
			// Skip templates because they are always offline.
			if qemu.Template == 1 {
				continue
			}

			labels := prometheus.Labels{
				"cluster": cluster.GetClusterName(),
				"node":    node.Name,
				"vmid":    strconv.Itoa(qemu.VMID),
				"name":    qemu.Name,
			}

			c.state.With(labels).Set(qemu.GetStatusNumeric())
			c.cpu.With(labels).Set(float64(qemu.CPUs))
			c.memBytes.With(labels).Set(float64(qemu.MaxMem))
			c.diskMax.With(labels).Set(float64(qemu.MaxDisk))

			// Metrics only on running virtual machines.
			if qemu.IsRunning() {
				c.uptime.With(labels).Set(float64(qemu.Uptime))
				c.cpuUsage.With(labels).Set(float64(qemu.CPU))
				c.memBytesUsed.With(labels).Set(float64(qemu.Mem))

				detail, err := c.apiClient.GetNodeQemu(node.Name, strconv.Itoa(qemu.VMID))
				if err != nil {
					return err
				}

				c.agent.With(labels).Set(float64(detail.Agent))

				for iface, value := range detail.Nics {
					labels := prometheus.Labels{
						"cluster":   cluster.GetClusterName(),
						"node":      node.Name,
						"vmid":      strconv.Itoa(qemu.VMID),
						"name":      qemu.Name,
						"interface": iface,
					}

					c.netReceive.With(labels).Set(float64(value.NetIn))
					c.netTransmit.With(labels).Set(float64(value.NetOut))
				}

				for device, value := range detail.BlockStat {
					labels := prometheus.Labels{
						"cluster": cluster.GetClusterName(),
						"node":    node.Name,
						"vmid":    strconv.Itoa(qemu.VMID),
						"name":    qemu.Name,
						"device":  device,
					}

					c.diskReadOps.With(labels).Set(float64(value.RdOperations))
					c.diskWriteOps.With(labels).Set(float64(value.WrOperations))

					c.diskReadBytes.With(labels).Set(float64(value.RdBytes))
					c.diskWriteBytes.With(labels).Set(float64(value.WrBytes))

					c.diskFailedReadOps.With(labels).Set(float64(value.FailedRdOperations))
					c.diskFailedWriteOps.With(labels).Set(float64(value.FailedWrOperations))

					c.diskReadTime.With(labels).Set(float64(value.RdTotalTimeNs) / float64(time.Second))
					c.diskWriteTime.With(labels).Set(float64(value.WrTotalTimeNs) / float64(time.Second))
				}
			}
		}
	}
	return nil
}

// PveMetricsCollector interface implementation.
func (c *PveVirtualMachineCollector) GetName() string {
	return "Virtual Machine"
}