package metrics import ( "strconv" "time" "github.com/prometheus/client_golang/prometheus" "lostak.dev/pve-exporter/proxmox" ) // PVE virtual machine collector. type PveVirtualMachineCollector struct { apiClient *proxmox.PveApiClient // PVE API client instance. registry *TTLRegistry // TTL metrics registry. state *TTLGaugeVec // Virtual machine state prometheus gauge. uptime *TTLGaugeVec // Virtual machine uptime prometheus gauge. cpu *TTLGaugeVec // Virtual machine count of CPUs prometheus gauge. cpuUsage *TTLGaugeVec // Virtual machine CPU usage % prometheus gauge. memBytes *TTLGaugeVec // Virtual machine memory in bytes prometheus gauge. memBytesUsed *TTLGaugeVec // Virtual machine memory usage in bytes prometheus gauge. disk *TTLGaugeVec // Virtual machine disk space usage in bytes prometheus gauge. diskMax *TTLGaugeVec // Virtual machine disk size in bytes prometheus gauge. swap *TTLGaugeVec // Virtual machine swap usage in bytes prometheus gauge. netReceive *TTLCounterVec // Virtual machine received network traffic in bytes prometheus counter. netTransmit *TTLCounterVec // Virtual machine transmitted network traffic in bytes prometheus counter. diskReadOps *TTLCounterVec // Virtual machine disk read operations prometheus counter. diskWriteOps *TTLCounterVec // Virtual machine disk write operations prometheus counter. diskReadBytes *TTLCounterVec // Virtual machine disk read bytes prometheus counter. diskWriteBytes *TTLCounterVec // Virtual machine disk written bytes prometheus counter. diskReadTime *TTLCounterVec // Virtual machine disk read time total in seconds prometheus counter. diskWriteTime *TTLCounterVec // Virtual machine disk write time total in seconds prometheus counter. diskFailedReadOps *TTLCounterVec // Virtual machine failed disk read operations prometheus counter. diskFailedWriteOps *TTLCounterVec // Virtual machine failed disk write operations prometheus counter. agent *TTLGaugeVec // Virtual machine agent enabled prometheus gauge. } // Create new instance of PVE virtual machine collector. func NewPveVirtualMachineCollector(apiClient *proxmox.PveApiClient, registry *TTLRegistry) *PveVirtualMachineCollector { c := PveVirtualMachineCollector{apiClient: apiClient} c.registry = registry // Virtual machine state. c.state = NewTTLGaugeVec( prometheus.GaugeOpts{ Name: "pve_vm_state", Help: "Virtual machine state.", }, []string{"cluster", "node", "vmid", "name"}, 5*time.Minute, ) c.registry.Register(c.state) // Virtual machine uptime. c.uptime = NewTTLGaugeVec( prometheus.GaugeOpts{ Name: "pve_vm_uptime_seconds", Help: "Virtual machine uptime in seconds.", }, []string{"cluster", "node", "vmid", "name"}, 5*time.Minute, ) c.registry.Register(c.uptime) // Virtual machine agent state. c.agent = NewTTLGaugeVec( prometheus.GaugeOpts{ Name: "pve_vm_agent", Help: "Virtual machine agent state.", }, []string{"cluster", "node", "vmid", "name"}, 5*time.Minute, ) c.registry.Register(c.agent) // Virtual machine CPU count. c.cpu = NewTTLGaugeVec( prometheus.GaugeOpts{ Name: "pve_vm_cpus", Help: "Virtual machine CPU count.", }, []string{"cluster", "node", "vmid", "name"}, 5*time.Minute, ) c.registry.Register(c.cpu) // Virtual machine CPU usage. c.cpuUsage = NewTTLGaugeVec( prometheus.GaugeOpts{ Name: "pve_vm_cpu_usage_ratio", Help: "Virtual machine CPU usage ratio (0-1).", }, []string{"cluster", "node", "vmid", "name"}, 5*time.Minute, ) c.registry.Register(c.cpuUsage) // Virtual machine memory total. c.memBytes = NewTTLGaugeVec( prometheus.GaugeOpts{ Name: "pve_vm_memory_total_bytes", Help: "Virtual machine total memory in bytes.", }, []string{"cluster", "node", "vmid", "name"}, 5*time.Minute, ) c.registry.Register(c.memBytes) // Virtual machine memory usage. c.memBytesUsed = NewTTLGaugeVec( prometheus.GaugeOpts{ Name: "pve_vm_memory_used_bytes", Help: "Virtual machine used memory in bytes.", }, []string{"cluster", "node", "vmid", "name"}, 5*time.Minute, ) c.registry.Register(c.memBytesUsed) // Virtual machine disk size. c.disk = NewTTLGaugeVec( prometheus.GaugeOpts{ Name: "pve_vm_disk_used_bytes", Help: "Virtual machine used disk space in bytes.", }, []string{"cluster", "node", "vmid", "name"}, 5*time.Minute, ) c.registry.Register(c.disk) // Virtual machine disk size. c.diskMax = NewTTLGaugeVec( prometheus.GaugeOpts{ Name: "pve_vm_disk_size_bytes", Help: "Virtual machine disk size bytes.", }, []string{"cluster", "node", "vmid", "name"}, 5*time.Minute, ) c.registry.Register(c.diskMax) // Virtual machine network receive bytes. c.netReceive = NewTTLCounterVec( prometheus.CounterOpts{ Name: "pve_vm_network_receive_bytes_total", Help: "Virtual machine received network traffic in bytes.", }, []string{"cluster", "node", "vmid", "name", "interface"}, 5*time.Minute, ) c.registry.Register(c.netReceive) // Virtual machine network transmit bytes. c.netTransmit = NewTTLCounterVec( prometheus.CounterOpts{ Name: "pve_vm_network_transmit_bytes_total", Help: "Virtual machine transmitted network traffic in bytes.", }, []string{"cluster", "node", "vmid", "name", "interface"}, 5*time.Minute, ) c.registry.Register(c.netTransmit) // Virtual machine disk read ops. c.diskReadOps = NewTTLCounterVec( prometheus.CounterOpts{ Name: "pve_vm_disk_read_operations_total", Help: "Virtual machine disk read operations.", }, []string{"cluster", "node", "vmid", "name", "device"}, 5*time.Minute, ) c.registry.Register(c.diskReadOps) // Virtual machine disk write ops. c.diskWriteOps = NewTTLCounterVec( prometheus.CounterOpts{ Name: "pve_vm_disk_write_operations_total", Help: "Virtual machine disk write operations.", }, []string{"cluster", "node", "vmid", "name", "device"}, 5*time.Minute, ) c.registry.Register(c.diskWriteOps) // Virtual machine disk read bytes. c.diskReadBytes = NewTTLCounterVec( prometheus.CounterOpts{ Name: "pve_vm_disk_read_bytes_total", Help: "Virtual machine disk read bytes.", }, []string{"cluster", "node", "vmid", "name", "device"}, 5*time.Minute, ) c.registry.Register(c.diskReadBytes) // Virtual machine disk write bytes. c.diskWriteBytes = NewTTLCounterVec( prometheus.CounterOpts{ Name: "pve_vm_disk_write_bytes_total", Help: "Virtual machine disk write bytes.", }, []string{"cluster", "node", "vmid", "name", "device"}, 5*time.Minute, ) c.registry.Register(c.diskWriteBytes) // Virtual machine failed disk read ops. c.diskFailedReadOps = NewTTLCounterVec( prometheus.CounterOpts{ Name: "pve_vm_disk_failed_read_operations_total", Help: "Virtual machine failed disk read operations.", }, []string{"cluster", "node", "vmid", "name", "device"}, 5*time.Minute, ) c.registry.Register(c.diskFailedReadOps) // Virtual machine failed disk write ops. c.diskFailedWriteOps = NewTTLCounterVec( prometheus.CounterOpts{ Name: "pve_vm_disk_failed_write_operations_total", Help: "Virtual machine failed disk write operations.", }, []string{"cluster", "node", "vmid", "name", "device"}, 5*time.Minute, ) c.registry.Register(c.diskFailedWriteOps) // Virtual machine disk read time total seconds. c.diskReadTime = NewTTLCounterVec( prometheus.CounterOpts{ Name: "pve_vm_disk_read_time_seconds_total", Help: "Virtual machine disk read time total in seconds.", }, []string{"cluster", "node", "vmid", "name", "device"}, 5*time.Minute, ) c.registry.Register(c.diskReadTime) // Virtual machine disk write time total seconds. c.diskWriteTime = NewTTLCounterVec( prometheus.CounterOpts{ Name: "pve_vm_disk_write_time_seconds_total", Help: "Virtual machine disk write time total in seconds.", }, []string{"cluster", "node", "vmid", "name", "device"}, 5*time.Minute, ) c.registry.Register(c.diskWriteTime) return &c } // PveMetricsCollector interface implementation. func (c *PveVirtualMachineCollector) CollectMetrics() error { cluster, err := c.apiClient.GetClusterStatus() if err != nil { return err } for _, node := range cluster.NodeStatuses { qemus, err := c.apiClient.GetNodeQemuList(node.Name) if err != nil { return err } for _, qemu := range *qemus { // Skip templates because they are always offline. if qemu.Template == 1 { continue } labels := prometheus.Labels{ "cluster": cluster.GetClusterName(), "node": node.Name, "vmid": strconv.Itoa(qemu.VMID), "name": qemu.Name, } c.state.With(labels).Set(qemu.GetStatusNumeric()) c.cpu.With(labels).Set(float64(qemu.CPUs)) c.memBytes.With(labels).Set(float64(qemu.MaxMem)) c.diskMax.With(labels).Set(float64(qemu.MaxDisk)) // Metrics only on running virtual machines. if qemu.IsRunning() { c.uptime.With(labels).Set(float64(qemu.Uptime)) c.cpuUsage.With(labels).Set(float64(qemu.CPU)) c.memBytesUsed.With(labels).Set(float64(qemu.Mem)) detail, err := c.apiClient.GetNodeQemu(node.Name, strconv.Itoa(qemu.VMID)) if err != nil { return err } c.agent.With(labels).Set(float64(detail.Agent)) for iface, value := range detail.Nics { labels := prometheus.Labels{ "cluster": cluster.GetClusterName(), "node": node.Name, "vmid": strconv.Itoa(qemu.VMID), "name": qemu.Name, "interface": iface, } c.netReceive.With(labels).Set(float64(value.NetIn)) c.netTransmit.With(labels).Set(float64(value.NetOut)) } for device, value := range detail.BlockStat { labels := prometheus.Labels{ "cluster": cluster.GetClusterName(), "node": node.Name, "vmid": strconv.Itoa(qemu.VMID), "name": qemu.Name, "device": device, } c.diskReadOps.With(labels).Set(float64(value.RdOperations)) c.diskWriteOps.With(labels).Set(float64(value.WrOperations)) c.diskReadBytes.With(labels).Set(float64(value.RdBytes)) c.diskWriteBytes.With(labels).Set(float64(value.WrBytes)) c.diskFailedReadOps.With(labels).Set(float64(value.FailedRdOperations)) c.diskFailedWriteOps.With(labels).Set(float64(value.FailedWrOperations)) c.diskReadTime.With(labels).Set(float64(value.RdTotalTimeNs) / float64(time.Second)) c.diskWriteTime.With(labels).Set(float64(value.WrTotalTimeNs) / float64(time.Second)) } } } } return nil } // PveMetricsCollector interface implementation. func (c *PveVirtualMachineCollector) GetName() string { return "Virtual Machine" }