Real-time monitoring dashboard for Proxmox VE, PBS, Docker, Kubernetes, TrueNAS and vSphere. Self-hosted, with alerts for failed backups, full disks, unhealthy containers and offline nodes.
Go aialertsdashboarddockerdocker-monitoringgohomelabinfrastructure-monitoringkubernetesmonitoringmspproxmoxproxmox-backup-serverproxmox-mail-gatewayproxmox-veself-hostedtruenastypescriptvspherewebhooks
package monitoring
import (
"strings"
"time"
"github.com/rcourtman/pulse-go-rewrite/internal/models"
"github.com/rcourtman/pulse-go-rewrite/internal/unifiedresources"
"github.com/rcourtman/pulse-go-rewrite/pkg/proxmox"
"github.com/rs/zerolog/log"
)
func (m *Monitor) preserveGuestsForGracePeriod(
instanceName string,
resources []proxmox.ClusterResource,
prevVMs []models.VM,
prevContainers []models.Container,
nodeEffectiveStatus map[string]string,
allVMs []models.VM,
allContainers []models.Container,
) ([]models.VM, []models.Container) {
// Preserve VMs and containers from nodes within grace period
// The cluster/resources endpoint doesn't return VMs/containers from nodes Proxmox considers offline,
// but we want to keep showing them if the node is within grace period
// Count previous resources for this instance
prevVMCount := len(prevVMs)
prevContainerCount := len(prevContainers)
// Build map of which nodes are covered by current resources
nodesWithResources := make(map[string]bool)
for _, res := range resources {
nodesWithResources[res.Node] = true
}
log.Debug().
Str("instance", instanceName).
Int("nodesInResources", len(nodesWithResources)).
Int("totalVMsFromResources", len(allVMs)).
Int("totalContainersFromResources", len(allContainers)).
Int("prevVMs", prevVMCount).
Int("prevContainers", prevContainerCount).
Msg("Cluster resources received, checking for grace period preservation")
// If we got ZERO resources but had resources before, and we have no node data,
// this likely means the cluster health check failed. Preserve everything.
if len(allVMs) == 0 && len(allContainers) == 0 &&
(prevVMCount > 0 || prevContainerCount > 0) &&
len(nodeEffectiveStatus) == 0 {
log.Warn().
Str("instance", instanceName).
Int("prevVMs", prevVMCount).
Int("prevContainers", prevContainerCount).
Msg("Cluster returned zero resources but had resources before - likely cluster health issue, preserving all previous resources")
// Preserve all previous VMs and containers for this instance
allVMs = append(allVMs, prevVMs...)
allContainers = append(allContainers, prevContainers...)
}
// Check for nodes that are within grace period but not in cluster/resources response
preservedVMCount := 0
preservedContainerCount := 0
for nodeName, effectiveStatus := range nodeEffectiveStatus {
if effectiveStatus == "online" && !nodesWithResources[nodeName] {
// This node is within grace period but Proxmox didn't return its resources
// Preserve previous VMs and containers from this node
vmsBefore := len(allVMs)
containersBefore := len(allContainers)
// Preserve VMs from this node
for _, vm := range prevVMs {
if vm.Node == nodeName {
allVMs = append(allVMs, vm)
}
}
// Preserve containers from this node
for _, container := range prevContainers {
if container.Node == nodeName {
allContainers = append(allContainers, container)
}
}
vmsPreserved := len(allVMs) - vmsBefore
containersPreserved := len(allContainers) - containersBefore
preservedVMCount += vmsPreserved
preservedContainerCount += containersPreserved
log.Info().
Str("instance", instanceName).
Str("node", nodeName).
Int("vmsPreserved", vmsPreserved).
Int("containersPreserved", containersPreserved).
Msg("Preserved VMs/containers from node in grace period")
}
}
if preservedVMCount > 0 || preservedContainerCount > 0 {
log.Info().
Str("instance", instanceName).
Int("totalPreservedVMs", preservedVMCount).
Int("totalPreservedContainers", preservedContainerCount).
Msg("Grace period preservation complete")
}
return allVMs, allContainers
}
// guestObservedInCycle reports whether a guest's telemetry came from this poll
// rather than being carried forward while its node sits in the grace period.
//
// Preserved guests keep the LastSeen of the cycle that actually observed them,
// so anything older than this cycle's start was not observed. Recording one as
// a sample fabricates a reading: the carried-forward projection has no counters,
// so it writes CPU, disk and network zeroes for a guest Pulse cannot currently
// see, and the history shows a collapse to zero instead of a gap.
//
// A zero LastSeen means we have no evidence either way, so this fails open and
// records. Dropping a legitimate sample is the worse error of the two, and
// harder to notice.
func guestObservedInCycle(lastSeen, cycleStart time.Time) bool {
if lastSeen.IsZero() {
return true
}
return !lastSeen.Before(cycleStart)
}
// recordGuestMetrics records metrics for all running VMs and containers that
// this cycle actually observed.
func (m *Monitor) recordGuestMetrics(allVMs []models.VM, allContainers []models.Container, cycleStart time.Time) {
now := time.Now()
for _, vm := range allVMs {
if !guestObservedInCycle(vm.LastSeen, cycleStart) {
continue
}
if vm.Status == "running" {
diskRead, diskWrite, networkIn, networkOut := guestHistoryRates(vm.DiskRead, vm.DiskWrite, vm.NetworkIn, vm.NetworkOut, vm.IORateValidity)
memoryUsage, memoryUsed, diskUsage := historyMemoryUsage(vm.Memory), historyMemoryUsed(vm.Memory), vm.Disk.Usage
// A failed filesystem read can leave a positive cluster allocation,
// not just a retained prev- reading. Neither is current usage.
if vm.DiskStatusReason != "" {
diskUsage = -1
}
// Only legacy unannotated memory needs the diagnostic heuristic.
// An explicit observation belongs to the selected source: deferred
// filesystem/QGA work cannot disqualify independent current memory.
if vm.GuestAgentStatus == "deferred" && vm.Memory.Observation.State == "" {
snapshot := m.previousGuestSnapshot(vm.Instance, "qemu", vm.Node, vm.VMID)
if snapshot == nil || snapshot.MemorySource == "previous-snapshot" || strings.HasPrefix(CanonicalMemorySource(snapshot.MemorySource), "guest-agent-meminfo") {
memoryUsage, memoryUsed = -1, -1
}
}
m.recordGuestMetric("vm", vm.ID, unifiedresources.ProxmoxGuestCPUPercent(vm.CPU), memoryUsage, memoryUsed, diskUsage, diskRead, diskWrite, networkIn, networkOut, now)
}
}
for _, ct := range allContainers {
if !guestObservedInCycle(ct.LastSeen, cycleStart) {
continue
}
if ct.Status == "running" {
diskRead, diskWrite, networkIn, networkOut := guestHistoryRates(ct.DiskRead, ct.DiskWrite, ct.NetworkIn, ct.NetworkOut, ct.IORateValidity)
m.recordGuestMetric("container", ct.ID, unifiedresources.ProxmoxGuestCPUPercent(ct.CPU), historyMemoryUsage(ct.Memory), historyMemoryUsed(ct.Memory), ct.Disk.Usage, diskRead, diskWrite, networkIn, networkOut, now)
}
}
}
func guestHistoryRates(diskRead, diskWrite, networkIn, networkOut int64, validity models.IORateValidity) (int64, int64, int64, int64) {
effective := validity.EffectiveForRates(diskRead, diskWrite, networkIn, networkOut)
if !effective.DiskRead {
diskRead = -1
}
if !effective.DiskWrite {
diskWrite = -1
}
if !effective.NetworkIn {
networkIn = -1
}
if !effective.NetworkOut {
networkOut = -1
}
return diskRead, diskWrite, networkIn, networkOut
}
// Retained numbers remain useful for display, but a newly refreshed guest row
// cannot turn them into a new percentage or byte observation. The producer owns
// freshness; ordinary current TTL-cache hits and legacy readings remain valid.
func historyMemoryUsage(memory models.Memory) float64 {
if !memory.HasKnownUsage() || (memory.Observation.State != "" && memory.Observation.State != "current") {
return -1
}
return memory.Usage
}
func historyMemoryUsed(memory models.Memory) float64 {
if !memory.HasKnownUsage() || (memory.Observation.State != "" && memory.Observation.State != "current") {
return -1
}
return float64(memory.Used)
}
// recordGuestMetric records metrics for a single guest (VM or container) to both
// the in-memory metrics history and the persistent metrics store.
func (m *Monitor) recordGuestMetric(
resourceType, resourceID string,
cpu, memory, memoryUsed, diskUsage float64,
diskRead, diskWrite, networkIn, networkOut int64,
now time.Time,
) {
if m == nil || shouldSkipNativeMockStateMetricWrites() {
return
}
if m.metricsHistory != nil {
m.metricsHistory.AddGuestMetric(resourceID, "cpu", cpu, now)
if memory >= 0 {
m.metricsHistory.AddGuestMetric(resourceID, "memory", memory, now)
}
if memoryUsed >= 0 {
m.metricsHistory.AddGuestMetric(resourceID, "memoryused", memoryUsed, now)
}
if diskUsage >= 0 {
m.metricsHistory.AddGuestMetric(resourceID, "disk", diskUsage, now)
}
if diskRead >= 0 {
m.metricsHistory.AddGuestMetric(resourceID, "diskread", float64(diskRead), now)
}
if diskWrite >= 0 {
m.metricsHistory.AddGuestMetric(resourceID, "diskwrite", float64(diskWrite), now)
}
if networkIn >= 0 {
m.metricsHistory.AddGuestMetric(resourceID, "netin", float64(networkIn), now)
}
if networkOut >= 0 {
m.metricsHistory.AddGuestMetric(resourceID, "netout", float64(networkOut), now)
}
}
if m.metricsStore != nil {
m.metricsStore.Write(resourceType, resourceID, "cpu", cpu, now)
if memory >= 0 {
m.metricsStore.Write(resourceType, resourceID, "memory", memory, now)
}
if memoryUsed >= 0 {
m.metricsStore.Write(resourceType, resourceID, "memoryused", memoryUsed, now)
}
if diskUsage >= 0 {
m.metricsStore.Write(resourceType, resourceID, "disk", diskUsage, now)
}
if diskRead >= 0 {
m.metricsStore.Write(resourceType, resourceID, "diskread", float64(diskRead), now)
}
if diskWrite >= 0 {
m.metricsStore.Write(resourceType, resourceID, "diskwrite", float64(diskWrite), now)
}
if networkIn >= 0 {
m.metricsStore.Write(resourceType, resourceID, "netin", float64(networkIn), now)
}
if networkOut >= 0 {
m.metricsStore.Write(resourceType, resourceID, "netout", float64(networkOut), now)
}
}
}