Skip to content

armgit / Pulse

Public Imported from GitHub
0

Real-time monitoring dashboard for Proxmox VE, PBS, Docker, Kubernetes, TrueNAS and vSphere. Self-hosted, with alerts for failed backups, full disks, unhealthy containers and offline nodes.

9.6 KB
Raw Blame
package monitoring

import (
	"strings"
	"time"

	"github.com/rcourtman/pulse-go-rewrite/internal/models"
	"github.com/rcourtman/pulse-go-rewrite/internal/unifiedresources"
	"github.com/rcourtman/pulse-go-rewrite/pkg/proxmox"
	"github.com/rs/zerolog/log"
)

func (m *Monitor) preserveGuestsForGracePeriod(
	instanceName string,
	resources []proxmox.ClusterResource,
	prevVMs []models.VM,
	prevContainers []models.Container,
	nodeEffectiveStatus map[string]string,
	allVMs []models.VM,
	allContainers []models.Container,
) ([]models.VM, []models.Container) {
	// Preserve VMs and containers from nodes within grace period
	// The cluster/resources endpoint doesn't return VMs/containers from nodes Proxmox considers offline,
	// but we want to keep showing them if the node is within grace period
	// Count previous resources for this instance
	prevVMCount := len(prevVMs)
	prevContainerCount := len(prevContainers)

	// Build map of which nodes are covered by current resources
	nodesWithResources := make(map[string]bool)
	for _, res := range resources {
		nodesWithResources[res.Node] = true
	}

	log.Debug().
		Str("instance", instanceName).
		Int("nodesInResources", len(nodesWithResources)).
		Int("totalVMsFromResources", len(allVMs)).
		Int("totalContainersFromResources", len(allContainers)).
		Int("prevVMs", prevVMCount).
		Int("prevContainers", prevContainerCount).
		Msg("Cluster resources received, checking for grace period preservation")

	// If we got ZERO resources but had resources before, and we have no node data,
	// this likely means the cluster health check failed. Preserve everything.
	if len(allVMs) == 0 && len(allContainers) == 0 &&
		(prevVMCount > 0 || prevContainerCount > 0) &&
		len(nodeEffectiveStatus) == 0 {
		log.Warn().
			Str("instance", instanceName).
			Int("prevVMs", prevVMCount).
			Int("prevContainers", prevContainerCount).
			Msg("Cluster returned zero resources but had resources before - likely cluster health issue, preserving all previous resources")

		// Preserve all previous VMs and containers for this instance
		allVMs = append(allVMs, prevVMs...)
		allContainers = append(allContainers, prevContainers...)
	}

	// Check for nodes that are within grace period but not in cluster/resources response
	preservedVMCount := 0
	preservedContainerCount := 0
	for nodeName, effectiveStatus := range nodeEffectiveStatus {
		if effectiveStatus == "online" && !nodesWithResources[nodeName] {
			// This node is within grace period but Proxmox didn't return its resources
			// Preserve previous VMs and containers from this node
			vmsBefore := len(allVMs)
			containersBefore := len(allContainers)

			// Preserve VMs from this node
			for _, vm := range prevVMs {
				if vm.Node == nodeName {
					allVMs = append(allVMs, vm)
				}
			}

			// Preserve containers from this node
			for _, container := range prevContainers {
				if container.Node == nodeName {
					allContainers = append(allContainers, container)
				}
			}

			vmsPreserved := len(allVMs) - vmsBefore
			containersPreserved := len(allContainers) - containersBefore
			preservedVMCount += vmsPreserved
			preservedContainerCount += containersPreserved

			log.Info().
				Str("instance", instanceName).
				Str("node", nodeName).
				Int("vmsPreserved", vmsPreserved).
				Int("containersPreserved", containersPreserved).
				Msg("Preserved VMs/containers from node in grace period")
		}
	}

	if preservedVMCount > 0 || preservedContainerCount > 0 {
		log.Info().
			Str("instance", instanceName).
			Int("totalPreservedVMs", preservedVMCount).
			Int("totalPreservedContainers", preservedContainerCount).
			Msg("Grace period preservation complete")
	}

	return allVMs, allContainers
}

// guestObservedInCycle reports whether a guest's telemetry came from this poll
// rather than being carried forward while its node sits in the grace period.
//
// Preserved guests keep the LastSeen of the cycle that actually observed them,
// so anything older than this cycle's start was not observed. Recording one as
// a sample fabricates a reading: the carried-forward projection has no counters,
// so it writes CPU, disk and network zeroes for a guest Pulse cannot currently
// see, and the history shows a collapse to zero instead of a gap.
//
// A zero LastSeen means we have no evidence either way, so this fails open and
// records. Dropping a legitimate sample is the worse error of the two, and
// harder to notice.
func guestObservedInCycle(lastSeen, cycleStart time.Time) bool {
	if lastSeen.IsZero() {
		return true
	}
	return !lastSeen.Before(cycleStart)
}

// recordGuestMetrics records metrics for all running VMs and containers that
// this cycle actually observed.
func (m *Monitor) recordGuestMetrics(allVMs []models.VM, allContainers []models.Container, cycleStart time.Time) {
	now := time.Now()
	for _, vm := range allVMs {
		if !guestObservedInCycle(vm.LastSeen, cycleStart) {
			continue
		}
		if vm.Status == "running" {
			diskRead, diskWrite, networkIn, networkOut := guestHistoryRates(vm.DiskRead, vm.DiskWrite, vm.NetworkIn, vm.NetworkOut, vm.IORateValidity)
			memoryUsage, memoryUsed, diskUsage := historyMemoryUsage(vm.Memory), historyMemoryUsed(vm.Memory), vm.Disk.Usage
			// A failed filesystem read can leave a positive cluster allocation,
			// not just a retained prev- reading. Neither is current usage.
			if vm.DiskStatusReason != "" {
				diskUsage = -1
			}
			// Only legacy unannotated memory needs the diagnostic heuristic.
			// An explicit observation belongs to the selected source: deferred
			// filesystem/QGA work cannot disqualify independent current memory.
			if vm.GuestAgentStatus == "deferred" && vm.Memory.Observation.State == "" {
				snapshot := m.previousGuestSnapshot(vm.Instance, "qemu", vm.Node, vm.VMID)
				if snapshot == nil || snapshot.MemorySource == "previous-snapshot" || strings.HasPrefix(CanonicalMemorySource(snapshot.MemorySource), "guest-agent-meminfo") {
					memoryUsage, memoryUsed = -1, -1
				}
			}
			m.recordGuestMetric("vm", vm.ID, unifiedresources.ProxmoxGuestCPUPercent(vm.CPU), memoryUsage, memoryUsed, diskUsage, diskRead, diskWrite, networkIn, networkOut, now)
		}
	}
	for _, ct := range allContainers {
		if !guestObservedInCycle(ct.LastSeen, cycleStart) {
			continue
		}
		if ct.Status == "running" {
			diskRead, diskWrite, networkIn, networkOut := guestHistoryRates(ct.DiskRead, ct.DiskWrite, ct.NetworkIn, ct.NetworkOut, ct.IORateValidity)
			m.recordGuestMetric("container", ct.ID, unifiedresources.ProxmoxGuestCPUPercent(ct.CPU), historyMemoryUsage(ct.Memory), historyMemoryUsed(ct.Memory), ct.Disk.Usage, diskRead, diskWrite, networkIn, networkOut, now)
		}
	}
}

func guestHistoryRates(diskRead, diskWrite, networkIn, networkOut int64, validity models.IORateValidity) (int64, int64, int64, int64) {
	effective := validity.EffectiveForRates(diskRead, diskWrite, networkIn, networkOut)
	if !effective.DiskRead {
		diskRead = -1
	}
	if !effective.DiskWrite {
		diskWrite = -1
	}
	if !effective.NetworkIn {
		networkIn = -1
	}
	if !effective.NetworkOut {
		networkOut = -1
	}
	return diskRead, diskWrite, networkIn, networkOut
}

// Retained numbers remain useful for display, but a newly refreshed guest row
// cannot turn them into a new percentage or byte observation. The producer owns
// freshness; ordinary current TTL-cache hits and legacy readings remain valid.
func historyMemoryUsage(memory models.Memory) float64 {
	if !memory.HasKnownUsage() || (memory.Observation.State != "" && memory.Observation.State != "current") {
		return -1
	}
	return memory.Usage
}

func historyMemoryUsed(memory models.Memory) float64 {
	if !memory.HasKnownUsage() || (memory.Observation.State != "" && memory.Observation.State != "current") {
		return -1
	}
	return float64(memory.Used)
}

// recordGuestMetric records metrics for a single guest (VM or container) to both
// the in-memory metrics history and the persistent metrics store.
func (m *Monitor) recordGuestMetric(
	resourceType, resourceID string,
	cpu, memory, memoryUsed, diskUsage float64,
	diskRead, diskWrite, networkIn, networkOut int64,
	now time.Time,
) {
	if m == nil || shouldSkipNativeMockStateMetricWrites() {
		return
	}

	if m.metricsHistory != nil {
		m.metricsHistory.AddGuestMetric(resourceID, "cpu", cpu, now)
		if memory >= 0 {
			m.metricsHistory.AddGuestMetric(resourceID, "memory", memory, now)
		}
		if memoryUsed >= 0 {
			m.metricsHistory.AddGuestMetric(resourceID, "memoryused", memoryUsed, now)
		}
		if diskUsage >= 0 {
			m.metricsHistory.AddGuestMetric(resourceID, "disk", diskUsage, now)
		}
		if diskRead >= 0 {
			m.metricsHistory.AddGuestMetric(resourceID, "diskread", float64(diskRead), now)
		}
		if diskWrite >= 0 {
			m.metricsHistory.AddGuestMetric(resourceID, "diskwrite", float64(diskWrite), now)
		}
		if networkIn >= 0 {
			m.metricsHistory.AddGuestMetric(resourceID, "netin", float64(networkIn), now)
		}
		if networkOut >= 0 {
			m.metricsHistory.AddGuestMetric(resourceID, "netout", float64(networkOut), now)
		}
	}

	if m.metricsStore != nil {
		m.metricsStore.Write(resourceType, resourceID, "cpu", cpu, now)
		if memory >= 0 {
			m.metricsStore.Write(resourceType, resourceID, "memory", memory, now)
		}
		if memoryUsed >= 0 {
			m.metricsStore.Write(resourceType, resourceID, "memoryused", memoryUsed, now)
		}
		if diskUsage >= 0 {
			m.metricsStore.Write(resourceType, resourceID, "disk", diskUsage, now)
		}
		if diskRead >= 0 {
			m.metricsStore.Write(resourceType, resourceID, "diskread", float64(diskRead), now)
		}
		if diskWrite >= 0 {
			m.metricsStore.Write(resourceType, resourceID, "diskwrite", float64(diskWrite), now)
		}
		if networkIn >= 0 {
			m.metricsStore.Write(resourceType, resourceID, "netin", float64(networkIn), now)
		}
		if networkOut >= 0 {
			m.metricsStore.Write(resourceType, resourceID, "netout", float64(networkOut), now)
		}
	}
}