Real-time monitoring dashboard for Proxmox VE, PBS, Docker, Kubernetes, TrueNAS and vSphere. Self-hosted, with alerts for failed backups, full disks, unhealthy containers and offline nodes.
Go aialertsdashboarddockerdocker-monitoringgohomelabinfrastructure-monitoringkubernetesmonitoringmspproxmoxproxmox-backup-serverproxmox-mail-gatewayproxmox-veself-hostedtruenastypescriptvspherewebhooks
package monitoring
import (
"math"
"time"
"github.com/rcourtman/pulse-go-rewrite/internal/models"
"github.com/rcourtman/pulse-go-rewrite/pkg/proxmox"
)
const (
guestMemoryCarryForwardMaxAge = 2 * time.Minute
guestMemoryHealthyGuestMaxAge = 10 * time.Minute
guestMemoryCarryForwardMinUsageDelta = 5.0
guestMemoryReliabilityLow = 0
guestMemoryReliabilityFallback = 1
guestMemoryReliabilityTrusted = 2
)
func guestMemorySourceReliability(source string) int {
switch CanonicalMemorySource(source) {
case "available-field", "derived-free-buffers-cached",
"guest-agent-meminfo", "guest-agent-meminfo-derived", "agent":
return guestMemoryReliabilityTrusted
case "derived-total-minus-used", "previous-snapshot":
return guestMemoryReliabilityFallback
case "unknown", "unavailable", "cluster-resources", "status-mem", "status-freemem", "status-unavailable":
return guestMemoryReliabilityLow
default:
return guestMemoryReliabilityFallback
}
}
// Configured capacity fences a VM resize; it is not the denominator of an
// in-guest reading. Older snapshots and containers keep their existing total.
func guestMemoryConfiguredCapacity(previous *GuestMemorySnapshot) uint64 {
if previous == nil || previous.Memory.Total <= 0 {
return 0
}
if previous.GuestType == "qemu" {
if previous.Raw.StatusMaxMem > 0 {
return previous.Raw.StatusMaxMem
}
if previous.Raw.ListingMaxMem > 0 {
return previous.Raw.ListingMaxMem
}
}
return uint64(previous.Memory.Total)
}
func guestMemoryCapacityMatches(previous *GuestMemorySnapshot, currentTotal uint64) bool {
return previous != nil && previous.Memory.Total > 0 &&
(uint64(previous.Memory.Total) == currentTotal || guestMemoryConfiguredCapacity(previous) == currentTotal)
}
func (m *Monitor) previousGuestSnapshot(instance, guestType, node string, vmid int) *GuestMemorySnapshot {
if m == nil {
return nil
}
key := makeGuestSnapshotKey(instance, guestType, node, vmid)
m.diagMu.RLock()
snapshot, ok := m.guestSnapshots[key]
m.diagMu.RUnlock()
if !ok {
return nil
}
normalized := normalizeGuestMemorySnapshot(snapshot)
return &normalized
}
func guestAgentSignalsHealthy(
detailedStatus *proxmox.VMStatus,
diskFromAgent bool,
ipAddresses []string,
networkInterfaces []models.GuestNetworkInterface,
osName, osVersion, agentVersion string,
) bool {
if detailedStatus != nil && detailedStatus.Agent.IsAvailable() {
return true
}
return diskFromAgent ||
len(ipAddresses) > 0 ||
len(networkInterfaces) > 0 ||
osName != "" ||
osVersion != "" ||
agentVersion != ""
}
func stabilizeGuestLowTrustMemory(
prev *GuestMemorySnapshot,
currentStatus string,
currentSource string,
currentTotal uint64,
currentUsed uint64,
now time.Time,
guestAgentHealthy bool,
) (uint64, string, []string) {
if shouldCarryForwardPreviousGuestMemory(prev, currentStatus, currentSource, currentTotal, currentUsed, now) {
return uint64(prev.Memory.Used), "previous-snapshot", []string{"preserved-previous-memory-after-repeated-low-trust-pattern"}
}
if shouldCarryForwardHealthyGuestLowTrustMemory(prev, currentStatus, currentSource, currentTotal, currentUsed, now, guestAgentHealthy) {
return uint64(prev.Memory.Used), "previous-snapshot", []string{"preserved-previous-memory-for-healthy-guest-low-trust-full-usage"}
}
return currentUsed, currentSource, nil
}
// A retained snapshot is a wrapper around an original observation, not a
// downgrade of that observation's origin. The caller still checks its original
// age and capacity; neither this wrapper nor RetrievedAt can renew the sample.
func guestMemorySnapshotReliability(prev *GuestMemorySnapshot) int {
source := CanonicalMemorySource(prev.MemorySource)
if source == "previous-snapshot" && (prev.Memory.Observation.State == "current" || prev.Memory.Observation.State == "last-known") {
source = CanonicalMemorySource(prev.Memory.Observation.Source)
}
return guestMemorySourceReliability(source)
}
func shouldCarryForwardPreviousGuestMemory(prev *GuestMemorySnapshot, currentStatus, currentSource string, currentTotal, currentUsed uint64, now time.Time) bool {
if prev == nil || currentStatus != "running" || prev.Status != "running" {
return false
}
if prev.Memory.Total <= 0 || prev.Memory.Used < 0 {
return false
}
if !guestMemorySnapshotWithinAge(prev, now, guestMemoryCarryForwardMaxAge) {
return false
}
prevReliability := guestMemorySnapshotReliability(prev)
currentReliability := guestMemorySourceReliability(currentSource)
if prevReliability < guestMemoryReliabilityTrusted || currentReliability > guestMemoryReliabilityFallback {
return false
}
if prevReliability <= currentReliability {
return false
}
if currentTotal > 0 && !guestMemoryCapacityMatches(prev, currentTotal) {
return false
}
currentUsage := safePercentage(float64(currentUsed), float64(currentTotal))
if prev.Memory.Usage > 0 && math.Abs(prev.Memory.Usage-currentUsage) < guestMemoryCarryForwardMinUsageDelta {
return false
}
return true
}
func shouldCarryForwardHealthyGuestLowTrustMemory(prev *GuestMemorySnapshot, currentStatus, currentSource string, currentTotal, currentUsed uint64, now time.Time, guestAgentHealthy bool) bool {
if prev == nil || !guestAgentHealthy || currentStatus != "running" || prev.Status != "running" {
return false
}
if prev.Memory.Total <= 0 || prev.Memory.Used < 0 {
return false
}
if !guestMemorySnapshotWithinAge(prev, now, guestMemoryHealthyGuestMaxAge) {
return false
}
if currentTotal == 0 || !guestMemoryCapacityMatches(prev, currentTotal) {
return false
}
if guestMemorySourceReliability(currentSource) != guestMemoryReliabilityLow {
return false
}
currentUsage := safePercentage(float64(currentUsed), float64(currentTotal))
if currentUsage < 99 {
return false
}
if prev.Memory.Usage >= 90 || math.Abs(prev.Memory.Usage-currentUsage) < guestMemoryCarryForwardMinUsageDelta {
return false
}
prevReliability := guestMemorySourceReliability(prev.MemorySource)
if CanonicalMemorySource(prev.MemorySource) != "previous-snapshot" && prevReliability < guestMemoryReliabilityTrusted {
return false
}
return true
}
// The diagnostic snapshot is refreshed on every poll, including when it only
// carries an old reading forward. It cannot extend the lifetime of that reading.
// Legacy direct readings can use their receipt time once; a legacy retained
// value has lost its original age and cannot borrow the latest poll's time.
func guestMemorySnapshotWithinAge(prev *GuestMemorySnapshot, now time.Time, maxAge time.Duration) bool {
observation := prev.Memory.Observation
observedAt := observation.ObservedAt
if observation.State != "" || observation.Source != "" || !observedAt.IsZero() {
if observation.State != "current" && observation.State != "last-known" {
return false
}
} else {
if CanonicalMemorySource(prev.MemorySource) == "previous-snapshot" {
return false
}
observedAt = prev.RetrievedAt
}
return !observedAt.IsZero() && !observedAt.After(now) && now.Sub(observedAt) <= maxAge
}