Real-time monitoring dashboard for Proxmox VE, PBS, Docker, Kubernetes, TrueNAS and vSphere. Self-hosted, with alerts for failed backups, full disks, unhealthy containers and offline nodes.
Go aialertsdashboarddockerdocker-monitoringgohomelabinfrastructure-monitoringkubernetesmonitoringmspproxmoxproxmox-backup-serverproxmox-mail-gatewayproxmox-veself-hostedtruenastypescriptvspherewebhooks
Preserve guest memory across repeated backup-lock polls
rcourtman0060daf585f9
diff --git a/docs/release-control/v6/internal/subsystems/monitoring.md b/docs/release-control/v6/internal/subsystems/monitoring.mdindex 4394455bd1..c957b50182 100644--- a/docs/release-control/v6/internal/subsystems/monitoring.md+++ b/docs/release-control/v6/internal/subsystems/monitoring.md@@ -138,6 +138,22 @@ Current non-QGA memory, CPU and I/O may still be observed. History never writesretained disk values or deferred cached memory as newly observed telemetry;normal unlocked polling resumes and replaces them with fresh data.+Repeated QGA deferrals preserve the complete last-known memory value, including+truly-free and reclaimable cache, using the original successful memory-read+cache as authority. The ordinary 60-second read TTL schedules refresh; retained+evidence remains bounded by the original cache's ten-minute cleanup age.+Deferral polls never renew that timestamp. Missing, failed, future-dated or+expired observations, a different instance/node/VM, changed capacity, or a+previous value inconsistent with the original read cannot support retention.+Current non-QGA memory remains authoritative when available. The retained+value stays `previous-snapshot` and does not produce fresh memory History.+`TestGuestAgentBackupMonitoringContract` includes eight consecutive locked polls+after ordinary memory-cache expiry, checks complete memory and unchanged cache+timestamps/History, and then exercises fresh unlocked resumption.+`TestDeferredVMGuestMemoryRequiresOriginalEvidence` covers the provenance,+age, identity and availability boundaries, including measured zero usage.+These are synthetic runtime proofs; they do not qualify the native freeze race.+`TestGuestAgentBackupMonitoringContract` exercises healthy → status lock →resource lock with failed status → node fallback → unlocked resumption throughthe real client/builders and History. The client controls cover every read,diff --git a/internal/monitoring/canonical_guardrails_test.go b/internal/monitoring/canonical_guardrails_test.goindex 3971067d22..f8c5d48ec8 100644--- a/internal/monitoring/canonical_guardrails_test.go+++ b/internal/monitoring/canonical_guardrails_test.go@@ -20,6 +20,7 @@ import (unifiedresources "github.com/rcourtman/pulse-go-rewrite/internal/unifiedresources""github.com/rcourtman/pulse-go-rewrite/internal/vmware"agentsdocker "github.com/rcourtman/pulse-go-rewrite/pkg/agents/docker"+ "github.com/rcourtman/pulse-go-rewrite/pkg/proxmox")var bannedSnapshotResourceAccessPatterns = []struct {@@ -2975,6 +2976,71 @@ func TestGuestAgentBackupMonitoringContract(t *testing.T) {testGuestAgentBackupMonitoringLifecycle(t)}+func TestDeferredVMGuestMemoryRequiresOriginalEvidence(t *testing.T) {+ now := time.Now()+ tests := []struct {+ name string+ change func(*agentMemCacheEntry, *GuestMemorySnapshot)+ key string+ want bool+ }{+ {name: "expired read TTL still supports last-known memory", want: true},+ {name: "derived meminfo", want: true, change: func(e *agentMemCacheEntry, _ *GuestMemorySnapshot) { e.info.Source = "meminfo-derived" }},+ {name: "last-known age boundary", want: true, change: func(e *agentMemCacheEntry, _ *GuestMemorySnapshot) { e.fetchedAt = now.Add(-vmAgentMemCleanupMaxAge) }},+ {name: "expired original observation", change: func(e *agentMemCacheEntry, _ *GuestMemorySnapshot) {+ e.fetchedAt = now.Add(-vmAgentMemCleanupMaxAge - time.Nanosecond)+ }},+ {name: "missing observation time", change: func(e *agentMemCacheEntry, _ *GuestMemorySnapshot) { e.fetchedAt = time.Time{} }},+ {name: "future observation time", change: func(e *agentMemCacheEntry, _ *GuestMemorySnapshot) { e.fetchedAt = now.Add(time.Second) }},+ {name: "failed guest read", change: func(e *agentMemCacheEntry, _ *GuestMemorySnapshot) { e.negative = true }},+ {name: "missing source", change: func(e *agentMemCacheEntry, _ *GuestMemorySnapshot) { e.info.Source = "" }},+ {name: "unsupported source", change: func(e *agentMemCacheEntry, _ *GuestMemorySnapshot) { e.info.Source = "unavailable" }},+ {name: "different instance cache", key: "other/node/105"},+ {name: "different node cache", key: "fixture/other/105"},+ {name: "different VM cache", key: "fixture/node/106"},+ {name: "different previous identity", change: func(_ *agentMemCacheEntry, v *GuestMemorySnapshot) { v.VMID++ }},+ {name: "different guest type", change: func(_ *agentMemCacheEntry, v *GuestMemorySnapshot) { v.GuestType = "lxc" }},+ {name: "stopped guest", change: func(_ *agentMemCacheEntry, v *GuestMemorySnapshot) { v.Status = "stopped" }},+ {name: "unrelated memory source", change: func(_ *agentMemCacheEntry, v *GuestMemorySnapshot) { v.MemorySource = "agent" }},+ {name: "previous-snapshot remains bound to original read", want: true, change: func(_ *agentMemCacheEntry, v *GuestMemorySnapshot) { v.MemorySource = "previous-snapshot" }},+ {name: "unknown previous memory", change: func(_ *agentMemCacheEntry, v *GuestMemorySnapshot) { v.Memory = models.UnavailableMemory(100) }},+ {name: "changed capacity", change: func(_ *agentMemCacheEntry, v *GuestMemorySnapshot) { v.Memory.Total++ }},+ {name: "different observation", change: func(_ *agentMemCacheEntry, v *GuestMemorySnapshot) { v.Memory.Used++ }},+ {name: "different free memory observation", change: func(_ *agentMemCacheEntry, v *GuestMemorySnapshot) { v.Memory.Free++ }},+ {name: "different reclaimable cache observation", change: func(_ *agentMemCacheEntry, v *GuestMemorySnapshot) { v.Memory.Cache++ }},+ {name: "impossible availability", change: func(e *agentMemCacheEntry, _ *GuestMemorySnapshot) { e.info.EffectiveAvailable = 101 }},+ {name: "observed zero usage", want: true, change: func(e *agentMemCacheEntry, v *GuestMemorySnapshot) {+ e.info.EffectiveAvailable = 100+ e.info.Free = 20+ v.Memory = models.Memory{Total: 100, Free: 20, Cache: 80}+ }},+ }+ for _, tt := range tests {+ t.Run(tt.name, func(t *testing.T) {+ entry := agentMemCacheEntry{info: proxmox.LinuxMemoryAvailability{Source: "meminfo-available", EffectiveAvailable: 60, Free: 10}, fetchedAt: now.Add(-2 * vmAgentMemCacheTTL)}+ previous := GuestMemorySnapshot{GuestType: "qemu", Status: "running", MemorySource: "guest-agent-meminfo", Instance: "fixture", Node: "node", VMID: 105, Memory: models.Memory{Total: 100, Used: 40, Free: 10, Cache: 50, Usage: 40}}+ if tt.change != nil {+ tt.change(&entry, &previous)+ }+ key := tt.key+ if key == "" {+ key = guestMemoryCacheKey("fixture", "node", 105)+ }+ m := &Monitor{vmAgentMemCache: map[string]agentMemCacheEntry{key: entry}}+ memory, ok := m.deferredVMGuestMemory("fixture", "node", 105, 100, &previous, now)+ if ok != tt.want || (ok && memory != previous.Memory) {+ t.Fatalf("retention = %#v, %t; want previous memory, %t", memory, ok, tt.want)+ }+ if m.vmAgentMemCache[key] != entry {+ t.Fatal("last-known evidence was renewed or changed")+ }+ })+ }+ if _, ok := (&Monitor{}).deferredVMGuestMemory("fixture", "node", 105, 100, nil, now); ok {+ t.Fatal("invented memory without a previous observation")+ }+}+func TestGuestAgentTransportMonitoringContract(t *testing.T) {for _, reason := range []string{"agent-redirect", "agent-transport-unverified"} {for _, prefix := range []string{"", "prev-"} {diff --git a/internal/monitoring/guest_agent_backup_integration_test.go b/internal/monitoring/guest_agent_backup_integration_test.goindex 349103ec48..f13b76eca4 100644--- a/internal/monitoring/guest_agent_backup_integration_test.go+++ b/internal/monitoring/guest_agent_backup_integration_test.go@@ -15,6 +15,7 @@ import ("github.com/rcourtman/pulse-go-rewrite/internal/alerts""github.com/rcourtman/pulse-go-rewrite/internal/config""github.com/rcourtman/pulse-go-rewrite/internal/models"+ "github.com/rcourtman/pulse-go-rewrite/internal/unifiedresources""github.com/rcourtman/pulse-go-rewrite/pkg/proxmox")@@ -52,7 +53,7 @@ func testGuestAgentBackupMonitoringLifecycle(t *testing.T) {if p == 3 {available = 4 * 1024}- json.NewEncoder(w).Encode(map[string]any{"data": map[string]any{"content": fmt.Sprintf("MemTotal: 8192 kB\nMemAvailable: %d kB\n", available)}})+ json.NewEncoder(w).Encode(map[string]any{"data": map[string]any{"content": fmt.Sprintf("MemTotal: 8192 kB\nMemFree: 1024 kB\nMemAvailable: %d kB\n", available)}})case strings.HasSuffix(r.URL.Path, "get-fsinfo"):used := 300 * mibif p == 3 {@@ -79,6 +80,7 @@ func testGuestAgentBackupMonitoringLifecycle(t *testing.T) {res := proxmox.ClusterResource{Type: "qemu", Node: "node", Name: "guest", VMID: 105, Status: "running", MaxMem: 8 * mib, Mem: 8 * mib, MaxDisk: 1000 * mib, CPU: 0.1}var previous *models.VMidentity := "fixture:node:105"+ registry := unifiedresources.NewRegistry(nil)build := func() (models.VM, string) {vm, raw, source, notes, at, ok := m.buildVMFromClusterResource(context.Background(), "fixture", res, client, identity, nil, previous)if !ok {@@ -86,7 +88,15 @@ func testGuestAgentBackupMonitoringLifecycle(t *testing.T) {}m.recordGuestSnapshot("fixture", "qemu", "node", 105, GuestMemorySnapshot{Name: vm.Name, Status: vm.Status, RetrievedAt: at, MemorySource: source, Memory: vm.Memory, Raw: raw, Notes: notes})m.recordGuestMetrics([]models.VM{vm}, nil, time.Now().Add(-time.Second))- previous = &vm+ // Production obtains previous guest context from the unified read view,+ // not the builder's raw model (the context need not contain memory).+ registry.IngestSnapshot(models.StateSnapshot{VMs: []models.VM{vm}})+ views := registry.VMs()+ if len(views) != 1 || views[0].MemoryUsed() != vm.Memory.Used {+ t.Fatal("memory did not reach the unified read view")+ }+ next := previousVMFromView(views[0])+ previous = &nextreturn vm, source}initial, source := build()@@ -119,12 +129,21 @@ func testGuestAgentBackupMonitoringLifecycle(t *testing.T) {}phase.Store(1)res.CPU = 0.2- locked, source := build()- if locked.ID != initial.ID || locked.GuestAgentStatus != "deferred" || locked.DiskStatusReason != "prev-vm-locked" {- t.Fatalf("lock lost continuity/truth: %#v", locked)- }- if locked.Memory.Used != initial.Memory.Used || source != "previous-snapshot" || locked.Disk.Used != initial.Disk.Used || !reflect.DeepEqual(locked.NetworkInterfaces, initial.NetworkInterfaces) {- t.Fatalf("last known evidence lost: %s %#v", source, locked)+ // A normal backup outlives the ordinary memory-read TTL. Deferrals must+ // keep using the original observation, never the previous poll's fallback+ // trust label or timestamp. Include truly-free/cache evidence in the check.+ originalMemory.fetchedAt = time.Now().Add(-2 * vmAgentMemCacheTTL)+ m.vmAgentMemCache[memKey] = originalMemory+ const lockedPolls = 8+ var locked models.VM+ for poll := 0; poll < lockedPolls; poll++ {+ locked, source = build()+ if locked.ID != initial.ID || locked.GuestAgentStatus != "deferred" || locked.DiskStatusReason != "prev-vm-locked" {+ t.Fatalf("locked poll %d lost continuity/truth: %#v", poll, locked)+ }+ if locked.Memory != initial.Memory || source != "previous-snapshot" || locked.Disk.Used != initial.Disk.Used || !reflect.DeepEqual(locked.NetworkInterfaces, initial.NetworkInterfaces) {+ t.Fatalf("locked poll %d lost last-known evidence: source=%s memory=%#v want=%#v", poll, source, locked.Memory, initial.Memory)+ }}if calls.Load() != before {t.Fatalf("status-lock sent agent commands: %d -> %d", before, calls.Load())@@ -132,7 +151,7 @@ func testGuestAgentBackupMonitoringLifecycle(t *testing.T) {if !reflect.DeepEqual(m.guestMetadataCache[metadataKey], originalMetadata) || !reflect.DeepEqual(m.vmAgentMemCache[memKey], originalMemory) {t.Fatal("lock renewed/replaced cached observations")}- for metric, want := range map[string]int{"cpu": 3, "memory": 1, "memoryused": 1, "disk": 1} {+ for metric, want := range map[string]int{"cpu": lockedPolls + 2, "memory": 1, "memoryused": 1, "disk": 1} {if got := len(m.metricsHistory.GetGuestMetrics(identity, metric, time.Hour)); got != want {t.Errorf("%s history points = %d, want %d", metric, got, want)}@@ -143,7 +162,7 @@ func testGuestAgentBackupMonitoringLifecycle(t *testing.T) {t.Fatal(err)}locked, _ = build()- if calls.Load() != before || locked.GuestAgentStatus != "deferred" || locked.AgentVersion != initial.AgentVersion {+ if calls.Load() != before || locked.GuestAgentStatus != "deferred" || locked.AgentVersion != initial.AgentVersion || locked.Memory != initial.Memory {t.Fatalf("resource-lock fallback queried/lost identity: calls=%d %#v", calls.Load(), locked)}// The per-node inventory fallback must propagate its lock to the same builder.diff --git a/internal/monitoring/guest_memory_agent.go b/internal/monitoring/guest_memory_agent.goindex 3439899636..f237c9e465 100644--- a/internal/monitoring/guest_memory_agent.go+++ b/internal/monitoring/guest_memory_agent.go@@ -7,6 +7,7 @@ import ("strings""time"+ "github.com/rcourtman/pulse-go-rewrite/internal/models""github.com/rcourtman/pulse-go-rewrite/pkg/proxmox")@@ -37,6 +38,39 @@ func guestMemoryCacheKey(instanceName, node string, vmid int) string {return fmt.Sprintf("%s/%s/%d", instanceName, node, vmid)}+// deferredVMGuestMemory retains a rendered observation only while the original+// successful guest read still supports it. The ordinary read TTL schedules a+// refresh; the cleanup age bounds last-known evidence during coordination.+// Neither a poll timestamp nor a previous-snapshot trust label renews that age.+func (m *Monitor) deferredVMGuestMemory(instanceName, node string, vmid int, total uint64, previous *GuestMemorySnapshot, now time.Time) (models.Memory, bool) {+ if previous == nil || previous.GuestType != "qemu" || previous.Status != "running" || previous.Instance != instanceName || previous.Node != node || previous.VMID != vmid || !previous.Memory.HasKnownUsage() || uint64(previous.Memory.Total) != total {+ return models.Memory{}, false+ }+ switch CanonicalMemorySource(previous.MemorySource) {+ case "guest-agent-meminfo", "guest-agent-meminfo-derived", "previous-snapshot":+ default:+ return models.Memory{}, false+ }+ m.rrdCacheMu.RLock()+ entry, ok := m.vmAgentMemCache[guestMemoryCacheKey(instanceName, node, vmid)]+ m.rrdCacheMu.RUnlock()+ if !ok || entry.negative || entry.fetchedAt.IsZero() || entry.fetchedAt.After(now) || now.Sub(entry.fetchedAt) > vmAgentMemCleanupMaxAge {+ return models.Memory{}, false+ }+ if entry.info.Source != "meminfo-available" && entry.info.Source != "meminfo-derived" {+ return models.Memory{}, false+ }+ if entry.info.EffectiveAvailable > total || previous.Memory.Used != int64(total-entry.info.EffectiveAvailable) {+ return models.Memory{}, false+ }+ expected := models.Memory{Free: int64(entry.info.EffectiveAvailable)}+ splitReclaimableMemory(&expected, entry.info.Free)+ if previous.Memory.Free != expected.Free || previous.Memory.Cache != expected.Cache {+ return models.Memory{}, false+ }+ return previous.Memory, true+}+func (m *Monitor) getVMAgentMemAvailable(ctx context.Context, client PVEClientInterface, instanceName, node string, vmid int) (uint64, error) {info, err := m.getVMAgentMemoryAvailability(ctx, client, instanceName, node, vmid)return info.EffectiveAvailable, errdiff --git a/internal/monitoring/monitor_pve_guest_builders.go b/internal/monitoring/monitor_pve_guest_builders.goindex 30081fbb2d..21aba527c7 100644--- a/internal/monitoring/monitor_pve_guest_builders.go+++ b/internal/monitoring/monitor_pve_guest_builders.go@@ -461,6 +461,13 @@ func (m *Monitor) buildVMFromClusterResource(if state.detailedStatus != nil && state.detailedStatus.Balloon > 0 {memory.Balloon = int64(state.detailedStatus.Balloon)}+ if state.guestAgentStatus == "deferred" && (CanonicalMemorySource(state.memorySource) == "unavailable" || state.memorySource == "previous-snapshot") {+ if retained, ok := m.deferredVMGuestMemory(instanceName, res.Node, res.VMID, state.memTotal, prevSnapshot, sampleTime); ok {+ memory = retained+ state.memorySource = "previous-snapshot"+ snapshotNotes = append(snapshotNotes, "preserved-original-guest-memory-during-deferral")+ }+ }vm := models.VM{ID: guestID,