Skip to content

armgit / Pulse

Public Imported from GitHub
0

Real-time monitoring dashboard for Proxmox VE, PBS, Docker, Kubernetes, TrueNAS and vSphere. Self-hosted, with alerts for failed backups, full disks, unhealthy containers and offline nodes.

Preserve guest memory across repeated backup-lock polls

rcourtman0060daf585f9
diff --git a/docs/release-control/v6/internal/subsystems/monitoring.md b/docs/release-control/v6/internal/subsystems/monitoring.md
index 4394455bd1..c957b50182 100644
--- a/docs/release-control/v6/internal/subsystems/monitoring.md
+++ b/docs/release-control/v6/internal/subsystems/monitoring.md
@@ -138,6 +138,22 @@ Current non-QGA memory, CPU and I/O may still be observed. History never writes
retained disk values or deferred cached memory as newly observed telemetry;
normal unlocked polling resumes and replaces them with fresh data.
+Repeated QGA deferrals preserve the complete last-known memory value, including
+truly-free and reclaimable cache, using the original successful memory-read
+cache as authority. The ordinary 60-second read TTL schedules refresh; retained
+evidence remains bounded by the original cache's ten-minute cleanup age.
+Deferral polls never renew that timestamp. Missing, failed, future-dated or
+expired observations, a different instance/node/VM, changed capacity, or a
+previous value inconsistent with the original read cannot support retention.
+Current non-QGA memory remains authoritative when available. The retained
+value stays `previous-snapshot` and does not produce fresh memory History.
+`TestGuestAgentBackupMonitoringContract` includes eight consecutive locked polls
+after ordinary memory-cache expiry, checks complete memory and unchanged cache
+timestamps/History, and then exercises fresh unlocked resumption.
+`TestDeferredVMGuestMemoryRequiresOriginalEvidence` covers the provenance,
+age, identity and availability boundaries, including measured zero usage.
+These are synthetic runtime proofs; they do not qualify the native freeze race.
+
`TestGuestAgentBackupMonitoringContract` exercises healthy → status lock →
resource lock with failed status → node fallback → unlocked resumption through
the real client/builders and History. The client controls cover every read,
diff --git a/internal/monitoring/canonical_guardrails_test.go b/internal/monitoring/canonical_guardrails_test.go
index 3971067d22..f8c5d48ec8 100644
--- a/internal/monitoring/canonical_guardrails_test.go
+++ b/internal/monitoring/canonical_guardrails_test.go
@@ -20,6 +20,7 @@ import (
unifiedresources "github.com/rcourtman/pulse-go-rewrite/internal/unifiedresources"
"github.com/rcourtman/pulse-go-rewrite/internal/vmware"
agentsdocker "github.com/rcourtman/pulse-go-rewrite/pkg/agents/docker"
+ "github.com/rcourtman/pulse-go-rewrite/pkg/proxmox"
)
var bannedSnapshotResourceAccessPatterns = []struct {
@@ -2975,6 +2976,71 @@ func TestGuestAgentBackupMonitoringContract(t *testing.T) {
testGuestAgentBackupMonitoringLifecycle(t)
}
+func TestDeferredVMGuestMemoryRequiresOriginalEvidence(t *testing.T) {
+ now := time.Now()
+ tests := []struct {
+ name string
+ change func(*agentMemCacheEntry, *GuestMemorySnapshot)
+ key string
+ want bool
+ }{
+ {name: "expired read TTL still supports last-known memory", want: true},
+ {name: "derived meminfo", want: true, change: func(e *agentMemCacheEntry, _ *GuestMemorySnapshot) { e.info.Source = "meminfo-derived" }},
+ {name: "last-known age boundary", want: true, change: func(e *agentMemCacheEntry, _ *GuestMemorySnapshot) { e.fetchedAt = now.Add(-vmAgentMemCleanupMaxAge) }},
+ {name: "expired original observation", change: func(e *agentMemCacheEntry, _ *GuestMemorySnapshot) {
+ e.fetchedAt = now.Add(-vmAgentMemCleanupMaxAge - time.Nanosecond)
+ }},
+ {name: "missing observation time", change: func(e *agentMemCacheEntry, _ *GuestMemorySnapshot) { e.fetchedAt = time.Time{} }},
+ {name: "future observation time", change: func(e *agentMemCacheEntry, _ *GuestMemorySnapshot) { e.fetchedAt = now.Add(time.Second) }},
+ {name: "failed guest read", change: func(e *agentMemCacheEntry, _ *GuestMemorySnapshot) { e.negative = true }},
+ {name: "missing source", change: func(e *agentMemCacheEntry, _ *GuestMemorySnapshot) { e.info.Source = "" }},
+ {name: "unsupported source", change: func(e *agentMemCacheEntry, _ *GuestMemorySnapshot) { e.info.Source = "unavailable" }},
+ {name: "different instance cache", key: "other/node/105"},
+ {name: "different node cache", key: "fixture/other/105"},
+ {name: "different VM cache", key: "fixture/node/106"},
+ {name: "different previous identity", change: func(_ *agentMemCacheEntry, v *GuestMemorySnapshot) { v.VMID++ }},
+ {name: "different guest type", change: func(_ *agentMemCacheEntry, v *GuestMemorySnapshot) { v.GuestType = "lxc" }},
+ {name: "stopped guest", change: func(_ *agentMemCacheEntry, v *GuestMemorySnapshot) { v.Status = "stopped" }},
+ {name: "unrelated memory source", change: func(_ *agentMemCacheEntry, v *GuestMemorySnapshot) { v.MemorySource = "agent" }},
+ {name: "previous-snapshot remains bound to original read", want: true, change: func(_ *agentMemCacheEntry, v *GuestMemorySnapshot) { v.MemorySource = "previous-snapshot" }},
+ {name: "unknown previous memory", change: func(_ *agentMemCacheEntry, v *GuestMemorySnapshot) { v.Memory = models.UnavailableMemory(100) }},
+ {name: "changed capacity", change: func(_ *agentMemCacheEntry, v *GuestMemorySnapshot) { v.Memory.Total++ }},
+ {name: "different observation", change: func(_ *agentMemCacheEntry, v *GuestMemorySnapshot) { v.Memory.Used++ }},
+ {name: "different free memory observation", change: func(_ *agentMemCacheEntry, v *GuestMemorySnapshot) { v.Memory.Free++ }},
+ {name: "different reclaimable cache observation", change: func(_ *agentMemCacheEntry, v *GuestMemorySnapshot) { v.Memory.Cache++ }},
+ {name: "impossible availability", change: func(e *agentMemCacheEntry, _ *GuestMemorySnapshot) { e.info.EffectiveAvailable = 101 }},
+ {name: "observed zero usage", want: true, change: func(e *agentMemCacheEntry, v *GuestMemorySnapshot) {
+ e.info.EffectiveAvailable = 100
+ e.info.Free = 20
+ v.Memory = models.Memory{Total: 100, Free: 20, Cache: 80}
+ }},
+ }
+ for _, tt := range tests {
+ t.Run(tt.name, func(t *testing.T) {
+ entry := agentMemCacheEntry{info: proxmox.LinuxMemoryAvailability{Source: "meminfo-available", EffectiveAvailable: 60, Free: 10}, fetchedAt: now.Add(-2 * vmAgentMemCacheTTL)}
+ previous := GuestMemorySnapshot{GuestType: "qemu", Status: "running", MemorySource: "guest-agent-meminfo", Instance: "fixture", Node: "node", VMID: 105, Memory: models.Memory{Total: 100, Used: 40, Free: 10, Cache: 50, Usage: 40}}
+ if tt.change != nil {
+ tt.change(&entry, &previous)
+ }
+ key := tt.key
+ if key == "" {
+ key = guestMemoryCacheKey("fixture", "node", 105)
+ }
+ m := &Monitor{vmAgentMemCache: map[string]agentMemCacheEntry{key: entry}}
+ memory, ok := m.deferredVMGuestMemory("fixture", "node", 105, 100, &previous, now)
+ if ok != tt.want || (ok && memory != previous.Memory) {
+ t.Fatalf("retention = %#v, %t; want previous memory, %t", memory, ok, tt.want)
+ }
+ if m.vmAgentMemCache[key] != entry {
+ t.Fatal("last-known evidence was renewed or changed")
+ }
+ })
+ }
+ if _, ok := (&Monitor{}).deferredVMGuestMemory("fixture", "node", 105, 100, nil, now); ok {
+ t.Fatal("invented memory without a previous observation")
+ }
+}
+
func TestGuestAgentTransportMonitoringContract(t *testing.T) {
for _, reason := range []string{"agent-redirect", "agent-transport-unverified"} {
for _, prefix := range []string{"", "prev-"} {
diff --git a/internal/monitoring/guest_agent_backup_integration_test.go b/internal/monitoring/guest_agent_backup_integration_test.go
index 349103ec48..f13b76eca4 100644
--- a/internal/monitoring/guest_agent_backup_integration_test.go
+++ b/internal/monitoring/guest_agent_backup_integration_test.go
@@ -15,6 +15,7 @@ import (
"github.com/rcourtman/pulse-go-rewrite/internal/alerts"
"github.com/rcourtman/pulse-go-rewrite/internal/config"
"github.com/rcourtman/pulse-go-rewrite/internal/models"
+ "github.com/rcourtman/pulse-go-rewrite/internal/unifiedresources"
"github.com/rcourtman/pulse-go-rewrite/pkg/proxmox"
)
@@ -52,7 +53,7 @@ func testGuestAgentBackupMonitoringLifecycle(t *testing.T) {
if p == 3 {
available = 4 * 1024
}
- json.NewEncoder(w).Encode(map[string]any{"data": map[string]any{"content": fmt.Sprintf("MemTotal: 8192 kB\nMemAvailable: %d kB\n", available)}})
+ json.NewEncoder(w).Encode(map[string]any{"data": map[string]any{"content": fmt.Sprintf("MemTotal: 8192 kB\nMemFree: 1024 kB\nMemAvailable: %d kB\n", available)}})
case strings.HasSuffix(r.URL.Path, "get-fsinfo"):
used := 300 * mib
if p == 3 {
@@ -79,6 +80,7 @@ func testGuestAgentBackupMonitoringLifecycle(t *testing.T) {
res := proxmox.ClusterResource{Type: "qemu", Node: "node", Name: "guest", VMID: 105, Status: "running", MaxMem: 8 * mib, Mem: 8 * mib, MaxDisk: 1000 * mib, CPU: 0.1}
var previous *models.VM
identity := "fixture:node:105"
+ registry := unifiedresources.NewRegistry(nil)
build := func() (models.VM, string) {
vm, raw, source, notes, at, ok := m.buildVMFromClusterResource(context.Background(), "fixture", res, client, identity, nil, previous)
if !ok {
@@ -86,7 +88,15 @@ func testGuestAgentBackupMonitoringLifecycle(t *testing.T) {
}
m.recordGuestSnapshot("fixture", "qemu", "node", 105, GuestMemorySnapshot{Name: vm.Name, Status: vm.Status, RetrievedAt: at, MemorySource: source, Memory: vm.Memory, Raw: raw, Notes: notes})
m.recordGuestMetrics([]models.VM{vm}, nil, time.Now().Add(-time.Second))
- previous = &vm
+ // Production obtains previous guest context from the unified read view,
+ // not the builder's raw model (the context need not contain memory).
+ registry.IngestSnapshot(models.StateSnapshot{VMs: []models.VM{vm}})
+ views := registry.VMs()
+ if len(views) != 1 || views[0].MemoryUsed() != vm.Memory.Used {
+ t.Fatal("memory did not reach the unified read view")
+ }
+ next := previousVMFromView(views[0])
+ previous = &next
return vm, source
}
initial, source := build()
@@ -119,12 +129,21 @@ func testGuestAgentBackupMonitoringLifecycle(t *testing.T) {
}
phase.Store(1)
res.CPU = 0.2
- locked, source := build()
- if locked.ID != initial.ID || locked.GuestAgentStatus != "deferred" || locked.DiskStatusReason != "prev-vm-locked" {
- t.Fatalf("lock lost continuity/truth: %#v", locked)
- }
- if locked.Memory.Used != initial.Memory.Used || source != "previous-snapshot" || locked.Disk.Used != initial.Disk.Used || !reflect.DeepEqual(locked.NetworkInterfaces, initial.NetworkInterfaces) {
- t.Fatalf("last known evidence lost: %s %#v", source, locked)
+ // A normal backup outlives the ordinary memory-read TTL. Deferrals must
+ // keep using the original observation, never the previous poll's fallback
+ // trust label or timestamp. Include truly-free/cache evidence in the check.
+ originalMemory.fetchedAt = time.Now().Add(-2 * vmAgentMemCacheTTL)
+ m.vmAgentMemCache[memKey] = originalMemory
+ const lockedPolls = 8
+ var locked models.VM
+ for poll := 0; poll < lockedPolls; poll++ {
+ locked, source = build()
+ if locked.ID != initial.ID || locked.GuestAgentStatus != "deferred" || locked.DiskStatusReason != "prev-vm-locked" {
+ t.Fatalf("locked poll %d lost continuity/truth: %#v", poll, locked)
+ }
+ if locked.Memory != initial.Memory || source != "previous-snapshot" || locked.Disk.Used != initial.Disk.Used || !reflect.DeepEqual(locked.NetworkInterfaces, initial.NetworkInterfaces) {
+ t.Fatalf("locked poll %d lost last-known evidence: source=%s memory=%#v want=%#v", poll, source, locked.Memory, initial.Memory)
+ }
}
if calls.Load() != before {
t.Fatalf("status-lock sent agent commands: %d -> %d", before, calls.Load())
@@ -132,7 +151,7 @@ func testGuestAgentBackupMonitoringLifecycle(t *testing.T) {
if !reflect.DeepEqual(m.guestMetadataCache[metadataKey], originalMetadata) || !reflect.DeepEqual(m.vmAgentMemCache[memKey], originalMemory) {
t.Fatal("lock renewed/replaced cached observations")
}
- for metric, want := range map[string]int{"cpu": 3, "memory": 1, "memoryused": 1, "disk": 1} {
+ for metric, want := range map[string]int{"cpu": lockedPolls + 2, "memory": 1, "memoryused": 1, "disk": 1} {
if got := len(m.metricsHistory.GetGuestMetrics(identity, metric, time.Hour)); got != want {
t.Errorf("%s history points = %d, want %d", metric, got, want)
}
@@ -143,7 +162,7 @@ func testGuestAgentBackupMonitoringLifecycle(t *testing.T) {
t.Fatal(err)
}
locked, _ = build()
- if calls.Load() != before || locked.GuestAgentStatus != "deferred" || locked.AgentVersion != initial.AgentVersion {
+ if calls.Load() != before || locked.GuestAgentStatus != "deferred" || locked.AgentVersion != initial.AgentVersion || locked.Memory != initial.Memory {
t.Fatalf("resource-lock fallback queried/lost identity: calls=%d %#v", calls.Load(), locked)
}
// The per-node inventory fallback must propagate its lock to the same builder.
diff --git a/internal/monitoring/guest_memory_agent.go b/internal/monitoring/guest_memory_agent.go
index 3439899636..f237c9e465 100644
--- a/internal/monitoring/guest_memory_agent.go
+++ b/internal/monitoring/guest_memory_agent.go
@@ -7,6 +7,7 @@ import (
"strings"
"time"
+ "github.com/rcourtman/pulse-go-rewrite/internal/models"
"github.com/rcourtman/pulse-go-rewrite/pkg/proxmox"
)
@@ -37,6 +38,39 @@ func guestMemoryCacheKey(instanceName, node string, vmid int) string {
return fmt.Sprintf("%s/%s/%d", instanceName, node, vmid)
}
+// deferredVMGuestMemory retains a rendered observation only while the original
+// successful guest read still supports it. The ordinary read TTL schedules a
+// refresh; the cleanup age bounds last-known evidence during coordination.
+// Neither a poll timestamp nor a previous-snapshot trust label renews that age.
+func (m *Monitor) deferredVMGuestMemory(instanceName, node string, vmid int, total uint64, previous *GuestMemorySnapshot, now time.Time) (models.Memory, bool) {
+ if previous == nil || previous.GuestType != "qemu" || previous.Status != "running" || previous.Instance != instanceName || previous.Node != node || previous.VMID != vmid || !previous.Memory.HasKnownUsage() || uint64(previous.Memory.Total) != total {
+ return models.Memory{}, false
+ }
+ switch CanonicalMemorySource(previous.MemorySource) {
+ case "guest-agent-meminfo", "guest-agent-meminfo-derived", "previous-snapshot":
+ default:
+ return models.Memory{}, false
+ }
+ m.rrdCacheMu.RLock()
+ entry, ok := m.vmAgentMemCache[guestMemoryCacheKey(instanceName, node, vmid)]
+ m.rrdCacheMu.RUnlock()
+ if !ok || entry.negative || entry.fetchedAt.IsZero() || entry.fetchedAt.After(now) || now.Sub(entry.fetchedAt) > vmAgentMemCleanupMaxAge {
+ return models.Memory{}, false
+ }
+ if entry.info.Source != "meminfo-available" && entry.info.Source != "meminfo-derived" {
+ return models.Memory{}, false
+ }
+ if entry.info.EffectiveAvailable > total || previous.Memory.Used != int64(total-entry.info.EffectiveAvailable) {
+ return models.Memory{}, false
+ }
+ expected := models.Memory{Free: int64(entry.info.EffectiveAvailable)}
+ splitReclaimableMemory(&expected, entry.info.Free)
+ if previous.Memory.Free != expected.Free || previous.Memory.Cache != expected.Cache {
+ return models.Memory{}, false
+ }
+ return previous.Memory, true
+}
+
func (m *Monitor) getVMAgentMemAvailable(ctx context.Context, client PVEClientInterface, instanceName, node string, vmid int) (uint64, error) {
info, err := m.getVMAgentMemoryAvailability(ctx, client, instanceName, node, vmid)
return info.EffectiveAvailable, err
diff --git a/internal/monitoring/monitor_pve_guest_builders.go b/internal/monitoring/monitor_pve_guest_builders.go
index 30081fbb2d..21aba527c7 100644
--- a/internal/monitoring/monitor_pve_guest_builders.go
+++ b/internal/monitoring/monitor_pve_guest_builders.go
@@ -461,6 +461,13 @@ func (m *Monitor) buildVMFromClusterResource(
if state.detailedStatus != nil && state.detailedStatus.Balloon > 0 {
memory.Balloon = int64(state.detailedStatus.Balloon)
}
+ if state.guestAgentStatus == "deferred" && (CanonicalMemorySource(state.memorySource) == "unavailable" || state.memorySource == "previous-snapshot") {
+ if retained, ok := m.deferredVMGuestMemory(instanceName, res.Node, res.VMID, state.memTotal, prevSnapshot, sampleTime); ok {
+ memory = retained
+ state.memorySource = "previous-snapshot"
+ snapshotNotes = append(snapshotNotes, "preserved-original-guest-memory-during-deferral")
+ }
+ }
vm := models.VM{
ID: guestID,