Real-time monitoring dashboard for Proxmox VE, PBS, Docker, Kubernetes, TrueNAS and vSphere. Self-hosted, with alerts for failed backups, full disks, unhealthy containers and offline nodes.
Go aialertsdashboarddockerdocker-monitoringgohomelabinfrastructure-monitoringkubernetesmonitoringmspproxmoxproxmox-backup-serverproxmox-mail-gatewayproxmox-veself-hostedtruenastypescriptvspherewebhooks
Reproduce lost guest-memory provenance through served projections
pulse-triage[bot]abee02363421
diff --git a/internal/monitoring/canonical_guardrails_test.go b/internal/monitoring/canonical_guardrails_test.goindex 30eeb870b5..60d78292b3 100644--- a/internal/monitoring/canonical_guardrails_test.go+++ b/internal/monitoring/canonical_guardrails_test.go@@ -2980,6 +2980,10 @@ func TestGuestAgentBackupMonitoringContract(t *testing.T) {testGuestAgentBackupMonitoringLifecycle(t)}+func TestGuestMemoryObservationContract(t *testing.T) {+ testGuestMemoryObservationLifecycle(t)+}+// Same-target, unambiguous lock evidence is required even with warm caches.func TestGuestAgentBackupLockEvidenceContract(t *testing.T) {for _, test := range []struct{ name, config string }{diff --git a/internal/monitoring/guest_memory_observation_test.go b/internal/monitoring/guest_memory_observation_test.gonew file mode 100644index 0000000000..8d58f06c66--- /dev/null+++ b/internal/monitoring/guest_memory_observation_test.go@@ -0,0 +1,155 @@+package monitoring++import (+ "context"+ "encoding/json"+ "testing"+ "time"++ "github.com/rcourtman/pulse-go-rewrite/internal/models"+ "github.com/rcourtman/pulse-go-rewrite/internal/unifiedresources"+ "github.com/rcourtman/pulse-go-rewrite/pkg/proxmox"+)++// Use the served JSON rather than the proposed Go type so this control also+// compiles against the parent that loses observation provenance entirely.+func assertGuestMemoryObservation(t *testing.T, memory models.Memory, state, source string, at time.Time) {+ t.Helper()+ data, err := json.Marshal(memory)+ if err != nil {+ t.Fatal(err)+ }+ var wire struct {+ Observation struct {+ State string `json:"state"`+ Source string `json:"source"`+ ObservedAt *time.Time `json:"observedAt"`+ } `json:"observation"`+ }+ if err := json.Unmarshal(data, &wire); err != nil {+ t.Fatal(err)+ }+ got := wire.Observation+ if got.State != state || got.Source != source {+ t.Errorf("memory observation = %s, want %s / %s", data, state, source)+ }+ if at.IsZero() {+ if got.ObservedAt != nil {+ t.Errorf("unobserved memory acquired a timestamp: %s", data)+ }+ } else if got.ObservedAt == nil || !got.ObservedAt.Equal(at) {+ t.Errorf("memory observation time = %v, want original %v", got.ObservedAt, at)+ }+}++func testGuestMemoryObservationLifecycle(t *testing.T) {+ const gib = uint64(1024 * 1024 * 1024)+ identity := makeGuestID("provenance", "node", 105)+ registry := unifiedresources.NewRegistry(nil)+ m := &Monitor{rateTracker: NewRateTracker(), metricsHistory: NewMetricsHistory(32, time.Hour), resourceStore: unifiedresources.NewMonitorAdapter(registry)}+ client := &vmMemoryTrustStubClient{stubPVEClient: &stubPVEClient{}, vmAgentMemAvailable: 5 * gib,+ vmStatus: &proxmox.VMStatus{Status: "running", MaxMem: 8 * gib, Mem: 8 * gib, Agent: proxmox.VMAgentField{Value: 1}}}+ res := proxmox.ClusterResource{Type: "qemu", Node: "node", Name: "guest", VMID: 105, Status: "running", MaxMem: 8 * gib, Mem: 8 * gib}+ var previous *models.VM+ build := func() models.VM {+ t.Helper()+ vm, raw, source, notes, at, ok := m.buildVMFromClusterResource(context.Background(), "provenance", res, client, identity, nil, previous)+ if !ok {+ t.Fatal("guest disappeared")+ }+ m.recordGuestSnapshot("provenance", "qemu", "node", 105, GuestMemorySnapshot{Status: vm.Status, RetrievedAt: at, MemorySource: source, Memory: vm.Memory, Raw: raw, Notes: notes})+ m.recordGuestMetrics([]models.VM{vm}, nil, time.Now().Add(-time.Second))+ registry.IngestSnapshot(models.StateSnapshot{VMs: []models.VM{vm}})+ views := registry.VMs()+ if len(views) != 1 {+ t.Fatal("guest did not reach unified view")+ }+ next := previousVMFromView(views[0])+ previous = &next+ return vm+ }+ assertProjection := func(vm models.VM, state, source string, at time.Time) {+ t.Helper()+ assertGuestMemoryObservation(t, vm.Memory, state, source, at)+ legacy := vm.ToFrontend()+ if legacy.Memory == nil {+ t.Fatal("legacy projection lost memory")+ }+ assertGuestMemoryObservation(t, *legacy.Memory, state, source, at)+ // REST and WebSocket state use this production conversion, including+ // registry cloning and the nested Proxmox payload the drawer consumes.+ front := m.buildBroadcastFrontendStateFromSnapshot(models.StateSnapshot{VMs: []models.VM{vm}})+ data, err := json.Marshal(front.Resources)+ if err != nil {+ t.Fatal(err)+ }+ var wire []struct {+ Proxmox struct {+ Memory models.Memory `json:"memory"`+ } `json:"proxmox"`+ }+ if err := json.Unmarshal(data, &wire); err != nil || len(wire) != 1 {+ t.Fatalf("broadcast guest memory unavailable: %s / %v", data, err)+ }+ assertGuestMemoryObservation(t, wire[0].Proxmox.Memory, state, source, at)+ if wire[0].Proxmox.Memory.Used != vm.Memory.Used || wire[0].Proxmox.Memory.UsageUnavailable != vm.Memory.UsageUnavailable {+ t.Fatal("provenance changed numeric/unknown memory projection")+ }+ }+ initial := build()+ key := guestMemoryCacheKey("provenance", "node", 105)+ original := m.vmAgentMemCache[key].fetchedAt+ if original.IsZero() || initial.Memory.Used != int64(3*gib) {+ t.Fatal("successful QGA observation missing")+ }+ assertProjection(initial, "current", "guest-agent-meminfo", original)+ cached := build()+ assertProjection(cached, "current", "guest-agent-meminfo", original)+ if client.vmAgentMemCalls != 1 {+ t.Fatal("normal cache caused a new guest read")+ }+ res.Lock, client.vmStatus.Lock = "backup", "backup"+ for poll := 0; poll < 4; poll++ {+ locked := build()+ assertProjection(locked, "last-known", "guest-agent-meminfo", original)+ if locked.Memory.Used != initial.Memory.Used || m.vmAgentMemCache[key].fetchedAt != original || client.vmAgentMemCalls != 1 {+ t.Fatal("deferral changed evidence, its age, or guest read admission")+ }+ }+ if got := len(m.metricsHistory.GetGuestMetrics(identity, "memory", time.Hour)); got != 2 {+ t.Errorf("deferred memory became fresh History points: %d, want two normal polls", got)+ }+ entry := m.vmAgentMemCache[key]+ entry.fetchedAt = time.Now().Add(-vmAgentMemCleanupMaxAge - time.Second)+ m.vmAgentMemCache[key] = entry+ expired := build()+ assertProjection(expired, "unavailable", "unavailable", time.Time{})+ if expired.Memory.HasKnownUsage() || client.vmAgentMemCalls != 1 {+ t.Fatal("expired evidence became usage or queued a guest read")+ }+ res.Lock, client.vmStatus.Lock = "", ""+ client.vmAgentMemAvailable = 4 * gib+ resumed := build()+ resumedAt := m.vmAgentMemCache[key].fetchedAt+ assertProjection(resumed, "current", "guest-agent-meminfo", resumedAt)+ if !resumedAt.After(original) || resumed.Memory.Used != int64(4*gib) || client.vmAgentMemCalls != 2 {+ t.Fatal("resumption did not replace original guest observation")+ }+ // Backup/disk deferral must not disqualify an independently current PVE+ // memory reading. Its timestamp belongs to that source, not this poll.+ statusAt := time.Now().Add(-time.Second)+ res.Lock, client.vmStatus.Lock = "backup", "backup"+ client.vmStatus.ObservedAt = statusAt+ client.vmStatus.MemInfo = &proxmox.VMMemInfo{Total: 8 * gib, Available: 6 * gib}+ live := build()+ assertProjection(live, "current", "available-field", statusAt)+ if live.Memory.Used != int64(2*gib) || live.GuestAgentStatus != "deferred" || client.vmAgentMemCalls != 2 {+ t.Fatal("independent PVE memory was replaced by deferred QGA evidence")+ }+ res.Status = "stopped"+ stopped := build()+ assertProjection(stopped, "unavailable", "powered-off", time.Time{})+ if stopped.Memory.Used != 0 {+ t.Fatal("powered-off numeric compatibility changed")+ }+}