Real-time monitoring dashboard for Proxmox VE, PBS, Docker, Kubernetes, TrueNAS and vSphere. Self-hosted, with alerts for failed backups, full disks, unhealthy containers and offline nodes.
Go aialertsdashboarddockerdocker-monitoringgohomelabinfrastructure-monitoringkubernetesmonitoringmspproxmoxproxmox-backup-serverproxmox-mail-gatewayproxmox-veself-hostedtruenastypescriptvspherewebhooks
Bound guest-memory fallbacks to their original observation age
pulse-triage[bot]2256ad7e8c78
diff --git a/docs/release-control/v6/internal/subsystems/monitoring.md b/docs/release-control/v6/internal/subsystems/monitoring.mdindex dc80ccbdd8..525d568088 100644--- a/docs/release-control/v6/internal/subsystems/monitoring.md+++ b/docs/release-control/v6/internal/subsystems/monitoring.md@@ -232,6 +232,23 @@ resumption through unified and served conversions; wire and source-selectioncontrols cover omission, cloning and independent live agent data. These aresynthetic source proofs, not native QGA/HAOS acceptance or delivered relief.+Low-trust memory stabilisation uses the original memory observation time for+its existing two-minute reconnect and ten-minute healthy-guest windows, not+the diagnostic poll's `RetrievedAt`. Repeated carried-forward polls cannot+extend either window. Missing/future original times and explicit unavailable+or incomplete observations cannot supply fallback usage. A legacy direct+reading can still use its own receipt time; a legacy `previous-snapshot`+without original provenance cannot acquire another lease from a new poll.+Expiry leaves the independently selected current source in place (including+unavailability when that source has no usage); it neither queues QGA commands+nor changes cache/admission, current PVE/agent selection, alert thresholds or+History's rejection of retained observations.+`TestGuestMemoryCarryForwardExpiryContract` covers the original-age boundaries,+repeated polling, real-client lock-unverified deferral, canonical served values,+in-memory History, independent PVE readings and ordinary QGA read recovery.+The existing cluster/node preservation tests use an original source timestamp;+these fixtures do not establish native thaw, HAOS/Windows readings or shipment.+### Unavailable backup-age evaluation is not silent recoveryPVE, PBS and mock backup checks share one failure-preserving evaluator. Thediff --git a/internal/monitoring/guest_memory_expiry_test.go b/internal/monitoring/guest_memory_expiry_test.gonew file mode 100644index 0000000000..e0115623f7--- /dev/null+++ b/internal/monitoring/guest_memory_expiry_test.go@@ -0,0 +1,221 @@+package monitoring++import (+ "context"+ "encoding/json"+ "fmt"+ "net/http"+ "net/http/httptest"+ "strings"+ "sync/atomic"+ "testing"+ "time"++ "github.com/rcourtman/pulse-go-rewrite/internal/config"+ "github.com/rcourtman/pulse-go-rewrite/internal/models"+ "github.com/rcourtman/pulse-go-rewrite/internal/unifiedresources"+ "github.com/rcourtman/pulse-go-rewrite/pkg/proxmox"+)++// A diagnostic refresh is not a new memory reading. These cases use the+// existing selectors, so the identical test also runs against the exact parent.+func testGuestMemoryCarryForwardOriginalAge(t *testing.T) {+ now := time.Date(2026, time.October, 9, 13, 0, 0, 0, time.UTC)+ for _, path := range []struct {+ name string+ window time.Duration+ read func(*GuestMemorySnapshot) bool+ }{+ {"trusted-reconnect", guestMemoryCarryForwardMaxAge, func(prev *GuestMemorySnapshot) bool {+ return shouldCarryForwardPreviousGuestMemory(prev, "running", "unavailable", 100, 0, now)+ }},+ {"healthy-guest", guestMemoryHealthyGuestMaxAge, func(prev *GuestMemorySnapshot) bool {+ return shouldCarryForwardHealthyGuestLowTrustMemory(prev, "running", "status-mem", 100, 100, now, true)+ }},+ } {+ t.Run(path.name, func(t *testing.T) {+ for _, tc := range []struct {+ name string+ observation models.MemoryObservation+ retrievedAt time.Time+ source string+ want bool+ }{+ {"recent-current", models.MemoryObservation{State: "current", Source: "guest-agent-meminfo", ObservedAt: now.Add(-path.window / 2)}, now, "guest-agent-meminfo", true},+ {"recent-last-known", models.MemoryObservation{State: "last-known", Source: "guest-agent-meminfo", ObservedAt: now.Add(-path.window / 2)}, now, "guest-agent-meminfo", true},+ {"inclusive-boundary", models.MemoryObservation{State: "last-known", Source: "guest-agent-meminfo", ObservedAt: now.Add(-path.window)}, now, "guest-agent-meminfo", true},+ {"expired-original-with-fresh-poll", models.MemoryObservation{State: "last-known", Source: "guest-agent-meminfo", ObservedAt: now.Add(-path.window - time.Nanosecond)}, now, "guest-agent-meminfo", false},+ {"future-original", models.MemoryObservation{State: "current", Source: "guest-agent-meminfo", ObservedAt: now.Add(time.Second)}, now, "guest-agent-meminfo", false},+ {"missing-original", models.MemoryObservation{State: "last-known", Source: "guest-agent-meminfo"}, now, "guest-agent-meminfo", false},+ {"explicit-unavailable", models.MemoryObservation{State: "unavailable", Source: "guest-agent-meminfo", ObservedAt: now}, now, "guest-agent-meminfo", false},+ {"unknown-state", models.MemoryObservation{State: "unknown", Source: "guest-agent-meminfo", ObservedAt: now}, now, "guest-agent-meminfo", false},+ {"partial-annotation", models.MemoryObservation{Source: "guest-agent-meminfo", ObservedAt: now}, now, "guest-agent-meminfo", false},+ {"legacy-direct-recent", models.MemoryObservation{}, now.Add(-path.window / 2), "guest-agent-meminfo", true},+ {"legacy-direct-expired", models.MemoryObservation{}, now.Add(-path.window - time.Second), "guest-agent-meminfo", false},+ {"legacy-direct-future", models.MemoryObservation{}, now.Add(time.Second), "guest-agent-meminfo", false},+ {"legacy-direct-missing", models.MemoryObservation{}, time.Time{}, "guest-agent-meminfo", false},+ {"legacy-retained-has-no-original-age", models.MemoryObservation{}, now, "previous-snapshot", false},+ } {+ t.Run(tc.name, func(t *testing.T) {+ prev := &GuestMemorySnapshot{Status: "running", RetrievedAt: tc.retrievedAt, MemorySource: tc.source,+ Memory: models.Memory{Total: 100, Used: 25, Free: 75, Usage: 25, Observation: tc.observation}}+ if got := path.read(prev); got != tc.want {+ t.Errorf("carry-forward = %t, want %t; original=%v poll=%v", got, tc.want, tc.observation, tc.retrievedAt)+ }+ })+ }+ })+ }+}++func testGuestMemoryCarryForwardRepeatedPollsDoNotExtendAge(t *testing.T) {+ original := time.Date(2026, time.October, 9, 13, 0, 0, 0, time.UTC)+ prev := &GuestMemorySnapshot{Status: "running", MemorySource: "previous-snapshot",+ Memory: models.Memory{Total: 100, Used: 25, Free: 75, Usage: 25,+ Observation: models.MemoryObservation{State: "last-known", Source: "available-field", ObservedAt: original}}}+ for minute := 1; minute <= 12; minute++ {+ now := original.Add(time.Duration(minute) * time.Minute)+ prev.RetrievedAt = now // Ordinary polling refreshes diagnostics every time.+ used, source, _ := stabilizeGuestLowTrustMemory(prev, "running", "status-mem", 100, 100, now, true)+ wantUsed, wantSource := uint64(25), "previous-snapshot"+ if minute > 10 {+ wantUsed, wantSource = 100, "status-mem"+ }+ if used != wantUsed || source != wantSource {+ t.Errorf("minute %d selected %d/%s, want %d/%s", minute, used, source, wantUsed, wantSource)+ }+ if prev.Memory.Observation.ObservedAt != original {+ t.Fatal("selector mutated the original observation")+ }+ }+}++// The real PVE client sees an unverified configuration and dispatches no guest+// commands. This exercises selection, source-owned cache age, served canonical+// projections, History and normal recovery without freezing/probing a real VM.+func testGuestMemoryCarryForwardOrdinaryDeferralAndRecovery(t *testing.T) {+ const gib = uint64(1024 * 1024 * 1024)+ for _, tc := range []struct {+ name string+ age time.Duration+ priorSource string+ missingOrigin bool+ wantUsed uint64+ wantSource string+ }{+ {"recent-retained", 3 * time.Minute, "previous-snapshot", false, 3 * gib, "previous-snapshot"},+ {"expired-retained", 11 * time.Minute, "previous-snapshot", false, 8 * gib, "status-mem"},+ {"expired-direct", 11 * time.Minute, "guest-agent-meminfo", false, 8 * gib, "status-mem"},+ {"retained-without-origin", 11 * time.Minute, "previous-snapshot", true, 8 * gib, "status-mem"},+ } {+ t.Run(tc.name, func(t *testing.T) {+ var phase, commands atomic.Int32+ server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {+ switch {+ case strings.HasSuffix(r.URL.Path, "/status/current"):+ memInfo := ""+ if phase.Load() == 2 {+ memInfo = fmt.Sprintf(`,"meminfo":{"total":%d,"available":%d}`, 8*gib, 6*gib)+ }+ fmt.Fprintf(w, `{"data":{"status":"running","agent":1,"maxmem":%d,"mem":%d%s}}`, 8*gib, 8*gib, memInfo)+ case strings.HasSuffix(r.URL.Path, "/config"):+ if phase.Load() == 1 {+ fmt.Fprint(w, `{"data":{}}`)+ } else {+ fmt.Fprint(w, `{"data":null}`)+ }+ case strings.Contains(r.URL.Path, "/agent/"):+ commands.Add(1)+ if strings.HasSuffix(r.URL.Path, "/file-read") {+ fmt.Fprint(w, `{"data":{"content":"MemTotal: 8388608 kB\nMemAvailable: 4194304 kB\n"}}`)+ } else {+ fmt.Fprint(w, `{"data":{"result":[]}}`)+ }+ default:+ http.NotFound(w, r)+ }+ }))+ defer server.Close()+ client, err := proxmox.NewClient(proxmox.ClientConfig{Host: server.URL, TokenName: "fixture@pve!pulse", TokenValue: "fixture", Timeout: time.Second})+ if err != nil {+ t.Fatal(err)+ }+ identity := makeGuestID("expiry", "node", 105)+ registry := unifiedresources.NewRegistry(nil)+ m := &Monitor{config: &config.Config{}, rateTracker: NewRateTracker(), metricsHistory: NewMetricsHistory(32, time.Hour),+ resourceStore: unifiedresources.NewMonitorAdapter(registry), guestMetadataLimiter: make(map[string]time.Time)}+ original := time.Now().Add(-tc.age)+ origin := original+ if tc.missingOrigin {+ origin = time.Time{}+ }+ m.recordGuestSnapshot("expiry", "qemu", "node", 105, GuestMemorySnapshot{Status: "running", RetrievedAt: time.Now(), MemorySource: tc.priorSource,+ Memory: models.Memory{Total: 8 * int64(gib), Used: 3 * int64(gib), Free: 5 * int64(gib), Usage: 37.5,+ Observation: models.MemoryObservation{State: "last-known", Source: "guest-agent-meminfo", ObservedAt: origin}}})+ key := guestMemoryCacheKey("expiry", "node", 105)+ m.vmAgentMemCache = map[string]agentMemCacheEntry{key: {fetchedAt: original,+ info: proxmox.LinuxMemoryAvailability{Source: "meminfo-available", EffectiveAvailable: 5 * gib}}}+ m.guestMetadataCache = map[string]guestMetadataCacheEntry{guestMetadataCacheKey("expiry", "node", 105): {fetchedAt: time.Now(), agentVersion: "fixture"}}+ res := proxmox.ClusterResource{Type: "qemu", Node: "node", Name: "guest", VMID: 105, Status: "running", MaxMem: 8 * gib, Mem: 8 * gib, MaxDisk: gib, CPU: 0.2}+ build := func() (models.VM, string) {+ vm, raw, source, notes, at, ok := m.buildVMFromClusterResource(context.Background(), "expiry", res, client, identity, nil, nil)+ if !ok {+ t.Fatal("guest disappeared")+ }+ m.recordGuestSnapshot("expiry", "qemu", "node", 105, GuestMemorySnapshot{Status: vm.Status, RetrievedAt: at, MemorySource: source, Memory: vm.Memory, Raw: raw, Notes: notes})+ m.recordGuestMetrics([]models.VM{vm}, nil, time.Now().Add(-time.Second))+ return vm, source+ }+ vm, source := build()+ if uint64(vm.Memory.Used) != tc.wantUsed || source != tc.wantSource || vm.GuestAgentStatus != "deferred" {+ t.Errorf("ordinary deferred poll selected %d/%s/%s, want %d/%s/deferred", vm.Memory.Used, source, vm.GuestAgentStatus, tc.wantUsed, tc.wantSource)+ }+ if commands.Load() != 0 || m.vmAgentMemCache[key].fetchedAt != original {+ t.Fatal("deferral sent a guest command or renewed the cache")+ }+ state, observedSource, at, history := "current", "status-mem", vm.Memory.Observation.ObservedAt, 1+ if tc.wantSource == "previous-snapshot" {+ state, observedSource, at, history = "last-known", "guest-agent-meminfo", original, 0+ }+ assertGuestMemoryObservation(t, vm.Memory, state, observedSource, at)+ front := m.buildBroadcastFrontendStateFromSnapshot(models.StateSnapshot{VMs: []models.VM{vm}}, m.mockModeFence.begin())+ wire, err := json.Marshal(front.Resources)+ if err != nil {+ t.Fatal(err)+ }+ var served []struct {+ Memory *models.ResourceMetricFrontend `json:"memory"`+ Proxmox struct {+ Memory models.Memory `json:"memory"`+ } `json:"proxmox"`+ }+ if err := json.Unmarshal(wire, &served); err != nil || len(served) != 1 || served[0].Memory == nil {+ t.Fatalf("served memory missing: %s / %v", wire, err)+ }+ assertGuestMemoryObservation(t, served[0].Proxmox.Memory, state, observedSource, at)+ assertGuestMemoryObservation(t, models.Memory{Observation: served[0].Memory.Observation}, state, observedSource, at)+ if served[0].Proxmox.Memory.Used != int64(tc.wantUsed) || served[0].Memory.Current != float64(tc.wantUsed)*100/float64(8*gib) {+ t.Error("served values do not match the selected source")+ }+ for metric, want := range map[string]int{"cpu": 1, "memory": history, "memoryused": history, "disk": 0} {+ if got := len(m.metricsHistory.GetGuestMetrics(identity, metric, time.Hour)); got != want {+ t.Errorf("%s History points = %d, want %d", metric, got, want)+ }+ }+ // Independent current PVE memory is not displaced by any retained+ // value or disk/QGA deferral, even before the fallback window expires.+ phase.Store(2)+ live, liveSource := build()+ if live.Memory.Used != int64(2*gib) || liveSource != "available-field" || live.Memory.Observation.State != "current" || commands.Load() != 0 {+ t.Error("independent PVE memory lost to guest deferral")+ }+ // An ordinary, newly admitted successful QGA read replaces the old+ // observation; no restart, manual probe or forced thaw is involved.+ phase.Store(1)+ resumed, resumedSource := build()+ if resumed.Memory.Used != int64(4*gib) || resumedSource != "guest-agent-meminfo" || resumed.Memory.Observation.State != "current" || !m.vmAgentMemCache[key].fetchedAt.After(original) || commands.Load() != 2 {+ t.Errorf("normal resumption failed: %d/%s/%s commands=%d", resumed.Memory.Used, resumedSource, resumed.Memory.Observation.State, commands.Load())+ }+ })+ }+}diff --git a/internal/monitoring/guest_memory_stability.go b/internal/monitoring/guest_memory_stability.goindex 7945594899..e20338986a 100644--- a/internal/monitoring/guest_memory_stability.go+++ b/internal/monitoring/guest_memory_stability.go@@ -92,7 +92,7 @@ func shouldCarryForwardPreviousGuestMemory(prev *GuestMemorySnapshot, currentStaif prev.Memory.Total <= 0 || prev.Memory.Used < 0 {return false}- if prev.RetrievedAt.IsZero() || now.Sub(prev.RetrievedAt) > guestMemoryCarryForwardMaxAge {+ if !guestMemorySnapshotWithinAge(prev, now, guestMemoryCarryForwardMaxAge) {return false}@@ -123,7 +123,7 @@ func shouldCarryForwardHealthyGuestLowTrustMemory(prev *GuestMemorySnapshot, curif prev.Memory.Total <= 0 || prev.Memory.Used < 0 {return false}- if prev.RetrievedAt.IsZero() || now.Sub(prev.RetrievedAt) > guestMemoryHealthyGuestMaxAge {+ if !guestMemorySnapshotWithinAge(prev, now, guestMemoryHealthyGuestMaxAge) {return false}if currentTotal == 0 || prev.Memory.Total != int64(currentTotal) {@@ -148,3 +148,23 @@ func shouldCarryForwardHealthyGuestLowTrustMemory(prev *GuestMemorySnapshot, curreturn true}++// The diagnostic snapshot is refreshed on every poll, including when it only+// carries an old reading forward. It cannot extend the lifetime of that reading.+// Legacy direct readings can use their receipt time once; a legacy retained+// value has lost its original age and cannot borrow the latest poll's time.+func guestMemorySnapshotWithinAge(prev *GuestMemorySnapshot, now time.Time, maxAge time.Duration) bool {+ observation := prev.Memory.Observation+ observedAt := observation.ObservedAt+ if observation.State != "" || observation.Source != "" || !observedAt.IsZero() {+ if observation.State != "current" && observation.State != "last-known" {+ return false+ }+ } else {+ if CanonicalMemorySource(prev.MemorySource) == "previous-snapshot" {+ return false+ }+ observedAt = prev.RetrievedAt+ }+ return !observedAt.IsZero() && !observedAt.After(now) && now.Sub(observedAt) <= maxAge+}diff --git a/internal/monitoring/guest_memory_stability_test.go b/internal/monitoring/guest_memory_stability_test.goindex f5aadddb7e..06ae9b8a51 100644--- a/internal/monitoring/guest_memory_stability_test.go+++ b/internal/monitoring/guest_memory_stability_test.go@@ -73,9 +73,10 @@ func TestStabilizeGuestLowTrustMemoryUsesHealthyGuestAgentEvidence(t *testing.T)RetrievedAt: now.Add(-2 * time.Minute),MemorySource: "previous-snapshot",Memory: models.Memory{- Total: 8 * int64(gib),- Used: 3 * int64(gib),- Usage: safePercentage(float64(3*gib), float64(8*gib)),+ Total: 8 * int64(gib),+ Used: 3 * int64(gib),+ Usage: safePercentage(float64(3*gib), float64(8*gib)),+ Observation: models.MemoryObservation{State: "last-known", Source: "guest-agent-meminfo", ObservedAt: now.Add(-2 * time.Minute)},},}@@ -107,10 +108,11 @@ func TestHandleClusterVMResourcePreservesHealthyGuestMemoryFromPreviousSnapshot(RetrievedAt: previousSample,MemorySource: "previous-snapshot",Memory: models.Memory{- Total: 8 * int64(gib),- Used: 3 * int64(gib),- Free: 5 * int64(gib),- Usage: safePercentage(float64(3*gib), float64(8*gib)),+ Total: 8 * int64(gib),+ Used: 3 * int64(gib),+ Free: 5 * int64(gib),+ Usage: safePercentage(float64(3*gib), float64(8*gib)),+ Observation: models.MemoryObservation{State: "last-known", Source: "guest-agent-meminfo", ObservedAt: previousSample},},})@@ -179,10 +181,11 @@ func TestPollVMsWithNodesPreservesHealthyGuestMemoryFromPreviousSnapshot(t *testRetrievedAt: previousSample,MemorySource: "previous-snapshot",Memory: models.Memory{- Total: 8 * int64(gib),- Used: 3 * int64(gib),- Free: 5 * int64(gib),- Usage: safePercentage(float64(3*gib), float64(8*gib)),+ Total: 8 * int64(gib),+ Used: 3 * int64(gib),+ Free: 5 * int64(gib),+ Usage: safePercentage(float64(3*gib), float64(8*gib)),+ Observation: models.MemoryObservation{State: "last-known", Source: "guest-agent-meminfo", ObservedAt: previousSample},},})diff --git a/internal/monitoring/monitor_pve_guest_lxc_test.go b/internal/monitoring/monitor_pve_guest_lxc_test.goindex 73a5ae371c..bdca67a7d6 100644--- a/internal/monitoring/monitor_pve_guest_lxc_test.go+++ b/internal/monitoring/monitor_pve_guest_lxc_test.go@@ -23,6 +23,12 @@ import ("github.com/rs/zerolog/log")+func TestGuestMemoryCarryForwardExpiryContract(t *testing.T) {+ t.Run("original-age", testGuestMemoryCarryForwardOriginalAge)+ t.Run("repeated-polls", testGuestMemoryCarryForwardRepeatedPollsDoNotExtendAge)+ t.Run("ordinary-deferral-recovery", testGuestMemoryCarryForwardOrdinaryDeferralAndRecovery)+}+type stubPVEClientLXCStatus struct {stubPVEClient