Real-time monitoring dashboard for Proxmox VE, PBS, Docker, Kubernetes, TrueNAS and vSphere. Self-hosted, with alerts for failed backups, full disks, unhealthy containers and offline nodes.
Go aialertsdashboarddockerdocker-monitoringgohomelabinfrastructure-monitoringkubernetesmonitoringmspproxmoxproxmox-backup-serverproxmox-mail-gatewayproxmox-veself-hostedtruenastypescriptvspherewebhooks
Bind guest memory usage to its sampled capacity
pulse-triage[bot]ec978c647f41
diff --git a/docs/release-control/v6/internal/subsystems/monitoring.md b/docs/release-control/v6/internal/subsystems/monitoring.mdindex 5f26dd6952..286ce796b1 100644--- a/docs/release-control/v6/internal/subsystems/monitoring.md+++ b/docs/release-control/v6/internal/subsystems/monitoring.md@@ -17,6 +17,33 @@## Purpose+### Guest memory readings keep their own capacity++The QEMU memory selector pairs guest availability with the total from the+same status meminfo or QGA `/proc/meminfo` sample, and linked in-guest agent+usage with that agent's total. A configured maximum is not guest usage:+ballooning and kernel reservations can make those capacities differ.+Existing availability-only payloads retain their configured-capacity fallback.+Reported totals larger than a known VM maximum are not accepted as guest+evidence, including a hotplug transition before configured status catches up.+The existing balloon/free-memory fallback also keeps the denominator it used+to derive usage, without being upgraded to cache-aware pressure evidence.+The diagnostic snapshot records `guestAgentMemTotal` alongside the already+retained configured maximum and memory fields; it adds no read or public log.++Retention keeps the selected total, used, free/cache and original observation+as one tuple. Configured-capacity changes, including a resize to the old guest+total, cannot borrow that tuple. The existing source age bounds, backup lock,+filesystem-first ordering and shared command fences are unchanged; a held or+expired sample cannot produce new History, breach or recovery evidence.+`TestGuestMemorySampleCapacityContract` and+`TestGuestMemorySampleCapacityCanonicalProjection` check selectors, coherent+cache binding and both ordinary collectors with direct/cluster HTTP clients,+canonical views, History and actual threshold decisions. Healthy ballooned,+full-pressure/zero-available, legacy, resize and age controls remain distinct.+These are secret-free source fixtures, not native balloon or freeze recovery,+reporter #2762 cause/resolution, or release acceptance.+### Existing memory origins cannot be renewed by AI consumersAI query/context/fact consumers now retain the selected guest observation anddiff --git a/internal/monitoring/canonical_guardrails_test.go b/internal/monitoring/canonical_guardrails_test.goindex 8142dbe0f8..4472c7f966 100644--- a/internal/monitoring/canonical_guardrails_test.go+++ b/internal/monitoring/canonical_guardrails_test.go@@ -3297,3 +3297,7 @@ func TestBackupAlertEvaluationCallersShareFailureVisibility(t *testing.T) {}}}++func TestGuestMemorySampleCapacityCanonicalProjection(t *testing.T) {+ testGuestMemorySampleCapacityPolling(t)+}diff --git a/internal/monitoring/diagnostic_snapshots.go b/internal/monitoring/diagnostic_snapshots.goindex ee9ba5b520..1bb10b7ccc 100644--- a/internal/monitoring/diagnostic_snapshots.go+++ b/internal/monitoring/diagnostic_snapshots.go@@ -63,6 +63,7 @@ type VMMemoryRaw struct {MemInfoCached uint64 `json:"meminfoCached,omitempty"`MemInfoShared uint64 `json:"meminfoShared,omitempty"`MemInfoTotalMinusUsed uint64 `json:"meminfoTotalMinusUsed,omitempty"`+ GuestAgentMemTotal uint64 `json:"guestAgentMemTotal,omitempty"`GuestAgentMemAvailable uint64 `json:"guestAgentMemAvailable,omitempty"`GuestAgentMemFree uint64 `json:"guestAgentMemFree,omitempty"`GuestAgentMemBuffers uint64 `json:"guestAgentMemBuffers,omitempty"`diff --git a/internal/monitoring/diagnostic_snapshots_test.go b/internal/monitoring/diagnostic_snapshots_test.goindex e7ab36a19b..d03f92e431 100644--- a/internal/monitoring/diagnostic_snapshots_test.go+++ b/internal/monitoring/diagnostic_snapshots_test.go@@ -1390,3 +1390,8 @@ func TestGetDiagnosticSnapshots(t *testing.T) {}})}++func TestGuestMemorySampleCapacityContract(t *testing.T) {+ t.Run("selection", testGuestMemorySampleCapacitySelection)+ t.Run("cache-binding", testGuestMemorySampleCapacityCacheBinding)+}diff --git a/internal/monitoring/guest_memory_agent.go b/internal/monitoring/guest_memory_agent.goindex e3f75f8a0d..4196aabb7a 100644--- a/internal/monitoring/guest_memory_agent.go+++ b/internal/monitoring/guest_memory_agent.go@@ -43,7 +43,12 @@ func guestMemoryCacheKey(instanceName, node string, vmid int) string {// refresh; the cleanup age bounds last-known evidence during coordination.// Neither a poll timestamp nor a previous-snapshot trust label renews that age.func (m *Monitor) deferredVMGuestMemory(instanceName, node string, vmid int, total uint64, previous *GuestMemorySnapshot, now time.Time) (models.Memory, bool) {- if previous == nil || previous.GuestType != "qemu" || previous.Status != "running" || previous.Instance != instanceName || previous.Node != node || previous.VMID != vmid || !previous.Memory.HasKnownUsage() || uint64(previous.Memory.Total) != total {+ if previous == nil || previous.GuestType != "qemu" || previous.Status != "running" || previous.Instance != instanceName || previous.Node != node || previous.VMID != vmid || !previous.Memory.HasKnownUsage() || previous.Memory.Total <= 0 {+ return models.Memory{}, false+ }+ // The last guest sample can be smaller than its configured maximum. Keep+ // the tuple intact, but still reject retention across a capacity change.+ if guestMemoryConfiguredCapacity(previous) != total {return models.Memory{}, false}switch CanonicalMemorySource(previous.MemorySource) {@@ -60,7 +65,8 @@ func (m *Monitor) deferredVMGuestMemory(instanceName, node string, vmid int, totif entry.info.Source != "meminfo-available" && entry.info.Source != "meminfo-derived" {return models.Memory{}, false}- if entry.info.EffectiveAvailable > total || previous.Memory.Used != int64(total-entry.info.EffectiveAvailable) {+ sampleTotal := uint64(previous.Memory.Total)+ if (entry.info.Total > 0 && entry.info.Total != sampleTotal) || entry.info.EffectiveAvailable > sampleTotal || previous.Memory.Used != int64(sampleTotal-entry.info.EffectiveAvailable) {return models.Memory{}, false}expected := models.Memory{Free: int64(entry.info.EffectiveAvailable)}diff --git a/internal/monitoring/guest_memory_capacity_test.go b/internal/monitoring/guest_memory_capacity_test.gonew file mode 100644index 0000000000..11b1b57888--- /dev/null+++ b/internal/monitoring/guest_memory_capacity_test.go@@ -0,0 +1,316 @@+package monitoring++import (+ "context"+ "encoding/json"+ "fmt"+ "math"+ "net/http"+ "net/http/httptest"+ "reflect"+ "strings"+ "sync/atomic"+ "testing"+ "time"++ "github.com/rcourtman/pulse-go-rewrite/internal/alerts"+ "github.com/rcourtman/pulse-go-rewrite/internal/config"+ "github.com/rcourtman/pulse-go-rewrite/internal/models"+ "github.com/rcourtman/pulse-go-rewrite/internal/unifiedresources"+ "github.com/rcourtman/pulse-go-rewrite/pkg/proxmox"+)++// These controls run unchanged against the parent. The configured ceiling is+// deliberately different from the guest sample; this is not a native VM probe.+func testGuestMemorySampleCapacitySelection(t *testing.T) {+ const gib = uint64(1024 * 1024 * 1024)+ id := makeGuestID("capacity", "node", 105)+ for _, tc := range []struct {+ name string+ status proxmox.VMStatus+ info *proxmox.LinuxMemoryAvailability+ agent *models.Host+ wantTotal, wantUsed uint64+ wantSource string+ calls int+ }{+ {name: "status-available", status: proxmox.VMStatus{MemInfo: &proxmox.VMMemInfo{Total: 4 * gib, Available: 3 * gib}}, wantTotal: 4 * gib, wantUsed: gib, wantSource: "available-field"},+ {name: "status-free-cache", status: proxmox.VMStatus{MemInfo: &proxmox.VMMemInfo{Total: 4 * gib, Free: gib / 2, Buffers: gib / 2, Cached: 2 * gib}}, wantTotal: 4 * gib, wantUsed: gib, wantSource: "derived-free-buffers-cached"},+ {name: "status-total-used-gap", status: proxmox.VMStatus{MemInfo: &proxmox.VMMemInfo{Total: 4 * gib, Used: gib, Free: gib / 2}}, wantTotal: 4 * gib, wantUsed: gib, wantSource: "derived-total-minus-used"},+ {name: "qga-available", status: proxmox.VMStatus{Agent: proxmox.VMAgentField{Value: 1}}, info: &proxmox.LinuxMemoryAvailability{Total: 4 * gib, EffectiveAvailable: 3 * gib, Source: "meminfo-available"}, wantTotal: 4 * gib, wantUsed: gib, wantSource: "guest-agent-meminfo", calls: 1},+ {name: "qga-derived", status: proxmox.VMStatus{Agent: proxmox.VMAgentField{Value: 1}}, info: &proxmox.LinuxMemoryAvailability{Total: 4 * gib, EffectiveAvailable: 3 * gib, Source: "meminfo-derived"}, wantTotal: 4 * gib, wantUsed: gib, wantSource: "guest-agent-meminfo-derived", calls: 1},+ {name: "qga-explicit-zero-available", status: proxmox.VMStatus{Agent: proxmox.VMAgentField{Value: 1}}, info: &proxmox.LinuxMemoryAvailability{Total: 4 * gib, Source: "meminfo-available"}, wantTotal: 4 * gib, wantUsed: 4 * gib, wantSource: "guest-agent-meminfo", calls: 1},+ {name: "qga-invalid-available-exceeds-sample-total", status: proxmox.VMStatus{Agent: proxmox.VMAgentField{Value: 1}}, info: &proxmox.LinuxMemoryAvailability{Total: 4 * gib, EffectiveAvailable: 5 * gib, Source: "meminfo-available"}, wantTotal: 20 * gib, wantUsed: 20 * gib, wantSource: "status-mem", calls: 1},+ {name: "qga-unrepresentable-sample-total", status: proxmox.VMStatus{Agent: proxmox.VMAgentField{Value: 1}}, info: &proxmox.LinuxMemoryAvailability{Total: uint64(math.MaxInt64) + 1, Source: "meminfo-available"}, wantTotal: 20 * gib, wantUsed: 20 * gib, wantSource: "status-mem", calls: 1},+ {name: "linked-agent", agent: &models.Host{Memory: models.Memory{Total: 4 * int64(gib), Used: int64(gib), Free: 3 * int64(gib), Usage: 25}}, wantTotal: 4 * gib, wantUsed: gib, wantSource: "agent"},+ {name: "linked-agent-host-sized-rejected", agent: &models.Host{Memory: models.Memory{Total: 64 * int64(gib), Used: int64(gib), Free: 63 * int64(gib), Usage: 100.0 / 64}}, wantTotal: 20 * gib, wantUsed: 20 * gib, wantSource: "status-mem"},+ {name: "balloon-freemem-keeps-low-trust", status: proxmox.VMStatus{Balloon: 4 * gib, FreeMem: 3 * gib}, wantTotal: 4 * gib, wantUsed: gib, wantSource: "status-freemem"},+ {name: "ballooninfo-freemem-keeps-low-trust", status: proxmox.VMStatus{BalloonInfo: &proxmox.VMBalloonInfo{TotalMem: 4 * gib, FreeMem: 3 * gib}}, wantTotal: 4 * gib, wantUsed: gib, wantSource: "status-freemem"},+ {name: "missing-status-total-compatibility", status: proxmox.VMStatus{MemInfo: &proxmox.VMMemInfo{Available: 3 * gib}}, wantTotal: 20 * gib, wantUsed: 17 * gib, wantSource: "available-field"},+ {name: "availability-only-compatibility", status: proxmox.VMStatus{Agent: proxmox.VMAgentField{Value: 1}}, info: &proxmox.LinuxMemoryAvailability{EffectiveAvailable: 3 * gib, Source: "meminfo-available"}, wantTotal: 20 * gib, wantUsed: 17 * gib, wantSource: "guest-agent-meminfo", calls: 1},+ {name: "matching-capacity-unchanged", status: proxmox.VMStatus{MemInfo: &proxmox.VMMemInfo{Total: 20 * gib, Available: 15 * gib}}, wantTotal: 20 * gib, wantUsed: 5 * gib, wantSource: "available-field"},+ {name: "status-total-above-configured-ceiling-rejected", status: proxmox.VMStatus{MemInfo: &proxmox.VMMemInfo{Total: 24 * gib, Available: 18 * gib}}, wantTotal: 20 * gib, wantSource: "unavailable"},+ {name: "qga-total-above-configured-ceiling-rejected", status: proxmox.VMStatus{Agent: proxmox.VMAgentField{Value: 1}}, info: &proxmox.LinuxMemoryAvailability{Total: 24 * gib, EffectiveAvailable: 18 * gib, Source: "meminfo-available"}, wantTotal: 20 * gib, wantUsed: 20 * gib, wantSource: "status-mem", calls: 1},+ } {+ t.Run(tc.name, func(t *testing.T) {+ m := &Monitor{}+ client := &guestMemoryAgentTestClient{stubPVEClient: &stubPVEClient{}, memInfo: tc.info}+ status := tc.status+ status.MaxMem, status.Mem = 20*gib, 20*gib+ linked := map[string]models.Host{}+ if tc.agent != nil {+ linked[id] = *tc.agent+ }+ total, used, source, deferred := m.resolveGuestStatusMemory(context.Background(), client, "capacity", "guest", "node", 105, id, &status, linked, 20*gib, "cluster-resources", &VMMemoryRaw{})+ if total != tc.wantTotal || used != tc.wantUsed || source != tc.wantSource || deferred || client.memCalls != tc.calls || client.rrdCalls != 0 {+ t.Errorf("selected %d/%d (%s, deferred=%t, calls=%d), want %d/%d (%s, calls=%d)", used, total, source, deferred, client.memCalls, tc.wantUsed, tc.wantTotal, tc.wantSource, tc.calls)+ }+ e := unifiedresources.QualifyGuestMemory(safePercentage(float64(used), float64(total)), models.MemoryObservation{State: "current", Source: source, ObservedAt: time.Now()}, true, true, time.Now())+ wantPressure := source == "available-field" || source == "derived-free-buffers-cached" || source == "guest-agent-meminfo" || source == "guest-agent-meminfo-derived" || source == "agent"+ if e.PressureKnown != wantPressure {+ t.Error("capacity selection changed source qualification")+ }+ })+ }+}++func testGuestMemorySampleCapacityCacheBinding(t *testing.T) {+ now := time.Now()+ memory := models.Memory{Total: 4000, Used: 1000, Free: 500, Cache: 2500, Usage: 25}+ key := guestMemoryCacheKey("capacity", "node", 105)+ for _, tc := range []struct {+ name string+ cachedTotal uint64+ want bool+ }{+ {"same-tuple", 4000, true}, {"different-denominator-same-available", 5000, false}, {"legacy-availability-only", 0, true},+ } {+ t.Run(tc.name, func(t *testing.T) {+ m := &Monitor{vmAgentMemCache: map[string]agentMemCacheEntry{key: {fetchedAt: now, info: proxmox.LinuxMemoryAvailability{Total: tc.cachedTotal, Free: 500, EffectiveAvailable: 3000, Source: "meminfo-available"}}}}+ observation, ok := m.cachedGuestMemoryObservation("capacity", "node", 105, memory, now)+ if ok != tc.want || ok && !observation.ObservedAt.Equal(now) {+ t.Errorf("origin bound to wrong tuple: %+v, %t", observation, ok)+ }+ prev := &GuestMemorySnapshot{GuestType: "qemu", Status: "running", Instance: "capacity", Node: "node", VMID: 105, MemorySource: "guest-agent-meminfo", Memory: memory, Raw: VMMemoryRaw{StatusMaxMem: 20000}}+ retained, ok := m.deferredVMGuestMemory("capacity", "node", 105, 20000, prev, now)+ if ok != tc.want || ok && !reflect.DeepEqual(retained, memory) {+ t.Errorf("deferral lost tuple: %+v, %t", retained, ok)+ }+ if _, ok := m.deferredVMGuestMemory("capacity", "node", 105, 4000, prev, now); ok {+ t.Error("resize to old guest total retained earlier tuple")+ }+ if _, ok := m.deferredVMGuestMemory("other", "node", 105, 20000, prev, now); ok {+ t.Error("retention crossed instance identity")+ }+ })+ }+}++// Both ordinary collectors and concrete HTTP clients feed canonical views,+// History and the production alert call. Lock deferral, age and resize controls+// keep a held value from becoming a new breach or recovery. No guest is frozen.+func testGuestMemorySampleCapacityPolling(t *testing.T) {+ const gib = uint64(1024 * 1024 * 1024)+ for _, clientKind := range []string{"direct", "cluster"} {+ for _, collector := range []string{"cluster-resources", "node-list"} {+ t.Run(clientKind+"/"+collector, func(t *testing.T) {+ var phase, commands, memoryReads atomic.Int32+ var configured atomic.Uint64+ configured.Store(20 * gib)+ server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {+ w.Header().Set("Content-Type", "application/json")+ switch {+ case strings.HasSuffix(r.URL.Path, "/status/current"):+ lock := ""+ if phase.Load() == 1 {+ lock = `,"lock":"backup"`+ }+ fmt.Fprintf(w, `{"data":{"status":"running","agent":1,"maxmem":%d,"mem":%d%s}}`, configured.Load(), configured.Load(), lock)+ case strings.HasSuffix(r.URL.Path, "/config"):+ if phase.Load() == 1 {+ fmt.Fprint(w, `{"data":{"lock":"backup"}}`)+ } else {+ fmt.Fprint(w, `{"data":{}}`)+ }+ case strings.Contains(r.URL.Path, "/agent/"):+ commands.Add(1)+ switch {+ case strings.HasSuffix(r.URL.Path, "/file-read"):+ memoryReads.Add(1)+ total, available, free := 4*gib, 3*gib, gib/2+ if phase.Load() == 2 {+ available, free = 0, 0+ }+ if phase.Load() == 4 {+ total, available, free = 8*gib, 6*gib, gib+ }+ json.NewEncoder(w).Encode(map[string]any{"data": map[string]any{"content": fmt.Sprintf("MemAvailable: %d kB\nMemFree: %d kB\nMemTotal: %d kB\n", available/1024, free/1024, total/1024)}})+ case strings.HasSuffix(r.URL.Path, "/get-fsinfo"):+ fmt.Fprintf(w, `{"data":{"result":[{"mountpoint":"/","type":"ext4","total-bytes":%d,"used-bytes":%d}]}}`, gib, gib/4)+ case strings.HasSuffix(r.URL.Path, "/get-osinfo"):+ fmt.Fprint(w, `{"data":{"name":"Linux","version":"fixture"}}`)+ case strings.HasSuffix(r.URL.Path, "/version"):+ fmt.Fprint(w, `{"data":{"result":{"version":"fixture"}}}`)+ default:+ fmt.Fprint(w, `{"data":{"result":[]}}`)+ }+ case strings.HasSuffix(r.URL.Path, "/version"):+ fmt.Fprint(w, `{"data":{"version":"9.2-fixture"}}`)+ case strings.HasSuffix(r.URL.Path, "/nodes"):+ fmt.Fprint(w, `{"data":[{"node":"node","status":"online"}]}`)+ default:+ http.NotFound(w, r)+ }+ }))+ defer server.Close()+ pcfg := proxmox.ClientConfig{Host: server.URL, TokenName: "fixture@pve!pulse", TokenValue: "fixture", Timeout: time.Second}+ var client PVEClientInterface+ if clientKind == "direct" {+ var err error+ client, err = proxmox.NewClient(pcfg)+ if err != nil {+ t.Fatal(err)+ }+ } else {+ client = proxmox.NewClusterClient("capacity", pcfg, []string{server.URL}, nil)+ }+ am := alerts.NewManagerWithDataDir(t.TempDir(), alerts.WithoutPersistedAlertRestore())+ defer am.Stop()+ acfg := am.GetConfig()+ acfg.Enabled = true+ acfg.GuestDefaults.Memory = &alerts.HysteresisThreshold{Trigger: 80, Clear: 75}+ acfg.GuestDefaults.CPU = &alerts.HysteresisThreshold{Trigger: 80, Clear: 75}+ acfg.GuestDefaults.Disk = &alerts.HysteresisThreshold{Trigger: 90, Clear: 85}+ acfg.MetricTimeThresholds = map[string]map[string]int{"guest": {"memory": 0, "cpu": 0, "disk": 0}}+ am.UpdateConfig(acfg)+ m := &Monitor{config: &config.Config{}, alertManager: am, rateTracker: NewRateTracker(), metricsHistory: NewMetricsHistory(32, time.Hour), guestMetadataLimiter: make(map[string]time.Time), vmAgentMemCache: make(map[string]agentMemCacheEntry)}+ registry := unifiedresources.NewRegistry(nil)+ id := makeGuestID("capacity", "node", 105)+ previous := map[string]models.VM{}+ build := func() models.VM {+ t.Helper()+ res := proxmox.ClusterResource{Type: "qemu", Node: "node", Name: "guest", VMID: 105, Status: "running", MaxMem: configured.Load(), Mem: configured.Load(), MaxDisk: gib, CPU: .1}+ var vms []models.VM+ if collector == "cluster-resources" {+ vms = m.collectClusterVMResources(context.Background(), "capacity", []indexedClusterResource{{resource: res, guestID: id}}, client, previous, nil)+ } else {+ vms, _ = m.pollNodeVMsWithClusterResourceBuilder(context.Background(), "capacity", "node", []proxmox.VM{{VMID: 105, Name: "guest", Status: "running", MaxMem: res.MaxMem, Mem: res.Mem, MaxDisk: gib, CPU: .1}}, client, previous, nil)+ }+ if len(vms) != 1 {+ t.Fatalf("collector lost VM: %v", vms)+ }+ vm := vms[0]+ m.recordGuestMetrics(vms, nil, time.Now().Add(-time.Second))+ registry.IngestSnapshot(models.StateSnapshot{VMs: vms})+ view := registry.VMs()[0]+ if vm.Memory.HasKnownUsage() && (view.MemoryTotal() != vm.Memory.Total || view.MemoryUsed() != vm.Memory.Used || view.MemoryPercent() != vm.Memory.Usage) {+ t.Error("canonical view changed selected tuple")+ }+ if !vm.Memory.HasKnownUsage() && view.MemoryEvidence(time.Now()).Available {+ t.Error("unavailable memory became a canonical measurement")+ }+ previous[id] = previousVMFromView(view)+ return vm+ }+ assertHealthy := func(vm models.VM, total uint64) {+ t.Helper()+ if vm.Memory.Total != int64(total) || vm.Memory.Used != int64(total/4) || vm.Memory.Usage != 25 || vm.Memory.Free != int64(total/8) || vm.Memory.Cache != int64(total*5/8) {+ t.Errorf("healthy sample mixed capacities: %+v", vm.Memory)+ }+ if !vm.Memory.HasKnownUsage() {+ t.Error("healthy guest became unavailable")+ }+ }+ initial := build()+ assertHealthy(initial, 4*gib)+ if len(am.GetActiveAlerts()) != 0 {+ t.Error("healthy ballooned guest created false memory alert")+ }+ snapshot := m.previousGuestSnapshot("capacity", "qemu", "node", 105)+ rawBytes, _ := json.Marshal(snapshot.Raw)+ var raw map[string]uint64+ json.Unmarshal(rawBytes, &raw)+ if raw["guestAgentMemTotal"] != 4*gib || raw["statusMaxmem"] != 20*gib {+ t.Errorf("raw evidence lost guest/configured capacity distinction: %s", rawBytes)+ }+ if memoryReads.Load() != 1 {+ t.Error("initial sample repeated optional memory read")+ }+ origin := initial.Memory.Observation.ObservedAt+ before := commands.Load()+ phase.Store(1)+ for i := 0; i < 3; i++ {+ held := build()+ assertHealthy(held, 4*gib)+ if held.Memory.Observation.State != "last-known" || !held.Memory.Observation.ObservedAt.Equal(origin) || held.GuestAgentStatus != "deferred" {+ t.Error("lock renewed memory origin or lost shared deferral")+ }+ }+ if commands.Load() != before || len(m.metricsHistory.GetGuestMetrics(id, "memory", time.Hour)) != 1 {+ t.Error("lock queued QGA or repeated memory History")+ }+ // A resize cannot retain the old tuple even when its new maximum+ // happens to equal the earlier guest sample's total.+ configured.Store(4 * gib)+ resized := build()+ if resized.Memory.HasKnownUsage() || resized.Memory.Observation.State != "unavailable" {+ t.Errorf("resize retained earlier tuple: %+v", resized.Memory)+ }+ configured.Store(20 * gib)+ m.recordGuestSnapshot("capacity", "qemu", "node", 105, *snapshot)+ aged := *snapshot+ aged.Memory.Observation.ObservedAt = time.Now().Add(-11 * time.Minute)+ m.recordGuestSnapshot("capacity", "qemu", "node", 105, aged)+ key := guestMemoryCacheKey("capacity", "node", 105)+ entry := m.vmAgentMemCache[key]+ entry.fetchedAt = aged.Memory.Observation.ObservedAt+ m.vmAgentMemCache[key] = entry+ expired := build()+ if expired.Memory.HasKnownUsage() || expired.Memory.Observation.State != "unavailable" || commands.Load() != before {+ t.Errorf("expiry fabricated observation or command: %+v", expired.Memory)+ }+ phase.Store(2)+ pressure := build()+ if pressure.Memory.Total != 4*int64(gib) || pressure.Memory.Used != 4*int64(gib) || pressure.Memory.Usage != 100 || pressure.Memory.Observation.State != "current" || len(am.GetActiveAlerts()) != 1 {+ t.Errorf("real full-pressure sample suppressed: %+v; alerts=%d", pressure.Memory, len(am.GetActiveAlerts()))+ }+ for _, a := range am.GetActiveAlerts() {+ if a.Type != "memory" || a.Value != 100 {+ t.Error("wrong pressure alert")+ }+ }+ phase.Store(3)+ entry = m.vmAgentMemCache[key]+ entry.fetchedAt = time.Now().Add(-2 * vmAgentMemCacheTTL)+ m.vmAgentMemCache[key] = entry+ recovered := build()+ assertHealthy(recovered, 4*gib)+ if len(am.GetActiveAlerts()) != 0 || !recovered.Memory.Observation.ObservedAt.After(pressure.Memory.Observation.ObservedAt) {+ t.Error("current healthy sample did not resolve pressure with a new origin")+ }+ phase.Store(4)+ configured.Store(24 * gib)+ entry = m.vmAgentMemCache[key]+ entry.fetchedAt = time.Now().Add(-2 * vmAgentMemCacheTTL)+ m.vmAgentMemCache[key] = entry+ assertHealthy(build(), 8*gib)+ if memoryReads.Load() != 4 {+ t.Errorf("normal refresh read count = %d, want 4", memoryReads.Load())+ }+ points := m.metricsHistory.GetGuestMetrics(id, "memory", time.Hour)+ if len(points) != 4 {+ t.Errorf("current History count = %d, want 4", len(points))+ }+ for i, p := range points {+ want := 25.0+ if i == 1 {+ want = 100+ }+ if p.Value != want {+ t.Errorf("History[%d]=%v, want %v", i, p.Value, want)+ }+ }+ })+ }+ }+}diff --git a/internal/monitoring/guest_memory_observation.go b/internal/monitoring/guest_memory_observation.goindex aa29e34b78..4dd1d0da7d 100644--- a/internal/monitoring/guest_memory_observation.go+++ b/internal/monitoring/guest_memory_observation.go@@ -77,7 +77,7 @@ func (m *Monitor) cachedGuestMemoryObservation(instance, node string, vmid int,m.rrdCacheMu.RLock()entry, ok := m.vmAgentMemCache[guestMemoryCacheKey(instance, node, vmid)]m.rrdCacheMu.RUnlock()- if !ok || entry.negative || entry.fetchedAt.IsZero() || entry.fetchedAt.After(now) || now.Sub(entry.fetchedAt) > vmAgentMemCleanupMaxAge || memory.Total <= 0 || entry.info.EffectiveAvailable > uint64(memory.Total) || memory.Used != memory.Total-int64(entry.info.EffectiveAvailable) {+ if !ok || entry.negative || entry.fetchedAt.IsZero() || entry.fetchedAt.After(now) || now.Sub(entry.fetchedAt) > vmAgentMemCleanupMaxAge || memory.Total <= 0 || (entry.info.Total > 0 && entry.info.Total != uint64(memory.Total)) || entry.info.EffectiveAvailable > uint64(memory.Total) || memory.Used != memory.Total-int64(entry.info.EffectiveAvailable) {return models.MemoryObservation{}, false}expected := models.Memory{Free: int64(entry.info.EffectiveAvailable)}diff --git a/internal/monitoring/guest_memory_sources.go b/internal/monitoring/guest_memory_sources.goindex 2f113eb4f7..93dc7d4080 100644--- a/internal/monitoring/guest_memory_sources.go+++ b/internal/monitoring/guest_memory_sources.go@@ -209,13 +209,21 @@ func (m *Monitor) tryGuestAgentMemAvailable(vmid int,memTotal uint64,guestRaw *VMMemoryRaw,-) (uint64, string, bool, bool) {+) (uint64, uint64, string, bool, bool) {availability, agentErr := m.getVMAgentMemoryAvailability(ctx, client, instanceName, node, vmid)if agentErr != nil || availability.Source == "" {- return 0, "", false, errors.Is(agentErr, proxmox.ErrGuestAgentDeferred)+ return 0, 0, "", false, errors.Is(agentErr, proxmox.ErrGuestAgentDeferred)+ }+ // MemAvailable belongs to this guest's MemTotal, not the configured+ // maximum. Ballooning and kernel-reserved pages can make them different.+ // Older availability-only clients retain their supplied capacity fallback.+ configuredTotal := memTotal+ if availability.Total > 0 {+ memTotal = availability.Total}agentAvailable := availability.EffectiveAvailableif guestRaw != nil {+ guestRaw.GuestAgentMemTotal = availability.TotalguestRaw.GuestAgentMemAvailable = agentAvailableguestRaw.GuestAgentMemFree = availability.FreeguestRaw.GuestAgentMemBuffers = availability.Buffers@@ -224,8 +232,9 @@ func (m *Monitor) tryGuestAgentMemAvailable(guestRaw.GuestAgentShmem = availability.ShmemguestRaw.GuestAgentDerived = availability.Source == "meminfo-derived"}- if memTotal == 0 || agentAvailable > memTotal {- return 0, "", false, false+ if memTotal == 0 || memTotal > math.MaxInt64 || agentAvailable > memTotal ||+ (configuredTotal > 0 && memTotal > configuredTotal) {+ return 0, 0, "", false, false}source := "guest-agent-meminfo"if availability.Source == "meminfo-derived" {@@ -239,7 +248,7 @@ func (m *Monitor) tryGuestAgentMemAvailable(Uint64("available", agentAvailable).Str("source", source).Msg("QEMU memory: using guest agent /proc/meminfo fallback (excludes reclaimable cache)")- return agentAvailable, source, true, false+ return memTotal, agentAvailable, source, true, false}// Return shared guest-command deferral even when independent memory is usable.@@ -281,21 +290,30 @@ func (m *Monitor) resolveGuestStatusMemory(memAvailable := uint64(0)hasMemAvailable := falsederivedTotalMinusUsedAvailable := uint64(0)+ derivedTotalMinusUsedTotal := uint64(0)selectedUsed := uint64(0)hasSelectedUsed := falseif status.MemInfo != nil {+ memInfoTotal := memTotal+ if status.MemInfo.Total > 0 {+ memInfoTotal = status.MemInfo.Total+ }memAvailable, memorySource = deriveGuestMemInfoAvailable(status.MemInfo, guestRaw)hasMemAvailable = memorySource != ""- if memAvailable > memTotal {+ if memInfoTotal == 0 || memInfoTotal > math.MaxInt64 || memAvailable > memInfoTotal ||+ (memTotal > 0 && memInfoTotal > memTotal) {memAvailable = 0hasMemAvailable = falsememorySource = ""}if memorySource == "derived-total-minus-used" {derivedTotalMinusUsedAvailable = memAvailable+ derivedTotalMinusUsedTotal = memInfoTotalmemAvailable = 0hasMemAvailable = falsememorySource = ""+ } else if hasMemAvailable {+ memTotal = memInfoTotal}}@@ -303,9 +321,10 @@ func (m *Monitor) resolveGuestStatusMemory(guestReadDeferred := falseif !hasMemAvailable && shouldPreferGuestAgentMemAvailable(status, memTotal) {triedGuestAgentMemAvailable = true- agentAvailable, agentSource, ok, deferred := m.tryGuestAgentMemAvailable(ctx, client, instanceName, guestName, node, vmid, memTotal, guestRaw)+ agentTotal, agentAvailable, agentSource, ok, deferred := m.tryGuestAgentMemAvailable(ctx, client, instanceName, guestName, node, vmid, memTotal, guestRaw)guestReadDeferred = guestReadDeferred || deferredif ok {+ memTotal = agentTotalmemAvailable = agentAvailablehasMemAvailable = truememorySource = agentSource@@ -318,9 +337,10 @@ func (m *Monitor) resolveGuestStatusMemory(// (#1634).if !hasMemAvailable && !hasSelectedUsed && status.Lock == "" && status.Agent.IsAvailable() && !triedGuestAgentMemAvailable {- agentAvailable, agentSource, ok, deferred := m.tryGuestAgentMemAvailable(ctx, client, instanceName, guestName, node, vmid, memTotal, guestRaw)+ agentTotal, agentAvailable, agentSource, ok, deferred := m.tryGuestAgentMemAvailable(ctx, client, instanceName, guestName, node, vmid, memTotal, guestRaw)guestReadDeferred = guestReadDeferred || deferredif ok {+ memTotal = agentTotalmemAvailable = agentAvailablehasMemAvailable = truememorySource = agentSource@@ -328,6 +348,7 @@ func (m *Monitor) resolveGuestStatusMemory(}if !hasMemAvailable && !hasSelectedUsed && derivedTotalMinusUsedAvailable > 0 {+ memTotal = derivedTotalMinusUsedTotalmemAvailable = derivedTotalMinusUsedAvailablehasMemAvailable = truememorySource = "derived-total-minus-used"@@ -344,8 +365,9 @@ func (m *Monitor) resolveGuestStatusMemory(if !hasMemAvailable && !hasSelectedUsed {if agentHost, ok := vmIDToHostAgent[guestID]; ok &&agentHost.Memory.HasKnownUsage() &&- agentHost.Memory.Used <= int64(memTotal) &&- agentHost.Memory.Total >= agentHost.Memory.Used {+ agentHost.Memory.Total > 0 &&+ (memTotal == 0 || uint64(agentHost.Memory.Total) <= memTotal) {+ memTotal = uint64(agentHost.Memory.Total)agentAvailable := uint64(agentHost.Memory.Total - agentHost.Memory.Used)memAvailable = agentAvailablehasMemAvailable = true@@ -379,6 +401,11 @@ func (m *Monitor) resolveGuestStatusMemory(memorySource = "unavailable"} else {memUsed, memorySource = selectGuestLowTrustUsedMemory(memTotal, status)+ if memorySource == "status-freemem" {+ // That fallback already derives used from the balloon capacity.+ // Keep its denominator with it, without upgrading its cache trust.+ memTotal = effectiveGuestFreeMemTotal(memTotal, status)+ }if memorySource == "" {memorySource = "unavailable"}diff --git a/internal/monitoring/guest_memory_stability.go b/internal/monitoring/guest_memory_stability.goindex e20338986a..750f8ee16c 100644--- a/internal/monitoring/guest_memory_stability.go+++ b/internal/monitoring/guest_memory_stability.go@@ -31,6 +31,28 @@ func guestMemorySourceReliability(source string) int {}}+// Configured capacity fences a VM resize; it is not the denominator of an+// in-guest reading. Older snapshots and containers keep their existing total.+func guestMemoryConfiguredCapacity(previous *GuestMemorySnapshot) uint64 {+ if previous == nil || previous.Memory.Total <= 0 {+ return 0+ }+ if previous.GuestType == "qemu" {+ if previous.Raw.StatusMaxMem > 0 {+ return previous.Raw.StatusMaxMem+ }+ if previous.Raw.ListingMaxMem > 0 {+ return previous.Raw.ListingMaxMem+ }+ }+ return uint64(previous.Memory.Total)+}++func guestMemoryCapacityMatches(previous *GuestMemorySnapshot, currentTotal uint64) bool {+ return previous != nil && previous.Memory.Total > 0 &&+ (uint64(previous.Memory.Total) == currentTotal || guestMemoryConfiguredCapacity(previous) == currentTotal)+}+func (m *Monitor) previousGuestSnapshot(instance, guestType, node string, vmid int) *GuestMemorySnapshot {if m == nil {return nil@@ -104,7 +126,7 @@ func shouldCarryForwardPreviousGuestMemory(prev *GuestMemorySnapshot, currentStaif prevReliability <= currentReliability {return false}- if currentTotal > 0 && prev.Memory.Total > 0 && prev.Memory.Total != int64(currentTotal) {+ if currentTotal > 0 && !guestMemoryCapacityMatches(prev, currentTotal) {return false}@@ -126,7 +148,7 @@ func shouldCarryForwardHealthyGuestLowTrustMemory(prev *GuestMemorySnapshot, curif !guestMemorySnapshotWithinAge(prev, now, guestMemoryHealthyGuestMaxAge) {return false}- if currentTotal == 0 || prev.Memory.Total != int64(currentTotal) {+ if currentTotal == 0 || !guestMemoryCapacityMatches(prev, currentTotal) {return false}if guestMemorySourceReliability(currentSource) != guestMemoryReliabilityLow {diff --git a/internal/monitoring/monitor_pve_guest_builders.go b/internal/monitoring/monitor_pve_guest_builders.goindex f5ce2b4ef2..f56dc2ff0c 100644--- a/internal/monitoring/monitor_pve_guest_builders.go+++ b/internal/monitoring/monitor_pve_guest_builders.go@@ -411,6 +411,15 @@ func (m *Monitor) buildVMFromClusterResource(}var snapshotNotes []string+ if prevSnapshot != nil && (prevSnapshot.Raw.StatusMaxMem > 0 || prevSnapshot.Raw.ListingMaxMem > 0) {+ currentCapacity := state.guestRaw.StatusMaxMem+ if currentCapacity == 0 {+ currentCapacity = state.guestRaw.ListingMaxMem+ }+ if guestMemoryConfiguredCapacity(prevSnapshot) != currentCapacity {+ prevSnapshot = nil // A resize cannot retain the earlier guest tuple.+ }+ }state.memUsed, state.memorySource, snapshotNotes = stabilizeGuestLowTrustMemory(prevSnapshot,res.Status,@@ -428,6 +437,11 @@ func (m *Monitor) buildVMFromClusterResource(state.agentVersion,),)+ if state.memorySource == "previous-snapshot" && prevSnapshot != nil {+ // Retention keeps the original guest denominator as well as its usage.+ // The selector has already checked capacity continuity and source age.+ state.memTotal = uint64(prevSnapshot.Memory.Total)+ }memFree := uint64(0)if state.memTotal >= state.memUsed {memFree = state.memTotal - state.memUsed@@ -478,6 +492,9 @@ func (m *Monitor) buildVMFromClusterResource(trulyFree = state.guestRaw.GuestAgentMemFree}splitReclaimableMemory(&memory, trulyFree)+ if state.memorySource == "previous-snapshot" && prevSnapshot != nil && prevSnapshot.Memory.Observation.State != "" {+ memory = prevSnapshot.Memory+ }if state.detailedStatus != nil && state.detailedStatus.Balloon > 0 {memory.Balloon = int64(state.detailedStatus.Balloon)}