Skip to content

armgit / Pulse

Public Imported from GitHub
0

Real-time monitoring dashboard for Proxmox VE, PBS, Docker, Kubernetes, TrueNAS and vSphere. Self-hosted, with alerts for failed backups, full disks, unhealthy containers and offline nodes.

Bind guest memory usage to its sampled capacity

pulse-triage[bot]ec978c647f41
diff --git a/docs/release-control/v6/internal/subsystems/monitoring.md b/docs/release-control/v6/internal/subsystems/monitoring.md
index 5f26dd6952..286ce796b1 100644
--- a/docs/release-control/v6/internal/subsystems/monitoring.md
+++ b/docs/release-control/v6/internal/subsystems/monitoring.md
@@ -17,6 +17,33 @@
## Purpose
+### Guest memory readings keep their own capacity
+
+The QEMU memory selector pairs guest availability with the total from the
+same status meminfo or QGA `/proc/meminfo` sample, and linked in-guest agent
+usage with that agent's total. A configured maximum is not guest usage:
+ballooning and kernel reservations can make those capacities differ.
+Existing availability-only payloads retain their configured-capacity fallback.
+Reported totals larger than a known VM maximum are not accepted as guest
+evidence, including a hotplug transition before configured status catches up.
+The existing balloon/free-memory fallback also keeps the denominator it used
+to derive usage, without being upgraded to cache-aware pressure evidence.
+The diagnostic snapshot records `guestAgentMemTotal` alongside the already
+retained configured maximum and memory fields; it adds no read or public log.
+
+Retention keeps the selected total, used, free/cache and original observation
+as one tuple. Configured-capacity changes, including a resize to the old guest
+total, cannot borrow that tuple. The existing source age bounds, backup lock,
+filesystem-first ordering and shared command fences are unchanged; a held or
+expired sample cannot produce new History, breach or recovery evidence.
+`TestGuestMemorySampleCapacityContract` and
+`TestGuestMemorySampleCapacityCanonicalProjection` check selectors, coherent
+cache binding and both ordinary collectors with direct/cluster HTTP clients,
+canonical views, History and actual threshold decisions. Healthy ballooned,
+full-pressure/zero-available, legacy, resize and age controls remain distinct.
+These are secret-free source fixtures, not native balloon or freeze recovery,
+reporter #2762 cause/resolution, or release acceptance.
+
### Existing memory origins cannot be renewed by AI consumers
AI query/context/fact consumers now retain the selected guest observation and
diff --git a/internal/monitoring/canonical_guardrails_test.go b/internal/monitoring/canonical_guardrails_test.go
index 8142dbe0f8..4472c7f966 100644
--- a/internal/monitoring/canonical_guardrails_test.go
+++ b/internal/monitoring/canonical_guardrails_test.go
@@ -3297,3 +3297,7 @@ func TestBackupAlertEvaluationCallersShareFailureVisibility(t *testing.T) {
}
}
}
+
+func TestGuestMemorySampleCapacityCanonicalProjection(t *testing.T) {
+ testGuestMemorySampleCapacityPolling(t)
+}
diff --git a/internal/monitoring/diagnostic_snapshots.go b/internal/monitoring/diagnostic_snapshots.go
index ee9ba5b520..1bb10b7ccc 100644
--- a/internal/monitoring/diagnostic_snapshots.go
+++ b/internal/monitoring/diagnostic_snapshots.go
@@ -63,6 +63,7 @@ type VMMemoryRaw struct {
MemInfoCached uint64 `json:"meminfoCached,omitempty"`
MemInfoShared uint64 `json:"meminfoShared,omitempty"`
MemInfoTotalMinusUsed uint64 `json:"meminfoTotalMinusUsed,omitempty"`
+ GuestAgentMemTotal uint64 `json:"guestAgentMemTotal,omitempty"`
GuestAgentMemAvailable uint64 `json:"guestAgentMemAvailable,omitempty"`
GuestAgentMemFree uint64 `json:"guestAgentMemFree,omitempty"`
GuestAgentMemBuffers uint64 `json:"guestAgentMemBuffers,omitempty"`
diff --git a/internal/monitoring/diagnostic_snapshots_test.go b/internal/monitoring/diagnostic_snapshots_test.go
index e7ab36a19b..d03f92e431 100644
--- a/internal/monitoring/diagnostic_snapshots_test.go
+++ b/internal/monitoring/diagnostic_snapshots_test.go
@@ -1390,3 +1390,8 @@ func TestGetDiagnosticSnapshots(t *testing.T) {
}
})
}
+
+func TestGuestMemorySampleCapacityContract(t *testing.T) {
+ t.Run("selection", testGuestMemorySampleCapacitySelection)
+ t.Run("cache-binding", testGuestMemorySampleCapacityCacheBinding)
+}
diff --git a/internal/monitoring/guest_memory_agent.go b/internal/monitoring/guest_memory_agent.go
index e3f75f8a0d..4196aabb7a 100644
--- a/internal/monitoring/guest_memory_agent.go
+++ b/internal/monitoring/guest_memory_agent.go
@@ -43,7 +43,12 @@ func guestMemoryCacheKey(instanceName, node string, vmid int) string {
// refresh; the cleanup age bounds last-known evidence during coordination.
// Neither a poll timestamp nor a previous-snapshot trust label renews that age.
func (m *Monitor) deferredVMGuestMemory(instanceName, node string, vmid int, total uint64, previous *GuestMemorySnapshot, now time.Time) (models.Memory, bool) {
- if previous == nil || previous.GuestType != "qemu" || previous.Status != "running" || previous.Instance != instanceName || previous.Node != node || previous.VMID != vmid || !previous.Memory.HasKnownUsage() || uint64(previous.Memory.Total) != total {
+ if previous == nil || previous.GuestType != "qemu" || previous.Status != "running" || previous.Instance != instanceName || previous.Node != node || previous.VMID != vmid || !previous.Memory.HasKnownUsage() || previous.Memory.Total <= 0 {
+ return models.Memory{}, false
+ }
+ // The last guest sample can be smaller than its configured maximum. Keep
+ // the tuple intact, but still reject retention across a capacity change.
+ if guestMemoryConfiguredCapacity(previous) != total {
return models.Memory{}, false
}
switch CanonicalMemorySource(previous.MemorySource) {
@@ -60,7 +65,8 @@ func (m *Monitor) deferredVMGuestMemory(instanceName, node string, vmid int, tot
if entry.info.Source != "meminfo-available" && entry.info.Source != "meminfo-derived" {
return models.Memory{}, false
}
- if entry.info.EffectiveAvailable > total || previous.Memory.Used != int64(total-entry.info.EffectiveAvailable) {
+ sampleTotal := uint64(previous.Memory.Total)
+ if (entry.info.Total > 0 && entry.info.Total != sampleTotal) || entry.info.EffectiveAvailable > sampleTotal || previous.Memory.Used != int64(sampleTotal-entry.info.EffectiveAvailable) {
return models.Memory{}, false
}
expected := models.Memory{Free: int64(entry.info.EffectiveAvailable)}
diff --git a/internal/monitoring/guest_memory_capacity_test.go b/internal/monitoring/guest_memory_capacity_test.go
new file mode 100644
index 0000000000..11b1b57888
--- /dev/null
+++ b/internal/monitoring/guest_memory_capacity_test.go
@@ -0,0 +1,316 @@
+package monitoring
+
+import (
+ "context"
+ "encoding/json"
+ "fmt"
+ "math"
+ "net/http"
+ "net/http/httptest"
+ "reflect"
+ "strings"
+ "sync/atomic"
+ "testing"
+ "time"
+
+ "github.com/rcourtman/pulse-go-rewrite/internal/alerts"
+ "github.com/rcourtman/pulse-go-rewrite/internal/config"
+ "github.com/rcourtman/pulse-go-rewrite/internal/models"
+ "github.com/rcourtman/pulse-go-rewrite/internal/unifiedresources"
+ "github.com/rcourtman/pulse-go-rewrite/pkg/proxmox"
+)
+
+// These controls run unchanged against the parent. The configured ceiling is
+// deliberately different from the guest sample; this is not a native VM probe.
+func testGuestMemorySampleCapacitySelection(t *testing.T) {
+ const gib = uint64(1024 * 1024 * 1024)
+ id := makeGuestID("capacity", "node", 105)
+ for _, tc := range []struct {
+ name string
+ status proxmox.VMStatus
+ info *proxmox.LinuxMemoryAvailability
+ agent *models.Host
+ wantTotal, wantUsed uint64
+ wantSource string
+ calls int
+ }{
+ {name: "status-available", status: proxmox.VMStatus{MemInfo: &proxmox.VMMemInfo{Total: 4 * gib, Available: 3 * gib}}, wantTotal: 4 * gib, wantUsed: gib, wantSource: "available-field"},
+ {name: "status-free-cache", status: proxmox.VMStatus{MemInfo: &proxmox.VMMemInfo{Total: 4 * gib, Free: gib / 2, Buffers: gib / 2, Cached: 2 * gib}}, wantTotal: 4 * gib, wantUsed: gib, wantSource: "derived-free-buffers-cached"},
+ {name: "status-total-used-gap", status: proxmox.VMStatus{MemInfo: &proxmox.VMMemInfo{Total: 4 * gib, Used: gib, Free: gib / 2}}, wantTotal: 4 * gib, wantUsed: gib, wantSource: "derived-total-minus-used"},
+ {name: "qga-available", status: proxmox.VMStatus{Agent: proxmox.VMAgentField{Value: 1}}, info: &proxmox.LinuxMemoryAvailability{Total: 4 * gib, EffectiveAvailable: 3 * gib, Source: "meminfo-available"}, wantTotal: 4 * gib, wantUsed: gib, wantSource: "guest-agent-meminfo", calls: 1},
+ {name: "qga-derived", status: proxmox.VMStatus{Agent: proxmox.VMAgentField{Value: 1}}, info: &proxmox.LinuxMemoryAvailability{Total: 4 * gib, EffectiveAvailable: 3 * gib, Source: "meminfo-derived"}, wantTotal: 4 * gib, wantUsed: gib, wantSource: "guest-agent-meminfo-derived", calls: 1},
+ {name: "qga-explicit-zero-available", status: proxmox.VMStatus{Agent: proxmox.VMAgentField{Value: 1}}, info: &proxmox.LinuxMemoryAvailability{Total: 4 * gib, Source: "meminfo-available"}, wantTotal: 4 * gib, wantUsed: 4 * gib, wantSource: "guest-agent-meminfo", calls: 1},
+ {name: "qga-invalid-available-exceeds-sample-total", status: proxmox.VMStatus{Agent: proxmox.VMAgentField{Value: 1}}, info: &proxmox.LinuxMemoryAvailability{Total: 4 * gib, EffectiveAvailable: 5 * gib, Source: "meminfo-available"}, wantTotal: 20 * gib, wantUsed: 20 * gib, wantSource: "status-mem", calls: 1},
+ {name: "qga-unrepresentable-sample-total", status: proxmox.VMStatus{Agent: proxmox.VMAgentField{Value: 1}}, info: &proxmox.LinuxMemoryAvailability{Total: uint64(math.MaxInt64) + 1, Source: "meminfo-available"}, wantTotal: 20 * gib, wantUsed: 20 * gib, wantSource: "status-mem", calls: 1},
+ {name: "linked-agent", agent: &models.Host{Memory: models.Memory{Total: 4 * int64(gib), Used: int64(gib), Free: 3 * int64(gib), Usage: 25}}, wantTotal: 4 * gib, wantUsed: gib, wantSource: "agent"},
+ {name: "linked-agent-host-sized-rejected", agent: &models.Host{Memory: models.Memory{Total: 64 * int64(gib), Used: int64(gib), Free: 63 * int64(gib), Usage: 100.0 / 64}}, wantTotal: 20 * gib, wantUsed: 20 * gib, wantSource: "status-mem"},
+ {name: "balloon-freemem-keeps-low-trust", status: proxmox.VMStatus{Balloon: 4 * gib, FreeMem: 3 * gib}, wantTotal: 4 * gib, wantUsed: gib, wantSource: "status-freemem"},
+ {name: "ballooninfo-freemem-keeps-low-trust", status: proxmox.VMStatus{BalloonInfo: &proxmox.VMBalloonInfo{TotalMem: 4 * gib, FreeMem: 3 * gib}}, wantTotal: 4 * gib, wantUsed: gib, wantSource: "status-freemem"},
+ {name: "missing-status-total-compatibility", status: proxmox.VMStatus{MemInfo: &proxmox.VMMemInfo{Available: 3 * gib}}, wantTotal: 20 * gib, wantUsed: 17 * gib, wantSource: "available-field"},
+ {name: "availability-only-compatibility", status: proxmox.VMStatus{Agent: proxmox.VMAgentField{Value: 1}}, info: &proxmox.LinuxMemoryAvailability{EffectiveAvailable: 3 * gib, Source: "meminfo-available"}, wantTotal: 20 * gib, wantUsed: 17 * gib, wantSource: "guest-agent-meminfo", calls: 1},
+ {name: "matching-capacity-unchanged", status: proxmox.VMStatus{MemInfo: &proxmox.VMMemInfo{Total: 20 * gib, Available: 15 * gib}}, wantTotal: 20 * gib, wantUsed: 5 * gib, wantSource: "available-field"},
+ {name: "status-total-above-configured-ceiling-rejected", status: proxmox.VMStatus{MemInfo: &proxmox.VMMemInfo{Total: 24 * gib, Available: 18 * gib}}, wantTotal: 20 * gib, wantSource: "unavailable"},
+ {name: "qga-total-above-configured-ceiling-rejected", status: proxmox.VMStatus{Agent: proxmox.VMAgentField{Value: 1}}, info: &proxmox.LinuxMemoryAvailability{Total: 24 * gib, EffectiveAvailable: 18 * gib, Source: "meminfo-available"}, wantTotal: 20 * gib, wantUsed: 20 * gib, wantSource: "status-mem", calls: 1},
+ } {
+ t.Run(tc.name, func(t *testing.T) {
+ m := &Monitor{}
+ client := &guestMemoryAgentTestClient{stubPVEClient: &stubPVEClient{}, memInfo: tc.info}
+ status := tc.status
+ status.MaxMem, status.Mem = 20*gib, 20*gib
+ linked := map[string]models.Host{}
+ if tc.agent != nil {
+ linked[id] = *tc.agent
+ }
+ total, used, source, deferred := m.resolveGuestStatusMemory(context.Background(), client, "capacity", "guest", "node", 105, id, &status, linked, 20*gib, "cluster-resources", &VMMemoryRaw{})
+ if total != tc.wantTotal || used != tc.wantUsed || source != tc.wantSource || deferred || client.memCalls != tc.calls || client.rrdCalls != 0 {
+ t.Errorf("selected %d/%d (%s, deferred=%t, calls=%d), want %d/%d (%s, calls=%d)", used, total, source, deferred, client.memCalls, tc.wantUsed, tc.wantTotal, tc.wantSource, tc.calls)
+ }
+ e := unifiedresources.QualifyGuestMemory(safePercentage(float64(used), float64(total)), models.MemoryObservation{State: "current", Source: source, ObservedAt: time.Now()}, true, true, time.Now())
+ wantPressure := source == "available-field" || source == "derived-free-buffers-cached" || source == "guest-agent-meminfo" || source == "guest-agent-meminfo-derived" || source == "agent"
+ if e.PressureKnown != wantPressure {
+ t.Error("capacity selection changed source qualification")
+ }
+ })
+ }
+}
+
+func testGuestMemorySampleCapacityCacheBinding(t *testing.T) {
+ now := time.Now()
+ memory := models.Memory{Total: 4000, Used: 1000, Free: 500, Cache: 2500, Usage: 25}
+ key := guestMemoryCacheKey("capacity", "node", 105)
+ for _, tc := range []struct {
+ name string
+ cachedTotal uint64
+ want bool
+ }{
+ {"same-tuple", 4000, true}, {"different-denominator-same-available", 5000, false}, {"legacy-availability-only", 0, true},
+ } {
+ t.Run(tc.name, func(t *testing.T) {
+ m := &Monitor{vmAgentMemCache: map[string]agentMemCacheEntry{key: {fetchedAt: now, info: proxmox.LinuxMemoryAvailability{Total: tc.cachedTotal, Free: 500, EffectiveAvailable: 3000, Source: "meminfo-available"}}}}
+ observation, ok := m.cachedGuestMemoryObservation("capacity", "node", 105, memory, now)
+ if ok != tc.want || ok && !observation.ObservedAt.Equal(now) {
+ t.Errorf("origin bound to wrong tuple: %+v, %t", observation, ok)
+ }
+ prev := &GuestMemorySnapshot{GuestType: "qemu", Status: "running", Instance: "capacity", Node: "node", VMID: 105, MemorySource: "guest-agent-meminfo", Memory: memory, Raw: VMMemoryRaw{StatusMaxMem: 20000}}
+ retained, ok := m.deferredVMGuestMemory("capacity", "node", 105, 20000, prev, now)
+ if ok != tc.want || ok && !reflect.DeepEqual(retained, memory) {
+ t.Errorf("deferral lost tuple: %+v, %t", retained, ok)
+ }
+ if _, ok := m.deferredVMGuestMemory("capacity", "node", 105, 4000, prev, now); ok {
+ t.Error("resize to old guest total retained earlier tuple")
+ }
+ if _, ok := m.deferredVMGuestMemory("other", "node", 105, 20000, prev, now); ok {
+ t.Error("retention crossed instance identity")
+ }
+ })
+ }
+}
+
+// Both ordinary collectors and concrete HTTP clients feed canonical views,
+// History and the production alert call. Lock deferral, age and resize controls
+// keep a held value from becoming a new breach or recovery. No guest is frozen.
+func testGuestMemorySampleCapacityPolling(t *testing.T) {
+ const gib = uint64(1024 * 1024 * 1024)
+ for _, clientKind := range []string{"direct", "cluster"} {
+ for _, collector := range []string{"cluster-resources", "node-list"} {
+ t.Run(clientKind+"/"+collector, func(t *testing.T) {
+ var phase, commands, memoryReads atomic.Int32
+ var configured atomic.Uint64
+ configured.Store(20 * gib)
+ server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+ w.Header().Set("Content-Type", "application/json")
+ switch {
+ case strings.HasSuffix(r.URL.Path, "/status/current"):
+ lock := ""
+ if phase.Load() == 1 {
+ lock = `,"lock":"backup"`
+ }
+ fmt.Fprintf(w, `{"data":{"status":"running","agent":1,"maxmem":%d,"mem":%d%s}}`, configured.Load(), configured.Load(), lock)
+ case strings.HasSuffix(r.URL.Path, "/config"):
+ if phase.Load() == 1 {
+ fmt.Fprint(w, `{"data":{"lock":"backup"}}`)
+ } else {
+ fmt.Fprint(w, `{"data":{}}`)
+ }
+ case strings.Contains(r.URL.Path, "/agent/"):
+ commands.Add(1)
+ switch {
+ case strings.HasSuffix(r.URL.Path, "/file-read"):
+ memoryReads.Add(1)
+ total, available, free := 4*gib, 3*gib, gib/2
+ if phase.Load() == 2 {
+ available, free = 0, 0
+ }
+ if phase.Load() == 4 {
+ total, available, free = 8*gib, 6*gib, gib
+ }
+ json.NewEncoder(w).Encode(map[string]any{"data": map[string]any{"content": fmt.Sprintf("MemAvailable: %d kB\nMemFree: %d kB\nMemTotal: %d kB\n", available/1024, free/1024, total/1024)}})
+ case strings.HasSuffix(r.URL.Path, "/get-fsinfo"):
+ fmt.Fprintf(w, `{"data":{"result":[{"mountpoint":"/","type":"ext4","total-bytes":%d,"used-bytes":%d}]}}`, gib, gib/4)
+ case strings.HasSuffix(r.URL.Path, "/get-osinfo"):
+ fmt.Fprint(w, `{"data":{"name":"Linux","version":"fixture"}}`)
+ case strings.HasSuffix(r.URL.Path, "/version"):
+ fmt.Fprint(w, `{"data":{"result":{"version":"fixture"}}}`)
+ default:
+ fmt.Fprint(w, `{"data":{"result":[]}}`)
+ }
+ case strings.HasSuffix(r.URL.Path, "/version"):
+ fmt.Fprint(w, `{"data":{"version":"9.2-fixture"}}`)
+ case strings.HasSuffix(r.URL.Path, "/nodes"):
+ fmt.Fprint(w, `{"data":[{"node":"node","status":"online"}]}`)
+ default:
+ http.NotFound(w, r)
+ }
+ }))
+ defer server.Close()
+ pcfg := proxmox.ClientConfig{Host: server.URL, TokenName: "fixture@pve!pulse", TokenValue: "fixture", Timeout: time.Second}
+ var client PVEClientInterface
+ if clientKind == "direct" {
+ var err error
+ client, err = proxmox.NewClient(pcfg)
+ if err != nil {
+ t.Fatal(err)
+ }
+ } else {
+ client = proxmox.NewClusterClient("capacity", pcfg, []string{server.URL}, nil)
+ }
+ am := alerts.NewManagerWithDataDir(t.TempDir(), alerts.WithoutPersistedAlertRestore())
+ defer am.Stop()
+ acfg := am.GetConfig()
+ acfg.Enabled = true
+ acfg.GuestDefaults.Memory = &alerts.HysteresisThreshold{Trigger: 80, Clear: 75}
+ acfg.GuestDefaults.CPU = &alerts.HysteresisThreshold{Trigger: 80, Clear: 75}
+ acfg.GuestDefaults.Disk = &alerts.HysteresisThreshold{Trigger: 90, Clear: 85}
+ acfg.MetricTimeThresholds = map[string]map[string]int{"guest": {"memory": 0, "cpu": 0, "disk": 0}}
+ am.UpdateConfig(acfg)
+ m := &Monitor{config: &config.Config{}, alertManager: am, rateTracker: NewRateTracker(), metricsHistory: NewMetricsHistory(32, time.Hour), guestMetadataLimiter: make(map[string]time.Time), vmAgentMemCache: make(map[string]agentMemCacheEntry)}
+ registry := unifiedresources.NewRegistry(nil)
+ id := makeGuestID("capacity", "node", 105)
+ previous := map[string]models.VM{}
+ build := func() models.VM {
+ t.Helper()
+ res := proxmox.ClusterResource{Type: "qemu", Node: "node", Name: "guest", VMID: 105, Status: "running", MaxMem: configured.Load(), Mem: configured.Load(), MaxDisk: gib, CPU: .1}
+ var vms []models.VM
+ if collector == "cluster-resources" {
+ vms = m.collectClusterVMResources(context.Background(), "capacity", []indexedClusterResource{{resource: res, guestID: id}}, client, previous, nil)
+ } else {
+ vms, _ = m.pollNodeVMsWithClusterResourceBuilder(context.Background(), "capacity", "node", []proxmox.VM{{VMID: 105, Name: "guest", Status: "running", MaxMem: res.MaxMem, Mem: res.Mem, MaxDisk: gib, CPU: .1}}, client, previous, nil)
+ }
+ if len(vms) != 1 {
+ t.Fatalf("collector lost VM: %v", vms)
+ }
+ vm := vms[0]
+ m.recordGuestMetrics(vms, nil, time.Now().Add(-time.Second))
+ registry.IngestSnapshot(models.StateSnapshot{VMs: vms})
+ view := registry.VMs()[0]
+ if vm.Memory.HasKnownUsage() && (view.MemoryTotal() != vm.Memory.Total || view.MemoryUsed() != vm.Memory.Used || view.MemoryPercent() != vm.Memory.Usage) {
+ t.Error("canonical view changed selected tuple")
+ }
+ if !vm.Memory.HasKnownUsage() && view.MemoryEvidence(time.Now()).Available {
+ t.Error("unavailable memory became a canonical measurement")
+ }
+ previous[id] = previousVMFromView(view)
+ return vm
+ }
+ assertHealthy := func(vm models.VM, total uint64) {
+ t.Helper()
+ if vm.Memory.Total != int64(total) || vm.Memory.Used != int64(total/4) || vm.Memory.Usage != 25 || vm.Memory.Free != int64(total/8) || vm.Memory.Cache != int64(total*5/8) {
+ t.Errorf("healthy sample mixed capacities: %+v", vm.Memory)
+ }
+ if !vm.Memory.HasKnownUsage() {
+ t.Error("healthy guest became unavailable")
+ }
+ }
+ initial := build()
+ assertHealthy(initial, 4*gib)
+ if len(am.GetActiveAlerts()) != 0 {
+ t.Error("healthy ballooned guest created false memory alert")
+ }
+ snapshot := m.previousGuestSnapshot("capacity", "qemu", "node", 105)
+ rawBytes, _ := json.Marshal(snapshot.Raw)
+ var raw map[string]uint64
+ json.Unmarshal(rawBytes, &raw)
+ if raw["guestAgentMemTotal"] != 4*gib || raw["statusMaxmem"] != 20*gib {
+ t.Errorf("raw evidence lost guest/configured capacity distinction: %s", rawBytes)
+ }
+ if memoryReads.Load() != 1 {
+ t.Error("initial sample repeated optional memory read")
+ }
+ origin := initial.Memory.Observation.ObservedAt
+ before := commands.Load()
+ phase.Store(1)
+ for i := 0; i < 3; i++ {
+ held := build()
+ assertHealthy(held, 4*gib)
+ if held.Memory.Observation.State != "last-known" || !held.Memory.Observation.ObservedAt.Equal(origin) || held.GuestAgentStatus != "deferred" {
+ t.Error("lock renewed memory origin or lost shared deferral")
+ }
+ }
+ if commands.Load() != before || len(m.metricsHistory.GetGuestMetrics(id, "memory", time.Hour)) != 1 {
+ t.Error("lock queued QGA or repeated memory History")
+ }
+ // A resize cannot retain the old tuple even when its new maximum
+ // happens to equal the earlier guest sample's total.
+ configured.Store(4 * gib)
+ resized := build()
+ if resized.Memory.HasKnownUsage() || resized.Memory.Observation.State != "unavailable" {
+ t.Errorf("resize retained earlier tuple: %+v", resized.Memory)
+ }
+ configured.Store(20 * gib)
+ m.recordGuestSnapshot("capacity", "qemu", "node", 105, *snapshot)
+ aged := *snapshot
+ aged.Memory.Observation.ObservedAt = time.Now().Add(-11 * time.Minute)
+ m.recordGuestSnapshot("capacity", "qemu", "node", 105, aged)
+ key := guestMemoryCacheKey("capacity", "node", 105)
+ entry := m.vmAgentMemCache[key]
+ entry.fetchedAt = aged.Memory.Observation.ObservedAt
+ m.vmAgentMemCache[key] = entry
+ expired := build()
+ if expired.Memory.HasKnownUsage() || expired.Memory.Observation.State != "unavailable" || commands.Load() != before {
+ t.Errorf("expiry fabricated observation or command: %+v", expired.Memory)
+ }
+ phase.Store(2)
+ pressure := build()
+ if pressure.Memory.Total != 4*int64(gib) || pressure.Memory.Used != 4*int64(gib) || pressure.Memory.Usage != 100 || pressure.Memory.Observation.State != "current" || len(am.GetActiveAlerts()) != 1 {
+ t.Errorf("real full-pressure sample suppressed: %+v; alerts=%d", pressure.Memory, len(am.GetActiveAlerts()))
+ }
+ for _, a := range am.GetActiveAlerts() {
+ if a.Type != "memory" || a.Value != 100 {
+ t.Error("wrong pressure alert")
+ }
+ }
+ phase.Store(3)
+ entry = m.vmAgentMemCache[key]
+ entry.fetchedAt = time.Now().Add(-2 * vmAgentMemCacheTTL)
+ m.vmAgentMemCache[key] = entry
+ recovered := build()
+ assertHealthy(recovered, 4*gib)
+ if len(am.GetActiveAlerts()) != 0 || !recovered.Memory.Observation.ObservedAt.After(pressure.Memory.Observation.ObservedAt) {
+ t.Error("current healthy sample did not resolve pressure with a new origin")
+ }
+ phase.Store(4)
+ configured.Store(24 * gib)
+ entry = m.vmAgentMemCache[key]
+ entry.fetchedAt = time.Now().Add(-2 * vmAgentMemCacheTTL)
+ m.vmAgentMemCache[key] = entry
+ assertHealthy(build(), 8*gib)
+ if memoryReads.Load() != 4 {
+ t.Errorf("normal refresh read count = %d, want 4", memoryReads.Load())
+ }
+ points := m.metricsHistory.GetGuestMetrics(id, "memory", time.Hour)
+ if len(points) != 4 {
+ t.Errorf("current History count = %d, want 4", len(points))
+ }
+ for i, p := range points {
+ want := 25.0
+ if i == 1 {
+ want = 100
+ }
+ if p.Value != want {
+ t.Errorf("History[%d]=%v, want %v", i, p.Value, want)
+ }
+ }
+ })
+ }
+ }
+}
diff --git a/internal/monitoring/guest_memory_observation.go b/internal/monitoring/guest_memory_observation.go
index aa29e34b78..4dd1d0da7d 100644
--- a/internal/monitoring/guest_memory_observation.go
+++ b/internal/monitoring/guest_memory_observation.go
@@ -77,7 +77,7 @@ func (m *Monitor) cachedGuestMemoryObservation(instance, node string, vmid int,
m.rrdCacheMu.RLock()
entry, ok := m.vmAgentMemCache[guestMemoryCacheKey(instance, node, vmid)]
m.rrdCacheMu.RUnlock()
- if !ok || entry.negative || entry.fetchedAt.IsZero() || entry.fetchedAt.After(now) || now.Sub(entry.fetchedAt) > vmAgentMemCleanupMaxAge || memory.Total <= 0 || entry.info.EffectiveAvailable > uint64(memory.Total) || memory.Used != memory.Total-int64(entry.info.EffectiveAvailable) {
+ if !ok || entry.negative || entry.fetchedAt.IsZero() || entry.fetchedAt.After(now) || now.Sub(entry.fetchedAt) > vmAgentMemCleanupMaxAge || memory.Total <= 0 || (entry.info.Total > 0 && entry.info.Total != uint64(memory.Total)) || entry.info.EffectiveAvailable > uint64(memory.Total) || memory.Used != memory.Total-int64(entry.info.EffectiveAvailable) {
return models.MemoryObservation{}, false
}
expected := models.Memory{Free: int64(entry.info.EffectiveAvailable)}
diff --git a/internal/monitoring/guest_memory_sources.go b/internal/monitoring/guest_memory_sources.go
index 2f113eb4f7..93dc7d4080 100644
--- a/internal/monitoring/guest_memory_sources.go
+++ b/internal/monitoring/guest_memory_sources.go
@@ -209,13 +209,21 @@ func (m *Monitor) tryGuestAgentMemAvailable(
vmid int,
memTotal uint64,
guestRaw *VMMemoryRaw,
-) (uint64, string, bool, bool) {
+) (uint64, uint64, string, bool, bool) {
availability, agentErr := m.getVMAgentMemoryAvailability(ctx, client, instanceName, node, vmid)
if agentErr != nil || availability.Source == "" {
- return 0, "", false, errors.Is(agentErr, proxmox.ErrGuestAgentDeferred)
+ return 0, 0, "", false, errors.Is(agentErr, proxmox.ErrGuestAgentDeferred)
+ }
+ // MemAvailable belongs to this guest's MemTotal, not the configured
+ // maximum. Ballooning and kernel-reserved pages can make them different.
+ // Older availability-only clients retain their supplied capacity fallback.
+ configuredTotal := memTotal
+ if availability.Total > 0 {
+ memTotal = availability.Total
}
agentAvailable := availability.EffectiveAvailable
if guestRaw != nil {
+ guestRaw.GuestAgentMemTotal = availability.Total
guestRaw.GuestAgentMemAvailable = agentAvailable
guestRaw.GuestAgentMemFree = availability.Free
guestRaw.GuestAgentMemBuffers = availability.Buffers
@@ -224,8 +232,9 @@ func (m *Monitor) tryGuestAgentMemAvailable(
guestRaw.GuestAgentShmem = availability.Shmem
guestRaw.GuestAgentDerived = availability.Source == "meminfo-derived"
}
- if memTotal == 0 || agentAvailable > memTotal {
- return 0, "", false, false
+ if memTotal == 0 || memTotal > math.MaxInt64 || agentAvailable > memTotal ||
+ (configuredTotal > 0 && memTotal > configuredTotal) {
+ return 0, 0, "", false, false
}
source := "guest-agent-meminfo"
if availability.Source == "meminfo-derived" {
@@ -239,7 +248,7 @@ func (m *Monitor) tryGuestAgentMemAvailable(
Uint64("available", agentAvailable).
Str("source", source).
Msg("QEMU memory: using guest agent /proc/meminfo fallback (excludes reclaimable cache)")
- return agentAvailable, source, true, false
+ return memTotal, agentAvailable, source, true, false
}
// Return shared guest-command deferral even when independent memory is usable.
@@ -281,21 +290,30 @@ func (m *Monitor) resolveGuestStatusMemory(
memAvailable := uint64(0)
hasMemAvailable := false
derivedTotalMinusUsedAvailable := uint64(0)
+ derivedTotalMinusUsedTotal := uint64(0)
selectedUsed := uint64(0)
hasSelectedUsed := false
if status.MemInfo != nil {
+ memInfoTotal := memTotal
+ if status.MemInfo.Total > 0 {
+ memInfoTotal = status.MemInfo.Total
+ }
memAvailable, memorySource = deriveGuestMemInfoAvailable(status.MemInfo, guestRaw)
hasMemAvailable = memorySource != ""
- if memAvailable > memTotal {
+ if memInfoTotal == 0 || memInfoTotal > math.MaxInt64 || memAvailable > memInfoTotal ||
+ (memTotal > 0 && memInfoTotal > memTotal) {
memAvailable = 0
hasMemAvailable = false
memorySource = ""
}
if memorySource == "derived-total-minus-used" {
derivedTotalMinusUsedAvailable = memAvailable
+ derivedTotalMinusUsedTotal = memInfoTotal
memAvailable = 0
hasMemAvailable = false
memorySource = ""
+ } else if hasMemAvailable {
+ memTotal = memInfoTotal
}
}
@@ -303,9 +321,10 @@ func (m *Monitor) resolveGuestStatusMemory(
guestReadDeferred := false
if !hasMemAvailable && shouldPreferGuestAgentMemAvailable(status, memTotal) {
triedGuestAgentMemAvailable = true
- agentAvailable, agentSource, ok, deferred := m.tryGuestAgentMemAvailable(ctx, client, instanceName, guestName, node, vmid, memTotal, guestRaw)
+ agentTotal, agentAvailable, agentSource, ok, deferred := m.tryGuestAgentMemAvailable(ctx, client, instanceName, guestName, node, vmid, memTotal, guestRaw)
guestReadDeferred = guestReadDeferred || deferred
if ok {
+ memTotal = agentTotal
memAvailable = agentAvailable
hasMemAvailable = true
memorySource = agentSource
@@ -318,9 +337,10 @@ func (m *Monitor) resolveGuestStatusMemory(
// (#1634).
if !hasMemAvailable && !hasSelectedUsed && status.Lock == "" && status.Agent.IsAvailable() && !triedGuestAgentMemAvailable {
- agentAvailable, agentSource, ok, deferred := m.tryGuestAgentMemAvailable(ctx, client, instanceName, guestName, node, vmid, memTotal, guestRaw)
+ agentTotal, agentAvailable, agentSource, ok, deferred := m.tryGuestAgentMemAvailable(ctx, client, instanceName, guestName, node, vmid, memTotal, guestRaw)
guestReadDeferred = guestReadDeferred || deferred
if ok {
+ memTotal = agentTotal
memAvailable = agentAvailable
hasMemAvailable = true
memorySource = agentSource
@@ -328,6 +348,7 @@ func (m *Monitor) resolveGuestStatusMemory(
}
if !hasMemAvailable && !hasSelectedUsed && derivedTotalMinusUsedAvailable > 0 {
+ memTotal = derivedTotalMinusUsedTotal
memAvailable = derivedTotalMinusUsedAvailable
hasMemAvailable = true
memorySource = "derived-total-minus-used"
@@ -344,8 +365,9 @@ func (m *Monitor) resolveGuestStatusMemory(
if !hasMemAvailable && !hasSelectedUsed {
if agentHost, ok := vmIDToHostAgent[guestID]; ok &&
agentHost.Memory.HasKnownUsage() &&
- agentHost.Memory.Used <= int64(memTotal) &&
- agentHost.Memory.Total >= agentHost.Memory.Used {
+ agentHost.Memory.Total > 0 &&
+ (memTotal == 0 || uint64(agentHost.Memory.Total) <= memTotal) {
+ memTotal = uint64(agentHost.Memory.Total)
agentAvailable := uint64(agentHost.Memory.Total - agentHost.Memory.Used)
memAvailable = agentAvailable
hasMemAvailable = true
@@ -379,6 +401,11 @@ func (m *Monitor) resolveGuestStatusMemory(
memorySource = "unavailable"
} else {
memUsed, memorySource = selectGuestLowTrustUsedMemory(memTotal, status)
+ if memorySource == "status-freemem" {
+ // That fallback already derives used from the balloon capacity.
+ // Keep its denominator with it, without upgrading its cache trust.
+ memTotal = effectiveGuestFreeMemTotal(memTotal, status)
+ }
if memorySource == "" {
memorySource = "unavailable"
}
diff --git a/internal/monitoring/guest_memory_stability.go b/internal/monitoring/guest_memory_stability.go
index e20338986a..750f8ee16c 100644
--- a/internal/monitoring/guest_memory_stability.go
+++ b/internal/monitoring/guest_memory_stability.go
@@ -31,6 +31,28 @@ func guestMemorySourceReliability(source string) int {
}
}
+// Configured capacity fences a VM resize; it is not the denominator of an
+// in-guest reading. Older snapshots and containers keep their existing total.
+func guestMemoryConfiguredCapacity(previous *GuestMemorySnapshot) uint64 {
+ if previous == nil || previous.Memory.Total <= 0 {
+ return 0
+ }
+ if previous.GuestType == "qemu" {
+ if previous.Raw.StatusMaxMem > 0 {
+ return previous.Raw.StatusMaxMem
+ }
+ if previous.Raw.ListingMaxMem > 0 {
+ return previous.Raw.ListingMaxMem
+ }
+ }
+ return uint64(previous.Memory.Total)
+}
+
+func guestMemoryCapacityMatches(previous *GuestMemorySnapshot, currentTotal uint64) bool {
+ return previous != nil && previous.Memory.Total > 0 &&
+ (uint64(previous.Memory.Total) == currentTotal || guestMemoryConfiguredCapacity(previous) == currentTotal)
+}
+
func (m *Monitor) previousGuestSnapshot(instance, guestType, node string, vmid int) *GuestMemorySnapshot {
if m == nil {
return nil
@@ -104,7 +126,7 @@ func shouldCarryForwardPreviousGuestMemory(prev *GuestMemorySnapshot, currentSta
if prevReliability <= currentReliability {
return false
}
- if currentTotal > 0 && prev.Memory.Total > 0 && prev.Memory.Total != int64(currentTotal) {
+ if currentTotal > 0 && !guestMemoryCapacityMatches(prev, currentTotal) {
return false
}
@@ -126,7 +148,7 @@ func shouldCarryForwardHealthyGuestLowTrustMemory(prev *GuestMemorySnapshot, cur
if !guestMemorySnapshotWithinAge(prev, now, guestMemoryHealthyGuestMaxAge) {
return false
}
- if currentTotal == 0 || prev.Memory.Total != int64(currentTotal) {
+ if currentTotal == 0 || !guestMemoryCapacityMatches(prev, currentTotal) {
return false
}
if guestMemorySourceReliability(currentSource) != guestMemoryReliabilityLow {
diff --git a/internal/monitoring/monitor_pve_guest_builders.go b/internal/monitoring/monitor_pve_guest_builders.go
index f5ce2b4ef2..f56dc2ff0c 100644
--- a/internal/monitoring/monitor_pve_guest_builders.go
+++ b/internal/monitoring/monitor_pve_guest_builders.go
@@ -411,6 +411,15 @@ func (m *Monitor) buildVMFromClusterResource(
}
var snapshotNotes []string
+ if prevSnapshot != nil && (prevSnapshot.Raw.StatusMaxMem > 0 || prevSnapshot.Raw.ListingMaxMem > 0) {
+ currentCapacity := state.guestRaw.StatusMaxMem
+ if currentCapacity == 0 {
+ currentCapacity = state.guestRaw.ListingMaxMem
+ }
+ if guestMemoryConfiguredCapacity(prevSnapshot) != currentCapacity {
+ prevSnapshot = nil // A resize cannot retain the earlier guest tuple.
+ }
+ }
state.memUsed, state.memorySource, snapshotNotes = stabilizeGuestLowTrustMemory(
prevSnapshot,
res.Status,
@@ -428,6 +437,11 @@ func (m *Monitor) buildVMFromClusterResource(
state.agentVersion,
),
)
+ if state.memorySource == "previous-snapshot" && prevSnapshot != nil {
+ // Retention keeps the original guest denominator as well as its usage.
+ // The selector has already checked capacity continuity and source age.
+ state.memTotal = uint64(prevSnapshot.Memory.Total)
+ }
memFree := uint64(0)
if state.memTotal >= state.memUsed {
memFree = state.memTotal - state.memUsed
@@ -478,6 +492,9 @@ func (m *Monitor) buildVMFromClusterResource(
trulyFree = state.guestRaw.GuestAgentMemFree
}
splitReclaimableMemory(&memory, trulyFree)
+ if state.memorySource == "previous-snapshot" && prevSnapshot != nil && prevSnapshot.Memory.Observation.State != "" {
+ memory = prevSnapshot.Memory
+ }
if state.detailedStatus != nil && state.detailedStatus.Balloon > 0 {
memory.Balloon = int64(state.detailedStatus.Balloon)
}