Real-time monitoring dashboard for Proxmox VE, PBS, Docker, Kubernetes, TrueNAS and vSphere. Self-hosted, with alerts for failed backups, full disks, unhealthy containers and offline nodes.
Go aialertsdashboarddockerdocker-monitoringgohomelabinfrastructure-monitoringkubernetesmonitoringmspproxmoxproxmox-backup-serverproxmox-mail-gatewayproxmox-veself-hostedtruenastypescriptvspherewebhooks
Learn guest OS before optional Linux memory polling
pulse-triage[bot]081c12f65c6b
diff --git a/docs/release-control/v6/internal/subsystems/monitoring.md b/docs/release-control/v6/internal/subsystems/monitoring.mdindex 39202ba490..c1ce355671 100644--- a/docs/release-control/v6/internal/subsystems/monitoring.md+++ b/docs/release-control/v6/internal/subsystems/monitoring.md@@ -174,7 +174,14 @@ It does not invent a negative memory observation or clear an existing pause.Independent status/balloon/linked-agent memory, filesystem-first ordering,unknown-OS compatibility and all command/backup-lock admission remain intact.The first unknown-OS poll can still attempt the existing Linux fallback; this-change does not add a probe or infer an OS from a display name/configuration hint.+requires completing ordinary metadata collection first. Both detailed-status+collectors now collect filesystems, then metadata, then optional Linux memory.+On an ordinary cold start or expired metadata refresh, the real OS reply can+therefore establish the existing Windows exclusion before an inappropriate+Linux read creates a shared pause. No extra probe, persistent OS hint or+configuration/display-name inference is added. If metadata itself is uncertain,+the same channel guard still prevents every later command; independent current+status/balloon/linked-agent memory is not turned into guest meminfo evidence.`TestGuestWindowsMeminfoPollingContract` covers the real direct/cluster clientsand both collectors after an actual OS read: repeated NTFS usage through zero,@@ -185,6 +192,19 @@ evidence, Linux recovery and cache/source compatibility. These are syntheticsource controls, not native QEMU-only Windows recovery, thaw, cross-process orrestart containment, reporter resolution or release acceptance.+`TestGuestWindowsRestartPollingContract` starts empty monitors through both real+clients and both collectors, ages actual OS receipts for the expiry control,+and runs two distinct fresh test processes against the same loopback endpoint.+It checks both NTFS volumes, healthy zero recovery, OS/canonical/public state,+memory source and in-memory/persistent History. The Linux command's fixture+response remains an unexplained HTTP 500: recovery comes from not sending it,+not from a completion exemption. Lock, in-flight/shared-client uncertainty,+original ages, Linux/unknown and replacement-identity controls remain covered.+The ordinary supported loopback process restart is source acceptance, not+native Pro/PVE restart recovery, every-filesystem writes/liveness, independent+thaw, prior-active restoration or a released fix. Existing OS-info safety+suppression and metadata backoff are not reset to force recovery.+### QGA fallback eligibility keeps its original observation ageA successful unlocked VM-status availability, usable filesystem reading ordiff --git a/internal/monitoring/canonical_guardrails_test.go b/internal/monitoring/canonical_guardrails_test.goindex 29dae0cd86..93474edaf1 100644--- a/internal/monitoring/canonical_guardrails_test.go+++ b/internal/monitoring/canonical_guardrails_test.go@@ -3254,6 +3254,7 @@ func TestGuestWindowsMeminfoPollingContract(t *testing.T) {func TestGuestWindowsRestartPollingContract(t *testing.T) {t.Run("cold-and-expired", testGuestWindowsBootstrapPolling)t.Run("process-restart", testGuestWindowsBootstrapProcessRestart)+ t.Run("admission-and-identity", testGuestWindowsBootstrapSafety)}func TestGuestMemoryObservationContract(t *testing.T) {diff --git a/internal/monitoring/guest_agent_optional_read_integration_test.go b/internal/monitoring/guest_agent_optional_read_integration_test.goindex 6d8145a3f5..5804ef02c6 100644--- a/internal/monitoring/guest_agent_optional_read_integration_test.go+++ b/internal/monitoring/guest_agent_optional_read_integration_test.go@@ -29,8 +29,10 @@ func testGuestAgentOptionalReadOrdering(t *testing.T, withoutStatus bool) {terminal bool}{{"windows-unsupported-memory", "file-read", "Microsoft Windows 11", "ntfs", true},+ {"linux-unsupported-memory", "file-read", "Linux", "ext4", true},{"android-unsupported-os", "get-osinfo", "Android", "ext4", true},- {"uncertain-memory", "file-read", "Microsoft Windows 11", "ntfs", false},+ {"uncertain-memory", "file-read", "Linux", "ext4", false},+ {"unknown-os-uncertain-memory", "file-read", "", "ext4", false},{"uncertain-network", "network-get-interfaces", "Android", "ext4", false},{"uncertain-os", "get-osinfo", "Android", "ext4", false},{"uncertain-version", "info", "Microsoft Windows 11", "ntfs", false},diff --git a/internal/monitoring/guest_windows_bootstrap_test.go b/internal/monitoring/guest_windows_bootstrap_test.goindex 4c4eb83795..61c46f3898 100644--- a/internal/monitoring/guest_windows_bootstrap_test.go+++ b/internal/monitoring/guest_windows_bootstrap_test.go@@ -72,6 +72,7 @@ func (f *windowsBootstrapFixture) serve(w http.ResponseWriter, r *http.Request)w.Header().Set("Content-Type", "application/json")f.mu.Lock()osName, uncertain, locked, available := f.osName, f.uncertain, f.locked, f.linuxAvailable+ fsEntered, fsRelease := f.fsEntered, f.fsReleasef.mu.Unlock()switch {case strings.HasSuffix(r.URL.Path, "/nodes"):@@ -94,9 +95,9 @@ func (f *windowsBootstrapFixture) serve(w http.ResponseWriter, r *http.Request)f.mu.Lock()f.commands = append(f.commands, cmd)f.mu.Unlock()- if cmd == "get-fsinfo" && f.fsEntered != nil {- f.fsOnce.Do(func() { close(f.fsEntered) })- <-f.fsRelease+ if cmd == "get-fsinfo" && fsEntered != nil {+ f.fsOnce.Do(func() { close(fsEntered) })+ <-fsRelease}if cmd == uncertain || cmd == "file-read" && osName == bootstrapWindowsOS {w.WriteHeader(http.StatusInternalServerError)@@ -209,6 +210,20 @@ func checkWindowsBootstrapReading(t *testing.T, vm models.VM, poll int) {if vm.OSName != bootstrapWindowsOS || vm.Disk.Total != int64(2000*bootstrapMiB) || vm.Disk.Used != wantDisk || len(vm.Disks) != 2 || vm.GuestAgentStatus != "available" || vm.DiskStatusReason != "" || vm.Memory.Used != int64(2+poll)*int64(bootstrapMiB) || vm.Memory.Observation.Source != "status-mem" || vm.Memory.Observation.State != "current" || vm.CPU != float64(poll)/10 {t.Errorf("bootstrap poll %d: OS=%q disk=%d/%d rows=%d guest=%q reason=%q memory=%d/%s/%s CPU=%v", poll, vm.OSName, vm.Disk.Used, vm.Disk.Total, len(vm.Disks), vm.GuestAgentStatus, vm.DiskStatusReason, vm.Memory.Used, vm.Memory.Observation.Source, vm.Memory.Observation.State, vm.CPU)}+ wantVolumes := map[string][2]int64{`C:\`: {int64(1500 * bootstrapMiB), int64(400 * bootstrapMiB)}, `D:\`: {int64(500 * bootstrapMiB), int64(100 * bootstrapMiB)}}+ if poll == 2 {+ for mount, volume := range wantVolumes {+ volume[1] = 0+ wantVolumes[mount] = volume+ }+ }+ volumes := make(map[string][2]int64)+ for _, disk := range vm.Disks {+ volumes[disk.Mountpoint] = [2]int64{disk.Total, disk.Used}+ }+ if !reflect.DeepEqual(volumes, wantVolumes) {+ t.Errorf("bootstrap poll %d lost individual NTFS volumes: %v, want %v", poll, volumes, wantVolumes)+ }}func checkBootstrapHistory(t *testing.T, m *Monitor, id, metric string, want []float64) {@@ -377,3 +392,158 @@ func testGuestWindowsBootstrapProcessRestart(t *testing.T) {}}}+func testGuestWindowsBootstrapSafety(t *testing.T) {+ for _, kind := range []string{"client", "cluster-client"} {+ for _, collector := range []string{"cluster", "node"} {+ t.Run(kind+"/"+collector, func(t *testing.T) {+ t.Run("authoritative-lock", func(t *testing.T) {+ f, s := newWindowsBootstrapFixture(t)+ f.mu.Lock()+ f.locked = true+ f.mu.Unlock()+ m := windowsBootstrapMonitor(t)+ // Only metadata scheduling is shortened in this fixture. Lock+ // checks, client deadlines and command fencing are unchanged.+ m.guestMetadataRetryBackoff = time.Nanosecond+ client := windowsBootstrapClient(t, s.URL, kind)+ vm := windowsBootstrapPoll(t, m, client, collector, "bootstrap", "node", 105, 1)+ if vm.GuestAgentStatus != "deferred" || vm.DiskStatusReason != "vm-locked" || len(f.trace()) != 0 || len(m.guestMetadataCache) != 0 || len(m.vmAgentMemCache) != 0 {+ t.Error("cold Windows bootstrap bypassed the backup lock or invented evidence")+ }+ f.mu.Lock()+ f.locked = false+ f.mu.Unlock()+ checkWindowsBootstrapReading(t, windowsBootstrapPoll(t, m, client, collector, "bootstrap", "node", 105, 1), 1)+ if got := f.trace(); !reflect.DeepEqual(got, []string{"get-fsinfo", "network-get-interfaces", "get-osinfo", "info"}) {+ t.Errorf("unlocked ordinary bootstrap trace=%v", got)+ }+ checkBootstrapHistory(t, m, makeGuestID("bootstrap", "node", 105), "disk", []float64{25})+ })+ t.Run("in-flight-channel", func(t *testing.T) {+ f, s := newWindowsBootstrapFixture(t)+ f.mu.Lock()+ f.fsEntered, f.fsRelease = make(chan struct{}), make(chan struct{})+ f.mu.Unlock()+ m := windowsBootstrapMonitor(t)+ client := windowsBootstrapClient(t, s.URL, kind)+ done := make(chan models.VM, 1)+ go func() { done <- windowsBootstrapPoll(t, m, client, collector, "bootstrap", "node", 105, 1) }()+ select {+ case <-f.fsEntered:+ case <-time.After(windowsPollingHangGuard):+ close(f.fsRelease)+ t.Fatal("filesystem request never entered")+ }+ _, err := windowsBootstrapClient(t, s.URL, kind).GetVMAgentInfo(context.Background(), "node", 105)+ if proxmox.GuestAgentDeferredReason(err) != "agent-busy" || !reflect.DeepEqual(f.trace(), []string{"get-fsinfo"}) {+ t.Errorf("new diagnostic bypassed in-flight bootstrap: %v trace=%v", err, f.trace())+ }+ close(f.fsRelease)+ select {+ case vm := <-done:+ checkWindowsBootstrapReading(t, vm, 1)+ case <-time.After(windowsPollingHangGuard):+ t.Fatal("bootstrap did not finish after its original request completed")+ }+ if got := f.trace(); !reflect.DeepEqual(got, []string{"get-fsinfo", "network-get-interfaces", "get-osinfo", "info"}) {+ t.Errorf("in-flight owner lost the ordinary bootstrap: %v", got)+ }+ })+ for _, uncertain := range []string{"get-fsinfo", "network-get-interfaces", "get-osinfo", "info"} {+ t.Run("uncertain-"+uncertain, func(t *testing.T) {+ f, s := newWindowsBootstrapFixture(t)+ f.mu.Lock()+ f.uncertain = uncertain+ f.mu.Unlock()+ m := windowsBootstrapMonitor(t)+ m.guestAgentRetries = 2 // No uncertain command may be retried.+ client := windowsBootstrapClient(t, s.URL, kind)+ first := windowsBootstrapPoll(t, m, client, collector, "bootstrap", "node", 105, 1)+ trace := f.trace()+ if first.GuestAgentStatus != "deferred" || len(trace) == 0 || trace[len(trace)-1] != uncertain || len(m.vmAgentMemCache) != 0 {+ t.Errorf("unknown completion was exempted, replayed or lost: %v guest=%q", trace, first.GuestAgentStatus)+ }+ for _, cmd := range trace {+ if cmd == "file-read" {+ t.Fatal("optional Linux read followed an uncertain bootstrap command")+ }+ }+ _, err := windowsBootstrapClient(t, s.URL, kind).GetVMAgentInfo(context.Background(), "node", 105)+ if proxmox.GuestAgentDeferredReason(err) != "agent-cooldown" || !reflect.DeepEqual(f.trace(), trace) {+ t.Errorf("new client escaped bootstrap uncertainty: %v trace=%v", err, f.trace())+ }+ originalOS := m.guestMetadataCache[guestMetadataCacheKey("bootstrap", "node", 105)]+ id := makeGuestID("bootstrap", "node", 105)+ if uncertain == "get-fsinfo" {+ checkBootstrapHistory(t, m, id, "disk", nil)+ } else {+ checkBootstrapHistory(t, m, id, "disk", []float64{25})+ }+ // Distinct History keys expose false renewal. The live one-+ // minute uncertainty fence is not advanced or overwritten.+ time.Sleep(1100 * time.Millisecond)+ f.poll.Store(2)+ second := windowsBootstrapPoll(t, m, windowsBootstrapClient(t, s.URL, kind), collector, "bootstrap", "node", 105, 2)+ if second.GuestAgentStatus != "deferred" || !reflect.DeepEqual(f.trace(), trace) || !reflect.DeepEqual(m.guestMetadataCache[guestMetadataCacheKey("bootstrap", "node", 105)], originalOS) || len(m.vmAgentMemCache) != 0 {+ t.Error("paused poll dispatched or renewed original OS/memory evidence")+ }+ if uncertain != "get-fsinfo" {+ if second.Disk.Used != first.Disk.Used || second.DiskObservation != first.DiskObservation || second.DiskStatusReason != "prev-agent-cooldown" {+ t.Error("paused poll lost or renewed the accepted filesystem receipt")+ }+ checkBootstrapHistory(t, m, id, "disk", []float64{25})+ } else {+ checkBootstrapHistory(t, m, id, "disk", nil)+ }+ // A new monitor is still in the same process: cold caches do+ // not permit it to bypass the original client's admission.+ cold := windowsBootstrapMonitor(t)+ blocked := windowsBootstrapPoll(t, cold, windowsBootstrapClient(t, s.URL, kind), collector, "bootstrap", "node", 105, 2)+ if blocked.GuestAgentStatus != "deferred" || !reflect.DeepEqual(f.trace(), trace) || len(cold.guestMetadataCache) != 0 || len(cold.vmAgentMemCache) != 0 {+ t.Error("cold monitor bypassed live process uncertainty")+ }+ checkBootstrapHistory(t, m, id, "cpu", []float64{10, 20})+ checkBootstrapHistory(t, m, id, "memory", []float64{37.5, 50})+ })+ }+ t.Run("linux-unknown-and-replacement", func(t *testing.T) {+ f, s := newWindowsBootstrapFixture(t)+ m := windowsBootstrapMonitor(t)+ client := windowsBootstrapClient(t, s.URL, kind)+ checkWindowsBootstrapReading(t, windowsBootstrapPoll(t, m, client, collector, "bootstrap", "node", 105, 1), 1)+ original := m.guestMetadataCache[guestMetadataCacheKey("bootstrap", "node", 105)]+ for _, identity := range []struct {+ instance, node, os string+ id int+ }{{"other", "node", "Linux", 105}, {"bootstrap", "other", "", 105}, {"bootstrap", "node", "Linux", 106}} {+ f.mu.Lock()+ f.osName = identity.os+ f.mu.Unlock()+ f.resetTrace()+ vm := windowsBootstrapPoll(t, m, client, collector, identity.instance, identity.node, identity.id, 1)+ if !reflect.DeepEqual(f.trace(), []string{"get-fsinfo", "network-get-interfaces", "get-osinfo", "info", "file-read"}) || vm.Memory.Used != int64(2*bootstrapMiB) || vm.Memory.Observation.Source != "guest-agent-meminfo" || vm.GuestAgentStatus != "available" {+ t.Errorf("Windows evidence/config hint suppressed another identity's Linux fallback: trace=%v memory=%+v", f.trace(), vm.Memory)+ }+ }+ if !reflect.DeepEqual(m.guestMetadataCache[guestMetadataCacheKey("bootstrap", "node", 105)], original) {+ t.Error("replacement identity renewed the original Windows evidence")+ }+ // A reinstalled guest at the same tuple must be reclassified+ // from its new OS receipt, including measured-zero availability.+ key := guestMetadataCacheKey("bootstrap", "node", 105)+ aged := time.Now().Add(-2 * vmAgentMemCleanupMaxAge)+ original.fetchedAt, original.osInfoObservedAt = aged, aged+ m.guestMetadataCache[key], m.guestMetadataLimiter[key] = original, aged+ f.mu.Lock()+ f.osName, f.linuxAvailable = "Linux", 0+ f.mu.Unlock()+ f.resetTrace()+ vm := windowsBootstrapPoll(t, m, client, collector, "bootstrap", "node", 105, 1)+ if vm.Memory.Used != int64(8*bootstrapMiB) || vm.Memory.Observation.Source != "guest-agent-meminfo" || m.guestMetadataCache[key].windowsGuest || vm.OSName != "Linux" || !reflect.DeepEqual(f.trace(), []string{"get-fsinfo", "network-get-interfaces", "get-osinfo", "info", "file-read"}) {+ t.Errorf("new actual Linux OS failed to invalidate expired Windows evidence: trace=%v OS=%q memory=%+v", f.trace(), vm.OSName, vm.Memory)+ }+ })+ })+ }+ }+}diff --git a/internal/monitoring/monitor_pve_guest_builders.go b/internal/monitoring/monitor_pve_guest_builders.goindex 39125532ac..4860bc91b2 100644--- a/internal/monitoring/monitor_pve_guest_builders.go+++ b/internal/monitoring/monitor_pve_guest_builders.go@@ -98,6 +98,31 @@ func (m *Monitor) applyVMStatusDetails(Msg("VM guest agent is not currently queryable")}+ // Learn or refresh the OS before the optional Linux-only memory fallback.+ // Otherwise an uncertain /proc/meminfo reply on a cold/expired Windows+ // guest can fence OS discovery on every poll, so the Windows exclusion+ // never becomes usable. Filesystems remain first; metadata and memory+ // retain the same per-command lock, completion and shared-channel guard.+ guestIPs, guestIfaces, guestOSName, guestOSVersion, guestAgentVersion, metadataDeferred := m.fetchGuestAgentMetadata(ctx, client, instanceName, res.Node, res.Name, res.VMID, status, false)+ if metadataDeferred {+ state.guestAgentStatus = "deferred"+ }+ if len(guestIPs) > 0 {+ state.ipAddresses = guestIPs+ }+ if len(guestIfaces) > 0 {+ state.networkInterfaces = guestIfaces+ }+ if guestOSName != "" {+ state.osName = guestOSName+ }+ if guestOSVersion != "" {+ state.osVersion = guestOSVersion+ }+ if guestAgentVersion != "" {+ state.agentVersion = guestAgentVersion+ }+var memoryDeferred boolstate.memTotal, state.memUsed, state.memorySource, memoryDeferred = m.resolveGuestStatusMemory(ctx,@@ -119,27 +144,6 @@ func (m *Monitor) applyVMStatusDetails(}mergeVMRuntimeCounters(state, status)-- // Gather guest metadata from the agent when available- guestIPs, guestIfaces, guestOSName, guestOSVersion, guestAgentVersion, metadataDeferred := m.fetchGuestAgentMetadata(ctx, client, instanceName, res.Node, res.Name, res.VMID, status, false)- if metadataDeferred {- state.guestAgentStatus = "deferred"- }- if len(guestIPs) > 0 {- state.ipAddresses = guestIPs- }- if len(guestIfaces) > 0 {- state.networkInterfaces = guestIfaces- }- if guestOSName != "" {- state.osName = guestOSName- }- if guestOSVersion != "" {- state.osVersion = guestOSVersion- }- if guestAgentVersion != "" {- state.agentVersion = guestAgentVersion- }}func (m *Monitor) applyVMGuestAgentFSInfo(ctx context.Context, instanceName string, res proxmox.ClusterResource, client PVEClientInterface, state *vmBuildState) {