Skip to content

armgit / Pulse

Public Imported from GitHub
0

Real-time monitoring dashboard for Proxmox VE, PBS, Docker, Kubernetes, TrueNAS and vSphere. Self-hosted, with alerts for failed backups, full disks, unhealthy containers and offline nodes.

Learn guest OS before optional Linux memory polling

pulse-triage[bot]081c12f65c6b
diff --git a/docs/release-control/v6/internal/subsystems/monitoring.md b/docs/release-control/v6/internal/subsystems/monitoring.md
index 39202ba490..c1ce355671 100644
--- a/docs/release-control/v6/internal/subsystems/monitoring.md
+++ b/docs/release-control/v6/internal/subsystems/monitoring.md
@@ -174,7 +174,14 @@ It does not invent a negative memory observation or clear an existing pause.
Independent status/balloon/linked-agent memory, filesystem-first ordering,
unknown-OS compatibility and all command/backup-lock admission remain intact.
The first unknown-OS poll can still attempt the existing Linux fallback; this
-change does not add a probe or infer an OS from a display name/configuration hint.
+requires completing ordinary metadata collection first. Both detailed-status
+collectors now collect filesystems, then metadata, then optional Linux memory.
+On an ordinary cold start or expired metadata refresh, the real OS reply can
+therefore establish the existing Windows exclusion before an inappropriate
+Linux read creates a shared pause. No extra probe, persistent OS hint or
+configuration/display-name inference is added. If metadata itself is uncertain,
+the same channel guard still prevents every later command; independent current
+status/balloon/linked-agent memory is not turned into guest meminfo evidence.
`TestGuestWindowsMeminfoPollingContract` covers the real direct/cluster clients
and both collectors after an actual OS read: repeated NTFS usage through zero,
@@ -185,6 +192,19 @@ evidence, Linux recovery and cache/source compatibility. These are synthetic
source controls, not native QEMU-only Windows recovery, thaw, cross-process or
restart containment, reporter resolution or release acceptance.
+`TestGuestWindowsRestartPollingContract` starts empty monitors through both real
+clients and both collectors, ages actual OS receipts for the expiry control,
+and runs two distinct fresh test processes against the same loopback endpoint.
+It checks both NTFS volumes, healthy zero recovery, OS/canonical/public state,
+memory source and in-memory/persistent History. The Linux command's fixture
+response remains an unexplained HTTP 500: recovery comes from not sending it,
+not from a completion exemption. Lock, in-flight/shared-client uncertainty,
+original ages, Linux/unknown and replacement-identity controls remain covered.
+The ordinary supported loopback process restart is source acceptance, not
+native Pro/PVE restart recovery, every-filesystem writes/liveness, independent
+thaw, prior-active restoration or a released fix. Existing OS-info safety
+suppression and metadata backoff are not reset to force recovery.
+
### QGA fallback eligibility keeps its original observation age
A successful unlocked VM-status availability, usable filesystem reading or
diff --git a/internal/monitoring/canonical_guardrails_test.go b/internal/monitoring/canonical_guardrails_test.go
index 29dae0cd86..93474edaf1 100644
--- a/internal/monitoring/canonical_guardrails_test.go
+++ b/internal/monitoring/canonical_guardrails_test.go
@@ -3254,6 +3254,7 @@ func TestGuestWindowsMeminfoPollingContract(t *testing.T) {
func TestGuestWindowsRestartPollingContract(t *testing.T) {
t.Run("cold-and-expired", testGuestWindowsBootstrapPolling)
t.Run("process-restart", testGuestWindowsBootstrapProcessRestart)
+ t.Run("admission-and-identity", testGuestWindowsBootstrapSafety)
}
func TestGuestMemoryObservationContract(t *testing.T) {
diff --git a/internal/monitoring/guest_agent_optional_read_integration_test.go b/internal/monitoring/guest_agent_optional_read_integration_test.go
index 6d8145a3f5..5804ef02c6 100644
--- a/internal/monitoring/guest_agent_optional_read_integration_test.go
+++ b/internal/monitoring/guest_agent_optional_read_integration_test.go
@@ -29,8 +29,10 @@ func testGuestAgentOptionalReadOrdering(t *testing.T, withoutStatus bool) {
terminal bool
}{
{"windows-unsupported-memory", "file-read", "Microsoft Windows 11", "ntfs", true},
+ {"linux-unsupported-memory", "file-read", "Linux", "ext4", true},
{"android-unsupported-os", "get-osinfo", "Android", "ext4", true},
- {"uncertain-memory", "file-read", "Microsoft Windows 11", "ntfs", false},
+ {"uncertain-memory", "file-read", "Linux", "ext4", false},
+ {"unknown-os-uncertain-memory", "file-read", "", "ext4", false},
{"uncertain-network", "network-get-interfaces", "Android", "ext4", false},
{"uncertain-os", "get-osinfo", "Android", "ext4", false},
{"uncertain-version", "info", "Microsoft Windows 11", "ntfs", false},
diff --git a/internal/monitoring/guest_windows_bootstrap_test.go b/internal/monitoring/guest_windows_bootstrap_test.go
index 4c4eb83795..61c46f3898 100644
--- a/internal/monitoring/guest_windows_bootstrap_test.go
+++ b/internal/monitoring/guest_windows_bootstrap_test.go
@@ -72,6 +72,7 @@ func (f *windowsBootstrapFixture) serve(w http.ResponseWriter, r *http.Request)
w.Header().Set("Content-Type", "application/json")
f.mu.Lock()
osName, uncertain, locked, available := f.osName, f.uncertain, f.locked, f.linuxAvailable
+ fsEntered, fsRelease := f.fsEntered, f.fsRelease
f.mu.Unlock()
switch {
case strings.HasSuffix(r.URL.Path, "/nodes"):
@@ -94,9 +95,9 @@ func (f *windowsBootstrapFixture) serve(w http.ResponseWriter, r *http.Request)
f.mu.Lock()
f.commands = append(f.commands, cmd)
f.mu.Unlock()
- if cmd == "get-fsinfo" && f.fsEntered != nil {
- f.fsOnce.Do(func() { close(f.fsEntered) })
- <-f.fsRelease
+ if cmd == "get-fsinfo" && fsEntered != nil {
+ f.fsOnce.Do(func() { close(fsEntered) })
+ <-fsRelease
}
if cmd == uncertain || cmd == "file-read" && osName == bootstrapWindowsOS {
w.WriteHeader(http.StatusInternalServerError)
@@ -209,6 +210,20 @@ func checkWindowsBootstrapReading(t *testing.T, vm models.VM, poll int) {
if vm.OSName != bootstrapWindowsOS || vm.Disk.Total != int64(2000*bootstrapMiB) || vm.Disk.Used != wantDisk || len(vm.Disks) != 2 || vm.GuestAgentStatus != "available" || vm.DiskStatusReason != "" || vm.Memory.Used != int64(2+poll)*int64(bootstrapMiB) || vm.Memory.Observation.Source != "status-mem" || vm.Memory.Observation.State != "current" || vm.CPU != float64(poll)/10 {
t.Errorf("bootstrap poll %d: OS=%q disk=%d/%d rows=%d guest=%q reason=%q memory=%d/%s/%s CPU=%v", poll, vm.OSName, vm.Disk.Used, vm.Disk.Total, len(vm.Disks), vm.GuestAgentStatus, vm.DiskStatusReason, vm.Memory.Used, vm.Memory.Observation.Source, vm.Memory.Observation.State, vm.CPU)
}
+ wantVolumes := map[string][2]int64{`C:\`: {int64(1500 * bootstrapMiB), int64(400 * bootstrapMiB)}, `D:\`: {int64(500 * bootstrapMiB), int64(100 * bootstrapMiB)}}
+ if poll == 2 {
+ for mount, volume := range wantVolumes {
+ volume[1] = 0
+ wantVolumes[mount] = volume
+ }
+ }
+ volumes := make(map[string][2]int64)
+ for _, disk := range vm.Disks {
+ volumes[disk.Mountpoint] = [2]int64{disk.Total, disk.Used}
+ }
+ if !reflect.DeepEqual(volumes, wantVolumes) {
+ t.Errorf("bootstrap poll %d lost individual NTFS volumes: %v, want %v", poll, volumes, wantVolumes)
+ }
}
func checkBootstrapHistory(t *testing.T, m *Monitor, id, metric string, want []float64) {
@@ -377,3 +392,158 @@ func testGuestWindowsBootstrapProcessRestart(t *testing.T) {
}
}
}
+func testGuestWindowsBootstrapSafety(t *testing.T) {
+ for _, kind := range []string{"client", "cluster-client"} {
+ for _, collector := range []string{"cluster", "node"} {
+ t.Run(kind+"/"+collector, func(t *testing.T) {
+ t.Run("authoritative-lock", func(t *testing.T) {
+ f, s := newWindowsBootstrapFixture(t)
+ f.mu.Lock()
+ f.locked = true
+ f.mu.Unlock()
+ m := windowsBootstrapMonitor(t)
+ // Only metadata scheduling is shortened in this fixture. Lock
+ // checks, client deadlines and command fencing are unchanged.
+ m.guestMetadataRetryBackoff = time.Nanosecond
+ client := windowsBootstrapClient(t, s.URL, kind)
+ vm := windowsBootstrapPoll(t, m, client, collector, "bootstrap", "node", 105, 1)
+ if vm.GuestAgentStatus != "deferred" || vm.DiskStatusReason != "vm-locked" || len(f.trace()) != 0 || len(m.guestMetadataCache) != 0 || len(m.vmAgentMemCache) != 0 {
+ t.Error("cold Windows bootstrap bypassed the backup lock or invented evidence")
+ }
+ f.mu.Lock()
+ f.locked = false
+ f.mu.Unlock()
+ checkWindowsBootstrapReading(t, windowsBootstrapPoll(t, m, client, collector, "bootstrap", "node", 105, 1), 1)
+ if got := f.trace(); !reflect.DeepEqual(got, []string{"get-fsinfo", "network-get-interfaces", "get-osinfo", "info"}) {
+ t.Errorf("unlocked ordinary bootstrap trace=%v", got)
+ }
+ checkBootstrapHistory(t, m, makeGuestID("bootstrap", "node", 105), "disk", []float64{25})
+ })
+ t.Run("in-flight-channel", func(t *testing.T) {
+ f, s := newWindowsBootstrapFixture(t)
+ f.mu.Lock()
+ f.fsEntered, f.fsRelease = make(chan struct{}), make(chan struct{})
+ f.mu.Unlock()
+ m := windowsBootstrapMonitor(t)
+ client := windowsBootstrapClient(t, s.URL, kind)
+ done := make(chan models.VM, 1)
+ go func() { done <- windowsBootstrapPoll(t, m, client, collector, "bootstrap", "node", 105, 1) }()
+ select {
+ case <-f.fsEntered:
+ case <-time.After(windowsPollingHangGuard):
+ close(f.fsRelease)
+ t.Fatal("filesystem request never entered")
+ }
+ _, err := windowsBootstrapClient(t, s.URL, kind).GetVMAgentInfo(context.Background(), "node", 105)
+ if proxmox.GuestAgentDeferredReason(err) != "agent-busy" || !reflect.DeepEqual(f.trace(), []string{"get-fsinfo"}) {
+ t.Errorf("new diagnostic bypassed in-flight bootstrap: %v trace=%v", err, f.trace())
+ }
+ close(f.fsRelease)
+ select {
+ case vm := <-done:
+ checkWindowsBootstrapReading(t, vm, 1)
+ case <-time.After(windowsPollingHangGuard):
+ t.Fatal("bootstrap did not finish after its original request completed")
+ }
+ if got := f.trace(); !reflect.DeepEqual(got, []string{"get-fsinfo", "network-get-interfaces", "get-osinfo", "info"}) {
+ t.Errorf("in-flight owner lost the ordinary bootstrap: %v", got)
+ }
+ })
+ for _, uncertain := range []string{"get-fsinfo", "network-get-interfaces", "get-osinfo", "info"} {
+ t.Run("uncertain-"+uncertain, func(t *testing.T) {
+ f, s := newWindowsBootstrapFixture(t)
+ f.mu.Lock()
+ f.uncertain = uncertain
+ f.mu.Unlock()
+ m := windowsBootstrapMonitor(t)
+ m.guestAgentRetries = 2 // No uncertain command may be retried.
+ client := windowsBootstrapClient(t, s.URL, kind)
+ first := windowsBootstrapPoll(t, m, client, collector, "bootstrap", "node", 105, 1)
+ trace := f.trace()
+ if first.GuestAgentStatus != "deferred" || len(trace) == 0 || trace[len(trace)-1] != uncertain || len(m.vmAgentMemCache) != 0 {
+ t.Errorf("unknown completion was exempted, replayed or lost: %v guest=%q", trace, first.GuestAgentStatus)
+ }
+ for _, cmd := range trace {
+ if cmd == "file-read" {
+ t.Fatal("optional Linux read followed an uncertain bootstrap command")
+ }
+ }
+ _, err := windowsBootstrapClient(t, s.URL, kind).GetVMAgentInfo(context.Background(), "node", 105)
+ if proxmox.GuestAgentDeferredReason(err) != "agent-cooldown" || !reflect.DeepEqual(f.trace(), trace) {
+ t.Errorf("new client escaped bootstrap uncertainty: %v trace=%v", err, f.trace())
+ }
+ originalOS := m.guestMetadataCache[guestMetadataCacheKey("bootstrap", "node", 105)]
+ id := makeGuestID("bootstrap", "node", 105)
+ if uncertain == "get-fsinfo" {
+ checkBootstrapHistory(t, m, id, "disk", nil)
+ } else {
+ checkBootstrapHistory(t, m, id, "disk", []float64{25})
+ }
+ // Distinct History keys expose false renewal. The live one-
+ // minute uncertainty fence is not advanced or overwritten.
+ time.Sleep(1100 * time.Millisecond)
+ f.poll.Store(2)
+ second := windowsBootstrapPoll(t, m, windowsBootstrapClient(t, s.URL, kind), collector, "bootstrap", "node", 105, 2)
+ if second.GuestAgentStatus != "deferred" || !reflect.DeepEqual(f.trace(), trace) || !reflect.DeepEqual(m.guestMetadataCache[guestMetadataCacheKey("bootstrap", "node", 105)], originalOS) || len(m.vmAgentMemCache) != 0 {
+ t.Error("paused poll dispatched or renewed original OS/memory evidence")
+ }
+ if uncertain != "get-fsinfo" {
+ if second.Disk.Used != first.Disk.Used || second.DiskObservation != first.DiskObservation || second.DiskStatusReason != "prev-agent-cooldown" {
+ t.Error("paused poll lost or renewed the accepted filesystem receipt")
+ }
+ checkBootstrapHistory(t, m, id, "disk", []float64{25})
+ } else {
+ checkBootstrapHistory(t, m, id, "disk", nil)
+ }
+ // A new monitor is still in the same process: cold caches do
+ // not permit it to bypass the original client's admission.
+ cold := windowsBootstrapMonitor(t)
+ blocked := windowsBootstrapPoll(t, cold, windowsBootstrapClient(t, s.URL, kind), collector, "bootstrap", "node", 105, 2)
+ if blocked.GuestAgentStatus != "deferred" || !reflect.DeepEqual(f.trace(), trace) || len(cold.guestMetadataCache) != 0 || len(cold.vmAgentMemCache) != 0 {
+ t.Error("cold monitor bypassed live process uncertainty")
+ }
+ checkBootstrapHistory(t, m, id, "cpu", []float64{10, 20})
+ checkBootstrapHistory(t, m, id, "memory", []float64{37.5, 50})
+ })
+ }
+ t.Run("linux-unknown-and-replacement", func(t *testing.T) {
+ f, s := newWindowsBootstrapFixture(t)
+ m := windowsBootstrapMonitor(t)
+ client := windowsBootstrapClient(t, s.URL, kind)
+ checkWindowsBootstrapReading(t, windowsBootstrapPoll(t, m, client, collector, "bootstrap", "node", 105, 1), 1)
+ original := m.guestMetadataCache[guestMetadataCacheKey("bootstrap", "node", 105)]
+ for _, identity := range []struct {
+ instance, node, os string
+ id int
+ }{{"other", "node", "Linux", 105}, {"bootstrap", "other", "", 105}, {"bootstrap", "node", "Linux", 106}} {
+ f.mu.Lock()
+ f.osName = identity.os
+ f.mu.Unlock()
+ f.resetTrace()
+ vm := windowsBootstrapPoll(t, m, client, collector, identity.instance, identity.node, identity.id, 1)
+ if !reflect.DeepEqual(f.trace(), []string{"get-fsinfo", "network-get-interfaces", "get-osinfo", "info", "file-read"}) || vm.Memory.Used != int64(2*bootstrapMiB) || vm.Memory.Observation.Source != "guest-agent-meminfo" || vm.GuestAgentStatus != "available" {
+ t.Errorf("Windows evidence/config hint suppressed another identity's Linux fallback: trace=%v memory=%+v", f.trace(), vm.Memory)
+ }
+ }
+ if !reflect.DeepEqual(m.guestMetadataCache[guestMetadataCacheKey("bootstrap", "node", 105)], original) {
+ t.Error("replacement identity renewed the original Windows evidence")
+ }
+ // A reinstalled guest at the same tuple must be reclassified
+ // from its new OS receipt, including measured-zero availability.
+ key := guestMetadataCacheKey("bootstrap", "node", 105)
+ aged := time.Now().Add(-2 * vmAgentMemCleanupMaxAge)
+ original.fetchedAt, original.osInfoObservedAt = aged, aged
+ m.guestMetadataCache[key], m.guestMetadataLimiter[key] = original, aged
+ f.mu.Lock()
+ f.osName, f.linuxAvailable = "Linux", 0
+ f.mu.Unlock()
+ f.resetTrace()
+ vm := windowsBootstrapPoll(t, m, client, collector, "bootstrap", "node", 105, 1)
+ if vm.Memory.Used != int64(8*bootstrapMiB) || vm.Memory.Observation.Source != "guest-agent-meminfo" || m.guestMetadataCache[key].windowsGuest || vm.OSName != "Linux" || !reflect.DeepEqual(f.trace(), []string{"get-fsinfo", "network-get-interfaces", "get-osinfo", "info", "file-read"}) {
+ t.Errorf("new actual Linux OS failed to invalidate expired Windows evidence: trace=%v OS=%q memory=%+v", f.trace(), vm.OSName, vm.Memory)
+ }
+ })
+ })
+ }
+ }
+}
diff --git a/internal/monitoring/monitor_pve_guest_builders.go b/internal/monitoring/monitor_pve_guest_builders.go
index 39125532ac..4860bc91b2 100644
--- a/internal/monitoring/monitor_pve_guest_builders.go
+++ b/internal/monitoring/monitor_pve_guest_builders.go
@@ -98,6 +98,31 @@ func (m *Monitor) applyVMStatusDetails(
Msg("VM guest agent is not currently queryable")
}
+ // Learn or refresh the OS before the optional Linux-only memory fallback.
+ // Otherwise an uncertain /proc/meminfo reply on a cold/expired Windows
+ // guest can fence OS discovery on every poll, so the Windows exclusion
+ // never becomes usable. Filesystems remain first; metadata and memory
+ // retain the same per-command lock, completion and shared-channel guard.
+ guestIPs, guestIfaces, guestOSName, guestOSVersion, guestAgentVersion, metadataDeferred := m.fetchGuestAgentMetadata(ctx, client, instanceName, res.Node, res.Name, res.VMID, status, false)
+ if metadataDeferred {
+ state.guestAgentStatus = "deferred"
+ }
+ if len(guestIPs) > 0 {
+ state.ipAddresses = guestIPs
+ }
+ if len(guestIfaces) > 0 {
+ state.networkInterfaces = guestIfaces
+ }
+ if guestOSName != "" {
+ state.osName = guestOSName
+ }
+ if guestOSVersion != "" {
+ state.osVersion = guestOSVersion
+ }
+ if guestAgentVersion != "" {
+ state.agentVersion = guestAgentVersion
+ }
+
var memoryDeferred bool
state.memTotal, state.memUsed, state.memorySource, memoryDeferred = m.resolveGuestStatusMemory(
ctx,
@@ -119,27 +144,6 @@ func (m *Monitor) applyVMStatusDetails(
}
mergeVMRuntimeCounters(state, status)
-
- // Gather guest metadata from the agent when available
- guestIPs, guestIfaces, guestOSName, guestOSVersion, guestAgentVersion, metadataDeferred := m.fetchGuestAgentMetadata(ctx, client, instanceName, res.Node, res.Name, res.VMID, status, false)
- if metadataDeferred {
- state.guestAgentStatus = "deferred"
- }
- if len(guestIPs) > 0 {
- state.ipAddresses = guestIPs
- }
- if len(guestIfaces) > 0 {
- state.networkInterfaces = guestIfaces
- }
- if guestOSName != "" {
- state.osName = guestOSName
- }
- if guestOSVersion != "" {
- state.osVersion = guestOSVersion
- }
- if guestAgentVersion != "" {
- state.agentVersion = guestAgentVersion
- }
}
func (m *Monitor) applyVMGuestAgentFSInfo(ctx context.Context, instanceName string, res proxmox.ClusterResource, client PVEClientInterface, state *vmBuildState) {