Real-time monitoring dashboard for Proxmox VE, PBS, Docker, Kubernetes, TrueNAS and vSphere. Self-hosted, with alerts for failed backups, full disks, unhealthy containers and offline nodes.
Go aialertsdashboarddockerdocker-monitoringgohomelabinfrastructure-monitoringkubernetesmonitoringmspproxmoxproxmox-backup-serverproxmox-mail-gatewayproxmox-veself-hostedtruenastypescriptvspherewebhooks
Keep guest OS discovery eligible after temporary rejections
pulse-triage[bot]3caea610ed67
diff --git a/docs/release-control/v6/internal/subsystems/monitoring.md b/docs/release-control/v6/internal/subsystems/monitoring.mdindex 2e41e917aa..9b645d110e 100644--- a/docs/release-control/v6/internal/subsystems/monitoring.md+++ b/docs/release-control/v6/internal/subsystems/monitoring.md@@ -666,10 +666,18 @@ required; all responses in this fixture are synthetic.### Guest-agent coordination around backups — issue #2439-A completed OS-info outcome keeps its existing suppression semantics even when+A completed OS-info outcome keeps its suppression semantics even whenthe later agent-version read is deferred. Unsupported OS-info still disables-that optional command; ordinary completed failures still accumulate towards-the existing three-failure limit, and a supported reply resets that count.+that optional command; ordinary completed command failures still accumulate+towards the existing three-failure limit, and a supported reply resets that count.+Actual HTTP 401/403 or exact command-bound stopped-agent rejections did not+exercise OS-info and therefore do not accumulate towards permanent suppression.+They preserve prior command failures and normal refresh/backoff timing; they+neither renew retained identity nor establish a Windows observation. Once ordinary+access/agent availability returns, an eligible accepted OS reply can recover+the identity and cross-platform memory classification without a Pulse restart.+Quoted status/diagnosis text does not grant this distinction. Unsupported,+repeated completed-failure and uncertain-command safety suppression is retained.Only these safety fields are retained on version deferral: partial network/OSmetadata is not published, old identity is not made fresh, and the shared pauseand configured retry backoff remain in force. A network or OS-info deferral@@ -683,6 +691,15 @@ suppressed OS-info request and with fresh disk History. It changes no nativeguest, backup policy, lock admission, cooldown, retry or single-attempt rule.These source controls do not establish native recovery or a reported cause.+`TestGuestMetadataRejectionRecoveryContract` verifies typed rejections with cold+and retained caches, prior genuine failures, unchanged backoff and fresh OS+receipt requirements. Both ordinary collectors and single/cluster HTTP clients+recover Windows OS, individual NTFS volumes and disk History after a stopped-agent+fixture, without dispatching Linux-only memory reads on the recovered Windows+guest. Existing suppression, backup-lock and shared uncertainty contracts remain+required. These synthetic source controls do not establish native restart/thaw,+the cause of #2822/#2619, or installed/released recovery.+The process-local admission table remains capped at 4,096 endpoint/VMIDentries. Expired entries replaced by a new admission consume no extra slot;when new aliases would exceed the bound, expired idle entries are reclaimeddiff --git a/internal/monitoring/canonical_guardrails_test.go b/internal/monitoring/canonical_guardrails_test.goindex 93474edaf1..ff1b4645fe 100644--- a/internal/monitoring/canonical_guardrails_test.go+++ b/internal/monitoring/canonical_guardrails_test.go@@ -3257,6 +3257,11 @@ func TestGuestWindowsRestartPollingContract(t *testing.T) {t.Run("admission-and-identity", testGuestWindowsBootstrapSafety)}+func TestGuestMetadataRejectionRecoveryContract(t *testing.T) {+ t.Run("typed-rejection", testGuestMetadataRejectedOSInfo)+ t.Run("ordinary-stopped-agent-recovery", testGuestMetadataStoppedAgentPollingRecovery)+}+func TestGuestMemoryObservationContract(t *testing.T) {t.Run("poll-to-served-observation", testGuestMemoryObservationLifecycle)t.Run("identity-and-origin-boundaries", testGuestMemoryObservationKeepsOriginsSeparate)diff --git a/internal/monitoring/guest_metadata.go b/internal/monitoring/guest_metadata.goindex 70b4ae2847..00c05ee685 100644--- a/internal/monitoring/guest_metadata.go+++ b/internal/monitoring/guest_metadata.go@@ -366,7 +366,18 @@ func (m *Monitor) fetchGuestAgentMetadata(ctx context.Context, client PVEClientIreturn cloneStringSlice(cached.ipAddresses), cloneGuestNetworkInterfaces(cached.networkInterfaces), cached.osName, cached.osVersion, cached.agentVersion, true}if err != nil {- if isGuestAgentOSInfoUnsupportedError(err) {+ if reason := proxmox.GuestAgentErrorReason(err); reason == "permission-denied" || reason == "agent-not-running" {+ // The request was not admitted or QGA was stopped, so this did+ // not exercise the OS-info command. Do not permanently disable+ // that command when ordinary access/agent availability returns.+ // Preserve any real prior failures and the normal refresh backoff.+ log.Debug().+ Str("instance", instanceName).+ Str("vm", vmName).+ Int("vmid", vmid).+ Err(err).+ Msg("Guest agent OS info request not admitted; preserving command failure state")+ } else if isGuestAgentOSInfoUnsupportedError(err) {osInfoSkip = trueosInfoFailureCount = guestAgentOSInfoFailureThresholdlog.Warn().diff --git a/internal/monitoring/guest_metadata_rejection_test.go b/internal/monitoring/guest_metadata_rejection_test.gonew file mode 100644index 0000000000..9e3bd95173--- /dev/null+++ b/internal/monitoring/guest_metadata_rejection_test.go@@ -0,0 +1,161 @@+package monitoring++import (+ "context"+ "fmt"+ "net/http"+ "net/http/httptest"+ "reflect"+ "strings"+ "sync/atomic"+ "testing"+ "time"++ "github.com/rcourtman/pulse-go-rewrite/pkg/proxmox"+)++// A complete access/agent-unavailable rejection did not exercise guest-get-osinfo.+// It must not be counted as evidence that this particular command is unsafe.+func testGuestMetadataRejectedOSInfo(t *testing.T) {+ for _, rejection := range []string{"unauthorized", "forbidden", "agent-not-running"} {+ for _, seeded := range []bool{false, true} {+ t.Run(fmt.Sprintf("%s/cached-%t", rejection, seeded), func(t *testing.T) {+ var recovered atomic.Bool+ var osCalls atomic.Int32+ server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {+ if r.Header.Get("Authorization") != "PVEAPIToken=fixture@pve!pulse=fixture" {+ t.Error("fixture authentication changed")+ w.WriteHeader(http.StatusForbidden)+ return+ }+ switch {+ case strings.HasSuffix(r.URL.Path, "/config"):+ fmt.Fprint(w, `{"data":{}}`)+ case strings.HasSuffix(r.URL.Path, "/get-osinfo"):+ osCalls.Add(1)+ if recovered.Load() {+ fmt.Fprintf(w, `{"data":{"name":%q,"version":"2025"}}`, bootstrapWindowsOS)+ return+ }+ switch rejection {+ case "unauthorized":+ w.WriteHeader(http.StatusUnauthorized)+ // Even OS-like body text is not a completed OS-info reply.+ fmt.Fprint(w, "unsupported command: guest-get-osinfo")+ case "forbidden":+ w.WriteHeader(http.StatusForbidden)+ fmt.Fprint(w, "Failed to open file '/etc/os-release': No such file or directory")+ default:+ w.WriteHeader(http.StatusInternalServerError)+ fmt.Fprint(w, `{"data":null,"message":"VM 105 qmp command 'guest-get-osinfo' failed - QEMU guest agent is not running"}`)+ }+ case strings.HasSuffix(r.URL.Path, "/network-get-interfaces"):+ fmt.Fprint(w, `{"data":{"result":[]}}`)+ case strings.HasSuffix(r.URL.Path, "/info"):+ fmt.Fprint(w, `{"data":{"result":{}}}`)+ default:+ t.Errorf("unexpected request: %s", r.URL.Path)+ http.NotFound(w, r)+ }+ }))+ defer server.Close()+ client, err := proxmox.NewClient(proxmox.ClientConfig{Host: server.URL, TokenName: "fixture@pve!pulse", TokenValue: "fixture", Timeout: windowsPollingHangGuard})+ if err != nil {+ t.Fatal(err)+ }+ key := guestMetadataCacheKey("rejection", "node", 105)+ original := guestMetadataCacheEntry{}+ if seeded {+ original = metadataObservationFixture(time.Now().Add(-2 * guestMetadataCacheTTL))+ original.osInfoFailureCount = guestAgentOSInfoFailureThreshold - 1+ }+ m := &Monitor{guestMetadataCache: map[string]guestMetadataCacheEntry{key: original}, guestMetadataLimiter: make(map[string]time.Time)}+ status := &proxmox.VMStatus{Agent: proxmox.VMAgentField{Value: 1}}+ for poll := 1; poll <= guestAgentOSInfoFailureThreshold; poll++ {+ // Represent separate eligible refreshes without waiting for the+ // production backoff. Never clear a QGA command fence or lock.+ m.guestMetadataLimiter[key] = time.Now().Add(-time.Second)+ start := time.Now()+ ips, ifaces, name, version, agent, deferred := m.fetchGuestAgentMetadata(context.Background(), client, "rejection", "node", "guest", 105, status, false)+ if deferred || !reflect.DeepEqual(m.guestMetadataCache[key], original) || !reflect.DeepEqual(ips, original.ipAddresses) || !reflect.DeepEqual(ifaces, original.networkInterfaces) || name != original.osName || version != original.osVersion || agent != original.agentVersion {+ t.Errorf("rejected poll %d poisoned OS state or renewed identity: %+v, deferred=%t", poll, m.guestMetadataCache[key], deferred)+ }+ if m.hasRecentGuestMetadataEvidence("rejection", "node", 105, time.Now()) || !m.guestMetadataLimiter[key].After(start) {+ t.Error("rejection became fresh metadata or removed normal backoff")+ }+ before := osCalls.Load()+ m.fetchGuestAgentMetadata(context.Background(), client, "rejection", "node", "guest", 105, status, false)+ if osCalls.Load() != before {+ t.Error("rejection bypassed normal refresh backoff")+ }+ }+ recovered.Store(true)+ m.guestMetadataLimiter[key] = time.Now().Add(-time.Second)+ before := time.Now()+ _, _, name, version, _, deferred := m.fetchGuestAgentMetadata(context.Background(), client, "rejection", "node", "guest", 105, status, false)+ entry := m.guestMetadataCache[key]+ if deferred || name != bootstrapWindowsOS || version != "2025" || !entry.windowsGuest || entry.osInfoObservedAt.Before(before) || entry.osInfoFailureCount != 0 || entry.osInfoSkip || osCalls.Load() != guestAgentOSInfoFailureThreshold+1 {+ t.Errorf("ordinary accepted OS reply did not recover: name=%q version=%q calls=%d entry=%+v", name, version, osCalls.Load(), entry)+ }+ })+ }+ }+}++// A stopped QGA can coexist with status/current's configured agent flag. Exercise+// actual HTTP errors, both ordinary collectors and both client paths. No native+// restart, freeze, extra probe, shortened uncertainty fence or ACL change.+func testGuestMetadataStoppedAgentPollingRecovery(t *testing.T) {+ for _, kind := range []string{"client", "cluster-client"} {+ for _, collector := range []string{"cluster", "node"} {+ t.Run(kind+"/"+collector, func(t *testing.T) {+ f := &windowsBootstrapFixture{t: t, osName: bootstrapWindowsOS, linuxAvailable: 6 * bootstrapMiB}+ f.poll.Store(1)+ var available atomic.Bool+ s := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {+ if !available.Load() && strings.Contains(r.URL.Path, "/agent/") {+ if r.Header.Get("Authorization") != "PVEAPIToken=fixture@pve!pulse=fixture" {+ t.Error("fixture authentication changed")+ w.WriteHeader(http.StatusForbidden)+ return+ }+ command := r.URL.Path[strings.LastIndex(r.URL.Path, "/")+1:]+ f.mu.Lock()+ f.commands = append(f.commands, command)+ f.mu.Unlock()+ w.WriteHeader(http.StatusInternalServerError)+ fmt.Fprintf(w, `{"data":null,"message":"VM 105 qmp command 'guest-%s' failed - QEMU guest agent is not running"}`, command)+ return+ }+ f.serve(w, r)+ }))+ defer s.Close()+ m := windowsBootstrapMonitor(t)+ m.guestMetadataLimiter = make(map[string]time.Time)+ client := windowsBootstrapClient(t, s.URL, kind)+ key := guestMetadataCacheKey("bootstrap", "node", 105)+ for poll := 0; poll < guestAgentOSInfoFailureThreshold; poll++ {+ m.guestMetadataLimiter[key] = time.Now().Add(-time.Second)+ vm := windowsBootstrapPoll(t, m, client, collector, "bootstrap", "node", 105, 1)+ entry := m.guestMetadataCache[key]+ if entry.osInfoFailureCount != 0 || entry.osInfoSkip || !entry.fetchedAt.IsZero() || vm.OSName != "" || vm.Disk.Usage >= 0 || vm.Memory.Observation.Source != "status-mem" || vm.Memory.Observation.State != "current" || vm.CPU != .1 {+ t.Errorf("stopped-agent poll %d became an OS command defect or lost independent counters: %+v, OS=%q disk=%v memory=%+v CPU=%v", poll, entry, vm.OSName, vm.Disk.Usage, vm.Memory.Observation, vm.CPU)+ }+ }+ id := makeGuestID("bootstrap", "node", 105)+ if len(guestHistoryStoredPoints(t, m, "vm", id, "disk")) != 0 {+ t.Error("stopped agent manufactured filesystem History")+ }+ available.Store(true)+ m.guestMetadataLimiter[key] = time.Now().Add(-time.Second)+ f.resetTrace()+ vm := windowsBootstrapPoll(t, m, client, collector, "bootstrap", "node", 105, 1)+ checkWindowsBootstrapReading(t, vm, 1)+ if got, want := f.trace(), []string{"get-fsinfo", "network-get-interfaces", "get-osinfo", "info"}; !reflect.DeepEqual(got, want) {+ t.Errorf("ordinary recovery did not learn Windows before optional Linux reads: %v, want %v", got, want)+ }+ checkBootstrapHistory(t, m, id, "disk", []float64{25})+ })+ }+ }+}