Skip to content

armgit / Pulse

Public Imported from GitHub
0

Real-time monitoring dashboard for Proxmox VE, PBS, Docker, Kubernetes, TrueNAS and vSphere. Self-hosted, with alerts for failed backups, full disks, unhealthy containers and offline nodes.

Keep guest OS discovery eligible after temporary rejections

pulse-triage[bot]3caea610ed67
diff --git a/docs/release-control/v6/internal/subsystems/monitoring.md b/docs/release-control/v6/internal/subsystems/monitoring.md
index 2e41e917aa..9b645d110e 100644
--- a/docs/release-control/v6/internal/subsystems/monitoring.md
+++ b/docs/release-control/v6/internal/subsystems/monitoring.md
@@ -666,10 +666,18 @@ required; all responses in this fixture are synthetic.
### Guest-agent coordination around backups — issue #2439
-A completed OS-info outcome keeps its existing suppression semantics even when
+A completed OS-info outcome keeps its suppression semantics even when
the later agent-version read is deferred. Unsupported OS-info still disables
-that optional command; ordinary completed failures still accumulate towards
-the existing three-failure limit, and a supported reply resets that count.
+that optional command; ordinary completed command failures still accumulate
+towards the existing three-failure limit, and a supported reply resets that count.
+Actual HTTP 401/403 or exact command-bound stopped-agent rejections did not
+exercise OS-info and therefore do not accumulate towards permanent suppression.
+They preserve prior command failures and normal refresh/backoff timing; they
+neither renew retained identity nor establish a Windows observation. Once ordinary
+access/agent availability returns, an eligible accepted OS reply can recover
+the identity and cross-platform memory classification without a Pulse restart.
+Quoted status/diagnosis text does not grant this distinction. Unsupported,
+repeated completed-failure and uncertain-command safety suppression is retained.
Only these safety fields are retained on version deferral: partial network/OS
metadata is not published, old identity is not made fresh, and the shared pause
and configured retry backoff remain in force. A network or OS-info deferral
@@ -683,6 +691,15 @@ suppressed OS-info request and with fresh disk History. It changes no native
guest, backup policy, lock admission, cooldown, retry or single-attempt rule.
These source controls do not establish native recovery or a reported cause.
+`TestGuestMetadataRejectionRecoveryContract` verifies typed rejections with cold
+and retained caches, prior genuine failures, unchanged backoff and fresh OS
+receipt requirements. Both ordinary collectors and single/cluster HTTP clients
+recover Windows OS, individual NTFS volumes and disk History after a stopped-agent
+fixture, without dispatching Linux-only memory reads on the recovered Windows
+guest. Existing suppression, backup-lock and shared uncertainty contracts remain
+required. These synthetic source controls do not establish native restart/thaw,
+the cause of #2822/#2619, or installed/released recovery.
+
The process-local admission table remains capped at 4,096 endpoint/VMID
entries. Expired entries replaced by a new admission consume no extra slot;
when new aliases would exceed the bound, expired idle entries are reclaimed
diff --git a/internal/monitoring/canonical_guardrails_test.go b/internal/monitoring/canonical_guardrails_test.go
index 93474edaf1..ff1b4645fe 100644
--- a/internal/monitoring/canonical_guardrails_test.go
+++ b/internal/monitoring/canonical_guardrails_test.go
@@ -3257,6 +3257,11 @@ func TestGuestWindowsRestartPollingContract(t *testing.T) {
t.Run("admission-and-identity", testGuestWindowsBootstrapSafety)
}
+func TestGuestMetadataRejectionRecoveryContract(t *testing.T) {
+ t.Run("typed-rejection", testGuestMetadataRejectedOSInfo)
+ t.Run("ordinary-stopped-agent-recovery", testGuestMetadataStoppedAgentPollingRecovery)
+}
+
func TestGuestMemoryObservationContract(t *testing.T) {
t.Run("poll-to-served-observation", testGuestMemoryObservationLifecycle)
t.Run("identity-and-origin-boundaries", testGuestMemoryObservationKeepsOriginsSeparate)
diff --git a/internal/monitoring/guest_metadata.go b/internal/monitoring/guest_metadata.go
index 70b4ae2847..00c05ee685 100644
--- a/internal/monitoring/guest_metadata.go
+++ b/internal/monitoring/guest_metadata.go
@@ -366,7 +366,18 @@ func (m *Monitor) fetchGuestAgentMetadata(ctx context.Context, client PVEClientI
return cloneStringSlice(cached.ipAddresses), cloneGuestNetworkInterfaces(cached.networkInterfaces), cached.osName, cached.osVersion, cached.agentVersion, true
}
if err != nil {
- if isGuestAgentOSInfoUnsupportedError(err) {
+ if reason := proxmox.GuestAgentErrorReason(err); reason == "permission-denied" || reason == "agent-not-running" {
+ // The request was not admitted or QGA was stopped, so this did
+ // not exercise the OS-info command. Do not permanently disable
+ // that command when ordinary access/agent availability returns.
+ // Preserve any real prior failures and the normal refresh backoff.
+ log.Debug().
+ Str("instance", instanceName).
+ Str("vm", vmName).
+ Int("vmid", vmid).
+ Err(err).
+ Msg("Guest agent OS info request not admitted; preserving command failure state")
+ } else if isGuestAgentOSInfoUnsupportedError(err) {
osInfoSkip = true
osInfoFailureCount = guestAgentOSInfoFailureThreshold
log.Warn().
diff --git a/internal/monitoring/guest_metadata_rejection_test.go b/internal/monitoring/guest_metadata_rejection_test.go
new file mode 100644
index 0000000000..9e3bd95173
--- /dev/null
+++ b/internal/monitoring/guest_metadata_rejection_test.go
@@ -0,0 +1,161 @@
+package monitoring
+
+import (
+ "context"
+ "fmt"
+ "net/http"
+ "net/http/httptest"
+ "reflect"
+ "strings"
+ "sync/atomic"
+ "testing"
+ "time"
+
+ "github.com/rcourtman/pulse-go-rewrite/pkg/proxmox"
+)
+
+// A complete access/agent-unavailable rejection did not exercise guest-get-osinfo.
+// It must not be counted as evidence that this particular command is unsafe.
+func testGuestMetadataRejectedOSInfo(t *testing.T) {
+ for _, rejection := range []string{"unauthorized", "forbidden", "agent-not-running"} {
+ for _, seeded := range []bool{false, true} {
+ t.Run(fmt.Sprintf("%s/cached-%t", rejection, seeded), func(t *testing.T) {
+ var recovered atomic.Bool
+ var osCalls atomic.Int32
+ server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+ if r.Header.Get("Authorization") != "PVEAPIToken=fixture@pve!pulse=fixture" {
+ t.Error("fixture authentication changed")
+ w.WriteHeader(http.StatusForbidden)
+ return
+ }
+ switch {
+ case strings.HasSuffix(r.URL.Path, "/config"):
+ fmt.Fprint(w, `{"data":{}}`)
+ case strings.HasSuffix(r.URL.Path, "/get-osinfo"):
+ osCalls.Add(1)
+ if recovered.Load() {
+ fmt.Fprintf(w, `{"data":{"name":%q,"version":"2025"}}`, bootstrapWindowsOS)
+ return
+ }
+ switch rejection {
+ case "unauthorized":
+ w.WriteHeader(http.StatusUnauthorized)
+ // Even OS-like body text is not a completed OS-info reply.
+ fmt.Fprint(w, "unsupported command: guest-get-osinfo")
+ case "forbidden":
+ w.WriteHeader(http.StatusForbidden)
+ fmt.Fprint(w, "Failed to open file '/etc/os-release': No such file or directory")
+ default:
+ w.WriteHeader(http.StatusInternalServerError)
+ fmt.Fprint(w, `{"data":null,"message":"VM 105 qmp command 'guest-get-osinfo' failed - QEMU guest agent is not running"}`)
+ }
+ case strings.HasSuffix(r.URL.Path, "/network-get-interfaces"):
+ fmt.Fprint(w, `{"data":{"result":[]}}`)
+ case strings.HasSuffix(r.URL.Path, "/info"):
+ fmt.Fprint(w, `{"data":{"result":{}}}`)
+ default:
+ t.Errorf("unexpected request: %s", r.URL.Path)
+ http.NotFound(w, r)
+ }
+ }))
+ defer server.Close()
+ client, err := proxmox.NewClient(proxmox.ClientConfig{Host: server.URL, TokenName: "fixture@pve!pulse", TokenValue: "fixture", Timeout: windowsPollingHangGuard})
+ if err != nil {
+ t.Fatal(err)
+ }
+ key := guestMetadataCacheKey("rejection", "node", 105)
+ original := guestMetadataCacheEntry{}
+ if seeded {
+ original = metadataObservationFixture(time.Now().Add(-2 * guestMetadataCacheTTL))
+ original.osInfoFailureCount = guestAgentOSInfoFailureThreshold - 1
+ }
+ m := &Monitor{guestMetadataCache: map[string]guestMetadataCacheEntry{key: original}, guestMetadataLimiter: make(map[string]time.Time)}
+ status := &proxmox.VMStatus{Agent: proxmox.VMAgentField{Value: 1}}
+ for poll := 1; poll <= guestAgentOSInfoFailureThreshold; poll++ {
+ // Represent separate eligible refreshes without waiting for the
+ // production backoff. Never clear a QGA command fence or lock.
+ m.guestMetadataLimiter[key] = time.Now().Add(-time.Second)
+ start := time.Now()
+ ips, ifaces, name, version, agent, deferred := m.fetchGuestAgentMetadata(context.Background(), client, "rejection", "node", "guest", 105, status, false)
+ if deferred || !reflect.DeepEqual(m.guestMetadataCache[key], original) || !reflect.DeepEqual(ips, original.ipAddresses) || !reflect.DeepEqual(ifaces, original.networkInterfaces) || name != original.osName || version != original.osVersion || agent != original.agentVersion {
+ t.Errorf("rejected poll %d poisoned OS state or renewed identity: %+v, deferred=%t", poll, m.guestMetadataCache[key], deferred)
+ }
+ if m.hasRecentGuestMetadataEvidence("rejection", "node", 105, time.Now()) || !m.guestMetadataLimiter[key].After(start) {
+ t.Error("rejection became fresh metadata or removed normal backoff")
+ }
+ before := osCalls.Load()
+ m.fetchGuestAgentMetadata(context.Background(), client, "rejection", "node", "guest", 105, status, false)
+ if osCalls.Load() != before {
+ t.Error("rejection bypassed normal refresh backoff")
+ }
+ }
+ recovered.Store(true)
+ m.guestMetadataLimiter[key] = time.Now().Add(-time.Second)
+ before := time.Now()
+ _, _, name, version, _, deferred := m.fetchGuestAgentMetadata(context.Background(), client, "rejection", "node", "guest", 105, status, false)
+ entry := m.guestMetadataCache[key]
+ if deferred || name != bootstrapWindowsOS || version != "2025" || !entry.windowsGuest || entry.osInfoObservedAt.Before(before) || entry.osInfoFailureCount != 0 || entry.osInfoSkip || osCalls.Load() != guestAgentOSInfoFailureThreshold+1 {
+ t.Errorf("ordinary accepted OS reply did not recover: name=%q version=%q calls=%d entry=%+v", name, version, osCalls.Load(), entry)
+ }
+ })
+ }
+ }
+}
+
+// A stopped QGA can coexist with status/current's configured agent flag. Exercise
+// actual HTTP errors, both ordinary collectors and both client paths. No native
+// restart, freeze, extra probe, shortened uncertainty fence or ACL change.
+func testGuestMetadataStoppedAgentPollingRecovery(t *testing.T) {
+ for _, kind := range []string{"client", "cluster-client"} {
+ for _, collector := range []string{"cluster", "node"} {
+ t.Run(kind+"/"+collector, func(t *testing.T) {
+ f := &windowsBootstrapFixture{t: t, osName: bootstrapWindowsOS, linuxAvailable: 6 * bootstrapMiB}
+ f.poll.Store(1)
+ var available atomic.Bool
+ s := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+ if !available.Load() && strings.Contains(r.URL.Path, "/agent/") {
+ if r.Header.Get("Authorization") != "PVEAPIToken=fixture@pve!pulse=fixture" {
+ t.Error("fixture authentication changed")
+ w.WriteHeader(http.StatusForbidden)
+ return
+ }
+ command := r.URL.Path[strings.LastIndex(r.URL.Path, "/")+1:]
+ f.mu.Lock()
+ f.commands = append(f.commands, command)
+ f.mu.Unlock()
+ w.WriteHeader(http.StatusInternalServerError)
+ fmt.Fprintf(w, `{"data":null,"message":"VM 105 qmp command 'guest-%s' failed - QEMU guest agent is not running"}`, command)
+ return
+ }
+ f.serve(w, r)
+ }))
+ defer s.Close()
+ m := windowsBootstrapMonitor(t)
+ m.guestMetadataLimiter = make(map[string]time.Time)
+ client := windowsBootstrapClient(t, s.URL, kind)
+ key := guestMetadataCacheKey("bootstrap", "node", 105)
+ for poll := 0; poll < guestAgentOSInfoFailureThreshold; poll++ {
+ m.guestMetadataLimiter[key] = time.Now().Add(-time.Second)
+ vm := windowsBootstrapPoll(t, m, client, collector, "bootstrap", "node", 105, 1)
+ entry := m.guestMetadataCache[key]
+ if entry.osInfoFailureCount != 0 || entry.osInfoSkip || !entry.fetchedAt.IsZero() || vm.OSName != "" || vm.Disk.Usage >= 0 || vm.Memory.Observation.Source != "status-mem" || vm.Memory.Observation.State != "current" || vm.CPU != .1 {
+ t.Errorf("stopped-agent poll %d became an OS command defect or lost independent counters: %+v, OS=%q disk=%v memory=%+v CPU=%v", poll, entry, vm.OSName, vm.Disk.Usage, vm.Memory.Observation, vm.CPU)
+ }
+ }
+ id := makeGuestID("bootstrap", "node", 105)
+ if len(guestHistoryStoredPoints(t, m, "vm", id, "disk")) != 0 {
+ t.Error("stopped agent manufactured filesystem History")
+ }
+ available.Store(true)
+ m.guestMetadataLimiter[key] = time.Now().Add(-time.Second)
+ f.resetTrace()
+ vm := windowsBootstrapPoll(t, m, client, collector, "bootstrap", "node", 105, 1)
+ checkWindowsBootstrapReading(t, vm, 1)
+ if got, want := f.trace(), []string{"get-fsinfo", "network-get-interfaces", "get-osinfo", "info"}; !reflect.DeepEqual(got, want) {
+ t.Errorf("ordinary recovery did not learn Windows before optional Linux reads: %v, want %v", got, want)
+ }
+ checkBootstrapHistory(t, m, id, "disk", []float64{25})
+ })
+ }
+ }
+}