Real-time monitoring dashboard for Proxmox VE, PBS, Docker, Kubernetes, TrueNAS and vSphere. Self-hosted, with alerts for failed backups, full disks, unhealthy containers and offline nodes.
Go aialertsdashboarddockerdocker-monitoringgohomelabinfrastructure-monitoringkubernetesmonitoringmspproxmoxproxmox-backup-serverproxmox-mail-gatewayproxmox-veself-hostedtruenastypescriptvspherewebhooks
Keep guest OS discovery eligible after rate refusals
pulse-triage[bot]63756a7017d0
diff --git a/docs/release-control/v6/internal/subsystems/monitoring.md b/docs/release-control/v6/internal/subsystems/monitoring.mdindex 63a65fd906..df3b0def77 100644--- a/docs/release-control/v6/internal/subsystems/monitoring.md+++ b/docs/release-control/v6/internal/subsystems/monitoring.md@@ -17,6 +17,23 @@## Purpose+### Rate refusals cannot permanently disable guest OS discovery++An observed HTTP 429 from guest OS-info preserves the existing OS-command+failure count and suppression state, like access refusal or stopped QGA. It+neither exercised the OS command nor supplies a fresh OS observation. Ordinary+metadata backoff remains in force; a subsequent admitted poll can learn the OS+without a restart or clearing command-safety state. Only the typed wire status+admits this distinction: provider text quoting 429 is not a rate refusal.++`TestGuestMetadataRateLimitRecoveryContract` covers both HTTP client types and+ordinary collectors with cold/stale metadata, OS-like 429 bodies, unchanged+backoff and accepted filesystem/memory readings, subsequent Windows discovery+before Linux-only memory admission, and genuine repeated-failure suppression+with a quoted 429. Existing unsupported-command, backup-lock and uncertain-reply+fences are unchanged. Its fixtures use local sockets and synthetic credentials,+not native guest recovery, History storage or a reported failure's cause.+### Cluster endpoint membership belongs to its client`NewClusterClient` snapshots the supplied endpoint slice before registeringdiff --git a/internal/monitoring/guest_metadata.go b/internal/monitoring/guest_metadata.goindex bcdbf8bff2..a56e39488a 100644--- a/internal/monitoring/guest_metadata.go+++ b/internal/monitoring/guest_metadata.go@@ -6,6 +6,7 @@ import ("errors""fmt""math"+ "net/http""sort""strconv""strings"@@ -468,7 +469,11 @@ func (m *Monitor) fetchGuestAgentMetadata(ctx context.Context, client PVEClientIreturn cloneStringSlice(cached.ipAddresses), cloneGuestNetworkInterfaces(cached.networkInterfaces), cached.osName, cached.osVersion, cached.agentVersion, true}if err != nil {- if reason := proxmox.GuestAgentErrorReason(err); reason == "permission-denied" || reason == "agent-not-running" {+ status, statusKnown := proxmox.APIErrorStatus(err)+ if reason := proxmox.GuestAgentErrorReason(err); reason == "permission-denied" || reason == "agent-not-running" || (statusKnown && status == http.StatusTooManyRequests) {+ // Access/rate refusal or stopped QGA did not exercise this+ // command. Use the wire status, not OS-like or status-like body+ // text: a temporary rate limit must not disable OS discovery.// The request was not admitted or QGA was stopped, so this did// not exercise the OS-info command. Do not permanently disable// that command when ordinary access/agent availability returns.diff --git a/internal/monitoring/guest_metadata_rate_limit_test.go b/internal/monitoring/guest_metadata_rate_limit_test.gonew file mode 100644index 0000000000..d2f80771ee--- /dev/null+++ b/internal/monitoring/guest_metadata_rate_limit_test.go@@ -0,0 +1,171 @@+package monitoring++import (+ "context"+ "encoding/json"+ "fmt"+ "net/http"+ "net/http/httptest"+ "os"+ "reflect"+ "strings"+ "sync/atomic"+ "testing"+ "time"++ "github.com/rcourtman/pulse-go-rewrite/internal/alerts"+ "github.com/rcourtman/pulse-go-rewrite/internal/config"+ "github.com/rcourtman/pulse-go-rewrite/internal/models"+ "github.com/rcourtman/pulse-go-rewrite/pkg/proxmox"+)++// Rate refusal is not a completed OS-command failure. Exercise the real HTTP+// clients and both ordinary builders without a guest, database, backup or restart.+func TestGuestMetadataRateLimitRecoveryContract(t *testing.T) {+ for _, kind := range []string{"client", "cluster-client"} {+ for _, collector := range []string{"cluster", "node"} {+ for _, seeded := range []bool{false, true} {+ for _, tc := range []struct {+ name string+ status int+ body string+ rateRefusal bool+ }{+ {"rate-limit", http.StatusTooManyRequests, "Too many requests", true},+ {"rate-limit-os-like-body", http.StatusTooManyRequests, "unsupported command: guest-get-osinfo", true},+ {"quoted-rate-is-not-wire-rate", http.StatusBadRequest, "API error 429: Too many requests", false},+ } {+ t.Run(fmt.Sprintf("%s/%s/cached-%t/%s", kind, collector, seeded, tc.name), func(t *testing.T) {+ const instance = "rate-recovery"+ var recovered atomic.Bool+ var osCalls, linuxCalls atomic.Int32+ server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {+ if r.Header.Get("Authorization") != "PVEAPIToken=fixture@pve!pulse=fixture" || r.Method != http.MethodGet {+ t.Error("fixture authentication or read-only method changed")+ w.WriteHeader(http.StatusForbidden)+ return+ }+ w.Header().Set("Content-Type", "application/json")+ switch {+ case strings.HasSuffix(r.URL.Path, "/nodes"):+ fmt.Fprint(w, `{"data":[]}`)+ case strings.HasSuffix(r.URL.Path, "/config"):+ fmt.Fprint(w, `{"data":{}}`)+ case strings.HasSuffix(r.URL.Path, "/status/current"):+ // Independent pressure evidence avoids an unnecessary optional Linux+ // read while the OS endpoint is temporarily refusing requests.+ if recovered.Load() && tc.rateRefusal {+ fmt.Fprint(w, `{"data":{"status":"running","agent":1,"maxmem":8388608,"mem":3145728}}`)+ } else {+ fmt.Fprint(w, `{"data":{"status":"running","agent":1,"maxmem":8388608,"mem":3145728,"meminfo":{"total":8388608,"available":5242880}}}`)+ }+ case strings.HasSuffix(r.URL.Path, "/get-fsinfo"):+ fmt.Fprint(w, `{"data":{"result":[{"name":"C:","mountpoint":"C:\\","type":"ntfs","total-bytes":1572864000,"used-bytes":419430400},{"name":"D:","mountpoint":"D:\\","type":"ntfs","total-bytes":524288000,"used-bytes":104857600}]}}`)+ case strings.HasSuffix(r.URL.Path, "/network-get-interfaces"):+ fmt.Fprint(w, `{"data":{"result":[]}}`)+ case strings.HasSuffix(r.URL.Path, "/get-osinfo"):+ osCalls.Add(1)+ if recovered.Load() {+ fmt.Fprintf(w, `{"data":{"result":{"name":%q,"version":"2025"}}}`, bootstrapWindowsOS)+ } else {+ w.WriteHeader(tc.status)+ fmt.Fprint(w, tc.body)+ }+ case strings.HasSuffix(r.URL.Path, "/info"):+ fmt.Fprint(w, `{"data":{"result":{}}}`)+ case strings.HasSuffix(r.URL.Path, "/file-read"):+ linuxCalls.Add(1)+ w.WriteHeader(http.StatusInternalServerError)+ fmt.Fprint(w, "unsupported command: guest-file-read")+ default:+ t.Errorf("unexpected request: %s", r.URL.Path)+ http.NotFound(w, r)+ }+ }))+ defer server.Close()+ client := windowsBootstrapClient(t, server.URL, kind)+ key := guestMetadataCacheKey(instance, "node", 105)+ original := guestMetadataCacheEntry{}+ if seeded {+ original = metadataObservationFixture(time.Now().Add(-2 * guestMetadataCacheTTL))+ original.osInfoFailureCount = guestAgentOSInfoFailureThreshold - 1+ }+ m := &Monitor{+ config: &config.Config{}, state: models.NewState(), rateTracker: NewRateTracker(),+ guestMetadataCache: map[string]guestMetadataCacheEntry{key: original},+ guestMetadataLimiter: make(map[string]time.Time), guestAgentRetries: 2,+ }+ m.alertManager = alerts.NewManagerWithDataDir(t.TempDir(), alerts.WithoutPersistedAlertRestore())+ t.Cleanup(m.alertManager.Stop)+ m.guestAgentNetworkTimeout, m.guestAgentOSInfoTimeout, m.guestAgentVersionTimeout = windowsPollingHangGuard, windowsPollingHangGuard, windowsPollingHangGuard+ id := makeGuestID(instance, "node", 105)+ var previous models.VM+ poll := func() models.VM {+ var vm models.VM+ if collector == "node" {+ vms, _ := m.pollNodeVMsWithClusterResourceBuilder(context.Background(), instance, "node", []proxmox.VM{{VMID: 105, Name: "rate-limited-guest", Status: "running", MaxMem: 8 * bootstrapMiB, MaxDisk: 2000 * bootstrapMiB, CPU: .1}}, client, map[string]models.VM{id: previous}, nil)+ if len(vms) != 1 {+ t.Fatal("ordinary node poll lost the guest")+ }+ vm = vms[0]+ } else {+ var ok bool+ vm, _, _, _, _, ok = m.buildVMFromClusterResource(context.Background(), instance, proxmox.ClusterResource{VMID: 105, Node: "node", Type: "qemu", Name: "rate-limited-guest", Status: "running", MaxMem: 8 * bootstrapMiB, MaxDisk: 2000 * bootstrapMiB, CPU: .1}, client, id, nil, &previous)+ if !ok {+ t.Fatal("ordinary cluster builder lost the guest")+ }+ }+ if vm.Disk.Used != int64(500*bootstrapMiB) || len(vm.Disks) != 2 || vm.DiskStatusReason != "" || vm.Memory.Used != int64(3*bootstrapMiB) || vm.CPU != .1 {+ t.Errorf("independent accepted readings were lost: disk=%+v rows=%d memory=%+v CPU=%v", vm.Disk, len(vm.Disks), vm.Memory, vm.CPU)+ }+ previous = vm+ return vm+ }+ rounds := guestAgentOSInfoFailureThreshold+ if !tc.rateRefusal && seeded {+ rounds = 1+ }+ var before models.VM+ for round := 0; round < rounds; round++ {+ // Advance only test-owned metadata scheduling. No lock, command-safety+ // suppression, uncertainty fence or guest command is cleared/replayed.+ m.guestMetadataLimiter[key] = time.Now().Add(-time.Second)+ before = poll()+ if tc.rateRefusal && !reflect.DeepEqual(m.guestMetadataCache[key], original) {+ t.Errorf("HTTP429 poisoned OS suppression or renewed identity: %+v, want %+v", m.guestMetadataCache[key], original)+ }+ if m.hasRecentGuestMetadataEvidence(instance, "node", 105, time.Now()) {+ t.Error("refusal became fresh metadata")+ }+ count := osCalls.Load()+ poll()+ if osCalls.Load() != count {+ t.Error("refusal bypassed ordinary refresh backoff")+ }+ }+ recovered.Store(true)+ m.guestMetadataLimiter[key] = time.Now().Add(-time.Second)+ after := poll()+ entry := m.guestMetadataCache[key]+ if tc.rateRefusal {+ if after.OSName != bootstrapWindowsOS || after.OSVersion != "2025" || entry.osInfoSkip || entry.osInfoFailureCount != 0 || !entry.windowsGuest || entry.osInfoObservedAt.IsZero() || osCalls.Load() != int32(rounds+1) || linuxCalls.Load() != 0 {+ t.Errorf("ordinary recovery stayed suppressed or sent Linux memory read: OS=%q version=%q calls=%d Linux=%d entry=%+v", after.OSName, after.OSVersion, osCalls.Load(), linuxCalls.Load(), entry)+ }+ } else if !entry.osInfoSkip || osCalls.Load() != int32(rounds) || after.OSName != original.osName {+ t.Error("unrelated genuine repeated failure suppression was weakened by a quoted 429")+ }+ // The optional, safe source projection is byte-for-byte producer JSON,+ // not a hand-authored browser replacement. No database/native acceptance.+ if os.Getenv("PULSE_TEST_RATE_LIMIT_PROJECTION") == "1" && kind == "client" && collector == "cluster" && !seeded && tc.name == "rate-limit" {+ data, err := json.Marshal(map[string]models.VM{"before": before, "after": after})+ if err != nil {+ t.Fatal(err)+ }+ t.Logf("RATE_LIMIT_PROJECTION %s", data)+ }+ })+ }+ }+ }+ }+}