Skip to content

armgit / Pulse

Public Imported from GitHub
0

Real-time monitoring dashboard for Proxmox VE, PBS, Docker, Kubernetes, TrueNAS and vSphere. Self-hosted, with alerts for failed backups, full disks, unhealthy containers and offline nodes.

Keep guest OS discovery eligible after rate refusals

pulse-triage[bot]63756a7017d0
diff --git a/docs/release-control/v6/internal/subsystems/monitoring.md b/docs/release-control/v6/internal/subsystems/monitoring.md
index 63a65fd906..df3b0def77 100644
--- a/docs/release-control/v6/internal/subsystems/monitoring.md
+++ b/docs/release-control/v6/internal/subsystems/monitoring.md
@@ -17,6 +17,23 @@
## Purpose
+### Rate refusals cannot permanently disable guest OS discovery
+
+An observed HTTP 429 from guest OS-info preserves the existing OS-command
+failure count and suppression state, like access refusal or stopped QGA. It
+neither exercised the OS command nor supplies a fresh OS observation. Ordinary
+metadata backoff remains in force; a subsequent admitted poll can learn the OS
+without a restart or clearing command-safety state. Only the typed wire status
+admits this distinction: provider text quoting 429 is not a rate refusal.
+
+`TestGuestMetadataRateLimitRecoveryContract` covers both HTTP client types and
+ordinary collectors with cold/stale metadata, OS-like 429 bodies, unchanged
+backoff and accepted filesystem/memory readings, subsequent Windows discovery
+before Linux-only memory admission, and genuine repeated-failure suppression
+with a quoted 429. Existing unsupported-command, backup-lock and uncertain-reply
+fences are unchanged. Its fixtures use local sockets and synthetic credentials,
+not native guest recovery, History storage or a reported failure's cause.
+
### Cluster endpoint membership belongs to its client
`NewClusterClient` snapshots the supplied endpoint slice before registering
diff --git a/internal/monitoring/guest_metadata.go b/internal/monitoring/guest_metadata.go
index bcdbf8bff2..a56e39488a 100644
--- a/internal/monitoring/guest_metadata.go
+++ b/internal/monitoring/guest_metadata.go
@@ -6,6 +6,7 @@ import (
"errors"
"fmt"
"math"
+ "net/http"
"sort"
"strconv"
"strings"
@@ -468,7 +469,11 @@ func (m *Monitor) fetchGuestAgentMetadata(ctx context.Context, client PVEClientI
return cloneStringSlice(cached.ipAddresses), cloneGuestNetworkInterfaces(cached.networkInterfaces), cached.osName, cached.osVersion, cached.agentVersion, true
}
if err != nil {
- if reason := proxmox.GuestAgentErrorReason(err); reason == "permission-denied" || reason == "agent-not-running" {
+ status, statusKnown := proxmox.APIErrorStatus(err)
+ if reason := proxmox.GuestAgentErrorReason(err); reason == "permission-denied" || reason == "agent-not-running" || (statusKnown && status == http.StatusTooManyRequests) {
+ // Access/rate refusal or stopped QGA did not exercise this
+ // command. Use the wire status, not OS-like or status-like body
+ // text: a temporary rate limit must not disable OS discovery.
// The request was not admitted or QGA was stopped, so this did
// not exercise the OS-info command. Do not permanently disable
// that command when ordinary access/agent availability returns.
diff --git a/internal/monitoring/guest_metadata_rate_limit_test.go b/internal/monitoring/guest_metadata_rate_limit_test.go
new file mode 100644
index 0000000000..d2f80771ee
--- /dev/null
+++ b/internal/monitoring/guest_metadata_rate_limit_test.go
@@ -0,0 +1,171 @@
+package monitoring
+
+import (
+ "context"
+ "encoding/json"
+ "fmt"
+ "net/http"
+ "net/http/httptest"
+ "os"
+ "reflect"
+ "strings"
+ "sync/atomic"
+ "testing"
+ "time"
+
+ "github.com/rcourtman/pulse-go-rewrite/internal/alerts"
+ "github.com/rcourtman/pulse-go-rewrite/internal/config"
+ "github.com/rcourtman/pulse-go-rewrite/internal/models"
+ "github.com/rcourtman/pulse-go-rewrite/pkg/proxmox"
+)
+
+// Rate refusal is not a completed OS-command failure. Exercise the real HTTP
+// clients and both ordinary builders without a guest, database, backup or restart.
+func TestGuestMetadataRateLimitRecoveryContract(t *testing.T) {
+ for _, kind := range []string{"client", "cluster-client"} {
+ for _, collector := range []string{"cluster", "node"} {
+ for _, seeded := range []bool{false, true} {
+ for _, tc := range []struct {
+ name string
+ status int
+ body string
+ rateRefusal bool
+ }{
+ {"rate-limit", http.StatusTooManyRequests, "Too many requests", true},
+ {"rate-limit-os-like-body", http.StatusTooManyRequests, "unsupported command: guest-get-osinfo", true},
+ {"quoted-rate-is-not-wire-rate", http.StatusBadRequest, "API error 429: Too many requests", false},
+ } {
+ t.Run(fmt.Sprintf("%s/%s/cached-%t/%s", kind, collector, seeded, tc.name), func(t *testing.T) {
+ const instance = "rate-recovery"
+ var recovered atomic.Bool
+ var osCalls, linuxCalls atomic.Int32
+ server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+ if r.Header.Get("Authorization") != "PVEAPIToken=fixture@pve!pulse=fixture" || r.Method != http.MethodGet {
+ t.Error("fixture authentication or read-only method changed")
+ w.WriteHeader(http.StatusForbidden)
+ return
+ }
+ w.Header().Set("Content-Type", "application/json")
+ switch {
+ case strings.HasSuffix(r.URL.Path, "/nodes"):
+ fmt.Fprint(w, `{"data":[]}`)
+ case strings.HasSuffix(r.URL.Path, "/config"):
+ fmt.Fprint(w, `{"data":{}}`)
+ case strings.HasSuffix(r.URL.Path, "/status/current"):
+ // Independent pressure evidence avoids an unnecessary optional Linux
+ // read while the OS endpoint is temporarily refusing requests.
+ if recovered.Load() && tc.rateRefusal {
+ fmt.Fprint(w, `{"data":{"status":"running","agent":1,"maxmem":8388608,"mem":3145728}}`)
+ } else {
+ fmt.Fprint(w, `{"data":{"status":"running","agent":1,"maxmem":8388608,"mem":3145728,"meminfo":{"total":8388608,"available":5242880}}}`)
+ }
+ case strings.HasSuffix(r.URL.Path, "/get-fsinfo"):
+ fmt.Fprint(w, `{"data":{"result":[{"name":"C:","mountpoint":"C:\\","type":"ntfs","total-bytes":1572864000,"used-bytes":419430400},{"name":"D:","mountpoint":"D:\\","type":"ntfs","total-bytes":524288000,"used-bytes":104857600}]}}`)
+ case strings.HasSuffix(r.URL.Path, "/network-get-interfaces"):
+ fmt.Fprint(w, `{"data":{"result":[]}}`)
+ case strings.HasSuffix(r.URL.Path, "/get-osinfo"):
+ osCalls.Add(1)
+ if recovered.Load() {
+ fmt.Fprintf(w, `{"data":{"result":{"name":%q,"version":"2025"}}}`, bootstrapWindowsOS)
+ } else {
+ w.WriteHeader(tc.status)
+ fmt.Fprint(w, tc.body)
+ }
+ case strings.HasSuffix(r.URL.Path, "/info"):
+ fmt.Fprint(w, `{"data":{"result":{}}}`)
+ case strings.HasSuffix(r.URL.Path, "/file-read"):
+ linuxCalls.Add(1)
+ w.WriteHeader(http.StatusInternalServerError)
+ fmt.Fprint(w, "unsupported command: guest-file-read")
+ default:
+ t.Errorf("unexpected request: %s", r.URL.Path)
+ http.NotFound(w, r)
+ }
+ }))
+ defer server.Close()
+ client := windowsBootstrapClient(t, server.URL, kind)
+ key := guestMetadataCacheKey(instance, "node", 105)
+ original := guestMetadataCacheEntry{}
+ if seeded {
+ original = metadataObservationFixture(time.Now().Add(-2 * guestMetadataCacheTTL))
+ original.osInfoFailureCount = guestAgentOSInfoFailureThreshold - 1
+ }
+ m := &Monitor{
+ config: &config.Config{}, state: models.NewState(), rateTracker: NewRateTracker(),
+ guestMetadataCache: map[string]guestMetadataCacheEntry{key: original},
+ guestMetadataLimiter: make(map[string]time.Time), guestAgentRetries: 2,
+ }
+ m.alertManager = alerts.NewManagerWithDataDir(t.TempDir(), alerts.WithoutPersistedAlertRestore())
+ t.Cleanup(m.alertManager.Stop)
+ m.guestAgentNetworkTimeout, m.guestAgentOSInfoTimeout, m.guestAgentVersionTimeout = windowsPollingHangGuard, windowsPollingHangGuard, windowsPollingHangGuard
+ id := makeGuestID(instance, "node", 105)
+ var previous models.VM
+ poll := func() models.VM {
+ var vm models.VM
+ if collector == "node" {
+ vms, _ := m.pollNodeVMsWithClusterResourceBuilder(context.Background(), instance, "node", []proxmox.VM{{VMID: 105, Name: "rate-limited-guest", Status: "running", MaxMem: 8 * bootstrapMiB, MaxDisk: 2000 * bootstrapMiB, CPU: .1}}, client, map[string]models.VM{id: previous}, nil)
+ if len(vms) != 1 {
+ t.Fatal("ordinary node poll lost the guest")
+ }
+ vm = vms[0]
+ } else {
+ var ok bool
+ vm, _, _, _, _, ok = m.buildVMFromClusterResource(context.Background(), instance, proxmox.ClusterResource{VMID: 105, Node: "node", Type: "qemu", Name: "rate-limited-guest", Status: "running", MaxMem: 8 * bootstrapMiB, MaxDisk: 2000 * bootstrapMiB, CPU: .1}, client, id, nil, &previous)
+ if !ok {
+ t.Fatal("ordinary cluster builder lost the guest")
+ }
+ }
+ if vm.Disk.Used != int64(500*bootstrapMiB) || len(vm.Disks) != 2 || vm.DiskStatusReason != "" || vm.Memory.Used != int64(3*bootstrapMiB) || vm.CPU != .1 {
+ t.Errorf("independent accepted readings were lost: disk=%+v rows=%d memory=%+v CPU=%v", vm.Disk, len(vm.Disks), vm.Memory, vm.CPU)
+ }
+ previous = vm
+ return vm
+ }
+ rounds := guestAgentOSInfoFailureThreshold
+ if !tc.rateRefusal && seeded {
+ rounds = 1
+ }
+ var before models.VM
+ for round := 0; round < rounds; round++ {
+ // Advance only test-owned metadata scheduling. No lock, command-safety
+ // suppression, uncertainty fence or guest command is cleared/replayed.
+ m.guestMetadataLimiter[key] = time.Now().Add(-time.Second)
+ before = poll()
+ if tc.rateRefusal && !reflect.DeepEqual(m.guestMetadataCache[key], original) {
+ t.Errorf("HTTP429 poisoned OS suppression or renewed identity: %+v, want %+v", m.guestMetadataCache[key], original)
+ }
+ if m.hasRecentGuestMetadataEvidence(instance, "node", 105, time.Now()) {
+ t.Error("refusal became fresh metadata")
+ }
+ count := osCalls.Load()
+ poll()
+ if osCalls.Load() != count {
+ t.Error("refusal bypassed ordinary refresh backoff")
+ }
+ }
+ recovered.Store(true)
+ m.guestMetadataLimiter[key] = time.Now().Add(-time.Second)
+ after := poll()
+ entry := m.guestMetadataCache[key]
+ if tc.rateRefusal {
+ if after.OSName != bootstrapWindowsOS || after.OSVersion != "2025" || entry.osInfoSkip || entry.osInfoFailureCount != 0 || !entry.windowsGuest || entry.osInfoObservedAt.IsZero() || osCalls.Load() != int32(rounds+1) || linuxCalls.Load() != 0 {
+ t.Errorf("ordinary recovery stayed suppressed or sent Linux memory read: OS=%q version=%q calls=%d Linux=%d entry=%+v", after.OSName, after.OSVersion, osCalls.Load(), linuxCalls.Load(), entry)
+ }
+ } else if !entry.osInfoSkip || osCalls.Load() != int32(rounds) || after.OSName != original.osName {
+ t.Error("unrelated genuine repeated failure suppression was weakened by a quoted 429")
+ }
+ // The optional, safe source projection is byte-for-byte producer JSON,
+ // not a hand-authored browser replacement. No database/native acceptance.
+ if os.Getenv("PULSE_TEST_RATE_LIMIT_PROJECTION") == "1" && kind == "client" && collector == "cluster" && !seeded && tc.name == "rate-limit" {
+ data, err := json.Marshal(map[string]models.VM{"before": before, "after": after})
+ if err != nil {
+ t.Fatal(err)
+ }
+ t.Logf("RATE_LIMIT_PROJECTION %s", data)
+ }
+ })
+ }
+ }
+ }
+ }
+}