Real-time monitoring dashboard for Proxmox VE, PBS, Docker, Kubernetes, TrueNAS and vSphere. Self-hosted, with alerts for failed backups, full disks, unhealthy containers and offline nodes.
Go aialertsdashboarddockerdocker-monitoringgohomelabinfrastructure-monitoringkubernetesmonitoringmspproxmoxproxmox-backup-serverproxmox-mail-gatewayproxmox-veself-hostedtruenastypescriptvspherewebhooks
Reclaim expired guest-command admission before capacity deferral
pulse-triage[bot]ef222a7c7b77
diff --git a/docs/release-control/v6/internal/subsystems/monitoring.md b/docs/release-control/v6/internal/subsystems/monitoring.mdindex ced11b1f90..969da5d259 100644--- a/docs/release-control/v6/internal/subsystems/monitoring.md+++ b/docs/release-control/v6/internal/subsystems/monitoring.md@@ -318,6 +318,22 @@ required; all responses in this fixture are synthetic.### Guest-agent coordination around backups — issue #2439+The process-local admission table remains capped at 4,096 endpoint/VMID+entries. Expired entries replaced by a new admission consume no extra slot;+when new aliases would exceed the bound, expired idle entries are reclaimed+before returning `agent-capacity`, even before the periodic cleanup deadline.+In-flight commands and unexpired uncertainty cooldowns are never evicted,+shortened or renewed to make space. An alias group is reserved atomically or+not at all; full protected capacity still defers before any HTTP request.++`pkg/proxmox/guest_agent_capacity_test.go` covers near-limit expired alias+replacement, immediate expiry reclamation through all six HTTP readers,+backup-lock rejection and ordinary resumption, plus fully protected capacity+and incomplete group admission. These are synthetic source controls, not+native recovery or a diagnosis of a reported shared pause. Existing lock,+no-replay, cooldown, endpoint/VMID isolation and restart limitations remain.++Filesystem-query diagnostics preserve this safety boundary. Missing, empty,unsupported or refused readings do not prescribe installing, activating orrestarting QGA, changing backup settings, re-running setup or broadening adiff --git a/pkg/proxmox/guest_agent_capacity_test.go b/pkg/proxmox/guest_agent_capacity_test.gonew file mode 100644index 0000000000..fc09a1bdc7--- /dev/null+++ b/pkg/proxmox/guest_agent_capacity_test.go@@ -0,0 +1,178 @@+package proxmox++import (+ "context"+ "fmt"+ "net/http"+ "net/http/httptest"+ "reflect"+ "strings"+ "sync/atomic"+ "testing"+ "time"+)++// These tests are serial, as are the other guest-guard tests. Restore the+// process-wide state before the package's parallel HTTP tests can start.+func isolateGuestAgentCapacity(t *testing.T) {+ t.Helper()+ guestAgentGuards.Lock()+ aliases, entries, next := guestAgentGuards.aliases, guestAgentGuards.entries, guestAgentGuards.nextCleanup+ guestAgentGuards.aliases = make(map[string][]string)+ guestAgentGuards.entries = make(map[guestAgentGuardKey]guestAgentGuardEntry)+ guestAgentGuards.nextCleanup = time.Now().Add(time.Hour)+ guestAgentGuards.Unlock()+ t.Cleanup(func() {+ guestAgentGuards.Lock()+ guestAgentGuards.aliases, guestAgentGuards.entries, guestAgentGuards.nextCleanup = aliases, entries, next+ guestAgentGuards.Unlock()+ })+}++// Half the protected entries are busy; half retain uncertain completion.+func seedProtectedGuestAgentCapacity(count int) map[guestAgentGuardKey]guestAgentGuardEntry {+ protected := make(map[guestAgentGuardKey]guestAgentGuardEntry, count)+ until := time.Now().Add(time.Hour)+ guestAgentGuards.Lock()+ defer guestAgentGuards.Unlock()+ for i := 0; i < count; i++ {+ key := guestAgentGuardKey{endpoint: "https://protected.capacity.invalid/api2/json", vmid: 1000 + i}+ entry := guestAgentGuardEntry{busy: i%2 == 0, until: until}+ protected[key] = entry+ guestAgentGuards.entries[key] = entry+ }+ return protected+}++func assertProtectedGuestAgentCapacity(t *testing.T, protected map[guestAgentGuardKey]guestAgentGuardEntry) {+ t.Helper()+ guestAgentGuards.Lock()+ defer guestAgentGuards.Unlock()+ if len(guestAgentGuards.entries) > maxGuestAgentGuardEntries {+ t.Errorf("guard bound exceeded: %d", len(guestAgentGuards.entries))+ }+ for key, want := range protected {+ if got, ok := guestAgentGuards.entries[key]; !ok || got != want {+ t.Fatalf("protected busy/cooldown entry evicted or renewed: %+v", key)+ }+ }+}++func TestGuestAgentCapacityReusesExpiredAliasEntries(t *testing.T) {+ for _, expiredAliases := range []int{1, 2} {+ t.Run(fmt.Sprintf("expired-%d", expiredAliases), func(t *testing.T) {+ isolateGuestAgentCapacity(t)+ first, second := "https://first.capacity.invalid/api2/json", "https://second.capacity.invalid/api2/json"+ registerGuestAgentEndpoints("https://first.capacity.invalid", []string{"https://second.capacity.invalid"})+ protected := seedProtectedGuestAgentCapacity(maxGuestAgentGuardEntries - 1 - expiredAliases)+ guestAgentGuards.Lock()+ for _, endpoint := range []string{first, second}[:expiredAliases] {+ guestAgentGuards.entries[guestAgentGuardKey{endpoint: endpoint, vmid: 105}] = guestAgentGuardEntry{until: time.Now().Add(-time.Second)}+ }+ guestAgentGuards.Unlock()+ release, err := acquireGuestAgent(first, 105)+ if err != nil {+ t.Fatalf("expired alias was charged twice against capacity: %v", err)+ }+ if next, err := acquireGuestAgent(second, 105); GuestAgentDeferredReason(err) != "agent-busy" {+ if next != nil {+ next(false)+ }+ t.Errorf("reused capacity lost shared admission: %v", err)+ }+ assertProtectedGuestAgentCapacity(t, protected)+ release(true)+ if next, err := acquireGuestAgent(second, 105); GuestAgentDeferredReason(err) != "agent-cooldown" {+ if next != nil {+ next(false)+ }+ t.Errorf("reused capacity lost uncertainty: %v", err)+ }+ assertProtectedGuestAgentCapacity(t, protected)+ })+ }+}++func TestGuestAgentCapacityReclaimsExpiredBeforeEveryHTTPRead(t *testing.T) {+ for name, read := range backupAgentReads() {+ t.Run(name, func(t *testing.T) {+ isolateGuestAgentCapacity(t)+ var commands, configs atomic.Int32+ var locked atomic.Bool+ server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {+ if strings.HasSuffix(r.URL.Path, "/config") {+ configs.Add(1)+ if locked.Load() {+ fmt.Fprint(w, `{"data":{"lock":"backup"}}`)+ } else {+ fmt.Fprint(w, `{"data":{}}`)+ }+ return+ }+ commands.Add(1)+ backupAgentPayload(w, r)+ }))+ defer server.Close()+ client := backupTestClient(t, server.URL)+ registerGuestAgentEndpoints(server.URL, []string{"https://alias.capacity.invalid"})+ protected := seedProtectedGuestAgentCapacity(maxGuestAgentGuardEntries - 2)+ guestAgentGuards.Lock()+ guestAgentGuards.entries[guestAgentGuardKey{endpoint: "https://expired.capacity.invalid/api2/json", vmid: 999}] = guestAgentGuardEntry{until: time.Now().Add(-time.Second)}+ guestAgentGuards.Unlock()+ if err := read(context.Background(), client, 105); err != nil {+ t.Fatalf("eligible read postponed until periodic cleanup: %v", err)+ }+ if commands.Load() != 1 || configs.Load() != 2 {+ t.Fatalf("single command and both lock checks required: commands=%d configs=%d", commands.Load(), configs.Load())+ }+ assertProtectedGuestAgentCapacity(t, protected)+ locked.Store(true)+ if err := read(context.Background(), client, 105); GuestAgentDeferredReason(err) != "vm-locked" {+ t.Errorf("capacity recovery bypassed backup admission: %v", err)+ }+ if commands.Load() != 1 || configs.Load() != 3 {+ t.Fatal("backup-locked read sent another command")+ }+ locked.Store(false)+ if err := read(context.Background(), client, 105); err != nil {+ t.Errorf("known-unlocked guest did not resume: %v", err)+ }+ if commands.Load() != 2 || configs.Load() != 5 {+ t.Fatal("resumption did not retain single-attempt/both-lock checks")+ }+ assertProtectedGuestAgentCapacity(t, protected)+ })+ }+}++func TestGuestAgentCapacityNeverEvictsProtectedWork(t *testing.T) {+ for _, slotsLeft := range []int{0, 1} {+ t.Run(fmt.Sprintf("slots-%d", slotsLeft), func(t *testing.T) {+ isolateGuestAgentCapacity(t)+ var requests atomic.Int32+ server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {+ requests.Add(1)+ backupAgentPayload(w, r)+ }))+ defer server.Close()+ client := backupTestClient(t, server.URL)+ registerGuestAgentEndpoints(server.URL, []string{"https://alias.capacity.invalid"})+ protected := seedProtectedGuestAgentCapacity(maxGuestAgentGuardEntries - slotsLeft)+ for name, read := range backupAgentReads() {+ if err := read(context.Background(), client, 105); GuestAgentDeferredReason(err) != "agent-capacity" {+ t.Errorf("%s evicted protected work instead of deferring: %v", name, err)+ }+ }+ if requests.Load() != 0 {+ t.Fatal("over-capacity guest sent config or agent request")+ }+ guestAgentGuards.Lock()+ unchanged := reflect.DeepEqual(guestAgentGuards.entries, protected)+ guestAgentGuards.Unlock()+ if !unchanged {+ t.Fatal("failed admission left partial alias reservations or changed protected entries")+ }+ assertProtectedGuestAgentCapacity(t, protected)+ })+ }+}diff --git a/pkg/proxmox/guest_agent_guard.go b/pkg/proxmox/guest_agent_guard.goindex 7451b02e04..5ba2f2c60c 100644--- a/pkg/proxmox/guest_agent_guard.go+++ b/pkg/proxmox/guest_agent_guard.go@@ -94,7 +94,7 @@ func acquireGuestAgent(endpoint string, vmid int) (func(bool), error) {guestAgentGuards.Lock()defer guestAgentGuards.Unlock()now := time.Now()- if !now.Before(guestAgentGuards.nextCleanup) || len(guestAgentGuards.entries) >= maxGuestAgentGuardEntries {+ cleanup := func() {for key, entry := range guestAgentGuards.entries {if !entry.busy && !now.Before(entry.until) {delete(guestAgentGuards.entries, key)@@ -102,6 +102,9 @@ func acquireGuestAgent(endpoint string, vmid int) (func(bool), error) {}guestAgentGuards.nextCleanup = now.Add(time.Minute)}+ if !now.Before(guestAgentGuards.nextCleanup) {+ cleanup()+ }endpoints := guestAgentGuards.aliases[endpoint]if len(endpoints) == 0 {endpoints = []string{endpoint}@@ -118,8 +121,24 @@ func acquireGuestAgent(endpoint string, vmid int) (func(bool), error) {}keys = append(keys, key)}- if len(guestAgentGuards.entries)+len(keys) > maxGuestAgentGuardEntries {- return nil, &guestAgentDeferredError{reason: "agent-capacity"}+ // Replacing an expired entry does not consume another slot. If new keys+ // would exceed the bound, reclaim expired entries now rather than defer+ // an otherwise eligible guest until the next periodic sweep. Never evict+ // in-flight work or an unexpired uncertainty fence to make room.+ additional := func() int {+ count := 0+ for _, key := range keys {+ if _, exists := guestAgentGuards.entries[key]; !exists {+ count+++ }+ }+ return count+ }+ if len(guestAgentGuards.entries)+additional() > maxGuestAgentGuardEntries {+ cleanup()+ if len(guestAgentGuards.entries)+additional() > maxGuestAgentGuardEntries {+ return nil, &guestAgentDeferredError{reason: "agent-capacity"}+ }}for _, key := range keys {guestAgentGuards.entries[key] = guestAgentGuardEntry{busy: true}