Skip to content

armgit / Pulse

Public Imported from GitHub
0

Real-time monitoring dashboard for Proxmox VE, PBS, Docker, Kubernetes, TrueNAS and vSphere. Self-hosted, with alerts for failed backups, full disks, unhealthy containers and offline nodes.

Reclaim expired guest-command admission before capacity deferral

pulse-triage[bot]ef222a7c7b77
diff --git a/docs/release-control/v6/internal/subsystems/monitoring.md b/docs/release-control/v6/internal/subsystems/monitoring.md
index ced11b1f90..969da5d259 100644
--- a/docs/release-control/v6/internal/subsystems/monitoring.md
+++ b/docs/release-control/v6/internal/subsystems/monitoring.md
@@ -318,6 +318,22 @@ required; all responses in this fixture are synthetic.
### Guest-agent coordination around backups — issue #2439
+The process-local admission table remains capped at 4,096 endpoint/VMID
+entries. Expired entries replaced by a new admission consume no extra slot;
+when new aliases would exceed the bound, expired idle entries are reclaimed
+before returning `agent-capacity`, even before the periodic cleanup deadline.
+In-flight commands and unexpired uncertainty cooldowns are never evicted,
+shortened or renewed to make space. An alias group is reserved atomically or
+not at all; full protected capacity still defers before any HTTP request.
+
+`pkg/proxmox/guest_agent_capacity_test.go` covers near-limit expired alias
+replacement, immediate expiry reclamation through all six HTTP readers,
+backup-lock rejection and ordinary resumption, plus fully protected capacity
+and incomplete group admission. These are synthetic source controls, not
+native recovery or a diagnosis of a reported shared pause. Existing lock,
+no-replay, cooldown, endpoint/VMID isolation and restart limitations remain.
+
+
Filesystem-query diagnostics preserve this safety boundary. Missing, empty,
unsupported or refused readings do not prescribe installing, activating or
restarting QGA, changing backup settings, re-running setup or broadening a
diff --git a/pkg/proxmox/guest_agent_capacity_test.go b/pkg/proxmox/guest_agent_capacity_test.go
new file mode 100644
index 0000000000..fc09a1bdc7
--- /dev/null
+++ b/pkg/proxmox/guest_agent_capacity_test.go
@@ -0,0 +1,178 @@
+package proxmox
+
+import (
+ "context"
+ "fmt"
+ "net/http"
+ "net/http/httptest"
+ "reflect"
+ "strings"
+ "sync/atomic"
+ "testing"
+ "time"
+)
+
+// These tests are serial, as are the other guest-guard tests. Restore the
+// process-wide state before the package's parallel HTTP tests can start.
+func isolateGuestAgentCapacity(t *testing.T) {
+ t.Helper()
+ guestAgentGuards.Lock()
+ aliases, entries, next := guestAgentGuards.aliases, guestAgentGuards.entries, guestAgentGuards.nextCleanup
+ guestAgentGuards.aliases = make(map[string][]string)
+ guestAgentGuards.entries = make(map[guestAgentGuardKey]guestAgentGuardEntry)
+ guestAgentGuards.nextCleanup = time.Now().Add(time.Hour)
+ guestAgentGuards.Unlock()
+ t.Cleanup(func() {
+ guestAgentGuards.Lock()
+ guestAgentGuards.aliases, guestAgentGuards.entries, guestAgentGuards.nextCleanup = aliases, entries, next
+ guestAgentGuards.Unlock()
+ })
+}
+
+// Half the protected entries are busy; half retain uncertain completion.
+func seedProtectedGuestAgentCapacity(count int) map[guestAgentGuardKey]guestAgentGuardEntry {
+ protected := make(map[guestAgentGuardKey]guestAgentGuardEntry, count)
+ until := time.Now().Add(time.Hour)
+ guestAgentGuards.Lock()
+ defer guestAgentGuards.Unlock()
+ for i := 0; i < count; i++ {
+ key := guestAgentGuardKey{endpoint: "https://protected.capacity.invalid/api2/json", vmid: 1000 + i}
+ entry := guestAgentGuardEntry{busy: i%2 == 0, until: until}
+ protected[key] = entry
+ guestAgentGuards.entries[key] = entry
+ }
+ return protected
+}
+
+func assertProtectedGuestAgentCapacity(t *testing.T, protected map[guestAgentGuardKey]guestAgentGuardEntry) {
+ t.Helper()
+ guestAgentGuards.Lock()
+ defer guestAgentGuards.Unlock()
+ if len(guestAgentGuards.entries) > maxGuestAgentGuardEntries {
+ t.Errorf("guard bound exceeded: %d", len(guestAgentGuards.entries))
+ }
+ for key, want := range protected {
+ if got, ok := guestAgentGuards.entries[key]; !ok || got != want {
+ t.Fatalf("protected busy/cooldown entry evicted or renewed: %+v", key)
+ }
+ }
+}
+
+func TestGuestAgentCapacityReusesExpiredAliasEntries(t *testing.T) {
+ for _, expiredAliases := range []int{1, 2} {
+ t.Run(fmt.Sprintf("expired-%d", expiredAliases), func(t *testing.T) {
+ isolateGuestAgentCapacity(t)
+ first, second := "https://first.capacity.invalid/api2/json", "https://second.capacity.invalid/api2/json"
+ registerGuestAgentEndpoints("https://first.capacity.invalid", []string{"https://second.capacity.invalid"})
+ protected := seedProtectedGuestAgentCapacity(maxGuestAgentGuardEntries - 1 - expiredAliases)
+ guestAgentGuards.Lock()
+ for _, endpoint := range []string{first, second}[:expiredAliases] {
+ guestAgentGuards.entries[guestAgentGuardKey{endpoint: endpoint, vmid: 105}] = guestAgentGuardEntry{until: time.Now().Add(-time.Second)}
+ }
+ guestAgentGuards.Unlock()
+ release, err := acquireGuestAgent(first, 105)
+ if err != nil {
+ t.Fatalf("expired alias was charged twice against capacity: %v", err)
+ }
+ if next, err := acquireGuestAgent(second, 105); GuestAgentDeferredReason(err) != "agent-busy" {
+ if next != nil {
+ next(false)
+ }
+ t.Errorf("reused capacity lost shared admission: %v", err)
+ }
+ assertProtectedGuestAgentCapacity(t, protected)
+ release(true)
+ if next, err := acquireGuestAgent(second, 105); GuestAgentDeferredReason(err) != "agent-cooldown" {
+ if next != nil {
+ next(false)
+ }
+ t.Errorf("reused capacity lost uncertainty: %v", err)
+ }
+ assertProtectedGuestAgentCapacity(t, protected)
+ })
+ }
+}
+
+func TestGuestAgentCapacityReclaimsExpiredBeforeEveryHTTPRead(t *testing.T) {
+ for name, read := range backupAgentReads() {
+ t.Run(name, func(t *testing.T) {
+ isolateGuestAgentCapacity(t)
+ var commands, configs atomic.Int32
+ var locked atomic.Bool
+ server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+ if strings.HasSuffix(r.URL.Path, "/config") {
+ configs.Add(1)
+ if locked.Load() {
+ fmt.Fprint(w, `{"data":{"lock":"backup"}}`)
+ } else {
+ fmt.Fprint(w, `{"data":{}}`)
+ }
+ return
+ }
+ commands.Add(1)
+ backupAgentPayload(w, r)
+ }))
+ defer server.Close()
+ client := backupTestClient(t, server.URL)
+ registerGuestAgentEndpoints(server.URL, []string{"https://alias.capacity.invalid"})
+ protected := seedProtectedGuestAgentCapacity(maxGuestAgentGuardEntries - 2)
+ guestAgentGuards.Lock()
+ guestAgentGuards.entries[guestAgentGuardKey{endpoint: "https://expired.capacity.invalid/api2/json", vmid: 999}] = guestAgentGuardEntry{until: time.Now().Add(-time.Second)}
+ guestAgentGuards.Unlock()
+ if err := read(context.Background(), client, 105); err != nil {
+ t.Fatalf("eligible read postponed until periodic cleanup: %v", err)
+ }
+ if commands.Load() != 1 || configs.Load() != 2 {
+ t.Fatalf("single command and both lock checks required: commands=%d configs=%d", commands.Load(), configs.Load())
+ }
+ assertProtectedGuestAgentCapacity(t, protected)
+ locked.Store(true)
+ if err := read(context.Background(), client, 105); GuestAgentDeferredReason(err) != "vm-locked" {
+ t.Errorf("capacity recovery bypassed backup admission: %v", err)
+ }
+ if commands.Load() != 1 || configs.Load() != 3 {
+ t.Fatal("backup-locked read sent another command")
+ }
+ locked.Store(false)
+ if err := read(context.Background(), client, 105); err != nil {
+ t.Errorf("known-unlocked guest did not resume: %v", err)
+ }
+ if commands.Load() != 2 || configs.Load() != 5 {
+ t.Fatal("resumption did not retain single-attempt/both-lock checks")
+ }
+ assertProtectedGuestAgentCapacity(t, protected)
+ })
+ }
+}
+
+func TestGuestAgentCapacityNeverEvictsProtectedWork(t *testing.T) {
+ for _, slotsLeft := range []int{0, 1} {
+ t.Run(fmt.Sprintf("slots-%d", slotsLeft), func(t *testing.T) {
+ isolateGuestAgentCapacity(t)
+ var requests atomic.Int32
+ server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+ requests.Add(1)
+ backupAgentPayload(w, r)
+ }))
+ defer server.Close()
+ client := backupTestClient(t, server.URL)
+ registerGuestAgentEndpoints(server.URL, []string{"https://alias.capacity.invalid"})
+ protected := seedProtectedGuestAgentCapacity(maxGuestAgentGuardEntries - slotsLeft)
+ for name, read := range backupAgentReads() {
+ if err := read(context.Background(), client, 105); GuestAgentDeferredReason(err) != "agent-capacity" {
+ t.Errorf("%s evicted protected work instead of deferring: %v", name, err)
+ }
+ }
+ if requests.Load() != 0 {
+ t.Fatal("over-capacity guest sent config or agent request")
+ }
+ guestAgentGuards.Lock()
+ unchanged := reflect.DeepEqual(guestAgentGuards.entries, protected)
+ guestAgentGuards.Unlock()
+ if !unchanged {
+ t.Fatal("failed admission left partial alias reservations or changed protected entries")
+ }
+ assertProtectedGuestAgentCapacity(t, protected)
+ })
+ }
+}
diff --git a/pkg/proxmox/guest_agent_guard.go b/pkg/proxmox/guest_agent_guard.go
index 7451b02e04..5ba2f2c60c 100644
--- a/pkg/proxmox/guest_agent_guard.go
+++ b/pkg/proxmox/guest_agent_guard.go
@@ -94,7 +94,7 @@ func acquireGuestAgent(endpoint string, vmid int) (func(bool), error) {
guestAgentGuards.Lock()
defer guestAgentGuards.Unlock()
now := time.Now()
- if !now.Before(guestAgentGuards.nextCleanup) || len(guestAgentGuards.entries) >= maxGuestAgentGuardEntries {
+ cleanup := func() {
for key, entry := range guestAgentGuards.entries {
if !entry.busy && !now.Before(entry.until) {
delete(guestAgentGuards.entries, key)
@@ -102,6 +102,9 @@ func acquireGuestAgent(endpoint string, vmid int) (func(bool), error) {
}
guestAgentGuards.nextCleanup = now.Add(time.Minute)
}
+ if !now.Before(guestAgentGuards.nextCleanup) {
+ cleanup()
+ }
endpoints := guestAgentGuards.aliases[endpoint]
if len(endpoints) == 0 {
endpoints = []string{endpoint}
@@ -118,8 +121,24 @@ func acquireGuestAgent(endpoint string, vmid int) (func(bool), error) {
}
keys = append(keys, key)
}
- if len(guestAgentGuards.entries)+len(keys) > maxGuestAgentGuardEntries {
- return nil, &guestAgentDeferredError{reason: "agent-capacity"}
+ // Replacing an expired entry does not consume another slot. If new keys
+ // would exceed the bound, reclaim expired entries now rather than defer
+ // an otherwise eligible guest until the next periodic sweep. Never evict
+ // in-flight work or an unexpired uncertainty fence to make room.
+ additional := func() int {
+ count := 0
+ for _, key := range keys {
+ if _, exists := guestAgentGuards.entries[key]; !exists {
+ count++
+ }
+ }
+ return count
+ }
+ if len(guestAgentGuards.entries)+additional() > maxGuestAgentGuardEntries {
+ cleanup()
+ if len(guestAgentGuards.entries)+additional() > maxGuestAgentGuardEntries {
+ return nil, &guestAgentDeferredError{reason: "agent-capacity"}
+ }
}
for _, key := range keys {
guestAgentGuards.entries[key] = guestAgentGuardEntry{busy: true}