Real-time monitoring dashboard for Proxmox VE, PBS, Docker, Kubernetes, TrueNAS and vSphere. Self-hosted, with alerts for failed backups, full disks, unhealthy containers and offline nodes.
Go aialertsdashboarddockerdocker-monitoringgohomelabinfrastructure-monitoringkubernetesmonitoringmspproxmoxproxmox-backup-serverproxmox-mail-gatewayproxmox-veself-hostedtruenastypescriptvspherewebhooks
Isolate guest HTTP fixtures from retained endpoint cooldowns
rcourtman667e98082e75
diff --git a/docs/release-control/v6/internal/records/guest-agent-http-fixture-isolation-2026-10-18.md b/docs/release-control/v6/internal/records/guest-agent-http-fixture-isolation-2026-10-18.mdnew file mode 100644index 0000000000..c225ba2871--- /dev/null+++ b/docs/release-control/v6/internal/records/guest-agent-http-fixture-isolation-2026-10-18.md@@ -0,0 +1,46 @@+# Guest-agent HTTP fixture isolation, 18 October 2026++This is a separately claimed L13 test-stabilization slice discovered while+checking the chart-read changes. It does not change runtime guest admission,+the monitoring lane's completion state or release qualification.++## Failure and correction++The required REST-0 check for chart head+`ea3d71a7990798c026a39480635a43ae1f3ea6c0` failed in+`TestGuestAgentHTTPFailureDefersEveryRead/unknown-client-error/os`.+Its first read returned `agent-cooldown` before sending any guest command,+where the fixture expected one command followed by completion uncertainty.+See [job 117775441898](https://github.com/rcourtman/Pulse/actions/runs/39161106967/job/117775441898).+The complete log is retained with the chart investigation evidence.++Runtime uncertainty fences intentionally outlive individual clients and+closed HTTP fixtures. The HTTP-status tests reused bare loopback endpoint+identities with the same VMID, so a recycled listener address could inherit+another fixture's retained cooldown. Lock-evidence tests already avoided+this through a per-case base-path namespace.++The existing helper is now shared between lock-evidence and HTTP-status+fixtures. All clients and declared cluster aliases within a case use the+same namespace; independent cases have separate identities. Existing+completion, VM-wide cooldown, no-failover, status/counter and recovery+assertions remain unchanged. No runtime guard is reset, expired or weakened+by the isolation mechanism.++## Verification++`TestGuestAgentHTTPFailureFixtureIsolation` gives two independent uncertain+HTTP cases one listener, then checks that both original fences still reject+every diagnostic guest command without another config or guest request.+The earlier entries must remain byte-for-byte equal. Replacing the helper+with the old bare-endpoint behaviour deterministically reproduced the same+pre-wire `agent-cooldown` failure. The expected failure log is retained as+`fixture-negative.log`; it is not a passing test or an idle-worker result.++With the namespaced helper restored, targeted HTTP/lock-evidence tests+passed in 0.779 seconds and the complete Proxmox package passed in 17.437+seconds. All three source hashes matched after the negative control and+the full suite. Raw output and `fixture-source.sha256` remain under+`/opt/pulse-release-worker/task-proofs/chart-latency-slo-20261018/`.+This establishes the fixture reuse mechanism and source correction, not+native guest-agent recovery or the chart's 90 ms performance claim.diff --git a/docs/release-control/v6/internal/subsystems/monitoring.md b/docs/release-control/v6/internal/subsystems/monitoring.mdindex 0b967c1326..4b625475ca 100644--- a/docs/release-control/v6/internal/subsystems/monitoring.md+++ b/docs/release-control/v6/internal/subsystems/monitoring.md@@ -88,6 +88,15 @@ an earlier synthetic guest's cooldown. `TestGuestAgentLockEvidenceFixtureIsolatishares one listener and retains an earlier uncertainty fence unchanged while anindependent fixture completes its lock/command checks. No runtime fence iscleared, expired or weakened; native guest recovery is not established.+HTTP-status fixtures share this same test-only endpoint namespace helper.+Independent cases keep distinct base-path identities even when a loopback+listener is reused; every diagnostic and cluster alias within a case retains+the same identity and VM-wide cooldown. `TestGuestAgentHTTPFailureFixtureIsolation`+uses one listener for two independent uncertain commands, then verifies both+unchanged fences still reject all follow-up guest reads without wire work.+The old bare-endpoint fixture behaviour fails that control. This is a separately+claimed test-stabilization correction; no runtime guard, cooldown or provider+completion rule is changed.This establishes source/consumer behaviour, not native recovery or attributionof a guest, PBS or certificate report.diff --git a/pkg/proxmox/guest_agent_backup_test.go b/pkg/proxmox/guest_agent_backup_test.goindex f4b1d3ff1a..123009211f 100644--- a/pkg/proxmox/guest_agent_backup_test.go+++ b/pkg/proxmox/guest_agent_backup_test.go@@ -2,6 +2,7 @@ package proxmoximport ("context"+ "crypto/sha256""fmt""net/http""net/http/httptest"@@ -21,6 +22,18 @@ func backupTestClient(t *testing.T, host string) *Client {return c}+// Shared guest fences intentionally outlive a closed fixture listener. A+// recycled loopback port must not make an independent guest fixture inherit an+// earlier case's VM-wide uncertainty. Every client within a case keeps its key.+func guestAgentFixtureEndpoint(t *testing.T, serverURL string) string {+ t.Helper()+ // Go appends #NN to repeated subtest names. Names are evidence labels,+ // not URL syntax; a digest keeps their full identity without fragments,+ // query parameters or ambiguous path separators.+ identity := sha256.Sum256([]byte(t.Name()))+ return serverURL + "/guest-agent-fixture/" + fmt.Sprintf("%x", identity)+}+func backupAgentReads() map[string]func(context.Context, *Client, int) error {return map[string]func(context.Context, *Client, int) error{"os": func(ctx context.Context, c *Client, id int) error {diff --git a/pkg/proxmox/guest_agent_http_status_test.go b/pkg/proxmox/guest_agent_http_status_test.goindex eda4066a06..d7617d5420 100644--- a/pkg/proxmox/guest_agent_http_status_test.go+++ b/pkg/proxmox/guest_agent_http_status_test.go@@ -56,13 +56,13 @@ func TestGuestAgentHTTPFailureDefersEveryRead(t *testing.T) {}}))defer server.Close()- c := backupTestClient(t, server.URL)+ c := backupTestClient(t, guestAgentFixtureEndpoint(t, server.URL))err := first(context.Background(), c, 105)if !errors.Is(err, ErrGuestAgentDeferred) || GuestAgentDeferredReason(err) != "agent-completion-unverified" {t.Errorf("unknown completion = %v, want completion-unverified deferral", err)}// A separately constructed diagnostic cannot follow with any command.- diagnostic := backupTestClient(t, server.URL)+ diagnostic := backupTestClient(t, guestAgentFixtureEndpoint(t, server.URL))for later, read := range backupAgentReads() {if err := read(context.Background(), diagnostic, 105); GuestAgentDeferredReason(err) != "agent-cooldown" {t.Errorf("next %s = %v, want VM-wide cooldown", later, err)@@ -104,7 +104,7 @@ func TestGuestAgentUnrecognisedHTTPStatusDefersEveryRead(t *testing.T) {}}))defer server.Close()- c := backupTestClient(t, server.URL)+ c := backupTestClient(t, guestAgentFixtureEndpoint(t, server.URL))err := first(context.Background(), c, 105)var response *apiResponseErrorif GuestAgentDeferredReason(err) != "agent-completion-unverified" || !errors.As(err, &response) || response.statusCode != status {@@ -112,7 +112,7 @@ func TestGuestAgentUnrecognisedHTTPStatusDefersEveryRead(t *testing.T) {}before := configs.Load()for later, read := range backupAgentReads() {- if err := read(context.Background(), backupTestClient(t, server.URL), 105); GuestAgentDeferredReason(err) != "agent-cooldown" {+ if err := read(context.Background(), backupTestClient(t, guestAgentFixtureEndpoint(t, server.URL)), 105); GuestAgentDeferredReason(err) != "agent-cooldown" {t.Errorf("next %s bypassed uncertainty: %v", later, err)}}@@ -154,14 +154,14 @@ func TestGuestAgentUnrecognisedHTTPStatusSharesAliasesAndResumes(t *testing.T) {defer a.Close()defer b.Close()defer independent.Close()- cc := NewClusterClient("unrecognised-http", ClientConfig{Host: a.URL, TokenName: "fixture@pve!pulse", TokenValue: "fixture"}, []string{a.URL, b.URL}, nil)+ cc := NewClusterClient("unrecognised-http", ClientConfig{Host: guestAgentFixtureEndpoint(t, a.URL), TokenName: "fixture@pve!pulse", TokenValue: "fixture"}, []string{guestAgentFixtureEndpoint(t, a.URL), guestAgentFixtureEndpoint(t, b.URL)}, nil)if _, err := cc.GetVMFSInfo(context.Background(), "node", 105); GuestAgentDeferredReason(err) != "agent-completion-unverified" {t.Errorf("cluster cleared unverified completion: %v", err)}if commands.Load() != 1 {t.Errorf("cluster failed over the guest command: %d", commands.Load())}- for _, endpoint := range []string{a.URL, b.URL} {+ for _, endpoint := range []string{guestAgentFixtureEndpoint(t, a.URL), guestAgentFixtureEndpoint(t, b.URL)} {for name, read := range backupAgentReads() {if err := read(context.Background(), backupTestClient(t, endpoint), 105); GuestAgentDeferredReason(err) != "agent-cooldown" {t.Errorf("alias %s bypassed uncertainty: %v", name, err)@@ -171,16 +171,16 @@ func TestGuestAgentUnrecognisedHTTPStatusSharesAliasesAndResumes(t *testing.T) {if commands.Load() != 1 {t.Errorf("aliases sent extra commands: %d", commands.Load())}- if _, err := backupTestClient(t, b.URL).GetVMAgentInfo(context.Background(), "node", 106); err != nil {+ if _, err := backupTestClient(t, guestAgentFixtureEndpoint(t, b.URL)).GetVMAgentInfo(context.Background(), "node", 106); err != nil {t.Errorf("independent VM was blocked: %v", err)}- if _, err := backupTestClient(t, independent.URL).GetVMAgentInfo(context.Background(), "node", 105); GuestAgentDeferredReason(err) != "agent-completion-unverified" {+ if _, err := backupTestClient(t, guestAgentFixtureEndpoint(t, independent.URL)).GetVMAgentInfo(context.Background(), "node", 105); GuestAgentDeferredReason(err) != "agent-completion-unverified" {t.Errorf("independent endpoint inherited another endpoint's cooldown: %v", err)}// Expire only this fixture's existing cooldown. This is not a// native recovery or a replay of an uncertain provider command.guestAgentGuards.Lock()- for _, endpoint := range []string{a.URL, b.URL} {+ for _, endpoint := range []string{guestAgentFixtureEndpoint(t, a.URL), guestAgentFixtureEndpoint(t, b.URL)} {key := guestAgentGuardKey{endpoint: endpoint + "/api2/json", vmid: 105}entry := guestAgentGuards.entries[key]entry.until = time.Now().Add(-time.Second)@@ -189,7 +189,7 @@ func TestGuestAgentUnrecognisedHTTPStatusSharesAliasesAndResumes(t *testing.T) {guestAgentGuards.Unlock()healthy.Store(true)for name, read := range backupAgentReads() {- if err := read(context.Background(), backupTestClient(t, b.URL), 105); err != nil {+ if err := read(context.Background(), backupTestClient(t, guestAgentFixtureEndpoint(t, b.URL)), 105); err != nil {t.Errorf("known healthy admission did not resume %s: %v", name, err)}}@@ -222,11 +222,11 @@ func TestGuestAgentHTTPFailureDoesNotFailOver(t *testing.T) {a, b := httptest.NewServer(handler), httptest.NewServer(handler)defer a.Close()defer b.Close()- cc := NewClusterClient("http-uncertainty", ClientConfig{Host: a.URL, TokenName: "fixture@pve!pulse", TokenValue: "fixture"}, []string{a.URL, b.URL}, nil)+ cc := NewClusterClient("http-uncertainty", ClientConfig{Host: guestAgentFixtureEndpoint(t, a.URL), TokenName: "fixture@pve!pulse", TokenValue: "fixture"}, []string{guestAgentFixtureEndpoint(t, a.URL), guestAgentFixtureEndpoint(t, b.URL)}, nil)if _, err := cc.GetVMFSInfo(context.Background(), "node", 105); GuestAgentDeferredReason(err) != "agent-completion-unverified" {t.Errorf("first command not deferred: %v", err)}- for _, endpoint := range []string{a.URL, b.URL} {+ for _, endpoint := range []string{guestAgentFixtureEndpoint(t, a.URL), guestAgentFixtureEndpoint(t, b.URL)} {if _, err := backupTestClient(t, endpoint).GetVMAgentInfo(context.Background(), "migrated", 105); GuestAgentDeferredReason(err) != "agent-cooldown" {t.Errorf("alias escaped cooldown: %v", err)}@@ -277,7 +277,7 @@ func TestGuestAgentHTTPExplicitRefusalsRemainErrors(t *testing.T) {backupAgentPayload(w, r)}))defer server.Close()- c := backupTestClient(t, server.URL)+ c := backupTestClient(t, guestAgentFixtureEndpoint(t, server.URL))if _, err := c.GetVMAgentInfo(context.Background(), "node", 105); err == nil || errors.Is(err, ErrGuestAgentDeferred) || !strings.Contains(err.Error(), fmt.Sprintf("API error %d", tc.status)) {t.Fatalf("explicit refusal lost: %v", err)}@@ -322,7 +322,7 @@ func TestGuestAgentHTTPAmbiguousFailuresDoNotClearAdmission(t *testing.T) {fmt.Fprint(w, tc.body)}))defer server.Close()- c := backupTestClient(t, server.URL)+ c := backupTestClient(t, guestAgentFixtureEndpoint(t, server.URL))if _, err := c.GetVMFSInfo(context.Background(), "node", 105); GuestAgentDeferredReason(err) != "agent-completion-unverified" {t.Errorf("ambiguous error released admission: %v", err)}@@ -361,7 +361,7 @@ func TestGuestAgentHTTPTerminalFailuresRemainCommandBound(t *testing.T) {backupAgentPayload(w, r)}))defer server.Close()- c := backupTestClient(t, server.URL)+ c := backupTestClient(t, guestAgentFixtureEndpoint(t, server.URL))if _, err := c.get(context.Background(), "/nodes/node/qemu/105/agent/"+tc.command); err == nil || errors.Is(err, ErrGuestAgentDeferred) {t.Fatalf("known terminal failure lost: %v", err)}@@ -381,7 +381,7 @@ func TestGuestAgentHTTPOrdinaryRequestPreservesStatusAndError(t *testing.T) {fmt.Fprint(w, "API error 403: permission denied")}))defer server.Close()- c := backupTestClient(t, server.URL)+ c := backupTestClient(t, guestAgentFixtureEndpoint(t, server.URL))_, err := c.get(context.Background(), "/nodes")var response *apiResponseErrorif !errors.As(err, &response) || response.statusCode != 502 || response.guestCommandRejected || err.Error() != "API error 502: API error 403: permission denied" {@@ -407,7 +407,7 @@ func TestGuestAgentHTTPUncertaintyExpiresAndKeepsOtherVMsIndependent(t *testing.backupAgentPayload(w, r)}))defer server.Close()- c := backupTestClient(t, server.URL)+ c := backupTestClient(t, guestAgentFixtureEndpoint(t, server.URL))if _, err := c.GetVMFSInfo(context.Background(), "node", 105); GuestAgentDeferredReason(err) != "agent-completion-unverified" {t.Fatal(err)}@@ -442,3 +442,57 @@ func TestGuestAgentHTTPUncertaintyExpiresAndKeepsOtherVMsIndependent(t *testing.t.Errorf("wire commands = %d, want uncertain/other/resumed", calls.Load())}}++// Sharing one listener makes endpoint reuse deterministic. Each independent+// fixture must dispatch its first command while every earlier uncertainty+// fence remains intact for that fixture's diagnostic clients.+func TestGuestAgentHTTPFailureFixtureIsolation(t *testing.T) {+ var configs, commands atomic.Int32+ server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {+ if strings.HasSuffix(r.URL.Path, "/config") {+ configs.Add(1)+ fmt.Fprint(w, `{"data":{}}`)+ return+ }+ commands.Add(1)+ w.WriteHeader(499)+ fmt.Fprint(w, "upstream unavailable")+ }))+ defer server.Close()+ retained := make(map[guestAgentGuardKey]guestAgentGuardEntry)+ var endpoints []string+ for _, name := range []string{"previous", "independent"} {+ t.Run(name, func(t *testing.T) {+ endpoint := guestAgentFixtureEndpoint(t, server.URL)+ endpoints = append(endpoints, endpoint)+ client := backupTestClient(t, endpoint)+ if _, err := client.GetVMAgentInfo(context.Background(), "node", 105); GuestAgentDeferredReason(err) != "agent-completion-unverified" {+ t.Fatalf("independent HTTP fixture inherited earlier uncertainty: %v", err)+ }+ key, valid := guestAgentEndpointKey(client.baseURL)+ if !valid {+ t.Fatal("invalid fixture endpoint")+ }+ guestAgentGuards.Lock()+ retained[guestAgentGuardKey{endpoint: key, vmid: 105}] = guestAgentGuards.entries[guestAgentGuardKey{endpoint: key, vmid: 105}]+ guestAgentGuards.Unlock()+ })+ }+ for _, endpoint := range endpoints {+ for name, read := range backupAgentReads() {+ if err := read(context.Background(), backupTestClient(t, endpoint), 105); GuestAgentDeferredReason(err) != "agent-cooldown" {+ t.Errorf("diagnostic %s bypassed retained fixture uncertainty: %v", name, err)+ }+ }+ }+ guestAgentGuards.Lock()+ for key, want := range retained {+ if guestAgentGuards.entries[key] != want {+ t.Errorf("fixture isolation changed an existing uncertainty fence")+ }+ }+ guestAgentGuards.Unlock()+ if configs.Load() != 2 || commands.Load() != 2 {+ t.Fatalf("independent fixtures dispatched configs=%d commands=%d, want 2/2", configs.Load(), commands.Load())+ }+}diff --git a/pkg/proxmox/guest_agent_lock_evidence_test.go b/pkg/proxmox/guest_agent_lock_evidence_test.goindex 39ede0fb06..fd97ab8ad8 100644--- a/pkg/proxmox/guest_agent_lock_evidence_test.go+++ b/pkg/proxmox/guest_agent_lock_evidence_test.go@@ -2,7 +2,6 @@ package proxmoximport ("context"- "crypto/sha256""fmt""net/http""net/http/httptest"@@ -55,7 +54,7 @@ func TestGuestAgentLockEvidenceRejectsAmbiguity(t *testing.T) {backupAgentPayload(w, r)}))defer server.Close()- c := backupTestClient(t, lockEvidenceFixtureEndpoint(t, server.URL))+ c := backupTestClient(t, guestAgentFixtureEndpoint(t, server.URL))if err := read(context.Background(), c, 105); GuestAgentDeferredReason(err) != "lock-unverified" {t.Errorf("ambiguous lock admitted/published a command: %v", err)}@@ -69,7 +68,7 @@ func TestGuestAgentLockEvidenceRejectsAmbiguity(t *testing.T) {healthy.Store(true)// A completed command's failed lock check is not transport// uncertainty. A new diagnostic must reverify and resume.- diagnostic := backupTestClient(t, lockEvidenceFixtureEndpoint(t, server.URL))+ diagnostic := backupTestClient(t, guestAgentFixtureEndpoint(t, server.URL))if err := read(context.Background(), diagnostic, 105); err != nil {t.Fatalf("complete unlocked config did not resume: %v", err)}@@ -117,7 +116,7 @@ func TestGuestAgentLockEvidenceRejectsRedirects(t *testing.T) {}}))defer server.Close()- c := backupTestClient(t, lockEvidenceFixtureEndpoint(t, server.URL))+ c := backupTestClient(t, guestAgentFixtureEndpoint(t, server.URL))if err := read(context.Background(), c, 105); GuestAgentDeferredReason(err) != "lock-unverified" {t.Errorf("redirected lock admitted/published a command: %v", err)}@@ -146,7 +145,7 @@ func TestGuestAgentLockEvidenceRejectsRedirects(t *testing.T) {t.Errorf("ordinary redirected requests = %d, want %d", redirected.Load(), wantRedirected)}healthy.Store(true)- if err := read(context.Background(), backupTestClient(t, lockEvidenceFixtureEndpoint(t, server.URL)), 105); err != nil {+ if err := read(context.Background(), backupTestClient(t, guestAgentFixtureEndpoint(t, server.URL)), 105); err != nil {t.Fatalf("valid same-target config did not resume: %v", err)}if commands.Load() != want+1 || redirected.Load() != wantRedirected {@@ -187,7 +186,7 @@ func TestGuestAgentLockEvidenceHealthyConfigAndSessionRecovery(t *testing.T) {}}))defer server.Close()- c, err := NewClient(ClientConfig{Host: lockEvidenceFixtureEndpoint(t, server.URL), User: "fixture@pam", Password: "fixture-password", Timeout: time.Second})+ c, err := NewClient(ClientConfig{Host: guestAgentFixtureEndpoint(t, server.URL), User: "fixture@pam", Password: "fixture-password", Timeout: time.Second})if err != nil {t.Fatal(err)}@@ -201,18 +200,6 @@ func TestGuestAgentLockEvidenceHealthyConfigAndSessionRecovery(t *testing.T) {}}-// Shared guest fences intentionally outlive a closed fixture listener. A-// recycled loopback port must not make an independent lock case inherit an-// earlier case's VM-wide uncertainty. Every client within a case keeps its key.-func lockEvidenceFixtureEndpoint(t *testing.T, serverURL string) string {- t.Helper()- // Go appends #NN to repeated subtest names. Names are evidence labels,- // not URL syntax; a digest keeps their full identity without fragments,- // query parameters or ambiguous path separators.- identity := sha256.Sum256([]byte(t.Name()))- return serverURL + "/lock-evidence-fixture/" + fmt.Sprintf("%x", identity)-}-func TestGuestAgentLockEvidenceFixtureIsolation(t *testing.T) {var configs, commands atomic.Int32server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {@@ -239,7 +226,7 @@ func TestGuestAgentLockEvidenceFixtureIsolation(t *testing.T) {guestAgentGuards.Lock()retained := guestAgentGuards.entries[guestAgentGuardKey{endpoint: key, vmid: 105}]guestAgentGuards.Unlock()- current := backupTestClient(t, lockEvidenceFixtureEndpoint(t, server.URL))+ current := backupTestClient(t, guestAgentFixtureEndpoint(t, server.URL))if _, err := current.GetVMAgentInfo(context.Background(), "node", 105); err != nil {t.Fatalf("independent lock fixture inherited earlier uncertainty: %v", err)}