Real-time monitoring dashboard for Proxmox VE, PBS, Docker, Kubernetes, TrueNAS and vSphere. Self-hosted, with alerts for failed backups, full disks, unhealthy containers and offline nodes.
Go aialertsdashboarddockerdocker-monitoringgohomelabinfrastructure-monitoringkubernetesmonitoringmspproxmoxproxmox-backup-serverproxmox-mail-gatewayproxmox-veself-hostedtruenastypescriptvspherewebhooks
Defer guest commands after unrecognised HTTP client errors
pulse-triage[bot]d16885bc35f4
diff --git a/docs/release-control/v6/internal/subsystems/monitoring.md b/docs/release-control/v6/internal/subsystems/monitoring.mdindex 7595b797d8..9c9c141c28 100644--- a/docs/release-control/v6/internal/subsystems/monitoring.md+++ b/docs/release-control/v6/internal/subsystems/monitoring.md@@ -573,8 +573,12 @@ consistency, not native payload-to-command provenance, thaw, cross-processcoordination or restart safety. The native recovery precaution below remains.The completed-error boundary uses the actual HTTP status retained by the request-layer, not an `API error` phrase in provider or proxy text. HTTP 408 and-unexplained server/proxy failures (including PVE 595) retain the same per-VM+layer, not an `API error` phrase in provider or proxy text. The supported+explicit request/access/command refusals (HTTP 400, 401, 403, 404, 405, 422 and+429) remain errors, never successful observations. Other client-error statuses,+including conflict/early-request responses and non-standard client-disconnect+statuses, cannot establish command completion merely by belonging to HTTP 4xx.+HTTP 408 and unexplained server/proxy failures (including PVE 595) retain the same per-VMuncertainty cooldown, even when their bodies finish: command completion is stillunverified. Another method, diagnostic client or configured cluster alias cannotimmediately send a command. Only complete, exact command-bound terminal QGA@@ -591,6 +595,14 @@ independent status counters and cross-client cooldown; cluster and semanticcontrols pin no failover, definitive refusals and conservative error provenance.The connected transport/History lifecycle and counter-receipt controls alsocover complete server/gateway failures without refreshing old guest evidence.+`TestGuestAgentUnrecognisedHTTPStatusDefersEveryRead` and its alias/resumption+control cover fourteen unrecognised 4xx statuses across all six readers, fixed+wire-status preservation, no follow-up command/config work, independent VM and+endpoint admission, normal post-cooldown admission and unchanged endpoint health.+`TestGuestAgentUnrecognisedHTTPStatusContract` connects conflict, early-request+and client-disconnect responses to polling, last-known disk/memory continuity,+unrenewed metadata/memory caches and History while live CPU advances. These+synthetic checks are not a diagnosis of a frozen guest or native recovery.Guest commands also prevent transport-level replay: each uses a fresh single-useHTTP/1 connection, with no pooled-connection or HTTP/2-stream retries and nodiff --git a/internal/monitoring/canonical_guardrails_test.go b/internal/monitoring/canonical_guardrails_test.goindex 089e4b4bc1..cb46cbecef 100644--- a/internal/monitoring/canonical_guardrails_test.go+++ b/internal/monitoring/canonical_guardrails_test.go@@ -3251,6 +3251,10 @@ func TestDeferredVMGuestMemoryRequiresOriginalEvidence(t *testing.T) {}}+func TestGuestAgentUnrecognisedHTTPStatusContract(t *testing.T) {+ testGuestAgentTransportDeferralKeepsLastKnownHistory(t, "conflict response", "too early response", "client closed response")+}+func TestGuestAgentTransportMonitoringContract(t *testing.T) {for _, reason := range []string{"agent-redirect", "agent-transport-unverified", "agent-completion-unverified"} {for _, prefix := range []string{"", "prev-"} {diff --git a/internal/monitoring/guest_agent_transport_integration_test.go b/internal/monitoring/guest_agent_transport_integration_test.goindex 8b0766228d..b0b1296dbf 100644--- a/internal/monitoring/guest_agent_transport_integration_test.go+++ b/internal/monitoring/guest_agent_transport_integration_test.go@@ -46,11 +46,13 @@ func testGuestAgentTransportDeferralKeepsLastKnownHistory(t *testing.T, kinds ..fmt.Fprint(w, `{"data":{"result":[]}} {}`)return}- if kind == "server error" || kind == "gateway error" {- status := http.StatusInternalServerError- if kind == "gateway error" {- status = http.StatusBadGateway- }+ if status := map[string]int{+ "server error": http.StatusInternalServerError,+ "gateway error": http.StatusBadGateway,+ "conflict response": http.StatusConflict,+ "too early response": http.StatusTooEarly,+ "client closed response": 499,+ }[kind]; status != 0 {w.WriteHeader(status)fmt.Fprint(w, "upstream unavailable")returndiff --git a/pkg/proxmox/guest_agent_guard.go b/pkg/proxmox/guest_agent_guard.goindex 5ba2f2c60c..2a5abe14cf 100644--- a/pkg/proxmox/guest_agent_guard.go+++ b/pkg/proxmox/guest_agent_guard.go@@ -238,8 +238,7 @@ func (c *Client) getGuestAgent(ctx context.Context, path, node string, vmid int)uncertain = truereturn nil, &guestAgentDeferredError{reason: "agent-timeout", cause: err}}- if response.statusCode == http.StatusRequestTimeout ||- (response.statusCode >= 500 && !response.guestCommandRejected) {+ if !guestAgentHTTPFailureCompleted(response) {uncertain = truereturn nil, &guestAgentDeferredError{reason: "agent-completion-unverified", cause: err}}@@ -275,6 +274,23 @@ func (c *Client) getGuestAgent(ctx context.Context, path, node string, vmid int)return resp, nil}+// A client-error class alone is not completion evidence: an intermediary can+// report a conflict, early request or client disconnect after accepting work.+// Keep the supported explicit request/access/command rejections as errors, but+// fence every other status just like an unexplained server/gateway failure.+func guestAgentHTTPFailureCompleted(response *apiResponseError) bool {+ switch response.statusCode {+ case http.StatusBadRequest, http.StatusUnauthorized, http.StatusForbidden,+ http.StatusNotFound, http.StatusMethodNotAllowed, http.StatusUnprocessableEntity,+ http.StatusTooManyRequests:+ return true+ case http.StatusInternalServerError:+ return response.guestCommandRejected+ default:+ return false+ }+}+// Guest calls do not fail over or replay: another HTTP worker still addresses// the same serial QGA channel. Transport recovery belongs to ordinary API reads.func (cc *ClusterClient) executeGuestAgent(ctx context.Context, fn func(*Client) error) error {diff --git a/pkg/proxmox/guest_agent_http_status_test.go b/pkg/proxmox/guest_agent_http_status_test.goindex 04ec3e859c..eda4066a06 100644--- a/pkg/proxmox/guest_agent_http_status_test.go+++ b/pkg/proxmox/guest_agent_http_status_test.go@@ -31,6 +31,9 @@ func TestGuestAgentHTTPFailureDefersEveryRead(t *testing.T) {{"gateway-quoting-refusal", 502, "upstream API error 403: permission denied"},{"gateway-quoting-unsupported", 502, "unsupported command: guest-get-osinfo"},{"server-quoting-refusal", 500, "upstream API error 403: permission denied"},+ {"unknown-client-error", 499, "upstream unavailable"},+ {"client-error-quoting-refusal", 409, "upstream API error 403: permission denied"},+ {"client-error-quoting-unsupported", 425, "unsupported command: guest-get-osinfo"},}for _, tc := range cases {t.Run(tc.name, func(t *testing.T) {@@ -78,6 +81,130 @@ func TestGuestAgentHTTPFailureDefersEveryRead(t *testing.T) {}}+func TestGuestAgentUnrecognisedHTTPStatusDefersEveryRead(t *testing.T) {+ for _, status := range []int{402, 407, 409, 410, 412, 418, 421, 423, 425, 426, 428, 431, 451, 499} {+ t.Run(fmt.Sprintf("status-%d", status), func(t *testing.T) {+ for name, first := range backupAgentReads() {+ t.Run(name, func(t *testing.T) {+ var commands, configs atomic.Int32+ server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {+ switch {+ case strings.HasSuffix(r.URL.Path, "/config"):+ configs.Add(1)+ fmt.Fprint(w, `{"data":{}}`)+ case strings.HasSuffix(r.URL.Path, "/status/current"):+ fmt.Fprint(w, `{"data":{"status":"running","cpu":0.25,"diskread":4096}}`)+ default:+ if commands.Add(1) == 1 {+ w.WriteHeader(status)+ fmt.Fprint(w, "upstream unavailable")+ return+ }+ backupAgentPayload(w, r)+ }+ }))+ defer server.Close()+ c := backupTestClient(t, server.URL)+ err := first(context.Background(), c, 105)+ var response *apiResponseError+ if GuestAgentDeferredReason(err) != "agent-completion-unverified" || !errors.As(err, &response) || response.statusCode != status {+ t.Errorf("unrecognised status lost its wire evidence or cleared admission: %v", err)+ }+ before := configs.Load()+ for later, read := range backupAgentReads() {+ if err := read(context.Background(), backupTestClient(t, server.URL), 105); GuestAgentDeferredReason(err) != "agent-cooldown" {+ t.Errorf("next %s bypassed uncertainty: %v", later, err)+ }+ }+ if commands.Load() != 1 || configs.Load() != before {+ t.Errorf("uncertainty dispatched follow-up work: commands=%d configs=%d, want 1/%d", commands.Load(), configs.Load(), before)+ }+ status, err := c.GetVMStatus(context.Background(), "node", 105)+ if err != nil || status.CPU != .25 || status.DiskRead != 4096 {+ t.Errorf("ordinary non-QGA reads changed: %+v %v", status, err)+ }+ })+ }+ })+ }+}++func TestGuestAgentUnrecognisedHTTPStatusSharesAliasesAndResumes(t *testing.T) {+ for _, status := range []int{409, 425, 499} {+ t.Run(fmt.Sprintf("status-%d", status), func(t *testing.T) {+ var commands atomic.Int32+ var healthy atomic.Bool+ handler := http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {+ switch {+ case strings.HasSuffix(r.URL.Path, "/nodes"):+ fmt.Fprint(w, `{"data":[]}`)+ case strings.HasSuffix(r.URL.Path, "/config"):+ fmt.Fprint(w, `{"data":{}}`)+ default:+ commands.Add(1)+ if strings.Contains(r.URL.Path, "/qemu/105/") && !healthy.Load() {+ w.WriteHeader(status)+ fmt.Fprint(w, "upstream unavailable")+ return+ }+ backupAgentPayload(w, r)+ }+ })+ a, b, independent := httptest.NewServer(handler), httptest.NewServer(handler), httptest.NewServer(handler)+ defer a.Close()+ defer b.Close()+ defer independent.Close()+ cc := NewClusterClient("unrecognised-http", ClientConfig{Host: a.URL, TokenName: "fixture@pve!pulse", TokenValue: "fixture"}, []string{a.URL, b.URL}, nil)+ if _, err := cc.GetVMFSInfo(context.Background(), "node", 105); GuestAgentDeferredReason(err) != "agent-completion-unverified" {+ t.Errorf("cluster cleared unverified completion: %v", err)+ }+ if commands.Load() != 1 {+ t.Errorf("cluster failed over the guest command: %d", commands.Load())+ }+ for _, endpoint := range []string{a.URL, b.URL} {+ for name, read := range backupAgentReads() {+ if err := read(context.Background(), backupTestClient(t, endpoint), 105); GuestAgentDeferredReason(err) != "agent-cooldown" {+ t.Errorf("alias %s bypassed uncertainty: %v", name, err)+ }+ }+ }+ if commands.Load() != 1 {+ t.Errorf("aliases sent extra commands: %d", commands.Load())+ }+ if _, err := backupTestClient(t, b.URL).GetVMAgentInfo(context.Background(), "node", 106); err != nil {+ t.Errorf("independent VM was blocked: %v", err)+ }+ if _, err := backupTestClient(t, independent.URL).GetVMAgentInfo(context.Background(), "node", 105); GuestAgentDeferredReason(err) != "agent-completion-unverified" {+ t.Errorf("independent endpoint inherited another endpoint's cooldown: %v", err)+ }+ // Expire only this fixture's existing cooldown. This is not a+ // native recovery or a replay of an uncertain provider command.+ guestAgentGuards.Lock()+ for _, endpoint := range []string{a.URL, b.URL} {+ key := guestAgentGuardKey{endpoint: endpoint + "/api2/json", vmid: 105}+ entry := guestAgentGuards.entries[key]+ entry.until = time.Now().Add(-time.Second)+ guestAgentGuards.entries[key] = entry+ }+ guestAgentGuards.Unlock()+ healthy.Store(true)+ for name, read := range backupAgentReads() {+ if err := read(context.Background(), backupTestClient(t, b.URL), 105); err != nil {+ t.Errorf("known healthy admission did not resume %s: %v", name, err)+ }+ }+ if commands.Load() != 9 {+ t.Errorf("uncertain/independent/resumed commands=%d, want 9", commands.Load())+ }+ for endpoint, healthy := range cc.GetHealthStatus() {+ if !healthy {+ t.Errorf("guest error poisoned endpoint health: %s", endpoint)+ }+ }+ })+ }+}+func TestGuestAgentHTTPFailureDoesNotFailOver(t *testing.T) {var calls atomic.Int32handler := http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {@@ -124,6 +251,8 @@ func TestGuestAgentHTTPExplicitRefusalsRemainErrors(t *testing.T) {{"unauthorized", 401, "unauthorized"},{"forbidden", 403, "permission denied"},{"missing-command", 404, "command not found"},+ {"method-not-allowed", 405, "method not allowed"},+ {"invalid-parameters", 422, "invalid parameters"},{"rate-limited", 429, "too many requests"},{"unsupported-plain", 500, "unsupported command: guest-get-osinfo"},{"unsupported-message", 500, `{"data":null,"message":"unsupported command: guest-get-osinfo"}`},