Real-time monitoring dashboard for Proxmox VE, PBS, Docker, Kubernetes, TrueNAS and vSphere. Self-hosted, with alerts for failed backups, full disks, unhealthy containers and offline nodes.
Go aialertsdashboarddockerdocker-monitoringgohomelabinfrastructure-monitoringkubernetesmonitoringmspproxmoxproxmox-backup-serverproxmox-mail-gatewayproxmox-veself-hostedtruenastypescriptvspherewebhooks
Defer guest commands after incomplete HTTP error responses
pulse-triage[bot]3c57c2d20728
diff --git a/pkg/proxmox/client.go b/pkg/proxmox/client.goindex 1b3dcee276..dec68ba461 100644--- a/pkg/proxmox/client.go+++ b/pkg/proxmox/client.go@@ -52,13 +52,20 @@ func classifyAPIErrorLog(status int, path string) (apiErrorLogLevel, string) {return apiErrorLogNone, ""}+// Preserve an incomplete body through the HTTP error-response wrapper too.+// An error status alone is not a completed refusal if its body was truncated.+type responseBodyReadError struct{ cause error }++func (e *responseBodyReadError) Error() string { return e.cause.Error() }+func (e *responseBodyReadError) Unwrap() error { return e.cause }+func readResponseBodyLimited(r io.Reader) ([]byte, error) {body, err := io.ReadAll(io.LimitReader(r, maxResponseBodyBytes+1))if err != nil {- return nil, err+ return nil, &responseBodyReadError{cause: err}}if int64(len(body)) > maxResponseBodyBytes {- return nil, fmt.Errorf("response body exceeds %d bytes", maxResponseBodyBytes)+ return nil, &responseBodyReadError{cause: fmt.Errorf("response body exceeds %d bytes", maxResponseBodyBytes)}}return body, nil}diff --git a/pkg/proxmox/guest_agent_guard.go b/pkg/proxmox/guest_agent_guard.goindex 1f8df97621..fce6be56b2 100644--- a/pkg/proxmox/guest_agent_guard.go+++ b/pkg/proxmox/guest_agent_guard.go@@ -176,6 +176,11 @@ func (c *Client) getGuestAgent(ctx context.Context, path, node string, vmid int)// own their usual session recovery; this guest admission is single-attempt.resp, err := c.requestWithRetry(ctx, http.MethodGet, path, nil, true)if err != nil {+ var incomplete *responseBodyReadError+ if errors.As(err, &incomplete) {+ uncertain = true+ return nil, &guestAgentDeferredError{reason: "agent-response-incomplete", cause: err}+ }// Explicit refusals/unsupported commands remain ordinary errors. An// uncertain completion blocks every command, not just this method.lower := strings.ToLower(err.Error())diff --git a/pkg/proxmox/guest_agent_guard_test.go b/pkg/proxmox/guest_agent_guard_test.goindex 62d77db187..7fcdc823d4 100644--- a/pkg/proxmox/guest_agent_guard_test.go+++ b/pkg/proxmox/guest_agent_guard_test.go@@ -101,6 +101,14 @@ func TestGuestAgentLockAppearingInFlightDiscardsPayload(t *testing.T) {}func TestGuestAgentIncompleteBodyBlocksWholeVM(t *testing.T) {+ for _, status := range []int{http.StatusOK, http.StatusInternalServerError} {+ t.Run(fmt.Sprintf("status-%d", status), func(t *testing.T) {+ testGuestAgentIncompleteBodyBlocksWholeVM(t, status)+ })+ }+}++func testGuestAgentIncompleteBodyBlocksWholeVM(t *testing.T, status int) {var calls atomic.Int32server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {if strings.HasSuffix(r.URL.Path, "/config") {@@ -109,12 +117,13 @@ func TestGuestAgentIncompleteBodyBlocksWholeVM(t *testing.T) {}calls.Add(1)w.Header().Set("Content-Length", "1000")+ w.WriteHeader(status)fmt.Fprint(w, `{"data":{"result":[]}}`)}))defer server.Close()c := backupTestClient(t, server.URL)if _, err := c.GetVMFSInfo(context.Background(), "node", 105); GuestAgentDeferredReason(err) != "agent-response-incomplete" {- t.Fatalf("incomplete body not deferred: %v", err)+ t.Errorf("incomplete body not deferred: %v", err)}for name, read := range backupAgentReads() {t.Run(name, func(t *testing.T) {@@ -128,6 +137,33 @@ func TestGuestAgentIncompleteBodyBlocksWholeVM(t *testing.T) {}}+func TestGuestAgentDeferralDoesNotHideCompletedRefusal(t *testing.T) {+ var calls atomic.Int32+ server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {+ if strings.HasSuffix(r.URL.Path, "/config") {+ fmt.Fprint(w, `{"data":{}}`)+ return+ }+ calls.Add(1)+ if strings.HasSuffix(r.URL.Path, "/get-fsinfo") {+ http.Error(w, "permission denied", http.StatusForbidden)+ return+ }+ backupAgentPayload(w, r)+ }))+ defer server.Close()+ c := backupTestClient(t, server.URL)+ if _, err := c.GetVMFSInfo(context.Background(), "node", 105); err == nil || errors.Is(err, ErrGuestAgentDeferred) || !strings.Contains(err.Error(), "403") {+ t.Fatalf("completed refusal hidden or lost: %v", err)+ }+ if _, err := c.GetVMAgentInfo(context.Background(), "node", 105); err != nil {+ t.Fatalf("completed refusal poisoned other healthy reads: %v", err)+ }+ if calls.Load() != 2 {+ t.Fatalf("refused/healthy command count = %d", calls.Load())+ }+}+func TestGuestAgentClusterDoesNotReplayUncertainCall(t *testing.T) {var calls atomic.Int32handler := http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {