Skip to content

armgit / Pulse

Public Imported from GitHub
0

Real-time monitoring dashboard for Proxmox VE, PBS, Docker, Kubernetes, TrueNAS and vSphere. Self-hosted, with alerts for failed backups, full disks, unhealthy containers and offline nodes.

Defer guest commands after unrecognised HTTP client errors

pulse-triage[bot]d16885bc35f4
diff --git a/docs/release-control/v6/internal/subsystems/monitoring.md b/docs/release-control/v6/internal/subsystems/monitoring.md
index 7595b797d8..9c9c141c28 100644
--- a/docs/release-control/v6/internal/subsystems/monitoring.md
+++ b/docs/release-control/v6/internal/subsystems/monitoring.md
@@ -573,8 +573,12 @@ consistency, not native payload-to-command provenance, thaw, cross-process
coordination or restart safety. The native recovery precaution below remains.
The completed-error boundary uses the actual HTTP status retained by the request
-layer, not an `API error` phrase in provider or proxy text. HTTP 408 and
-unexplained server/proxy failures (including PVE 595) retain the same per-VM
+layer, not an `API error` phrase in provider or proxy text. The supported
+explicit request/access/command refusals (HTTP 400, 401, 403, 404, 405, 422 and
+429) remain errors, never successful observations. Other client-error statuses,
+including conflict/early-request responses and non-standard client-disconnect
+statuses, cannot establish command completion merely by belonging to HTTP 4xx.
+HTTP 408 and unexplained server/proxy failures (including PVE 595) retain the same per-VM
uncertainty cooldown, even when their bodies finish: command completion is still
unverified. Another method, diagnostic client or configured cluster alias cannot
immediately send a command. Only complete, exact command-bound terminal QGA
@@ -591,6 +595,14 @@ independent status counters and cross-client cooldown; cluster and semantic
controls pin no failover, definitive refusals and conservative error provenance.
The connected transport/History lifecycle and counter-receipt controls also
cover complete server/gateway failures without refreshing old guest evidence.
+`TestGuestAgentUnrecognisedHTTPStatusDefersEveryRead` and its alias/resumption
+control cover fourteen unrecognised 4xx statuses across all six readers, fixed
+wire-status preservation, no follow-up command/config work, independent VM and
+endpoint admission, normal post-cooldown admission and unchanged endpoint health.
+`TestGuestAgentUnrecognisedHTTPStatusContract` connects conflict, early-request
+and client-disconnect responses to polling, last-known disk/memory continuity,
+unrenewed metadata/memory caches and History while live CPU advances. These
+synthetic checks are not a diagnosis of a frozen guest or native recovery.
Guest commands also prevent transport-level replay: each uses a fresh single-use
HTTP/1 connection, with no pooled-connection or HTTP/2-stream retries and no
diff --git a/internal/monitoring/canonical_guardrails_test.go b/internal/monitoring/canonical_guardrails_test.go
index 089e4b4bc1..cb46cbecef 100644
--- a/internal/monitoring/canonical_guardrails_test.go
+++ b/internal/monitoring/canonical_guardrails_test.go
@@ -3251,6 +3251,10 @@ func TestDeferredVMGuestMemoryRequiresOriginalEvidence(t *testing.T) {
}
}
+func TestGuestAgentUnrecognisedHTTPStatusContract(t *testing.T) {
+ testGuestAgentTransportDeferralKeepsLastKnownHistory(t, "conflict response", "too early response", "client closed response")
+}
+
func TestGuestAgentTransportMonitoringContract(t *testing.T) {
for _, reason := range []string{"agent-redirect", "agent-transport-unverified", "agent-completion-unverified"} {
for _, prefix := range []string{"", "prev-"} {
diff --git a/internal/monitoring/guest_agent_transport_integration_test.go b/internal/monitoring/guest_agent_transport_integration_test.go
index 8b0766228d..b0b1296dbf 100644
--- a/internal/monitoring/guest_agent_transport_integration_test.go
+++ b/internal/monitoring/guest_agent_transport_integration_test.go
@@ -46,11 +46,13 @@ func testGuestAgentTransportDeferralKeepsLastKnownHistory(t *testing.T, kinds ..
fmt.Fprint(w, `{"data":{"result":[]}} {}`)
return
}
- if kind == "server error" || kind == "gateway error" {
- status := http.StatusInternalServerError
- if kind == "gateway error" {
- status = http.StatusBadGateway
- }
+ if status := map[string]int{
+ "server error": http.StatusInternalServerError,
+ "gateway error": http.StatusBadGateway,
+ "conflict response": http.StatusConflict,
+ "too early response": http.StatusTooEarly,
+ "client closed response": 499,
+ }[kind]; status != 0 {
w.WriteHeader(status)
fmt.Fprint(w, "upstream unavailable")
return
diff --git a/pkg/proxmox/guest_agent_guard.go b/pkg/proxmox/guest_agent_guard.go
index 5ba2f2c60c..2a5abe14cf 100644
--- a/pkg/proxmox/guest_agent_guard.go
+++ b/pkg/proxmox/guest_agent_guard.go
@@ -238,8 +238,7 @@ func (c *Client) getGuestAgent(ctx context.Context, path, node string, vmid int)
uncertain = true
return nil, &guestAgentDeferredError{reason: "agent-timeout", cause: err}
}
- if response.statusCode == http.StatusRequestTimeout ||
- (response.statusCode >= 500 && !response.guestCommandRejected) {
+ if !guestAgentHTTPFailureCompleted(response) {
uncertain = true
return nil, &guestAgentDeferredError{reason: "agent-completion-unverified", cause: err}
}
@@ -275,6 +274,23 @@ func (c *Client) getGuestAgent(ctx context.Context, path, node string, vmid int)
return resp, nil
}
+// A client-error class alone is not completion evidence: an intermediary can
+// report a conflict, early request or client disconnect after accepting work.
+// Keep the supported explicit request/access/command rejections as errors, but
+// fence every other status just like an unexplained server/gateway failure.
+func guestAgentHTTPFailureCompleted(response *apiResponseError) bool {
+ switch response.statusCode {
+ case http.StatusBadRequest, http.StatusUnauthorized, http.StatusForbidden,
+ http.StatusNotFound, http.StatusMethodNotAllowed, http.StatusUnprocessableEntity,
+ http.StatusTooManyRequests:
+ return true
+ case http.StatusInternalServerError:
+ return response.guestCommandRejected
+ default:
+ return false
+ }
+}
+
// Guest calls do not fail over or replay: another HTTP worker still addresses
// the same serial QGA channel. Transport recovery belongs to ordinary API reads.
func (cc *ClusterClient) executeGuestAgent(ctx context.Context, fn func(*Client) error) error {
diff --git a/pkg/proxmox/guest_agent_http_status_test.go b/pkg/proxmox/guest_agent_http_status_test.go
index 04ec3e859c..eda4066a06 100644
--- a/pkg/proxmox/guest_agent_http_status_test.go
+++ b/pkg/proxmox/guest_agent_http_status_test.go
@@ -31,6 +31,9 @@ func TestGuestAgentHTTPFailureDefersEveryRead(t *testing.T) {
{"gateway-quoting-refusal", 502, "upstream API error 403: permission denied"},
{"gateway-quoting-unsupported", 502, "unsupported command: guest-get-osinfo"},
{"server-quoting-refusal", 500, "upstream API error 403: permission denied"},
+ {"unknown-client-error", 499, "upstream unavailable"},
+ {"client-error-quoting-refusal", 409, "upstream API error 403: permission denied"},
+ {"client-error-quoting-unsupported", 425, "unsupported command: guest-get-osinfo"},
}
for _, tc := range cases {
t.Run(tc.name, func(t *testing.T) {
@@ -78,6 +81,130 @@ func TestGuestAgentHTTPFailureDefersEveryRead(t *testing.T) {
}
}
+func TestGuestAgentUnrecognisedHTTPStatusDefersEveryRead(t *testing.T) {
+ for _, status := range []int{402, 407, 409, 410, 412, 418, 421, 423, 425, 426, 428, 431, 451, 499} {
+ t.Run(fmt.Sprintf("status-%d", status), func(t *testing.T) {
+ for name, first := range backupAgentReads() {
+ t.Run(name, func(t *testing.T) {
+ var commands, configs atomic.Int32
+ server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+ switch {
+ case strings.HasSuffix(r.URL.Path, "/config"):
+ configs.Add(1)
+ fmt.Fprint(w, `{"data":{}}`)
+ case strings.HasSuffix(r.URL.Path, "/status/current"):
+ fmt.Fprint(w, `{"data":{"status":"running","cpu":0.25,"diskread":4096}}`)
+ default:
+ if commands.Add(1) == 1 {
+ w.WriteHeader(status)
+ fmt.Fprint(w, "upstream unavailable")
+ return
+ }
+ backupAgentPayload(w, r)
+ }
+ }))
+ defer server.Close()
+ c := backupTestClient(t, server.URL)
+ err := first(context.Background(), c, 105)
+ var response *apiResponseError
+ if GuestAgentDeferredReason(err) != "agent-completion-unverified" || !errors.As(err, &response) || response.statusCode != status {
+ t.Errorf("unrecognised status lost its wire evidence or cleared admission: %v", err)
+ }
+ before := configs.Load()
+ for later, read := range backupAgentReads() {
+ if err := read(context.Background(), backupTestClient(t, server.URL), 105); GuestAgentDeferredReason(err) != "agent-cooldown" {
+ t.Errorf("next %s bypassed uncertainty: %v", later, err)
+ }
+ }
+ if commands.Load() != 1 || configs.Load() != before {
+ t.Errorf("uncertainty dispatched follow-up work: commands=%d configs=%d, want 1/%d", commands.Load(), configs.Load(), before)
+ }
+ status, err := c.GetVMStatus(context.Background(), "node", 105)
+ if err != nil || status.CPU != .25 || status.DiskRead != 4096 {
+ t.Errorf("ordinary non-QGA reads changed: %+v %v", status, err)
+ }
+ })
+ }
+ })
+ }
+}
+
+func TestGuestAgentUnrecognisedHTTPStatusSharesAliasesAndResumes(t *testing.T) {
+ for _, status := range []int{409, 425, 499} {
+ t.Run(fmt.Sprintf("status-%d", status), func(t *testing.T) {
+ var commands atomic.Int32
+ var healthy atomic.Bool
+ handler := http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+ switch {
+ case strings.HasSuffix(r.URL.Path, "/nodes"):
+ fmt.Fprint(w, `{"data":[]}`)
+ case strings.HasSuffix(r.URL.Path, "/config"):
+ fmt.Fprint(w, `{"data":{}}`)
+ default:
+ commands.Add(1)
+ if strings.Contains(r.URL.Path, "/qemu/105/") && !healthy.Load() {
+ w.WriteHeader(status)
+ fmt.Fprint(w, "upstream unavailable")
+ return
+ }
+ backupAgentPayload(w, r)
+ }
+ })
+ a, b, independent := httptest.NewServer(handler), httptest.NewServer(handler), httptest.NewServer(handler)
+ defer a.Close()
+ defer b.Close()
+ defer independent.Close()
+ cc := NewClusterClient("unrecognised-http", ClientConfig{Host: a.URL, TokenName: "fixture@pve!pulse", TokenValue: "fixture"}, []string{a.URL, b.URL}, nil)
+ if _, err := cc.GetVMFSInfo(context.Background(), "node", 105); GuestAgentDeferredReason(err) != "agent-completion-unverified" {
+ t.Errorf("cluster cleared unverified completion: %v", err)
+ }
+ if commands.Load() != 1 {
+ t.Errorf("cluster failed over the guest command: %d", commands.Load())
+ }
+ for _, endpoint := range []string{a.URL, b.URL} {
+ for name, read := range backupAgentReads() {
+ if err := read(context.Background(), backupTestClient(t, endpoint), 105); GuestAgentDeferredReason(err) != "agent-cooldown" {
+ t.Errorf("alias %s bypassed uncertainty: %v", name, err)
+ }
+ }
+ }
+ if commands.Load() != 1 {
+ t.Errorf("aliases sent extra commands: %d", commands.Load())
+ }
+ if _, err := backupTestClient(t, b.URL).GetVMAgentInfo(context.Background(), "node", 106); err != nil {
+ t.Errorf("independent VM was blocked: %v", err)
+ }
+ if _, err := backupTestClient(t, independent.URL).GetVMAgentInfo(context.Background(), "node", 105); GuestAgentDeferredReason(err) != "agent-completion-unverified" {
+ t.Errorf("independent endpoint inherited another endpoint's cooldown: %v", err)
+ }
+ // Expire only this fixture's existing cooldown. This is not a
+ // native recovery or a replay of an uncertain provider command.
+ guestAgentGuards.Lock()
+ for _, endpoint := range []string{a.URL, b.URL} {
+ key := guestAgentGuardKey{endpoint: endpoint + "/api2/json", vmid: 105}
+ entry := guestAgentGuards.entries[key]
+ entry.until = time.Now().Add(-time.Second)
+ guestAgentGuards.entries[key] = entry
+ }
+ guestAgentGuards.Unlock()
+ healthy.Store(true)
+ for name, read := range backupAgentReads() {
+ if err := read(context.Background(), backupTestClient(t, b.URL), 105); err != nil {
+ t.Errorf("known healthy admission did not resume %s: %v", name, err)
+ }
+ }
+ if commands.Load() != 9 {
+ t.Errorf("uncertain/independent/resumed commands=%d, want 9", commands.Load())
+ }
+ for endpoint, healthy := range cc.GetHealthStatus() {
+ if !healthy {
+ t.Errorf("guest error poisoned endpoint health: %s", endpoint)
+ }
+ }
+ })
+ }
+}
+
func TestGuestAgentHTTPFailureDoesNotFailOver(t *testing.T) {
var calls atomic.Int32
handler := http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
@@ -124,6 +251,8 @@ func TestGuestAgentHTTPExplicitRefusalsRemainErrors(t *testing.T) {
{"unauthorized", 401, "unauthorized"},
{"forbidden", 403, "permission denied"},
{"missing-command", 404, "command not found"},
+ {"method-not-allowed", 405, "method not allowed"},
+ {"invalid-parameters", 422, "invalid parameters"},
{"rate-limited", 429, "too many requests"},
{"unsupported-plain", 500, "unsupported command: guest-get-osinfo"},
{"unsupported-message", 500, `{"data":null,"message":"unsupported command: guest-get-osinfo"}`},