Real-time monitoring dashboard for Proxmox VE, PBS, Docker, Kubernetes, TrueNAS and vSphere. Self-hosted, with alerts for failed backups, full disks, unhealthy containers and offline nodes.
Go aialertsdashboarddockerdocker-monitoringgohomelabinfrastructure-monitoringkubernetesmonitoringmspproxmoxproxmox-backup-serverproxmox-mail-gatewayproxmox-veself-hostedtruenastypescriptvspherewebhooks
Defer guest reads after unverified successful replies
pulse-triage[bot]caceb1d26695
diff --git a/docs/release-control/v6/internal/subsystems/monitoring.md b/docs/release-control/v6/internal/subsystems/monitoring.mdindex 414f4dff78..6f835e2ac5 100644--- a/docs/release-control/v6/internal/subsystems/monitoring.md+++ b/docs/release-control/v6/internal/subsystems/monitoring.md@@ -296,6 +296,29 @@ replay through another cluster endpoint. Coordination retains at most 4,096active/cooling endpoint-guest entries and expires cooldowns. Explicit APIpermission refusals and unsupported commands remain errors, not successes.+A successful guest reply also needs direct HTTP 200 and exactly one complete+UTF-8 JSON envelope with one canonical `data` dictionary. Malformed, missing,+null/scalar data, extra values, duplicate/escaped-duplicate envelope or data+fields, competing error envelopes, or differently cased `data`, `result`,+`content` or `truncated` fields cannot establish a completed reply. The check+runs while admission is held, before method decoding or postflight lock reads;+failure retains `agent-completion-unverified` and the existing one-minute+VM-wide uncertainty pause. Other methods, fresh clients and configured cluster+aliases cannot immediately follow it with QGA work. Unexpected 2xx statuses+(including accepted/no-content/partial responses) have the same boundary.++Valid empty dictionaries, null/empty results, object-style rows, Unicode and+trailing whitespace retain existing method semantics, including partial row+admission and known truncated-file rejection. Ordinary API reads, permissions,+budgets, trust and backup settings are unchanged. The six-reader wire controls+pin no immediate redispatch, independent live counters, alias/VM/source isolation+and fresh admission after cooldown. `TestGuestAgentSuccessEnvelopeMonitoringContract`+connects malformed, duplicate and prefix-only replies to real polling builders:+original metadata/memory timestamps and guest History are not renewed, labelled+last-known values remain and current CPU advances. These checks establish reply+consistency, not native payload-to-command provenance, thaw, cross-process+coordination or restart safety. The native recovery precaution below remains.+The completed-error boundary uses the actual HTTP status retained by the requestlayer, not an `API error` phrase in provider or proxy text. HTTP 408 andunexplained server/proxy failures (including PVE 595) retain the same per-VMdiff --git a/internal/monitoring/canonical_guardrails_test.go b/internal/monitoring/canonical_guardrails_test.goindex b240785511..e8ee4ae41e 100644--- a/internal/monitoring/canonical_guardrails_test.go+++ b/internal/monitoring/canonical_guardrails_test.go@@ -3084,7 +3084,14 @@ func TestGuestAgentTransportMonitoringContract(t *testing.T) {t.Errorf("completed/ordinary error %q became transport uncertainty", reason)}}- testGuestAgentTransportDeferralKeepsLastKnownHistory(t)+ testGuestAgentTransportDeferralKeepsLastKnownHistory(t, "lost reply", "redirect", "server error", "gateway error")+}++func TestGuestAgentSuccessEnvelopeMonitoringContract(t *testing.T) {+ // A complete HTTP 200 is not enough to renew guest caches or History when+ // it only supplies a malformed, ambiguous or prefix-only reply. These+ // real client/builder controls also keep independent CPU progressing.+ testGuestAgentTransportDeferralKeepsLastKnownHistory(t, "malformed success", "ambiguous success", "trailing success")}func TestBackupAlertEvaluationCallersShareFailureVisibility(t *testing.T) {diff --git a/internal/monitoring/guest_agent_transport_integration_test.go b/internal/monitoring/guest_agent_transport_integration_test.goindex 30da065b05..8b0766228d 100644--- a/internal/monitoring/guest_agent_transport_integration_test.go+++ b/internal/monitoring/guest_agent_transport_integration_test.go@@ -17,8 +17,8 @@ import ("github.com/rcourtman/pulse-go-rewrite/pkg/proxmox")-func testGuestAgentTransportDeferralKeepsLastKnownHistory(t *testing.T) {- for _, kind := range []string{"lost reply", "redirect", "server error", "gateway error"} {+func testGuestAgentTransportDeferralKeepsLastKnownHistory(t *testing.T, kinds ...string) {+ for _, kind := range kinds {t.Run(kind, func(t *testing.T) {const mib = uint64(1024 * 1024)var phase, calls, redirected atomic.Int32@@ -35,6 +35,17 @@ func testGuestAgentTransportDeferralKeepsLastKnownHistory(t *testing.T) {redirected.Add(1)}if phase.Load() == 1 && lost.CompareAndSwap(false, true) {+ switch kind {+ case "malformed success":+ fmt.Fprint(w, `{"data":{"result":`)+ return+ case "ambiguous success":+ fmt.Fprint(w, `{"data":null,"data":{"result":[]}}`)+ return+ case "trailing success":+ fmt.Fprint(w, `{"data":{"result":[]}} {}`)+ return+ }if kind == "server error" || kind == "gateway error" {status := http.StatusInternalServerErrorif kind == "gateway error" {diff --git a/pkg/proxmox/guest_agent_guard.go b/pkg/proxmox/guest_agent_guard.goindex 59bcff1a3d..fa8660563c 100644--- a/pkg/proxmox/guest_agent_guard.go+++ b/pkg/proxmox/guest_agent_guard.go@@ -233,6 +233,13 @@ func (c *Client) getGuestAgent(ctx context.Context, path, node string, vmid int)uncertain = truereturn nil, &guestAgentDeferredError{reason: "agent-redirect"}}+ // A completed HTTP body is not a completed QGA reply if it is malformed,+ // ambiguous, or only a JSON prefix. Validate before releasing admission,+ // not in the method decoder after another guest command can be queued.+ if resp.StatusCode != http.StatusOK || !guestAgentSuccessResponse(body) {+ uncertain = true+ return nil, &guestAgentDeferredError{reason: "agent-completion-unverified"}+ }// A backup may have started while the command was in flight. Do not publish// its payload as fresh telemetry if lock clearance cannot still be verified.if err := c.verifyGuestAgentUnlocked(ctx, node, vmid); err != nil {diff --git a/pkg/proxmox/guest_agent_response.go b/pkg/proxmox/guest_agent_response.goindex 9ca881dd12..eade9e3025 100644--- a/pkg/proxmox/guest_agent_response.go+++ b/pkg/proxmox/guest_agent_response.go@@ -8,6 +8,7 @@ import ("io""net/http""strings"+ "unicode/utf8")// Keep the wire status separate from the existing diagnostic text. In@@ -112,6 +113,33 @@ func guestAgentErrorMessage(body []byte) (string, bool) {return strings.TrimSpace(text), true}+// PVE guest-command success has one data dictionary. Preserve its existing+// method-specific decoding (including empty/null result, object-style rows,+// and partial row admission), but do not treat a proxy response, duplicate+// envelope or case-insensitive overwrite as a completed reply. This is not+// payload-to-command provenance or proof of native QGA completion/thaw.+func guestAgentSuccessResponse(body []byte) bool {+ if !utf8.Valid(body) {+ return false+ }+ envelope, ok := guestAgentResponseFields(body, "data")+ if !ok {+ return false+ }+ data, ok := guestAgentResponseFields(envelope["data"])+ if !ok {+ return false+ }+ for name := range data {+ for _, canonical := range []string{"result", "content", "truncated"} {+ if name != canonical && strings.EqualFold(name, canonical) {+ return false+ }+ }+ }+ return true+}+// Reject ambiguous envelopes (including duplicate keys and trailing values)// instead of letting a favourable final field override other error evidence.// Without an allowlist, arbitrary unique configuration fields are permitted.diff --git a/pkg/proxmox/guest_agent_success_response_test.go b/pkg/proxmox/guest_agent_success_response_test.gonew file mode 100644index 0000000000..205f3fa674--- /dev/null+++ b/pkg/proxmox/guest_agent_success_response_test.go@@ -0,0 +1,209 @@+package proxmox++import (+ "context"+ "fmt"+ "net/http"+ "net/http/httptest"+ "strings"+ "sync/atomic"+ "testing"+ "time"+)++// HTTP 200 and a finished body do not establish a completed guest reply if+// the envelope is missing, ambiguous, or only a valid prefix. All six readers+// must retain uncertainty before another client/method can queue QGA work.+func TestGuestAgentInvalidSuccessDefersEveryRead(t *testing.T) {+ for name, body := range map[string]string{+ "empty": "",+ "html": "<html>upstream unavailable</html>",+ "truncated-json": `{"data":{"result":`,+ "trailing-value": `{"data":{"result":[]}} {"data":{"result":[]}}`,+ "trailing-garbage": `{"data":{"result":[]}} unavailable`,+ "missing-data": `{}`,+ "null-data": `{"data":null}`,+ "scalar-data": `{"data":"upstream unavailable"}`,+ "duplicate-data": `{"data":null,"data":{"result":[]}}`,+ "escaped-duplicate": `{"data":null,"\u0064ata":{"result":[]}}`,+ "case-conflict": `{"Data":null,"data":{"result":[]}}`,+ "case-only": `{"Data":{"result":[]}}`,+ "competing-error": `{"errors":{"message":"upstream unavailable"},"data":{"result":[]}}`,+ "duplicate-result": `{"data":{"result":null,"result":[]}}`,+ "result-case": `{"data":{"Result":null,"result":[]}}`,+ "content-case": `{"data":{"Content":"stale","content":"fresh"}}`,+ "duplicate-content": `{"data":{"content":"stale","content":"fresh"}}`,+ "non-utf8": "{\"data\":{\"content\":\"\xff\"}}",+ } {+ t.Run(name, func(t *testing.T) {+ for reader, first := range backupAgentReads() {+ t.Run(reader, func(t *testing.T) {+ var calls, configs atomic.Int32+ server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {+ switch {+ case strings.HasSuffix(r.URL.Path, "/config"):+ configs.Add(1)+ fmt.Fprint(w, `{"data":{}}`)+ case strings.HasSuffix(r.URL.Path, "/status/current"):+ fmt.Fprint(w, `{"data":{"status":"running","cpu":0.25,"diskread":4096}}`)+ default:+ if calls.Add(1) == 1 {+ fmt.Fprint(w, body)+ return+ }+ backupAgentPayload(w, r)+ }+ }))+ defer server.Close()+ c := backupTestClient(t, server.URL)+ if err := first(context.Background(), c, 105); GuestAgentDeferredReason(err) != "agent-completion-unverified" {+ t.Errorf("unverified success released admission: %v", err)+ }+ before := configs.Load()+ // The next complete reply would be healthy; it must not be+ // requested while the earlier command remains uncertain.+ diagnostic := backupTestClient(t, server.URL)+ for later, read := range backupAgentReads() {+ if err := read(context.Background(), diagnostic, 105); GuestAgentDeferredReason(err) != "agent-cooldown" {+ t.Errorf("next %s escaped cooldown: %v", later, err)+ }+ }+ if calls.Load() != 1 || configs.Load() != before {+ t.Errorf("uncertain reply caused follow-up commands/config reads: %d/%d -> %d", calls.Load(), before, configs.Load())+ }+ status, err := c.GetVMStatus(context.Background(), "node", 105)+ if err != nil || status.CPU != 0.25 || status.DiskRead != 4096 {+ t.Errorf("live non-agent counters lost: %+v %v", status, err)+ }+ })+ }+ })+ }+}++func TestGuestAgentInvalidSuccessSharesAliasesAndResumes(t *testing.T) {+ var calls, otherCalls atomic.Int32+ var healthy atomic.Bool+ handler := func(counter *atomic.Int32, broken bool) http.HandlerFunc {+ return func(w http.ResponseWriter, r *http.Request) {+ switch {+ case strings.HasSuffix(r.URL.Path, "/config"):+ fmt.Fprint(w, `{"data":{}}`)+ case strings.HasSuffix(r.URL.Path, "/nodes"):+ fmt.Fprint(w, `{"data":[]}`)+ default:+ counter.Add(1)+ if broken && !healthy.Load() {+ fmt.Fprint(w, `{"data":null,"data":{"result":[]}}`)+ return+ }+ backupAgentPayload(w, r)+ }+ }+ }+ a := httptest.NewServer(handler(&calls, true))+ b := httptest.NewServer(handler(&calls, false))+ independent := httptest.NewServer(handler(&otherCalls, false))+ defer a.Close()+ defer b.Close()+ defer independent.Close()+ cc := NewClusterClient("success-envelope", ClientConfig{Host: a.URL, TokenName: "fixture@pve!pulse", TokenValue: "fixture"}, []string{a.URL, b.URL}, nil)+ if _, err := backupTestClient(t, a.URL).GetVMFSInfo(context.Background(), "node", 105); GuestAgentDeferredReason(err) != "agent-completion-unverified" {+ t.Fatal(err)+ }+ if _, err := cc.GetVMAgentInfo(context.Background(), "migrated", 105); GuestAgentDeferredReason(err) != "agent-cooldown" {+ t.Fatalf("cluster alias replayed uncertain success: %v", err)+ }+ if _, err := backupTestClient(t, b.URL).GetVMFSInfo(context.Background(), "migrated", 105); GuestAgentDeferredReason(err) != "agent-cooldown" {+ t.Fatalf("fresh client/alias bypassed uncertainty: %v", err)+ }+ if calls.Load() != 1 {+ t.Fatalf("cluster wire commands after uncertainty = %d", calls.Load())+ }+ for _, c := range []*Client{backupTestClient(t, b.URL), backupTestClient(t, independent.URL)} {+ if _, err := c.GetVMFSInfo(context.Background(), "node", 106); err != nil {+ t.Fatalf("independent guest blocked: %v", err)+ }+ }+ if _, err := backupTestClient(t, independent.URL).GetVMFSInfo(context.Background(), "node", 105); err != nil {+ t.Fatalf("independent source blocked: %v", err)+ }+ // Expire only this fixture's actual shared cooldown. No sleep, replay of+ // the uncertain request, or replacement of the guard/transport is needed.+ guestAgentGuards.Lock()+ for _, endpoint := range []string{a.URL, b.URL} {+ key := guestAgentGuardKey{endpoint: endpoint + "/api2/json", vmid: 105}+ entry := guestAgentGuards.entries[key]+ entry.until = time.Now().Add(-time.Second)+ guestAgentGuards.entries[key] = entry+ }+ guestAgentGuards.Unlock()+ healthy.Store(true)+ for name, read := range backupAgentReads() {+ if err := read(context.Background(), backupTestClient(t, a.URL), 105); err != nil {+ t.Errorf("fresh %s after cooldown: %v", name, err)+ }+ }+ if calls.Load() != 8 || otherCalls.Load() != 2 {+ t.Fatalf("isolated/resumed commands = %d/%d, want 8/2", calls.Load(), otherCalls.Load())+ }+}++func TestGuestAgentUnexpectedSuccessStatusDefersEveryRead(t *testing.T) {+ for _, status := range []int{201, 202, 204, 206} {+ t.Run(fmt.Sprintf("status-%d", status), func(t *testing.T) {+ for name, first := range backupAgentReads() {+ t.Run(name, func(t *testing.T) {+ var calls atomic.Int32+ server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {+ if strings.HasSuffix(r.URL.Path, "/config") {+ fmt.Fprint(w, `{"data":{}}`)+ return+ }+ if calls.Add(1) == 1 {+ w.WriteHeader(status)+ backupAgentPayload(w, r)+ return+ }+ backupAgentPayload(w, r)+ }))+ defer server.Close()+ c := backupTestClient(t, server.URL)+ if err := first(context.Background(), c, 105); GuestAgentDeferredReason(err) != "agent-completion-unverified" {+ t.Fatalf("unexpected status became completed reply: %v", err)+ }+ for name, next := range backupAgentReads() {+ if err := next(context.Background(), c, 105); GuestAgentDeferredReason(err) != "agent-cooldown" {+ t.Errorf("next %s escaped cooldown: %v", name, err)+ }+ }+ if calls.Load() != 1 {+ t.Fatalf("unexpected success caused another command: %d", calls.Load())+ }+ })+ }+ })+ }+}++func TestGuestAgentSuccessEnvelopePreservesSupportedPayloads(t *testing.T) {+ for name, body := range map[string]string{+ "empty-dictionary": `{"data":{}}`,+ "null-result": `{"data":{"result":null}}`,+ "empty-result": `{"data":{"result":[]}}`,+ "object-result": `{"data":{"result":{"mountpoint":"/","total-bytes":1024,"used-bytes":512}}}`,+ "partial-rows": `{"data":{"result":[null,{"name":"root","mountpoint":"/","used-bytes":512}]}}`,+ "os-info": `{"data":{"id":"linux","name":"Linux"}}`,+ "version": `{"data":{"result":{"qemu-ga":{"version":"9.2"}}}}`,+ "file-read": `{"data":{"content":"MemTotal: 1024 kB\nMemAvailable: 512 kB\n","truncated":false}}`,+ "truncated-file": `{"data":{"content":"partial","truncated":true}}`,+ "unicode": `{"data":{"name":"café"}}`,+ "whitespace": " \n{\"data\":{\"result\":[]}}\r\n\t ",+ } {+ t.Run(name, func(t *testing.T) {+ if !guestAgentSuccessResponse([]byte(body)) {+ t.Fatal("valid envelope rejected before method-specific decoding")+ }+ })+ }+}