Skip to content

armgit / Pulse

Public Imported from GitHub
0

Real-time monitoring dashboard for Proxmox VE, PBS, Docker, Kubernetes, TrueNAS and vSphere. Self-hosted, with alerts for failed backups, full disks, unhealthy containers and offline nodes.

Defer guest reads after unverified successful replies

pulse-triage[bot]caceb1d26695
diff --git a/docs/release-control/v6/internal/subsystems/monitoring.md b/docs/release-control/v6/internal/subsystems/monitoring.md
index 414f4dff78..6f835e2ac5 100644
--- a/docs/release-control/v6/internal/subsystems/monitoring.md
+++ b/docs/release-control/v6/internal/subsystems/monitoring.md
@@ -296,6 +296,29 @@ replay through another cluster endpoint. Coordination retains at most 4,096
active/cooling endpoint-guest entries and expires cooldowns. Explicit API
permission refusals and unsupported commands remain errors, not successes.
+A successful guest reply also needs direct HTTP 200 and exactly one complete
+UTF-8 JSON envelope with one canonical `data` dictionary. Malformed, missing,
+null/scalar data, extra values, duplicate/escaped-duplicate envelope or data
+fields, competing error envelopes, or differently cased `data`, `result`,
+`content` or `truncated` fields cannot establish a completed reply. The check
+runs while admission is held, before method decoding or postflight lock reads;
+failure retains `agent-completion-unverified` and the existing one-minute
+VM-wide uncertainty pause. Other methods, fresh clients and configured cluster
+aliases cannot immediately follow it with QGA work. Unexpected 2xx statuses
+(including accepted/no-content/partial responses) have the same boundary.
+
+Valid empty dictionaries, null/empty results, object-style rows, Unicode and
+trailing whitespace retain existing method semantics, including partial row
+admission and known truncated-file rejection. Ordinary API reads, permissions,
+budgets, trust and backup settings are unchanged. The six-reader wire controls
+pin no immediate redispatch, independent live counters, alias/VM/source isolation
+and fresh admission after cooldown. `TestGuestAgentSuccessEnvelopeMonitoringContract`
+connects malformed, duplicate and prefix-only replies to real polling builders:
+original metadata/memory timestamps and guest History are not renewed, labelled
+last-known values remain and current CPU advances. These checks establish reply
+consistency, not native payload-to-command provenance, thaw, cross-process
+coordination or restart safety. The native recovery precaution below remains.
+
The completed-error boundary uses the actual HTTP status retained by the request
layer, not an `API error` phrase in provider or proxy text. HTTP 408 and
unexplained server/proxy failures (including PVE 595) retain the same per-VM
diff --git a/internal/monitoring/canonical_guardrails_test.go b/internal/monitoring/canonical_guardrails_test.go
index b240785511..e8ee4ae41e 100644
--- a/internal/monitoring/canonical_guardrails_test.go
+++ b/internal/monitoring/canonical_guardrails_test.go
@@ -3084,7 +3084,14 @@ func TestGuestAgentTransportMonitoringContract(t *testing.T) {
t.Errorf("completed/ordinary error %q became transport uncertainty", reason)
}
}
- testGuestAgentTransportDeferralKeepsLastKnownHistory(t)
+ testGuestAgentTransportDeferralKeepsLastKnownHistory(t, "lost reply", "redirect", "server error", "gateway error")
+}
+
+func TestGuestAgentSuccessEnvelopeMonitoringContract(t *testing.T) {
+ // A complete HTTP 200 is not enough to renew guest caches or History when
+ // it only supplies a malformed, ambiguous or prefix-only reply. These
+ // real client/builder controls also keep independent CPU progressing.
+ testGuestAgentTransportDeferralKeepsLastKnownHistory(t, "malformed success", "ambiguous success", "trailing success")
}
func TestBackupAlertEvaluationCallersShareFailureVisibility(t *testing.T) {
diff --git a/internal/monitoring/guest_agent_transport_integration_test.go b/internal/monitoring/guest_agent_transport_integration_test.go
index 30da065b05..8b0766228d 100644
--- a/internal/monitoring/guest_agent_transport_integration_test.go
+++ b/internal/monitoring/guest_agent_transport_integration_test.go
@@ -17,8 +17,8 @@ import (
"github.com/rcourtman/pulse-go-rewrite/pkg/proxmox"
)
-func testGuestAgentTransportDeferralKeepsLastKnownHistory(t *testing.T) {
- for _, kind := range []string{"lost reply", "redirect", "server error", "gateway error"} {
+func testGuestAgentTransportDeferralKeepsLastKnownHistory(t *testing.T, kinds ...string) {
+ for _, kind := range kinds {
t.Run(kind, func(t *testing.T) {
const mib = uint64(1024 * 1024)
var phase, calls, redirected atomic.Int32
@@ -35,6 +35,17 @@ func testGuestAgentTransportDeferralKeepsLastKnownHistory(t *testing.T) {
redirected.Add(1)
}
if phase.Load() == 1 && lost.CompareAndSwap(false, true) {
+ switch kind {
+ case "malformed success":
+ fmt.Fprint(w, `{"data":{"result":`)
+ return
+ case "ambiguous success":
+ fmt.Fprint(w, `{"data":null,"data":{"result":[]}}`)
+ return
+ case "trailing success":
+ fmt.Fprint(w, `{"data":{"result":[]}} {}`)
+ return
+ }
if kind == "server error" || kind == "gateway error" {
status := http.StatusInternalServerError
if kind == "gateway error" {
diff --git a/pkg/proxmox/guest_agent_guard.go b/pkg/proxmox/guest_agent_guard.go
index 59bcff1a3d..fa8660563c 100644
--- a/pkg/proxmox/guest_agent_guard.go
+++ b/pkg/proxmox/guest_agent_guard.go
@@ -233,6 +233,13 @@ func (c *Client) getGuestAgent(ctx context.Context, path, node string, vmid int)
uncertain = true
return nil, &guestAgentDeferredError{reason: "agent-redirect"}
}
+ // A completed HTTP body is not a completed QGA reply if it is malformed,
+ // ambiguous, or only a JSON prefix. Validate before releasing admission,
+ // not in the method decoder after another guest command can be queued.
+ if resp.StatusCode != http.StatusOK || !guestAgentSuccessResponse(body) {
+ uncertain = true
+ return nil, &guestAgentDeferredError{reason: "agent-completion-unverified"}
+ }
// A backup may have started while the command was in flight. Do not publish
// its payload as fresh telemetry if lock clearance cannot still be verified.
if err := c.verifyGuestAgentUnlocked(ctx, node, vmid); err != nil {
diff --git a/pkg/proxmox/guest_agent_response.go b/pkg/proxmox/guest_agent_response.go
index 9ca881dd12..eade9e3025 100644
--- a/pkg/proxmox/guest_agent_response.go
+++ b/pkg/proxmox/guest_agent_response.go
@@ -8,6 +8,7 @@ import (
"io"
"net/http"
"strings"
+ "unicode/utf8"
)
// Keep the wire status separate from the existing diagnostic text. In
@@ -112,6 +113,33 @@ func guestAgentErrorMessage(body []byte) (string, bool) {
return strings.TrimSpace(text), true
}
+// PVE guest-command success has one data dictionary. Preserve its existing
+// method-specific decoding (including empty/null result, object-style rows,
+// and partial row admission), but do not treat a proxy response, duplicate
+// envelope or case-insensitive overwrite as a completed reply. This is not
+// payload-to-command provenance or proof of native QGA completion/thaw.
+func guestAgentSuccessResponse(body []byte) bool {
+ if !utf8.Valid(body) {
+ return false
+ }
+ envelope, ok := guestAgentResponseFields(body, "data")
+ if !ok {
+ return false
+ }
+ data, ok := guestAgentResponseFields(envelope["data"])
+ if !ok {
+ return false
+ }
+ for name := range data {
+ for _, canonical := range []string{"result", "content", "truncated"} {
+ if name != canonical && strings.EqualFold(name, canonical) {
+ return false
+ }
+ }
+ }
+ return true
+}
+
// Reject ambiguous envelopes (including duplicate keys and trailing values)
// instead of letting a favourable final field override other error evidence.
// Without an allowlist, arbitrary unique configuration fields are permitted.
diff --git a/pkg/proxmox/guest_agent_success_response_test.go b/pkg/proxmox/guest_agent_success_response_test.go
new file mode 100644
index 0000000000..205f3fa674
--- /dev/null
+++ b/pkg/proxmox/guest_agent_success_response_test.go
@@ -0,0 +1,209 @@
+package proxmox
+
+import (
+ "context"
+ "fmt"
+ "net/http"
+ "net/http/httptest"
+ "strings"
+ "sync/atomic"
+ "testing"
+ "time"
+)
+
+// HTTP 200 and a finished body do not establish a completed guest reply if
+// the envelope is missing, ambiguous, or only a valid prefix. All six readers
+// must retain uncertainty before another client/method can queue QGA work.
+func TestGuestAgentInvalidSuccessDefersEveryRead(t *testing.T) {
+ for name, body := range map[string]string{
+ "empty": "",
+ "html": "<html>upstream unavailable</html>",
+ "truncated-json": `{"data":{"result":`,
+ "trailing-value": `{"data":{"result":[]}} {"data":{"result":[]}}`,
+ "trailing-garbage": `{"data":{"result":[]}} unavailable`,
+ "missing-data": `{}`,
+ "null-data": `{"data":null}`,
+ "scalar-data": `{"data":"upstream unavailable"}`,
+ "duplicate-data": `{"data":null,"data":{"result":[]}}`,
+ "escaped-duplicate": `{"data":null,"\u0064ata":{"result":[]}}`,
+ "case-conflict": `{"Data":null,"data":{"result":[]}}`,
+ "case-only": `{"Data":{"result":[]}}`,
+ "competing-error": `{"errors":{"message":"upstream unavailable"},"data":{"result":[]}}`,
+ "duplicate-result": `{"data":{"result":null,"result":[]}}`,
+ "result-case": `{"data":{"Result":null,"result":[]}}`,
+ "content-case": `{"data":{"Content":"stale","content":"fresh"}}`,
+ "duplicate-content": `{"data":{"content":"stale","content":"fresh"}}`,
+ "non-utf8": "{\"data\":{\"content\":\"\xff\"}}",
+ } {
+ t.Run(name, func(t *testing.T) {
+ for reader, first := range backupAgentReads() {
+ t.Run(reader, func(t *testing.T) {
+ var calls, configs atomic.Int32
+ server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+ switch {
+ case strings.HasSuffix(r.URL.Path, "/config"):
+ configs.Add(1)
+ fmt.Fprint(w, `{"data":{}}`)
+ case strings.HasSuffix(r.URL.Path, "/status/current"):
+ fmt.Fprint(w, `{"data":{"status":"running","cpu":0.25,"diskread":4096}}`)
+ default:
+ if calls.Add(1) == 1 {
+ fmt.Fprint(w, body)
+ return
+ }
+ backupAgentPayload(w, r)
+ }
+ }))
+ defer server.Close()
+ c := backupTestClient(t, server.URL)
+ if err := first(context.Background(), c, 105); GuestAgentDeferredReason(err) != "agent-completion-unverified" {
+ t.Errorf("unverified success released admission: %v", err)
+ }
+ before := configs.Load()
+ // The next complete reply would be healthy; it must not be
+ // requested while the earlier command remains uncertain.
+ diagnostic := backupTestClient(t, server.URL)
+ for later, read := range backupAgentReads() {
+ if err := read(context.Background(), diagnostic, 105); GuestAgentDeferredReason(err) != "agent-cooldown" {
+ t.Errorf("next %s escaped cooldown: %v", later, err)
+ }
+ }
+ if calls.Load() != 1 || configs.Load() != before {
+ t.Errorf("uncertain reply caused follow-up commands/config reads: %d/%d -> %d", calls.Load(), before, configs.Load())
+ }
+ status, err := c.GetVMStatus(context.Background(), "node", 105)
+ if err != nil || status.CPU != 0.25 || status.DiskRead != 4096 {
+ t.Errorf("live non-agent counters lost: %+v %v", status, err)
+ }
+ })
+ }
+ })
+ }
+}
+
+func TestGuestAgentInvalidSuccessSharesAliasesAndResumes(t *testing.T) {
+ var calls, otherCalls atomic.Int32
+ var healthy atomic.Bool
+ handler := func(counter *atomic.Int32, broken bool) http.HandlerFunc {
+ return func(w http.ResponseWriter, r *http.Request) {
+ switch {
+ case strings.HasSuffix(r.URL.Path, "/config"):
+ fmt.Fprint(w, `{"data":{}}`)
+ case strings.HasSuffix(r.URL.Path, "/nodes"):
+ fmt.Fprint(w, `{"data":[]}`)
+ default:
+ counter.Add(1)
+ if broken && !healthy.Load() {
+ fmt.Fprint(w, `{"data":null,"data":{"result":[]}}`)
+ return
+ }
+ backupAgentPayload(w, r)
+ }
+ }
+ }
+ a := httptest.NewServer(handler(&calls, true))
+ b := httptest.NewServer(handler(&calls, false))
+ independent := httptest.NewServer(handler(&otherCalls, false))
+ defer a.Close()
+ defer b.Close()
+ defer independent.Close()
+ cc := NewClusterClient("success-envelope", ClientConfig{Host: a.URL, TokenName: "fixture@pve!pulse", TokenValue: "fixture"}, []string{a.URL, b.URL}, nil)
+ if _, err := backupTestClient(t, a.URL).GetVMFSInfo(context.Background(), "node", 105); GuestAgentDeferredReason(err) != "agent-completion-unverified" {
+ t.Fatal(err)
+ }
+ if _, err := cc.GetVMAgentInfo(context.Background(), "migrated", 105); GuestAgentDeferredReason(err) != "agent-cooldown" {
+ t.Fatalf("cluster alias replayed uncertain success: %v", err)
+ }
+ if _, err := backupTestClient(t, b.URL).GetVMFSInfo(context.Background(), "migrated", 105); GuestAgentDeferredReason(err) != "agent-cooldown" {
+ t.Fatalf("fresh client/alias bypassed uncertainty: %v", err)
+ }
+ if calls.Load() != 1 {
+ t.Fatalf("cluster wire commands after uncertainty = %d", calls.Load())
+ }
+ for _, c := range []*Client{backupTestClient(t, b.URL), backupTestClient(t, independent.URL)} {
+ if _, err := c.GetVMFSInfo(context.Background(), "node", 106); err != nil {
+ t.Fatalf("independent guest blocked: %v", err)
+ }
+ }
+ if _, err := backupTestClient(t, independent.URL).GetVMFSInfo(context.Background(), "node", 105); err != nil {
+ t.Fatalf("independent source blocked: %v", err)
+ }
+ // Expire only this fixture's actual shared cooldown. No sleep, replay of
+ // the uncertain request, or replacement of the guard/transport is needed.
+ guestAgentGuards.Lock()
+ for _, endpoint := range []string{a.URL, b.URL} {
+ key := guestAgentGuardKey{endpoint: endpoint + "/api2/json", vmid: 105}
+ entry := guestAgentGuards.entries[key]
+ entry.until = time.Now().Add(-time.Second)
+ guestAgentGuards.entries[key] = entry
+ }
+ guestAgentGuards.Unlock()
+ healthy.Store(true)
+ for name, read := range backupAgentReads() {
+ if err := read(context.Background(), backupTestClient(t, a.URL), 105); err != nil {
+ t.Errorf("fresh %s after cooldown: %v", name, err)
+ }
+ }
+ if calls.Load() != 8 || otherCalls.Load() != 2 {
+ t.Fatalf("isolated/resumed commands = %d/%d, want 8/2", calls.Load(), otherCalls.Load())
+ }
+}
+
+func TestGuestAgentUnexpectedSuccessStatusDefersEveryRead(t *testing.T) {
+ for _, status := range []int{201, 202, 204, 206} {
+ t.Run(fmt.Sprintf("status-%d", status), func(t *testing.T) {
+ for name, first := range backupAgentReads() {
+ t.Run(name, func(t *testing.T) {
+ var calls atomic.Int32
+ server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+ if strings.HasSuffix(r.URL.Path, "/config") {
+ fmt.Fprint(w, `{"data":{}}`)
+ return
+ }
+ if calls.Add(1) == 1 {
+ w.WriteHeader(status)
+ backupAgentPayload(w, r)
+ return
+ }
+ backupAgentPayload(w, r)
+ }))
+ defer server.Close()
+ c := backupTestClient(t, server.URL)
+ if err := first(context.Background(), c, 105); GuestAgentDeferredReason(err) != "agent-completion-unverified" {
+ t.Fatalf("unexpected status became completed reply: %v", err)
+ }
+ for name, next := range backupAgentReads() {
+ if err := next(context.Background(), c, 105); GuestAgentDeferredReason(err) != "agent-cooldown" {
+ t.Errorf("next %s escaped cooldown: %v", name, err)
+ }
+ }
+ if calls.Load() != 1 {
+ t.Fatalf("unexpected success caused another command: %d", calls.Load())
+ }
+ })
+ }
+ })
+ }
+}
+
+func TestGuestAgentSuccessEnvelopePreservesSupportedPayloads(t *testing.T) {
+ for name, body := range map[string]string{
+ "empty-dictionary": `{"data":{}}`,
+ "null-result": `{"data":{"result":null}}`,
+ "empty-result": `{"data":{"result":[]}}`,
+ "object-result": `{"data":{"result":{"mountpoint":"/","total-bytes":1024,"used-bytes":512}}}`,
+ "partial-rows": `{"data":{"result":[null,{"name":"root","mountpoint":"/","used-bytes":512}]}}`,
+ "os-info": `{"data":{"id":"linux","name":"Linux"}}`,
+ "version": `{"data":{"result":{"qemu-ga":{"version":"9.2"}}}}`,
+ "file-read": `{"data":{"content":"MemTotal: 1024 kB\nMemAvailable: 512 kB\n","truncated":false}}`,
+ "truncated-file": `{"data":{"content":"partial","truncated":true}}`,
+ "unicode": `{"data":{"name":"café"}}`,
+ "whitespace": " \n{\"data\":{\"result\":[]}}\r\n\t ",
+ } {
+ t.Run(name, func(t *testing.T) {
+ if !guestAgentSuccessResponse([]byte(body)) {
+ t.Fatal("valid envelope rejected before method-specific decoding")
+ }
+ })
+ }
+}