Skip to content

armgit / Pulse

Public Imported from GitHub
0

Real-time monitoring dashboard for Proxmox VE, PBS, Docker, Kubernetes, TrueNAS and vSphere. Self-hosted, with alerts for failed backups, full disks, unhealthy containers and offline nodes.

Keep impossible guest reply types behind the uncertainty fence

pulse-triage[bot]83d51d75a5db
diff --git a/docs/release-control/v6/internal/subsystems/monitoring.md b/docs/release-control/v6/internal/subsystems/monitoring.md
index 86b17319b7..17ca448023 100644
--- a/docs/release-control/v6/internal/subsystems/monitoring.md
+++ b/docs/release-control/v6/internal/subsystems/monitoring.md
@@ -993,6 +993,30 @@ last-known values remain and current CPU advances. These checks establish reply
consistency, not native payload-to-command provenance, thaw, cross-process
coordination or restart safety. The native recovery precaution below remains.
+The same pre-release check validates the types of present canonical reply
+fields: `result` is an object, array or null, `content` a string or null, and
+`truncated` a boolean or null. Primitive results and wrongly typed file fields
+cannot become empty filesystem/network inventories or release admission before
+a file decoder reports its error. They retain `agent-completion-unverified` and
+the original one-minute fence; blocked follow-ups do not renew that deadline.
+Absent/null fields, compatible object/array rows, unknown metadata and completed
+but unusable memory content keep their existing semantics. A measurement
+rejection alone does not pause ordinary OS discovery.
+
+`TestGuestAgentTypedSuccessDefersEveryRead` checks sixteen impossible field
+shapes through all six real HTTP readers, including no postflight/follow-up
+request and unaffected live counters.
+`TestGuestAgentTypedSuccessSharesUnrenewedFenceAndResumes` checks fresh clients,
+cluster aliases, VM/source isolation, unchanged expiry and valid resumption.
+`TestGuestAgentTypedSuccessPreservesFieldAndMeasurementSemantics` keeps empty,
+null, partial-row, escaped-text and truncated-measurement compatibility.
+`TestGuestAgentTypedReplyMonitoringContract` connects all three field kinds to
+the real poll builder, broadcast resource and History, with untouched
+healthy-read cache and observation origins: original memory/disk and
+metadata evidence remain last-known, CPU advances, and guest metrics are not
+renewed. This is synthetic source/consuming-surface proof, not attribution of
+the original freeze or native thaw/restart recovery.
+
The completed-error boundary uses the actual HTTP status retained by the request
layer, not an `API error` phrase in provider or proxy text. The supported
explicit request/access/command refusals (HTTP 400, 401, 403, 404, 405, 422 and
diff --git a/internal/monitoring/canonical_guardrails_test.go b/internal/monitoring/canonical_guardrails_test.go
index e6cd1787e5..e6be9d046a 100644
--- a/internal/monitoring/canonical_guardrails_test.go
+++ b/internal/monitoring/canonical_guardrails_test.go
@@ -3378,6 +3378,12 @@ func TestGuestAgentSuccessEnvelopeMonitoringContract(t *testing.T) {
testGuestAgentTransportDeferralKeepsLastKnownHistory(t, "malformed success", "ambiguous success", "trailing success")
}
+func TestGuestAgentTypedReplyMonitoringContract(t *testing.T) {
+ // Valid JSON with impossible result/file field types is still unreadable
+ // command evidence, not an empty current inventory or a completion fence.
+ testGuestAgentTransportDeferralKeepsLastKnownHistory(t, "typed result", "typed content", "typed truncated")
+}
+
func TestBackupAlertEvaluationCallersShareFailureVisibility(t *testing.T) {
for path, want := range map[string]int{"monitor_backups.go": 2, "monitor_alerts.go": 1} {
raw, err := os.ReadFile(path)
diff --git a/internal/monitoring/guest_agent_transport_integration_test.go b/internal/monitoring/guest_agent_transport_integration_test.go
index b0b1296dbf..26933d59d8 100644
--- a/internal/monitoring/guest_agent_transport_integration_test.go
+++ b/internal/monitoring/guest_agent_transport_integration_test.go
@@ -45,6 +45,15 @@ func testGuestAgentTransportDeferralKeepsLastKnownHistory(t *testing.T, kinds ..
case "trailing success":
fmt.Fprint(w, `{"data":{"result":[]}} {}`)
return
+ case "typed result":
+ fmt.Fprint(w, `{"data":{"result":"wrong command id"}}`)
+ return
+ case "typed content":
+ fmt.Fprint(w, `{"data":{"result":[],"content":false}}`)
+ return
+ case "typed truncated":
+ fmt.Fprint(w, `{"data":{"result":[],"truncated":"false"}}`)
+ return
}
if status := map[string]int{
"server error": http.StatusInternalServerError,
@@ -95,6 +104,9 @@ func testGuestAgentTransportDeferralKeepsLastKnownHistory(t *testing.T, kinds ..
t.Fatal(err)
}
m := &Monitor{config: &config.Config{}, rateTracker: NewRateTracker(), metricsHistory: NewMetricsHistory(32, time.Hour), guestMetadataLimiter: make(map[string]time.Time)}
+ if strings.HasPrefix(kind, "typed ") {
+ m.state = models.NewState()
+ }
res := proxmox.ClusterResource{Type: "qemu", Node: "node", Name: "guest", VMID: 105, Status: "running", MaxMem: 8 * mib, Mem: 8 * mib, MaxDisk: 1000 * mib, CPU: 0.1}
identity := "transport:node:105"
var previous *models.VM
@@ -106,6 +118,25 @@ func testGuestAgentTransportDeferralKeepsLastKnownHistory(t *testing.T, kinds ..
m.recordGuestSnapshot("transport", "qemu", "node", 105, GuestMemorySnapshot{Name: vm.Name, Status: vm.Status, RetrievedAt: at, MemorySource: source, Memory: vm.Memory, Raw: raw, Notes: notes})
m.recordGuestMetrics([]models.VM{vm}, nil, time.Now().Add(-time.Second))
previous = &vm
+ if strings.HasPrefix(kind, "typed ") {
+ // The safe fixture's actual builder/broadcast output is also
+ // available for consuming browser proof. Do not hand-author a
+ // fresh/last-known carrier or replace its observation time.
+ m.state.UpdateVMs([]models.VM{vm})
+ front := m.buildBroadcastFrontendStateFromSnapshot(m.state.GetSnapshot(), m.mockModeFence.begin())
+ if len(front.Resources) != 1 {
+ t.Fatal("typed-reply producer lost the canonical guest")
+ }
+ wire, err := json.Marshal(map[string]any{
+ "kind": kind, "phase": phase.Load(), "guestCommands": calls.Load(),
+ "vm": vm, "resources": front.Resources,
+ "history": m.metricsHistory.GetAllGuestMetrics(identity, time.Hour),
+ })
+ if err != nil {
+ t.Fatal(err)
+ }
+ t.Logf("TYPED_REPLY_PRODUCER %s", wire)
+ }
return vm, source
}
initial, source := build()
@@ -114,11 +145,18 @@ func testGuestAgentTransportDeferralKeepsLastKnownHistory(t *testing.T, kinds ..
}
memKey, metadataKey := guestMemoryCacheKey("transport", "node", 105), guestMetadataCacheKey("transport", "node", 105)
memory := m.vmAgentMemCache[memKey]
- memory.fetchedAt = time.Now().Add(-2 * vmAgentMemCacheTTL)
- m.vmAgentMemCache[memKey] = memory
metadata := m.guestMetadataCache[metadataKey]
- metadata.fetchedAt = time.Now().Add(-2 * guestMetadataCacheTTL)
- m.guestMetadataCache[metadataKey] = metadata
+ if !strings.HasPrefix(kind, "typed ") {
+ memory.fetchedAt = time.Now().Add(-2 * vmAgentMemCacheTTL)
+ m.vmAgentMemCache[memKey] = memory
+ metadata.fetchedAt = time.Now().Add(-2 * guestMetadataCacheTTL)
+ m.guestMetadataCache[metadataKey] = metadata
+ }
+ // Typed-reply controls retain the real healthy read's original cache
+ // and observation times. fetchedAt also owns measurement provenance;
+ // manually aging it would invalidate an unchanged-origin assertion.
+ // The filesystem is always read before these optional cached fields,
+ // while the six-reader wire controls exercise fresh uncached clients.
m.guestMetadataLimiter = make(map[string]time.Time)
phase.Store(1)
res.CPU = 0.2
@@ -138,6 +176,9 @@ func testGuestAgentTransportDeferralKeepsLastKnownHistory(t *testing.T, kinds ..
if deferred.ID != initial.ID || deferred.GuestAgentStatus != "deferred" || deferred.DiskStatusReason != wantReason || deferred.Memory.Used != initial.Memory.Used || source != "previous-snapshot" || deferred.Disk.Used != initial.Disk.Used || !reflect.DeepEqual(deferred.NetworkInterfaces, initial.NetworkInterfaces) {
t.Fatalf("uncertain command lost truthful continuity: source=%s vm=%+v", source, deferred)
}
+ if strings.HasPrefix(kind, "typed ") && (deferred.Memory.Observation.State != "last-known" || !deferred.Memory.Observation.ObservedAt.Equal(initial.Memory.Observation.ObservedAt)) {
+ t.Fatal("typed reply renewed or promoted the original memory observation")
+ }
if !reflect.DeepEqual(m.vmAgentMemCache[memKey], memory) || !reflect.DeepEqual(m.guestMetadataCache[metadataKey], metadata) {
t.Fatal("uncertain command renewed or replaced cached evidence")
}
diff --git a/pkg/proxmox/guest_agent_response.go b/pkg/proxmox/guest_agent_response.go
index eade9e3025..1b7443b566 100644
--- a/pkg/proxmox/guest_agent_response.go
+++ b/pkg/proxmox/guest_agent_response.go
@@ -130,12 +130,33 @@ func guestAgentSuccessResponse(body []byte) bool {
if !ok {
return false
}
- for name := range data {
+ for name, value := range data {
for _, canonical := range []string{"result", "content", "truncated"} {
if name != canonical && strings.EqualFold(name, canonical) {
return false
}
}
+ // A well-formed envelope can still contain an impossible guest reply.
+ // The filesystem/network compatibility decoders otherwise turn scalar
+ // results into empty inventories, and the file decoder reports a type
+ // error only after admission has already been released. Neither proves
+ // that the command finished. Keep this check before release, while
+ // preserving absent/null fields and the existing row/content semantics.
+ value = bytes.TrimSpace(value)
+ switch name {
+ case "result":
+ if !bytes.Equal(value, []byte("null")) && value[0] != '{' && value[0] != '[' {
+ return false
+ }
+ case "content":
+ if !bytes.Equal(value, []byte("null")) && value[0] != '"' {
+ return false
+ }
+ case "truncated":
+ if !bytes.Equal(value, []byte("null")) && !bytes.Equal(value, []byte("true")) && !bytes.Equal(value, []byte("false")) {
+ return false
+ }
+ }
}
return true
}
diff --git a/pkg/proxmox/guest_agent_typed_response_test.go b/pkg/proxmox/guest_agent_typed_response_test.go
new file mode 100644
index 0000000000..6b992a51d5
--- /dev/null
+++ b/pkg/proxmox/guest_agent_typed_response_test.go
@@ -0,0 +1,208 @@
+package proxmox
+
+import (
+ "context"
+ "fmt"
+ "net/http"
+ "net/http/httptest"
+ "reflect"
+ "strings"
+ "sync/atomic"
+ "testing"
+ "time"
+)
+
+func invalidGuestAgentTypedReplies() map[string]string {
+ return map[string]string{
+ "string-result": `{"data":{"result":"wrong command id"}}`,
+ "empty-text-result": `{"data":{"result":""}}`,
+ "true-result": `{"data":{"result":true}}`,
+ "false-result": `{"data":{"result":false}}`,
+ "number-result": `{"data":{"result":17}}`,
+ "zero-result": `{"data":{"result":0}}`,
+ "float-result": `{"data":{"result":1.5}}`,
+ "object-content": `{"data":{"content":{"error":"wrong command id"}}}`,
+ "array-content": `{"data":{"content":[]}}`,
+ "boolean-content": `{"data":{"content":false}}`,
+ "number-content": `{"data":{"content":0}}`,
+ "text-truncated": `{"data":{"content":"MemTotal: 1024 kB\nMemAvailable: 512 kB\n","truncated":"false"}}`,
+ "number-truncated": `{"data":{"truncated":0}}`,
+ "object-truncated": `{"data":{"truncated":{}}}`,
+ "array-truncated": `{"data":{"truncated":[]}}`,
+ "space-result": "{\"data\":{\"result\": \n 17 \t}}",
+ }
+}
+
+// Exercise the actual guarded HTTP methods, not just the envelope helper.
+// No command, diagnostic or postflight lock read may follow an impossible
+// typed reply until its existing uncertainty deadline has expired.
+func TestGuestAgentTypedSuccessDefersEveryRead(t *testing.T) {
+ for name, body := range invalidGuestAgentTypedReplies() {
+ t.Run(name, func(t *testing.T) {
+ if guestAgentSuccessResponse([]byte(body)) {
+ t.Fatal("impossible guest reply accepted as a successful envelope")
+ }
+ for reader, first := range backupAgentReads() {
+ t.Run(reader, func(t *testing.T) {
+ var commands, configs atomic.Int32
+ server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+ switch {
+ case strings.HasSuffix(r.URL.Path, "/config"):
+ configs.Add(1)
+ fmt.Fprint(w, `{"data":{}}`)
+ case strings.HasSuffix(r.URL.Path, "/status/current"):
+ fmt.Fprint(w, `{"data":{"status":"running","cpu":0.25,"diskread":4096}}`)
+ default:
+ if commands.Add(1) == 1 {
+ fmt.Fprint(w, body)
+ return
+ }
+ backupAgentPayload(w, r)
+ }
+ }))
+ defer server.Close()
+ c := backupTestClient(t, server.URL)
+ if err := first(context.Background(), c, 105); GuestAgentDeferredReason(err) != "agent-completion-unverified" {
+ t.Fatalf("typed reply released admission: %v", err)
+ }
+ if commands.Load() != 1 || configs.Load() != 1 {
+ t.Fatal("reply was treated as complete and received a postflight lock read")
+ }
+ diagnostic := backupTestClient(t, server.URL)
+ for later, read := range backupAgentReads() {
+ if err := read(context.Background(), diagnostic, 105); GuestAgentDeferredReason(err) != "agent-cooldown" {
+ t.Errorf("next %s escaped uncertainty: %v", later, err)
+ }
+ }
+ if commands.Load() != 1 || configs.Load() != 1 {
+ t.Fatal("uncertain reply dispatched more guest commands/config reads")
+ }
+ status, err := c.GetVMStatus(context.Background(), "node", 105)
+ if err != nil || status.CPU != 0.25 || status.DiskRead != 4096 {
+ t.Errorf("independent non-QGA counters changed: %+v %v", status, err)
+ }
+ })
+ }
+ })
+ }
+}
+
+func TestGuestAgentTypedSuccessSharesUnrenewedFenceAndResumes(t *testing.T) {
+ isolateGuestAgentCapacity(t)
+ var commands atomic.Int32
+ var healthy atomic.Bool
+ handler := http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+ switch {
+ case strings.HasSuffix(r.URL.Path, "/nodes"):
+ fmt.Fprint(w, `{"data":[]}`)
+ case strings.HasSuffix(r.URL.Path, "/config"):
+ fmt.Fprint(w, `{"data":{}}`)
+ default:
+ commands.Add(1)
+ if strings.Contains(r.URL.Path, "/qemu/105/") && !healthy.Load() {
+ fmt.Fprint(w, `{"data":{"result":"wrong command id"}}`)
+ return
+ }
+ backupAgentPayload(w, r)
+ }
+ })
+ a, b, independent := httptest.NewServer(handler), httptest.NewServer(handler), httptest.NewServer(handler)
+ defer a.Close()
+ defer b.Close()
+ defer independent.Close()
+ cc := NewClusterClient("typed-success", ClientConfig{Host: a.URL, TokenName: "fixture@pve!pulse", TokenValue: "fixture"}, []string{a.URL, b.URL}, nil)
+ if _, err := cc.GetVMFSInfo(context.Background(), "node", 105); GuestAgentDeferredReason(err) != "agent-completion-unverified" || commands.Load() != 1 {
+ t.Fatalf("cluster failed over or cleared uncertain reply: %v commands=%d", err, commands.Load())
+ }
+ guestAgentGuards.Lock()
+ original := make(map[guestAgentGuardKey]guestAgentGuardEntry)
+ for key, entry := range guestAgentGuards.entries {
+ original[key] = entry
+ }
+ guestAgentGuards.Unlock()
+ for _, endpoint := range []string{a.URL, b.URL} {
+ for name, read := range backupAgentReads() {
+ if err := read(context.Background(), backupTestClient(t, endpoint), 105); GuestAgentDeferredReason(err) != "agent-cooldown" {
+ t.Errorf("fresh alias %s escaped uncertainty: %v", name, err)
+ }
+ }
+ }
+ guestAgentGuards.Lock()
+ unchanged := reflect.DeepEqual(original, guestAgentGuards.entries)
+ guestAgentGuards.Unlock()
+ if !unchanged || commands.Load() != 1 {
+ t.Fatal("deferral dispatched work or renewed the original cooldown")
+ }
+ if _, err := cc.GetVMAgentInfo(context.Background(), "node", 106); err != nil {
+ t.Fatalf("another VM blocked: %v", err)
+ }
+ // The unrelated source may issue its own command; it receives its own
+ // uncertainty, not the original cluster's pre-dispatch cooldown.
+ if _, err := backupTestClient(t, independent.URL).GetVMFSInfo(context.Background(), "node", 105); GuestAgentDeferredReason(err) != "agent-completion-unverified" || commands.Load() != 3 {
+ t.Fatalf("independent source inherited another cluster's fence: %v", err)
+ }
+ // Advance only the existing fixture-owned deadlines, not a native guest
+ // or the process clock. Recovery uses new valid observations.
+ guestAgentGuards.Lock()
+ for key, entry := range guestAgentGuards.entries {
+ entry.until = time.Now().Add(-time.Second)
+ guestAgentGuards.entries[key] = entry
+ }
+ guestAgentGuards.Unlock()
+ healthy.Store(true)
+ for name, read := range backupAgentReads() {
+ if err := read(context.Background(), backupTestClient(t, b.URL), 105); err != nil {
+ t.Errorf("fresh %s after expiry did not resume: %v", name, err)
+ }
+ }
+ if commands.Load() != 9 {
+ t.Fatalf("independent and resumed commands=%d, want9", commands.Load())
+ }
+}
+
+func TestGuestAgentTypedSuccessPreservesFieldAndMeasurementSemantics(t *testing.T) {
+ for name, body := range map[string]string{
+ "null-fields": `{"data":{"result":null,"content":null,"truncated":null}}`,
+ "empty-fields": `{"data":{"result":[],"content":"","truncated":false}}`,
+ "empty-object": `{"data":{"result":{}}}`,
+ "partial-rows": `{"data":{"result":[null,{"name":"root","mountpoint":"/","used-bytes":512}]}}`,
+ "escaped-content": `{"data":{"content":"caf\u00e9\n","truncated":true}}`,
+ "unknown-metadata": `{"data":{"future-version":{"enabled":true},"result":[]}}`,
+ } {
+ t.Run(name, func(t *testing.T) {
+ if !guestAgentSuccessResponse([]byte(body)) {
+ t.Fatal("compatible typed reply rejected")
+ }
+ })
+ }
+ for name, body := range map[string]string{
+ "empty": `{"data":{"content":""}}`,
+ "null": `{"data":{"content":null,"truncated":null}}`,
+ "truncated": `{"data":{"content":"MemTotal: 1024 kB\nMemFree: 100 kB\nBuffers: 100 kB\nCached: 100 kB\n","truncated":true}}`,
+ } {
+ t.Run("completed-file-"+name, func(t *testing.T) {
+ var commands atomic.Int32
+ server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+ if strings.HasSuffix(r.URL.Path, "/config") {
+ fmt.Fprint(w, `{"data":{}}`)
+ return
+ }
+ commands.Add(1)
+ if strings.HasSuffix(r.URL.Path, "/file-read") {
+ fmt.Fprint(w, body)
+ return
+ }
+ backupAgentPayload(w, r)
+ }))
+ defer server.Close()
+ c := backupTestClient(t, server.URL)
+ info, err := c.GetVMMemoryAvailabilityFromAgent(context.Background(), "node", 105)
+ if err == nil || GuestAgentDeferredReason(err) != "" || info != (LinuxMemoryAvailability{}) {
+ t.Fatalf("completed but unusable memory became telemetry or uncertainty: %+v %v", info, err)
+ }
+ if _, err := c.GetVMAgentInfo(context.Background(), "node", 105); err != nil || commands.Load() != 2 {
+ t.Fatalf("measurement rejection unnecessarily paused OS recovery: %v", err)
+ }
+ })
+ }
+}