Real-time monitoring dashboard for Proxmox VE, PBS, Docker, Kubernetes, TrueNAS and vSphere. Self-hosted, with alerts for failed backups, full disks, unhealthy containers and offline nodes.
Go aialertsdashboarddockerdocker-monitoringgohomelabinfrastructure-monitoringkubernetesmonitoringmspproxmoxproxmox-backup-serverproxmox-mail-gatewayproxmox-veself-hostedtruenastypescriptvspherewebhooks
Keep impossible guest reply types behind the uncertainty fence
pulse-triage[bot]83d51d75a5db
diff --git a/docs/release-control/v6/internal/subsystems/monitoring.md b/docs/release-control/v6/internal/subsystems/monitoring.mdindex 86b17319b7..17ca448023 100644--- a/docs/release-control/v6/internal/subsystems/monitoring.md+++ b/docs/release-control/v6/internal/subsystems/monitoring.md@@ -993,6 +993,30 @@ last-known values remain and current CPU advances. These checks establish replyconsistency, not native payload-to-command provenance, thaw, cross-processcoordination or restart safety. The native recovery precaution below remains.+The same pre-release check validates the types of present canonical reply+fields: `result` is an object, array or null, `content` a string or null, and+`truncated` a boolean or null. Primitive results and wrongly typed file fields+cannot become empty filesystem/network inventories or release admission before+a file decoder reports its error. They retain `agent-completion-unverified` and+the original one-minute fence; blocked follow-ups do not renew that deadline.+Absent/null fields, compatible object/array rows, unknown metadata and completed+but unusable memory content keep their existing semantics. A measurement+rejection alone does not pause ordinary OS discovery.++`TestGuestAgentTypedSuccessDefersEveryRead` checks sixteen impossible field+shapes through all six real HTTP readers, including no postflight/follow-up+request and unaffected live counters.+`TestGuestAgentTypedSuccessSharesUnrenewedFenceAndResumes` checks fresh clients,+cluster aliases, VM/source isolation, unchanged expiry and valid resumption.+`TestGuestAgentTypedSuccessPreservesFieldAndMeasurementSemantics` keeps empty,+null, partial-row, escaped-text and truncated-measurement compatibility.+`TestGuestAgentTypedReplyMonitoringContract` connects all three field kinds to+the real poll builder, broadcast resource and History, with untouched+healthy-read cache and observation origins: original memory/disk and+metadata evidence remain last-known, CPU advances, and guest metrics are not+renewed. This is synthetic source/consuming-surface proof, not attribution of+the original freeze or native thaw/restart recovery.+The completed-error boundary uses the actual HTTP status retained by the requestlayer, not an `API error` phrase in provider or proxy text. The supportedexplicit request/access/command refusals (HTTP 400, 401, 403, 404, 405, 422 anddiff --git a/internal/monitoring/canonical_guardrails_test.go b/internal/monitoring/canonical_guardrails_test.goindex e6cd1787e5..e6be9d046a 100644--- a/internal/monitoring/canonical_guardrails_test.go+++ b/internal/monitoring/canonical_guardrails_test.go@@ -3378,6 +3378,12 @@ func TestGuestAgentSuccessEnvelopeMonitoringContract(t *testing.T) {testGuestAgentTransportDeferralKeepsLastKnownHistory(t, "malformed success", "ambiguous success", "trailing success")}+func TestGuestAgentTypedReplyMonitoringContract(t *testing.T) {+ // Valid JSON with impossible result/file field types is still unreadable+ // command evidence, not an empty current inventory or a completion fence.+ testGuestAgentTransportDeferralKeepsLastKnownHistory(t, "typed result", "typed content", "typed truncated")+}+func TestBackupAlertEvaluationCallersShareFailureVisibility(t *testing.T) {for path, want := range map[string]int{"monitor_backups.go": 2, "monitor_alerts.go": 1} {raw, err := os.ReadFile(path)diff --git a/internal/monitoring/guest_agent_transport_integration_test.go b/internal/monitoring/guest_agent_transport_integration_test.goindex b0b1296dbf..26933d59d8 100644--- a/internal/monitoring/guest_agent_transport_integration_test.go+++ b/internal/monitoring/guest_agent_transport_integration_test.go@@ -45,6 +45,15 @@ func testGuestAgentTransportDeferralKeepsLastKnownHistory(t *testing.T, kinds ..case "trailing success":fmt.Fprint(w, `{"data":{"result":[]}} {}`)return+ case "typed result":+ fmt.Fprint(w, `{"data":{"result":"wrong command id"}}`)+ return+ case "typed content":+ fmt.Fprint(w, `{"data":{"result":[],"content":false}}`)+ return+ case "typed truncated":+ fmt.Fprint(w, `{"data":{"result":[],"truncated":"false"}}`)+ return}if status := map[string]int{"server error": http.StatusInternalServerError,@@ -95,6 +104,9 @@ func testGuestAgentTransportDeferralKeepsLastKnownHistory(t *testing.T, kinds ..t.Fatal(err)}m := &Monitor{config: &config.Config{}, rateTracker: NewRateTracker(), metricsHistory: NewMetricsHistory(32, time.Hour), guestMetadataLimiter: make(map[string]time.Time)}+ if strings.HasPrefix(kind, "typed ") {+ m.state = models.NewState()+ }res := proxmox.ClusterResource{Type: "qemu", Node: "node", Name: "guest", VMID: 105, Status: "running", MaxMem: 8 * mib, Mem: 8 * mib, MaxDisk: 1000 * mib, CPU: 0.1}identity := "transport:node:105"var previous *models.VM@@ -106,6 +118,25 @@ func testGuestAgentTransportDeferralKeepsLastKnownHistory(t *testing.T, kinds ..m.recordGuestSnapshot("transport", "qemu", "node", 105, GuestMemorySnapshot{Name: vm.Name, Status: vm.Status, RetrievedAt: at, MemorySource: source, Memory: vm.Memory, Raw: raw, Notes: notes})m.recordGuestMetrics([]models.VM{vm}, nil, time.Now().Add(-time.Second))previous = &vm+ if strings.HasPrefix(kind, "typed ") {+ // The safe fixture's actual builder/broadcast output is also+ // available for consuming browser proof. Do not hand-author a+ // fresh/last-known carrier or replace its observation time.+ m.state.UpdateVMs([]models.VM{vm})+ front := m.buildBroadcastFrontendStateFromSnapshot(m.state.GetSnapshot(), m.mockModeFence.begin())+ if len(front.Resources) != 1 {+ t.Fatal("typed-reply producer lost the canonical guest")+ }+ wire, err := json.Marshal(map[string]any{+ "kind": kind, "phase": phase.Load(), "guestCommands": calls.Load(),+ "vm": vm, "resources": front.Resources,+ "history": m.metricsHistory.GetAllGuestMetrics(identity, time.Hour),+ })+ if err != nil {+ t.Fatal(err)+ }+ t.Logf("TYPED_REPLY_PRODUCER %s", wire)+ }return vm, source}initial, source := build()@@ -114,11 +145,18 @@ func testGuestAgentTransportDeferralKeepsLastKnownHistory(t *testing.T, kinds ..}memKey, metadataKey := guestMemoryCacheKey("transport", "node", 105), guestMetadataCacheKey("transport", "node", 105)memory := m.vmAgentMemCache[memKey]- memory.fetchedAt = time.Now().Add(-2 * vmAgentMemCacheTTL)- m.vmAgentMemCache[memKey] = memorymetadata := m.guestMetadataCache[metadataKey]- metadata.fetchedAt = time.Now().Add(-2 * guestMetadataCacheTTL)- m.guestMetadataCache[metadataKey] = metadata+ if !strings.HasPrefix(kind, "typed ") {+ memory.fetchedAt = time.Now().Add(-2 * vmAgentMemCacheTTL)+ m.vmAgentMemCache[memKey] = memory+ metadata.fetchedAt = time.Now().Add(-2 * guestMetadataCacheTTL)+ m.guestMetadataCache[metadataKey] = metadata+ }+ // Typed-reply controls retain the real healthy read's original cache+ // and observation times. fetchedAt also owns measurement provenance;+ // manually aging it would invalidate an unchanged-origin assertion.+ // The filesystem is always read before these optional cached fields,+ // while the six-reader wire controls exercise fresh uncached clients.m.guestMetadataLimiter = make(map[string]time.Time)phase.Store(1)res.CPU = 0.2@@ -138,6 +176,9 @@ func testGuestAgentTransportDeferralKeepsLastKnownHistory(t *testing.T, kinds ..if deferred.ID != initial.ID || deferred.GuestAgentStatus != "deferred" || deferred.DiskStatusReason != wantReason || deferred.Memory.Used != initial.Memory.Used || source != "previous-snapshot" || deferred.Disk.Used != initial.Disk.Used || !reflect.DeepEqual(deferred.NetworkInterfaces, initial.NetworkInterfaces) {t.Fatalf("uncertain command lost truthful continuity: source=%s vm=%+v", source, deferred)}+ if strings.HasPrefix(kind, "typed ") && (deferred.Memory.Observation.State != "last-known" || !deferred.Memory.Observation.ObservedAt.Equal(initial.Memory.Observation.ObservedAt)) {+ t.Fatal("typed reply renewed or promoted the original memory observation")+ }if !reflect.DeepEqual(m.vmAgentMemCache[memKey], memory) || !reflect.DeepEqual(m.guestMetadataCache[metadataKey], metadata) {t.Fatal("uncertain command renewed or replaced cached evidence")}diff --git a/pkg/proxmox/guest_agent_response.go b/pkg/proxmox/guest_agent_response.goindex eade9e3025..1b7443b566 100644--- a/pkg/proxmox/guest_agent_response.go+++ b/pkg/proxmox/guest_agent_response.go@@ -130,12 +130,33 @@ func guestAgentSuccessResponse(body []byte) bool {if !ok {return false}- for name := range data {+ for name, value := range data {for _, canonical := range []string{"result", "content", "truncated"} {if name != canonical && strings.EqualFold(name, canonical) {return false}}+ // A well-formed envelope can still contain an impossible guest reply.+ // The filesystem/network compatibility decoders otherwise turn scalar+ // results into empty inventories, and the file decoder reports a type+ // error only after admission has already been released. Neither proves+ // that the command finished. Keep this check before release, while+ // preserving absent/null fields and the existing row/content semantics.+ value = bytes.TrimSpace(value)+ switch name {+ case "result":+ if !bytes.Equal(value, []byte("null")) && value[0] != '{' && value[0] != '[' {+ return false+ }+ case "content":+ if !bytes.Equal(value, []byte("null")) && value[0] != '"' {+ return false+ }+ case "truncated":+ if !bytes.Equal(value, []byte("null")) && !bytes.Equal(value, []byte("true")) && !bytes.Equal(value, []byte("false")) {+ return false+ }+ }}return true}diff --git a/pkg/proxmox/guest_agent_typed_response_test.go b/pkg/proxmox/guest_agent_typed_response_test.gonew file mode 100644index 0000000000..6b992a51d5--- /dev/null+++ b/pkg/proxmox/guest_agent_typed_response_test.go@@ -0,0 +1,208 @@+package proxmox++import (+ "context"+ "fmt"+ "net/http"+ "net/http/httptest"+ "reflect"+ "strings"+ "sync/atomic"+ "testing"+ "time"+)++func invalidGuestAgentTypedReplies() map[string]string {+ return map[string]string{+ "string-result": `{"data":{"result":"wrong command id"}}`,+ "empty-text-result": `{"data":{"result":""}}`,+ "true-result": `{"data":{"result":true}}`,+ "false-result": `{"data":{"result":false}}`,+ "number-result": `{"data":{"result":17}}`,+ "zero-result": `{"data":{"result":0}}`,+ "float-result": `{"data":{"result":1.5}}`,+ "object-content": `{"data":{"content":{"error":"wrong command id"}}}`,+ "array-content": `{"data":{"content":[]}}`,+ "boolean-content": `{"data":{"content":false}}`,+ "number-content": `{"data":{"content":0}}`,+ "text-truncated": `{"data":{"content":"MemTotal: 1024 kB\nMemAvailable: 512 kB\n","truncated":"false"}}`,+ "number-truncated": `{"data":{"truncated":0}}`,+ "object-truncated": `{"data":{"truncated":{}}}`,+ "array-truncated": `{"data":{"truncated":[]}}`,+ "space-result": "{\"data\":{\"result\": \n 17 \t}}",+ }+}++// Exercise the actual guarded HTTP methods, not just the envelope helper.+// No command, diagnostic or postflight lock read may follow an impossible+// typed reply until its existing uncertainty deadline has expired.+func TestGuestAgentTypedSuccessDefersEveryRead(t *testing.T) {+ for name, body := range invalidGuestAgentTypedReplies() {+ t.Run(name, func(t *testing.T) {+ if guestAgentSuccessResponse([]byte(body)) {+ t.Fatal("impossible guest reply accepted as a successful envelope")+ }+ for reader, first := range backupAgentReads() {+ t.Run(reader, func(t *testing.T) {+ var commands, configs atomic.Int32+ server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {+ switch {+ case strings.HasSuffix(r.URL.Path, "/config"):+ configs.Add(1)+ fmt.Fprint(w, `{"data":{}}`)+ case strings.HasSuffix(r.URL.Path, "/status/current"):+ fmt.Fprint(w, `{"data":{"status":"running","cpu":0.25,"diskread":4096}}`)+ default:+ if commands.Add(1) == 1 {+ fmt.Fprint(w, body)+ return+ }+ backupAgentPayload(w, r)+ }+ }))+ defer server.Close()+ c := backupTestClient(t, server.URL)+ if err := first(context.Background(), c, 105); GuestAgentDeferredReason(err) != "agent-completion-unverified" {+ t.Fatalf("typed reply released admission: %v", err)+ }+ if commands.Load() != 1 || configs.Load() != 1 {+ t.Fatal("reply was treated as complete and received a postflight lock read")+ }+ diagnostic := backupTestClient(t, server.URL)+ for later, read := range backupAgentReads() {+ if err := read(context.Background(), diagnostic, 105); GuestAgentDeferredReason(err) != "agent-cooldown" {+ t.Errorf("next %s escaped uncertainty: %v", later, err)+ }+ }+ if commands.Load() != 1 || configs.Load() != 1 {+ t.Fatal("uncertain reply dispatched more guest commands/config reads")+ }+ status, err := c.GetVMStatus(context.Background(), "node", 105)+ if err != nil || status.CPU != 0.25 || status.DiskRead != 4096 {+ t.Errorf("independent non-QGA counters changed: %+v %v", status, err)+ }+ })+ }+ })+ }+}++func TestGuestAgentTypedSuccessSharesUnrenewedFenceAndResumes(t *testing.T) {+ isolateGuestAgentCapacity(t)+ var commands atomic.Int32+ var healthy atomic.Bool+ handler := http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {+ switch {+ case strings.HasSuffix(r.URL.Path, "/nodes"):+ fmt.Fprint(w, `{"data":[]}`)+ case strings.HasSuffix(r.URL.Path, "/config"):+ fmt.Fprint(w, `{"data":{}}`)+ default:+ commands.Add(1)+ if strings.Contains(r.URL.Path, "/qemu/105/") && !healthy.Load() {+ fmt.Fprint(w, `{"data":{"result":"wrong command id"}}`)+ return+ }+ backupAgentPayload(w, r)+ }+ })+ a, b, independent := httptest.NewServer(handler), httptest.NewServer(handler), httptest.NewServer(handler)+ defer a.Close()+ defer b.Close()+ defer independent.Close()+ cc := NewClusterClient("typed-success", ClientConfig{Host: a.URL, TokenName: "fixture@pve!pulse", TokenValue: "fixture"}, []string{a.URL, b.URL}, nil)+ if _, err := cc.GetVMFSInfo(context.Background(), "node", 105); GuestAgentDeferredReason(err) != "agent-completion-unverified" || commands.Load() != 1 {+ t.Fatalf("cluster failed over or cleared uncertain reply: %v commands=%d", err, commands.Load())+ }+ guestAgentGuards.Lock()+ original := make(map[guestAgentGuardKey]guestAgentGuardEntry)+ for key, entry := range guestAgentGuards.entries {+ original[key] = entry+ }+ guestAgentGuards.Unlock()+ for _, endpoint := range []string{a.URL, b.URL} {+ for name, read := range backupAgentReads() {+ if err := read(context.Background(), backupTestClient(t, endpoint), 105); GuestAgentDeferredReason(err) != "agent-cooldown" {+ t.Errorf("fresh alias %s escaped uncertainty: %v", name, err)+ }+ }+ }+ guestAgentGuards.Lock()+ unchanged := reflect.DeepEqual(original, guestAgentGuards.entries)+ guestAgentGuards.Unlock()+ if !unchanged || commands.Load() != 1 {+ t.Fatal("deferral dispatched work or renewed the original cooldown")+ }+ if _, err := cc.GetVMAgentInfo(context.Background(), "node", 106); err != nil {+ t.Fatalf("another VM blocked: %v", err)+ }+ // The unrelated source may issue its own command; it receives its own+ // uncertainty, not the original cluster's pre-dispatch cooldown.+ if _, err := backupTestClient(t, independent.URL).GetVMFSInfo(context.Background(), "node", 105); GuestAgentDeferredReason(err) != "agent-completion-unverified" || commands.Load() != 3 {+ t.Fatalf("independent source inherited another cluster's fence: %v", err)+ }+ // Advance only the existing fixture-owned deadlines, not a native guest+ // or the process clock. Recovery uses new valid observations.+ guestAgentGuards.Lock()+ for key, entry := range guestAgentGuards.entries {+ entry.until = time.Now().Add(-time.Second)+ guestAgentGuards.entries[key] = entry+ }+ guestAgentGuards.Unlock()+ healthy.Store(true)+ for name, read := range backupAgentReads() {+ if err := read(context.Background(), backupTestClient(t, b.URL), 105); err != nil {+ t.Errorf("fresh %s after expiry did not resume: %v", name, err)+ }+ }+ if commands.Load() != 9 {+ t.Fatalf("independent and resumed commands=%d, want9", commands.Load())+ }+}++func TestGuestAgentTypedSuccessPreservesFieldAndMeasurementSemantics(t *testing.T) {+ for name, body := range map[string]string{+ "null-fields": `{"data":{"result":null,"content":null,"truncated":null}}`,+ "empty-fields": `{"data":{"result":[],"content":"","truncated":false}}`,+ "empty-object": `{"data":{"result":{}}}`,+ "partial-rows": `{"data":{"result":[null,{"name":"root","mountpoint":"/","used-bytes":512}]}}`,+ "escaped-content": `{"data":{"content":"caf\u00e9\n","truncated":true}}`,+ "unknown-metadata": `{"data":{"future-version":{"enabled":true},"result":[]}}`,+ } {+ t.Run(name, func(t *testing.T) {+ if !guestAgentSuccessResponse([]byte(body)) {+ t.Fatal("compatible typed reply rejected")+ }+ })+ }+ for name, body := range map[string]string{+ "empty": `{"data":{"content":""}}`,+ "null": `{"data":{"content":null,"truncated":null}}`,+ "truncated": `{"data":{"content":"MemTotal: 1024 kB\nMemFree: 100 kB\nBuffers: 100 kB\nCached: 100 kB\n","truncated":true}}`,+ } {+ t.Run("completed-file-"+name, func(t *testing.T) {+ var commands atomic.Int32+ server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {+ if strings.HasSuffix(r.URL.Path, "/config") {+ fmt.Fprint(w, `{"data":{}}`)+ return+ }+ commands.Add(1)+ if strings.HasSuffix(r.URL.Path, "/file-read") {+ fmt.Fprint(w, body)+ return+ }+ backupAgentPayload(w, r)+ }))+ defer server.Close()+ c := backupTestClient(t, server.URL)+ info, err := c.GetVMMemoryAvailabilityFromAgent(context.Background(), "node", 105)+ if err == nil || GuestAgentDeferredReason(err) != "" || info != (LinuxMemoryAvailability{}) {+ t.Fatalf("completed but unusable memory became telemetry or uncertainty: %+v %v", info, err)+ }+ if _, err := c.GetVMAgentInfo(context.Background(), "node", 105); err != nil || commands.Load() != 2 {+ t.Fatalf("measurement rejection unnecessarily paused OS recovery: %v", err)+ }+ })+ }+}