Real-time monitoring dashboard for Proxmox VE, PBS, Docker, Kubernetes, TrueNAS and vSphere. Self-hosted, with alerts for failed backups, full disks, unhealthy containers and offline nodes.
Go aialertsdashboarddockerdocker-monitoringgohomelabinfrastructure-monitoringkubernetesmonitoringmspproxmoxproxmox-backup-serverproxmox-mail-gatewayproxmox-veself-hostedtruenastypescriptvspherewebhooks
Renew QGA fallback eligibility only from real agent replies
rcourtman1d28b1e513b4
diff --git a/docs/release-control/v6/internal/subsystems/monitoring.md b/docs/release-control/v6/internal/subsystems/monitoring.mdindex 5cc1ad375a..4cce41269e 100644--- a/docs/release-control/v6/internal/subsystems/monitoring.md+++ b/docs/release-control/v6/internal/subsystems/monitoring.md@@ -238,9 +238,11 @@ suppression and metadata backoff are not reset to force recovery.### QGA fallback eligibility keeps its original observation age-A successful unlocked VM-status availability, usable filesystem reading or-useful guest metadata can establish fallback query eligibility. A later poll-that preserves optional IP/OS/version strings does not renew that evidence.+Only an actual guest-agent reply, a usable filesystem reading or useful guest+metadata, can establish fallback query eligibility. The `status/current` `agent`+flag means the agent is enabled in the VM config, not that the in-guest service+answers, so a successful VM-status read does not establish or renew it. A later+poll that preserves optional IP/OS/version strings does not renew that evidence.Internal `GuestAgentEvidence` crosses model, canonical and previous-statesnapshots, including an explicit missing origin. Legacy direct snapshots importtheir receipt once, while already annotated disk-only evidence uses its recorded@@ -255,7 +257,10 @@ remain independent.`TestGuestAgentAdmissionEvidenceContract` checks the original-age boundaries,repeated canonical replacement and both cluster/node collectors through acancelled detail cycle, unavailable status, missing status and ordinary recovery-with measured-zero disk usage. This is synthetic source proof, not native thaw,+with measured-zero disk usage. Its `config-flag-cannot-renew` case polls with the+real integer `agent: 1` flag: an enabled agent that never replied admits no+fallback command once status fails, while one that answered keeps its fallback.+This is synthetic source proof, not native thaw,QEMU-only Windows acceptance or reporter resolution.diff --git a/internal/monitoring/canonical_guardrails_test.go b/internal/monitoring/canonical_guardrails_test.goindex 49aebf65be..87b9b9ae78 100644--- a/internal/monitoring/canonical_guardrails_test.go+++ b/internal/monitoring/canonical_guardrails_test.go@@ -1136,6 +1136,7 @@ func TestGuestAgentAdmissionEvidenceContract(t *testing.T) {t.Run("original-age", testGuestAgentEvidenceOriginalAge)t.Run("canonical-continuity", testGuestAgentEvidenceCanonicalContinuity)t.Run("accepted-read-origins", testGuestAgentEvidenceAcceptedReadOrigins)+ t.Run("config-flag-cannot-renew", testGuestAgentConfigFlagCannotRenewAdmission)}func TestProxmoxGuestDiskInventoryPrefersCanonicalLinkedHostAgentSource(t *testing.T) {diff --git a/internal/monitoring/guest_agent_evidence_config_flag_test.go b/internal/monitoring/guest_agent_evidence_config_flag_test.gonew file mode 100644index 0000000000..0eb084fee8--- /dev/null+++ b/internal/monitoring/guest_agent_evidence_config_flag_test.go@@ -0,0 +1,109 @@+package monitoring++import (+ "context"+ "fmt"+ "net/http"+ "net/http/httptest"+ "strings"+ "sync/atomic"+ "testing"+ "time"++ "github.com/rcourtman/pulse-go-rewrite/internal/alerts"+ "github.com/rcourtman/pulse-go-rewrite/internal/config"+ "github.com/rcourtman/pulse-go-rewrite/internal/models"+ "github.com/rcourtman/pulse-go-rewrite/internal/unifiedresources"+ "github.com/rcourtman/pulse-go-rewrite/pkg/proxmox"+)++// Real qemu-server status/current reports `"agent": 1` whenever the guest+// agent is enabled in the VM config (PVE/API2/Qemu.pm vm_status), whether or+// not the in-guest service ever answers. That flag must not renew fallback+// eligibility: only an actual agent reply may. All HTTP endpoints are synthetic.+func testGuestAgentConfigFlagCannotRenewAdmission(t *testing.T) {+ for _, agentAnswers := range []bool{false, true} {+ name := "enabled-but-silent"+ if agentAnswers {+ name = "answered"+ }+ t.Run(name, func(t *testing.T) {+ var statusFails atomic.Bool+ var fallbackGuestCalls atomic.Int32+ server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {+ w.Header().Set("Content-Type", "application/json")+ switch {+ case strings.HasSuffix(r.URL.Path, "/config"):+ fmt.Fprint(w, `{"data":{"agent":"1"}}`)+ case strings.HasSuffix(r.URL.Path, "/status/current"):+ if statusFails.Load() {+ http.Error(w, "permission denied", http.StatusForbidden)+ return+ }+ fmt.Fprint(w, `{"data":{"status":"running","agent":1,"maxmem":8000,"meminfo":{"available":6000}}}`)+ case strings.Contains(r.URL.Path, "/agent/"):+ if statusFails.Load() {+ fallbackGuestCalls.Add(1)+ }+ if !agentAnswers {+ http.Error(w, "QEMU guest agent is not running", http.StatusInternalServerError)+ return+ }+ switch {+ case strings.HasSuffix(r.URL.Path, "get-fsinfo"):+ fmt.Fprint(w, `{"data":{"result":[{"mountpoint":"/","type":"ext4","total-bytes":1000,"used-bytes":250}]}}`)+ case strings.HasSuffix(r.URL.Path, "network-get-interfaces"):+ fmt.Fprint(w, `{"data":{"result":[{"name":"eth0","ip-addresses":[{"ip-address":"192.0.2.12","ip-address-type":"ipv4","prefix":24}]}]}}`)+ case strings.HasSuffix(r.URL.Path, "get-osinfo"):+ fmt.Fprint(w, `{"data":{"result":{"name":"Debian","version":"fixture"}}}`)+ case strings.HasSuffix(r.URL.Path, "info"):+ fmt.Fprint(w, `{"data":{"result":{"version":"2.0"}}}`)+ default:+ http.NotFound(w, r)+ }+ default:+ http.NotFound(w, r)+ }+ }))+ defer server.Close()+ client, err := proxmox.NewClient(proxmox.ClientConfig{Host: server.URL, TokenName: "fixture@pve!pulse", TokenValue: "fixture", Timeout: 5 * time.Second})+ if err != nil {+ t.Fatal(err)+ }+ m := &Monitor{config: &config.Config{}, state: models.NewState(), rateTracker: NewRateTracker(), metricsHistory: NewMetricsHistory(32, time.Hour), guestMetadataLimiter: make(map[string]time.Time)}+ m.alertManager = alerts.NewManagerWithDataDir(t.TempDir(), alerts.WithoutPersistedAlertRestore())+ t.Cleanup(m.alertManager.Stop)+ m.guestMetadataRetryBackoff = time.Nanosecond+ id := makeGuestID("flag", "node", 107)+ prev := models.VM{ID: id, Instance: "flag", Node: "node", VMID: 107, Type: "qemu", Status: "running", LastSeen: time.Now()}+ res := proxmox.ClusterResource{Type: "qemu", Node: "node", VMID: 107, Name: "guest", Status: "running", MaxMem: 8000, Mem: 2000, MaxDisk: 1000, CPU: .2}+ registry := unifiedresources.NewRegistry(nil)+ build := func() models.VM {+ t.Helper()+ vms := m.collectClusterVMResources(context.Background(), "flag", []indexedClusterResource{{resource: res, guestID: id}}, client, map[string]models.VM{id: prev}, nil)+ if len(vms) != 1 {+ t.Fatal("guest disappeared")+ }+ registry.IngestSnapshot(models.StateSnapshot{VMs: []models.VM{vms[0]}})+ prev = previousVMFromView(registry.VMs()[0])+ return vms[0]+ }++ for poll := 0; poll < 2; poll++ {+ build()+ }+ if got := hasRecentGuestAgentEvidence(&prev, time.Now()); got != agentAnswers {+ t.Fatalf("eligibility after status reads=%t, want %t (evidence %+v)", got, agentAnswers, prev.GuestAgentEvidence)+ }++ statusFails.Store(true)+ build()+ if agentAnswers && fallbackGuestCalls.Load() == 0 {+ t.Fatal("recent real agent reply did not keep fallback eligibility after status failed")+ }+ if !agentAnswers && fallbackGuestCalls.Load() != 0 {+ t.Fatalf("config-level agent flag admitted %d fallback guest commands after status failed", fallbackGuestCalls.Load())+ }+ })+ }+}diff --git a/internal/monitoring/monitor_pve_guest_builders.go b/internal/monitoring/monitor_pve_guest_builders.goindex e17178b5ac..ff3c10f5cc 100644--- a/internal/monitoring/monitor_pve_guest_builders.go+++ b/internal/monitoring/monitor_pve_guest_builders.go@@ -70,10 +70,9 @@ func (m *Monitor) applyVMStatusDetails(res.Lock = status.Lockstate.detailedStatus = statusstate.guestAgentStatus, state.guestAgentExpected = vmGuestAgentRuntimeState(status, recentGuestAgentEvidence)- if status.Lock == "" && status.Agent.IsAvailable() {- now := time.Now()- renewGuestAgentEvidence(&state.guestAgentEvidence, observedAtOr(status.ObservedAt, now), now)- }+ // status/current reports agent=1 whenever the VM config enables the guest+ // agent, not when the in-guest service answers. Fallback eligibility is+ // renewed only by an actual reply (filesystems below, metadata at the end).// Filesystem usage is cross-platform. Read it before optional Linux memory// and metadata commands can start a shared uncertainty pause. Every command// still verifies the same operation lock and is admitted only once.