Real-time monitoring dashboard for Proxmox VE, PBS, Docker, Kubernetes, TrueNAS and vSphere. Self-hosted, with alerts for failed backups, full disks, unhealthy containers and offline nodes.
Go aialertsdashboarddockerdocker-monitoringgohomelabinfrastructure-monitoringkubernetesmonitoringmspproxmoxproxmox-backup-serverproxmox-mail-gatewayproxmox-veself-hostedtruenastypescriptvspherewebhooks
Scope guest safety retirement to its exact PVE connection
pulse-triage[bot]58d4920de440
diff --git a/docs/release-control/v6/internal/subsystems/monitoring.md b/docs/release-control/v6/internal/subsystems/monitoring.mdindex df3b0def77..368ad868ad 100644--- a/docs/release-control/v6/internal/subsystems/monitoring.md+++ b/docs/release-control/v6/internal/subsystems/monitoring.md@@ -17,6 +17,26 @@## Purpose+### Connection retirement preserves unrelated guest safety state++Retiring a consolidated PVE connection cancels and clears only its own guest+metadata lifecycle. Connection labels may contain `|`; matching a label prefix+cannot establish ownership. The metadata key's two fixed node/VMID suffixes+are read from the right, while diagnostic memory snapshots use their recorded+`Instance`. A separate `site|other` connection keeps its outstanding owner,+OS-command suppression, original observations and refresh/backoff deadlines+when `site` is retired. The retired connection's cancelled fetch still owns+its key until completion, and shared transport uncertainty is never cleared.++`TestGuestMetadataRetirementOwnershipContract` covers exact cleanup, nested and+Unicode labels, diagnostic origins and in-flight HTTP bodies at all three+metadata stages. `TestGuestMetadataRetirementPollingKeepsSuppression` exercises+both clients and ordinary collectors: no reissued suppressed OS command, no+unrelated identity loss, and continued independent filesystem/CPU/memory reads.+Wire formats, TTLs, guest-command/backup-lock/TLS fences and stored History are+unchanged. These are synthetic source controls, not native thaw/restart recovery,+a reporter's failure cause or installed/release acceptance.+### Rate refusals cannot permanently disable guest OS discoveryAn observed HTTP 429 from guest OS-info preserves the existing OS-commanddiff --git a/internal/monitoring/guest_metadata.go b/internal/monitoring/guest_metadata.goindex a56e39488a..dce9e01f62 100644--- a/internal/monitoring/guest_metadata.go+++ b/internal/monitoring/guest_metadata.go@@ -621,6 +621,19 @@ func guestMetadataCacheKey(instanceName, nodeName string, vmid int) string {return fmt.Sprintf("%s|%s|%d", instanceName, nodeName, vmid)}+// PVE node names and VM IDs cannot contain '|', but connection labels can.+// Read the two fixed suffixes from the right: "site|other" is not owned by+// "site". Retirement must never erase another connection's command-safety+// state, cancel its fetch, or discard its refresh/backoff deadline.+func guestMetadataCacheBelongsToInstance(key, instanceName string) bool {+ end := strings.LastIndexByte(key, '|')+ if end < 0 {+ return false+ }+ end = strings.LastIndexByte(key[:end], '|')+ return end >= 0 && key[:end] == instanceName+}+func (m *Monitor) clearGuestMetadataCache(instanceName, nodeName string, vmid int) {if m == nil {returndiff --git a/internal/monitoring/guest_metadata_retirement_scope_test.go b/internal/monitoring/guest_metadata_retirement_scope_test.gonew file mode 100644index 0000000000..f9596680dd--- /dev/null+++ b/internal/monitoring/guest_metadata_retirement_scope_test.go@@ -0,0 +1,242 @@+package monitoring++import (+ "context"+ "encoding/json"+ "fmt"+ "net/http"+ "net/http/httptest"+ "os"+ "reflect"+ "strings"+ "sync"+ "sync/atomic"+ "testing"+ "time"++ "github.com/rcourtman/pulse-go-rewrite/internal/models"+ "github.com/rcourtman/pulse-go-rewrite/pkg/proxmox"+)++// Connection labels are not hostname components. A delimiter in a label must+// not turn retirement into cancellation or loss of a different QGA safety state.+func TestGuestMetadataRetirementOwnershipContract(t *testing.T) {+ for _, instance := range []string{"site", "site|branch", "région | zone"} {+ t.Run("cached/"+instance, func(t *testing.T) {+ at := time.Now().Add(-2 * guestMetadataCacheTTL)+ entry := metadataObservationFixture(at)+ entry.osInfoSkip, entry.osInfoFailureCount = true, guestAgentOSInfoFailureThreshold+ deadline, access := time.Now().Add(time.Hour), time.Now()+ m := &Monitor{+ guestMetadataCache: make(map[string]guestMetadataCacheEntry),+ guestMetadataLimiter: make(map[string]time.Time),+ guestMetadataLastAccess: make(map[string]time.Time),+ guestMetadataInFlight: make(map[string]*guestMetadataFetch),+ }+ contexts := make(map[string]context.Context)+ own := []string{guestMetadataCacheKey(instance, "node-a", 105), guestMetadataCacheKey(instance, "node-b", 106)}+ unrelated := []string{guestMetadataCacheKey(instance+"|other", "node-a", 105), guestMetadataCacheKey(instance+"|other|nested", "node-a", 105), guestMetadataCacheKey(instance+"-other", "node-a", 105), guestMetadataCacheKey(instance+"/other", "node-a", 105), guestMetadataCacheKey(strings.ToUpper(instance), "node-a", 105)}+ for _, key := range append(append([]string{}, own...), unrelated...) {+ ctx, cancel := context.WithCancel(context.Background())+ defer cancel()+ contexts[key] = ctx+ m.guestMetadataCache[key], m.guestMetadataLimiter[key], m.guestMetadataLastAccess[key] = entry, deadline, access+ m.guestMetadataInFlight[key] = &guestMetadataFetch{cancel: cancel}+ }+ m.retirePVEInstanceRuntime(instance)+ for _, key := range own {+ if _, ok := m.guestMetadataCache[key]; ok || !m.guestMetadataLimiter[key].IsZero() || !m.guestMetadataLastAccess[key].IsZero() {+ t.Errorf("retired connection retained metadata, deadline or activity: %s", key)+ }+ if fetch := m.guestMetadataInFlight[key]; fetch == nil || !fetch.invalidated || contexts[key].Err() == nil {+ t.Errorf("retired connection lost its outstanding owner or did not cancel it: %s", key)+ }+ }+ for _, key := range unrelated {+ if !reflect.DeepEqual(m.guestMetadataCache[key], entry) || m.guestMetadataLimiter[key] != deadline || m.guestMetadataLastAccess[key] != access {+ t.Errorf("retirement erased or renewed another connection's safety/evidence/backoff: %s", key)+ }+ if fetch := m.guestMetadataInFlight[key]; fetch == nil || fetch.invalidated || contexts[key].Err() != nil {+ t.Errorf("retirement invalidated another connection's outstanding read: %s", key)+ }+ }+ })+ }+ t.Run("diagnostic-origin", func(t *testing.T) {+ m := &Monitor{}+ at := time.Now()+ memory := models.Memory{Total: 8192, Used: 1024, Free: 7168, Usage: 12.5}+ for _, instance := range []string{"site", "site|other", "site|other|nested"} {+ m.recordNodeSnapshot(instance, "node-a", NodeMemorySnapshot{RetrievedAt: at, Memory: memory, MemorySource: "status"})+ m.recordGuestSnapshot(instance, "qemu", "node-a", 105, GuestMemorySnapshot{RetrievedAt: at, Memory: memory, MemorySource: "status-mem"})+ }+ before := m.GetDiagnosticSnapshots()+ m.retirePVEInstanceRuntime("site")+ got := m.GetDiagnosticSnapshots()+ if len(got.Nodes) != 2 || len(got.Guests) != 2 {+ t.Errorf("retirement removed another connection's diagnostic origin: nodes=%d guests=%d", len(got.Nodes), len(got.Guests))+ }+ for _, snapshot := range before.Nodes {+ if snapshot.Instance != "site" && !reflect.DeepEqual(m.nodeSnapshots[makeNodeSnapshotKey(snapshot.Instance, snapshot.Node)], snapshot) {+ t.Errorf("unrelated node observation changed: %s", snapshot.Instance)+ }+ }+ for _, snapshot := range before.Guests {+ if snapshot.Instance != "site" && !reflect.DeepEqual(m.guestSnapshots[makeGuestSnapshotKey(snapshot.Instance, snapshot.GuestType, snapshot.Node, snapshot.VMID)], snapshot) {+ t.Errorf("unrelated guest observation changed: %s", snapshot.Instance)+ }+ }+ })+ for _, kind := range []string{"client", "cluster-client"} {+ for _, phase := range []string{"network-get-interfaces", "get-osinfo", "info"} {+ t.Run("in-flight/"+kind+"/"+phase, func(t *testing.T) {+ entered, release := make(chan struct{}), make(chan struct{})+ var once sync.Once+ unblock := func() { once.Do(func() { close(release) }) }+ var commands atomic.Int32+ server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {+ if strings.HasSuffix(r.URL.Path, "/nodes") {+ fmt.Fprint(w, `{"data":[]}`)+ return+ }+ if strings.Contains(r.URL.Path, "/agent/") {+ commands.Add(1)+ if strings.HasSuffix(r.URL.Path, "/"+phase) {+ w.WriteHeader(http.StatusOK)+ w.(http.Flusher).Flush()+ close(entered)+ <-release+ }+ }+ metadataReservationPayload(w, r)+ }))+ defer server.Close()+ defer unblock()+ client := windowsBootstrapClient(t, server.URL, kind)+ key := guestMetadataCacheKey("site|other", "node", 105)+ m := &Monitor{guestMetadataSlots: make(chan struct{}, 1)}+ status := &proxmox.VMStatus{Agent: proxmox.VMAgentField{Value: 1}}+ done := make(chan bool, 1)+ go func() {+ _, _, name, _, agent, paused := m.fetchGuestAgentMetadata(context.Background(), client, "site|other", "node", "guest", 105, status, false)+ done <- !paused && name == "FixtureOS" && agent == "2.0"+ }()+ defer func() {+ unblock()+ select {+ case <-done:+ case <-time.After(5 * time.Second):+ t.Error("owned synthetic HTTP fetch did not finish")+ }+ }()+ select {+ case <-entered:+ case <-time.After(5 * time.Second):+ t.Fatal("selected HTTP response body did not start")+ }+ m.retirePVEInstanceRuntime("site")+ m.guestMetadataLimiterMu.Lock()+ fetch := m.guestMetadataInFlight[key]+ invalidated := fetch == nil || fetch.invalidated+ m.guestMetadataLimiterMu.Unlock()+ if invalidated {+ t.Error("unrelated retirement invalidated the live HTTP owner")+ }+ before := commands.Load()+ m.fetchGuestAgentMetadata(context.Background(), client, "site|other", "node", "guest", 105, status, false)+ if commands.Load() != before {+ t.Error("overlapping refresh bypassed the outstanding owner")+ }+ unblock()+ select {+ case accepted := <-done:+ if !accepted || commands.Load() != 3 || len(m.guestMetadataSlots) != 0 {+ t.Error("unrelated retirement lost the completed metadata, queued work or leaked a slot")+ }+ done <- accepted // Leave the completed result for cleanup's join.+ case <-time.After(5 * time.Second):+ t.Fatal("unrelated fetch did not complete")+ }+ assertMetadataReservationReleased(t, m)+ })+ }+ }+}++// Exercise the actual HTTP clients and both ordinary guest collectors. The+// endpoint represents an already-suppressed unsafe OS command; none of these+// controls sends a native guest command, changes an ACL or clears uncertainty.+func TestGuestMetadataRetirementPollingKeepsSuppression(t *testing.T) {+ for _, kind := range []string{"client", "cluster-client"} {+ for _, collector := range []string{"cluster", "node"} {+ for _, schedule := range []string{"backoff", "eligible-refresh"} {+ t.Run(kind+"/"+collector+"/"+schedule, func(t *testing.T) {+ f := &windowsBootstrapFixture{t: t, osName: bootstrapWindowsOS, linuxAvailable: 6 * bootstrapMiB}+ f.poll.Store(1)+ var osCalls atomic.Int32+ server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {+ if strings.HasSuffix(r.URL.Path, "/get-osinfo") {+ osCalls.Add(1)+ w.WriteHeader(http.StatusInternalServerError)+ fmt.Fprint(w, `{"data":null,"message":"unsupported command: guest-get-osinfo"}`)+ return+ }+ f.serve(w, r)+ }))+ defer server.Close()+ m := windowsBootstrapMonitor(t)+ client := windowsBootstrapClient(t, server.URL, kind)+ const instance = "site|other"+ key := guestMetadataCacheKey(instance, "node", 105)+ original := metadataObservationFixture(time.Now().Add(-2 * guestMetadataCacheTTL))+ original.osName, original.osVersion = "Windows", "2025"+ original.windowsGuest, original.osInfoObservedAt = true, time.Now()+ original.osInfoSkip, original.osInfoFailureCount = true, guestAgentOSInfoFailureThreshold+ m.guestMetadataCache = map[string]guestMetadataCacheEntry{key: original}+ m.guestMetadataLimiter = map[string]time.Time{key: time.Now().Add(time.Hour)}+ first := windowsBootstrapPoll(t, m, client, collector, instance, "node", 105, 1)+ if first.OSName != original.osName || osCalls.Load() != 0 {+ t.Fatal("initial ordinary poll did not use the suppressed identity")+ }+ if schedule == "eligible-refresh" {+ m.guestMetadataLimiter[key] = time.Now().Add(-time.Second)+ }+ deadline := m.guestMetadataLimiter[key]+ m.retirePVEInstanceRuntime("site")+ if m.guestMetadataLimiter[key] != deadline {+ t.Error("retirement changed a different connection's refresh schedule")+ }+ f.resetTrace()+ f.poll.Store(2)+ second := windowsBootstrapPoll(t, m, client, collector, instance, "node", 105, 2)+ if osCalls.Load() != 0 || second.OSName != original.osName || second.OSVersion != original.osVersion {+ t.Errorf("retirement reissued a suppressed OS command or lost display identity: calls=%d OS=%q/%q", osCalls.Load(), second.OSName, second.OSVersion)+ }+ entry := m.guestMetadataCache[key]+ if !entry.osInfoSkip || entry.osInfoFailureCount != original.osInfoFailureCount || entry.osInfoObservedAt != original.osInfoObservedAt {+ t.Error("retirement changed command suppression or renewed its original OS receipt")+ }+ if second.CPU != .2 || len(second.Disks) != 2 || second.Memory.Observation.Source != "status-mem" || second.Memory.Observation.State != "current" {+ t.Error("retaining suppression lost independent ordinary filesystem/CPU/memory readings")+ }+ want := []string{"get-fsinfo"}+ if schedule == "eligible-refresh" {+ want = append(want, "network-get-interfaces", "info")+ }+ if got := f.trace(); !reflect.DeepEqual(got, want) {+ t.Errorf("ordinary command trace=%v, want %v", got, want)+ }+ if output := os.Getenv("PULSE_GUEST_RETIREMENT_SURFACE_OUTPUT"); output != "" && kind == "client" && collector == "node" && schedule == "backoff" {+ body, err := json.MarshalIndent(struct{ Before, After models.VM }{first, second}, "", " ")+ if err != nil {+ t.Fatal(err)+ }+ if err := os.WriteFile(output, body, 0600); err != nil {+ t.Fatal(err)+ }+ }+ })+ }+ }+ }+}diff --git a/internal/monitoring/monitor_cluster_dedupe.go b/internal/monitoring/monitor_cluster_dedupe.goindex 9068ba9ba6..77d70d947b 100644--- a/internal/monitoring/monitor_cluster_dedupe.go+++ b/internal/monitoring/monitor_cluster_dedupe.go@@ -130,13 +130,13 @@ func (m *Monitor) retirePVEInstanceRuntime(instanceName string) {m.mu.Unlock()m.diagMu.Lock()- for key := range m.nodeSnapshots {- if strings.HasPrefix(key, instanceName+"|") {+ for key, snapshot := range m.nodeSnapshots {+ if snapshot.Instance == instanceName {delete(m.nodeSnapshots, key)}}- for key := range m.guestSnapshots {- if strings.HasPrefix(key, instanceName+"|") {+ for key, snapshot := range m.guestSnapshots {+ if snapshot.Instance == instanceName {delete(m.guestSnapshots, key)}}@@ -144,25 +144,25 @@ func (m *Monitor) retirePVEInstanceRuntime(instanceName string) {m.guestMetadataLimiterMu.Lock()for key := range m.guestMetadataInFlight {- if strings.HasPrefix(key, instanceName+"|") {+ if guestMetadataCacheBelongsToInstance(key, instanceName) {m.invalidateGuestMetadataFetchLocked(key)}}m.guestMetadataMu.Lock()for key := range m.guestMetadataCache {- if strings.HasPrefix(key, instanceName+"|") {+ if guestMetadataCacheBelongsToInstance(key, instanceName) {delete(m.guestMetadataCache, key)}}m.guestMetadataMu.Unlock()for key := range m.guestMetadataLimiter {- if strings.HasPrefix(key, instanceName+"|") {+ if guestMetadataCacheBelongsToInstance(key, instanceName) {delete(m.guestMetadataLimiter, key)}}for key := range m.guestMetadataLastAccess {- if strings.HasPrefix(key, instanceName+"|") {+ if guestMetadataCacheBelongsToInstance(key, instanceName) {delete(m.guestMetadataLastAccess, key)}}diff --git a/internal/monitoring/monitor_extra_coverage_test.go b/internal/monitoring/monitor_extra_coverage_test.goindex 23caafd87e..1d675570ed 100644--- a/internal/monitoring/monitor_extra_coverage_test.go+++ b/internal/monitoring/monitor_extra_coverage_test.go@@ -314,8 +314,8 @@ func TestMonitor_ConsolidateDuplicateClusters_RetiresRemovedStandaloneRuntimeStabackupPermissionWarnings: map[string]string{"minipc-standalone": "warn"},nodeLastOnline: map[string]time.Time{"minipc-standalone-minipc": time.Now()},nodePendingUpdatesCache: map[string]pendingUpdatesCache{"minipc-standalone-minipc": {count: 1, checkedAt: time.Now()}},- nodeSnapshots: map[string]NodeMemorySnapshot{"minipc-standalone|minipc": {RetrievedAt: time.Now()}},- guestSnapshots: map[string]GuestMemorySnapshot{"minipc-standalone|qemu|minipc|100": {RetrievedAt: time.Now()}},+ nodeSnapshots: map[string]NodeMemorySnapshot{"minipc-standalone|minipc": {Instance: "minipc-standalone", RetrievedAt: time.Now()}},+ guestSnapshots: map[string]GuestMemorySnapshot{"minipc-standalone|qemu|minipc|100": {Instance: "minipc-standalone", RetrievedAt: time.Now()}},guestMetadataCache: map[string]guestMetadataCacheEntry{"minipc-standalone|minipc|100": {fetchedAt: time.Now()}},guestMetadataLimiter: map[string]time.Time{"minipc-standalone|minipc|100": time.Now().Add(time.Minute)},}