Real-time monitoring dashboard for Proxmox VE, PBS, Docker, Kubernetes, TrueNAS and vSphere. Self-hosted, with alerts for failed backups, full disks, unhealthy containers and offline nodes.
Go aialertsdashboarddockerdocker-monitoringgohomelabinfrastructure-monitoringkubernetesmonitoringmspproxmoxproxmox-backup-serverproxmox-mail-gatewayproxmox-veself-hostedtruenastypescriptvspherewebhooks
Retain accepted guest observations when node enumeration fails
pulse-triage[bot]4b205a8968cd
diff --git a/docs/release-control/v6/internal/subsystems/monitoring.md b/docs/release-control/v6/internal/subsystems/monitoring.mdindex 466c366b66..4b54080404 100644--- a/docs/release-control/v6/internal/subsystems/monitoring.md+++ b/docs/release-control/v6/internal/subsystems/monitoring.md@@ -7688,8 +7688,22 @@ publication ownership or grace-period rules.`guest_inventory_response_test.go` covers complete-body and all-or-nothingadmission on all three real HTTP readers; `io_counters_test.go` coversscoped-empty, extended fields, counter presence and receipt-time controls.-`guest_inventory_admission_test.go` covers actual efficient/fallback collection,+`diagnostic_snapshots_test.go` covers actual efficient/fallback collection,owned publication, retained observation times, breach/History/templatecontinuity, independent-connection preservation and normal recovery/removal.These controlled source proofs do not establish native thaw/restart recovery,installed trust, report attribution or release availability.++Failed-node retention selects canonical guest membership but takes the unchanged+complete tuple from that connection's last accepted producer state. Enrichment's+partial previous-state projection is not an observation: it must not turn an+unavailable/retained memory reading into zero, lose CPU/disk usage or change the+original in-guest collection state. A node-scoped detached snapshot copies only+selected guests, not unrelated host disks or Kubernetes state. The existing+canonical/enrichment path remains the fallback when no accepted source tuple+exists; linked-agent evidence is never promoted to Proxmox ownership. The+same publication owner still admits or withdraws the eventual generation.+`proxmox_guest_observations_test.go` checks scope, original memory provenance,+CPU/disk/read time and nested detachment, alongside the end-to-end admission+controls above. The first complete admission proof exposed the former partial+retention and is retained as failed evidence, not weakened into acceptance.diff --git a/internal/models/proxmox_guest_observations.go b/internal/models/proxmox_guest_observations.gonew file mode 100644index 0000000000..dac8a769ec--- /dev/null+++ b/internal/models/proxmox_guest_observations.go@@ -0,0 +1,32 @@+package models++// GuestObservationsForNodes detaches the last accepted producer tuples for one+// connection's failed nodes. It is not a canonical/merged consumer projection:+// missing enumeration must not rebuild memory, disk or CPU from partial views.+// Copy only these guests, not unrelated hosts, disks or Kubernetes inventories.+func (s *State) GuestObservationsForNodes(instance string, nodes map[string]struct{}) ([]VM, []Container) {+ if s == nil || len(nodes) == 0 {+ return nil, nil+ }+ s.mu.RLock()+ defer s.mu.RUnlock()+ var vms []VM+ var containers []Container+ for _, vm := range s.VMs {+ if _, selected := nodes[vm.Node]; !selected || vm.Instance != instance {+ continue+ }+ copied := cloneVM(vm)+ copied.OnBoot = cloneBoolPtr(vm.OnBoot)+ vms = append(vms, copied)+ }+ for _, ct := range s.Containers {+ if _, selected := nodes[ct.Node]; !selected || ct.Instance != instance {+ continue+ }+ copied := cloneContainer(ct)+ copied.OnBoot = cloneBoolPtr(ct.OnBoot)+ containers = append(containers, copied)+ }+ return vms, containers+}diff --git a/internal/models/proxmox_guest_observations_test.go b/internal/models/proxmox_guest_observations_test.gonew file mode 100644index 0000000000..9ee624bf36--- /dev/null+++ b/internal/models/proxmox_guest_observations_test.go@@ -0,0 +1,47 @@+package models++import (+ "reflect"+ "testing"+ "time"+)++func TestGuestObservationsForNodesOwnsOnlyAcceptedScopedTuples(t *testing.T) {+ s := NewState()+ at := time.Date(2026, 10, 14, 10, 0, 0, 0, time.UTC)+ onBoot := true+ vm := VM{ID: "site:member:105", Instance: "site", Node: "member", VMID: 105, CPU: .95, LastSeen: at, OnBoot: &onBoot,+ Memory: Memory{Used: 40, Total: 100, Usage: 40, Observation: MemoryObservation{State: "last-known", Source: "guest-agent-meminfo", ObservedAt: at.Add(-time.Minute)}},+ Disk: Disk{Used: 20, Total: 100, Usage: 20}, Disks: []Disk{{Used: 20, Total: 100, Mountpoint: "C:\\"}}, IPAddresses: []string{"192.0.2.1"}, Tags: []string{"production"}, NetworkInterfaces: []GuestNetworkInterface{{Name: "Ethernet", Addresses: []string{"192.0.2.1"}}}}+ ct := Container{ID: "site:member:205", Instance: "site", Node: "member", VMID: 205, CPU: .25, LastSeen: at, OnBoot: &onBoot, Memory: vm.Memory, Disk: vm.Disk, Disks: vm.Disks, Tags: vm.Tags, IPAddresses: vm.IPAddresses, NetworkInterfaces: vm.NetworkInterfaces}+ s.UpdateGuestsForInstance("site", []VM{vm, {ID: "site:other:106", Instance: "site", Node: "other", VMID: 106}}, []Container{ct})+ s.UpdateGuestsForInstance("peer", []VM{{ID: "peer:member:105", Instance: "peer", Node: "member", VMID: 105}}, []Container{{ID: "peer:member:205", Instance: "peer", Node: "member", VMID: 205}})+ before := s.GetSnapshot()+ gotVMs, gotCTs := s.GuestObservationsForNodes("site", map[string]struct{}{"member": {}})+ if len(gotVMs) != 1 || len(gotCTs) != 1 || gotVMs[0].CPU != .95 || !reflect.DeepEqual(gotVMs[0].Memory, vm.Memory) || gotCTs[0].Memory.Usage != 40 || !gotVMs[0].LastSeen.Equal(at) {+ t.Fatalf("source tuple or scope lost: %+v %+v", gotVMs, gotCTs)+ }+ gotVMs[0].Disks[0].Used = 99+ gotVMs[0].Tags[0] = "changed"+ gotVMs[0].IPAddresses[0] = "192.0.2.99"+ gotVMs[0].NetworkInterfaces[0].Addresses[0] = "192.0.2.99"+ *gotVMs[0].OnBoot = false+ gotCTs[0].Disks[0].Used = 99+ gotCTs[0].Tags[0] = "changed"+ gotCTs[0].NetworkInterfaces[0].Addresses[0] = "192.0.2.99"+ *gotCTs[0].OnBoot = false+ if !reflect.DeepEqual(before, s.GetSnapshot()) {+ t.Fatal("retained tuples alias accepted source state")+ }+ for _, nodes := range []map[string]struct{}{nil, {"absent": {}}} {+ v, c := s.GuestObservationsForNodes("site", nodes)+ if len(v)+len(c) != 0 {+ t.Fatal("unselected nodes returned observations")+ }+ }+ var missing *State+ v, c := missing.GuestObservationsForNodes("site", map[string]struct{}{"member": {}})+ if len(v)+len(c) != 0 {+ t.Fatal("missing state manufactured observations")+ }+}diff --git a/internal/monitoring/diagnostic_snapshots_test.go b/internal/monitoring/diagnostic_snapshots_test.goindex b6ec68cfdb..b9f8b830aa 100644--- a/internal/monitoring/diagnostic_snapshots_test.go+++ b/internal/monitoring/diagnostic_snapshots_test.go@@ -3,7 +3,16 @@ package monitoringimport ("context""encoding/json"+ "fmt"+ "github.com/rcourtman/pulse-go-rewrite/internal/alerts"+ "github.com/rcourtman/pulse-go-rewrite/internal/config"+ "github.com/rcourtman/pulse-go-rewrite/internal/unifiedresources"+ "github.com/rcourtman/pulse-go-rewrite/pkg/proxmox"+ "net/http"+ "net/http/httptest"+ "reflect""strings"+ "sync/atomic""testing""time"@@ -1423,3 +1432,198 @@ func TestGuestPollOwnershipStagesAllReadingEffects(t *testing.T) {t.Fatal("accepted generation lost diagnostics or in-memory/persistent History")}}++// Walk the actual efficient enumeration -> fallback -> accepted publication ->+// canonical projection. Malformed successes cannot remove guests, refresh their+// last observation, clear a breach or invalidate template backup eligibility.+// All endpoints are synthetic; no backup, guest command or live send occurs.+func TestGuestInventoryFailurePreservesOwnedReadingsAndRecovers(t *testing.T) {+ for _, failure := range []string{"missing-data", "trailing-value", "truncated-prefix", "duplicate-id"} {+ t.Run(failure, func(t *testing.T) {+ t.Setenv("PULSE_DATA_DIR", t.TempDir())+ var phase, commands, clusterReads, listReads atomic.Int32+ corrupt := func(w http.ResponseWriter, prefix string) {+ switch failure {+ case "missing-data":+ fmt.Fprint(w, `{}`)+ case "trailing-value":+ fmt.Fprint(w, prefix+` {"errors":{"message":"fixture-private-detail"}}`)+ case "truncated-prefix":+ w.Header().Set("Content-Length", fmt.Sprint(len(prefix)+100))+ fmt.Fprint(w, prefix)+ case "duplicate-id":+ var payload struct {+ Data []json.RawMessage `json:"data"`+ }+ if err := json.Unmarshal([]byte(prefix), &payload); err != nil {+ t.Error(err)+ return+ }+ payload.Data = append(payload.Data, payload.Data[0])+ _ = json.NewEncoder(w).Encode(payload)+ }+ }+ const vm = `{"vmid":105,"type":"qemu","node":"member","name":"kept-vm","status":"running","lock":"backup","cpu":0.95,"maxmem":1073741824,"maxdisk":2147483648}`+ const extra = `{"vmid":106,"type":"qemu","node":"member","name":"later-deleted-vm","status":"running","lock":"backup","cpu":0.1,"maxmem":1073741824}`+ const ct = `{"vmid":205,"type":"lxc","node":"member","name":"kept-container","status":"running","lock":"backup","cpu":0.15,"mem":104857600,"maxmem":1073741824,"disk":268435456,"maxdisk":2147483648}`+ const template = `{"vmid":999,"type":"qemu","node":"member","template":1}`+ server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {+ w.Header().Set("Content-Type", "application/json")+ p := phase.Load()+ switch {+ case strings.HasSuffix(r.URL.Path, "/cluster/resources"):+ clusterReads.Add(1)+ if p == 1 || p == 2 {+ corrupt(w, `{"data":[`+vm+`]}`)+ return+ }+ if p == 4 {+ http.NotFound(w, r)+ return+ }+ rows := vm + `,` + ct+ if p == 0 {+ rows += `,` + extra + `,` + template+ }+ if p == 3 {+ rows += `,{"vmid":108,"type":"qemu","node":"member","name":"new-vm","status":"running","lock":"backup","cpu":0,"maxmem":1073741824}`+ }+ fmt.Fprint(w, `{"data":[`+rows+`]}`)+ case strings.HasSuffix(r.URL.Path, "/member/qemu"):+ listReads.Add(1)+ if p == 1 {+ corrupt(w, `{"data":[`+vm+`]}`)+ return+ }+ if p == 4 {+ fmt.Fprint(w, `{"data":[]}`)+ return+ }+ fmt.Fprint(w, `{"data":[`+vm+`]}`)+ case strings.HasSuffix(r.URL.Path, "/member/lxc"):+ listReads.Add(1)+ if p == 1 {+ corrupt(w, `{"data":[`+ct+`]}`)+ return+ }+ if p == 4 {+ fmt.Fprint(w, `{"data":[]}`)+ return+ }+ // Keep the existing older-PVE string identifier compatibility on fallback.+ fmt.Fprint(w, `{"data":[`+strings.Replace(ct, `"vmid":205`, `"vmid":"205"`, 1)+`]}`)+ case strings.Contains(r.URL.Path, "/agent/"):+ commands.Add(1)+ t.Error("inventory proof dispatched an in-guest command")+ http.Error(w, "unexpected command", 500)+ case strings.HasSuffix(r.URL.Path, "/qemu/105/status/current"), strings.HasSuffix(r.URL.Path, "/qemu/106/status/current"), strings.HasSuffix(r.URL.Path, "/qemu/108/status/current"):+ fmt.Fprint(w, `{"data":{"status":"running","lock":"backup","maxmem":1073741824,"mem":104857600}}`)+ case strings.HasSuffix(r.URL.Path, "/lxc/205/status/current"):+ fmt.Fprint(w, `{"data":{"vmid":205,"status":"running","lock":"backup","maxmem":1073741824,"mem":104857600,"maxdisk":2147483648,"disk":268435456}}`)+ case strings.HasSuffix(r.URL.Path, "/config"):+ fmt.Fprint(w, `{"data":{"lock":"backup"}}`)+ default:+ http.NotFound(w, r)+ }+ }))+ defer server.Close()+ client, err := proxmox.NewClient(proxmox.ClientConfig{Host: server.URL, TokenName: "fixture@pve!pulse", TokenValue: "fixture", Timeout: 10 * time.Second})+ if err != nil {+ t.Fatal(err)+ }+ m := newTestPVEMonitor("inventory")+ m.resourceStore = unifiedresources.NewMonitorAdapter(unifiedresources.NewRegistry(nil))+ t.Cleanup(m.alertManager.Stop)+ t.Cleanup(m.notificationMgr.Stop)+ m.replacePVEClient("inventory", client)+ alertCfg := m.alertManager.GetConfig()+ alertCfg.Enabled = true+ alertCfg.GuestDefaults.CPU = &alerts.HysteresisThreshold{Trigger: 80, Clear: 75}+ alertCfg.MetricTimeThresholds = map[string]map[string]int{"guest": {"cpu": 0}}+ m.alertManager.UpdateConfig(alertCfg)+ run := func() models.StateSnapshot {+ t.Helper()+ if err := m.pollGuestsWithFallback(context.Background(), "inventory", &config.PVEInstance{MonitorVMs: true, MonitorContainers: true}, client, []proxmox.Node{{Node: "member", Status: "online"}}, map[string]string{"member": "online"}); err != nil {+ t.Fatal(err)+ }+ snapshot := m.state.GetSnapshot()+ m.updateResourceStore(m.currentStateWithScope())+ return snapshot+ }+ // A second registered connection has its own observation and must survive+ // both failed enumerations and actual removals on this one.+ peer := models.VM{ID: "peer:other:305", Instance: "peer", Node: "other", VMID: 305, Name: "independent-peer", Type: "qemu", Status: "running", LastSeen: time.Now()}+ m.state.UpdateVMsForInstance("peer", []models.VM{peer})+ original := run()+ if len(original.VMs) != 3 || len(original.Containers) != 1 {+ t.Fatalf("initial complete inventory not established: %+v", original)+ }+ templateKey := pveBackupTemplateSubjectKey("inventory", "qemu", "member", 999)+ if _, ok := m.pveBackupTemplateSubjects["inventory"][templateKey]; !ok {+ t.Fatal("initial template not established")+ }+ originalDiagnostics := m.GetDiagnosticSnapshots().Guests+ originalHistory := m.metricsHistory.GetGuestMetrics("inventory:member:105", "cpu", time.Hour)+ originalAlerts := m.alertManager.GetActiveAlerts()+ if len(originalHistory) != 1 || len(originalAlerts) != 1 {+ t.Fatalf("initial history/breach not established: %d/%d", len(originalHistory), len(originalAlerts))+ }+ emit := func(label string, snapshot models.StateSnapshot) {+ registry := unifiedresources.NewRegistry(nil)+ registry.IngestSnapshot(snapshot)+ projection, err := json.Marshal(map[string]any{"case": failure, "phase": label, "snapshot": snapshot, "resources": registry.List()})+ if err != nil {+ t.Fatal(err)+ }+ t.Logf("GUEST_INVENTORY_PROJECTION %s", projection)+ }+ emit("initial", original)+ phase.Store(1)+ failed := run()+ if !reflect.DeepEqual(failed.VMs, original.VMs) || !reflect.DeepEqual(failed.Containers, original.Containers) {+ t.Error("incomplete inventory changed retained guests or their observation time")+ }+ if !reflect.DeepEqual(m.metricsHistory.GetGuestMetrics("inventory:member:105", "cpu", time.Hour), originalHistory) {+ t.Error("failed inventory fabricated a new History sample")+ }+ if !reflect.DeepEqual(m.alertManager.GetActiveAlerts(), originalAlerts) {+ t.Error("failed inventory changed an existing breach")+ }+ if _, ok := m.pveBackupTemplateSubjects["inventory"][templateKey]; !ok {+ t.Error("failed inventory erased backup-template eligibility")+ }+ if !reflect.DeepEqual(m.GetDiagnosticSnapshots().Guests, originalDiagnostics) {+ t.Error("failed inventory changed original diagnostic memory/source/time")+ }+ emit("both-inventories-refused", failed)+ phase.Store(2)+ fallback := run()+ if len(fallback.VMs) != 2 || len(fallback.Containers) != 1 || fallback.VMs[0].VMID == 106 {+ t.Error("complete fallback failed to publish a real deletion")+ }+ if _, ok := m.pveBackupTemplateSubjects["inventory"][templateKey]; ok {+ t.Error("complete fallback retained a deleted template")+ }+ if len(m.metricsHistory.GetGuestMetrics("inventory:member:105", "cpu", time.Hour)) != 2 {+ t.Error("complete fallback failed to resume History")+ }+ emit("complete-node-fallback", fallback)+ before := listReads.Load()+ phase.Store(3)+ recovered := run()+ if len(recovered.VMs) != 3 || len(recovered.Containers) != 1 || listReads.Load() != before {+ t.Error("complete cluster recovery did not return to the efficient route")+ }+ emit("complete-cluster-recovery", recovered)+ phase.Store(4)+ empty := run()+ if len(empty.VMs) != 1 || empty.VMs[0].ID != peer.ID || len(empty.Containers) != 0 {+ t.Error("explicit empty node inventories failed to remove guests within their scope")+ }+ if commands.Load() != 0 || clusterReads.Load() != 5 || listReads.Load() != 6 || len(m.pveGuestPolls) != 0 {+ t.Errorf("unexpected command/retry/owner leak: %d/%d/%d/%d", commands.Load(), clusterReads.Load(), listReads.Load(), len(m.pveGuestPolls))+ }+ emit("complete-scoped-empty", empty)+ })+ }+}diff --git a/internal/monitoring/guest_inventory_admission_test.go b/internal/monitoring/guest_inventory_admission_test.godeleted file mode 100644index c88f2993e0..0000000000--- a/internal/monitoring/guest_inventory_admission_test.go+++ /dev/null@@ -1,208 +0,0 @@-package monitoring--import (- "context"- "encoding/json"- "fmt"- "net/http"- "net/http/httptest"- "reflect"- "strings"- "sync/atomic"- "testing"- "time"-- "github.com/rcourtman/pulse-go-rewrite/internal/alerts"- "github.com/rcourtman/pulse-go-rewrite/internal/config"- "github.com/rcourtman/pulse-go-rewrite/internal/models"- "github.com/rcourtman/pulse-go-rewrite/internal/unifiedresources"- "github.com/rcourtman/pulse-go-rewrite/pkg/proxmox"-)--// Walk the actual efficient enumeration -> fallback -> accepted publication ->-// canonical projection. Malformed successes cannot remove guests, refresh their-// last observation, clear a breach or invalidate template backup eligibility.-// All endpoints are synthetic; no backup, guest command or live send occurs.-func TestGuestInventoryFailurePreservesOwnedReadingsAndRecovers(t *testing.T) {- for _, failure := range []string{"missing-data", "trailing-value", "truncated-prefix", "duplicate-id"} {- t.Run(failure, func(t *testing.T) {- t.Setenv("PULSE_DATA_DIR", t.TempDir())- var phase, commands, clusterReads, listReads atomic.Int32- corrupt := func(w http.ResponseWriter, prefix string) {- switch failure {- case "missing-data":- fmt.Fprint(w, `{}`)- case "trailing-value":- fmt.Fprint(w, prefix+` {"errors":{"message":"fixture-private-detail"}}`)- case "truncated-prefix":- w.Header().Set("Content-Length", fmt.Sprint(len(prefix)+100))- fmt.Fprint(w, prefix)- case "duplicate-id":- var payload struct {- Data []json.RawMessage `json:"data"`- }- if err := json.Unmarshal([]byte(prefix), &payload); err != nil {- t.Error(err)- return- }- payload.Data = append(payload.Data, payload.Data[0])- _ = json.NewEncoder(w).Encode(payload)- }- }- const vm = `{"vmid":105,"type":"qemu","node":"member","name":"kept-vm","status":"running","lock":"backup","cpu":0.95,"maxmem":1073741824,"maxdisk":2147483648}`- const extra = `{"vmid":106,"type":"qemu","node":"member","name":"later-deleted-vm","status":"running","lock":"backup","cpu":0.1,"maxmem":1073741824}`- const ct = `{"vmid":205,"type":"lxc","node":"member","name":"kept-container","status":"running","lock":"backup","cpu":0.15,"mem":104857600,"maxmem":1073741824,"disk":268435456,"maxdisk":2147483648}`- const template = `{"vmid":999,"type":"qemu","node":"member","template":1}`- server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {- w.Header().Set("Content-Type", "application/json")- p := phase.Load()- switch {- case strings.HasSuffix(r.URL.Path, "/cluster/resources"):- clusterReads.Add(1)- if p == 1 || p == 2 {- corrupt(w, `{"data":[`+vm+`]}`)- return- }- if p == 4 {- http.NotFound(w, r)- return- }- rows := vm + `,` + ct- if p == 0 {- rows += `,` + extra + `,` + template- }- if p == 3 {- rows += `,{"vmid":108,"type":"qemu","node":"member","name":"new-vm","status":"running","lock":"backup","cpu":0,"maxmem":1073741824}`- }- fmt.Fprint(w, `{"data":[`+rows+`]}`)- case strings.HasSuffix(r.URL.Path, "/member/qemu"):- listReads.Add(1)- if p == 1 {- corrupt(w, `{"data":[`+vm+`]}`)- return- }- if p == 4 {- fmt.Fprint(w, `{"data":[]}`)- return- }- fmt.Fprint(w, `{"data":[`+vm+`]}`)- case strings.HasSuffix(r.URL.Path, "/member/lxc"):- listReads.Add(1)- if p == 1 {- corrupt(w, `{"data":[`+ct+`]}`)- return- }- if p == 4 {- fmt.Fprint(w, `{"data":[]}`)- return- }- // Keep the existing older-PVE string identifier compatibility on fallback.- fmt.Fprint(w, `{"data":[`+strings.Replace(ct, `"vmid":205`, `"vmid":"205"`, 1)+`]}`)- case strings.Contains(r.URL.Path, "/agent/"):- commands.Add(1)- t.Error("inventory proof dispatched an in-guest command")- http.Error(w, "unexpected command", 500)- case strings.HasSuffix(r.URL.Path, "/qemu/105/status/current"), strings.HasSuffix(r.URL.Path, "/qemu/106/status/current"), strings.HasSuffix(r.URL.Path, "/qemu/108/status/current"):- fmt.Fprint(w, `{"data":{"status":"running","lock":"backup","maxmem":1073741824,"mem":104857600}}`)- case strings.HasSuffix(r.URL.Path, "/lxc/205/status/current"):- fmt.Fprint(w, `{"data":{"vmid":205,"status":"running","lock":"backup","maxmem":1073741824,"mem":104857600,"maxdisk":2147483648,"disk":268435456}}`)- case strings.HasSuffix(r.URL.Path, "/config"):- fmt.Fprint(w, `{"data":{"lock":"backup"}}`)- default:- http.NotFound(w, r)- }- }))- defer server.Close()- client, err := proxmox.NewClient(proxmox.ClientConfig{Host: server.URL, TokenName: "fixture@pve!pulse", TokenValue: "fixture", Timeout: 10 * time.Second})- if err != nil {- t.Fatal(err)- }- m := newTestPVEMonitor("inventory")- t.Cleanup(m.alertManager.Stop)- t.Cleanup(m.notificationMgr.Stop)- m.replacePVEClient("inventory", client)- alertCfg := m.alertManager.GetConfig()- alertCfg.Enabled = true- alertCfg.GuestDefaults.CPU = &alerts.HysteresisThreshold{Trigger: 80, Clear: 75}- alertCfg.MetricTimeThresholds = map[string]map[string]int{"guest": {"cpu": 0}}- m.alertManager.UpdateConfig(alertCfg)- run := func() models.StateSnapshot {- t.Helper()- if err := m.pollGuestsWithFallback(context.Background(), "inventory", &config.PVEInstance{MonitorVMs: true, MonitorContainers: true}, client, []proxmox.Node{{Node: "member", Status: "online"}}, map[string]string{"member": "online"}); err != nil {- t.Fatal(err)- }- return m.state.GetSnapshot()- }- // A second registered connection has its own observation and must survive- // both failed enumerations and actual removals on this one.- peer := models.VM{ID: "peer:other:305", Instance: "peer", Node: "other", VMID: 305, Name: "independent-peer", Type: "qemu", Status: "running", LastSeen: time.Now()}- m.state.UpdateVMsForInstance("peer", []models.VM{peer})- original := run()- if len(original.VMs) != 3 || len(original.Containers) != 1 {- t.Fatalf("initial complete inventory not established: %+v", original)- }- templateKey := pveBackupTemplateSubjectKey("inventory", "qemu", "member", 999)- if _, ok := m.pveBackupTemplateSubjects["inventory"][templateKey]; !ok {- t.Fatal("initial template not established")- }- originalHistory := m.metricsHistory.GetGuestMetrics("inventory:member:105", "cpu", time.Hour)- originalAlerts := m.alertManager.GetActiveAlerts()- if len(originalHistory) != 1 || len(originalAlerts) != 1 {- t.Fatalf("initial history/breach not established: %d/%d", len(originalHistory), len(originalAlerts))- }- emit := func(label string, snapshot models.StateSnapshot) {- registry := unifiedresources.NewRegistry(nil)- registry.IngestSnapshot(snapshot)- projection, err := json.Marshal(map[string]any{"case": failure, "phase": label, "snapshot": snapshot, "resources": registry.List()})- if err != nil {- t.Fatal(err)- }- t.Logf("GUEST_INVENTORY_PROJECTION %s", projection)- }- emit("initial", original)- phase.Store(1)- failed := run()- if !reflect.DeepEqual(failed.VMs, original.VMs) || !reflect.DeepEqual(failed.Containers, original.Containers) {- t.Error("incomplete inventory changed retained guests or their observation time")- }- if !reflect.DeepEqual(m.metricsHistory.GetGuestMetrics("inventory:member:105", "cpu", time.Hour), originalHistory) {- t.Error("failed inventory fabricated a new History sample")- }- if !reflect.DeepEqual(m.alertManager.GetActiveAlerts(), originalAlerts) {- t.Error("failed inventory changed an existing breach")- }- if _, ok := m.pveBackupTemplateSubjects["inventory"][templateKey]; !ok {- t.Error("failed inventory erased backup-template eligibility")- }- emit("both-inventories-refused", failed)- phase.Store(2)- fallback := run()- if len(fallback.VMs) != 2 || len(fallback.Containers) != 1 || fallback.VMs[0].VMID == 106 {- t.Error("complete fallback failed to publish a real deletion")- }- if _, ok := m.pveBackupTemplateSubjects["inventory"][templateKey]; ok {- t.Error("complete fallback retained a deleted template")- }- if len(m.metricsHistory.GetGuestMetrics("inventory:member:105", "cpu", time.Hour)) != 2 {- t.Error("complete fallback failed to resume History")- }- emit("complete-node-fallback", fallback)- before := listReads.Load()- phase.Store(3)- recovered := run()- if len(recovered.VMs) != 3 || len(recovered.Containers) != 1 || listReads.Load() != before {- t.Error("complete cluster recovery did not return to the efficient route")- }- emit("complete-cluster-recovery", recovered)- phase.Store(4)- empty := run()- if len(empty.VMs) != 1 || empty.VMs[0].ID != peer.ID || len(empty.Containers) != 0 {- t.Error("explicit empty node inventories failed to remove guests within their scope")- }- if commands.Load() != 0 || clusterReads.Load() != 5 || listReads.Load() != 6 || len(m.pveGuestPolls) != 0 {- t.Errorf("unexpected command/retry/owner leak: %d/%d/%d/%d", commands.Load(), clusterReads.Load(), listReads.Load(), len(m.pveGuestPolls))- }- emit("complete-scoped-empty", empty)- })- }-}diff --git a/internal/monitoring/guest_inventory_retention.go b/internal/monitoring/guest_inventory_retention.gonew file mode 100644index 0000000000..af30439ace--- /dev/null+++ b/internal/monitoring/guest_inventory_retention.go@@ -0,0 +1,40 @@+package monitoring++import "github.com/rcourtman/pulse-go-rewrite/internal/models"++// Enrichment's previous-state views intentionally carry only selected fields.+// Failed enumeration needs the complete last accepted source tuple instead.+// Keep canonical membership/coordinate selection; do not carry a merged linked+// agent's metric back into Proxmox ownership or renew any observation time.+func (m *Monitor) retainedGuestsForFailedNodes(instance string, nodes map[string]struct{}, previous previousGuestContext) ([]models.VM, []models.Container) {+ vms, containers := m.state.GuestObservationsForNodes(instance, nodes)+ fullVMs := make(map[string]models.VM, len(vms))+ fullContainers := make(map[string]models.Container, len(containers))+ for _, vm := range vms {+ fullVMs[makeGuestID(instance, vm.Node, vm.VMID)] = vm+ }+ for _, ct := range containers {+ fullContainers[makeGuestID(instance, ct.Node, ct.VMID)] = ct+ }+ var retainedVMs []models.VM+ var retainedContainers []models.Container+ for _, vm := range previous.vms {+ if _, failed := nodes[vm.Node]; !failed {+ continue+ }+ if full, exists := fullVMs[makeGuestID(instance, vm.Node, vm.VMID)]; exists {+ vm = full+ }+ retainedVMs = append(retainedVMs, vm)+ }+ for _, ct := range previous.containers {+ if _, failed := nodes[ct.Node]; !failed {+ continue+ }+ if full, exists := fullContainers[makeGuestID(instance, ct.Node, ct.VMID)]; exists {+ ct = full+ }+ retainedContainers = append(retainedContainers, ct)+ }+ return retainedVMs, retainedContainers+}diff --git a/internal/monitoring/guest_poll_ownership_test.go b/internal/monitoring/guest_poll_ownership_test.goindex 25310359d6..c5e4716aaa 100644--- a/internal/monitoring/guest_poll_ownership_test.go+++ b/internal/monitoring/guest_poll_ownership_test.go@@ -449,3 +449,25 @@ func TestGuestPollOwnershipRejectsObsoletePublication(t *testing.T) {}}}++// Failed collection still follows canonical membership, but a merged/enrichment+// view cannot relabel another source's metric as the last Proxmox observation.+func TestGuestPollFailedEnumerationRetainsProducerOwnership(t *testing.T) {+ at := time.Date(2026, 10, 14, 10, 0, 0, 0, time.UTC)+ original := models.VM{ID: "site-a:node:105", Instance: "site-a", Node: "node", VMID: 105, Type: "qemu", CPU: .95, LastSeen: at,+ Memory: models.Memory{Total: 100, Used: 25, Usage: 25, Observation: models.MemoryObservation{State: "last-known", Source: "guest-agent-meminfo", ObservedAt: at.Add(-time.Minute)}},+ Disk: models.Disk{Total: 100, Used: 40, Usage: 40}, GuestAgentStatus: "deferred"}+ m := &Monitor{state: models.NewState()}+ ghost := original+ ghost.ID = "site-a:node:106"+ ghost.VMID = 106+ m.state.UpdateGuestsForInstance("site-a", []models.VM{original, ghost}, nil)+ merged := original+ merged.CPU = .01+ merged.Memory = models.Memory{Total: 100, Used: 99, Usage: 99, Observation: models.MemoryObservation{State: "current", Source: "agent", ObservedAt: at.Add(time.Minute)}}+ previous := previousGuestContext{vms: []models.VM{merged}}+ vms, containers := m.retainedGuestsForFailedNodes("site-a", map[string]struct{}{"node": {}}, previous)+ if len(vms) != 1 || len(containers) != 0 || !reflect.DeepEqual(vms[0].Memory, original.Memory) || vms[0].CPU != .95 || vms[0].GuestAgentStatus != "deferred" || vms[0].Disk.Used != 40 || !vms[0].LastSeen.Equal(at) {+ t.Fatalf("failed enumeration changed source ownership/membership: %+v", vms)+ }+}diff --git a/internal/monitoring/monitor_polling_containers.go b/internal/monitoring/monitor_polling_containers.goindex d0e32c3e7b..d26f0b8762 100644--- a/internal/monitoring/monitor_polling_containers.go+++ b/internal/monitoring/monitor_polling_containers.go@@ -364,12 +364,9 @@ func (m *Monitor) collectContainersWithNodes(ctx context.Context, instanceName spreservedContainers := 0if len(failedNodeNames) > 0 {- for _, container := range prevGuests.containers {- if _, failed := failedNodeNames[container.Node]; failed {- allContainers = append(allContainers, container)- preservedContainers++- }- }+ _, retained := m.retainedGuestsForFailedNodes(instanceName, failedNodeNames, prevGuests)+ allContainers = append(allContainers, retained...)+ preservedContainers = len(retained)}if preservedContainers > 0 {log.Warn().diff --git a/internal/monitoring/monitor_polling_vm.go b/internal/monitoring/monitor_polling_vm.goindex 5ed8d32db2..def9948e1c 100644--- a/internal/monitoring/monitor_polling_vm.go+++ b/internal/monitoring/monitor_polling_vm.go@@ -105,12 +105,9 @@ func (m *Monitor) collectVMsWithNodes(ctx context.Context, instanceName string,preservedVMs := 0if len(failedNodeNames) > 0 {- for _, vm := range prevGuests.vms {- if _, failed := failedNodeNames[vm.Node]; failed {- allVMs = append(allVMs, vm)- preservedVMs++- }- }+ retained, _ := m.retainedGuestsForFailedNodes(instanceName, failedNodeNames, prevGuests)+ allVMs = append(allVMs, retained...)+ preservedVMs = len(retained)}if preservedVMs > 0 {log.Warn().