Skip to content

armgit / Pulse

Public Imported from GitHub
0

Real-time monitoring dashboard for Proxmox VE, PBS, Docker, Kubernetes, TrueNAS and vSphere. Self-hosted, with alerts for failed backups, full disks, unhealthy containers and offline nodes.

Retain accepted guest observations when node enumeration fails

pulse-triage[bot]4b205a8968cd
diff --git a/docs/release-control/v6/internal/subsystems/monitoring.md b/docs/release-control/v6/internal/subsystems/monitoring.md
index 466c366b66..4b54080404 100644
--- a/docs/release-control/v6/internal/subsystems/monitoring.md
+++ b/docs/release-control/v6/internal/subsystems/monitoring.md
@@ -7688,8 +7688,22 @@ publication ownership or grace-period rules.
`guest_inventory_response_test.go` covers complete-body and all-or-nothing
admission on all three real HTTP readers; `io_counters_test.go` covers
scoped-empty, extended fields, counter presence and receipt-time controls.
-`guest_inventory_admission_test.go` covers actual efficient/fallback collection,
+`diagnostic_snapshots_test.go` covers actual efficient/fallback collection,
owned publication, retained observation times, breach/History/template
continuity, independent-connection preservation and normal recovery/removal.
These controlled source proofs do not establish native thaw/restart recovery,
installed trust, report attribution or release availability.
+
+Failed-node retention selects canonical guest membership but takes the unchanged
+complete tuple from that connection's last accepted producer state. Enrichment's
+partial previous-state projection is not an observation: it must not turn an
+unavailable/retained memory reading into zero, lose CPU/disk usage or change the
+original in-guest collection state. A node-scoped detached snapshot copies only
+selected guests, not unrelated host disks or Kubernetes state. The existing
+canonical/enrichment path remains the fallback when no accepted source tuple
+exists; linked-agent evidence is never promoted to Proxmox ownership. The
+same publication owner still admits or withdraws the eventual generation.
+`proxmox_guest_observations_test.go` checks scope, original memory provenance,
+CPU/disk/read time and nested detachment, alongside the end-to-end admission
+controls above. The first complete admission proof exposed the former partial
+retention and is retained as failed evidence, not weakened into acceptance.
diff --git a/internal/models/proxmox_guest_observations.go b/internal/models/proxmox_guest_observations.go
new file mode 100644
index 0000000000..dac8a769ec
--- /dev/null
+++ b/internal/models/proxmox_guest_observations.go
@@ -0,0 +1,32 @@
+package models
+
+// GuestObservationsForNodes detaches the last accepted producer tuples for one
+// connection's failed nodes. It is not a canonical/merged consumer projection:
+// missing enumeration must not rebuild memory, disk or CPU from partial views.
+// Copy only these guests, not unrelated hosts, disks or Kubernetes inventories.
+func (s *State) GuestObservationsForNodes(instance string, nodes map[string]struct{}) ([]VM, []Container) {
+ if s == nil || len(nodes) == 0 {
+ return nil, nil
+ }
+ s.mu.RLock()
+ defer s.mu.RUnlock()
+ var vms []VM
+ var containers []Container
+ for _, vm := range s.VMs {
+ if _, selected := nodes[vm.Node]; !selected || vm.Instance != instance {
+ continue
+ }
+ copied := cloneVM(vm)
+ copied.OnBoot = cloneBoolPtr(vm.OnBoot)
+ vms = append(vms, copied)
+ }
+ for _, ct := range s.Containers {
+ if _, selected := nodes[ct.Node]; !selected || ct.Instance != instance {
+ continue
+ }
+ copied := cloneContainer(ct)
+ copied.OnBoot = cloneBoolPtr(ct.OnBoot)
+ containers = append(containers, copied)
+ }
+ return vms, containers
+}
diff --git a/internal/models/proxmox_guest_observations_test.go b/internal/models/proxmox_guest_observations_test.go
new file mode 100644
index 0000000000..9ee624bf36
--- /dev/null
+++ b/internal/models/proxmox_guest_observations_test.go
@@ -0,0 +1,47 @@
+package models
+
+import (
+ "reflect"
+ "testing"
+ "time"
+)
+
+func TestGuestObservationsForNodesOwnsOnlyAcceptedScopedTuples(t *testing.T) {
+ s := NewState()
+ at := time.Date(2026, 10, 14, 10, 0, 0, 0, time.UTC)
+ onBoot := true
+ vm := VM{ID: "site:member:105", Instance: "site", Node: "member", VMID: 105, CPU: .95, LastSeen: at, OnBoot: &onBoot,
+ Memory: Memory{Used: 40, Total: 100, Usage: 40, Observation: MemoryObservation{State: "last-known", Source: "guest-agent-meminfo", ObservedAt: at.Add(-time.Minute)}},
+ Disk: Disk{Used: 20, Total: 100, Usage: 20}, Disks: []Disk{{Used: 20, Total: 100, Mountpoint: "C:\\"}}, IPAddresses: []string{"192.0.2.1"}, Tags: []string{"production"}, NetworkInterfaces: []GuestNetworkInterface{{Name: "Ethernet", Addresses: []string{"192.0.2.1"}}}}
+ ct := Container{ID: "site:member:205", Instance: "site", Node: "member", VMID: 205, CPU: .25, LastSeen: at, OnBoot: &onBoot, Memory: vm.Memory, Disk: vm.Disk, Disks: vm.Disks, Tags: vm.Tags, IPAddresses: vm.IPAddresses, NetworkInterfaces: vm.NetworkInterfaces}
+ s.UpdateGuestsForInstance("site", []VM{vm, {ID: "site:other:106", Instance: "site", Node: "other", VMID: 106}}, []Container{ct})
+ s.UpdateGuestsForInstance("peer", []VM{{ID: "peer:member:105", Instance: "peer", Node: "member", VMID: 105}}, []Container{{ID: "peer:member:205", Instance: "peer", Node: "member", VMID: 205}})
+ before := s.GetSnapshot()
+ gotVMs, gotCTs := s.GuestObservationsForNodes("site", map[string]struct{}{"member": {}})
+ if len(gotVMs) != 1 || len(gotCTs) != 1 || gotVMs[0].CPU != .95 || !reflect.DeepEqual(gotVMs[0].Memory, vm.Memory) || gotCTs[0].Memory.Usage != 40 || !gotVMs[0].LastSeen.Equal(at) {
+ t.Fatalf("source tuple or scope lost: %+v %+v", gotVMs, gotCTs)
+ }
+ gotVMs[0].Disks[0].Used = 99
+ gotVMs[0].Tags[0] = "changed"
+ gotVMs[0].IPAddresses[0] = "192.0.2.99"
+ gotVMs[0].NetworkInterfaces[0].Addresses[0] = "192.0.2.99"
+ *gotVMs[0].OnBoot = false
+ gotCTs[0].Disks[0].Used = 99
+ gotCTs[0].Tags[0] = "changed"
+ gotCTs[0].NetworkInterfaces[0].Addresses[0] = "192.0.2.99"
+ *gotCTs[0].OnBoot = false
+ if !reflect.DeepEqual(before, s.GetSnapshot()) {
+ t.Fatal("retained tuples alias accepted source state")
+ }
+ for _, nodes := range []map[string]struct{}{nil, {"absent": {}}} {
+ v, c := s.GuestObservationsForNodes("site", nodes)
+ if len(v)+len(c) != 0 {
+ t.Fatal("unselected nodes returned observations")
+ }
+ }
+ var missing *State
+ v, c := missing.GuestObservationsForNodes("site", map[string]struct{}{"member": {}})
+ if len(v)+len(c) != 0 {
+ t.Fatal("missing state manufactured observations")
+ }
+}
diff --git a/internal/monitoring/diagnostic_snapshots_test.go b/internal/monitoring/diagnostic_snapshots_test.go
index b6ec68cfdb..b9f8b830aa 100644
--- a/internal/monitoring/diagnostic_snapshots_test.go
+++ b/internal/monitoring/diagnostic_snapshots_test.go
@@ -3,7 +3,16 @@ package monitoring
import (
"context"
"encoding/json"
+ "fmt"
+ "github.com/rcourtman/pulse-go-rewrite/internal/alerts"
+ "github.com/rcourtman/pulse-go-rewrite/internal/config"
+ "github.com/rcourtman/pulse-go-rewrite/internal/unifiedresources"
+ "github.com/rcourtman/pulse-go-rewrite/pkg/proxmox"
+ "net/http"
+ "net/http/httptest"
+ "reflect"
"strings"
+ "sync/atomic"
"testing"
"time"
@@ -1423,3 +1432,198 @@ func TestGuestPollOwnershipStagesAllReadingEffects(t *testing.T) {
t.Fatal("accepted generation lost diagnostics or in-memory/persistent History")
}
}
+
+// Walk the actual efficient enumeration -> fallback -> accepted publication ->
+// canonical projection. Malformed successes cannot remove guests, refresh their
+// last observation, clear a breach or invalidate template backup eligibility.
+// All endpoints are synthetic; no backup, guest command or live send occurs.
+func TestGuestInventoryFailurePreservesOwnedReadingsAndRecovers(t *testing.T) {
+ for _, failure := range []string{"missing-data", "trailing-value", "truncated-prefix", "duplicate-id"} {
+ t.Run(failure, func(t *testing.T) {
+ t.Setenv("PULSE_DATA_DIR", t.TempDir())
+ var phase, commands, clusterReads, listReads atomic.Int32
+ corrupt := func(w http.ResponseWriter, prefix string) {
+ switch failure {
+ case "missing-data":
+ fmt.Fprint(w, `{}`)
+ case "trailing-value":
+ fmt.Fprint(w, prefix+` {"errors":{"message":"fixture-private-detail"}}`)
+ case "truncated-prefix":
+ w.Header().Set("Content-Length", fmt.Sprint(len(prefix)+100))
+ fmt.Fprint(w, prefix)
+ case "duplicate-id":
+ var payload struct {
+ Data []json.RawMessage `json:"data"`
+ }
+ if err := json.Unmarshal([]byte(prefix), &payload); err != nil {
+ t.Error(err)
+ return
+ }
+ payload.Data = append(payload.Data, payload.Data[0])
+ _ = json.NewEncoder(w).Encode(payload)
+ }
+ }
+ const vm = `{"vmid":105,"type":"qemu","node":"member","name":"kept-vm","status":"running","lock":"backup","cpu":0.95,"maxmem":1073741824,"maxdisk":2147483648}`
+ const extra = `{"vmid":106,"type":"qemu","node":"member","name":"later-deleted-vm","status":"running","lock":"backup","cpu":0.1,"maxmem":1073741824}`
+ const ct = `{"vmid":205,"type":"lxc","node":"member","name":"kept-container","status":"running","lock":"backup","cpu":0.15,"mem":104857600,"maxmem":1073741824,"disk":268435456,"maxdisk":2147483648}`
+ const template = `{"vmid":999,"type":"qemu","node":"member","template":1}`
+ server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+ w.Header().Set("Content-Type", "application/json")
+ p := phase.Load()
+ switch {
+ case strings.HasSuffix(r.URL.Path, "/cluster/resources"):
+ clusterReads.Add(1)
+ if p == 1 || p == 2 {
+ corrupt(w, `{"data":[`+vm+`]}`)
+ return
+ }
+ if p == 4 {
+ http.NotFound(w, r)
+ return
+ }
+ rows := vm + `,` + ct
+ if p == 0 {
+ rows += `,` + extra + `,` + template
+ }
+ if p == 3 {
+ rows += `,{"vmid":108,"type":"qemu","node":"member","name":"new-vm","status":"running","lock":"backup","cpu":0,"maxmem":1073741824}`
+ }
+ fmt.Fprint(w, `{"data":[`+rows+`]}`)
+ case strings.HasSuffix(r.URL.Path, "/member/qemu"):
+ listReads.Add(1)
+ if p == 1 {
+ corrupt(w, `{"data":[`+vm+`]}`)
+ return
+ }
+ if p == 4 {
+ fmt.Fprint(w, `{"data":[]}`)
+ return
+ }
+ fmt.Fprint(w, `{"data":[`+vm+`]}`)
+ case strings.HasSuffix(r.URL.Path, "/member/lxc"):
+ listReads.Add(1)
+ if p == 1 {
+ corrupt(w, `{"data":[`+ct+`]}`)
+ return
+ }
+ if p == 4 {
+ fmt.Fprint(w, `{"data":[]}`)
+ return
+ }
+ // Keep the existing older-PVE string identifier compatibility on fallback.
+ fmt.Fprint(w, `{"data":[`+strings.Replace(ct, `"vmid":205`, `"vmid":"205"`, 1)+`]}`)
+ case strings.Contains(r.URL.Path, "/agent/"):
+ commands.Add(1)
+ t.Error("inventory proof dispatched an in-guest command")
+ http.Error(w, "unexpected command", 500)
+ case strings.HasSuffix(r.URL.Path, "/qemu/105/status/current"), strings.HasSuffix(r.URL.Path, "/qemu/106/status/current"), strings.HasSuffix(r.URL.Path, "/qemu/108/status/current"):
+ fmt.Fprint(w, `{"data":{"status":"running","lock":"backup","maxmem":1073741824,"mem":104857600}}`)
+ case strings.HasSuffix(r.URL.Path, "/lxc/205/status/current"):
+ fmt.Fprint(w, `{"data":{"vmid":205,"status":"running","lock":"backup","maxmem":1073741824,"mem":104857600,"maxdisk":2147483648,"disk":268435456}}`)
+ case strings.HasSuffix(r.URL.Path, "/config"):
+ fmt.Fprint(w, `{"data":{"lock":"backup"}}`)
+ default:
+ http.NotFound(w, r)
+ }
+ }))
+ defer server.Close()
+ client, err := proxmox.NewClient(proxmox.ClientConfig{Host: server.URL, TokenName: "fixture@pve!pulse", TokenValue: "fixture", Timeout: 10 * time.Second})
+ if err != nil {
+ t.Fatal(err)
+ }
+ m := newTestPVEMonitor("inventory")
+ m.resourceStore = unifiedresources.NewMonitorAdapter(unifiedresources.NewRegistry(nil))
+ t.Cleanup(m.alertManager.Stop)
+ t.Cleanup(m.notificationMgr.Stop)
+ m.replacePVEClient("inventory", client)
+ alertCfg := m.alertManager.GetConfig()
+ alertCfg.Enabled = true
+ alertCfg.GuestDefaults.CPU = &alerts.HysteresisThreshold{Trigger: 80, Clear: 75}
+ alertCfg.MetricTimeThresholds = map[string]map[string]int{"guest": {"cpu": 0}}
+ m.alertManager.UpdateConfig(alertCfg)
+ run := func() models.StateSnapshot {
+ t.Helper()
+ if err := m.pollGuestsWithFallback(context.Background(), "inventory", &config.PVEInstance{MonitorVMs: true, MonitorContainers: true}, client, []proxmox.Node{{Node: "member", Status: "online"}}, map[string]string{"member": "online"}); err != nil {
+ t.Fatal(err)
+ }
+ snapshot := m.state.GetSnapshot()
+ m.updateResourceStore(m.currentStateWithScope())
+ return snapshot
+ }
+ // A second registered connection has its own observation and must survive
+ // both failed enumerations and actual removals on this one.
+ peer := models.VM{ID: "peer:other:305", Instance: "peer", Node: "other", VMID: 305, Name: "independent-peer", Type: "qemu", Status: "running", LastSeen: time.Now()}
+ m.state.UpdateVMsForInstance("peer", []models.VM{peer})
+ original := run()
+ if len(original.VMs) != 3 || len(original.Containers) != 1 {
+ t.Fatalf("initial complete inventory not established: %+v", original)
+ }
+ templateKey := pveBackupTemplateSubjectKey("inventory", "qemu", "member", 999)
+ if _, ok := m.pveBackupTemplateSubjects["inventory"][templateKey]; !ok {
+ t.Fatal("initial template not established")
+ }
+ originalDiagnostics := m.GetDiagnosticSnapshots().Guests
+ originalHistory := m.metricsHistory.GetGuestMetrics("inventory:member:105", "cpu", time.Hour)
+ originalAlerts := m.alertManager.GetActiveAlerts()
+ if len(originalHistory) != 1 || len(originalAlerts) != 1 {
+ t.Fatalf("initial history/breach not established: %d/%d", len(originalHistory), len(originalAlerts))
+ }
+ emit := func(label string, snapshot models.StateSnapshot) {
+ registry := unifiedresources.NewRegistry(nil)
+ registry.IngestSnapshot(snapshot)
+ projection, err := json.Marshal(map[string]any{"case": failure, "phase": label, "snapshot": snapshot, "resources": registry.List()})
+ if err != nil {
+ t.Fatal(err)
+ }
+ t.Logf("GUEST_INVENTORY_PROJECTION %s", projection)
+ }
+ emit("initial", original)
+ phase.Store(1)
+ failed := run()
+ if !reflect.DeepEqual(failed.VMs, original.VMs) || !reflect.DeepEqual(failed.Containers, original.Containers) {
+ t.Error("incomplete inventory changed retained guests or their observation time")
+ }
+ if !reflect.DeepEqual(m.metricsHistory.GetGuestMetrics("inventory:member:105", "cpu", time.Hour), originalHistory) {
+ t.Error("failed inventory fabricated a new History sample")
+ }
+ if !reflect.DeepEqual(m.alertManager.GetActiveAlerts(), originalAlerts) {
+ t.Error("failed inventory changed an existing breach")
+ }
+ if _, ok := m.pveBackupTemplateSubjects["inventory"][templateKey]; !ok {
+ t.Error("failed inventory erased backup-template eligibility")
+ }
+ if !reflect.DeepEqual(m.GetDiagnosticSnapshots().Guests, originalDiagnostics) {
+ t.Error("failed inventory changed original diagnostic memory/source/time")
+ }
+ emit("both-inventories-refused", failed)
+ phase.Store(2)
+ fallback := run()
+ if len(fallback.VMs) != 2 || len(fallback.Containers) != 1 || fallback.VMs[0].VMID == 106 {
+ t.Error("complete fallback failed to publish a real deletion")
+ }
+ if _, ok := m.pveBackupTemplateSubjects["inventory"][templateKey]; ok {
+ t.Error("complete fallback retained a deleted template")
+ }
+ if len(m.metricsHistory.GetGuestMetrics("inventory:member:105", "cpu", time.Hour)) != 2 {
+ t.Error("complete fallback failed to resume History")
+ }
+ emit("complete-node-fallback", fallback)
+ before := listReads.Load()
+ phase.Store(3)
+ recovered := run()
+ if len(recovered.VMs) != 3 || len(recovered.Containers) != 1 || listReads.Load() != before {
+ t.Error("complete cluster recovery did not return to the efficient route")
+ }
+ emit("complete-cluster-recovery", recovered)
+ phase.Store(4)
+ empty := run()
+ if len(empty.VMs) != 1 || empty.VMs[0].ID != peer.ID || len(empty.Containers) != 0 {
+ t.Error("explicit empty node inventories failed to remove guests within their scope")
+ }
+ if commands.Load() != 0 || clusterReads.Load() != 5 || listReads.Load() != 6 || len(m.pveGuestPolls) != 0 {
+ t.Errorf("unexpected command/retry/owner leak: %d/%d/%d/%d", commands.Load(), clusterReads.Load(), listReads.Load(), len(m.pveGuestPolls))
+ }
+ emit("complete-scoped-empty", empty)
+ })
+ }
+}
diff --git a/internal/monitoring/guest_inventory_admission_test.go b/internal/monitoring/guest_inventory_admission_test.go
deleted file mode 100644
index c88f2993e0..0000000000
--- a/internal/monitoring/guest_inventory_admission_test.go
+++ /dev/null
@@ -1,208 +0,0 @@
-package monitoring
-
-import (
- "context"
- "encoding/json"
- "fmt"
- "net/http"
- "net/http/httptest"
- "reflect"
- "strings"
- "sync/atomic"
- "testing"
- "time"
-
- "github.com/rcourtman/pulse-go-rewrite/internal/alerts"
- "github.com/rcourtman/pulse-go-rewrite/internal/config"
- "github.com/rcourtman/pulse-go-rewrite/internal/models"
- "github.com/rcourtman/pulse-go-rewrite/internal/unifiedresources"
- "github.com/rcourtman/pulse-go-rewrite/pkg/proxmox"
-)
-
-// Walk the actual efficient enumeration -> fallback -> accepted publication ->
-// canonical projection. Malformed successes cannot remove guests, refresh their
-// last observation, clear a breach or invalidate template backup eligibility.
-// All endpoints are synthetic; no backup, guest command or live send occurs.
-func TestGuestInventoryFailurePreservesOwnedReadingsAndRecovers(t *testing.T) {
- for _, failure := range []string{"missing-data", "trailing-value", "truncated-prefix", "duplicate-id"} {
- t.Run(failure, func(t *testing.T) {
- t.Setenv("PULSE_DATA_DIR", t.TempDir())
- var phase, commands, clusterReads, listReads atomic.Int32
- corrupt := func(w http.ResponseWriter, prefix string) {
- switch failure {
- case "missing-data":
- fmt.Fprint(w, `{}`)
- case "trailing-value":
- fmt.Fprint(w, prefix+` {"errors":{"message":"fixture-private-detail"}}`)
- case "truncated-prefix":
- w.Header().Set("Content-Length", fmt.Sprint(len(prefix)+100))
- fmt.Fprint(w, prefix)
- case "duplicate-id":
- var payload struct {
- Data []json.RawMessage `json:"data"`
- }
- if err := json.Unmarshal([]byte(prefix), &payload); err != nil {
- t.Error(err)
- return
- }
- payload.Data = append(payload.Data, payload.Data[0])
- _ = json.NewEncoder(w).Encode(payload)
- }
- }
- const vm = `{"vmid":105,"type":"qemu","node":"member","name":"kept-vm","status":"running","lock":"backup","cpu":0.95,"maxmem":1073741824,"maxdisk":2147483648}`
- const extra = `{"vmid":106,"type":"qemu","node":"member","name":"later-deleted-vm","status":"running","lock":"backup","cpu":0.1,"maxmem":1073741824}`
- const ct = `{"vmid":205,"type":"lxc","node":"member","name":"kept-container","status":"running","lock":"backup","cpu":0.15,"mem":104857600,"maxmem":1073741824,"disk":268435456,"maxdisk":2147483648}`
- const template = `{"vmid":999,"type":"qemu","node":"member","template":1}`
- server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
- w.Header().Set("Content-Type", "application/json")
- p := phase.Load()
- switch {
- case strings.HasSuffix(r.URL.Path, "/cluster/resources"):
- clusterReads.Add(1)
- if p == 1 || p == 2 {
- corrupt(w, `{"data":[`+vm+`]}`)
- return
- }
- if p == 4 {
- http.NotFound(w, r)
- return
- }
- rows := vm + `,` + ct
- if p == 0 {
- rows += `,` + extra + `,` + template
- }
- if p == 3 {
- rows += `,{"vmid":108,"type":"qemu","node":"member","name":"new-vm","status":"running","lock":"backup","cpu":0,"maxmem":1073741824}`
- }
- fmt.Fprint(w, `{"data":[`+rows+`]}`)
- case strings.HasSuffix(r.URL.Path, "/member/qemu"):
- listReads.Add(1)
- if p == 1 {
- corrupt(w, `{"data":[`+vm+`]}`)
- return
- }
- if p == 4 {
- fmt.Fprint(w, `{"data":[]}`)
- return
- }
- fmt.Fprint(w, `{"data":[`+vm+`]}`)
- case strings.HasSuffix(r.URL.Path, "/member/lxc"):
- listReads.Add(1)
- if p == 1 {
- corrupt(w, `{"data":[`+ct+`]}`)
- return
- }
- if p == 4 {
- fmt.Fprint(w, `{"data":[]}`)
- return
- }
- // Keep the existing older-PVE string identifier compatibility on fallback.
- fmt.Fprint(w, `{"data":[`+strings.Replace(ct, `"vmid":205`, `"vmid":"205"`, 1)+`]}`)
- case strings.Contains(r.URL.Path, "/agent/"):
- commands.Add(1)
- t.Error("inventory proof dispatched an in-guest command")
- http.Error(w, "unexpected command", 500)
- case strings.HasSuffix(r.URL.Path, "/qemu/105/status/current"), strings.HasSuffix(r.URL.Path, "/qemu/106/status/current"), strings.HasSuffix(r.URL.Path, "/qemu/108/status/current"):
- fmt.Fprint(w, `{"data":{"status":"running","lock":"backup","maxmem":1073741824,"mem":104857600}}`)
- case strings.HasSuffix(r.URL.Path, "/lxc/205/status/current"):
- fmt.Fprint(w, `{"data":{"vmid":205,"status":"running","lock":"backup","maxmem":1073741824,"mem":104857600,"maxdisk":2147483648,"disk":268435456}}`)
- case strings.HasSuffix(r.URL.Path, "/config"):
- fmt.Fprint(w, `{"data":{"lock":"backup"}}`)
- default:
- http.NotFound(w, r)
- }
- }))
- defer server.Close()
- client, err := proxmox.NewClient(proxmox.ClientConfig{Host: server.URL, TokenName: "fixture@pve!pulse", TokenValue: "fixture", Timeout: 10 * time.Second})
- if err != nil {
- t.Fatal(err)
- }
- m := newTestPVEMonitor("inventory")
- t.Cleanup(m.alertManager.Stop)
- t.Cleanup(m.notificationMgr.Stop)
- m.replacePVEClient("inventory", client)
- alertCfg := m.alertManager.GetConfig()
- alertCfg.Enabled = true
- alertCfg.GuestDefaults.CPU = &alerts.HysteresisThreshold{Trigger: 80, Clear: 75}
- alertCfg.MetricTimeThresholds = map[string]map[string]int{"guest": {"cpu": 0}}
- m.alertManager.UpdateConfig(alertCfg)
- run := func() models.StateSnapshot {
- t.Helper()
- if err := m.pollGuestsWithFallback(context.Background(), "inventory", &config.PVEInstance{MonitorVMs: true, MonitorContainers: true}, client, []proxmox.Node{{Node: "member", Status: "online"}}, map[string]string{"member": "online"}); err != nil {
- t.Fatal(err)
- }
- return m.state.GetSnapshot()
- }
- // A second registered connection has its own observation and must survive
- // both failed enumerations and actual removals on this one.
- peer := models.VM{ID: "peer:other:305", Instance: "peer", Node: "other", VMID: 305, Name: "independent-peer", Type: "qemu", Status: "running", LastSeen: time.Now()}
- m.state.UpdateVMsForInstance("peer", []models.VM{peer})
- original := run()
- if len(original.VMs) != 3 || len(original.Containers) != 1 {
- t.Fatalf("initial complete inventory not established: %+v", original)
- }
- templateKey := pveBackupTemplateSubjectKey("inventory", "qemu", "member", 999)
- if _, ok := m.pveBackupTemplateSubjects["inventory"][templateKey]; !ok {
- t.Fatal("initial template not established")
- }
- originalHistory := m.metricsHistory.GetGuestMetrics("inventory:member:105", "cpu", time.Hour)
- originalAlerts := m.alertManager.GetActiveAlerts()
- if len(originalHistory) != 1 || len(originalAlerts) != 1 {
- t.Fatalf("initial history/breach not established: %d/%d", len(originalHistory), len(originalAlerts))
- }
- emit := func(label string, snapshot models.StateSnapshot) {
- registry := unifiedresources.NewRegistry(nil)
- registry.IngestSnapshot(snapshot)
- projection, err := json.Marshal(map[string]any{"case": failure, "phase": label, "snapshot": snapshot, "resources": registry.List()})
- if err != nil {
- t.Fatal(err)
- }
- t.Logf("GUEST_INVENTORY_PROJECTION %s", projection)
- }
- emit("initial", original)
- phase.Store(1)
- failed := run()
- if !reflect.DeepEqual(failed.VMs, original.VMs) || !reflect.DeepEqual(failed.Containers, original.Containers) {
- t.Error("incomplete inventory changed retained guests or their observation time")
- }
- if !reflect.DeepEqual(m.metricsHistory.GetGuestMetrics("inventory:member:105", "cpu", time.Hour), originalHistory) {
- t.Error("failed inventory fabricated a new History sample")
- }
- if !reflect.DeepEqual(m.alertManager.GetActiveAlerts(), originalAlerts) {
- t.Error("failed inventory changed an existing breach")
- }
- if _, ok := m.pveBackupTemplateSubjects["inventory"][templateKey]; !ok {
- t.Error("failed inventory erased backup-template eligibility")
- }
- emit("both-inventories-refused", failed)
- phase.Store(2)
- fallback := run()
- if len(fallback.VMs) != 2 || len(fallback.Containers) != 1 || fallback.VMs[0].VMID == 106 {
- t.Error("complete fallback failed to publish a real deletion")
- }
- if _, ok := m.pveBackupTemplateSubjects["inventory"][templateKey]; ok {
- t.Error("complete fallback retained a deleted template")
- }
- if len(m.metricsHistory.GetGuestMetrics("inventory:member:105", "cpu", time.Hour)) != 2 {
- t.Error("complete fallback failed to resume History")
- }
- emit("complete-node-fallback", fallback)
- before := listReads.Load()
- phase.Store(3)
- recovered := run()
- if len(recovered.VMs) != 3 || len(recovered.Containers) != 1 || listReads.Load() != before {
- t.Error("complete cluster recovery did not return to the efficient route")
- }
- emit("complete-cluster-recovery", recovered)
- phase.Store(4)
- empty := run()
- if len(empty.VMs) != 1 || empty.VMs[0].ID != peer.ID || len(empty.Containers) != 0 {
- t.Error("explicit empty node inventories failed to remove guests within their scope")
- }
- if commands.Load() != 0 || clusterReads.Load() != 5 || listReads.Load() != 6 || len(m.pveGuestPolls) != 0 {
- t.Errorf("unexpected command/retry/owner leak: %d/%d/%d/%d", commands.Load(), clusterReads.Load(), listReads.Load(), len(m.pveGuestPolls))
- }
- emit("complete-scoped-empty", empty)
- })
- }
-}
diff --git a/internal/monitoring/guest_inventory_retention.go b/internal/monitoring/guest_inventory_retention.go
new file mode 100644
index 0000000000..af30439ace
--- /dev/null
+++ b/internal/monitoring/guest_inventory_retention.go
@@ -0,0 +1,40 @@
+package monitoring
+
+import "github.com/rcourtman/pulse-go-rewrite/internal/models"
+
+// Enrichment's previous-state views intentionally carry only selected fields.
+// Failed enumeration needs the complete last accepted source tuple instead.
+// Keep canonical membership/coordinate selection; do not carry a merged linked
+// agent's metric back into Proxmox ownership or renew any observation time.
+func (m *Monitor) retainedGuestsForFailedNodes(instance string, nodes map[string]struct{}, previous previousGuestContext) ([]models.VM, []models.Container) {
+ vms, containers := m.state.GuestObservationsForNodes(instance, nodes)
+ fullVMs := make(map[string]models.VM, len(vms))
+ fullContainers := make(map[string]models.Container, len(containers))
+ for _, vm := range vms {
+ fullVMs[makeGuestID(instance, vm.Node, vm.VMID)] = vm
+ }
+ for _, ct := range containers {
+ fullContainers[makeGuestID(instance, ct.Node, ct.VMID)] = ct
+ }
+ var retainedVMs []models.VM
+ var retainedContainers []models.Container
+ for _, vm := range previous.vms {
+ if _, failed := nodes[vm.Node]; !failed {
+ continue
+ }
+ if full, exists := fullVMs[makeGuestID(instance, vm.Node, vm.VMID)]; exists {
+ vm = full
+ }
+ retainedVMs = append(retainedVMs, vm)
+ }
+ for _, ct := range previous.containers {
+ if _, failed := nodes[ct.Node]; !failed {
+ continue
+ }
+ if full, exists := fullContainers[makeGuestID(instance, ct.Node, ct.VMID)]; exists {
+ ct = full
+ }
+ retainedContainers = append(retainedContainers, ct)
+ }
+ return retainedVMs, retainedContainers
+}
diff --git a/internal/monitoring/guest_poll_ownership_test.go b/internal/monitoring/guest_poll_ownership_test.go
index 25310359d6..c5e4716aaa 100644
--- a/internal/monitoring/guest_poll_ownership_test.go
+++ b/internal/monitoring/guest_poll_ownership_test.go
@@ -449,3 +449,25 @@ func TestGuestPollOwnershipRejectsObsoletePublication(t *testing.T) {
}
}
}
+
+// Failed collection still follows canonical membership, but a merged/enrichment
+// view cannot relabel another source's metric as the last Proxmox observation.
+func TestGuestPollFailedEnumerationRetainsProducerOwnership(t *testing.T) {
+ at := time.Date(2026, 10, 14, 10, 0, 0, 0, time.UTC)
+ original := models.VM{ID: "site-a:node:105", Instance: "site-a", Node: "node", VMID: 105, Type: "qemu", CPU: .95, LastSeen: at,
+ Memory: models.Memory{Total: 100, Used: 25, Usage: 25, Observation: models.MemoryObservation{State: "last-known", Source: "guest-agent-meminfo", ObservedAt: at.Add(-time.Minute)}},
+ Disk: models.Disk{Total: 100, Used: 40, Usage: 40}, GuestAgentStatus: "deferred"}
+ m := &Monitor{state: models.NewState()}
+ ghost := original
+ ghost.ID = "site-a:node:106"
+ ghost.VMID = 106
+ m.state.UpdateGuestsForInstance("site-a", []models.VM{original, ghost}, nil)
+ merged := original
+ merged.CPU = .01
+ merged.Memory = models.Memory{Total: 100, Used: 99, Usage: 99, Observation: models.MemoryObservation{State: "current", Source: "agent", ObservedAt: at.Add(time.Minute)}}
+ previous := previousGuestContext{vms: []models.VM{merged}}
+ vms, containers := m.retainedGuestsForFailedNodes("site-a", map[string]struct{}{"node": {}}, previous)
+ if len(vms) != 1 || len(containers) != 0 || !reflect.DeepEqual(vms[0].Memory, original.Memory) || vms[0].CPU != .95 || vms[0].GuestAgentStatus != "deferred" || vms[0].Disk.Used != 40 || !vms[0].LastSeen.Equal(at) {
+ t.Fatalf("failed enumeration changed source ownership/membership: %+v", vms)
+ }
+}
diff --git a/internal/monitoring/monitor_polling_containers.go b/internal/monitoring/monitor_polling_containers.go
index d0e32c3e7b..d26f0b8762 100644
--- a/internal/monitoring/monitor_polling_containers.go
+++ b/internal/monitoring/monitor_polling_containers.go
@@ -364,12 +364,9 @@ func (m *Monitor) collectContainersWithNodes(ctx context.Context, instanceName s
preservedContainers := 0
if len(failedNodeNames) > 0 {
- for _, container := range prevGuests.containers {
- if _, failed := failedNodeNames[container.Node]; failed {
- allContainers = append(allContainers, container)
- preservedContainers++
- }
- }
+ _, retained := m.retainedGuestsForFailedNodes(instanceName, failedNodeNames, prevGuests)
+ allContainers = append(allContainers, retained...)
+ preservedContainers = len(retained)
}
if preservedContainers > 0 {
log.Warn().
diff --git a/internal/monitoring/monitor_polling_vm.go b/internal/monitoring/monitor_polling_vm.go
index 5ed8d32db2..def9948e1c 100644
--- a/internal/monitoring/monitor_polling_vm.go
+++ b/internal/monitoring/monitor_polling_vm.go
@@ -105,12 +105,9 @@ func (m *Monitor) collectVMsWithNodes(ctx context.Context, instanceName string,
preservedVMs := 0
if len(failedNodeNames) > 0 {
- for _, vm := range prevGuests.vms {
- if _, failed := failedNodeNames[vm.Node]; failed {
- allVMs = append(allVMs, vm)
- preservedVMs++
- }
- }
+ retained, _ := m.retainedGuestsForFailedNodes(instanceName, failedNodeNames, prevGuests)
+ allVMs = append(allVMs, retained...)
+ preservedVMs = len(retained)
}
if preservedVMs > 0 {
log.Warn().