Skip to content

armgit / Pulse

Public Imported from GitHub
0

Real-time monitoring dashboard for Proxmox VE, PBS, Docker, Kubernetes, TrueNAS and vSphere. Self-hosted, with alerts for failed backups, full disks, unhealthy containers and offline nodes.

Reproduce obsolete Proxmox guest poll publication

pulse-triage[bot]1ccc3651fdd6
diff --git a/internal/monitoring/guest_poll_ownership_test.go b/internal/monitoring/guest_poll_ownership_test.go
new file mode 100644
index 0000000000..e1537047d4
--- /dev/null
+++ b/internal/monitoring/guest_poll_ownership_test.go
@@ -0,0 +1,200 @@
+package monitoring
+
+import (
+ "context"
+ "fmt"
+ "reflect"
+ "sync"
+ "testing"
+ "time"
+
+ "github.com/rcourtman/pulse-go-rewrite/internal/alerts"
+ "github.com/rcourtman/pulse-go-rewrite/internal/config"
+ "github.com/rcourtman/pulse-go-rewrite/internal/models"
+ "github.com/rcourtman/pulse-go-rewrite/pkg/proxmox"
+)
+
+// Ignore cancellation deliberately: a late inventory reply cannot prove that
+// its connection/runtime still owns publication. No native guest is involved.
+type guestPollOwnershipClient struct {
+ stubPVEClient
+ route string
+ cpu float64
+ empty bool
+ entered chan context.Context
+ release chan struct{}
+ block string
+ once sync.Once
+}
+
+func (c *guestPollOwnershipClient) wait(ctx context.Context, phase string) {
+ if c.block == phase {
+ c.once.Do(func() { c.entered <- ctx })
+ <-c.release
+ }
+}
+
+func (c *guestPollOwnershipClient) resources() []proxmox.ClusterResource {
+ if c.empty {
+ return nil
+ }
+ return []proxmox.ClusterResource{
+ {Type: "qemu", Node: "node", VMID: 105, Name: "vm", Status: "running", CPU: c.cpu, MaxMem: 1000, Mem: 200, MaxDisk: 1000},
+ {Type: "lxc", Node: "node", VMID: 106, Name: "ct", Status: "running", CPU: c.cpu, MaxMem: 1000, Mem: 200, MaxDisk: 1000, Disk: 200},
+ {Type: "qemu", Node: "node", VMID: 107, Template: 1},
+ }
+}
+
+func (c *guestPollOwnershipClient) GetClusterResources(ctx context.Context, _ string) ([]proxmox.ClusterResource, error) {
+ if c.route == "node" {
+ return nil, fmt.Errorf("cluster/resources unavailable")
+ }
+ c.wait(ctx, "inventory")
+ return c.resources(), nil
+}
+
+func (c *guestPollOwnershipClient) GetVMs(ctx context.Context, _ string) ([]proxmox.VM, error) {
+ c.wait(ctx, "inventory")
+ var rows []proxmox.VM
+ for _, r := range c.resources() {
+ if r.Type == "qemu" {
+ rows = append(rows, proxmox.VM{VMID: r.VMID, Name: r.Name, Status: r.Status, CPU: r.CPU, MaxMem: r.MaxMem, Mem: r.Mem, MaxDisk: r.MaxDisk, Template: r.Template})
+ }
+ }
+ return rows, nil
+}
+
+func (c *guestPollOwnershipClient) GetContainers(context.Context, string) ([]proxmox.Container, error) {
+ if c.empty {
+ return nil, nil
+ }
+ return []proxmox.Container{{VMID: 106, Name: "ct", Status: "running", CPU: c.cpu, MaxMem: 1000, Mem: 200, MaxDisk: 1000, Disk: 200}}, nil
+}
+
+func (c *guestPollOwnershipClient) GetVMStatus(ctx context.Context, _ string, _ int) (*proxmox.VMStatus, error) {
+ c.wait(ctx, "detail")
+ return &proxmox.VMStatus{Status: "running", MaxMem: 1000, Mem: 200}, nil
+}
+
+func guestPollOwnershipMonitor(t *testing.T, c PVEClientInterface) *Monitor {
+ t.Helper()
+ m := windowsBootstrapMonitor(t)
+ m.pveClients = map[string]PVEClientInterface{"site-a": c}
+ m.setRuntimeContext(context.Background(), nil)
+ ac := m.alertManager.GetConfig()
+ ac.Enabled = true
+ ac.GuestDefaults.CPU = &alerts.HysteresisThreshold{Trigger: 80, Clear: 75}
+ ac.MetricTimeThresholds = map[string]map[string]int{"guest": {"cpu": 0}}
+ m.alertManager.UpdateConfig(ac)
+ return m
+}
+
+func runGuestOwnershipPoll(m *Monitor, ctx context.Context, c PVEClientInterface) error {
+ return m.pollGuestsWithFallback(ctx, "site-a", &config.PVEInstance{MonitorVMs: true, MonitorContainers: true}, c,
+ []proxmox.Node{{Node: "node", Status: "online"}}, map[string]string{"node": "online"})
+}
+
+func waitGuestOwnership(t *testing.T, done <-chan error) {
+ t.Helper()
+ select {
+ case <-done:
+ case <-time.After(5 * time.Second):
+ t.Fatal("controlled guest poll did not finish")
+ }
+}
+
+// Use existing entry points only, so this same assertion runs on the original
+// implementation. Identity, diagnostics, History and breach continuity are all
+// part of the user's reading, not just the returned builder value.
+func TestGuestPollOwnershipRejectsObsoletePublication(t *testing.T) {
+ for _, route := range []string{"cluster", "node"} {
+ for _, mode := range []string{"retire", "replace", "runtime", "cancel", "cancel-empty"} {
+ for _, phase := range []string{"inventory", "detail"} {
+ if mode == "cancel-empty" && phase == "detail" {
+ continue
+ }
+ t.Run(route+"/"+mode+"/"+phase, func(t *testing.T) {
+ old := &guestPollOwnershipClient{route: route, cpu: .1, block: phase, entered: make(chan context.Context, 1), release: make(chan struct{}), empty: mode == "cancel-empty"}
+ m := guestPollOwnershipMonitor(t, old)
+ seed := models.VM{ID: "site-b:node:105", Instance: "site-b", Node: "node", VMID: 105, Name: "peer", Type: "qemu", Status: "running", CPU: .3}
+ m.state.UpdateVMsForInstance("site-b", []models.VM{seed})
+ m.state.UpdateVMsForInstance("site-a", []models.VM{{ID: "site-a:node:105", Instance: "site-a", Node: "node", VMID: 105, Type: "qemu", Status: "running", CPU: .95}})
+ m.updatePVEBackupTemplateSubjectsForType("site-a", "qemu", map[string]struct{}{"original": {}})
+ ctx, cancel := context.WithCancel(context.Background())
+ defer cancel()
+ var once sync.Once
+ unblock := func() { once.Do(func() { close(old.release) }) }
+ defer unblock()
+ done := make(chan error, 1)
+ go func() { done <- runGuestOwnershipPoll(m, ctx, old) }()
+ var readCtx context.Context
+ select {
+ case readCtx = <-old.entered:
+ case <-time.After(5 * time.Second):
+ t.Fatal("controlled old inventory/detail did not start")
+ }
+ switch mode {
+ case "retire":
+ m.retirePVEInstanceRuntime("site-a")
+ case "runtime":
+ m.setRuntimeContext(context.Background(), nil)
+ case "cancel", "cancel-empty":
+ cancel()
+ }
+ fresh := &guestPollOwnershipClient{route: route, cpu: .95}
+ if mode != "cancel" && mode != "cancel-empty" {
+ // Tests change the map only while the old read is blocked;
+ // production replacement uses the publication lifecycle gate.
+ m.mu.Lock()
+ m.pveClients["site-a"] = fresh
+ m.mu.Unlock()
+ if err := runGuestOwnershipPoll(m, context.Background(), fresh); err != nil {
+ t.Fatal(err)
+ }
+ // A threshold observation needs two evaluations to become a
+ // breach, regardless of its zero duration.
+ for _, vm := range m.state.GetSnapshot().VMs {
+ if vm.Instance == "site-a" {
+ m.checkGuestAlertsForVM("site-a", vm)
+ }
+ }
+ }
+ before := m.state.GetSnapshot()
+ diagnostics := m.GetDiagnosticSnapshots().Guests
+ vmHistory := m.metricsHistory.GetGuestMetrics("site-a:node:105", "cpu", time.Hour)
+ ctHistory := m.metricsHistory.GetGuestMetrics("site-a:node:106", "cpu", time.Hour)
+ active := m.alertManager.GetActiveAlerts()
+ m.mu.RLock()
+ ready := fmt.Sprint(m.pveBackupInventoryReady["site-a"])
+ templates := fmt.Sprint(m.pveBackupTemplateSubjects["site-a"])
+ m.mu.RUnlock()
+ unblock()
+ waitGuestOwnership(t, done)
+ got := m.state.GetSnapshot()
+ if !reflect.DeepEqual(got.VMs, before.VMs) || !reflect.DeepEqual(got.Containers, before.Containers) {
+ t.Error("obsolete guest poll replaced or erased accepted inventory")
+ }
+ if !reflect.DeepEqual(m.GetDiagnosticSnapshots().Guests, diagnostics) {
+ t.Error("obsolete guest poll replaced accepted diagnostics")
+ }
+ if !reflect.DeepEqual(m.metricsHistory.GetGuestMetrics("site-a:node:105", "cpu", time.Hour), vmHistory) || !reflect.DeepEqual(m.metricsHistory.GetGuestMetrics("site-a:node:106", "cpu", time.Hour), ctHistory) {
+ t.Error("obsolete guest poll appended History")
+ }
+ if !reflect.DeepEqual(m.alertManager.GetActiveAlerts(), active) {
+ t.Error("obsolete guest poll changed accepted alert breaches")
+ }
+ m.mu.RLock()
+ if fmt.Sprint(m.pveBackupInventoryReady["site-a"]) != ready || fmt.Sprint(m.pveBackupTemplateSubjects["site-a"]) != templates {
+ t.Error("obsolete guest poll changed backup inventory eligibility")
+ }
+ m.mu.RUnlock()
+ if mode == "retire" || mode == "runtime" {
+ if readCtx.Err() == nil {
+ t.Error("ended connection/runtime did not cancel its outstanding guest read")
+ }
+ }
+ })
+ }
+ }
+ }
+}