Skip to content

armgit / Pulse

Public Imported from GitHub
0

Real-time monitoring dashboard for Proxmox VE, PBS, Docker, Kubernetes, TrueNAS and vSphere. Self-hosted, with alerts for failed backups, full disks, unhealthy containers and offline nodes.

Cover failed snapshot listings without treating errors as deletion

pulse-triage[bot]95e9996f0a36
diff --git a/internal/monitoring/snapshot_collection_observation_test.go b/internal/monitoring/snapshot_collection_observation_test.go
new file mode 100644
index 0000000000..c91d5a38e2
--- /dev/null
+++ b/internal/monitoring/snapshot_collection_observation_test.go
@@ -0,0 +1,216 @@
+package monitoring
+
+import (
+ "context"
+ "encoding/json"
+ "errors"
+ "fmt"
+ "os"
+ "reflect"
+ "testing"
+ "time"
+
+ "github.com/rcourtman/pulse-go-rewrite/internal/alerts"
+ "github.com/rcourtman/pulse-go-rewrite/internal/models"
+ "github.com/rcourtman/pulse-go-rewrite/pkg/proxmox"
+)
+
+type snapshotCollectionReply struct {
+ snapshots []proxmox.Snapshot
+ err error
+}
+
+type snapshotCollectionClient struct {
+ mockPVEClientExtra
+ replies map[string]snapshotCollectionReply
+}
+
+func (c *snapshotCollectionClient) GetVMSnapshots(_ context.Context, node string, vmid int) ([]proxmox.Snapshot, error) {
+ r := c.replies[fmt.Sprintf("qemu/%s/%d", node, vmid)]
+ return r.snapshots, r.err
+}
+
+func (c *snapshotCollectionClient) GetContainerSnapshots(_ context.Context, node string, vmid int) ([]proxmox.Snapshot, error) {
+ r := c.replies[fmt.Sprintf("lxc/%s/%d", node, vmid)]
+ return r.snapshots, r.err
+}
+
+func (c *snapshotCollectionClient) GetStorage(context.Context, string) ([]proxmox.Storage, error) {
+ return nil, nil // These observations do not require any storage or guest-agent operation.
+}
+
+func snapshotCollectionMonitor(t *testing.T, kind string, withPeers bool) *Monitor {
+ t.Helper()
+ am := alerts.NewManagerWithDataDir(t.TempDir())
+ t.Cleanup(am.Stop)
+ cfg := am.GetConfig()
+ cfg.Enabled, cfg.ActivationState = true, alerts.ActivationActive
+ cfg.SnapshotDefaults = alerts.SnapshotAlertConfig{Enabled: true, WarningDays: 7, CriticalDays: 14}
+ cfg.TimeThresholds = map[string]int{}
+ am.UpdateConfig(cfg)
+ m := &Monitor{state: models.NewState(), alertManager: am}
+ for _, site := range []string{"site", "other-site"} {
+ if site == "other-site" && !withPeers {
+ continue
+ }
+ var vms []models.VM
+ var containers []models.Container
+ for _, node := range []string{"node-a", "node-b"} {
+ if node == "node-b" && (!withPeers || site == "other-site") {
+ continue
+ }
+ id := alerts.BuildGuestKey(site, node, 101)
+ if kind == "qemu" {
+ vms = append(vms, models.VM{ID: id, VMID: 101, Name: "database", Node: node, Instance: site, Type: kind, Status: "running"})
+ } else {
+ containers = append(containers, models.Container{ID: id, VMID: 101, Name: "database", Node: node, Instance: site, Type: kind, Status: "running"})
+ }
+ }
+ m.state.UpdateGuestsForInstance(site, vms, containers)
+ }
+ return m
+}
+
+func snapshotCollectionByID(m *Monitor) map[string]models.GuestSnapshot {
+ result := map[string]models.GuestSnapshot{}
+ for _, snapshot := range m.state.GetSnapshot().PVEBackups.GuestSnapshots {
+ result[snapshot.ID] = snapshot
+ }
+ return result
+}
+
+func snapshotCollectionAlert(t *testing.T, m *Monitor, resourceID string) alerts.Alert {
+ t.Helper()
+ for _, alert := range m.GetState().ActiveAlerts {
+ if alert.Type == "snapshot-age" && alert.ResourceID == resourceID {
+ return alert
+ }
+ }
+ t.Fatalf("snapshot warning missing for %s: %+v", resourceID, m.GetState().ActiveAlerts)
+ return alerts.Alert{}
+}
+
+func TestSnapshotCollectionFailureIsNotRemoval(t *testing.T) {
+ for _, kind := range []string{"qemu", "lxc"} {
+ for _, message := range []string{"API error 596: node offline", "snapshot service not available", "permission denied", "request timed out", "unexpected response"} {
+ t.Run(kind+"/"+message, func(t *testing.T) {
+ m := snapshotCollectionMonitor(t, kind, true)
+ old := proxmox.Snapshot{Name: "weekly", SnapTime: time.Now().Add(-16 * 24 * time.Hour).Unix(), Description: "last accepted listing", Parent: "base"}
+ key := kind + "/node-a/101"
+ peerKey := kind + "/node-b/101"
+ client := &snapshotCollectionClient{replies: map[string]snapshotCollectionReply{key: {snapshots: []proxmox.Snapshot{old}}, peerKey: {snapshots: []proxmox.Snapshot{old}}}}
+ m.pollGuestSnapshots(context.Background(), "site", client)
+ m.pollGuestSnapshots(context.Background(), "other-site", client)
+ resourceID := "site:node-a:101"
+ initial := snapshotCollectionAlert(t, m, resourceID)
+ if err := m.alertManager.AcknowledgeAlert(initial.ID, "operator"); err != nil {
+ t.Fatal(err)
+ }
+ initial = snapshotCollectionAlert(t, m, resourceID)
+ before := snapshotCollectionByID(m)
+ targetID := "site-node-a-101-weekly"
+ otherID := "other-site-node-a-101-weekly"
+ // Discard even a partial result returned alongside an error.
+ client.replies[key] = snapshotCollectionReply{snapshots: []proxmox.Snapshot{{Name: "unaccepted", SnapTime: time.Now().Unix()}}, err: errors.New(message)}
+ client.replies[peerKey] = snapshotCollectionReply{snapshots: []proxmox.Snapshot{{Name: "fresh-peer", SnapTime: time.Now().Add(-time.Hour).Unix()}}}
+ for range 3 {
+ m.pollGuestSnapshots(context.Background(), "site", client)
+ }
+ after := snapshotCollectionByID(m)
+ if !reflect.DeepEqual(after[targetID], before[targetID]) {
+ t.Fatalf("failed %s listing removed or changed accepted snapshot: before=%+v after=%+v", kind, before[targetID], after[targetID])
+ }
+ if len(after) != 3 || !reflect.DeepEqual(after[otherID], before[otherID]) || after["site-node-b-101-fresh-peer"].Name != "fresh-peer" {
+ t.Fatalf("failed listing leaked across node/instance or blocked accepted peer: %+v", after)
+ }
+ current := snapshotCollectionAlert(t, m, resourceID)
+ if current.ID != initial.ID || !current.StartTime.Equal(initial.StartTime) || !current.Acknowledged || !reflect.DeepEqual(current.Metadata["snapshotCreatedAt"], initial.Metadata["snapshotCreatedAt"]) {
+ t.Fatalf("failed listing replaced occurrence, acknowledgement or creation evidence: before=%+v after=%+v", initial, current)
+ }
+ if len(m.GetState().ActiveAlerts) != 2 {
+ t.Fatalf("only failed guest and untouched other instance should retain warnings: %+v", m.GetState().ActiveAlerts)
+ }
+ // A later accepted listing can still confirm the old snapshot.
+ client.replies[key] = snapshotCollectionReply{snapshots: []proxmox.Snapshot{old}}
+ m.pollGuestSnapshots(context.Background(), "site", client)
+ if current = snapshotCollectionAlert(t, m, resourceID); current.ID != initial.ID || !current.StartTime.Equal(initial.StartTime) || !current.Acknowledged {
+ t.Fatal("read recovery replaced an existing warning")
+ }
+ // Only a successful empty listing establishes removal for this guest.
+ client.replies[key] = snapshotCollectionReply{}
+ m.pollGuestSnapshots(context.Background(), "site", client)
+ m.pollGuestSnapshots(context.Background(), "site", client)
+ if _, exists := snapshotCollectionByID(m)[targetID]; exists || len(m.GetState().ActiveAlerts) != 1 {
+ t.Fatalf("accepted removal failed or cleared unrelated warning: %+v", m.GetState())
+ }
+ snapshotCollectionAlert(t, m, "other-site:node-a:101")
+ })
+ }
+ }
+}
+
+func TestSnapshotCollectionFailureWithoutPreviousEvidenceDoesNotInventSnapshot(t *testing.T) {
+ m := snapshotCollectionMonitor(t, "lxc", false)
+ client := &snapshotCollectionClient{replies: map[string]snapshotCollectionReply{"lxc/node-a/101": {err: errors.New("596: snapshot API not available")}}}
+ m.pollGuestSnapshots(context.Background(), "site", client)
+ if len(snapshotCollectionByID(m)) != 0 || len(m.GetState().ActiveAlerts) != 0 {
+ t.Fatal("a failed first read invented a snapshot or warning")
+ }
+}
+
+func TestSnapshotCollectionFailureStillHonoursDisabledAlerts(t *testing.T) {
+ m := snapshotCollectionMonitor(t, "lxc", false)
+ client := &snapshotCollectionClient{replies: map[string]snapshotCollectionReply{"lxc/node-a/101": {snapshots: []proxmox.Snapshot{{Name: "weekly", SnapTime: time.Now().Add(-16 * 24 * time.Hour).Unix()}}}}}
+ m.pollGuestSnapshots(context.Background(), "site", client)
+ snapshotCollectionAlert(t, m, "site:node-a:101")
+ cfg := m.alertManager.GetConfig()
+ cfg.SnapshotDefaults.Enabled = false
+ m.alertManager.UpdateConfig(cfg)
+ client.replies["lxc/node-a/101"] = snapshotCollectionReply{err: errors.New("596: snapshot API not available")}
+ m.pollGuestSnapshots(context.Background(), "site", client)
+ if len(snapshotCollectionByID(m)) != 1 || len(m.GetState().ActiveAlerts) != 0 {
+ t.Fatal("retained inventory overrode explicit alert disablement")
+ }
+}
+
+// Opt-in frames connect the actual poller, live manager snapshot and browser.
+// No guest-agent, backup, freeze/thaw or external service is invoked.
+func TestSnapshotCollectionBrowserProducer(t *testing.T) {
+ m := snapshotCollectionMonitor(t, "lxc", false)
+ old := proxmox.Snapshot{Name: "weekly", SnapTime: time.Now().Add(-16 * 24 * time.Hour).Unix(), Description: "last accepted listing"}
+ client := &snapshotCollectionClient{replies: map[string]snapshotCollectionReply{}}
+ type frame struct {
+ Snapshots []models.GuestSnapshot `json:"snapshots"`
+ Alerts []alerts.Alert `json:"alerts"`
+ }
+ frames := map[string]frame{}
+ for _, phase := range []string{"observed", "failed-596", "failed-unavailable", "confirmed-empty"} {
+ reply := snapshotCollectionReply{snapshots: []proxmox.Snapshot{old}}
+ switch phase {
+ case "failed-596":
+ reply = snapshotCollectionReply{err: errors.New("API error 596: node offline")}
+ case "failed-unavailable":
+ reply = snapshotCollectionReply{err: errors.New("snapshot service not available")}
+ case "confirmed-empty":
+ reply = snapshotCollectionReply{}
+ }
+ client.replies["lxc/node-a/101"] = reply
+ m.pollGuestSnapshots(context.Background(), "site", client)
+ state := m.GetState()
+ frames[phase] = frame{Snapshots: state.PVEBackups.GuestSnapshots, Alerts: state.ActiveAlerts}
+ want := 1
+ if phase == "confirmed-empty" {
+ want = 0
+ }
+ if len(state.PVEBackups.GuestSnapshots) != want || len(state.ActiveAlerts) != want {
+ t.Fatalf("phase %s: want %d snapshots and warnings, got %d/%d", phase, want, len(state.PVEBackups.GuestSnapshots), len(state.ActiveAlerts))
+ }
+ }
+ if os.Getenv("PULSE_SNAPSHOT_COLLECTION_FRAMES") == "1" {
+ data, err := json.Marshal(frames)
+ if err != nil {
+ t.Fatal(err)
+ }
+ fmt.Printf("SNAPSHOT_COLLECTION_FRAMES=%s\n", data)
+ }
+}