Skip to content

armgit / Pulse

Public Imported from GitHub
0

Real-time monitoring dashboard for Proxmox VE, PBS, Docker, Kubernetes, TrueNAS and vSphere. Self-hosted, with alerts for failed backups, full disks, unhealthy containers and offline nodes.

Preserve PVE snapshot history across incomplete reads

pulse-triage[bot]dd8f492aae8c
diff --git a/docs/release-control/v6/internal/subsystems/monitoring.md b/docs/release-control/v6/internal/subsystems/monitoring.md
index 97a578c53d..a4c84ffc76 100644
--- a/docs/release-control/v6/internal/subsystems/monitoring.md
+++ b/docs/release-control/v6/internal/subsystems/monitoring.md
@@ -5575,7 +5575,21 @@ snapshot enumeration. `internal/monitoring/monitor_backups.go` must query VM
and LXC snapshot endpoints through one capped worker pool and preserve
previously-known snapshots only for guests that were not successfully polled in
the current cycle, so large or slow clusters do not starve later guests while
-transient misses still avoid destructive state churn.
+transient misses still avoid destructive state churn. Durable snapshot recovery
+points also outlive that live cache: an incomplete, cancelled or unobserved guest
+inventory must not reconcile missing durable points, including after a monitor
+replacement. Only snapshots returned by an accepted listing may renew point-level
+evidence; carried-forward rows remain live continuity, not fresh observations.
+Instance-wide deletion waits for a nonempty target inventory whose listings all
+succeeded, without cancellation or carried-forward rows. An accepted complete
+empty snapshot listing still clears that instance's snapshot scope, preserving
+other instances and backup/task classes. `TestPVESnapshotIncompletePollRetainsRecoveryHistory`,
+`TestPVESnapshotUnobservedInventoryPreservesRecoveryHistory` and
+`TestPVESnapshotCancelledAfterListingsCannotReconcile` in
+`internal/monitoring/recovery_ingest_test.go` exercise the actual poll, queue and
+retained reader across warm/cold VM and LXC inventories, rejected partial data,
+accepted peer updates and complete-empty recovery. These are synthetic reads,
+not native guest recovery, backup/restore or filesystem/thaw acceptance.
That same guest-monitoring boundary also owns linked host-agent precedence for
Proxmox VMs. When a VM has a live linked Pulse host agent, the canonical disk
inventory and aggregate disk summary must prefer that linked host-agent disk
diff --git a/internal/monitoring/monitor_backups.go b/internal/monitoring/monitor_backups.go
index ea48d4b8f4..6bba5d68f7 100644
--- a/internal/monitoring/monitor_backups.go
+++ b/internal/monitoring/monitor_backups.go
@@ -1248,6 +1248,10 @@ func (m *Monitor) pollGuestSnapshots(ctx context.Context, instanceName string, c
}
}
+ // Retained live rows are continuity, not newly accepted provider evidence.
+ // Keep their count separate before merging so a failed listing cannot renew
+ // durable point evidence merely because the raw row survived in memory.
+ observedSnapshotCount := len(allSnapshots)
carriedForward := 0
for _, prev := range previousSnapshots {
if _, polled := polledGuestKeys[guestKey(instanceName, prev.Node, prev.VMID)]; polled {
@@ -1287,7 +1291,7 @@ func (m *Monitor) pollGuestSnapshots(ctx context.Context, instanceName string, c
// Best-effort ingestion into recovery store (for rollups / unified backups UX).
guestInfo := buildProxmoxGuestInfoIndex(readState)
points, evidenceErr := proxmoxrecoverymapper.FromPVEGuestSnapshotsWithEvidence(
- allSnapshots,
+ allSnapshots[:observedSnapshotCount],
guestInfo,
time.Now().UTC(),
)
@@ -1296,7 +1300,12 @@ func (m *Monitor) pollGuestSnapshots(ctx context.Context, instanceName string, c
Err(evidenceErr).
Str("instance", instanceName).
Msg("Failed to attach PVE snapshot recovery evidence; preserving existing points")
- } else {
+ } else if len(targets) > 0 && len(polledGuestKeys) == len(targets) &&
+ carriedForward == 0 && snapshotCtx.Err() == nil {
+ // Only an accepted complete listing can establish instance-wide
+ // absence. In particular, after a monitor replacement the live cache
+ // can be empty while durable snapshots still exist for a failed guest.
+ // No observed guest inventory is not a successful empty enumeration.
m.ingestAndReconcileRecoveryPointsAsync(
points,
recoveryReconcileScope{
@@ -1305,6 +1314,8 @@ func (m *Monitor) pollGuestSnapshots(ctx context.Context, instanceName string, c
instance: instanceName,
},
)
+ } else {
+ m.ingestRecoveryPointsAsync(points)
}
if m.alertManager != nil {
diff --git a/internal/monitoring/recovery_ingest_test.go b/internal/monitoring/recovery_ingest_test.go
index 2d4df96266..a416d77a3c 100644
--- a/internal/monitoring/recovery_ingest_test.go
+++ b/internal/monitoring/recovery_ingest_test.go
@@ -2,13 +2,18 @@ package monitoring
import (
"context"
+ "fmt"
+ "reflect"
"testing"
"time"
"github.com/rcourtman/pulse-go-rewrite/internal/config"
+ "github.com/rcourtman/pulse-go-rewrite/internal/models"
"github.com/rcourtman/pulse-go-rewrite/internal/operationaltrust"
"github.com/rcourtman/pulse-go-rewrite/internal/recovery"
recoverymanager "github.com/rcourtman/pulse-go-rewrite/internal/recovery/manager"
+ recoverystore "github.com/rcourtman/pulse-go-rewrite/internal/recovery/store"
+ "github.com/rcourtman/pulse-go-rewrite/pkg/proxmox"
)
func recoveryIngestTestMonitor(t *testing.T) (*Monitor, *recoverymanager.Manager) {
@@ -352,3 +357,238 @@ func TestRecoveryIngestCompletePollCannotOvertakePartialBatch(t *testing.T) {
t.Fatalf("partial history resurrected after authoritative empty recovery: %v", ids)
}
}
+
+// Snapshot inventory in memory can be empty after an ordinary monitor
+// replacement while its durable recovery points still exist. Exercise the
+// actual poll, ingest queue and public store reader, not database internals.
+type snapshotRecoveryClient struct {
+ *snapshotCollectionClient
+ afterStorage func()
+}
+
+func (c *snapshotRecoveryClient) GetStorage(context.Context, string) ([]proxmox.Storage, error) {
+ if c.afterStorage != nil {
+ c.afterStorage()
+ }
+ return nil, nil
+}
+
+type snapshotRecoveryFixture struct {
+ m *Monitor
+ store *recoverystore.Store
+ client *snapshotRecoveryClient
+ kind string
+}
+
+func newSnapshotRecoveryFixture(t *testing.T, kind string) *snapshotRecoveryFixture {
+ t.Helper()
+ m, manager := recoveryIngestTestMonitor(t)
+ store, err := manager.StoreForOrg("default")
+ if err != nil {
+ t.Fatal(err)
+ }
+ f := &snapshotRecoveryFixture{m: m, store: store, kind: kind,
+ client: &snapshotRecoveryClient{snapshotCollectionClient: &snapshotCollectionClient{replies: map[string]snapshotCollectionReply{}}}}
+ f.coldState(true)
+ at := time.Now().UTC().Add(-16 * 24 * time.Hour).Truncate(time.Second)
+ for _, guest := range []string{"node-a/101", "node-b/202"} {
+ f.client.replies[kind+"/"+guest] = snapshotCollectionReply{snapshots: []proxmox.Snapshot{{Name: "weekly", SnapTime: at.Unix(), Description: "accepted listing"}}}
+ }
+ t.Cleanup(func() { f.wait(t); _ = store.Close() })
+ return f
+}
+
+func (f *snapshotRecoveryFixture) coldState(withInventory bool) {
+ f.m.state = models.NewState()
+ if !withInventory {
+ return
+ }
+ for _, site := range []string{"site", "other-site"} {
+ var vms []models.VM
+ var containers []models.Container
+ for _, guest := range []struct {
+ node string
+ id int
+ }{{"node-a", 101}, {"node-b", 202}} {
+ if site == "other-site" && guest.id != 101 {
+ continue
+ }
+ id := makeGuestID(site, guest.node, guest.id)
+ if f.kind == "qemu" {
+ vms = append(vms, models.VM{ID: id, Instance: site, Node: guest.node, VMID: guest.id, Name: "database", Type: f.kind})
+ } else {
+ containers = append(containers, models.Container{ID: id, Instance: site, Node: guest.node, VMID: guest.id, Name: "database", Type: f.kind})
+ }
+ }
+ f.m.state.UpdateGuestsForInstance(site, vms, containers)
+ }
+}
+
+func (f *snapshotRecoveryFixture) wait(t *testing.T) {
+ t.Helper()
+ deadline := time.Now().Add(10 * time.Second)
+ for {
+ f.m.recoveryIngestMu.Lock()
+ running := f.m.recoveryIngestRunning
+ f.m.recoveryIngestMu.Unlock()
+ if !running {
+ return
+ }
+ if time.Now().After(deadline) {
+ t.Fatal("snapshot recovery ingest did not finish")
+ }
+ time.Sleep(time.Millisecond)
+ }
+}
+
+func (f *snapshotRecoveryFixture) poll(t *testing.T, site string) {
+ t.Helper()
+ f.m.pollGuestSnapshots(context.Background(), site, f.client)
+ f.wait(t)
+}
+
+func (f *snapshotRecoveryFixture) points(t *testing.T) map[string]recovery.RecoveryPoint {
+ t.Helper()
+ points, _, err := f.store.ListPoints(context.Background(), recovery.ListPointsOptions{Limit: 1000})
+ if err != nil {
+ t.Fatal(err)
+ }
+ out := make(map[string]recovery.RecoveryPoint, len(points))
+ for _, point := range points {
+ out[point.ID] = point
+ }
+ return out
+}
+
+func TestPVESnapshotIncompletePollRetainsRecoveryHistory(t *testing.T) {
+ for _, kind := range []string{"qemu", "lxc"} {
+ for _, cold := range []bool{false, true} {
+ for _, failure := range []string{"503 unavailable", "403 forbidden", "596 node unavailable", "invalid snapshot response"} {
+ t.Run(fmt.Sprintf("%s/cold=%t/%s", kind, cold, failure), func(t *testing.T) {
+ f := newSnapshotRecoveryFixture(t, kind)
+ f.poll(t, "site")
+ f.poll(t, "other-site")
+ before := f.points(t)
+ if len(before) != 3 {
+ t.Fatalf("control has %d points, want three", len(before))
+ }
+ if cold {
+ f.coldState(true)
+ }
+ // A rejected response may contain data, but neither its rows
+ // nor its absence can replace the last accepted observation.
+ f.client.replies[kind+"/node-a/101"] = snapshotCollectionReply{
+ snapshots: []proxmox.Snapshot{{Name: "unaccepted", SnapTime: time.Now().Unix()}}, err: fmt.Errorf("%s", failure)}
+ f.client.replies[kind+"/node-b/202"] = snapshotCollectionReply{snapshots: []proxmox.Snapshot{{Name: "fresh-peer", SnapTime: time.Now().Add(-time.Hour).Unix()}}}
+ for range 3 {
+ f.poll(t, "site")
+ }
+ after := f.points(t)
+ failedID, otherID := "pve-snapshot:site-node-a-101-weekly", "pve-snapshot:other-site-node-a-101-weekly"
+ if !reflect.DeepEqual(before[failedID], after[failedID]) {
+ t.Fatalf("failed guest lost or renewed its durable point: before=%+v after=%+v", before[failedID], after[failedID])
+ }
+ if !reflect.DeepEqual(before[otherID], after[otherID]) || len(after) != 4 || after["pve-snapshot:site-node-b-202-fresh-peer"].ID == "" {
+ t.Fatalf("partial poll changed other scope or failed to ingest accepted peer: %+v", after)
+ }
+ // Deletion waits for a complete pass, not for another guest
+ // to happen to succeed. Accepted empty lists still remove
+ // this instance's snapshots once the full scope is readable.
+ f.client.replies[kind+"/node-a/101"] = snapshotCollectionReply{}
+ f.client.replies[kind+"/node-b/202"] = snapshotCollectionReply{}
+ f.poll(t, "site")
+ after = f.points(t)
+ if len(after) != 1 || !reflect.DeepEqual(before[otherID], after[otherID]) {
+ t.Fatalf("complete empty listing failed to clear only its own scope: %+v", after)
+ }
+ })
+ }
+ }
+ }
+}
+
+func TestPVESnapshotUnobservedInventoryPreservesRecoveryHistory(t *testing.T) {
+ for _, kind := range []string{"qemu", "lxc"} {
+ for _, noInventory := range []bool{false, true} {
+ t.Run(fmt.Sprintf("%s/no-inventory=%t", kind, noInventory), func(t *testing.T) {
+ f := newSnapshotRecoveryFixture(t, kind)
+ f.poll(t, "site")
+ before := f.points(t)
+ f.coldState(!noInventory)
+ for key := range f.client.replies {
+ f.client.replies[key] = snapshotCollectionReply{err: fmt.Errorf("503 unavailable")}
+ }
+ f.poll(t, "site")
+ if !reflect.DeepEqual(before, f.points(t)) {
+ t.Fatal("unobserved scope deleted or renewed durable snapshots")
+ }
+ })
+ }
+ }
+}
+
+func TestPVESnapshotCancelledAfterListingsCannotReconcile(t *testing.T) {
+ f := newSnapshotRecoveryFixture(t, "qemu")
+ f.poll(t, "site")
+ before := f.points(t)
+ f.coldState(true)
+ for key := range f.client.replies {
+ f.client.replies[key] = snapshotCollectionReply{snapshots: []proxmox.Snapshot{{Name: "new", SnapTime: time.Now().Unix()}}}
+ }
+ ctx, cancel := context.WithCancel(context.Background())
+ defer cancel()
+ // Sizing runs after every successful listing, before durable ingestion.
+ f.client.afterStorage = cancel
+ f.m.pollGuestSnapshots(ctx, "site", f.client)
+ f.wait(t)
+ if ctx.Err() == nil {
+ t.Fatal("control did not cancel the poll after the listings")
+ }
+ after := f.points(t)
+ for id, point := range before {
+ if !reflect.DeepEqual(point, after[id]) {
+ t.Fatalf("cancelled poll reconciled away accepted history %s", id)
+ }
+ }
+ if len(after) != 4 {
+ t.Fatalf("accepted pre-cancellation listings should remain useful, points=%d", len(after))
+ }
+}
+
+func TestPVESnapshotCompleteRecoveryKeepsOtherClassesAndUnpolledRows(t *testing.T) {
+ f := newSnapshotRecoveryFixture(t, "lxc")
+ f.poll(t, "site")
+ at := time.Now().UTC().Add(-time.Hour)
+ unrelated := []recovery.RecoveryPoint{
+ recoveryIngestTestPoint("pve-backup:site-archive", "site", at),
+ recoveryIngestTestPoint("pve-task:site-job", "site", at),
+ }
+ if err := f.store.UpsertPoints(context.Background(), unrelated); err != nil {
+ t.Fatal(err)
+ }
+ before := f.points(t)
+ // Even if every current target succeeds, a carried-forward row belongs to
+ // a guest that was not read and cannot authorise instance-wide deletion.
+ f.m.state.UpdateGuestsForInstance("site", nil, []models.Container{{ID: makeGuestID("site", "node-a", 101), Instance: "site", Node: "node-a", VMID: 101, Type: "lxc"}})
+ f.client.replies["lxc/node-a/101"] = snapshotCollectionReply{}
+ f.poll(t, "site")
+ if !reflect.DeepEqual(before, f.points(t)) {
+ t.Fatal("unpolled carried-forward guest permitted durable reconciliation")
+ }
+ // Restoring the same inventory and accepting new listings replaces old
+ // snapshots without deleting independent archive or execution records.
+ f.coldState(true)
+ for key := range f.client.replies {
+ f.client.replies[key] = snapshotCollectionReply{snapshots: []proxmox.Snapshot{{Name: "recovered", SnapTime: at.Unix()}}}
+ }
+ f.poll(t, "site")
+ after := f.points(t)
+ for _, point := range unrelated {
+ if !reflect.DeepEqual(before[point.ID], after[point.ID]) {
+ t.Fatalf("snapshot recovery changed another class: %s", point.ID)
+ }
+ }
+ if len(after) != 4 || after["pve-snapshot:site-node-a-101-recovered"].ID == "" || after["pve-snapshot:site-node-b-202-recovered"].ID == "" {
+ t.Fatalf("ordinary complete recovery failed to replace snapshot identities: %+v", after)
+ }
+}