Skip to content

armgit / Pulse

Public Imported from GitHub
0

Real-time monitoring dashboard for Proxmox VE, PBS, Docker, Kubernetes, TrueNAS and vSphere. Self-hosted, with alerts for failed backups, full disks, unhealthy containers and offline nodes.

Preserve accepted snapshot listings through optional sizing expiry

pulse-triage[bot]32dbd95a4d15
diff --git a/docs/release-control/v6/internal/subsystems/monitoring.md b/docs/release-control/v6/internal/subsystems/monitoring.md
index 894b7d1dfc..9abaf7ef0b 100644
--- a/docs/release-control/v6/internal/subsystems/monitoring.md
+++ b/docs/release-control/v6/internal/subsystems/monitoring.md
@@ -39,11 +39,20 @@ The existing two-to-five-minute shared task/storage budget, independent bounded
snapshot budget, read caps, partial/unread preservation, alert policy, identity,
auth and guest-operation rules are unchanged. A task listing or log accepted
before a later log exhausts the shared budget remains usable while its owner
-is current; a read returning after cancellation supplies no new guest facts. No guest operation is introduced.
+is current; a read returning after cancellation supplies no new guest facts.
+Snapshot listings accepted before optional size enrichment exhausts its child
+budget likewise remain usable, with missing sizes unknown. Neither a cancelled
+listing nor an expired sizing budget can authorise inventory reconciliation;
+runtime/connection withdrawal still fences every effect, including accepted
+listings whose sizing returns late. No guest operation is introduced.
`pve_backup_lifecycle_test.go` covers saturated/concurrent admission, late reads
-at all five read boundaries, VM/LXC replacement/runtime/retirement, original
+at all six read boundaries (including snapshot sizing), VM/LXC
+replacement/runtime/retirement, original
observation preservation, reused UPIDs, independent connection state and ordinary
recovery. Existing storage-cancellation and snapshot-budget controls still apply.
+`TestPVESnapshotCancelledAfterListingsCannotReconcile` keeps accepted listings
+without removing durable history; `TestPVEBackupLifecycleSizingBudgetKeepsAcceptedListings`
+checks that independent sizing expiry loses neither listings nor the next poll.
Desktop/narrow consuming proof uses actual produced inventories before and after
a late superseded read, not invented restore points or native recovery.
diff --git a/internal/monitoring/canonical_guardrails_test.go b/internal/monitoring/canonical_guardrails_test.go
index 7531c0d5e3..be52e1a229 100644
--- a/internal/monitoring/canonical_guardrails_test.go
+++ b/internal/monitoring/canonical_guardrails_test.go
@@ -2599,7 +2599,12 @@ func TestReloadAndRuntimeContextStayOnCanonicalMonitoringPath(t *testing.T) {
"m.setRuntimeContext(ctx, wsHub)",
},
"monitor_backups.go": {
- "parentCtx := m.getRuntimeContext()",
+ "owner := m.beginPVEBackupPoll(instanceName, client, lastPoll, newLast)",
+ "ctx := context.WithValue(owner.ctx, pveBackupPollContextKey{}, owner)",
+ },
+ "pve_backup_poll.go": {
+ "parent := m.runtimeCtx",
+ "ctx, cancel := context.WithCancel(parent)",
},
"monitor_pve.go": {
"parentCtx := m.getRuntimeContext()",
diff --git a/internal/monitoring/monitor_backups.go b/internal/monitoring/monitor_backups.go
index e707ae1bc3..8e460d9d41 100644
--- a/internal/monitoring/monitor_backups.go
+++ b/internal/monitoring/monitor_backups.go
@@ -1424,6 +1424,12 @@ func (m *Monitor) pollGuestSnapshots(ctx context.Context, instanceName string, c
defer func() { <-sem }()
snapshots, err := fetchSnapshots(target)
+ // A provider returning nil error after cancellation did not supply
+ // an accepted listing. Optional sizing can exhaust its budget later,
+ // but cannot retroactively invalidate a listing accepted here.
+ if err == nil {
+ err = snapshotCtx.Err()
+ }
result := guestSnapshotPollResult{target: target, err: err}
if err == nil {
result.polled = true
@@ -1520,9 +1526,11 @@ func (m *Monitor) pollGuestSnapshots(ctx context.Context, instanceName string, c
}
}
- // A child snapshot budget may yield accepted partial results; a canceled
- // runtime/connection owner may not publish even those results.
- unlock, current := m.beginPVEBackupPublication(ctx, instanceName)
+ // Keep listings accepted before optional sizing exhausted the child budget.
+ // WithoutCancel retains the owner value: runtime cancellation, retirement
+ // and connection replacement still fence every local effect. A cancelled
+ // listing above is not observed evidence and cannot authorise removal.
+ unlock, current := m.beginPVEBackupPublication(context.WithoutCancel(ctx), instanceName)
if !current {
return
}
diff --git a/internal/monitoring/monitor_backups_readstate_test.go b/internal/monitoring/monitor_backups_readstate_test.go
index 0828cc62da..d2774759c7 100644
--- a/internal/monitoring/monitor_backups_readstate_test.go
+++ b/internal/monitoring/monitor_backups_readstate_test.go
@@ -698,7 +698,10 @@ func TestMonitorPollGuestSnapshots_RefreshesStaleCanonicalStoreForClusterGuest(t
resourceStore: adapter,
}
- client := &backupStorageTimeoutSnapshotClient{
+ // This checks canonical guest lookup, not storage-budget exhaustion. The
+ // timeout fake deliberately blocks snapshot sizing until the caller ends,
+ // conflating an accepted listing with a cancelled publication.
+ client := &mockPVEClientSnapshots{
snapshots: []pveapi.Snapshot{{
Name: "cluster-snap",
SnapTime: now.Unix(),
@@ -710,10 +713,6 @@ func TestMonitorPollGuestSnapshots_RefreshesStaleCanonicalStoreForClusterGuest(t
defer cancel()
m.pollGuestSnapshots(ctx, "homelab", client)
- if client.snapshotCalls == 0 {
- t.Fatal("expected guest snapshot polling to use fresh clustered guest state")
- }
-
snapshot := state.GetSnapshot()
if len(snapshot.PVEBackups.GuestSnapshots) != 1 {
t.Fatalf("expected one guest snapshot from fresh clustered guest state, got %+v", snapshot.PVEBackups.GuestSnapshots)
diff --git a/internal/monitoring/pve_backup_lifecycle_test.go b/internal/monitoring/pve_backup_lifecycle_test.go
index 26d0225de2..70cd26ea59 100644
--- a/internal/monitoring/pve_backup_lifecycle_test.go
+++ b/internal/monitoring/pve_backup_lifecycle_test.go
@@ -108,7 +108,7 @@ func TestPVEBackupLifecycleSingleFlight(t *testing.T) {
func TestPVEBackupLifecycleLateReadsCannotReplaceCurrentEvidence(t *testing.T) {
for _, kind := range []string{"qemu", "lxc"} {
- for _, phase := range []string{"tasks", "log", "storage", "content", "snapshots"} {
+ for _, phase := range []string{"tasks", "log", "storage", "content", "snapshots", "sizing"} {
for _, transition := range []string{"client", "runtime", "retire"} {
t.Run(kind+"/"+phase+"/"+transition, func(t *testing.T) {
synctest.Test(t, func(t *testing.T) {
@@ -120,9 +120,11 @@ func TestPVEBackupLifecycleLateReadsCannotReplaceCurrentEvidence(t *testing.T) {
}
}()
var reached atomic.Bool
+ var storageReads atomic.Int32
var oldCtx context.Context
old := &backupLifecycleClient{label: "old", read: func(ctx context.Context, p string) {
- if p == phase {
+ sizing := p == "storage" && storageReads.Add(1) == 2
+ if p == phase || (phase == "sizing" && sizing) {
oldCtx = ctx
reached.Store(true)
<-release
@@ -314,6 +316,43 @@ func TestPVEBackupLifecycleCanceledCollectorsKeepEvidence(t *testing.T) {
}
}
+func TestPVEBackupLifecycleSizingBudgetKeepsAcceptedListings(t *testing.T) {
+ for _, kind := range []string{"qemu", "lxc"} {
+ t.Run(kind, func(t *testing.T) {
+ synctest.Test(t, func(t *testing.T) {
+ var storageReads atomic.Int32
+ c := &backupLifecycleClient{label: "accepted", read: func(ctx context.Context, phase string) {
+ if phase == "storage" && storageReads.Add(1) == 2 {
+ // The listing succeeded before this optional enrichment.
+ // Exhaust only the snapshot child, not its runtime owner.
+ <-ctx.Done()
+ }
+ }}
+ m := newBackupLifecycleMonitor(c, kind)
+ scheduleBackupLifecycle(m, c)
+ synctest.Wait()
+ require.Equal(t, int32(2), storageReads.Load(), "fixture must reach snapshot sizing")
+ time.Sleep(61 * time.Second)
+ synctest.Wait()
+ state := m.state.GetSnapshot()
+ require.Len(t, state.PVEBackups.GuestSnapshots, 2, "accepted listing and independent peer survive sizing timeout")
+ for _, snap := range state.PVEBackups.GuestSnapshots {
+ if snap.Instance == "site-a" {
+ require.Equal(t, "accepted", snap.Name)
+ require.Zero(t, snap.SizeBytes, "missing enrichment must not invent a size")
+ }
+ }
+ require.Empty(t, m.pveBackupPolls, "timed-out sizing must release the completed owner")
+ if os.Getenv("PULSE_BACKUP_LIFECYCLE_FRAMES") == "1" && kind == "qemu" {
+ data, err := json.Marshal(map[string]any{"accepted": state.PVEBackups, "vms": state.VMs, "containers": state.Containers})
+ require.NoError(t, err)
+ t.Log("BACKUP_SNAPSHOT_BUDGET_FRAME=" + string(data))
+ }
+ })
+ })
+ }
+}
+
func TestPVEBackupLifecycleCompletedReplacementRefreshesImmediately(t *testing.T) {
synctest.Test(t, func(t *testing.T) {
old := &backupLifecycleClient{label: "old"}