Real-time monitoring dashboard for Proxmox VE, PBS, Docker, Kubernetes, TrueNAS and vSphere. Self-hosted, with alerts for failed backups, full disks, unhealthy containers and offline nodes.
Go aialertsdashboarddockerdocker-monitoringgohomelabinfrastructure-monitoringkubernetesmonitoringmspproxmoxproxmox-backup-serverproxmox-mail-gatewayproxmox-veself-hostedtruenastypescriptvspherewebhooks
Preserve accepted snapshot listings through optional sizing expiry
pulse-triage[bot]32dbd95a4d15
diff --git a/docs/release-control/v6/internal/subsystems/monitoring.md b/docs/release-control/v6/internal/subsystems/monitoring.mdindex 894b7d1dfc..9abaf7ef0b 100644--- a/docs/release-control/v6/internal/subsystems/monitoring.md+++ b/docs/release-control/v6/internal/subsystems/monitoring.md@@ -39,11 +39,20 @@ The existing two-to-five-minute shared task/storage budget, independent boundedsnapshot budget, read caps, partial/unread preservation, alert policy, identity,auth and guest-operation rules are unchanged. A task listing or log acceptedbefore a later log exhausts the shared budget remains usable while its owner-is current; a read returning after cancellation supplies no new guest facts. No guest operation is introduced.+is current; a read returning after cancellation supplies no new guest facts.+Snapshot listings accepted before optional size enrichment exhausts its child+budget likewise remain usable, with missing sizes unknown. Neither a cancelled+listing nor an expired sizing budget can authorise inventory reconciliation;+runtime/connection withdrawal still fences every effect, including accepted+listings whose sizing returns late. No guest operation is introduced.`pve_backup_lifecycle_test.go` covers saturated/concurrent admission, late reads-at all five read boundaries, VM/LXC replacement/runtime/retirement, original+at all six read boundaries (including snapshot sizing), VM/LXC+replacement/runtime/retirement, originalobservation preservation, reused UPIDs, independent connection state and ordinaryrecovery. Existing storage-cancellation and snapshot-budget controls still apply.+`TestPVESnapshotCancelledAfterListingsCannotReconcile` keeps accepted listings+without removing durable history; `TestPVEBackupLifecycleSizingBudgetKeepsAcceptedListings`+checks that independent sizing expiry loses neither listings nor the next poll.Desktop/narrow consuming proof uses actual produced inventories before and aftera late superseded read, not invented restore points or native recovery.diff --git a/internal/monitoring/canonical_guardrails_test.go b/internal/monitoring/canonical_guardrails_test.goindex 7531c0d5e3..be52e1a229 100644--- a/internal/monitoring/canonical_guardrails_test.go+++ b/internal/monitoring/canonical_guardrails_test.go@@ -2599,7 +2599,12 @@ func TestReloadAndRuntimeContextStayOnCanonicalMonitoringPath(t *testing.T) {"m.setRuntimeContext(ctx, wsHub)",},"monitor_backups.go": {- "parentCtx := m.getRuntimeContext()",+ "owner := m.beginPVEBackupPoll(instanceName, client, lastPoll, newLast)",+ "ctx := context.WithValue(owner.ctx, pveBackupPollContextKey{}, owner)",+ },+ "pve_backup_poll.go": {+ "parent := m.runtimeCtx",+ "ctx, cancel := context.WithCancel(parent)",},"monitor_pve.go": {"parentCtx := m.getRuntimeContext()",diff --git a/internal/monitoring/monitor_backups.go b/internal/monitoring/monitor_backups.goindex e707ae1bc3..8e460d9d41 100644--- a/internal/monitoring/monitor_backups.go+++ b/internal/monitoring/monitor_backups.go@@ -1424,6 +1424,12 @@ func (m *Monitor) pollGuestSnapshots(ctx context.Context, instanceName string, cdefer func() { <-sem }()snapshots, err := fetchSnapshots(target)+ // A provider returning nil error after cancellation did not supply+ // an accepted listing. Optional sizing can exhaust its budget later,+ // but cannot retroactively invalidate a listing accepted here.+ if err == nil {+ err = snapshotCtx.Err()+ }result := guestSnapshotPollResult{target: target, err: err}if err == nil {result.polled = true@@ -1520,9 +1526,11 @@ func (m *Monitor) pollGuestSnapshots(ctx context.Context, instanceName string, c}}- // A child snapshot budget may yield accepted partial results; a canceled- // runtime/connection owner may not publish even those results.- unlock, current := m.beginPVEBackupPublication(ctx, instanceName)+ // Keep listings accepted before optional sizing exhausted the child budget.+ // WithoutCancel retains the owner value: runtime cancellation, retirement+ // and connection replacement still fence every local effect. A cancelled+ // listing above is not observed evidence and cannot authorise removal.+ unlock, current := m.beginPVEBackupPublication(context.WithoutCancel(ctx), instanceName)if !current {return}diff --git a/internal/monitoring/monitor_backups_readstate_test.go b/internal/monitoring/monitor_backups_readstate_test.goindex 0828cc62da..d2774759c7 100644--- a/internal/monitoring/monitor_backups_readstate_test.go+++ b/internal/monitoring/monitor_backups_readstate_test.go@@ -698,7 +698,10 @@ func TestMonitorPollGuestSnapshots_RefreshesStaleCanonicalStoreForClusterGuest(tresourceStore: adapter,}- client := &backupStorageTimeoutSnapshotClient{+ // This checks canonical guest lookup, not storage-budget exhaustion. The+ // timeout fake deliberately blocks snapshot sizing until the caller ends,+ // conflating an accepted listing with a cancelled publication.+ client := &mockPVEClientSnapshots{snapshots: []pveapi.Snapshot{{Name: "cluster-snap",SnapTime: now.Unix(),@@ -710,10 +713,6 @@ func TestMonitorPollGuestSnapshots_RefreshesStaleCanonicalStoreForClusterGuest(tdefer cancel()m.pollGuestSnapshots(ctx, "homelab", client)- if client.snapshotCalls == 0 {- t.Fatal("expected guest snapshot polling to use fresh clustered guest state")- }-snapshot := state.GetSnapshot()if len(snapshot.PVEBackups.GuestSnapshots) != 1 {t.Fatalf("expected one guest snapshot from fresh clustered guest state, got %+v", snapshot.PVEBackups.GuestSnapshots)diff --git a/internal/monitoring/pve_backup_lifecycle_test.go b/internal/monitoring/pve_backup_lifecycle_test.goindex 26d0225de2..70cd26ea59 100644--- a/internal/monitoring/pve_backup_lifecycle_test.go+++ b/internal/monitoring/pve_backup_lifecycle_test.go@@ -108,7 +108,7 @@ func TestPVEBackupLifecycleSingleFlight(t *testing.T) {func TestPVEBackupLifecycleLateReadsCannotReplaceCurrentEvidence(t *testing.T) {for _, kind := range []string{"qemu", "lxc"} {- for _, phase := range []string{"tasks", "log", "storage", "content", "snapshots"} {+ for _, phase := range []string{"tasks", "log", "storage", "content", "snapshots", "sizing"} {for _, transition := range []string{"client", "runtime", "retire"} {t.Run(kind+"/"+phase+"/"+transition, func(t *testing.T) {synctest.Test(t, func(t *testing.T) {@@ -120,9 +120,11 @@ func TestPVEBackupLifecycleLateReadsCannotReplaceCurrentEvidence(t *testing.T) {}}()var reached atomic.Bool+ var storageReads atomic.Int32var oldCtx context.Contextold := &backupLifecycleClient{label: "old", read: func(ctx context.Context, p string) {- if p == phase {+ sizing := p == "storage" && storageReads.Add(1) == 2+ if p == phase || (phase == "sizing" && sizing) {oldCtx = ctxreached.Store(true)<-release@@ -314,6 +316,43 @@ func TestPVEBackupLifecycleCanceledCollectorsKeepEvidence(t *testing.T) {}}+func TestPVEBackupLifecycleSizingBudgetKeepsAcceptedListings(t *testing.T) {+ for _, kind := range []string{"qemu", "lxc"} {+ t.Run(kind, func(t *testing.T) {+ synctest.Test(t, func(t *testing.T) {+ var storageReads atomic.Int32+ c := &backupLifecycleClient{label: "accepted", read: func(ctx context.Context, phase string) {+ if phase == "storage" && storageReads.Add(1) == 2 {+ // The listing succeeded before this optional enrichment.+ // Exhaust only the snapshot child, not its runtime owner.+ <-ctx.Done()+ }+ }}+ m := newBackupLifecycleMonitor(c, kind)+ scheduleBackupLifecycle(m, c)+ synctest.Wait()+ require.Equal(t, int32(2), storageReads.Load(), "fixture must reach snapshot sizing")+ time.Sleep(61 * time.Second)+ synctest.Wait()+ state := m.state.GetSnapshot()+ require.Len(t, state.PVEBackups.GuestSnapshots, 2, "accepted listing and independent peer survive sizing timeout")+ for _, snap := range state.PVEBackups.GuestSnapshots {+ if snap.Instance == "site-a" {+ require.Equal(t, "accepted", snap.Name)+ require.Zero(t, snap.SizeBytes, "missing enrichment must not invent a size")+ }+ }+ require.Empty(t, m.pveBackupPolls, "timed-out sizing must release the completed owner")+ if os.Getenv("PULSE_BACKUP_LIFECYCLE_FRAMES") == "1" && kind == "qemu" {+ data, err := json.Marshal(map[string]any{"accepted": state.PVEBackups, "vms": state.VMs, "containers": state.Containers})+ require.NoError(t, err)+ t.Log("BACKUP_SNAPSHOT_BUDGET_FRAME=" + string(data))+ }+ })+ })+ }+}+func TestPVEBackupLifecycleCompletedReplacementRefreshesImmediately(t *testing.T) {synctest.Test(t, func(t *testing.T) {old := &backupLifecycleClient{label: "old"}