Real-time monitoring dashboard for Proxmox VE, PBS, Docker, Kubernetes, TrueNAS and vSphere. Self-hosted, with alerts for failed backups, full disks, unhealthy containers and offline nodes.
Go aialertsdashboarddockerdocker-monitoringgohomelabinfrastructure-monitoringkubernetesmonitoringmspproxmoxproxmox-backup-serverproxmox-mail-gatewayproxmox-veself-hostedtruenastypescriptvspherewebhooks
Bind background backup scans to their connection lifetime
pulse-triage[bot]34fac9890459
diff --git a/docs/release-control/v6/internal/subsystems/monitoring.md b/docs/release-control/v6/internal/subsystems/monitoring.mdindex b7436c8322..045f8fdffd 100644--- a/docs/release-control/v6/internal/subsystems/monitoring.md+++ b/docs/release-control/v6/internal/subsystems/monitoring.md@@ -17,6 +17,35 @@## Purpose+### Background backup scans belong to their configured connection++PVE tasks, storage backups and guest snapshots share one in-flight owner per+connection. Admission and the cadence timestamp are atomic. Cycle-based or+short-interval scheduling cannot stack scans while an earlier client has not+returned, even after cancellation. Completion, not launch, starts the next+ordinary interval. The owner captures the runtime before dispatch and copies+node origins/statuses; a later core cycle cannot mutate its read inputs.++Client replacement (including fallback), runtime replacement and connection+retirement cancel/invalidate that owner. Each local acceptance boundary checks+the owner and registered client under the same publication fence used by those+transitions. Late task/log/storage/snapshot reads cannot replace live inventory,+renew accepted log observations, enqueue new recovery evidence, change access+warnings, or rewrite a replacement's scheduling/cache state. Reusing a connection+name/UPID does not reuse another binding's immutable log cache. Already accepted+historical observations remain history; retirement does not delete them.++The existing two-to-five-minute shared task/storage budget, independent bounded+snapshot budget, read caps, partial/unread preservation, alert policy, identity,+auth and guest-operation rules are unchanged. No guest operation is introduced.+`pve_backup_lifecycle_test.go` covers saturated/concurrent admission, late reads+at all five read boundaries, VM/LXC replacement/runtime/retirement, original+observation preservation, reused UPIDs, independent connection state and ordinary+recovery. Existing storage-cancellation and snapshot-budget controls still apply.+Desktop/narrow consuming proof uses actual produced inventories before and after+a late superseded read, not invented restore points or native recovery.++### Backup guest results require an unambiguous task logThe task-log reader requests the prefix from line one and accepts only a whole,diff --git a/docs/release-control/v6/internal/subsystems/registry.json b/docs/release-control/v6/internal/subsystems/registry.jsonindex 14ab871da8..c4cf593aa1 100644--- a/docs/release-control/v6/internal/subsystems/registry.json+++ b/docs/release-control/v6/internal/subsystems/registry.json@@ -6426,6 +6426,23 @@"exact_files": [],"require_explicit_path_policy_coverage": true,"path_policies": [+ {+ "id": "pve-backup-poll-ownership",+ "label": "connection-scoped single-flight backup scan and late publication proof",+ "match_prefixes": [],+ "match_files": [+ "internal/monitoring/pve_backup_poll.go",+ "internal/monitoring/guest_poll_publication.go",+ "internal/monitoring/monitor_cluster_dedupe.go",+ "internal/monitoring/monitor_pve.go",+ "internal/monitoring/monitor.go"+ ],+ "allow_same_subsystem_tests": false,+ "test_prefixes": [],+ "exact_files": [+ "internal/monitoring/pve_backup_lifecycle_test.go"+ ]+ },{"id": "task-log-response","label": "bounded ordered task log admission and guest evidence continuity proof",@@ -6920,6 +6937,7 @@"internal/monitoring/monitor_backups_dir_storage_test.go","internal/monitoring/monitor_full_coverage_test.go","internal/monitoring/pbs_protection_observation_test.go",+ "internal/monitoring/pve_backup_lifecycle_test.go","internal/monitoring/pve_protection_observation_test.go","internal/monitoring/recovery_ingest_test.go","internal/monitoring/vzdump_cached_guest_test.go",diff --git a/internal/monitoring/guest_poll_publication.go b/internal/monitoring/guest_poll_publication.goindex a4bee87850..925c22f95f 100644--- a/internal/monitoring/guest_poll_publication.go+++ b/internal/monitoring/guest_poll_publication.go@@ -144,6 +144,8 @@ func (m *Monitor) replacePVEClient(instance string, client PVEClientInterface) {m.mu.Lock()defer m.mu.Unlock()m.cancelPVEGuestPollLocked(instance)+ m.cancelPVEBackupPollLocked(instance)+ m.discardPVEBackupTaskCacheLocked(instance)if m.pveClients == nil {m.pveClients = make(map[string]PVEClientInterface)}diff --git a/internal/monitoring/monitor.go b/internal/monitoring/monitor.goindex 176056b6cc..f5c7b7346a 100644--- a/internal/monitoring/monitor.go+++ b/internal/monitoring/monitor.go@@ -1306,7 +1306,7 @@ type Monitor struct {mockUnifiedViewFixtureAt time.Time // fixture freshness the published view was built frommockUnifiedViewValid boolpveClients map[string]PVEClientInterface- pveGuestPublicationMu sync.RWMutex // Orders guest publication with connection/runtime replacement and retirement.+ pveGuestPublicationMu sync.RWMutex // Orders guest/backup publication with connection/runtime replacement and retirement.pveGuestPolls map[string]*pveGuestPoll // Guarded by m.mu; canceled cycles retain ownership until completion unless the binding ends.pbsClients map[string]*pbs.ClientpmgClients map[string]*pmg.Client@@ -1368,6 +1368,7 @@ type Monitor struct {lastAuthAttempt map[string]time.Time // Track last auth attempt timelastClusterCheck map[string]time.Time // Track last cluster check for standalone nodeslastPhysicalDiskPoll map[string]time.Time // Track last physical disk poll time per instance+ pveBackupPolls map[string]*pveBackupPoll // Owned runtime/client-scoped backup scans; guarded by mupveReplicationPolls map[string]*pveReplicationPoll // Owned runtime/client-scoped replication reads; guarded by mulastPVEBackupPoll map[string]time.Time // Track last PVE backup poll per instancevzdumpJobTaskCache map[string]vzdumpJobTaskCacheEntry // Cache synthesized per-guest tasks for multi-guest vzdump job runs, keyed by instance|UPID@@ -1521,6 +1522,9 @@ func (m *Monitor) setRuntimeContext(ctx context.Context, hub *websocket.Hub) {poll.cancel()delete(m.pveReplicationPolls, name)}+ for name := range m.pveBackupPolls {+ m.cancelPVEBackupPollLocked(name)+ }m.runtimeCtx = ctxm.wsHub = hub}diff --git a/internal/monitoring/monitor_backups.go b/internal/monitoring/monitor_backups.goindex a9eaadd456..89668e8da6 100644--- a/internal/monitoring/monitor_backups.go+++ b/internal/monitoring/monitor_backups.go@@ -137,7 +137,12 @@ func pveBackupAccessWarning(instanceCfg *config.PVEInstance, endpoint string, erreturn cause + " Rejected endpoint: " + strconv.Quote(endpoint) + ". " + pveBackupPermissionWarning(instanceCfg)}-func (m *Monitor) recordPVEBackupAccessWarning(instanceName, endpoint string, err error) {+func (m *Monitor) recordPVEBackupAccessWarning(ctx context.Context, instanceName, endpoint string, err error) {+ unlock, current := m.beginPVEBackupPublication(ctx, instanceName)+ if !current {+ return+ }+ defer unlock()warning := pveBackupAccessWarning(m.getInstanceConfig(instanceName), endpoint, err)m.mu.Lock()if m.backupPermissionWarnings == nil {@@ -367,7 +372,7 @@ func (m *Monitor) pollStorageBackupsWithNodes(ctx context.Context, instanceNameif isPVEBackupPermissionError(err) {hadPermissionError = truepermissionFailureCount++- m.recordPVEBackupAccessWarning(instanceName, fmt.Sprintf("/nodes/%s/storage", node.Node), err)+ m.recordPVEBackupAccessWarning(ctx, instanceName, fmt.Sprintf("/nodes/%s/storage", node.Node), err)}monErr := errors.NewMonitorError(errors.ErrorTypeAPI, "get_storage_for_backups", instanceName, err).WithNode(node.Node)log.Warn().Err(monErr).Str("node", node.Node).Msg("failed to get storage for backups - skipping node")@@ -422,7 +427,7 @@ func (m *Monitor) pollStorageBackupsWithNodes(ctx context.Context, instanceNameif isPVEBackupPermissionError(err) {hadPermissionError = truepermissionFailureCount++- m.recordPVEBackupAccessWarning(instanceName, fmt.Sprintf("/nodes/%s/storage/%s/content", node.Node, storage.Storage), err)+ m.recordPVEBackupAccessWarning(ctx, instanceName, fmt.Sprintf("/nodes/%s/storage/%s/content", node.Node, storage.Storage), err)} else {log.Debug().Err(monErr).Str("node", node.Node).@@ -598,6 +603,11 @@ func (m *Monitor) pollStorageBackupsWithNodes(ctx context.Context, instanceName// or renew provider evidence for a payload we did not accept.return}+ unlock, current := m.beginPVEBackupPublication(ctx, instanceName)+ if !current {+ return+ }+ defer unlock()for scope := range skippedLocalStorages {if assignment, known := storageAssignments[scope.storage]; known &&clusterStorageRestrictedToOtherNodes(assignment, scope.node) {@@ -1208,6 +1218,9 @@ func (m *Monitor) pollPVEBackupsAndSnapshots(parentCtx context.Context, instance// pollGuestSnapshots polls snapshots for all VMs and containersfunc (m *Monitor) pollGuestSnapshots(ctx context.Context, instanceName string, client PVEClientInterface) {+ if ctx.Err() != nil {+ return+ }log.Debug().Str("instance", instanceName).Msg("polling guest snapshots")// The guest inventory comes from the fixture read view in mock mode.@@ -1507,6 +1520,14 @@ func (m *Monitor) pollGuestSnapshots(ctx context.Context, instanceName string, c}}+ // A child snapshot budget may yield accepted partial results; a canceled+ // runtime/connection owner may not publish even those results.+ unlock, current := m.beginPVEBackupPublication(ctx, instanceName)+ if !current {+ return+ }+ defer unlock()+// Update state with guest snapshots for this instancem.state.UpdateGuestSnapshotsForInstance(instanceName, allSnapshots)@@ -2603,11 +2624,17 @@ func appendPBSBackupsWithinLimit(backups []models.PBSBackup, additions []models.// pollBackupTasks polls backup tasks from a PVE instancefunc (m *Monitor) pollBackupTasks(ctx context.Context, instanceName string, client PVEClientInterface) {+ if ctx.Err() != nil {+ return+ }log.Debug().Str("instance", instanceName).Msg("polling backup tasks")history, collection, err := m.readBackupTaskHistory(ctx, instanceName, client)if err != nil {- m.persistBackupTaskHistory(ctx, nil, collection)+ if unlock, current := m.beginPVEBackupPublication(ctx, instanceName); current {+ m.persistBackupTaskHistory(ctx, nil, collection)+ unlock()+ }monErr := errors.WrapAPIError("get_backup_tasks", instanceName, err, 0)log.Error().Err(monErr).Str("instance", instanceName).Msg("failed to get backup tasks")return@@ -2673,6 +2700,12 @@ func (m *Monitor) pollBackupTasks(ctx context.Context, instanceName string, clie}}+ unlock, current := m.beginPVEBackupPublication(ctx, instanceName)+ if !current {+ return+ }+ defer unlock()+// Update state with new backup tasks for this instancem.state.UpdateBackupTasksForInstance(instanceName, m.retainUnreadBackupTasks(instanceName, history.Nodes, liveTasks))@@ -2717,8 +2750,14 @@ const maxVzdumpJobLogFetchesPerPoll = 25// fetched only once. Running jobs are re-parsed every cycle to pick up guests// as the job progresses.func (m *Monitor) synthesizeVzdumpJobGuestTasks(ctx context.Context, instanceName string, client PVEClientInterface, jobTasks []models.BackupTask, jobUPIDs map[string]string) []models.BackupTask {+ if ctx.Err() != nil {+ return nil+ }if len(jobTasks) == 0 {- m.pruneVzdumpJobTaskCache(instanceName, nil)+ if unlock, current := m.beginPVEBackupPublication(ctx, instanceName); current {+ m.pruneVzdumpJobTaskCache(instanceName, nil)+ unlock()+ }return nil}@@ -2763,6 +2802,10 @@ func (m *Monitor) synthesizeVzdumpJobGuestTasks(ctx context.Context, instanceNam}continue}+ unlock, current := m.beginPVEBackupPublication(ctx, instanceName)+ if !current {+ return nil+ }fetches++delete(fetchJobs, key) // duplicate provider rows cannot spend the same turn twice// A failed or incomplete read still spent the allowance. Keep attempt@@ -2779,6 +2822,7 @@ func (m *Monitor) synthesizeVzdumpJobGuestTasks(ctx context.Context, instanceNam}m.vzdumpJobLogHistoryFirst[instanceName] = jobTask.EndTime.IsZero()m.mu.Unlock()+ unlock()logLines, err := client.GetTaskLog(ctx, jobTask.Node, upid)if err != nil {@@ -2792,6 +2836,10 @@ func (m *Monitor) synthesizeVzdumpJobGuestTasks(ctx context.Context, instanceNamcontinue}+ unlock, current = m.beginPVEBackupPublication(ctx, instanceName)+ if !current {+ return nil+ }guestTasks := refreshObservedAt(parseVzdumpJobLog(jobTask, upid, logLines), now)historyComplete := backupGuestLogComplete(jobTask, logLines, guestTasks)m.mu.Lock()@@ -2806,9 +2854,13 @@ func (m *Monitor) synthesizeVzdumpJobGuestTasks(ctx context.Context, instanceNam}m.mu.Unlock()byJob[index] = guestTasks+ unlock()}- m.pruneVzdumpJobTaskCache(instanceName, seen)+ if unlock, current := m.beginPVEBackupPublication(ctx, instanceName); current {+ m.pruneVzdumpJobTaskCache(instanceName, seen)+ unlock()+ }var synthesized []models.BackupTaskfor _, tasks := range byJob {synthesized = append(synthesized, tasks...)@@ -3086,16 +3138,24 @@ func (m *Monitor) pollPVEBackupsAsync(return}- // Set initial timestamp before starting goroutine (prevents concurrent starts).- // This optional work owns a runtime-scoped timeout below; the completed core- // PVE generation must not be reclassified as unreachable merely because its- // per-cycle context expired before background scheduling.- m.mu.Lock()- m.lastPVEBackupPoll[instanceName] = newLast- m.mu.Unlock()+ // Atomically claim this connection before dispatch. A timestamp is cadence,+ // not an in-flight fence: a scan can outlast the next eligible core cycle.+ owner := m.beginPVEBackupPoll(instanceName, client, lastPoll, newLast)+ if owner == nil {+ return+ }+ // Core-cycle cancellation must not cancel optional background inventory.+ ctx := context.WithValue(owner.ctx, pveBackupPollContextKey{}, owner)+ // The caller may reuse its inventory slices/maps on the next core cycle.+ nodes = append([]proxmox.Node(nil), nodes...)+ statuses := make(map[string]string, len(nodeEffectiveStatus))+ for node, status := range nodeEffectiveStatus {+ statuses[node] = status+ }// Run backup polling in a separate goroutine to avoid blocking real-time statsgo func(startTime time.Time, inst string, pveClient PVEClientInterface) {+ defer m.finishPVEBackupPoll(inst, owner)defer recoverFromPanic(fmt.Sprintf("pollPVEBackups-%s", inst))timeout := m.calculateBackupOperationTimeout(inst)log.Info().@@ -3103,14 +3163,7 @@ func (m *Monitor) pollPVEBackupsAsync(Dur("timeout", timeout).Msg("Starting background backup/snapshot polling")- // The per-cycle ctx is canceled as soon as the main polling loop finishes,- // so derive the backup poll context from the long-lived runtime context instead.- parentCtx := m.getRuntimeContext()- if parentCtx == nil {- parentCtx = context.Background()- }-- m.pollPVEBackupsAndSnapshots(parentCtx, inst, pveClient, nodes, nodeEffectiveStatus, timeout)+ m.pollPVEBackupsAndSnapshots(ctx, inst, pveClient, nodes, statuses, timeout)duration := time.Since(startTime)log.Info().@@ -3118,10 +3171,6 @@ func (m *Monitor) pollPVEBackupsAsync(Dur("duration", duration).Msg("Completed background backup/snapshot polling")- // Update timestamp after completion for accurate interval scheduling- m.mu.Lock()- m.lastPVEBackupPoll[inst] = time.Now()- m.mu.Unlock()}(now, instanceName, client)}diff --git a/internal/monitoring/monitor_backups_dir_storage_test.go b/internal/monitoring/monitor_backups_dir_storage_test.goindex 2cb50232ca..1890ab2c05 100644--- a/internal/monitoring/monitor_backups_dir_storage_test.go+++ b/internal/monitoring/monitor_backups_dir_storage_test.go@@ -231,7 +231,7 @@ func TestPollStorageBackups_Issue1639HarvestsPBSGuestConfirmations(t *testing.T)}}-func TestPollPVEBackupsAsyncUsesRuntimeContextIndependentOfCycle(t *testing.T) {+func TestPollPVEBackupsAsyncRejectsStoppedRuntime(t *testing.T) {runtimeCtx, cancelRuntime := context.WithCancel(context.Background())cancelRuntime()m := &Monitor{@@ -243,18 +243,20 @@ func TestPollPVEBackupsAsyncUsesRuntimeContextIndependentOfCycle(t *testing.T) {BackupPollingCycles: 1,},}+ client := &mockPVEClientExtra{}+ m.pveClients = map[string]PVEClientInterface{"cluster-a": client}m.pollPVEBackupsAsync("cluster-a",&config.PVEInstance{MonitorBackups: true},- &mockPVEClientExtra{},+ client,nil,map[string]string{},)m.mu.RLock()scheduledAt := m.lastPVEBackupPoll["cluster-a"]m.mu.RUnlock()- if scheduledAt.IsZero() {- t.Fatal("backup polling was not scheduled on the monitor runtime context")+ if !scheduledAt.IsZero() {+ t.Fatal("stopped runtime scheduled a new backup poll")}}diff --git a/internal/monitoring/monitor_cluster_dedupe.go b/internal/monitoring/monitor_cluster_dedupe.goindex 77d70d947b..450550fe9e 100644--- a/internal/monitoring/monitor_cluster_dedupe.go+++ b/internal/monitoring/monitor_cluster_dedupe.go@@ -91,6 +91,8 @@ func (m *Monitor) retirePVEInstanceRuntime(instanceName string) {m.mu.Lock()m.cancelPVEGuestPollLocked(instanceName)+ m.cancelPVEBackupPollLocked(instanceName)+ m.discardPVEBackupTaskCacheLocked(instanceName)if poll := m.pveReplicationPolls[instanceName]; poll != nil {poll.cancel()delete(m.pveReplicationPolls, instanceName)diff --git a/internal/monitoring/monitor_pve.go b/internal/monitoring/monitor_pve.goindex 5815241c1e..e4647f3a84 100644--- a/internal/monitoring/monitor_pve.go+++ b/internal/monitoring/monitor_pve.go@@ -445,6 +445,8 @@ func (m *Monitor) retryPVEPortFallback(ctx context.Context, instanceName string,m.mu.Lock()instanceCfg.Host = persistHostm.cancelPVEGuestPollLocked(instanceName)+ m.cancelPVEBackupPollLocked(instanceName)+ m.discardPVEBackupTaskCacheLocked(instanceName)m.pveClients[instanceName] = fallbackClient// Update in-memory config so subsequent polls build clients against the working port.diff --git a/internal/monitoring/pve_backup_lifecycle_test.go b/internal/monitoring/pve_backup_lifecycle_test.gonew file mode 100644index 0000000000..5ede7724ab--- /dev/null+++ b/internal/monitoring/pve_backup_lifecycle_test.go@@ -0,0 +1,294 @@+package monitoring++import (+ "context"+ "encoding/json"+ "os"+ "sync"+ "sync/atomic"+ "testing"+ "testing/synctest"+ "time"++ "github.com/rcourtman/pulse-go-rewrite/internal/config"+ "github.com/rcourtman/pulse-go-rewrite/internal/models"+ "github.com/rcourtman/pulse-go-rewrite/pkg/proxmox"+ "github.com/stretchr/testify/require"+)++// No guest-agent, task operation, external connection or installed target.+// The same behavioural tests also compile against the exact original source.+type backupLifecycleClient struct {+ mockPVEClientExtra+ label string+ read func(context.Context, string)+ calls atomic.Int32+}++func (c *backupLifecycleClient) before(ctx context.Context, phase string) {+ c.calls.Add(1)+ if c.read != nil {+ c.read(ctx, phase)+ }+}+func (c *backupLifecycleClient) GetBackupTasks(ctx context.Context) ([]proxmox.Task, error) {+ c.before(ctx, "tasks")+ return []proxmox.Task{{UPID: testJobUPID, Node: "node-a", Type: "vzdump", StartTime: time.Now().Add(-time.Hour).Unix(), EndTime: time.Now().Add(-time.Minute).Unix(), Status: "job errors"}}, nil+}+func (c *backupLifecycleClient) GetTaskLog(ctx context.Context, _, _ string) ([]proxmox.TaskLogLine, error) {+ c.before(ctx, "log")+ return []proxmox.TaskLogLine{{LineNumber: 1, Text: "INFO: Starting Backup of VM 101 (qemu)"}, {LineNumber: 2, Text: "ERROR: Backup of VM 101 failed - " + c.label + " result"}, {LineNumber: 3, Text: "TASK ERROR: job errors"}}, nil+}+func (c *backupLifecycleClient) GetStorage(ctx context.Context, _ string) ([]proxmox.Storage, error) {+ c.before(ctx, "storage")+ return []proxmox.Storage{{Storage: "archive", Type: "dir", Content: "backup", Active: 1, Enabled: 1}}, nil+}+func (c *backupLifecycleClient) GetStorageContent(ctx context.Context, _, _ string) ([]proxmox.StorageContent, error) {+ c.before(ctx, "content")+ return []proxmox.StorageContent{{Volid: "archive:backup/vzdump-qemu-101-" + c.label + ".vma.zst", VMID: 101, CTime: time.Now().Add(-2 * time.Hour).Unix(), Size: 1024, Content: "backup"}}, nil+}+func (c *backupLifecycleClient) GetVMSnapshots(ctx context.Context, _ string, _ int) ([]proxmox.Snapshot, error) {+ c.before(ctx, "snapshots")+ return []proxmox.Snapshot{{Name: c.label, SnapTime: time.Now().Add(-time.Hour).Unix()}}, nil+}+func (c *backupLifecycleClient) GetContainerSnapshots(ctx context.Context, node string, id int) ([]proxmox.Snapshot, error) {+ return c.GetVMSnapshots(ctx, node, id)+}+func newBackupLifecycleMonitor(c PVEClientInterface, kind string) *Monitor {+ m := &Monitor{state: models.NewState(), pveClients: map[string]PVEClientInterface{"site-a": c}, config: &config.Config{EnableBackupPolling: true, BackupPollingCycles: 1}, lastPVEBackupPoll: map[string]time.Time{}, runtimeCtx: context.Background()}+ if kind == "qemu" {+ m.state.UpdateVMsForInstance("site-a", []models.VM{{ID: "site-a-node-a-101", VMID: 101, Node: "node-a", Instance: "site-a", Name: "workload", Status: "running"}})+ } else {+ m.state.UpdateContainersForInstance("site-a", []models.Container{{ID: "site-a-node-a-101", VMID: 101, Node: "node-a", Instance: "site-a", Name: "workload", Status: "running"}})+ }+ m.state.UpdateStorageBackupsForInstance("site-b", []models.StorageBackup{{ID: "peer-file", Instance: "site-b", VMID: 101, Storage: "archive", Time: time.Now().Add(-time.Hour)}})+ m.state.UpdateGuestSnapshotsForInstance("site-b", []models.GuestSnapshot{{ID: "peer-snap", Instance: "site-b", VMID: 101, Name: "independent", Time: time.Now().Add(-time.Hour)}})+ return m+}+func scheduleBackupLifecycle(m *Monitor, c PVEClientInterface) {+ m.pollPVEBackupsAsync("site-a", &config.PVEInstance{MonitorBackups: true}, c, []proxmox.Node{{Node: "node-a", Status: "online"}}, map[string]string{"node-a": "online"})+}++func TestPVEBackupLifecycleSingleFlight(t *testing.T) {+ for _, interval := range []time.Duration{0, time.Minute} {+ t.Run(interval.String(), func(t *testing.T) {+ synctest.Test(t, func(t *testing.T) {+ release := make(chan struct{})+ var tasks atomic.Int32+ c := &backupLifecycleClient{label: "accepted", read: func(_ context.Context, phase string) {+ if phase == "tasks" {+ tasks.Add(1)+ <-release+ }+ }}+ m := newBackupLifecycleMonitor(c, "qemu")+ m.config.BackupPollingInterval = interval+ scheduleBackupLifecycle(m, c)+ synctest.Wait()+ // A slow scan outlasts cadence, but it still owns the connection.+ time.Sleep(90 * time.Second)+ for i := 0; i < 8; i++ {+ scheduleBackupLifecycle(m, c)+ }+ synctest.Wait()+ got := tasks.Load()+ close(release)+ synctest.Wait()+ require.Equal(t, int32(1), got, "cadence must not stack scans")+ completed := m.lastPVEBackupPoll["site-a"]+ require.Equal(t, time.Now(), completed, "cadence starts at accepted completion")+ time.Sleep(interval + time.Second)+ scheduleBackupLifecycle(m, c)+ synctest.Wait()+ require.Equal(t, int32(2), tasks.Load(), "ordinary next eligible cycle recovers")+ })+ })+ }+}++func TestPVEBackupLifecycleLateReadsCannotReplaceCurrentEvidence(t *testing.T) {+ for _, kind := range []string{"qemu", "lxc"} {+ for _, phase := range []string{"tasks", "log", "storage", "content", "snapshots"} {+ for _, transition := range []string{"client", "runtime", "retire"} {+ t.Run(kind+"/"+phase+"/"+transition, func(t *testing.T) {+ synctest.Test(t, func(t *testing.T) {+ release := make(chan struct{})+ released := false+ defer func() {+ if !released {+ close(release)+ }+ }()+ var reached atomic.Bool+ var oldCtx context.Context+ old := &backupLifecycleClient{label: "old", read: func(ctx context.Context, p string) {+ if p == phase {+ oldCtx = ctx+ reached.Store(true)+ <-release+ }+ }}+ m := newBackupLifecycleMonitor(old, kind)+ scheduleBackupLifecycle(m, old)+ synctest.Wait()+ require.True(t, reached.Load())+ fresh := &backupLifecycleClient{label: "current"}+ switch transition {+ case "retire":+ m.retirePVEInstanceRuntime("site-a")+ case "runtime":+ m.setRuntimeContext(context.Background(), nil) // Deliberately don't cancel the old parent.+ }+ m.replacePVEClient("site-a", fresh)+ if transition == "retire" {+ // Fresh configuration rebuilds its guest inventory without a live guest operation.+ source := newBackupLifecycleMonitor(fresh, kind).state.GetSnapshot()+ m.state.UpdateVMsForInstance("site-a", source.VMs)+ m.state.UpdateContainersForInstance("site-a", source.Containers)+ }+ scheduleBackupLifecycle(m, fresh)+ synctest.Wait()+ before := m.state.GetSnapshot().PVEBackups+ require.NotZero(t, fresh.calls.Load(), "replacement must not wait for the late client")+ currentStamp := m.lastPVEBackupPoll["site-a"]+ cache := m.vzdumpJobTaskCache["site-a|"+testJobUPID]+ close(release)+ released = true+ synctest.Wait()+ require.NotNil(t, oldCtx.Err(), "binding transition must cancel the old read")+ require.Equal(t, before, m.state.GetSnapshot().PVEBackups, "late read changed accepted inventories")+ require.Equal(t, currentStamp, m.lastPVEBackupPoll["site-a"], "old finish rewrote replacement cadence")+ require.Equal(t, cache, m.vzdumpJobTaskCache["site-a|"+testJobUPID], "old result poisoned the immutable cache")+ require.Len(t, before.StorageBackups, 2)+ require.Len(t, before.GuestSnapshots, 2)+ found := false+ for _, task := range before.BackupTasks {+ if task.VMID == 101 {+ require.Equal(t, "current result", task.Error)+ found = true+ }+ }+ require.True(t, found, "new client must read its own guest log even for a reused UPID")+ if os.Getenv("PULSE_BACKUP_LIFECYCLE_FRAMES") == "1" && kind == "qemu" && phase == "tasks" && transition == "client" {+ data, err := json.Marshal(map[string]any{"accepted": before, "after_late": m.state.GetSnapshot().PVEBackups, "vms": m.state.GetSnapshot().VMs, "containers": m.state.GetSnapshot().Containers})+ require.NoError(t, err)+ t.Log("BACKUP_LIFECYCLE_FRAME=" + string(data))+ }+ })+ })+ }+ }+ }+}++func TestPVEBackupLifecycleRetirementDoesNotResurrectState(t *testing.T) {+ synctest.Test(t, func(t *testing.T) {+ release := make(chan struct{})+ old := &backupLifecycleClient{label: "retired", read: func(_ context.Context, p string) {+ if p == "tasks" {+ <-release+ }+ }}+ m := newBackupLifecycleMonitor(old, "qemu")+ scheduleBackupLifecycle(m, old)+ synctest.Wait()+ m.retirePVEInstanceRuntime("site-a")+ before := m.state.GetSnapshot().PVEBackups+ close(release)+ synctest.Wait()+ require.Equal(t, before, m.state.GetSnapshot().PVEBackups)+ _, exists := m.lastPVEBackupPoll["site-a"]+ require.False(t, exists, "retired completion recreated scheduling state")+ require.Empty(t, m.vzdumpJobTaskCache)+ })+}++func TestPVEBackupLifecycleConcurrentAdmission(t *testing.T) {+ synctest.Test(t, func(t *testing.T) {+ release := make(chan struct{})+ var tasks atomic.Int32+ c := &backupLifecycleClient{label: "accepted", read: func(_ context.Context, p string) {+ if p == "tasks" {+ tasks.Add(1)+ <-release+ }+ }}+ m := newBackupLifecycleMonitor(c, "qemu")+ var callers sync.WaitGroup+ for i := 0; i < 32; i++ {+ callers.Add(1)+ go func() { defer callers.Done(); scheduleBackupLifecycle(m, c) }()+ }+ callers.Wait()+ synctest.Wait()+ count := tasks.Load()+ close(release)+ synctest.Wait()+ require.Equal(t, int32(1), count)+ })+}++func TestPVEBackupLifecycleCopiesSchedulingInventory(t *testing.T) {+ synctest.Test(t, func(t *testing.T) {+ release := make(chan struct{})+ c := &backupLifecycleClient{label: "accepted", read: func(_ context.Context, p string) {+ if p == "tasks" {+ <-release+ }+ }}+ m := newBackupLifecycleMonitor(c, "qemu")+ nodes := []proxmox.Node{{Node: "node-a", Status: "online"}}+ statuses := map[string]string{"node-a": "online"}+ m.pollPVEBackupsAsync("site-a", &config.PVEInstance{MonitorBackups: true}, c, nodes, statuses)+ synctest.Wait()+ nodes[0].Node = "reused-core-buffer"+ statuses["node-a"] = "offline"+ close(release)+ synctest.Wait()+ found := false+ for _, row := range m.state.GetSnapshot().PVEBackups.StorageBackups {+ if row.Instance == "site-a" {+ found = true+ require.Equal(t, "node-a", row.StorageNode)+ }+ }+ require.True(t, found, "next core cycle mutated the owned storage origin")+ })+}++func TestPVEBackupLifecycleCanceledCollectorsKeepEvidence(t *testing.T) {+ for _, phase := range []string{"tasks", "log", "snapshots"} {+ for _, kind := range []string{"qemu", "lxc"} {+ t.Run(phase+"/"+kind, func(t *testing.T) {+ c := &backupLifecycleClient{label: "accepted"}+ m := newBackupLifecycleMonitor(c, kind)+ m.pollBackupTasks(context.Background(), "site-a", c)+ m.pollGuestSnapshots(context.Background(), "site-a", c)+ before := m.state.GetSnapshot().PVEBackups+ ctx, cancel := context.WithCancel(context.Background())+ defer cancel()+ c.label = "late"+ c.read = func(_ context.Context, p string) {+ if p == phase {+ cancel()+ }+ }+ // Non-finalise the prior cache to force another real log read.+ key := "site-a|" + testJobUPID+ entry := m.vzdumpJobTaskCache[key]+ entry.final = false+ m.vzdumpJobTaskCache[key] = entry+ cache := m.vzdumpJobTaskCache[key]+ if phase == "snapshots" {+ m.pollGuestSnapshots(ctx, "site-a", c)+ } else {+ m.pollBackupTasks(ctx, "site-a", c)+ }+ require.Equal(t, before, m.state.GetSnapshot().PVEBackups)+ after := m.vzdumpJobTaskCache[key]+ require.Equal(t, cache.tasks, after.tasks, "cancellation replaced accepted log evidence")+ require.Equal(t, cache.final, after.final)+ })+ }+ }+}diff --git a/internal/monitoring/pve_backup_poll.go b/internal/monitoring/pve_backup_poll.gonew file mode 100644index 0000000000..434a4b3337--- /dev/null+++ b/internal/monitoring/pve_backup_poll.go@@ -0,0 +1,109 @@+package monitoring++import (+ "context"+ "strings"+ "time"+)++// One owner covers tasks, storage and snapshots. Their existing independent+// budgets remain children of this runtime-scoped context, not a core cycle.+// Cancellation is not completion: ordinary cycles cannot stack another scan+// behind a client that has not returned yet.+type pveBackupPoll struct {+ ctx context.Context+ cancel context.CancelFunc+ client PVEClientInterface+ done chan struct{}+}++type pveBackupPollContextKey struct{}++func (m *Monitor) beginPVEBackupPoll(instance string, client PVEClientInterface, last, next time.Time) *pveBackupPoll {+ m.pveGuestPublicationMu.RLock()+ defer m.pveGuestPublicationMu.RUnlock()+ m.mu.Lock()+ defer m.mu.Unlock()+ if client == nil || m.pveClients[instance] != client || !m.lastPVEBackupPoll[instance].Equal(last) {+ return nil+ }+ parent := m.runtimeCtx+ if parent == nil {+ parent = context.Background()+ }+ if parent.Err() != nil {+ return nil+ }+ if previous := m.pveBackupPolls[instance]; previous != nil {+ return nil+ }+ ctx, cancel := context.WithCancel(parent)+ owner := &pveBackupPoll{ctx: ctx, cancel: cancel, client: client, done: make(chan struct{})}+ if m.pveBackupPolls == nil {+ m.pveBackupPolls = make(map[string]*pveBackupPoll)+ }+ if m.lastPVEBackupPoll == nil {+ m.lastPVEBackupPoll = make(map[string]time.Time)+ }+ m.pveBackupPolls[instance] = owner+ m.lastPVEBackupPoll[instance] = next+ return owner+}++func (m *Monitor) finishPVEBackupPoll(instance string, owner *pveBackupPoll) {+ m.mu.Lock()+ if m.pveBackupPolls[instance] == owner {+ delete(m.pveBackupPolls, instance)+ if owner.ctx.Err() == nil && m.pveClients[instance] == owner.client {+ m.lastPVEBackupPoll[instance] = time.Now()+ }+ }+ m.mu.Unlock()+ owner.cancel()+ close(owner.done)+}++// Called with m.mu and the connection-publication writer held. A replacement+// may start immediately; the old owner cannot publish or release the new claim.+func (m *Monitor) cancelPVEBackupPollLocked(instance string) {+ if owner := m.pveBackupPolls[instance]; owner != nil {+ owner.cancel()+ delete(m.pveBackupPolls, instance)+ delete(m.lastPVEBackupPoll, instance)+ }+}++// Hold the connection publication reader across local acceptance, never across+// a provider read. Retirement/replacement uses the writer, so checking the+// owner and publishing its effects is atomic with respect to either transition.+// Already accepted/enqueued historical evidence remains valid; late reads do not.+func (m *Monitor) beginPVEBackupPublication(ctx context.Context, instance string) (func(), bool) {+ if ctx.Err() != nil {+ return nil, false+ }+ owner, _ := ctx.Value(pveBackupPollContextKey{}).(*pveBackupPoll)+ if owner == nil {+ return func() {}, true // focused collectors retain their normal behaviour+ }+ m.pveGuestPublicationMu.RLock()+ m.mu.RLock()+ current := ctx.Err() == nil && owner.ctx.Err() == nil && m.pveBackupPolls[instance] == owner && m.pveClients[instance] == owner.client+ m.mu.RUnlock()+ if !current {+ m.pveGuestPublicationMu.RUnlock()+ return nil, false+ }+ return m.pveGuestPublicationMu.RUnlock, true+}++// Connection identity, not just a transport pointer, bounds cached task logs.+// Keep accepted live state until an ordinary new read replaces it, but never+// let a reused name/UPID borrow another connection's immutable guest results.+func (m *Monitor) discardPVEBackupTaskCacheLocked(instance string) {+ delete(m.vzdumpJobLogHistoryFirst, instance)+ for key := range m.vzdumpJobTaskCache {+ if strings.HasPrefix(key, instance+"|") {+ delete(m.vzdumpJobTaskCache, key)+ }+ }+}