Skip to content

armgit / Pulse

Public Imported from GitHub
0

Real-time monitoring dashboard for Proxmox VE, PBS, Docker, Kubernetes, TrueNAS and vSphere. Self-hosted, with alerts for failed backups, full disks, unhealthy containers and offline nodes.

Bind background backup scans to their connection lifetime

pulse-triage[bot]34fac9890459
diff --git a/docs/release-control/v6/internal/subsystems/monitoring.md b/docs/release-control/v6/internal/subsystems/monitoring.md
index b7436c8322..045f8fdffd 100644
--- a/docs/release-control/v6/internal/subsystems/monitoring.md
+++ b/docs/release-control/v6/internal/subsystems/monitoring.md
@@ -17,6 +17,35 @@
## Purpose
+### Background backup scans belong to their configured connection
+
+PVE tasks, storage backups and guest snapshots share one in-flight owner per
+connection. Admission and the cadence timestamp are atomic. Cycle-based or
+short-interval scheduling cannot stack scans while an earlier client has not
+returned, even after cancellation. Completion, not launch, starts the next
+ordinary interval. The owner captures the runtime before dispatch and copies
+node origins/statuses; a later core cycle cannot mutate its read inputs.
+
+Client replacement (including fallback), runtime replacement and connection
+retirement cancel/invalidate that owner. Each local acceptance boundary checks
+the owner and registered client under the same publication fence used by those
+transitions. Late task/log/storage/snapshot reads cannot replace live inventory,
+renew accepted log observations, enqueue new recovery evidence, change access
+warnings, or rewrite a replacement's scheduling/cache state. Reusing a connection
+name/UPID does not reuse another binding's immutable log cache. Already accepted
+historical observations remain history; retirement does not delete them.
+
+The existing two-to-five-minute shared task/storage budget, independent bounded
+snapshot budget, read caps, partial/unread preservation, alert policy, identity,
+auth and guest-operation rules are unchanged. No guest operation is introduced.
+`pve_backup_lifecycle_test.go` covers saturated/concurrent admission, late reads
+at all five read boundaries, VM/LXC replacement/runtime/retirement, original
+observation preservation, reused UPIDs, independent connection state and ordinary
+recovery. Existing storage-cancellation and snapshot-budget controls still apply.
+Desktop/narrow consuming proof uses actual produced inventories before and after
+a late superseded read, not invented restore points or native recovery.
+
+
### Backup guest results require an unambiguous task log
The task-log reader requests the prefix from line one and accepts only a whole,
diff --git a/docs/release-control/v6/internal/subsystems/registry.json b/docs/release-control/v6/internal/subsystems/registry.json
index 14ab871da8..c4cf593aa1 100644
--- a/docs/release-control/v6/internal/subsystems/registry.json
+++ b/docs/release-control/v6/internal/subsystems/registry.json
@@ -6426,6 +6426,23 @@
"exact_files": [],
"require_explicit_path_policy_coverage": true,
"path_policies": [
+ {
+ "id": "pve-backup-poll-ownership",
+ "label": "connection-scoped single-flight backup scan and late publication proof",
+ "match_prefixes": [],
+ "match_files": [
+ "internal/monitoring/pve_backup_poll.go",
+ "internal/monitoring/guest_poll_publication.go",
+ "internal/monitoring/monitor_cluster_dedupe.go",
+ "internal/monitoring/monitor_pve.go",
+ "internal/monitoring/monitor.go"
+ ],
+ "allow_same_subsystem_tests": false,
+ "test_prefixes": [],
+ "exact_files": [
+ "internal/monitoring/pve_backup_lifecycle_test.go"
+ ]
+ },
{
"id": "task-log-response",
"label": "bounded ordered task log admission and guest evidence continuity proof",
@@ -6920,6 +6937,7 @@
"internal/monitoring/monitor_backups_dir_storage_test.go",
"internal/monitoring/monitor_full_coverage_test.go",
"internal/monitoring/pbs_protection_observation_test.go",
+ "internal/monitoring/pve_backup_lifecycle_test.go",
"internal/monitoring/pve_protection_observation_test.go",
"internal/monitoring/recovery_ingest_test.go",
"internal/monitoring/vzdump_cached_guest_test.go",
diff --git a/internal/monitoring/guest_poll_publication.go b/internal/monitoring/guest_poll_publication.go
index a4bee87850..925c22f95f 100644
--- a/internal/monitoring/guest_poll_publication.go
+++ b/internal/monitoring/guest_poll_publication.go
@@ -144,6 +144,8 @@ func (m *Monitor) replacePVEClient(instance string, client PVEClientInterface) {
m.mu.Lock()
defer m.mu.Unlock()
m.cancelPVEGuestPollLocked(instance)
+ m.cancelPVEBackupPollLocked(instance)
+ m.discardPVEBackupTaskCacheLocked(instance)
if m.pveClients == nil {
m.pveClients = make(map[string]PVEClientInterface)
}
diff --git a/internal/monitoring/monitor.go b/internal/monitoring/monitor.go
index 176056b6cc..f5c7b7346a 100644
--- a/internal/monitoring/monitor.go
+++ b/internal/monitoring/monitor.go
@@ -1306,7 +1306,7 @@ type Monitor struct {
mockUnifiedViewFixtureAt time.Time // fixture freshness the published view was built from
mockUnifiedViewValid bool
pveClients map[string]PVEClientInterface
- pveGuestPublicationMu sync.RWMutex // Orders guest publication with connection/runtime replacement and retirement.
+ pveGuestPublicationMu sync.RWMutex // Orders guest/backup publication with connection/runtime replacement and retirement.
pveGuestPolls map[string]*pveGuestPoll // Guarded by m.mu; canceled cycles retain ownership until completion unless the binding ends.
pbsClients map[string]*pbs.Client
pmgClients map[string]*pmg.Client
@@ -1368,6 +1368,7 @@ type Monitor struct {
lastAuthAttempt map[string]time.Time // Track last auth attempt time
lastClusterCheck map[string]time.Time // Track last cluster check for standalone nodes
lastPhysicalDiskPoll map[string]time.Time // Track last physical disk poll time per instance
+ pveBackupPolls map[string]*pveBackupPoll // Owned runtime/client-scoped backup scans; guarded by mu
pveReplicationPolls map[string]*pveReplicationPoll // Owned runtime/client-scoped replication reads; guarded by mu
lastPVEBackupPoll map[string]time.Time // Track last PVE backup poll per instance
vzdumpJobTaskCache map[string]vzdumpJobTaskCacheEntry // Cache synthesized per-guest tasks for multi-guest vzdump job runs, keyed by instance|UPID
@@ -1521,6 +1522,9 @@ func (m *Monitor) setRuntimeContext(ctx context.Context, hub *websocket.Hub) {
poll.cancel()
delete(m.pveReplicationPolls, name)
}
+ for name := range m.pveBackupPolls {
+ m.cancelPVEBackupPollLocked(name)
+ }
m.runtimeCtx = ctx
m.wsHub = hub
}
diff --git a/internal/monitoring/monitor_backups.go b/internal/monitoring/monitor_backups.go
index a9eaadd456..89668e8da6 100644
--- a/internal/monitoring/monitor_backups.go
+++ b/internal/monitoring/monitor_backups.go
@@ -137,7 +137,12 @@ func pveBackupAccessWarning(instanceCfg *config.PVEInstance, endpoint string, er
return cause + " Rejected endpoint: " + strconv.Quote(endpoint) + ". " + pveBackupPermissionWarning(instanceCfg)
}
-func (m *Monitor) recordPVEBackupAccessWarning(instanceName, endpoint string, err error) {
+func (m *Monitor) recordPVEBackupAccessWarning(ctx context.Context, instanceName, endpoint string, err error) {
+ unlock, current := m.beginPVEBackupPublication(ctx, instanceName)
+ if !current {
+ return
+ }
+ defer unlock()
warning := pveBackupAccessWarning(m.getInstanceConfig(instanceName), endpoint, err)
m.mu.Lock()
if m.backupPermissionWarnings == nil {
@@ -367,7 +372,7 @@ func (m *Monitor) pollStorageBackupsWithNodes(ctx context.Context, instanceName
if isPVEBackupPermissionError(err) {
hadPermissionError = true
permissionFailureCount++
- m.recordPVEBackupAccessWarning(instanceName, fmt.Sprintf("/nodes/%s/storage", node.Node), err)
+ m.recordPVEBackupAccessWarning(ctx, instanceName, fmt.Sprintf("/nodes/%s/storage", node.Node), err)
}
monErr := errors.NewMonitorError(errors.ErrorTypeAPI, "get_storage_for_backups", instanceName, err).WithNode(node.Node)
log.Warn().Err(monErr).Str("node", node.Node).Msg("failed to get storage for backups - skipping node")
@@ -422,7 +427,7 @@ func (m *Monitor) pollStorageBackupsWithNodes(ctx context.Context, instanceName
if isPVEBackupPermissionError(err) {
hadPermissionError = true
permissionFailureCount++
- m.recordPVEBackupAccessWarning(instanceName, fmt.Sprintf("/nodes/%s/storage/%s/content", node.Node, storage.Storage), err)
+ m.recordPVEBackupAccessWarning(ctx, instanceName, fmt.Sprintf("/nodes/%s/storage/%s/content", node.Node, storage.Storage), err)
} else {
log.Debug().Err(monErr).
Str("node", node.Node).
@@ -598,6 +603,11 @@ func (m *Monitor) pollStorageBackupsWithNodes(ctx context.Context, instanceName
// or renew provider evidence for a payload we did not accept.
return
}
+ unlock, current := m.beginPVEBackupPublication(ctx, instanceName)
+ if !current {
+ return
+ }
+ defer unlock()
for scope := range skippedLocalStorages {
if assignment, known := storageAssignments[scope.storage]; known &&
clusterStorageRestrictedToOtherNodes(assignment, scope.node) {
@@ -1208,6 +1218,9 @@ func (m *Monitor) pollPVEBackupsAndSnapshots(parentCtx context.Context, instance
// pollGuestSnapshots polls snapshots for all VMs and containers
func (m *Monitor) pollGuestSnapshots(ctx context.Context, instanceName string, client PVEClientInterface) {
+ if ctx.Err() != nil {
+ return
+ }
log.Debug().Str("instance", instanceName).Msg("polling guest snapshots")
// The guest inventory comes from the fixture read view in mock mode.
@@ -1507,6 +1520,14 @@ func (m *Monitor) pollGuestSnapshots(ctx context.Context, instanceName string, c
}
}
+ // A child snapshot budget may yield accepted partial results; a canceled
+ // runtime/connection owner may not publish even those results.
+ unlock, current := m.beginPVEBackupPublication(ctx, instanceName)
+ if !current {
+ return
+ }
+ defer unlock()
+
// Update state with guest snapshots for this instance
m.state.UpdateGuestSnapshotsForInstance(instanceName, allSnapshots)
@@ -2603,11 +2624,17 @@ func appendPBSBackupsWithinLimit(backups []models.PBSBackup, additions []models.
// pollBackupTasks polls backup tasks from a PVE instance
func (m *Monitor) pollBackupTasks(ctx context.Context, instanceName string, client PVEClientInterface) {
+ if ctx.Err() != nil {
+ return
+ }
log.Debug().Str("instance", instanceName).Msg("polling backup tasks")
history, collection, err := m.readBackupTaskHistory(ctx, instanceName, client)
if err != nil {
- m.persistBackupTaskHistory(ctx, nil, collection)
+ if unlock, current := m.beginPVEBackupPublication(ctx, instanceName); current {
+ m.persistBackupTaskHistory(ctx, nil, collection)
+ unlock()
+ }
monErr := errors.WrapAPIError("get_backup_tasks", instanceName, err, 0)
log.Error().Err(monErr).Str("instance", instanceName).Msg("failed to get backup tasks")
return
@@ -2673,6 +2700,12 @@ func (m *Monitor) pollBackupTasks(ctx context.Context, instanceName string, clie
}
}
+ unlock, current := m.beginPVEBackupPublication(ctx, instanceName)
+ if !current {
+ return
+ }
+ defer unlock()
+
// Update state with new backup tasks for this instance
m.state.UpdateBackupTasksForInstance(instanceName, m.retainUnreadBackupTasks(instanceName, history.Nodes, liveTasks))
@@ -2717,8 +2750,14 @@ const maxVzdumpJobLogFetchesPerPoll = 25
// fetched only once. Running jobs are re-parsed every cycle to pick up guests
// as the job progresses.
func (m *Monitor) synthesizeVzdumpJobGuestTasks(ctx context.Context, instanceName string, client PVEClientInterface, jobTasks []models.BackupTask, jobUPIDs map[string]string) []models.BackupTask {
+ if ctx.Err() != nil {
+ return nil
+ }
if len(jobTasks) == 0 {
- m.pruneVzdumpJobTaskCache(instanceName, nil)
+ if unlock, current := m.beginPVEBackupPublication(ctx, instanceName); current {
+ m.pruneVzdumpJobTaskCache(instanceName, nil)
+ unlock()
+ }
return nil
}
@@ -2763,6 +2802,10 @@ func (m *Monitor) synthesizeVzdumpJobGuestTasks(ctx context.Context, instanceNam
}
continue
}
+ unlock, current := m.beginPVEBackupPublication(ctx, instanceName)
+ if !current {
+ return nil
+ }
fetches++
delete(fetchJobs, key) // duplicate provider rows cannot spend the same turn twice
// A failed or incomplete read still spent the allowance. Keep attempt
@@ -2779,6 +2822,7 @@ func (m *Monitor) synthesizeVzdumpJobGuestTasks(ctx context.Context, instanceNam
}
m.vzdumpJobLogHistoryFirst[instanceName] = jobTask.EndTime.IsZero()
m.mu.Unlock()
+ unlock()
logLines, err := client.GetTaskLog(ctx, jobTask.Node, upid)
if err != nil {
@@ -2792,6 +2836,10 @@ func (m *Monitor) synthesizeVzdumpJobGuestTasks(ctx context.Context, instanceNam
continue
}
+ unlock, current = m.beginPVEBackupPublication(ctx, instanceName)
+ if !current {
+ return nil
+ }
guestTasks := refreshObservedAt(parseVzdumpJobLog(jobTask, upid, logLines), now)
historyComplete := backupGuestLogComplete(jobTask, logLines, guestTasks)
m.mu.Lock()
@@ -2806,9 +2854,13 @@ func (m *Monitor) synthesizeVzdumpJobGuestTasks(ctx context.Context, instanceNam
}
m.mu.Unlock()
byJob[index] = guestTasks
+ unlock()
}
- m.pruneVzdumpJobTaskCache(instanceName, seen)
+ if unlock, current := m.beginPVEBackupPublication(ctx, instanceName); current {
+ m.pruneVzdumpJobTaskCache(instanceName, seen)
+ unlock()
+ }
var synthesized []models.BackupTask
for _, tasks := range byJob {
synthesized = append(synthesized, tasks...)
@@ -3086,16 +3138,24 @@ func (m *Monitor) pollPVEBackupsAsync(
return
}
- // Set initial timestamp before starting goroutine (prevents concurrent starts).
- // This optional work owns a runtime-scoped timeout below; the completed core
- // PVE generation must not be reclassified as unreachable merely because its
- // per-cycle context expired before background scheduling.
- m.mu.Lock()
- m.lastPVEBackupPoll[instanceName] = newLast
- m.mu.Unlock()
+ // Atomically claim this connection before dispatch. A timestamp is cadence,
+ // not an in-flight fence: a scan can outlast the next eligible core cycle.
+ owner := m.beginPVEBackupPoll(instanceName, client, lastPoll, newLast)
+ if owner == nil {
+ return
+ }
+ // Core-cycle cancellation must not cancel optional background inventory.
+ ctx := context.WithValue(owner.ctx, pveBackupPollContextKey{}, owner)
+ // The caller may reuse its inventory slices/maps on the next core cycle.
+ nodes = append([]proxmox.Node(nil), nodes...)
+ statuses := make(map[string]string, len(nodeEffectiveStatus))
+ for node, status := range nodeEffectiveStatus {
+ statuses[node] = status
+ }
// Run backup polling in a separate goroutine to avoid blocking real-time stats
go func(startTime time.Time, inst string, pveClient PVEClientInterface) {
+ defer m.finishPVEBackupPoll(inst, owner)
defer recoverFromPanic(fmt.Sprintf("pollPVEBackups-%s", inst))
timeout := m.calculateBackupOperationTimeout(inst)
log.Info().
@@ -3103,14 +3163,7 @@ func (m *Monitor) pollPVEBackupsAsync(
Dur("timeout", timeout).
Msg("Starting background backup/snapshot polling")
- // The per-cycle ctx is canceled as soon as the main polling loop finishes,
- // so derive the backup poll context from the long-lived runtime context instead.
- parentCtx := m.getRuntimeContext()
- if parentCtx == nil {
- parentCtx = context.Background()
- }
-
- m.pollPVEBackupsAndSnapshots(parentCtx, inst, pveClient, nodes, nodeEffectiveStatus, timeout)
+ m.pollPVEBackupsAndSnapshots(ctx, inst, pveClient, nodes, statuses, timeout)
duration := time.Since(startTime)
log.Info().
@@ -3118,10 +3171,6 @@ func (m *Monitor) pollPVEBackupsAsync(
Dur("duration", duration).
Msg("Completed background backup/snapshot polling")
- // Update timestamp after completion for accurate interval scheduling
- m.mu.Lock()
- m.lastPVEBackupPoll[inst] = time.Now()
- m.mu.Unlock()
}(now, instanceName, client)
}
diff --git a/internal/monitoring/monitor_backups_dir_storage_test.go b/internal/monitoring/monitor_backups_dir_storage_test.go
index 2cb50232ca..1890ab2c05 100644
--- a/internal/monitoring/monitor_backups_dir_storage_test.go
+++ b/internal/monitoring/monitor_backups_dir_storage_test.go
@@ -231,7 +231,7 @@ func TestPollStorageBackups_Issue1639HarvestsPBSGuestConfirmations(t *testing.T)
}
}
-func TestPollPVEBackupsAsyncUsesRuntimeContextIndependentOfCycle(t *testing.T) {
+func TestPollPVEBackupsAsyncRejectsStoppedRuntime(t *testing.T) {
runtimeCtx, cancelRuntime := context.WithCancel(context.Background())
cancelRuntime()
m := &Monitor{
@@ -243,18 +243,20 @@ func TestPollPVEBackupsAsyncUsesRuntimeContextIndependentOfCycle(t *testing.T) {
BackupPollingCycles: 1,
},
}
+ client := &mockPVEClientExtra{}
+ m.pveClients = map[string]PVEClientInterface{"cluster-a": client}
m.pollPVEBackupsAsync(
"cluster-a",
&config.PVEInstance{MonitorBackups: true},
- &mockPVEClientExtra{},
+ client,
nil,
map[string]string{},
)
m.mu.RLock()
scheduledAt := m.lastPVEBackupPoll["cluster-a"]
m.mu.RUnlock()
- if scheduledAt.IsZero() {
- t.Fatal("backup polling was not scheduled on the monitor runtime context")
+ if !scheduledAt.IsZero() {
+ t.Fatal("stopped runtime scheduled a new backup poll")
}
}
diff --git a/internal/monitoring/monitor_cluster_dedupe.go b/internal/monitoring/monitor_cluster_dedupe.go
index 77d70d947b..450550fe9e 100644
--- a/internal/monitoring/monitor_cluster_dedupe.go
+++ b/internal/monitoring/monitor_cluster_dedupe.go
@@ -91,6 +91,8 @@ func (m *Monitor) retirePVEInstanceRuntime(instanceName string) {
m.mu.Lock()
m.cancelPVEGuestPollLocked(instanceName)
+ m.cancelPVEBackupPollLocked(instanceName)
+ m.discardPVEBackupTaskCacheLocked(instanceName)
if poll := m.pveReplicationPolls[instanceName]; poll != nil {
poll.cancel()
delete(m.pveReplicationPolls, instanceName)
diff --git a/internal/monitoring/monitor_pve.go b/internal/monitoring/monitor_pve.go
index 5815241c1e..e4647f3a84 100644
--- a/internal/monitoring/monitor_pve.go
+++ b/internal/monitoring/monitor_pve.go
@@ -445,6 +445,8 @@ func (m *Monitor) retryPVEPortFallback(ctx context.Context, instanceName string,
m.mu.Lock()
instanceCfg.Host = persistHost
m.cancelPVEGuestPollLocked(instanceName)
+ m.cancelPVEBackupPollLocked(instanceName)
+ m.discardPVEBackupTaskCacheLocked(instanceName)
m.pveClients[instanceName] = fallbackClient
// Update in-memory config so subsequent polls build clients against the working port.
diff --git a/internal/monitoring/pve_backup_lifecycle_test.go b/internal/monitoring/pve_backup_lifecycle_test.go
new file mode 100644
index 0000000000..5ede7724ab
--- /dev/null
+++ b/internal/monitoring/pve_backup_lifecycle_test.go
@@ -0,0 +1,294 @@
+package monitoring
+
+import (
+ "context"
+ "encoding/json"
+ "os"
+ "sync"
+ "sync/atomic"
+ "testing"
+ "testing/synctest"
+ "time"
+
+ "github.com/rcourtman/pulse-go-rewrite/internal/config"
+ "github.com/rcourtman/pulse-go-rewrite/internal/models"
+ "github.com/rcourtman/pulse-go-rewrite/pkg/proxmox"
+ "github.com/stretchr/testify/require"
+)
+
+// No guest-agent, task operation, external connection or installed target.
+// The same behavioural tests also compile against the exact original source.
+type backupLifecycleClient struct {
+ mockPVEClientExtra
+ label string
+ read func(context.Context, string)
+ calls atomic.Int32
+}
+
+func (c *backupLifecycleClient) before(ctx context.Context, phase string) {
+ c.calls.Add(1)
+ if c.read != nil {
+ c.read(ctx, phase)
+ }
+}
+func (c *backupLifecycleClient) GetBackupTasks(ctx context.Context) ([]proxmox.Task, error) {
+ c.before(ctx, "tasks")
+ return []proxmox.Task{{UPID: testJobUPID, Node: "node-a", Type: "vzdump", StartTime: time.Now().Add(-time.Hour).Unix(), EndTime: time.Now().Add(-time.Minute).Unix(), Status: "job errors"}}, nil
+}
+func (c *backupLifecycleClient) GetTaskLog(ctx context.Context, _, _ string) ([]proxmox.TaskLogLine, error) {
+ c.before(ctx, "log")
+ return []proxmox.TaskLogLine{{LineNumber: 1, Text: "INFO: Starting Backup of VM 101 (qemu)"}, {LineNumber: 2, Text: "ERROR: Backup of VM 101 failed - " + c.label + " result"}, {LineNumber: 3, Text: "TASK ERROR: job errors"}}, nil
+}
+func (c *backupLifecycleClient) GetStorage(ctx context.Context, _ string) ([]proxmox.Storage, error) {
+ c.before(ctx, "storage")
+ return []proxmox.Storage{{Storage: "archive", Type: "dir", Content: "backup", Active: 1, Enabled: 1}}, nil
+}
+func (c *backupLifecycleClient) GetStorageContent(ctx context.Context, _, _ string) ([]proxmox.StorageContent, error) {
+ c.before(ctx, "content")
+ return []proxmox.StorageContent{{Volid: "archive:backup/vzdump-qemu-101-" + c.label + ".vma.zst", VMID: 101, CTime: time.Now().Add(-2 * time.Hour).Unix(), Size: 1024, Content: "backup"}}, nil
+}
+func (c *backupLifecycleClient) GetVMSnapshots(ctx context.Context, _ string, _ int) ([]proxmox.Snapshot, error) {
+ c.before(ctx, "snapshots")
+ return []proxmox.Snapshot{{Name: c.label, SnapTime: time.Now().Add(-time.Hour).Unix()}}, nil
+}
+func (c *backupLifecycleClient) GetContainerSnapshots(ctx context.Context, node string, id int) ([]proxmox.Snapshot, error) {
+ return c.GetVMSnapshots(ctx, node, id)
+}
+func newBackupLifecycleMonitor(c PVEClientInterface, kind string) *Monitor {
+ m := &Monitor{state: models.NewState(), pveClients: map[string]PVEClientInterface{"site-a": c}, config: &config.Config{EnableBackupPolling: true, BackupPollingCycles: 1}, lastPVEBackupPoll: map[string]time.Time{}, runtimeCtx: context.Background()}
+ if kind == "qemu" {
+ m.state.UpdateVMsForInstance("site-a", []models.VM{{ID: "site-a-node-a-101", VMID: 101, Node: "node-a", Instance: "site-a", Name: "workload", Status: "running"}})
+ } else {
+ m.state.UpdateContainersForInstance("site-a", []models.Container{{ID: "site-a-node-a-101", VMID: 101, Node: "node-a", Instance: "site-a", Name: "workload", Status: "running"}})
+ }
+ m.state.UpdateStorageBackupsForInstance("site-b", []models.StorageBackup{{ID: "peer-file", Instance: "site-b", VMID: 101, Storage: "archive", Time: time.Now().Add(-time.Hour)}})
+ m.state.UpdateGuestSnapshotsForInstance("site-b", []models.GuestSnapshot{{ID: "peer-snap", Instance: "site-b", VMID: 101, Name: "independent", Time: time.Now().Add(-time.Hour)}})
+ return m
+}
+func scheduleBackupLifecycle(m *Monitor, c PVEClientInterface) {
+ m.pollPVEBackupsAsync("site-a", &config.PVEInstance{MonitorBackups: true}, c, []proxmox.Node{{Node: "node-a", Status: "online"}}, map[string]string{"node-a": "online"})
+}
+
+func TestPVEBackupLifecycleSingleFlight(t *testing.T) {
+ for _, interval := range []time.Duration{0, time.Minute} {
+ t.Run(interval.String(), func(t *testing.T) {
+ synctest.Test(t, func(t *testing.T) {
+ release := make(chan struct{})
+ var tasks atomic.Int32
+ c := &backupLifecycleClient{label: "accepted", read: func(_ context.Context, phase string) {
+ if phase == "tasks" {
+ tasks.Add(1)
+ <-release
+ }
+ }}
+ m := newBackupLifecycleMonitor(c, "qemu")
+ m.config.BackupPollingInterval = interval
+ scheduleBackupLifecycle(m, c)
+ synctest.Wait()
+ // A slow scan outlasts cadence, but it still owns the connection.
+ time.Sleep(90 * time.Second)
+ for i := 0; i < 8; i++ {
+ scheduleBackupLifecycle(m, c)
+ }
+ synctest.Wait()
+ got := tasks.Load()
+ close(release)
+ synctest.Wait()
+ require.Equal(t, int32(1), got, "cadence must not stack scans")
+ completed := m.lastPVEBackupPoll["site-a"]
+ require.Equal(t, time.Now(), completed, "cadence starts at accepted completion")
+ time.Sleep(interval + time.Second)
+ scheduleBackupLifecycle(m, c)
+ synctest.Wait()
+ require.Equal(t, int32(2), tasks.Load(), "ordinary next eligible cycle recovers")
+ })
+ })
+ }
+}
+
+func TestPVEBackupLifecycleLateReadsCannotReplaceCurrentEvidence(t *testing.T) {
+ for _, kind := range []string{"qemu", "lxc"} {
+ for _, phase := range []string{"tasks", "log", "storage", "content", "snapshots"} {
+ for _, transition := range []string{"client", "runtime", "retire"} {
+ t.Run(kind+"/"+phase+"/"+transition, func(t *testing.T) {
+ synctest.Test(t, func(t *testing.T) {
+ release := make(chan struct{})
+ released := false
+ defer func() {
+ if !released {
+ close(release)
+ }
+ }()
+ var reached atomic.Bool
+ var oldCtx context.Context
+ old := &backupLifecycleClient{label: "old", read: func(ctx context.Context, p string) {
+ if p == phase {
+ oldCtx = ctx
+ reached.Store(true)
+ <-release
+ }
+ }}
+ m := newBackupLifecycleMonitor(old, kind)
+ scheduleBackupLifecycle(m, old)
+ synctest.Wait()
+ require.True(t, reached.Load())
+ fresh := &backupLifecycleClient{label: "current"}
+ switch transition {
+ case "retire":
+ m.retirePVEInstanceRuntime("site-a")
+ case "runtime":
+ m.setRuntimeContext(context.Background(), nil) // Deliberately don't cancel the old parent.
+ }
+ m.replacePVEClient("site-a", fresh)
+ if transition == "retire" {
+ // Fresh configuration rebuilds its guest inventory without a live guest operation.
+ source := newBackupLifecycleMonitor(fresh, kind).state.GetSnapshot()
+ m.state.UpdateVMsForInstance("site-a", source.VMs)
+ m.state.UpdateContainersForInstance("site-a", source.Containers)
+ }
+ scheduleBackupLifecycle(m, fresh)
+ synctest.Wait()
+ before := m.state.GetSnapshot().PVEBackups
+ require.NotZero(t, fresh.calls.Load(), "replacement must not wait for the late client")
+ currentStamp := m.lastPVEBackupPoll["site-a"]
+ cache := m.vzdumpJobTaskCache["site-a|"+testJobUPID]
+ close(release)
+ released = true
+ synctest.Wait()
+ require.NotNil(t, oldCtx.Err(), "binding transition must cancel the old read")
+ require.Equal(t, before, m.state.GetSnapshot().PVEBackups, "late read changed accepted inventories")
+ require.Equal(t, currentStamp, m.lastPVEBackupPoll["site-a"], "old finish rewrote replacement cadence")
+ require.Equal(t, cache, m.vzdumpJobTaskCache["site-a|"+testJobUPID], "old result poisoned the immutable cache")
+ require.Len(t, before.StorageBackups, 2)
+ require.Len(t, before.GuestSnapshots, 2)
+ found := false
+ for _, task := range before.BackupTasks {
+ if task.VMID == 101 {
+ require.Equal(t, "current result", task.Error)
+ found = true
+ }
+ }
+ require.True(t, found, "new client must read its own guest log even for a reused UPID")
+ if os.Getenv("PULSE_BACKUP_LIFECYCLE_FRAMES") == "1" && kind == "qemu" && phase == "tasks" && transition == "client" {
+ data, err := json.Marshal(map[string]any{"accepted": before, "after_late": m.state.GetSnapshot().PVEBackups, "vms": m.state.GetSnapshot().VMs, "containers": m.state.GetSnapshot().Containers})
+ require.NoError(t, err)
+ t.Log("BACKUP_LIFECYCLE_FRAME=" + string(data))
+ }
+ })
+ })
+ }
+ }
+ }
+}
+
+func TestPVEBackupLifecycleRetirementDoesNotResurrectState(t *testing.T) {
+ synctest.Test(t, func(t *testing.T) {
+ release := make(chan struct{})
+ old := &backupLifecycleClient{label: "retired", read: func(_ context.Context, p string) {
+ if p == "tasks" {
+ <-release
+ }
+ }}
+ m := newBackupLifecycleMonitor(old, "qemu")
+ scheduleBackupLifecycle(m, old)
+ synctest.Wait()
+ m.retirePVEInstanceRuntime("site-a")
+ before := m.state.GetSnapshot().PVEBackups
+ close(release)
+ synctest.Wait()
+ require.Equal(t, before, m.state.GetSnapshot().PVEBackups)
+ _, exists := m.lastPVEBackupPoll["site-a"]
+ require.False(t, exists, "retired completion recreated scheduling state")
+ require.Empty(t, m.vzdumpJobTaskCache)
+ })
+}
+
+func TestPVEBackupLifecycleConcurrentAdmission(t *testing.T) {
+ synctest.Test(t, func(t *testing.T) {
+ release := make(chan struct{})
+ var tasks atomic.Int32
+ c := &backupLifecycleClient{label: "accepted", read: func(_ context.Context, p string) {
+ if p == "tasks" {
+ tasks.Add(1)
+ <-release
+ }
+ }}
+ m := newBackupLifecycleMonitor(c, "qemu")
+ var callers sync.WaitGroup
+ for i := 0; i < 32; i++ {
+ callers.Add(1)
+ go func() { defer callers.Done(); scheduleBackupLifecycle(m, c) }()
+ }
+ callers.Wait()
+ synctest.Wait()
+ count := tasks.Load()
+ close(release)
+ synctest.Wait()
+ require.Equal(t, int32(1), count)
+ })
+}
+
+func TestPVEBackupLifecycleCopiesSchedulingInventory(t *testing.T) {
+ synctest.Test(t, func(t *testing.T) {
+ release := make(chan struct{})
+ c := &backupLifecycleClient{label: "accepted", read: func(_ context.Context, p string) {
+ if p == "tasks" {
+ <-release
+ }
+ }}
+ m := newBackupLifecycleMonitor(c, "qemu")
+ nodes := []proxmox.Node{{Node: "node-a", Status: "online"}}
+ statuses := map[string]string{"node-a": "online"}
+ m.pollPVEBackupsAsync("site-a", &config.PVEInstance{MonitorBackups: true}, c, nodes, statuses)
+ synctest.Wait()
+ nodes[0].Node = "reused-core-buffer"
+ statuses["node-a"] = "offline"
+ close(release)
+ synctest.Wait()
+ found := false
+ for _, row := range m.state.GetSnapshot().PVEBackups.StorageBackups {
+ if row.Instance == "site-a" {
+ found = true
+ require.Equal(t, "node-a", row.StorageNode)
+ }
+ }
+ require.True(t, found, "next core cycle mutated the owned storage origin")
+ })
+}
+
+func TestPVEBackupLifecycleCanceledCollectorsKeepEvidence(t *testing.T) {
+ for _, phase := range []string{"tasks", "log", "snapshots"} {
+ for _, kind := range []string{"qemu", "lxc"} {
+ t.Run(phase+"/"+kind, func(t *testing.T) {
+ c := &backupLifecycleClient{label: "accepted"}
+ m := newBackupLifecycleMonitor(c, kind)
+ m.pollBackupTasks(context.Background(), "site-a", c)
+ m.pollGuestSnapshots(context.Background(), "site-a", c)
+ before := m.state.GetSnapshot().PVEBackups
+ ctx, cancel := context.WithCancel(context.Background())
+ defer cancel()
+ c.label = "late"
+ c.read = func(_ context.Context, p string) {
+ if p == phase {
+ cancel()
+ }
+ }
+ // Non-finalise the prior cache to force another real log read.
+ key := "site-a|" + testJobUPID
+ entry := m.vzdumpJobTaskCache[key]
+ entry.final = false
+ m.vzdumpJobTaskCache[key] = entry
+ cache := m.vzdumpJobTaskCache[key]
+ if phase == "snapshots" {
+ m.pollGuestSnapshots(ctx, "site-a", c)
+ } else {
+ m.pollBackupTasks(ctx, "site-a", c)
+ }
+ require.Equal(t, before, m.state.GetSnapshot().PVEBackups)
+ after := m.vzdumpJobTaskCache[key]
+ require.Equal(t, cache.tasks, after.tasks, "cancellation replaced accepted log evidence")
+ require.Equal(t, cache.final, after.final)
+ })
+ }
+ }
+}
diff --git a/internal/monitoring/pve_backup_poll.go b/internal/monitoring/pve_backup_poll.go
new file mode 100644
index 0000000000..434a4b3337
--- /dev/null
+++ b/internal/monitoring/pve_backup_poll.go
@@ -0,0 +1,109 @@
+package monitoring
+
+import (
+ "context"
+ "strings"
+ "time"
+)
+
+// One owner covers tasks, storage and snapshots. Their existing independent
+// budgets remain children of this runtime-scoped context, not a core cycle.
+// Cancellation is not completion: ordinary cycles cannot stack another scan
+// behind a client that has not returned yet.
+type pveBackupPoll struct {
+ ctx context.Context
+ cancel context.CancelFunc
+ client PVEClientInterface
+ done chan struct{}
+}
+
+type pveBackupPollContextKey struct{}
+
+func (m *Monitor) beginPVEBackupPoll(instance string, client PVEClientInterface, last, next time.Time) *pveBackupPoll {
+ m.pveGuestPublicationMu.RLock()
+ defer m.pveGuestPublicationMu.RUnlock()
+ m.mu.Lock()
+ defer m.mu.Unlock()
+ if client == nil || m.pveClients[instance] != client || !m.lastPVEBackupPoll[instance].Equal(last) {
+ return nil
+ }
+ parent := m.runtimeCtx
+ if parent == nil {
+ parent = context.Background()
+ }
+ if parent.Err() != nil {
+ return nil
+ }
+ if previous := m.pveBackupPolls[instance]; previous != nil {
+ return nil
+ }
+ ctx, cancel := context.WithCancel(parent)
+ owner := &pveBackupPoll{ctx: ctx, cancel: cancel, client: client, done: make(chan struct{})}
+ if m.pveBackupPolls == nil {
+ m.pveBackupPolls = make(map[string]*pveBackupPoll)
+ }
+ if m.lastPVEBackupPoll == nil {
+ m.lastPVEBackupPoll = make(map[string]time.Time)
+ }
+ m.pveBackupPolls[instance] = owner
+ m.lastPVEBackupPoll[instance] = next
+ return owner
+}
+
+func (m *Monitor) finishPVEBackupPoll(instance string, owner *pveBackupPoll) {
+ m.mu.Lock()
+ if m.pveBackupPolls[instance] == owner {
+ delete(m.pveBackupPolls, instance)
+ if owner.ctx.Err() == nil && m.pveClients[instance] == owner.client {
+ m.lastPVEBackupPoll[instance] = time.Now()
+ }
+ }
+ m.mu.Unlock()
+ owner.cancel()
+ close(owner.done)
+}
+
+// Called with m.mu and the connection-publication writer held. A replacement
+// may start immediately; the old owner cannot publish or release the new claim.
+func (m *Monitor) cancelPVEBackupPollLocked(instance string) {
+ if owner := m.pveBackupPolls[instance]; owner != nil {
+ owner.cancel()
+ delete(m.pveBackupPolls, instance)
+ delete(m.lastPVEBackupPoll, instance)
+ }
+}
+
+// Hold the connection publication reader across local acceptance, never across
+// a provider read. Retirement/replacement uses the writer, so checking the
+// owner and publishing its effects is atomic with respect to either transition.
+// Already accepted/enqueued historical evidence remains valid; late reads do not.
+func (m *Monitor) beginPVEBackupPublication(ctx context.Context, instance string) (func(), bool) {
+ if ctx.Err() != nil {
+ return nil, false
+ }
+ owner, _ := ctx.Value(pveBackupPollContextKey{}).(*pveBackupPoll)
+ if owner == nil {
+ return func() {}, true // focused collectors retain their normal behaviour
+ }
+ m.pveGuestPublicationMu.RLock()
+ m.mu.RLock()
+ current := ctx.Err() == nil && owner.ctx.Err() == nil && m.pveBackupPolls[instance] == owner && m.pveClients[instance] == owner.client
+ m.mu.RUnlock()
+ if !current {
+ m.pveGuestPublicationMu.RUnlock()
+ return nil, false
+ }
+ return m.pveGuestPublicationMu.RUnlock, true
+}
+
+// Connection identity, not just a transport pointer, bounds cached task logs.
+// Keep accepted live state until an ordinary new read replaces it, but never
+// let a reused name/UPID borrow another connection's immutable guest results.
+func (m *Monitor) discardPVEBackupTaskCacheLocked(instance string) {
+ delete(m.vzdumpJobLogHistoryFirst, instance)
+ for key := range m.vzdumpJobTaskCache {
+ if strings.HasPrefix(key, instance+"|") {
+ delete(m.vzdumpJobTaskCache, key)
+ }
+ }
+}