Real-time monitoring dashboard for Proxmox VE, PBS, Docker, Kubernetes, TrueNAS and vSphere. Self-hosted, with alerts for failed backups, full disks, unhealthy containers and offline nodes.
Go aialertsdashboarddockerdocker-monitoringgohomelabinfrastructure-monitoringkubernetesmonitoringmspproxmoxproxmox-backup-serverproxmox-mail-gatewayproxmox-veself-hostedtruenastypescriptvspherewebhooks
Preserve accepted task progress when a later log exhausts its budget
pulse-triage[bot]036c8e7f5404
diff --git a/docs/release-control/v6/internal/subsystems/monitoring.md b/docs/release-control/v6/internal/subsystems/monitoring.mdindex 045f8fdffd..894b7d1dfc 100644--- a/docs/release-control/v6/internal/subsystems/monitoring.md+++ b/docs/release-control/v6/internal/subsystems/monitoring.md@@ -37,7 +37,9 @@ historical observations remain history; retirement does not delete them.The existing two-to-five-minute shared task/storage budget, independent boundedsnapshot budget, read caps, partial/unread preservation, alert policy, identity,-auth and guest-operation rules are unchanged. No guest operation is introduced.+auth and guest-operation rules are unchanged. A task listing or log accepted+before a later log exhausts the shared budget remains usable while its owner+is current; a read returning after cancellation supplies no new guest facts. No guest operation is introduced.`pve_backup_lifecycle_test.go` covers saturated/concurrent admission, late readsat all five read boundaries, VM/LXC replacement/runtime/retirement, originalobservation preservation, reused UPIDs, independent connection state and ordinarydiff --git a/internal/monitoring/monitor_backups.go b/internal/monitoring/monitor_backups.goindex 89668e8da6..e707ae1bc3 100644--- a/internal/monitoring/monitor_backups.go+++ b/internal/monitoring/monitor_backups.go@@ -2639,6 +2639,11 @@ func (m *Monitor) pollBackupTasks(ctx context.Context, instanceName string, clielog.Error().Err(monErr).Str("instance", instanceName).Msg("failed to get backup tasks")return}+ // A read returning after cancellation is not accepted history. Once the+ // listing is accepted, later log-budget exhaustion must not hide it.+ if ctx.Err() != nil {+ return+ }var backupTasks []models.BackupTaskvar liveTasks []models.BackupTask@@ -2700,7 +2705,10 @@ func (m *Monitor) pollBackupTasks(ctx context.Context, instanceName string, clie}}- unlock, current := m.beginPVEBackupPublication(ctx, instanceName)+ // Preserve the accepted listing and earlier log reads if a later log used+ // the shared budget. WithoutCancel drops only that child budget: the owner+ // value still fences explicit connection/runtime withdrawal below.+ unlock, current := m.beginPVEBackupPublication(context.WithoutCancel(ctx), instanceName)if !current {return}@@ -2750,9 +2758,6 @@ const maxVzdumpJobLogFetchesPerPoll = 25// fetched only once. Running jobs are re-parsed every cycle to pick up guests// as the job progresses.func (m *Monitor) synthesizeVzdumpJobGuestTasks(ctx context.Context, instanceName string, client PVEClientInterface, jobTasks []models.BackupTask, jobUPIDs map[string]string) []models.BackupTask {- if ctx.Err() != nil {- return nil- }if len(jobTasks) == 0 {if unlock, current := m.beginPVEBackupPublication(ctx, instanceName); current {m.pruneVzdumpJobTaskCache(instanceName, nil)@@ -2804,7 +2809,10 @@ func (m *Monitor) synthesizeVzdumpJobGuestTasks(ctx context.Context, instanceNam}unlock, current := m.beginPVEBackupPublication(ctx, instanceName)if !current {- return nil+ if cached {+ byJob[index] = cachedVzdumpGuestTasks(entry.tasks, jobTask)+ }+ continue}fetches++delete(fetchJobs, key) // duplicate provider rows cannot spend the same turn twice@@ -2838,7 +2846,10 @@ func (m *Monitor) synthesizeVzdumpJobGuestTasks(ctx context.Context, instanceNamunlock, current = m.beginPVEBackupPublication(ctx, instanceName)if !current {- return nil+ if cached {+ byJob[index] = cachedVzdumpGuestTasks(entry.tasks, jobTask)+ }+ continue}guestTasks := refreshObservedAt(parseVzdumpJobLog(jobTask, upid, logLines), now)historyComplete := backupGuestLogComplete(jobTask, logLines, guestTasks)diff --git a/internal/monitoring/pve_backup_lifecycle_test.go b/internal/monitoring/pve_backup_lifecycle_test.goindex 9825be89a4..26d0225de2 100644--- a/internal/monitoring/pve_backup_lifecycle_test.go+++ b/internal/monitoring/pve_backup_lifecycle_test.go@@ -284,7 +284,28 @@ func TestPVEBackupLifecycleCanceledCollectorsKeepEvidence(t *testing.T) {} else {m.pollBackupTasks(ctx, "site-a", c)}- require.Equal(t, before, m.state.GetSnapshot().PVEBackups)+ observed := m.state.GetSnapshot().PVEBackups+ if phase == "log" {+ // The parent listing was accepted before this log exhausted+ // the budget. Its observation may advance; cached guest facts+ // may not. A canceled read supplied no new guest verdict.+ var beforeGuests, afterGuests []models.BackupTask+ for _, row := range before.BackupTasks {+ if row.VMID > 0 {+ beforeGuests = append(beforeGuests, row)+ }+ }+ for _, row := range observed.BackupTasks {+ if row.VMID > 0 {+ afterGuests = append(afterGuests, row)+ }+ }+ require.Equal(t, beforeGuests, afterGuests)+ require.Equal(t, before.StorageBackups, observed.StorageBackups)+ require.Equal(t, before.GuestSnapshots, observed.GuestSnapshots)+ } else {+ require.Equal(t, before, observed)+ }after := m.vzdumpJobTaskCache[key]require.Equal(t, cache.tasks, after.tasks, "cancellation replaced accepted log evidence")require.Equal(t, cache.final, after.final)