Skip to content

armgit / Pulse

Public Imported from GitHub
0

Real-time monitoring dashboard for Proxmox VE, PBS, Docker, Kubernetes, TrueNAS and vSphere. Self-hosted, with alerts for failed backups, full disks, unhealthy containers and offline nodes.

Preserve accepted task progress when a later log exhausts its budget

pulse-triage[bot]036c8e7f5404
diff --git a/docs/release-control/v6/internal/subsystems/monitoring.md b/docs/release-control/v6/internal/subsystems/monitoring.md
index 045f8fdffd..894b7d1dfc 100644
--- a/docs/release-control/v6/internal/subsystems/monitoring.md
+++ b/docs/release-control/v6/internal/subsystems/monitoring.md
@@ -37,7 +37,9 @@ historical observations remain history; retirement does not delete them.
The existing two-to-five-minute shared task/storage budget, independent bounded
snapshot budget, read caps, partial/unread preservation, alert policy, identity,
-auth and guest-operation rules are unchanged. No guest operation is introduced.
+auth and guest-operation rules are unchanged. A task listing or log accepted
+before a later log exhausts the shared budget remains usable while its owner
+is current; a read returning after cancellation supplies no new guest facts. No guest operation is introduced.
`pve_backup_lifecycle_test.go` covers saturated/concurrent admission, late reads
at all five read boundaries, VM/LXC replacement/runtime/retirement, original
observation preservation, reused UPIDs, independent connection state and ordinary
diff --git a/internal/monitoring/monitor_backups.go b/internal/monitoring/monitor_backups.go
index 89668e8da6..e707ae1bc3 100644
--- a/internal/monitoring/monitor_backups.go
+++ b/internal/monitoring/monitor_backups.go
@@ -2639,6 +2639,11 @@ func (m *Monitor) pollBackupTasks(ctx context.Context, instanceName string, clie
log.Error().Err(monErr).Str("instance", instanceName).Msg("failed to get backup tasks")
return
}
+ // A read returning after cancellation is not accepted history. Once the
+ // listing is accepted, later log-budget exhaustion must not hide it.
+ if ctx.Err() != nil {
+ return
+ }
var backupTasks []models.BackupTask
var liveTasks []models.BackupTask
@@ -2700,7 +2705,10 @@ func (m *Monitor) pollBackupTasks(ctx context.Context, instanceName string, clie
}
}
- unlock, current := m.beginPVEBackupPublication(ctx, instanceName)
+ // Preserve the accepted listing and earlier log reads if a later log used
+ // the shared budget. WithoutCancel drops only that child budget: the owner
+ // value still fences explicit connection/runtime withdrawal below.
+ unlock, current := m.beginPVEBackupPublication(context.WithoutCancel(ctx), instanceName)
if !current {
return
}
@@ -2750,9 +2758,6 @@ const maxVzdumpJobLogFetchesPerPoll = 25
// fetched only once. Running jobs are re-parsed every cycle to pick up guests
// as the job progresses.
func (m *Monitor) synthesizeVzdumpJobGuestTasks(ctx context.Context, instanceName string, client PVEClientInterface, jobTasks []models.BackupTask, jobUPIDs map[string]string) []models.BackupTask {
- if ctx.Err() != nil {
- return nil
- }
if len(jobTasks) == 0 {
if unlock, current := m.beginPVEBackupPublication(ctx, instanceName); current {
m.pruneVzdumpJobTaskCache(instanceName, nil)
@@ -2804,7 +2809,10 @@ func (m *Monitor) synthesizeVzdumpJobGuestTasks(ctx context.Context, instanceNam
}
unlock, current := m.beginPVEBackupPublication(ctx, instanceName)
if !current {
- return nil
+ if cached {
+ byJob[index] = cachedVzdumpGuestTasks(entry.tasks, jobTask)
+ }
+ continue
}
fetches++
delete(fetchJobs, key) // duplicate provider rows cannot spend the same turn twice
@@ -2838,7 +2846,10 @@ func (m *Monitor) synthesizeVzdumpJobGuestTasks(ctx context.Context, instanceNam
unlock, current = m.beginPVEBackupPublication(ctx, instanceName)
if !current {
- return nil
+ if cached {
+ byJob[index] = cachedVzdumpGuestTasks(entry.tasks, jobTask)
+ }
+ continue
}
guestTasks := refreshObservedAt(parseVzdumpJobLog(jobTask, upid, logLines), now)
historyComplete := backupGuestLogComplete(jobTask, logLines, guestTasks)
diff --git a/internal/monitoring/pve_backup_lifecycle_test.go b/internal/monitoring/pve_backup_lifecycle_test.go
index 9825be89a4..26d0225de2 100644
--- a/internal/monitoring/pve_backup_lifecycle_test.go
+++ b/internal/monitoring/pve_backup_lifecycle_test.go
@@ -284,7 +284,28 @@ func TestPVEBackupLifecycleCanceledCollectorsKeepEvidence(t *testing.T) {
} else {
m.pollBackupTasks(ctx, "site-a", c)
}
- require.Equal(t, before, m.state.GetSnapshot().PVEBackups)
+ observed := m.state.GetSnapshot().PVEBackups
+ if phase == "log" {
+ // The parent listing was accepted before this log exhausted
+ // the budget. Its observation may advance; cached guest facts
+ // may not. A canceled read supplied no new guest verdict.
+ var beforeGuests, afterGuests []models.BackupTask
+ for _, row := range before.BackupTasks {
+ if row.VMID > 0 {
+ beforeGuests = append(beforeGuests, row)
+ }
+ }
+ for _, row := range observed.BackupTasks {
+ if row.VMID > 0 {
+ afterGuests = append(afterGuests, row)
+ }
+ }
+ require.Equal(t, beforeGuests, afterGuests)
+ require.Equal(t, before.StorageBackups, observed.StorageBackups)
+ require.Equal(t, before.GuestSnapshots, observed.GuestSnapshots)
+ } else {
+ require.Equal(t, before, observed)
+ }
after := m.vzdumpJobTaskCache[key]
require.Equal(t, cache.tasks, after.tasks, "cancellation replaced accepted log evidence")
require.Equal(t, cache.final, after.final)