Skip to content

armgit / Pulse

Public Imported from GitHub
0

Real-time monitoring dashboard for Proxmox VE, PBS, Docker, Kubernetes, TrueNAS and vSphere. Self-hosted, with alerts for failed backups, full disks, unhealthy containers and offline nodes.

Keep live backup reads current without starving ended jobs

pulse-triage[bot]9f48d8a8fa09
diff --git a/docs/release-control/v6/internal/subsystems/monitoring.md b/docs/release-control/v6/internal/subsystems/monitoring.md
index eb5e7d2d41..49d4e90b36 100644
--- a/docs/release-control/v6/internal/subsystems/monitoring.md
+++ b/docs/release-control/v6/internal/subsystems/monitoring.md
@@ -21,7 +21,10 @@
Multi-guest vzdump log collection keeps its 25-request cap per poll. Pending
jobs are selected by least recent attempt within the connection, with never-read
-jobs first and provider order breaking ties. Failures, empty, capped and
+jobs first and provider order breaking ties. Running jobs use up to 24 turns
+first, preserving ordinary live reads; at least one turn remains for pending
+ended jobs even when live work fills the allowance. Unused turns serve either
+queue. Failures, empty, capped and
incomplete logs spend their turn without becoming final; an unread older job
must not be hidden forever by the same failing prefix. Only an actual request
advances scheduling, not selection or cancellation. Selection leaves input and
@@ -33,7 +36,8 @@ the original guest evidence and its age; neither a retry nor an ended parent
supplies a missing guest verdict. No allowance, provider permission, task/result
schema, identity, protection rule or guest operation changes.
`vzdump_job_log_scheduling_test.go` checks bounded progress, poll-to-guest results,
-immutable completion, cancellation, pruning and concurrent connection isolation.
+immutable completion, live/history progress under saturation, cancellation,
+pruning and concurrent connection isolation.
The existing cached-guest tests retain their observation/intent assertions.
Desktop/narrow proof consumes actual poll frames through Backups Coverage,
including scoped task results without inventing a restore artifact or protection.
diff --git a/internal/monitoring/monitor_backups.go b/internal/monitoring/monitor_backups.go
index de834f8bdf..e4b8446e48 100644
--- a/internal/monitoring/monitor_backups.go
+++ b/internal/monitoring/monitor_backups.go
@@ -2797,13 +2797,15 @@ func (m *Monitor) synthesizeVzdumpJobGuestTasks(ctx context.Context, instanceNam
}
// Select without reordering the returned guest tasks or mutating the caller's
-// history. Never-read jobs come first; ties retain provider order. Thereafter
-// the least recently attempted reads win, including failed reads. Only actual
-// calls advance scheduling, so cancellation cannot spend another job's turn.
+// history. Keep ordinary running jobs current, but reserve at least one turn
+// for pending ended jobs even when live work fills the allowance. Within each
+// group, never-read then least recently attempted reads win; ties retain provider
+// order. Only actual calls advance scheduling, not selection or cancellation.
func (m *Monitor) selectVzdumpJobLogReads(instance string, jobs []models.BackupTask, upids map[string]string) map[string]bool {
type pendingRead struct {
key string
attempt time.Time
+ running bool
}
pending := make([]pendingRead, 0, len(jobs))
seen := make(map[string]bool, len(jobs))
@@ -2814,14 +2816,27 @@ func (m *Monitor) selectVzdumpJobLogReads(instance string, jobs []models.BackupT
continue
}
seen[key] = true
- pending = append(pending, pendingRead{key: key, attempt: m.vzdumpJobTaskCache[key].lastAttempt})
+ pending = append(pending, pendingRead{key: key, attempt: m.vzdumpJobTaskCache[key].lastAttempt, running: job.EndTime.IsZero()})
}
m.mu.RUnlock()
sort.SliceStable(pending, func(i, j int) bool {
return pending[i].attempt.Before(pending[j].attempt)
})
selected := make(map[string]bool, min(len(pending), maxVzdumpJobLogFetchesPerPoll))
- for _, read := range pending[:min(len(pending), maxVzdumpJobLogFetchesPerPoll)] {
+ for _, read := range pending {
+ if read.running && len(selected) < maxVzdumpJobLogFetchesPerPoll-1 {
+ selected[read.key] = true
+ }
+ }
+ for _, read := range pending {
+ if !read.running && len(selected) < maxVzdumpJobLogFetchesPerPoll {
+ selected[read.key] = true
+ }
+ }
+ for _, read := range pending {
+ if len(selected) >= maxVzdumpJobLogFetchesPerPoll {
+ break
+ }
selected[read.key] = true
}
return selected
diff --git a/internal/monitoring/vzdump_job_log_scheduling_test.go b/internal/monitoring/vzdump_job_log_scheduling_test.go
index 6a972bc50f..20d8b6b1aa 100644
--- a/internal/monitoring/vzdump_job_log_scheduling_test.go
+++ b/internal/monitoring/vzdump_job_log_scheduling_test.go
@@ -181,3 +181,40 @@ func TestVzdumpJobLogSchedulingConcurrentConnections(t *testing.T) {
wg.Wait()
require.Len(t, m.vzdumpJobTaskCache, 3*(maxVzdumpJobLogFetchesPerPoll+1))
}
+
+func TestVzdumpJobLogSchedulingKeepsLiveReadsAndHistoryMoving(t *testing.T) {
+ for _, liveCount := range []int{1, maxVzdumpJobLogFetchesPerPoll + 1} {
+ t.Run(fmt.Sprint(liveCount), func(t *testing.T) {
+ jobs, upids := scheduledVzdumpJobs(2*maxVzdumpJobLogFetchesPerPoll+liveCount, "site")
+ live := make(map[string]bool)
+ for i := 2 * maxVzdumpJobLogFetchesPerPoll; i < len(jobs); i++ {
+ jobs[i].EndTime = time.Time{}
+ jobs[i].Status = "running"
+ live[upids[jobs[i].ID]] = true
+ }
+ client := &scheduledVzdumpClient{read: func(string) ([]proxmox.TaskLogLine, error) { return nil, nil }}
+ m := &Monitor{}
+ seen := make(map[string]bool)
+ // Even a saturated live queue leaves one history turn per poll.
+ for cycle := 0; cycle < 2*maxVzdumpJobLogFetchesPerPoll; cycle++ {
+ before := len(client.requests)
+ m.synthesizeVzdumpJobGuestTasks(context.Background(), "site", client, jobs, upids)
+ reads := client.requests[before:]
+ require.Len(t, reads, maxVzdumpJobLogFetchesPerPoll)
+ liveReads := 0
+ for _, upid := range reads {
+ seen[upid] = true
+ if live[upid] {
+ liveReads++
+ }
+ }
+ if liveCount == 1 {
+ require.Equal(t, 1, liveReads, "ordinary live work is not delayed behind history")
+ } else {
+ require.Equal(t, maxVzdumpJobLogFetchesPerPoll-1, liveReads, "saturated live work still reserves history progress")
+ }
+ }
+ require.Len(t, seen, len(jobs), "both live and ended queues must make progress")
+ })
+ }
+}