Real-time monitoring dashboard for Proxmox VE, PBS, Docker, Kubernetes, TrueNAS and vSphere. Self-hosted, with alerts for failed backups, full disks, unhealthy containers and offline nodes.
Go aialertsdashboarddockerdocker-monitoringgohomelabinfrastructure-monitoringkubernetesmonitoringmspproxmoxproxmox-backup-serverproxmox-mail-gatewayproxmox-veself-hostedtruenastypescriptvspherewebhooks
Keep live backup reads current without starving ended jobs
pulse-triage[bot]9f48d8a8fa09
diff --git a/docs/release-control/v6/internal/subsystems/monitoring.md b/docs/release-control/v6/internal/subsystems/monitoring.mdindex eb5e7d2d41..49d4e90b36 100644--- a/docs/release-control/v6/internal/subsystems/monitoring.md+++ b/docs/release-control/v6/internal/subsystems/monitoring.md@@ -21,7 +21,10 @@Multi-guest vzdump log collection keeps its 25-request cap per poll. Pendingjobs are selected by least recent attempt within the connection, with never-read-jobs first and provider order breaking ties. Failures, empty, capped and+jobs first and provider order breaking ties. Running jobs use up to 24 turns+first, preserving ordinary live reads; at least one turn remains for pending+ended jobs even when live work fills the allowance. Unused turns serve either+queue. Failures, empty, capped andincomplete logs spend their turn without becoming final; an unread older jobmust not be hidden forever by the same failing prefix. Only an actual requestadvances scheduling, not selection or cancellation. Selection leaves input and@@ -33,7 +36,8 @@ the original guest evidence and its age; neither a retry nor an ended parentsupplies a missing guest verdict. No allowance, provider permission, task/resultschema, identity, protection rule or guest operation changes.`vzdump_job_log_scheduling_test.go` checks bounded progress, poll-to-guest results,-immutable completion, cancellation, pruning and concurrent connection isolation.+immutable completion, live/history progress under saturation, cancellation,+pruning and concurrent connection isolation.The existing cached-guest tests retain their observation/intent assertions.Desktop/narrow proof consumes actual poll frames through Backups Coverage,including scoped task results without inventing a restore artifact or protection.diff --git a/internal/monitoring/monitor_backups.go b/internal/monitoring/monitor_backups.goindex de834f8bdf..e4b8446e48 100644--- a/internal/monitoring/monitor_backups.go+++ b/internal/monitoring/monitor_backups.go@@ -2797,13 +2797,15 @@ func (m *Monitor) synthesizeVzdumpJobGuestTasks(ctx context.Context, instanceNam}// Select without reordering the returned guest tasks or mutating the caller's-// history. Never-read jobs come first; ties retain provider order. Thereafter-// the least recently attempted reads win, including failed reads. Only actual-// calls advance scheduling, so cancellation cannot spend another job's turn.+// history. Keep ordinary running jobs current, but reserve at least one turn+// for pending ended jobs even when live work fills the allowance. Within each+// group, never-read then least recently attempted reads win; ties retain provider+// order. Only actual calls advance scheduling, not selection or cancellation.func (m *Monitor) selectVzdumpJobLogReads(instance string, jobs []models.BackupTask, upids map[string]string) map[string]bool {type pendingRead struct {key stringattempt time.Time+ running bool}pending := make([]pendingRead, 0, len(jobs))seen := make(map[string]bool, len(jobs))@@ -2814,14 +2816,27 @@ func (m *Monitor) selectVzdumpJobLogReads(instance string, jobs []models.BackupTcontinue}seen[key] = true- pending = append(pending, pendingRead{key: key, attempt: m.vzdumpJobTaskCache[key].lastAttempt})+ pending = append(pending, pendingRead{key: key, attempt: m.vzdumpJobTaskCache[key].lastAttempt, running: job.EndTime.IsZero()})}m.mu.RUnlock()sort.SliceStable(pending, func(i, j int) bool {return pending[i].attempt.Before(pending[j].attempt)})selected := make(map[string]bool, min(len(pending), maxVzdumpJobLogFetchesPerPoll))- for _, read := range pending[:min(len(pending), maxVzdumpJobLogFetchesPerPoll)] {+ for _, read := range pending {+ if read.running && len(selected) < maxVzdumpJobLogFetchesPerPoll-1 {+ selected[read.key] = true+ }+ }+ for _, read := range pending {+ if !read.running && len(selected) < maxVzdumpJobLogFetchesPerPoll {+ selected[read.key] = true+ }+ }+ for _, read := range pending {+ if len(selected) >= maxVzdumpJobLogFetchesPerPoll {+ break+ }selected[read.key] = true}return selecteddiff --git a/internal/monitoring/vzdump_job_log_scheduling_test.go b/internal/monitoring/vzdump_job_log_scheduling_test.goindex 6a972bc50f..20d8b6b1aa 100644--- a/internal/monitoring/vzdump_job_log_scheduling_test.go+++ b/internal/monitoring/vzdump_job_log_scheduling_test.go@@ -181,3 +181,40 @@ func TestVzdumpJobLogSchedulingConcurrentConnections(t *testing.T) {wg.Wait()require.Len(t, m.vzdumpJobTaskCache, 3*(maxVzdumpJobLogFetchesPerPoll+1))}++func TestVzdumpJobLogSchedulingKeepsLiveReadsAndHistoryMoving(t *testing.T) {+ for _, liveCount := range []int{1, maxVzdumpJobLogFetchesPerPoll + 1} {+ t.Run(fmt.Sprint(liveCount), func(t *testing.T) {+ jobs, upids := scheduledVzdumpJobs(2*maxVzdumpJobLogFetchesPerPoll+liveCount, "site")+ live := make(map[string]bool)+ for i := 2 * maxVzdumpJobLogFetchesPerPoll; i < len(jobs); i++ {+ jobs[i].EndTime = time.Time{}+ jobs[i].Status = "running"+ live[upids[jobs[i].ID]] = true+ }+ client := &scheduledVzdumpClient{read: func(string) ([]proxmox.TaskLogLine, error) { return nil, nil }}+ m := &Monitor{}+ seen := make(map[string]bool)+ // Even a saturated live queue leaves one history turn per poll.+ for cycle := 0; cycle < 2*maxVzdumpJobLogFetchesPerPoll; cycle++ {+ before := len(client.requests)+ m.synthesizeVzdumpJobGuestTasks(context.Background(), "site", client, jobs, upids)+ reads := client.requests[before:]+ require.Len(t, reads, maxVzdumpJobLogFetchesPerPoll)+ liveReads := 0+ for _, upid := range reads {+ seen[upid] = true+ if live[upid] {+ liveReads+++ }+ }+ if liveCount == 1 {+ require.Equal(t, 1, liveReads, "ordinary live work is not delayed behind history")+ } else {+ require.Equal(t, maxVzdumpJobLogFetchesPerPoll-1, liveReads, "saturated live work still reserves history progress")+ }+ }+ require.Len(t, seen, len(jobs), "both live and ended queues must make progress")+ })+ }+}