Real-time monitoring dashboard for Proxmox VE, PBS, Docker, Kubernetes, TrueNAS and vSphere. Self-hosted, with alerts for failed backups, full disks, unhealthy containers and offline nodes.
Go aialertsdashboarddockerdocker-monitoringgohomelabinfrastructure-monitoringkubernetesmonitoringmspproxmoxproxmox-backup-serverproxmox-mail-gatewayproxmox-veself-hostedtruenastypescriptvspherewebhooks
Share the backup log context between live and ended reads
pulse-triage[bot]48c50d69fe87
diff --git a/docs/release-control/v6/internal/subsystems/monitoring.md b/docs/release-control/v6/internal/subsystems/monitoring.mdindex 053e0a863b..30d2d2dd79 100644--- a/docs/release-control/v6/internal/subsystems/monitoring.md+++ b/docs/release-control/v6/internal/subsystems/monitoring.md@@ -21,16 +21,22 @@Multi-guest vzdump log collection keeps its 25-request cap per poll. Pendingjobs are selected by least recent attempt within the connection, with never-read-jobs first and provider order breaking ties. Running jobs use up to 24 turns-first, preserving ordinary live reads; at least one turn remains for pending-ended jobs even when live work fills the allowance. Unused turns serve either-queue. Failures, empty, capped and-incomplete logs spend their turn without becoming final; an unread older job-must not be hidden forever by the same failing prefix. Only an actual request-advances scheduling, not selection or cancellation. Execution follows that-priority too, so one slow read exhausting the context cannot monopolise even a-small queue. Selection leaves input and guest output ordering unchanged, completed attributed logs remain cached, and-removed jobs discard their scheduling state with the existing cache pruning.+jobs first and provider order breaking ties. While ended work waits, admission keeps up to 24 running+reads and reserves at least one of the 25 turns for pending ended history;+unused turns serve either queue. Execution starts with the queue not served by+the last actual request (live first on a new connection), then uses the admitted+live-first order. Thus a read exhausting the shared context cannot indefinitely+prevent the other queue from running. When both queues exhaust it on every+request, their first turns alternate; when just one queue exhausts it, the+other gets a request before it on the next poll. The request cap is not a+promise that all admitted reads fit in the shared time budget.++Failures, empty, capped and incomplete logs spend their turn without becoming+final. Only an actual request advances either per-job age or the connection's+queue cursor, not selection or cancellation before execution. The cursor is+connection-local and discarded with an empty history window. Selection leaves+input and guest output ordering unchanged, completed attributed logs remain+cached, and removed jobs discard their scheduling state with cache pruning.Scheduling metadata is not an accepted observation. Skipped/failed reads retainthe original guest evidence and its age; neither a retry nor an ended parent@@ -38,7 +44,8 @@ supplies a missing guest verdict. No allowance, provider permission, task/resultschema, identity, protection rule or guest operation changes.`vzdump_job_log_scheduling_test.go` checks bounded progress, poll-to-guest results,immutable completion, live/history progress under saturation, cancellation,-context-budget fairness, stable output order, pruning and concurrent connection+same-queue and mixed live/ended context exhaustion, fast live reads with a+slow ended backlog, stable output order, cursor pruning and concurrent connectionisolation.The existing cached-guest tests retain their observation/intent assertions.Desktop/narrow proof consumes actual poll frames through Backups Coverage,diff --git a/internal/monitoring/monitor.go b/internal/monitoring/monitor.goindex 7afe0f68ed..176056b6cc 100644--- a/internal/monitoring/monitor.go+++ b/internal/monitoring/monitor.go@@ -1371,6 +1371,7 @@ type Monitor struct {pveReplicationPolls map[string]*pveReplicationPoll // Owned runtime/client-scoped replication reads; guarded by mulastPVEBackupPoll map[string]time.Time // Track last PVE backup poll per instancevzdumpJobTaskCache map[string]vzdumpJobTaskCacheEntry // Cache synthesized per-guest tasks for multi-guest vzdump job runs, keyed by instance|UPID+ vzdumpJobLogHistoryFirst map[string]bool // Last actual log read was running; next poll offers ended history the first turn. Guarded by mu.lastPBSBackupPoll map[string]time.Time // Track last PBS backup poll per instancepveBackupInventoryReady map[string]map[string]bool // Track PVE guest inventory readiness for backup orphan detectionpveBackupTemplateSubjects map[string]map[string]struct{} // Track template VMIDs excluded from runtime workloads but valid for backupsdiff --git a/internal/monitoring/monitor_backups.go b/internal/monitoring/monitor_backups.goindex fa6e13ffe5..53d86cfe4c 100644--- a/internal/monitoring/monitor_backups.go+++ b/internal/monitoring/monitor_backups.go@@ -2774,6 +2774,10 @@ func (m *Monitor) synthesizeVzdumpJobGuestTasks(ctx context.Context, instanceNamattempted := m.vzdumpJobTaskCache[key]attempted.lastAttempt = nowm.vzdumpJobTaskCache[key] = attempted+ if m.vzdumpJobLogHistoryFirst == nil {+ m.vzdumpJobLogHistoryFirst = make(map[string]bool)+ }+ m.vzdumpJobLogHistoryFirst[instanceName] = jobTask.EndTime.IsZero()m.mu.Unlock()logLines, err := client.GetTaskLog(ctx, jobTask.Node, upid)@@ -2816,7 +2820,9 @@ func (m *Monitor) synthesizeVzdumpJobGuestTasks(ctx context.Context, instanceNam// history. Keep ordinary running jobs current, but reserve at least one turn// for pending ended jobs even when live work fills the allowance. Within each// group, never-read then least recently attempted reads win; ties retain provider-// order. Only actual calls advance scheduling, not selection or cancellation.+// order. The queue not served by the last actual request gets the first turn,+// so either queue can progress even when one read exhausts the shared context.+// Only actual calls advance scheduling, not selection or cancellation.func (m *Monitor) selectVzdumpJobLogReads(instance string, jobs []models.BackupTask, upids map[string]string) map[string]int {type pendingRead struct {key string@@ -2826,6 +2832,7 @@ func (m *Monitor) selectVzdumpJobLogReads(instance string, jobs []models.BackupTpending := make([]pendingRead, 0, len(jobs))seen := make(map[string]bool, len(jobs))m.mu.RLock()+ historyFirst := m.vzdumpJobLogHistoryFirst[instance]for _, job := range jobs {key := instance + "|" + upids[job.ID]if seen[key] || m.vzdumpJobTaskCache[key].final {@@ -2858,6 +2865,24 @@ func (m *Monitor) selectVzdumpJobLogReads(instance string, jobs []models.BackupTselected[read.key] = len(selected) + 1}}+ // Admission reserves live capacity and an ended turn; execution must share+ // the context too. Move only the oldest selected ended read ahead of live+ // work when the last actual request was live. If either queue exhausts the+ // context, the other starts next poll. Fast live reads still run every poll,+ // without waiting behind a whole backlog of slow ended reads (or vice versa).+ if historyFirst {+ for _, read := range pending {+ if rank := selected[read.key]; !read.running && rank > 0 {+ for key, priority := range selected {+ if priority < rank {+ selected[key] = priority + 1+ }+ }+ selected[read.key] = 1+ break+ }+ }+ }return selected}@@ -2887,6 +2912,9 @@ func cachedVzdumpGuestTasks(tasks []models.BackupTask, job models.BackupTask) []func (m *Monitor) pruneVzdumpJobTaskCache(instanceName string, seen map[string]struct{}) {prefix := instanceName + "|"m.mu.Lock()+ if len(seen) == 0 {+ delete(m.vzdumpJobLogHistoryFirst, instanceName)+ }for key := range m.vzdumpJobTaskCache {if !strings.HasPrefix(key, prefix) {continuediff --git a/internal/monitoring/vzdump_job_log_scheduling_test.go b/internal/monitoring/vzdump_job_log_scheduling_test.goindex a3cd61e276..9b9f1f568d 100644--- a/internal/monitoring/vzdump_job_log_scheduling_test.go+++ b/internal/monitoring/vzdump_job_log_scheduling_test.go@@ -172,6 +172,8 @@ func TestVzdumpJobLogSchedulingConcurrentConnections(t *testing.T) {go func(instance string) {defer wg.Done()jobs, upids := scheduledVzdumpJobs(maxVzdumpJobLogFetchesPerPoll+1, instance)+ jobs[len(jobs)-1].EndTime = time.Time{}+ jobs[len(jobs)-1].Status = "running"client := &scheduledVzdumpClient{read: func(string) ([]proxmox.TaskLogLine, error) { return nil, nil }}for cycle := 0; cycle < 3; cycle++ {m.synthesizeVzdumpJobGuestTasks(context.Background(), instance, client, jobs, upids)@@ -256,3 +258,152 @@ func TestVzdumpJobLogSchedulingOutputRetainsProviderOrder(t *testing.T) {}require.Equal(t, []string{upids[jobs[0].ID], upids[jobs[1].ID], upids[jobs[1].ID], upids[jobs[0].ID]}, client.requests)}++// A never-read ended job must run even if a live request consumes every poll's+// context. This exercises the real poll-to-state path, not just admission ranks.+func TestVzdumpJobLogSchedulingMixedQueuesContextProgress(t *testing.T) {+ jobs, upids := scheduledVzdumpJobs(2, "site")+ jobs[1].EndTime = time.Time{}+ jobs[1].Status = "running"+ client := &scheduledVzdumpClient{history: proxmox.BackupTaskHistory{Nodes: []proxmox.BackupTaskHistoryNode{{Node: "pve1", Complete: true}}}}+ for _, job := range jobs {+ end := int64(0)+ if !job.EndTime.IsZero() {+ end = job.EndTime.Unix()+ }+ client.history.Tasks = append(client.history.Tasks, proxmox.Task{UPID: upids[job.ID], Node: job.Node, Type: job.Type, Status: job.Status, StartTime: job.StartTime.Unix(), EndTime: end})+ }+ m := &Monitor{state: models.NewState()}+ for cycle := 0; cycle < 4; cycle++ {+ ctx, cancel := context.WithCancel(context.Background())+ client.read = func(upid string) ([]proxmox.TaskLogLine, error) {+ if upid == upids[jobs[1].ID] {+ cancel()+ return nil, context.DeadlineExceeded+ }+ return testLogLines("INFO: Starting Backup of VM 501 (qemu)", "INFO: Finished Backup of VM 501 (00:00:30)", "TASK OK"), nil+ }+ m.pollBackupTasks(ctx, "site", client)+ cancel()+ }+ require.Equal(t, []string{upids[jobs[1].ID], upids[jobs[0].ID], upids[jobs[1].ID], upids[jobs[1].ID], upids[jobs[1].ID]}, client.requests,+ "ended evidence gets through on the second poll without delaying ordinary live attempts")+ var guests []models.BackupTask+ for _, task := range m.state.GetBackupTasks() {+ if task.VMID > 0 {+ guests = append(guests, task)+ }+ }+ require.Len(t, guests, 1, "a timed-out live read supplies no invented guest outcome")+ require.Equal(t, jobs[0].ID+"-vm501", guests[0].ID)+ require.Equal(t, "OK", guests[0].Status)+ require.True(t, m.vzdumpJobTaskCache["site|"+upids[jobs[0].ID]].historyComplete)+ require.False(t, m.vzdumpJobTaskCache["site|"+upids[jobs[1].ID]].historyComplete)+}++func TestVzdumpJobLogSchedulingMixedQueuesBothExhaustContext(t *testing.T) {+ for _, counts := range [][2]int{{1, 1}, {3, 2}, {maxVzdumpJobLogFetchesPerPoll + 1, maxVzdumpJobLogFetchesPerPoll + 2}} {+ for _, liveFirst := range []bool{false, true} {+ t.Run(fmt.Sprintf("live=%d/ended=%d/live-first=%t", counts[0], counts[1], liveFirst), func(t *testing.T) {+ liveCount, endedCount := counts[0], counts[1]+ jobs, upids := scheduledVzdumpJobs(liveCount+endedCount, "site")+ live := make(map[string]bool)+ for i := range jobs {+ if (liveFirst && i < liveCount) || (!liveFirst && i >= endedCount) {+ jobs[i].EndTime = time.Time{}+ jobs[i].Status = "running"+ live[upids[jobs[i].ID]] = true+ }+ }+ original := append([]models.BackupTask(nil), jobs...)+ client := &scheduledVzdumpClient{}+ m := &Monitor{}+ for cycle := 0; cycle < 2*max(liveCount, endedCount); cycle++ {+ ctx, cancel := context.WithCancel(context.Background())+ client.read = func(string) ([]proxmox.TaskLogLine, error) {+ cancel()+ return nil, context.DeadlineExceeded+ }+ before := len(client.requests)+ tasks := m.synthesizeVzdumpJobGuestTasks(ctx, "site", client, jobs, upids)+ cancel()+ require.Empty(t, tasks)+ require.Len(t, client.requests[before:], 1)+ require.Equal(t, cycle%2 == 0, live[client.requests[before]], "actual first turns alternate even under saturation")+ }+ seen := make(map[string]bool)+ for _, upid := range client.requests {+ seen[upid] = true+ }+ require.Len(t, seen, len(jobs), "neither queue nor any pending job can be hidden forever")+ require.Equal(t, original, jobs)+ })+ }+ }+}++func TestVzdumpJobLogSchedulingFastLiveSurvivesSlowEndedBacklog(t *testing.T) {+ const liveCount = 2+ endedCount := maxVzdumpJobLogFetchesPerPoll + 2+ jobs, upids := scheduledVzdumpJobs(endedCount+liveCount, "site")+ live := make(map[string]bool)+ for i := endedCount; i < len(jobs); i++ {+ jobs[i].EndTime = time.Time{}+ jobs[i].Status = "running"+ live[upids[jobs[i].ID]] = true+ }+ client := &scheduledVzdumpClient{}+ m := &Monitor{}+ for cycle := 0; cycle < endedCount; cycle++ {+ ctx, cancel := context.WithCancel(context.Background())+ client.read = func(upid string) ([]proxmox.TaskLogLine, error) {+ if live[upid] {+ return nil, nil+ }+ cancel()+ return nil, context.DeadlineExceeded+ }+ before := len(client.requests)+ m.synthesizeVzdumpJobGuestTasks(ctx, "site", client, jobs, upids)+ cancel()+ reads := client.requests[before:]+ require.Len(t, reads, liveCount+1)+ for _, job := range jobs[endedCount:] {+ require.Contains(t, reads[:liveCount], upids[job.ID], "live work must not wait behind unread slow history")+ }+ require.Equal(t, upids[jobs[cycle].ID], reads[liveCount], "ended reads keep their own least-attempted order")+ }+}++func TestVzdumpJobLogSchedulingQueueCursorOnlyTracksActualScopedReads(t *testing.T) {+ jobs, upids := scheduledVzdumpJobs(2, "site")+ jobs[1].EndTime = time.Time{}+ jobs[1].Status = "running"+ ctx, cancel := context.WithCancel(context.Background())+ defer cancel()+ client := &scheduledVzdumpClient{read: func(string) ([]proxmox.TaskLogLine, error) {+ cancel()+ return nil, context.DeadlineExceeded+ }}+ m := &Monitor{}+ m.synthesizeVzdumpJobGuestTasks(ctx, "site", client, jobs, upids)+ require.Equal(t, []string{upids[jobs[1].ID]}, client.requests)+ require.True(t, m.vzdumpJobLogHistoryFirst["site"])+ // Repeated selection and a cancelled poll cannot spend the ended turn.+ for i := 0; i < 2; i++ {+ require.Equal(t, 1, m.selectVzdumpJobLogReads("site", jobs, upids)["site|"+upids[jobs[0].ID]])+ }+ m.synthesizeVzdumpJobGuestTasks(ctx, "site", client, jobs, upids)+ require.Len(t, client.requests, 1)+ require.True(t, m.vzdumpJobLogHistoryFirst["site"])+ client.read = func(string) ([]proxmox.TaskLogLine, error) { return nil, nil }+ m.synthesizeVzdumpJobGuestTasks(context.Background(), "other", client, jobs, upids)+ require.Equal(t, upids[jobs[1].ID], client.requests[1], "a new connection starts live, regardless of its peer's cursor")+ require.True(t, m.vzdumpJobLogHistoryFirst["site"])+ m.synthesizeVzdumpJobGuestTasks(context.Background(), "site", client, nil, nil)+ require.NotContains(t, m.vzdumpJobLogHistoryFirst, "site")+ require.Contains(t, m.vzdumpJobLogHistoryFirst, "other")+ for key := range m.vzdumpJobTaskCache {+ require.NotContains(t, key, "site|")+ }+}