Skip to content

armgit / Pulse

Public Imported from GitHub
0

Real-time monitoring dashboard for Proxmox VE, PBS, Docker, Kubernetes, TrueNAS and vSphere. Self-hosted, with alerts for failed backups, full disks, unhealthy containers and offline nodes.

Rotate backup log execution as well as admission

pulse-triage[bot]281b04486b85
diff --git a/docs/release-control/v6/internal/subsystems/monitoring.md b/docs/release-control/v6/internal/subsystems/monitoring.md
index 49d4e90b36..053e0a863b 100644
--- a/docs/release-control/v6/internal/subsystems/monitoring.md
+++ b/docs/release-control/v6/internal/subsystems/monitoring.md
@@ -27,8 +27,9 @@ ended jobs even when live work fills the allowance. Unused turns serve either
queue. Failures, empty, capped and
incomplete logs spend their turn without becoming final; an unread older job
must not be hidden forever by the same failing prefix. Only an actual request
-advances scheduling, not selection or cancellation. Selection leaves input and
-guest output ordering unchanged, completed attributed logs remain cached, and
+advances scheduling, not selection or cancellation. Execution follows that
+priority too, so one slow read exhausting the context cannot monopolise even a
+small queue. Selection leaves input and guest output ordering unchanged, completed attributed logs remain cached, and
removed jobs discard their scheduling state with the existing cache pruning.
Scheduling metadata is not an accepted observation. Skipped/failed reads retain
@@ -37,7 +38,8 @@ supplies a missing guest verdict. No allowance, provider permission, task/result
schema, identity, protection rule or guest operation changes.
`vzdump_job_log_scheduling_test.go` checks bounded progress, poll-to-guest results,
immutable completion, live/history progress under saturation, cancellation,
-pruning and concurrent connection isolation.
+context-budget fairness, stable output order, pruning and concurrent connection
+isolation.
The existing cached-guest tests retain their observation/intent assertions.
Desktop/narrow proof consumes actual poll frames through Backups Coverage,
including scoped task results without inventing a restore artifact or protection.
diff --git a/internal/monitoring/monitor_backups.go b/internal/monitoring/monitor_backups.go
index e4b8446e48..fa6e13ffe5 100644
--- a/internal/monitoring/monitor_backups.go
+++ b/internal/monitoring/monitor_backups.go
@@ -2725,10 +2725,22 @@ func (m *Monitor) synthesizeVzdumpJobGuestTasks(ctx context.Context, instanceNam
now := time.Now().UTC()
seen := make(map[string]struct{}, len(jobTasks))
fetchJobs := m.selectVzdumpJobLogReads(instanceName, jobTasks, jobUPIDs)
- var synthesized []models.BackupTask
+ order := make([]int, len(jobTasks))
+ for i := range order {
+ order[i] = i
+ }
+ // Execute selected reads in scheduling order as well: an early slow read
+ // can exhaust the shared context before the request-count cap is reached.
+ sort.SliceStable(order, func(i, j int) bool {
+ left := fetchJobs[instanceName+"|"+jobUPIDs[jobTasks[order[i]].ID]]
+ right := fetchJobs[instanceName+"|"+jobUPIDs[jobTasks[order[j]].ID]]
+ return left > 0 && (right == 0 || left < right)
+ })
+ byJob := make([][]models.BackupTask, len(jobTasks))
fetches := 0
- for _, jobTask := range jobTasks {
+ for _, index := range order {
+ jobTask := jobTasks[index]
upid := jobUPIDs[jobTask.ID]
key := instanceName + "|" + upid
seen[key] = struct{}{}
@@ -2738,16 +2750,16 @@ func (m *Monitor) synthesizeVzdumpJobGuestTasks(ctx context.Context, instanceNam
m.mu.RUnlock()
if cached && entry.final {
- synthesized = append(synthesized, refreshObservedAt(entry.tasks, now)...)
+ byJob[index] = refreshObservedAt(entry.tasks, now)
continue
}
- if !fetchJobs[key] || fetches >= maxVzdumpJobLogFetchesPerPoll || ctx.Err() != nil {
+ if fetchJobs[key] == 0 || fetches >= maxVzdumpJobLogFetchesPerPoll || ctx.Err() != nil {
// Serve a stale non-final entry if we have one; the next cycle
// will catch up. The current parent can end a cached running
// guest, but cannot supply its missing verdict or a fresh log read.
if cached {
- synthesized = append(synthesized, cachedVzdumpGuestTasks(entry.tasks, jobTask)...)
+ byJob[index] = cachedVzdumpGuestTasks(entry.tasks, jobTask)
}
continue
}
@@ -2771,7 +2783,7 @@ func (m *Monitor) synthesizeVzdumpJobGuestTasks(ctx context.Context, instanceNam
Str("upid", upid).
Msg("failed to fetch vzdump job task log; per-guest status unavailable this cycle")
if cached {
- synthesized = append(synthesized, cachedVzdumpGuestTasks(entry.tasks, jobTask)...)
+ byJob[index] = cachedVzdumpGuestTasks(entry.tasks, jobTask)
}
continue
}
@@ -2789,10 +2801,14 @@ func (m *Monitor) synthesizeVzdumpJobGuestTasks(ctx context.Context, instanceNam
lastAttempt: now,
}
m.mu.Unlock()
- synthesized = append(synthesized, guestTasks...)
+ byJob[index] = guestTasks
}
m.pruneVzdumpJobTaskCache(instanceName, seen)
+ var synthesized []models.BackupTask
+ for _, tasks := range byJob {
+ synthesized = append(synthesized, tasks...)
+ }
return synthesized
}
@@ -2801,7 +2817,7 @@ func (m *Monitor) synthesizeVzdumpJobGuestTasks(ctx context.Context, instanceNam
// for pending ended jobs even when live work fills the allowance. Within each
// group, never-read then least recently attempted reads win; ties retain provider
// order. Only actual calls advance scheduling, not selection or cancellation.
-func (m *Monitor) selectVzdumpJobLogReads(instance string, jobs []models.BackupTask, upids map[string]string) map[string]bool {
+func (m *Monitor) selectVzdumpJobLogReads(instance string, jobs []models.BackupTask, upids map[string]string) map[string]int {
type pendingRead struct {
key string
attempt time.Time
@@ -2822,22 +2838,25 @@ func (m *Monitor) selectVzdumpJobLogReads(instance string, jobs []models.BackupT
sort.SliceStable(pending, func(i, j int) bool {
return pending[i].attempt.Before(pending[j].attempt)
})
- selected := make(map[string]bool, min(len(pending), maxVzdumpJobLogFetchesPerPoll))
+ // Positive ranks bind both admission and execution order; zero is unselected.
+ selected := make(map[string]int, min(len(pending), maxVzdumpJobLogFetchesPerPoll))
for _, read := range pending {
if read.running && len(selected) < maxVzdumpJobLogFetchesPerPoll-1 {
- selected[read.key] = true
+ selected[read.key] = len(selected) + 1
}
}
for _, read := range pending {
if !read.running && len(selected) < maxVzdumpJobLogFetchesPerPoll {
- selected[read.key] = true
+ selected[read.key] = len(selected) + 1
}
}
for _, read := range pending {
if len(selected) >= maxVzdumpJobLogFetchesPerPoll {
break
}
- selected[read.key] = true
+ if selected[read.key] == 0 {
+ selected[read.key] = len(selected) + 1
+ }
}
return selected
}
diff --git a/internal/monitoring/vzdump_job_log_scheduling_test.go b/internal/monitoring/vzdump_job_log_scheduling_test.go
index 20d8b6b1aa..a3cd61e276 100644
--- a/internal/monitoring/vzdump_job_log_scheduling_test.go
+++ b/internal/monitoring/vzdump_job_log_scheduling_test.go
@@ -218,3 +218,41 @@ func TestVzdumpJobLogSchedulingKeepsLiveReadsAndHistoryMoving(t *testing.T) {
})
}
}
+
+func TestVzdumpJobLogSchedulingContextBudgetRotatesInsideAllowance(t *testing.T) {
+ jobs, upids := scheduledVzdumpJobs(2, "site")
+ client := &scheduledVzdumpClient{}
+ m := &Monitor{}
+ for cycle := 0; cycle < 2; cycle++ {
+ ctx, cancel := context.WithCancel(context.Background())
+ client.read = func(string) ([]proxmox.TaskLogLine, error) {
+ cancel() // one slow request consumed this cycle's context budget
+ return nil, context.DeadlineExceeded
+ }
+ m.synthesizeVzdumpJobGuestTasks(ctx, "site", client, jobs, upids)
+ cancel()
+ }
+ require.Equal(t, []string{upids[jobs[0].ID], upids[jobs[1].ID]}, client.requests,
+ "a small queue must rotate actual execution, not only admission")
+}
+
+func TestVzdumpJobLogSchedulingOutputRetainsProviderOrder(t *testing.T) {
+ jobs, upids := scheduledVzdumpJobs(2, "site")
+ index := map[string]int{upids[jobs[0].ID]: 0, upids[jobs[1].ID]: 1}
+ client := &scheduledVzdumpClient{read: func(upid string) ([]proxmox.TaskLogLine, error) {
+ return testLogLines(fmt.Sprintf("INFO: Starting Backup of VM %d (qemu)", 501+index[upid])), nil
+ }}
+ m := &Monitor{}
+ for cycle := 0; cycle < 2; cycle++ {
+ tasks := m.synthesizeVzdumpJobGuestTasks(context.Background(), "site", client, jobs, upids)
+ require.Len(t, tasks, 2)
+ require.Equal(t, []int{501, 502}, []int{tasks[0].VMID, tasks[1].VMID})
+ // Force a different scheduling order without changing accepted tasks.
+ m.mu.Lock()
+ entry := m.vzdumpJobTaskCache["site|"+upids[jobs[1].ID]]
+ entry.lastAttempt = time.Time{}
+ m.vzdumpJobTaskCache["site|"+upids[jobs[1].ID]] = entry
+ m.mu.Unlock()
+ }
+ require.Equal(t, []string{upids[jobs[0].ID], upids[jobs[1].ID], upids[jobs[1].ID], upids[jobs[0].ID]}, client.requests)
+}