Skip to content

armgit / Pulse

Public Imported from GitHub
0

Real-time monitoring dashboard for Proxmox VE, PBS, Docker, Kubernetes, TrueNAS and vSphere. Self-hosted, with alerts for failed backups, full disks, unhealthy containers and offline nodes.

Rotate unfinished vzdump log reads within the existing allowance

pulse-triage[bot]58868f500e1c
diff --git a/docs/release-control/v6/internal/subsystems/monitoring.md b/docs/release-control/v6/internal/subsystems/monitoring.md
index 0badb88278..eb5e7d2d41 100644
--- a/docs/release-control/v6/internal/subsystems/monitoring.md
+++ b/docs/release-control/v6/internal/subsystems/monitoring.md
@@ -17,6 +17,28 @@
## Purpose
+### Unfinished backup logs cannot monopolise the read allowance
+
+Multi-guest vzdump log collection keeps its 25-request cap per poll. Pending
+jobs are selected by least recent attempt within the connection, with never-read
+jobs first and provider order breaking ties. Failures, empty, capped and
+incomplete logs spend their turn without becoming final; an unread older job
+must not be hidden forever by the same failing prefix. Only an actual request
+advances scheduling, not selection or cancellation. Selection leaves input and
+guest output ordering unchanged, completed attributed logs remain cached, and
+removed jobs discard their scheduling state with the existing cache pruning.
+
+Scheduling metadata is not an accepted observation. Skipped/failed reads retain
+the original guest evidence and its age; neither a retry nor an ended parent
+supplies a missing guest verdict. No allowance, provider permission, task/result
+schema, identity, protection rule or guest operation changes.
+`vzdump_job_log_scheduling_test.go` checks bounded progress, poll-to-guest results,
+immutable completion, cancellation, pruning and concurrent connection isolation.
+The existing cached-guest tests retain their observation/intent assertions.
+Desktop/narrow proof consumes actual poll frames through Backups Coverage,
+including scoped task results without inventing a restore artifact or protection.
+
+
### Backup task results agree with their completion clocks
The PVE backup-history reader accepts a live task only with an absent/zero end
@@ -4771,9 +4793,11 @@ named and default organizations. The custom event envelope stays unchanged.
task listing queries `source=all` so running jobs are visible: a guest
an in-progress job is currently backing up carries a `running`
synthetic task, which is valid backup-intent evidence for
- `resolveBackupIntentContext`. Finished job logs are immutable and are
+ `resolveBackupIntentContext`. Completely attributed finished job logs are immutable and are
fetched at most once per instance|UPID, with a per-cycle fetch cap so
a historical backlog cannot stall the shared backup poll budget.
+ Unfinished reads rotate by last attempt, including failures, so a
+ persistent prefix cannot prevent later jobs from supplying guest results.
Regression coverage: `TestParseVzdumpJobLogFinishedJob`,
`TestPollBackupTasksSynthesizesJobGuestTasks`, and
`TestPollBackupTasksRunningJobRefetchesAndSuppressesAlerts` in
diff --git a/docs/release-control/v6/internal/subsystems/registry.json b/docs/release-control/v6/internal/subsystems/registry.json
index 260e235249..2f06c9eb7a 100644
--- a/docs/release-control/v6/internal/subsystems/registry.json
+++ b/docs/release-control/v6/internal/subsystems/registry.json
@@ -6906,6 +6906,8 @@
"internal/monitoring/pbs_protection_observation_test.go",
"internal/monitoring/pve_protection_observation_test.go",
"internal/monitoring/recovery_ingest_test.go",
+ "internal/monitoring/vzdump_cached_guest_test.go",
+ "internal/monitoring/vzdump_job_log_scheduling_test.go",
"internal/recovery/mapper/proxmox/mapper_test.go",
"internal/recovery/posture_test.go"
]
diff --git a/internal/monitoring/monitor_backups.go b/internal/monitoring/monitor_backups.go
index bb765d8281..de834f8bdf 100644
--- a/internal/monitoring/monitor_backups.go
+++ b/internal/monitoring/monitor_backups.go
@@ -2701,11 +2701,14 @@ type vzdumpJobTaskCacheEntry struct {
tasks []models.BackupTask
final bool
historyComplete bool
+ lastAttempt time.Time // scheduling metadata, not an accepted log observation
}
// maxVzdumpJobLogFetchesPerPoll bounds task-log requests per poll cycle so a
// large backlog of historical job runs (e.g. right after startup) trickles in
-// over a few cycles instead of stalling the shared backup-poll budget.
+// over a few cycles instead of stalling the shared backup-poll budget. Pending
+// reads rotate by last attempt so errors and incomplete logs cannot consume the
+// entire allowance every cycle and permanently hide older guest results.
const maxVzdumpJobLogFetchesPerPoll = 25
// synthesizeVzdumpJobGuestTasks expands multi-guest vzdump job runs into one
@@ -2721,6 +2724,7 @@ func (m *Monitor) synthesizeVzdumpJobGuestTasks(ctx context.Context, instanceNam
now := time.Now().UTC()
seen := make(map[string]struct{}, len(jobTasks))
+ fetchJobs := m.selectVzdumpJobLogReads(instanceName, jobTasks, jobUPIDs)
var synthesized []models.BackupTask
fetches := 0
@@ -2738,7 +2742,7 @@ func (m *Monitor) synthesizeVzdumpJobGuestTasks(ctx context.Context, instanceNam
continue
}
- if fetches >= maxVzdumpJobLogFetchesPerPoll || ctx.Err() != nil {
+ if !fetchJobs[key] || fetches >= maxVzdumpJobLogFetchesPerPoll || ctx.Err() != nil {
// Serve a stale non-final entry if we have one; the next cycle
// will catch up. The current parent can end a cached running
// guest, but cannot supply its missing verdict or a fresh log read.
@@ -2748,6 +2752,17 @@ func (m *Monitor) synthesizeVzdumpJobGuestTasks(ctx context.Context, instanceNam
continue
}
fetches++
+ delete(fetchJobs, key) // duplicate provider rows cannot spend the same turn twice
+ // A failed or incomplete read still spent the allowance. Keep attempt
+ // order separate from accepted observations so failure never freshens evidence.
+ m.mu.Lock()
+ if m.vzdumpJobTaskCache == nil {
+ m.vzdumpJobTaskCache = make(map[string]vzdumpJobTaskCacheEntry)
+ }
+ attempted := m.vzdumpJobTaskCache[key]
+ attempted.lastAttempt = now
+ m.vzdumpJobTaskCache[key] = attempted
+ m.mu.Unlock()
logLines, err := client.GetTaskLog(ctx, jobTask.Node, upid)
if err != nil {
@@ -2771,6 +2786,7 @@ func (m *Monitor) synthesizeVzdumpJobGuestTasks(ctx context.Context, instanceNam
tasks: guestTasks,
final: historyComplete,
historyComplete: historyComplete,
+ lastAttempt: now,
}
m.mu.Unlock()
synthesized = append(synthesized, guestTasks...)
@@ -2780,10 +2796,41 @@ func (m *Monitor) synthesizeVzdumpJobGuestTasks(ctx context.Context, instanceNam
return synthesized
}
+// Select without reordering the returned guest tasks or mutating the caller's
+// history. Never-read jobs come first; ties retain provider order. Thereafter
+// the least recently attempted reads win, including failed reads. Only actual
+// calls advance scheduling, so cancellation cannot spend another job's turn.
+func (m *Monitor) selectVzdumpJobLogReads(instance string, jobs []models.BackupTask, upids map[string]string) map[string]bool {
+ type pendingRead struct {
+ key string
+ attempt time.Time
+ }
+ pending := make([]pendingRead, 0, len(jobs))
+ seen := make(map[string]bool, len(jobs))
+ m.mu.RLock()
+ for _, job := range jobs {
+ key := instance + "|" + upids[job.ID]
+ if seen[key] || m.vzdumpJobTaskCache[key].final {
+ continue
+ }
+ seen[key] = true
+ pending = append(pending, pendingRead{key: key, attempt: m.vzdumpJobTaskCache[key].lastAttempt})
+ }
+ m.mu.RUnlock()
+ sort.SliceStable(pending, func(i, j int) bool {
+ return pending[i].attempt.Before(pending[j].attempt)
+ })
+ selected := make(map[string]bool, min(len(pending), maxVzdumpJobLogFetchesPerPoll))
+ for _, read := range pending[:min(len(pending), maxVzdumpJobLogFetchesPerPoll)] {
+ selected[read.key] = true
+ }
+ return selected
+}
+
// Retain the last successful log observation when a non-final log is unread or
// unavailable. A freshly ended parent proves the guest is no longer running,
-// not whether it succeeded. Do not mutate the cache: only another accepted log
-// read can complete attribution and make the entry final.
+// not whether it succeeded. Do not mutate accepted task evidence: only another
+// accepted log read can complete attribution and make the entry final.
func cachedVzdumpGuestTasks(tasks []models.BackupTask, job models.BackupTask) []models.BackupTask {
out := append([]models.BackupTask(nil), tasks...)
if job.EndTime.IsZero() {
diff --git a/internal/monitoring/vzdump_cached_guest_test.go b/internal/monitoring/vzdump_cached_guest_test.go
index 1877a32697..12e5b25dc1 100644
--- a/internal/monitoring/vzdump_cached_guest_test.go
+++ b/internal/monitoring/vzdump_cached_guest_test.go
@@ -131,7 +131,10 @@ func TestVzdumpCachedGuestReadFailureDoesNotRenewRunningIntent(t *testing.T) {
require.False(t, active, "parent termination withdraws intent even while the cached observation is fresh")
}
checkStored(recovery.OutcomeUnknown, false)
- require.Equal(t, entry, m.vzdumpJobTaskCache[key], "fallback must not mutate/finalise accepted log data")
+ cached := m.vzdumpJobTaskCache[key]
+ require.True(t, cached.lastAttempt.After(entry.lastAttempt), "failed reads still advance scheduling")
+ entry.lastAttempt = cached.lastAttempt // scheduling is not guest observation evidence
+ require.Equal(t, entry, cached, "fallback must not mutate/finalise accepted log data")
require.Equal(t, 3, client.logRequests)
// Only an ordinary accepted log supplies the missing guest verdict.
diff --git a/internal/monitoring/vzdump_job_log_scheduling_test.go b/internal/monitoring/vzdump_job_log_scheduling_test.go
new file mode 100644
index 0000000000..6a972bc50f
--- /dev/null
+++ b/internal/monitoring/vzdump_job_log_scheduling_test.go
@@ -0,0 +1,183 @@
+package monitoring
+
+import (
+ "context"
+ "fmt"
+ "sync"
+ "testing"
+ "time"
+
+ "github.com/rcourtman/pulse-go-rewrite/internal/models"
+ "github.com/rcourtman/pulse-go-rewrite/pkg/proxmox"
+ "github.com/stretchr/testify/require"
+)
+
+type scheduledVzdumpClient struct {
+ PVEClientInterface
+ history proxmox.BackupTaskHistory
+ requests []string
+ read func(string) ([]proxmox.TaskLogLine, error)
+}
+
+func (c *scheduledVzdumpClient) GetBackupTaskHistory(context.Context, time.Time) (proxmox.BackupTaskHistory, error) {
+ return c.history, nil
+}
+
+func (c *scheduledVzdumpClient) GetTaskLog(_ context.Context, _, upid string) ([]proxmox.TaskLogLine, error) {
+ c.requests = append(c.requests, upid)
+ return c.read(upid)
+}
+
+func scheduledVzdumpJobs(count int, instance string) ([]models.BackupTask, map[string]string) {
+ jobs := make([]models.BackupTask, count)
+ upids := make(map[string]string, count)
+ start := time.Date(2026, 10, 29, 2, 0, 0, 0, time.UTC)
+ for i := range jobs {
+ upid := fmt.Sprintf("UPID:pve1:%08X:00000001:69016420:vzdump::root@pam:", i+1)
+ id := instance + "-" + upid
+ jobs[i] = models.BackupTask{ID: id, Instance: instance, Node: "pve1", Type: "vzdump", Status: "OK", StartTime: start, EndTime: start.Add(time.Minute)}
+ upids[id] = upid
+ }
+ return jobs, upids
+}
+
+func TestVzdumpJobLogSchedulingPersistentReadsDoNotStarveHistory(t *testing.T) {
+ for _, shape := range []string{"error", "incomplete", "empty", "capped"} {
+ t.Run(shape, func(t *testing.T) {
+ jobs, upids := scheduledVzdumpJobs(3*maxVzdumpJobLogFetchesPerPoll+1, "site")
+ original := append([]models.BackupTask(nil), jobs...)
+ lines := testLogLines("INFO: Starting Backup of VM 101 (qemu)")
+ if shape == "capped" {
+ for len(lines) < 19999 {
+ lines = append(lines, proxmox.TaskLogLine{LineNumber: int64(len(lines) + 1), Text: "INFO: progress"})
+ }
+ lines = append(lines, proxmox.TaskLogLine{LineNumber: 20000, Text: "TASK OK"})
+ }
+ client := &scheduledVzdumpClient{read: func(string) ([]proxmox.TaskLogLine, error) {
+ switch shape {
+ case "error":
+ return nil, fmt.Errorf("synthetic task log unavailable")
+ case "empty":
+ return nil, nil
+ default:
+ return lines, nil
+ }
+ }}
+ m := &Monitor{}
+ for cycle := 0; cycle < 4; cycle++ {
+ before := len(client.requests)
+ m.synthesizeVzdumpJobGuestTasks(context.Background(), "site", client, jobs, upids)
+ require.Len(t, client.requests[before:], maxVzdumpJobLogFetchesPerPoll)
+ }
+ counts := make(map[string]int)
+ for _, upid := range client.requests {
+ counts[upid]++
+ }
+ require.Len(t, counts, len(jobs), "persistent early failures must not hide later jobs")
+ for _, count := range counts {
+ require.LessOrEqual(t, count, 2, "retry only after every pending job has had a turn")
+ }
+ require.Equal(t, original, jobs, "selection must not reorder provider history")
+ for _, entry := range m.vzdumpJobTaskCache {
+ require.False(t, entry.final, "an attempt is not complete guest evidence")
+ require.False(t, entry.historyComplete)
+ }
+ })
+ }
+}
+
+func TestVzdumpJobLogSchedulingPollReachesLaterGuestResults(t *testing.T) {
+ jobs, upids := scheduledVzdumpJobs(maxVzdumpJobLogFetchesPerPoll+2, "site")
+ client := &scheduledVzdumpClient{history: proxmox.BackupTaskHistory{Nodes: []proxmox.BackupTaskHistoryNode{{Node: "pve1", Complete: true}}}}
+ for _, job := range jobs {
+ client.history.Tasks = append(client.history.Tasks, proxmox.Task{UPID: upids[job.ID], Node: job.Node, Type: job.Type, Status: job.Status, StartTime: job.StartTime.Unix(), EndTime: job.EndTime.Unix()})
+ }
+ good, failed := upids[jobs[len(jobs)-2].ID], upids[jobs[len(jobs)-1].ID]
+ client.read = func(upid string) ([]proxmox.TaskLogLine, error) {
+ switch upid {
+ case good:
+ return testLogLines("INFO: Starting Backup of VM 501 (qemu)", "INFO: Finished Backup of VM 501 (00:00:30)", "TASK OK"), nil
+ case failed:
+ return testLogLines("INFO: Starting Backup of VM 502 (lxc)", "ERROR: Backup of VM 502 failed - storage unavailable", "TASK ERROR: job errors"), nil
+ default:
+ return nil, fmt.Errorf("synthetic task log unavailable")
+ }
+ }
+ m := &Monitor{state: models.NewState()}
+ other := models.BackupTask{ID: "other-task", Instance: "other", VMID: 501, Status: "error"}
+ m.state.UpdateBackupTasksForInstance("other", []models.BackupTask{other})
+ m.pollBackupTasks(context.Background(), "site", client)
+ require.Len(t, m.state.GetBackupTasks(), len(jobs)+1, "no guest outcome can be inferred from an unread log")
+ m.pollBackupTasks(context.Background(), "site", client)
+ m.pollBackupTasks(context.Background(), "site", client)
+ byID := make(map[string]models.BackupTask)
+ for _, task := range m.state.GetBackupTasks() {
+ byID[task.ID] = task
+ }
+ require.Len(t, byID, len(jobs)+3)
+ require.Equal(t, other, byID[other.ID])
+ require.Equal(t, "OK", byID[jobs[len(jobs)-2].ID+"-vm501"].Status)
+ require.Equal(t, "error", byID[jobs[len(jobs)-1].ID+"-vm502"].Status)
+ require.Equal(t, "storage unavailable", byID[jobs[len(jobs)-1].ID+"-vm502"].Error)
+ for _, upid := range []string{good, failed} {
+ count := 0
+ for _, read := range client.requests {
+ if read == upid {
+ count++
+ }
+ }
+ require.Equal(t, 1, count, "complete terminal logs retain their immutable cache")
+ require.True(t, m.vzdumpJobTaskCache["site|"+upid].historyComplete)
+ }
+}
+
+func TestVzdumpJobLogSchedulingCancellationKeepsUnspentTurns(t *testing.T) {
+ jobs, upids := scheduledVzdumpJobs(maxVzdumpJobLogFetchesPerPoll+1, "site")
+ ctx, cancel := context.WithCancel(context.Background())
+ defer cancel()
+ client := &scheduledVzdumpClient{read: func(string) ([]proxmox.TaskLogLine, error) {
+ cancel()
+ return nil, context.Canceled
+ }}
+ m := &Monitor{}
+ m.synthesizeVzdumpJobGuestTasks(ctx, "site", client, jobs, upids)
+ require.Equal(t, []string{upids[jobs[0].ID]}, client.requests)
+ client.requests = nil
+ client.read = func(string) ([]proxmox.TaskLogLine, error) { return nil, nil }
+ m.synthesizeVzdumpJobGuestTasks(context.Background(), "site", client, jobs, upids)
+ require.Len(t, client.requests, maxVzdumpJobLogFetchesPerPoll)
+ require.NotContains(t, client.requests, upids[jobs[0].ID], "only an attempted read spends its turn")
+}
+
+func TestVzdumpJobLogSchedulingPrunesAndSeparatesConnections(t *testing.T) {
+ jobs, upids := scheduledVzdumpJobs(maxVzdumpJobLogFetchesPerPoll+1, "site")
+ client := &scheduledVzdumpClient{read: func(string) ([]proxmox.TaskLogLine, error) { return nil, nil }}
+ m := &Monitor{}
+ m.synthesizeVzdumpJobGuestTasks(context.Background(), "site", client, jobs, upids)
+ client.requests = nil
+ m.synthesizeVzdumpJobGuestTasks(context.Background(), "other", client, jobs, upids)
+ require.Equal(t, upids[jobs[0].ID], client.requests[0], "another connection has its own request turns")
+ m.synthesizeVzdumpJobGuestTasks(context.Background(), "site", client, nil, nil)
+ require.Len(t, m.vzdumpJobTaskCache, maxVzdumpJobLogFetchesPerPoll, "removed jobs discard scheduling state, not another connection")
+ client.requests = nil
+ m.synthesizeVzdumpJobGuestTasks(context.Background(), "site", client, jobs, upids)
+ require.Equal(t, upids[jobs[0].ID], client.requests[0], "a new history window has no stale attempt priority")
+}
+
+func TestVzdumpJobLogSchedulingConcurrentConnections(t *testing.T) {
+ m := &Monitor{}
+ var wg sync.WaitGroup
+ for i := 0; i < 3; i++ {
+ wg.Add(1)
+ go func(instance string) {
+ defer wg.Done()
+ jobs, upids := scheduledVzdumpJobs(maxVzdumpJobLogFetchesPerPoll+1, instance)
+ client := &scheduledVzdumpClient{read: func(string) ([]proxmox.TaskLogLine, error) { return nil, nil }}
+ for cycle := 0; cycle < 3; cycle++ {
+ m.synthesizeVzdumpJobGuestTasks(context.Background(), instance, client, jobs, upids)
+ }
+ }(fmt.Sprintf("site-%d", i))
+ }
+ wg.Wait()
+ require.Len(t, m.vzdumpJobTaskCache, 3*(maxVzdumpJobLogFetchesPerPoll+1))
+}