Real-time monitoring dashboard for Proxmox VE, PBS, Docker, Kubernetes, TrueNAS and vSphere. Self-hosted, with alerts for failed backups, full disks, unhealthy containers and offline nodes.
Go aialertsdashboarddockerdocker-monitoringgohomelabinfrastructure-monitoringkubernetesmonitoringmspproxmoxproxmox-backup-serverproxmox-mail-gatewayproxmox-veself-hostedtruenastypescriptvspherewebhooks
Rotate unfinished vzdump log reads within the existing allowance
pulse-triage[bot]58868f500e1c
diff --git a/docs/release-control/v6/internal/subsystems/monitoring.md b/docs/release-control/v6/internal/subsystems/monitoring.mdindex 0badb88278..eb5e7d2d41 100644--- a/docs/release-control/v6/internal/subsystems/monitoring.md+++ b/docs/release-control/v6/internal/subsystems/monitoring.md@@ -17,6 +17,28 @@## Purpose+### Unfinished backup logs cannot monopolise the read allowance++Multi-guest vzdump log collection keeps its 25-request cap per poll. Pending+jobs are selected by least recent attempt within the connection, with never-read+jobs first and provider order breaking ties. Failures, empty, capped and+incomplete logs spend their turn without becoming final; an unread older job+must not be hidden forever by the same failing prefix. Only an actual request+advances scheduling, not selection or cancellation. Selection leaves input and+guest output ordering unchanged, completed attributed logs remain cached, and+removed jobs discard their scheduling state with the existing cache pruning.++Scheduling metadata is not an accepted observation. Skipped/failed reads retain+the original guest evidence and its age; neither a retry nor an ended parent+supplies a missing guest verdict. No allowance, provider permission, task/result+schema, identity, protection rule or guest operation changes.+`vzdump_job_log_scheduling_test.go` checks bounded progress, poll-to-guest results,+immutable completion, cancellation, pruning and concurrent connection isolation.+The existing cached-guest tests retain their observation/intent assertions.+Desktop/narrow proof consumes actual poll frames through Backups Coverage,+including scoped task results without inventing a restore artifact or protection.++### Backup task results agree with their completion clocksThe PVE backup-history reader accepts a live task only with an absent/zero end@@ -4771,9 +4793,11 @@ named and default organizations. The custom event envelope stays unchanged.task listing queries `source=all` so running jobs are visible: a guestan in-progress job is currently backing up carries a `running`synthetic task, which is valid backup-intent evidence for- `resolveBackupIntentContext`. Finished job logs are immutable and are+ `resolveBackupIntentContext`. Completely attributed finished job logs are immutable and arefetched at most once per instance|UPID, with a per-cycle fetch cap soa historical backlog cannot stall the shared backup poll budget.+ Unfinished reads rotate by last attempt, including failures, so a+ persistent prefix cannot prevent later jobs from supplying guest results.Regression coverage: `TestParseVzdumpJobLogFinishedJob`,`TestPollBackupTasksSynthesizesJobGuestTasks`, and`TestPollBackupTasksRunningJobRefetchesAndSuppressesAlerts` indiff --git a/docs/release-control/v6/internal/subsystems/registry.json b/docs/release-control/v6/internal/subsystems/registry.jsonindex 260e235249..2f06c9eb7a 100644--- a/docs/release-control/v6/internal/subsystems/registry.json+++ b/docs/release-control/v6/internal/subsystems/registry.json@@ -6906,6 +6906,8 @@"internal/monitoring/pbs_protection_observation_test.go","internal/monitoring/pve_protection_observation_test.go","internal/monitoring/recovery_ingest_test.go",+ "internal/monitoring/vzdump_cached_guest_test.go",+ "internal/monitoring/vzdump_job_log_scheduling_test.go","internal/recovery/mapper/proxmox/mapper_test.go","internal/recovery/posture_test.go"]diff --git a/internal/monitoring/monitor_backups.go b/internal/monitoring/monitor_backups.goindex bb765d8281..de834f8bdf 100644--- a/internal/monitoring/monitor_backups.go+++ b/internal/monitoring/monitor_backups.go@@ -2701,11 +2701,14 @@ type vzdumpJobTaskCacheEntry struct {tasks []models.BackupTaskfinal boolhistoryComplete bool+ lastAttempt time.Time // scheduling metadata, not an accepted log observation}// maxVzdumpJobLogFetchesPerPoll bounds task-log requests per poll cycle so a// large backlog of historical job runs (e.g. right after startup) trickles in-// over a few cycles instead of stalling the shared backup-poll budget.+// over a few cycles instead of stalling the shared backup-poll budget. Pending+// reads rotate by last attempt so errors and incomplete logs cannot consume the+// entire allowance every cycle and permanently hide older guest results.const maxVzdumpJobLogFetchesPerPoll = 25// synthesizeVzdumpJobGuestTasks expands multi-guest vzdump job runs into one@@ -2721,6 +2724,7 @@ func (m *Monitor) synthesizeVzdumpJobGuestTasks(ctx context.Context, instanceNamnow := time.Now().UTC()seen := make(map[string]struct{}, len(jobTasks))+ fetchJobs := m.selectVzdumpJobLogReads(instanceName, jobTasks, jobUPIDs)var synthesized []models.BackupTaskfetches := 0@@ -2738,7 +2742,7 @@ func (m *Monitor) synthesizeVzdumpJobGuestTasks(ctx context.Context, instanceNamcontinue}- if fetches >= maxVzdumpJobLogFetchesPerPoll || ctx.Err() != nil {+ if !fetchJobs[key] || fetches >= maxVzdumpJobLogFetchesPerPoll || ctx.Err() != nil {// Serve a stale non-final entry if we have one; the next cycle// will catch up. The current parent can end a cached running// guest, but cannot supply its missing verdict or a fresh log read.@@ -2748,6 +2752,17 @@ func (m *Monitor) synthesizeVzdumpJobGuestTasks(ctx context.Context, instanceNamcontinue}fetches+++ delete(fetchJobs, key) // duplicate provider rows cannot spend the same turn twice+ // A failed or incomplete read still spent the allowance. Keep attempt+ // order separate from accepted observations so failure never freshens evidence.+ m.mu.Lock()+ if m.vzdumpJobTaskCache == nil {+ m.vzdumpJobTaskCache = make(map[string]vzdumpJobTaskCacheEntry)+ }+ attempted := m.vzdumpJobTaskCache[key]+ attempted.lastAttempt = now+ m.vzdumpJobTaskCache[key] = attempted+ m.mu.Unlock()logLines, err := client.GetTaskLog(ctx, jobTask.Node, upid)if err != nil {@@ -2771,6 +2786,7 @@ func (m *Monitor) synthesizeVzdumpJobGuestTasks(ctx context.Context, instanceNamtasks: guestTasks,final: historyComplete,historyComplete: historyComplete,+ lastAttempt: now,}m.mu.Unlock()synthesized = append(synthesized, guestTasks...)@@ -2780,10 +2796,41 @@ func (m *Monitor) synthesizeVzdumpJobGuestTasks(ctx context.Context, instanceNamreturn synthesized}+// Select without reordering the returned guest tasks or mutating the caller's+// history. Never-read jobs come first; ties retain provider order. Thereafter+// the least recently attempted reads win, including failed reads. Only actual+// calls advance scheduling, so cancellation cannot spend another job's turn.+func (m *Monitor) selectVzdumpJobLogReads(instance string, jobs []models.BackupTask, upids map[string]string) map[string]bool {+ type pendingRead struct {+ key string+ attempt time.Time+ }+ pending := make([]pendingRead, 0, len(jobs))+ seen := make(map[string]bool, len(jobs))+ m.mu.RLock()+ for _, job := range jobs {+ key := instance + "|" + upids[job.ID]+ if seen[key] || m.vzdumpJobTaskCache[key].final {+ continue+ }+ seen[key] = true+ pending = append(pending, pendingRead{key: key, attempt: m.vzdumpJobTaskCache[key].lastAttempt})+ }+ m.mu.RUnlock()+ sort.SliceStable(pending, func(i, j int) bool {+ return pending[i].attempt.Before(pending[j].attempt)+ })+ selected := make(map[string]bool, min(len(pending), maxVzdumpJobLogFetchesPerPoll))+ for _, read := range pending[:min(len(pending), maxVzdumpJobLogFetchesPerPoll)] {+ selected[read.key] = true+ }+ return selected+}+// Retain the last successful log observation when a non-final log is unread or// unavailable. A freshly ended parent proves the guest is no longer running,-// not whether it succeeded. Do not mutate the cache: only another accepted log-// read can complete attribution and make the entry final.+// not whether it succeeded. Do not mutate accepted task evidence: only another+// accepted log read can complete attribution and make the entry final.func cachedVzdumpGuestTasks(tasks []models.BackupTask, job models.BackupTask) []models.BackupTask {out := append([]models.BackupTask(nil), tasks...)if job.EndTime.IsZero() {diff --git a/internal/monitoring/vzdump_cached_guest_test.go b/internal/monitoring/vzdump_cached_guest_test.goindex 1877a32697..12e5b25dc1 100644--- a/internal/monitoring/vzdump_cached_guest_test.go+++ b/internal/monitoring/vzdump_cached_guest_test.go@@ -131,7 +131,10 @@ func TestVzdumpCachedGuestReadFailureDoesNotRenewRunningIntent(t *testing.T) {require.False(t, active, "parent termination withdraws intent even while the cached observation is fresh")}checkStored(recovery.OutcomeUnknown, false)- require.Equal(t, entry, m.vzdumpJobTaskCache[key], "fallback must not mutate/finalise accepted log data")+ cached := m.vzdumpJobTaskCache[key]+ require.True(t, cached.lastAttempt.After(entry.lastAttempt), "failed reads still advance scheduling")+ entry.lastAttempt = cached.lastAttempt // scheduling is not guest observation evidence+ require.Equal(t, entry, cached, "fallback must not mutate/finalise accepted log data")require.Equal(t, 3, client.logRequests)// Only an ordinary accepted log supplies the missing guest verdict.diff --git a/internal/monitoring/vzdump_job_log_scheduling_test.go b/internal/monitoring/vzdump_job_log_scheduling_test.gonew file mode 100644index 0000000000..6a972bc50f--- /dev/null+++ b/internal/monitoring/vzdump_job_log_scheduling_test.go@@ -0,0 +1,183 @@+package monitoring++import (+ "context"+ "fmt"+ "sync"+ "testing"+ "time"++ "github.com/rcourtman/pulse-go-rewrite/internal/models"+ "github.com/rcourtman/pulse-go-rewrite/pkg/proxmox"+ "github.com/stretchr/testify/require"+)++type scheduledVzdumpClient struct {+ PVEClientInterface+ history proxmox.BackupTaskHistory+ requests []string+ read func(string) ([]proxmox.TaskLogLine, error)+}++func (c *scheduledVzdumpClient) GetBackupTaskHistory(context.Context, time.Time) (proxmox.BackupTaskHistory, error) {+ return c.history, nil+}++func (c *scheduledVzdumpClient) GetTaskLog(_ context.Context, _, upid string) ([]proxmox.TaskLogLine, error) {+ c.requests = append(c.requests, upid)+ return c.read(upid)+}++func scheduledVzdumpJobs(count int, instance string) ([]models.BackupTask, map[string]string) {+ jobs := make([]models.BackupTask, count)+ upids := make(map[string]string, count)+ start := time.Date(2026, 10, 29, 2, 0, 0, 0, time.UTC)+ for i := range jobs {+ upid := fmt.Sprintf("UPID:pve1:%08X:00000001:69016420:vzdump::root@pam:", i+1)+ id := instance + "-" + upid+ jobs[i] = models.BackupTask{ID: id, Instance: instance, Node: "pve1", Type: "vzdump", Status: "OK", StartTime: start, EndTime: start.Add(time.Minute)}+ upids[id] = upid+ }+ return jobs, upids+}++func TestVzdumpJobLogSchedulingPersistentReadsDoNotStarveHistory(t *testing.T) {+ for _, shape := range []string{"error", "incomplete", "empty", "capped"} {+ t.Run(shape, func(t *testing.T) {+ jobs, upids := scheduledVzdumpJobs(3*maxVzdumpJobLogFetchesPerPoll+1, "site")+ original := append([]models.BackupTask(nil), jobs...)+ lines := testLogLines("INFO: Starting Backup of VM 101 (qemu)")+ if shape == "capped" {+ for len(lines) < 19999 {+ lines = append(lines, proxmox.TaskLogLine{LineNumber: int64(len(lines) + 1), Text: "INFO: progress"})+ }+ lines = append(lines, proxmox.TaskLogLine{LineNumber: 20000, Text: "TASK OK"})+ }+ client := &scheduledVzdumpClient{read: func(string) ([]proxmox.TaskLogLine, error) {+ switch shape {+ case "error":+ return nil, fmt.Errorf("synthetic task log unavailable")+ case "empty":+ return nil, nil+ default:+ return lines, nil+ }+ }}+ m := &Monitor{}+ for cycle := 0; cycle < 4; cycle++ {+ before := len(client.requests)+ m.synthesizeVzdumpJobGuestTasks(context.Background(), "site", client, jobs, upids)+ require.Len(t, client.requests[before:], maxVzdumpJobLogFetchesPerPoll)+ }+ counts := make(map[string]int)+ for _, upid := range client.requests {+ counts[upid]+++ }+ require.Len(t, counts, len(jobs), "persistent early failures must not hide later jobs")+ for _, count := range counts {+ require.LessOrEqual(t, count, 2, "retry only after every pending job has had a turn")+ }+ require.Equal(t, original, jobs, "selection must not reorder provider history")+ for _, entry := range m.vzdumpJobTaskCache {+ require.False(t, entry.final, "an attempt is not complete guest evidence")+ require.False(t, entry.historyComplete)+ }+ })+ }+}++func TestVzdumpJobLogSchedulingPollReachesLaterGuestResults(t *testing.T) {+ jobs, upids := scheduledVzdumpJobs(maxVzdumpJobLogFetchesPerPoll+2, "site")+ client := &scheduledVzdumpClient{history: proxmox.BackupTaskHistory{Nodes: []proxmox.BackupTaskHistoryNode{{Node: "pve1", Complete: true}}}}+ for _, job := range jobs {+ client.history.Tasks = append(client.history.Tasks, proxmox.Task{UPID: upids[job.ID], Node: job.Node, Type: job.Type, Status: job.Status, StartTime: job.StartTime.Unix(), EndTime: job.EndTime.Unix()})+ }+ good, failed := upids[jobs[len(jobs)-2].ID], upids[jobs[len(jobs)-1].ID]+ client.read = func(upid string) ([]proxmox.TaskLogLine, error) {+ switch upid {+ case good:+ return testLogLines("INFO: Starting Backup of VM 501 (qemu)", "INFO: Finished Backup of VM 501 (00:00:30)", "TASK OK"), nil+ case failed:+ return testLogLines("INFO: Starting Backup of VM 502 (lxc)", "ERROR: Backup of VM 502 failed - storage unavailable", "TASK ERROR: job errors"), nil+ default:+ return nil, fmt.Errorf("synthetic task log unavailable")+ }+ }+ m := &Monitor{state: models.NewState()}+ other := models.BackupTask{ID: "other-task", Instance: "other", VMID: 501, Status: "error"}+ m.state.UpdateBackupTasksForInstance("other", []models.BackupTask{other})+ m.pollBackupTasks(context.Background(), "site", client)+ require.Len(t, m.state.GetBackupTasks(), len(jobs)+1, "no guest outcome can be inferred from an unread log")+ m.pollBackupTasks(context.Background(), "site", client)+ m.pollBackupTasks(context.Background(), "site", client)+ byID := make(map[string]models.BackupTask)+ for _, task := range m.state.GetBackupTasks() {+ byID[task.ID] = task+ }+ require.Len(t, byID, len(jobs)+3)+ require.Equal(t, other, byID[other.ID])+ require.Equal(t, "OK", byID[jobs[len(jobs)-2].ID+"-vm501"].Status)+ require.Equal(t, "error", byID[jobs[len(jobs)-1].ID+"-vm502"].Status)+ require.Equal(t, "storage unavailable", byID[jobs[len(jobs)-1].ID+"-vm502"].Error)+ for _, upid := range []string{good, failed} {+ count := 0+ for _, read := range client.requests {+ if read == upid {+ count+++ }+ }+ require.Equal(t, 1, count, "complete terminal logs retain their immutable cache")+ require.True(t, m.vzdumpJobTaskCache["site|"+upid].historyComplete)+ }+}++func TestVzdumpJobLogSchedulingCancellationKeepsUnspentTurns(t *testing.T) {+ jobs, upids := scheduledVzdumpJobs(maxVzdumpJobLogFetchesPerPoll+1, "site")+ ctx, cancel := context.WithCancel(context.Background())+ defer cancel()+ client := &scheduledVzdumpClient{read: func(string) ([]proxmox.TaskLogLine, error) {+ cancel()+ return nil, context.Canceled+ }}+ m := &Monitor{}+ m.synthesizeVzdumpJobGuestTasks(ctx, "site", client, jobs, upids)+ require.Equal(t, []string{upids[jobs[0].ID]}, client.requests)+ client.requests = nil+ client.read = func(string) ([]proxmox.TaskLogLine, error) { return nil, nil }+ m.synthesizeVzdumpJobGuestTasks(context.Background(), "site", client, jobs, upids)+ require.Len(t, client.requests, maxVzdumpJobLogFetchesPerPoll)+ require.NotContains(t, client.requests, upids[jobs[0].ID], "only an attempted read spends its turn")+}++func TestVzdumpJobLogSchedulingPrunesAndSeparatesConnections(t *testing.T) {+ jobs, upids := scheduledVzdumpJobs(maxVzdumpJobLogFetchesPerPoll+1, "site")+ client := &scheduledVzdumpClient{read: func(string) ([]proxmox.TaskLogLine, error) { return nil, nil }}+ m := &Monitor{}+ m.synthesizeVzdumpJobGuestTasks(context.Background(), "site", client, jobs, upids)+ client.requests = nil+ m.synthesizeVzdumpJobGuestTasks(context.Background(), "other", client, jobs, upids)+ require.Equal(t, upids[jobs[0].ID], client.requests[0], "another connection has its own request turns")+ m.synthesizeVzdumpJobGuestTasks(context.Background(), "site", client, nil, nil)+ require.Len(t, m.vzdumpJobTaskCache, maxVzdumpJobLogFetchesPerPoll, "removed jobs discard scheduling state, not another connection")+ client.requests = nil+ m.synthesizeVzdumpJobGuestTasks(context.Background(), "site", client, jobs, upids)+ require.Equal(t, upids[jobs[0].ID], client.requests[0], "a new history window has no stale attempt priority")+}++func TestVzdumpJobLogSchedulingConcurrentConnections(t *testing.T) {+ m := &Monitor{}+ var wg sync.WaitGroup+ for i := 0; i < 3; i++ {+ wg.Add(1)+ go func(instance string) {+ defer wg.Done()+ jobs, upids := scheduledVzdumpJobs(maxVzdumpJobLogFetchesPerPoll+1, instance)+ client := &scheduledVzdumpClient{read: func(string) ([]proxmox.TaskLogLine, error) { return nil, nil }}+ for cycle := 0; cycle < 3; cycle++ {+ m.synthesizeVzdumpJobGuestTasks(context.Background(), instance, client, jobs, upids)+ }+ }(fmt.Sprintf("site-%d", i))+ }+ wg.Wait()+ require.Len(t, m.vzdumpJobTaskCache, 3*(maxVzdumpJobLogFetchesPerPoll+1))+}