Real-time monitoring dashboard for Proxmox VE, PBS, Docker, Kubernetes, TrueNAS and vSphere. Self-hosted, with alerts for failed backups, full disks, unhealthy containers and offline nodes.
Go aialertsdashboarddockerdocker-monitoringgohomelabinfrastructure-monitoringkubernetesmonitoringmspproxmoxproxmox-backup-serverproxmox-mail-gatewayproxmox-veself-hostedtruenastypescriptvspherewebhooks
Keep unread vzdump guest logs from renewing running state
pulse-triage[bot]eb13a00e4fcd
diff --git a/docs/release-control/v6/internal/subsystems/monitoring.md b/docs/release-control/v6/internal/subsystems/monitoring.mdindex 1461d97c5b..97a578c53d 100644--- a/docs/release-control/v6/internal/subsystems/monitoring.md+++ b/docs/release-control/v6/internal/subsystems/monitoring.md@@ -40,6 +40,17 @@ has an explicit result and the existing contiguous, bounded terminal-log checkspass. An ordinary later complete read repairs the result under the same taskidentity; only then may a finished log be cached as final.+A non-final cached log follows the same rule when a read fails, is cancelled,+or is deferred by the per-poll request bound. The current parent end withdraws+cached running state without inventing a guest verdict. Explicit cached results+remain intact; the cache stays incomplete and retryable. These fallback rows+retain the last accepted log observation time instead of renewing backup-intent+freshness on an unread log. Successful reads record that time in the cache;+complete immutable logs retain their existing final-cache behaviour.+`TestVzdumpCachedGuest*` checks live intent expiry, ended-parent fallback,+connection isolation, mixed results, durable history, request/cancellation+bounds and ordinary complete-read recovery.+`TestVzdumpGuestResult*` covers ended/running jobs, truncated/gapped/capped logs,explicit mixed results, retries and cache finalisation through the real poller,state JSON and recovery mapper. These are offline log fixtures, not native backupdiff --git a/internal/monitoring/monitor_backups.go b/internal/monitoring/monitor_backups.goindex 939f05ef2c..ea48d4b8f4 100644--- a/internal/monitoring/monitor_backups.go+++ b/internal/monitoring/monitor_backups.go@@ -2472,9 +2472,10 @@ func (m *Monitor) synthesizeVzdumpJobGuestTasks(ctx context.Context, instanceNamif fetches >= maxVzdumpJobLogFetchesPerPoll || ctx.Err() != nil {// Serve a stale non-final entry if we have one; the next cycle- // will catch up.+ // will catch up. The current parent can end a cached running+ // guest, but cannot supply its missing verdict or a fresh log read.if cached {- synthesized = append(synthesized, refreshObservedAt(entry.tasks, now)...)+ synthesized = append(synthesized, cachedVzdumpGuestTasks(entry.tasks, jobTask)...)}continue}@@ -2487,12 +2488,12 @@ func (m *Monitor) synthesizeVzdumpJobGuestTasks(ctx context.Context, instanceNamStr("upid", upid).Msg("failed to fetch vzdump job task log; per-guest status unavailable this cycle")if cached {- synthesized = append(synthesized, refreshObservedAt(entry.tasks, now)...)+ synthesized = append(synthesized, cachedVzdumpGuestTasks(entry.tasks, jobTask)...)}continue}- guestTasks := parseVzdumpJobLog(jobTask, upid, logLines)+ guestTasks := refreshObservedAt(parseVzdumpJobLog(jobTask, upid, logLines), now)historyComplete := backupGuestLogComplete(jobTask, logLines, guestTasks)m.mu.Lock()if m.vzdumpJobTaskCache == nil {@@ -2504,13 +2505,34 @@ func (m *Monitor) synthesizeVzdumpJobGuestTasks(ctx context.Context, instanceNamhistoryComplete: historyComplete,}m.mu.Unlock()- synthesized = append(synthesized, refreshObservedAt(guestTasks, now)...)+ synthesized = append(synthesized, guestTasks...)}m.pruneVzdumpJobTaskCache(instanceName, seen)return synthesized}+// Retain the last successful log observation when a non-final log is unread or+// unavailable. A freshly ended parent proves the guest is no longer running,+// not whether it succeeded. Do not mutate the cache: only another accepted log+// read can complete attribution and make the entry final.+func cachedVzdumpGuestTasks(tasks []models.BackupTask, job models.BackupTask) []models.BackupTask {+ out := append([]models.BackupTask(nil), tasks...)+ if job.EndTime.IsZero() {+ return out+ }+ for i := range out {+ if out[i].Status == "running" {+ out[i].Status = "unknown"+ out[i].Error = "backup job ended without an observed guest result"+ }+ if out[i].EndTime.IsZero() {+ out[i].EndTime = job.EndTime+ }+ }+ return out+}+// pruneVzdumpJobTaskCache drops cached job entries for this instance whose// UPIDs are no longer present in the task list (fell out of the window).func (m *Monitor) pruneVzdumpJobTaskCache(instanceName string, seen map[string]struct{}) {diff --git a/internal/monitoring/vzdump_cached_guest_test.go b/internal/monitoring/vzdump_cached_guest_test.gonew file mode 100644index 0000000000..0931eae15a--- /dev/null+++ b/internal/monitoring/vzdump_cached_guest_test.go@@ -0,0 +1,175 @@+package monitoring++import (+ "context"+ "fmt"+ "testing"+ "time"++ "github.com/rcourtman/pulse-go-rewrite/internal/config"+ "github.com/rcourtman/pulse-go-rewrite/internal/models"+ "github.com/rcourtman/pulse-go-rewrite/internal/recovery"+ recoverymanager "github.com/rcourtman/pulse-go-rewrite/internal/recovery/manager"+ "github.com/rcourtman/pulse-go-rewrite/pkg/proxmox"+ "github.com/stretchr/testify/require"+)++type cachedVzdumpClient struct {+ jobBackupTaskClient+ logErr error+}++func (c *cachedVzdumpClient) GetTaskLog(context.Context, string, string) ([]proxmox.TaskLogLine, error) {+ c.logRequests+++ return c.logLines, c.logErr+}++func TestVzdumpCachedGuestReadFailureDoesNotRenewRunningIntent(t *testing.T) {+ for _, failure := range []string{"503 unavailable", "403 denied"} {+ for _, parentStatus := range []string{"OK", "job errors"} {+ t.Run(failure+"/"+parentStatus, func(t *testing.T) {+ ctx := context.Background()+ job, lines := vzdumpGuestResultFixture()+ client := &cachedVzdumpClient{jobBackupTaskClient: jobBackupTaskClient{+ task: proxmox.Task{UPID: testJobUPID, Node: job.Node, Type: "vzdump", StartTime: job.StartTime.Unix()},+ logLines: lines,+ }}+ manager := recoverymanager.New(config.NewMultiTenantPersistence(t.TempDir()))+ m := &Monitor{state: models.NewState(), recoveryManager: manager}+ store, err := manager.StoreForOrg("default")+ require.NoError(t, err)+ defer store.Close()+ other := models.BackupTask{ID: "other-job", Instance: "other", Node: job.Node, VMID: 103, Status: "running", ObservedAt: time.Now()}+ m.state.UpdateBackupTasksForInstance("other", []models.BackupTask{other})++ guests := func() []models.BackupTask {+ var result []models.BackupTask+ for _, task := range m.state.GetBackupTasks() {+ if task.Instance == "site" && task.VMID > 0 {+ result = append(result, task)+ }+ if task.Instance == "other" {+ require.Equal(t, other, task, "another connection must remain untouched")+ }+ }+ return result+ }+ checkStored := func(last recovery.Outcome, complete bool) {+ points, total, err := store.ListPoints(ctx, recovery.ListPointsOptions{Kind: recovery.KindOther})+ require.NoError(t, err)+ require.Equal(t, 4, total, "parent and three guest identities survive every poll")+ for _, point := range points {+ switch point.ID {+ case "pve-task:" + job.ID:+ require.Equal(t, complete, point.Details["guestTaskDetailsComplete"])+ case "pve-task:" + job.ID + "-vm101":+ require.Equal(t, recovery.OutcomeSuccess, point.Outcome)+ case "pve-task:" + job.ID + "-vm102":+ require.Equal(t, recovery.OutcomeFailed, point.Outcome)+ case "pve-task:" + job.ID + "-vm103":+ require.Equal(t, last, point.Outcome)+ default:+ t.Fatalf("unexpected point: %s", point.ID)+ }+ }+ }+ m.pollBackupTasks(ctx, "site", client)+ assertVzdumpGuestResults(t, guests(), "running")+ checkStored(recovery.OutcomeRunning, false)+ _, active := m.resolveBackupIntentContext("", "site", job.Node, 103, time.Now())+ require.True(t, active, "an accepted live log still supplies backup intent")++ // Simulate passage of the evidence lease without a timing-dependent sleep.+ key := "site|" + testJobUPID+ entry := m.vzdumpJobTaskCache[key]+ observed := time.Now().UTC().Add(-2 * backupIntentEvidenceMaxAge)+ for i := range entry.tasks {+ require.False(t, entry.tasks[i].ObservedAt.IsZero(), "cache must retain its accepted log observation")+ entry.tasks[i].ObservedAt = observed+ }+ m.vzdumpJobTaskCache[key] = entry+ client.logErr = fmt.Errorf("task log %s", failure)+ m.pollBackupTasks(ctx, "site", client)+ assertVzdumpGuestResults(t, guests(), "running")+ for _, task := range guests() {+ require.Equal(t, observed, task.ObservedAt, "failed reads cannot freshen the last log observation")+ }+ _, active = m.resolveBackupIntentContext("", "site", job.Node, 103, time.Now())+ require.False(t, active, "unread live logs must not indefinitely renew alert deferral")++ client.task.EndTime = job.EndTime.Unix()+ client.task.Status = parentStatus+ m.pollBackupTasks(ctx, "site", client)+ assertVzdumpGuestResults(t, guests(), "unknown")+ for _, task := range guests() {+ require.Equal(t, observed, task.ObservedAt)+ require.False(t, task.EndTime.IsZero(), "ended parent must not leave an apparently active guest")+ if task.VMID == 103 {+ require.Equal(t, job.EndTime, task.EndTime)+ require.Equal(t, "backup job ended without an observed guest result", task.Error)+ }+ _, active = m.resolveBackupIntentContext("", "site", job.Node, task.VMID, observed)+ require.False(t, active, "parent termination withdraws intent even while the cached observation is fresh")+ }+ checkStored(recovery.OutcomeUnknown, false)+ require.Equal(t, entry, m.vzdumpJobTaskCache[key], "fallback must not mutate/finalise accepted log data")+ require.Equal(t, 3, client.logRequests)++ // Only an ordinary accepted log supplies the missing guest verdict.+ client.logErr = nil+ client.logLines = append(lines, proxmox.TaskLogLine{LineNumber: 6, Text: "INFO: Finished Backup of VM 103 (00:02:00)"}, proxmox.TaskLogLine{LineNumber: 7, Text: "TASK ERROR: job errors"})+ m.pollBackupTasks(ctx, "site", client)+ assertVzdumpGuestResults(t, guests(), "OK")+ checkStored(recovery.OutcomeSuccess, true)+ require.True(t, m.vzdumpJobTaskCache[key].final)+ for _, task := range guests() {+ require.True(t, task.ObservedAt.After(observed))+ }+ m.pollBackupTasks(ctx, "site", client)+ require.Equal(t, 4, client.logRequests, "complete immutable logs still use the final cache")+ checkStored(recovery.OutcomeSuccess, true)+ })+ }+ }+}++func TestVzdumpCachedGuestUnreadLogUsesCurrentParent(t *testing.T) {+ for _, blocked := range []string{"budget", "cancelled"} {+ t.Run(blocked, func(t *testing.T) {+ job, lines := vzdumpGuestResultFixture()+ running := job+ running.EndTime = time.Time{}+ observed := time.Now().UTC().Add(-time.Minute)+ entry := vzdumpJobTaskCacheEntry{tasks: refreshObservedAt(parseVzdumpJobLog(running, testJobUPID, lines), observed)}+ m := &Monitor{vzdumpJobTaskCache: map[string]vzdumpJobTaskCacheEntry{"site|" + testJobUPID: entry}}+ client := &cachedVzdumpClient{}+ ctx := context.Background()+ jobs := []models.BackupTask{}+ upids := map[string]string{job.ID: testJobUPID}+ if blocked == "budget" {+ for i := 0; i < maxVzdumpJobLogFetchesPerPoll; i++ {+ filler := models.BackupTask{ID: fmt.Sprintf("filler-%d", i)}+ jobs = append(jobs, filler)+ upids[filler.ID] = filler.ID+ }+ } else {+ cancelled, cancel := context.WithCancel(ctx)+ cancel()+ ctx = cancelled+ }+ jobs = append(jobs, job)+ tasks := m.synthesizeVzdumpJobGuestTasks(ctx, "site", client, jobs, upids)+ assertVzdumpGuestResults(t, tasks, "unknown")+ for _, task := range tasks {+ require.Equal(t, observed, task.ObservedAt, "an unread log is not a fresh guest observation")+ require.False(t, task.EndTime.IsZero())+ }+ require.Equal(t, entry, m.vzdumpJobTaskCache["site|"+testJobUPID])+ wantRequests := 0+ if blocked == "budget" {+ wantRequests = maxVzdumpJobLogFetchesPerPoll+ }+ require.Equal(t, wantRequests, client.logRequests, "repair must not bypass request or cancellation bounds")+ })+ }+}