Skip to content

armgit / Pulse

Public Imported from GitHub
0

Real-time monitoring dashboard for Proxmox VE, PBS, Docker, Kubernetes, TrueNAS and vSphere. Self-hosted, with alerts for failed backups, full disks, unhealthy containers and offline nodes.

Keep unread vzdump guest logs from renewing running state

pulse-triage[bot]eb13a00e4fcd
diff --git a/docs/release-control/v6/internal/subsystems/monitoring.md b/docs/release-control/v6/internal/subsystems/monitoring.md
index 1461d97c5b..97a578c53d 100644
--- a/docs/release-control/v6/internal/subsystems/monitoring.md
+++ b/docs/release-control/v6/internal/subsystems/monitoring.md
@@ -40,6 +40,17 @@ has an explicit result and the existing contiguous, bounded terminal-log checks
pass. An ordinary later complete read repairs the result under the same task
identity; only then may a finished log be cached as final.
+A non-final cached log follows the same rule when a read fails, is cancelled,
+or is deferred by the per-poll request bound. The current parent end withdraws
+cached running state without inventing a guest verdict. Explicit cached results
+remain intact; the cache stays incomplete and retryable. These fallback rows
+retain the last accepted log observation time instead of renewing backup-intent
+freshness on an unread log. Successful reads record that time in the cache;
+complete immutable logs retain their existing final-cache behaviour.
+`TestVzdumpCachedGuest*` checks live intent expiry, ended-parent fallback,
+connection isolation, mixed results, durable history, request/cancellation
+bounds and ordinary complete-read recovery.
+
`TestVzdumpGuestResult*` covers ended/running jobs, truncated/gapped/capped logs,
explicit mixed results, retries and cache finalisation through the real poller,
state JSON and recovery mapper. These are offline log fixtures, not native backup
diff --git a/internal/monitoring/monitor_backups.go b/internal/monitoring/monitor_backups.go
index 939f05ef2c..ea48d4b8f4 100644
--- a/internal/monitoring/monitor_backups.go
+++ b/internal/monitoring/monitor_backups.go
@@ -2472,9 +2472,10 @@ func (m *Monitor) synthesizeVzdumpJobGuestTasks(ctx context.Context, instanceNam
if fetches >= maxVzdumpJobLogFetchesPerPoll || ctx.Err() != nil {
// Serve a stale non-final entry if we have one; the next cycle
- // will catch up.
+ // will catch up. The current parent can end a cached running
+ // guest, but cannot supply its missing verdict or a fresh log read.
if cached {
- synthesized = append(synthesized, refreshObservedAt(entry.tasks, now)...)
+ synthesized = append(synthesized, cachedVzdumpGuestTasks(entry.tasks, jobTask)...)
}
continue
}
@@ -2487,12 +2488,12 @@ func (m *Monitor) synthesizeVzdumpJobGuestTasks(ctx context.Context, instanceNam
Str("upid", upid).
Msg("failed to fetch vzdump job task log; per-guest status unavailable this cycle")
if cached {
- synthesized = append(synthesized, refreshObservedAt(entry.tasks, now)...)
+ synthesized = append(synthesized, cachedVzdumpGuestTasks(entry.tasks, jobTask)...)
}
continue
}
- guestTasks := parseVzdumpJobLog(jobTask, upid, logLines)
+ guestTasks := refreshObservedAt(parseVzdumpJobLog(jobTask, upid, logLines), now)
historyComplete := backupGuestLogComplete(jobTask, logLines, guestTasks)
m.mu.Lock()
if m.vzdumpJobTaskCache == nil {
@@ -2504,13 +2505,34 @@ func (m *Monitor) synthesizeVzdumpJobGuestTasks(ctx context.Context, instanceNam
historyComplete: historyComplete,
}
m.mu.Unlock()
- synthesized = append(synthesized, refreshObservedAt(guestTasks, now)...)
+ synthesized = append(synthesized, guestTasks...)
}
m.pruneVzdumpJobTaskCache(instanceName, seen)
return synthesized
}
+// Retain the last successful log observation when a non-final log is unread or
+// unavailable. A freshly ended parent proves the guest is no longer running,
+// not whether it succeeded. Do not mutate the cache: only another accepted log
+// read can complete attribution and make the entry final.
+func cachedVzdumpGuestTasks(tasks []models.BackupTask, job models.BackupTask) []models.BackupTask {
+ out := append([]models.BackupTask(nil), tasks...)
+ if job.EndTime.IsZero() {
+ return out
+ }
+ for i := range out {
+ if out[i].Status == "running" {
+ out[i].Status = "unknown"
+ out[i].Error = "backup job ended without an observed guest result"
+ }
+ if out[i].EndTime.IsZero() {
+ out[i].EndTime = job.EndTime
+ }
+ }
+ return out
+}
+
// pruneVzdumpJobTaskCache drops cached job entries for this instance whose
// UPIDs are no longer present in the task list (fell out of the window).
func (m *Monitor) pruneVzdumpJobTaskCache(instanceName string, seen map[string]struct{}) {
diff --git a/internal/monitoring/vzdump_cached_guest_test.go b/internal/monitoring/vzdump_cached_guest_test.go
new file mode 100644
index 0000000000..0931eae15a
--- /dev/null
+++ b/internal/monitoring/vzdump_cached_guest_test.go
@@ -0,0 +1,175 @@
+package monitoring
+
+import (
+ "context"
+ "fmt"
+ "testing"
+ "time"
+
+ "github.com/rcourtman/pulse-go-rewrite/internal/config"
+ "github.com/rcourtman/pulse-go-rewrite/internal/models"
+ "github.com/rcourtman/pulse-go-rewrite/internal/recovery"
+ recoverymanager "github.com/rcourtman/pulse-go-rewrite/internal/recovery/manager"
+ "github.com/rcourtman/pulse-go-rewrite/pkg/proxmox"
+ "github.com/stretchr/testify/require"
+)
+
+type cachedVzdumpClient struct {
+ jobBackupTaskClient
+ logErr error
+}
+
+func (c *cachedVzdumpClient) GetTaskLog(context.Context, string, string) ([]proxmox.TaskLogLine, error) {
+ c.logRequests++
+ return c.logLines, c.logErr
+}
+
+func TestVzdumpCachedGuestReadFailureDoesNotRenewRunningIntent(t *testing.T) {
+ for _, failure := range []string{"503 unavailable", "403 denied"} {
+ for _, parentStatus := range []string{"OK", "job errors"} {
+ t.Run(failure+"/"+parentStatus, func(t *testing.T) {
+ ctx := context.Background()
+ job, lines := vzdumpGuestResultFixture()
+ client := &cachedVzdumpClient{jobBackupTaskClient: jobBackupTaskClient{
+ task: proxmox.Task{UPID: testJobUPID, Node: job.Node, Type: "vzdump", StartTime: job.StartTime.Unix()},
+ logLines: lines,
+ }}
+ manager := recoverymanager.New(config.NewMultiTenantPersistence(t.TempDir()))
+ m := &Monitor{state: models.NewState(), recoveryManager: manager}
+ store, err := manager.StoreForOrg("default")
+ require.NoError(t, err)
+ defer store.Close()
+ other := models.BackupTask{ID: "other-job", Instance: "other", Node: job.Node, VMID: 103, Status: "running", ObservedAt: time.Now()}
+ m.state.UpdateBackupTasksForInstance("other", []models.BackupTask{other})
+
+ guests := func() []models.BackupTask {
+ var result []models.BackupTask
+ for _, task := range m.state.GetBackupTasks() {
+ if task.Instance == "site" && task.VMID > 0 {
+ result = append(result, task)
+ }
+ if task.Instance == "other" {
+ require.Equal(t, other, task, "another connection must remain untouched")
+ }
+ }
+ return result
+ }
+ checkStored := func(last recovery.Outcome, complete bool) {
+ points, total, err := store.ListPoints(ctx, recovery.ListPointsOptions{Kind: recovery.KindOther})
+ require.NoError(t, err)
+ require.Equal(t, 4, total, "parent and three guest identities survive every poll")
+ for _, point := range points {
+ switch point.ID {
+ case "pve-task:" + job.ID:
+ require.Equal(t, complete, point.Details["guestTaskDetailsComplete"])
+ case "pve-task:" + job.ID + "-vm101":
+ require.Equal(t, recovery.OutcomeSuccess, point.Outcome)
+ case "pve-task:" + job.ID + "-vm102":
+ require.Equal(t, recovery.OutcomeFailed, point.Outcome)
+ case "pve-task:" + job.ID + "-vm103":
+ require.Equal(t, last, point.Outcome)
+ default:
+ t.Fatalf("unexpected point: %s", point.ID)
+ }
+ }
+ }
+ m.pollBackupTasks(ctx, "site", client)
+ assertVzdumpGuestResults(t, guests(), "running")
+ checkStored(recovery.OutcomeRunning, false)
+ _, active := m.resolveBackupIntentContext("", "site", job.Node, 103, time.Now())
+ require.True(t, active, "an accepted live log still supplies backup intent")
+
+ // Simulate passage of the evidence lease without a timing-dependent sleep.
+ key := "site|" + testJobUPID
+ entry := m.vzdumpJobTaskCache[key]
+ observed := time.Now().UTC().Add(-2 * backupIntentEvidenceMaxAge)
+ for i := range entry.tasks {
+ require.False(t, entry.tasks[i].ObservedAt.IsZero(), "cache must retain its accepted log observation")
+ entry.tasks[i].ObservedAt = observed
+ }
+ m.vzdumpJobTaskCache[key] = entry
+ client.logErr = fmt.Errorf("task log %s", failure)
+ m.pollBackupTasks(ctx, "site", client)
+ assertVzdumpGuestResults(t, guests(), "running")
+ for _, task := range guests() {
+ require.Equal(t, observed, task.ObservedAt, "failed reads cannot freshen the last log observation")
+ }
+ _, active = m.resolveBackupIntentContext("", "site", job.Node, 103, time.Now())
+ require.False(t, active, "unread live logs must not indefinitely renew alert deferral")
+
+ client.task.EndTime = job.EndTime.Unix()
+ client.task.Status = parentStatus
+ m.pollBackupTasks(ctx, "site", client)
+ assertVzdumpGuestResults(t, guests(), "unknown")
+ for _, task := range guests() {
+ require.Equal(t, observed, task.ObservedAt)
+ require.False(t, task.EndTime.IsZero(), "ended parent must not leave an apparently active guest")
+ if task.VMID == 103 {
+ require.Equal(t, job.EndTime, task.EndTime)
+ require.Equal(t, "backup job ended without an observed guest result", task.Error)
+ }
+ _, active = m.resolveBackupIntentContext("", "site", job.Node, task.VMID, observed)
+ require.False(t, active, "parent termination withdraws intent even while the cached observation is fresh")
+ }
+ checkStored(recovery.OutcomeUnknown, false)
+ require.Equal(t, entry, m.vzdumpJobTaskCache[key], "fallback must not mutate/finalise accepted log data")
+ require.Equal(t, 3, client.logRequests)
+
+ // Only an ordinary accepted log supplies the missing guest verdict.
+ client.logErr = nil
+ client.logLines = append(lines, proxmox.TaskLogLine{LineNumber: 6, Text: "INFO: Finished Backup of VM 103 (00:02:00)"}, proxmox.TaskLogLine{LineNumber: 7, Text: "TASK ERROR: job errors"})
+ m.pollBackupTasks(ctx, "site", client)
+ assertVzdumpGuestResults(t, guests(), "OK")
+ checkStored(recovery.OutcomeSuccess, true)
+ require.True(t, m.vzdumpJobTaskCache[key].final)
+ for _, task := range guests() {
+ require.True(t, task.ObservedAt.After(observed))
+ }
+ m.pollBackupTasks(ctx, "site", client)
+ require.Equal(t, 4, client.logRequests, "complete immutable logs still use the final cache")
+ checkStored(recovery.OutcomeSuccess, true)
+ })
+ }
+ }
+}
+
+func TestVzdumpCachedGuestUnreadLogUsesCurrentParent(t *testing.T) {
+ for _, blocked := range []string{"budget", "cancelled"} {
+ t.Run(blocked, func(t *testing.T) {
+ job, lines := vzdumpGuestResultFixture()
+ running := job
+ running.EndTime = time.Time{}
+ observed := time.Now().UTC().Add(-time.Minute)
+ entry := vzdumpJobTaskCacheEntry{tasks: refreshObservedAt(parseVzdumpJobLog(running, testJobUPID, lines), observed)}
+ m := &Monitor{vzdumpJobTaskCache: map[string]vzdumpJobTaskCacheEntry{"site|" + testJobUPID: entry}}
+ client := &cachedVzdumpClient{}
+ ctx := context.Background()
+ jobs := []models.BackupTask{}
+ upids := map[string]string{job.ID: testJobUPID}
+ if blocked == "budget" {
+ for i := 0; i < maxVzdumpJobLogFetchesPerPoll; i++ {
+ filler := models.BackupTask{ID: fmt.Sprintf("filler-%d", i)}
+ jobs = append(jobs, filler)
+ upids[filler.ID] = filler.ID
+ }
+ } else {
+ cancelled, cancel := context.WithCancel(ctx)
+ cancel()
+ ctx = cancelled
+ }
+ jobs = append(jobs, job)
+ tasks := m.synthesizeVzdumpJobGuestTasks(ctx, "site", client, jobs, upids)
+ assertVzdumpGuestResults(t, tasks, "unknown")
+ for _, task := range tasks {
+ require.Equal(t, observed, task.ObservedAt, "an unread log is not a fresh guest observation")
+ require.False(t, task.EndTime.IsZero())
+ }
+ require.Equal(t, entry, m.vzdumpJobTaskCache["site|"+testJobUPID])
+ wantRequests := 0
+ if blocked == "budget" {
+ wantRequests = maxVzdumpJobLogFetchesPerPoll
+ }
+ require.Equal(t, wantRequests, client.logRequests, "repair must not bypass request or cancellation bounds")
+ })
+ }
+}