Real-time monitoring dashboard for Proxmox VE, PBS, Docker, Kubernetes, TrueNAS and vSphere. Self-hosted, with alerts for failed backups, full disks, unhealthy containers and offline nodes.
Go aialertsdashboarddockerdocker-monitoringgohomelabinfrastructure-monitoringkubernetesmonitoringmspproxmoxproxmox-backup-serverproxmox-mail-gatewayproxmox-veself-hostedtruenastypescriptvspherewebhooks
Keep carried PVE rows from renewing recovery observations
pulse-triage[bot]10f570a69837
diff --git a/docs/release-control/v6/internal/subsystems/monitoring.md b/docs/release-control/v6/internal/subsystems/monitoring.mdindex bdd2770a06..034b60d2f8 100644--- a/docs/release-control/v6/internal/subsystems/monitoring.md+++ b/docs/release-control/v6/internal/subsystems/monitoring.md@@ -23,6 +23,9 @@ Only a successfully built, complete PVE backup-history observation admitsinstance-wide recovery-point reconciliation. Partial or unavailable scans maypublish accepted rows and their actual provider-quality evidence, but cannotdelete unseen historical points, including after live inventory is rebuilt.+Carried live rows do not renew point-level observation evidence: only rows+actually returned by this scan enter recovery ingest, while historical points+retain their previous evidence under the current incomplete provider observation.Cancellation before publication leaves the previous payload and providerobservation untouched. Complete empty scans still remove absent PVE backuppoints without crossing connection or evidence-class boundaries.diff --git a/internal/monitoring/monitor_backups.go b/internal/monitoring/monitor_backups.goindex 01ef0f379a..e3c5e57d79 100644--- a/internal/monitoring/monitor_backups.go+++ b/internal/monitoring/monitor_backups.go@@ -442,6 +442,9 @@ func (m *Monitor) pollStorageBackupsWithNodes(ctx context.Context, instanceNamereturn}+ // Carry-forward serves the live inventory, not a new provider observation.+ // Only rows actually returned this poll may renew point-level evidence.+ observedBackups := allBackupsallBackups, preservedStorages := preserveFailedStorageBackups(instanceName, snapshot, storagePreserveNeeded, allBackups)if len(preservedStorages) > 0 {log.Warn().@@ -509,7 +512,7 @@ func (m *Monitor) pollStorageBackupsWithNodes(ctx context.Context, instanceName// Best-effort ingestion into recovery store (for rollups / unified backups UX).guestInfo := buildProxmoxGuestInfoIndex(readState)points, evidenceErr := proxmoxrecoverymapper.FromPVEStorageBackupsWithEvidence(- allBackups,+ observedBackups,guestInfo,protectionObservedAt,)diff --git a/internal/monitoring/pve_backup_reconciliation_test.go b/internal/monitoring/pve_backup_reconciliation_test.goindex 2c59fcf61f..c17764df0b 100644--- a/internal/monitoring/pve_backup_reconciliation_test.go+++ b/internal/monitoring/pve_backup_reconciliation_test.go@@ -202,6 +202,26 @@ func TestPVEBackupIncompletePollRetainsRecoveryHistory(t *testing.T) {}}+func TestPVEBackupRetainedRowsCannotRenewRecoveryEvidence(t *testing.T) {+ f := newPVEReconciliationFixture(t)+ f.poll(t)+ before := f.points(t)+ f.client.contentErr = fmt.Errorf("503 unavailable")+ f.client.at = f.client.at.Add(15 * time.Minute)+ for range 2 {+ f.poll(t)+ if len(f.m.state.GetSnapshot().PVEBackups.StorageBackups) != 2 {+ t.Fatal("warm failed read lost its last accepted live inventory")+ }+ for id, point := range before {+ if point.SubjectResourceID == f.ids[1] && !reflect.DeepEqual(f.points(t)[id], point) {+ t.Error("carried row renewed point evidence without a successful provider read")+ }+ }+ f.assertHistory(t, recovery.ProtectionHistoryPartial, operationaltrust.EvidencePermissionsUnknown)+ }+}+func TestPVEBackupCompletePollStillReconcilesOnlyItsScope(t *testing.T) {for _, noStorages := range []bool{false, true} {t.Run(fmt.Sprintf("no-storages=%t", noStorages), func(t *testing.T) {@@ -268,6 +288,10 @@ func TestPVEBackupReconciliationProducer(t *testing.T) {}f.poll(t)capture("observed")+ f.client.contentErr = fmt.Errorf("503 unavailable")+ f.poll(t)+ capture("warm-partial")+ f.client.contentErr = nilf.coldState()f.client.nodeErr = fmt.Errorf("503 unavailable")f.poll(t)diff --git a/scripts/check-pve-backup-reconciliation.cjs b/scripts/check-pve-backup-reconciliation.cjsindex d6573c70aa..7302030d3a 100644--- a/scripts/check-pve-backup-reconciliation.cjs+++ b/scripts/check-pve-backup-reconciliation.cjs@@ -82,7 +82,7 @@ let browser, server;result.requests.push({ viewport: name, phase, path: url.pathname });return route.fulfill({ json: body });});- for (phase of ['observed', 'partial', 'denied', 'recovered', 'confirmed-empty']) {+ for (phase of ['observed', 'warm-partial', 'partial', 'denied', 'recovered', 'confirmed-empty']) {if (phase === 'observed') await page.goto(origin + '/proxmox/backups/coverage');else await page.reload();const expand = page.getByRole('button', { name: 'Expand details for guest-102', exact: true });@@ -99,7 +99,7 @@ let browser, server;assert(body.includes(`History ${cap(provider.historyCompleteness)}`));assert(body.includes(`Access ${cap(provider.permissions)}`));}- if (phase === 'partial' || phase === 'denied') {+ if (phase === 'warm-partial' || phase === 'partial' || phase === 'denied') {assert.notEqual(posture.state, 'protected');assert.equal(posture.providerStates[0].historyCompleteness, 'partial');assert.equal(Object.keys(producer.frames[phase].points).length, 2, 'unseen historical point disappeared');