Skip to content

armgit / Pulse

Public Imported from GitHub
0

Real-time monitoring dashboard for Proxmox VE, PBS, Docker, Kubernetes, TrueNAS and vSphere. Self-hosted, with alerts for failed backups, full disks, unhealthy containers and offline nodes.

Keep carried PVE rows from renewing recovery observations

pulse-triage[bot]10f570a69837
diff --git a/docs/release-control/v6/internal/subsystems/monitoring.md b/docs/release-control/v6/internal/subsystems/monitoring.md
index bdd2770a06..034b60d2f8 100644
--- a/docs/release-control/v6/internal/subsystems/monitoring.md
+++ b/docs/release-control/v6/internal/subsystems/monitoring.md
@@ -23,6 +23,9 @@ Only a successfully built, complete PVE backup-history observation admits
instance-wide recovery-point reconciliation. Partial or unavailable scans may
publish accepted rows and their actual provider-quality evidence, but cannot
delete unseen historical points, including after live inventory is rebuilt.
+Carried live rows do not renew point-level observation evidence: only rows
+actually returned by this scan enter recovery ingest, while historical points
+retain their previous evidence under the current incomplete provider observation.
Cancellation before publication leaves the previous payload and provider
observation untouched. Complete empty scans still remove absent PVE backup
points without crossing connection or evidence-class boundaries.
diff --git a/internal/monitoring/monitor_backups.go b/internal/monitoring/monitor_backups.go
index 01ef0f379a..e3c5e57d79 100644
--- a/internal/monitoring/monitor_backups.go
+++ b/internal/monitoring/monitor_backups.go
@@ -442,6 +442,9 @@ func (m *Monitor) pollStorageBackupsWithNodes(ctx context.Context, instanceName
return
}
+ // Carry-forward serves the live inventory, not a new provider observation.
+ // Only rows actually returned this poll may renew point-level evidence.
+ observedBackups := allBackups
allBackups, preservedStorages := preserveFailedStorageBackups(instanceName, snapshot, storagePreserveNeeded, allBackups)
if len(preservedStorages) > 0 {
log.Warn().
@@ -509,7 +512,7 @@ func (m *Monitor) pollStorageBackupsWithNodes(ctx context.Context, instanceName
// Best-effort ingestion into recovery store (for rollups / unified backups UX).
guestInfo := buildProxmoxGuestInfoIndex(readState)
points, evidenceErr := proxmoxrecoverymapper.FromPVEStorageBackupsWithEvidence(
- allBackups,
+ observedBackups,
guestInfo,
protectionObservedAt,
)
diff --git a/internal/monitoring/pve_backup_reconciliation_test.go b/internal/monitoring/pve_backup_reconciliation_test.go
index 2c59fcf61f..c17764df0b 100644
--- a/internal/monitoring/pve_backup_reconciliation_test.go
+++ b/internal/monitoring/pve_backup_reconciliation_test.go
@@ -202,6 +202,26 @@ func TestPVEBackupIncompletePollRetainsRecoveryHistory(t *testing.T) {
}
}
+func TestPVEBackupRetainedRowsCannotRenewRecoveryEvidence(t *testing.T) {
+ f := newPVEReconciliationFixture(t)
+ f.poll(t)
+ before := f.points(t)
+ f.client.contentErr = fmt.Errorf("503 unavailable")
+ f.client.at = f.client.at.Add(15 * time.Minute)
+ for range 2 {
+ f.poll(t)
+ if len(f.m.state.GetSnapshot().PVEBackups.StorageBackups) != 2 {
+ t.Fatal("warm failed read lost its last accepted live inventory")
+ }
+ for id, point := range before {
+ if point.SubjectResourceID == f.ids[1] && !reflect.DeepEqual(f.points(t)[id], point) {
+ t.Error("carried row renewed point evidence without a successful provider read")
+ }
+ }
+ f.assertHistory(t, recovery.ProtectionHistoryPartial, operationaltrust.EvidencePermissionsUnknown)
+ }
+}
+
func TestPVEBackupCompletePollStillReconcilesOnlyItsScope(t *testing.T) {
for _, noStorages := range []bool{false, true} {
t.Run(fmt.Sprintf("no-storages=%t", noStorages), func(t *testing.T) {
@@ -268,6 +288,10 @@ func TestPVEBackupReconciliationProducer(t *testing.T) {
}
f.poll(t)
capture("observed")
+ f.client.contentErr = fmt.Errorf("503 unavailable")
+ f.poll(t)
+ capture("warm-partial")
+ f.client.contentErr = nil
f.coldState()
f.client.nodeErr = fmt.Errorf("503 unavailable")
f.poll(t)
diff --git a/scripts/check-pve-backup-reconciliation.cjs b/scripts/check-pve-backup-reconciliation.cjs
index d6573c70aa..7302030d3a 100644
--- a/scripts/check-pve-backup-reconciliation.cjs
+++ b/scripts/check-pve-backup-reconciliation.cjs
@@ -82,7 +82,7 @@ let browser, server;
result.requests.push({ viewport: name, phase, path: url.pathname });
return route.fulfill({ json: body });
});
- for (phase of ['observed', 'partial', 'denied', 'recovered', 'confirmed-empty']) {
+ for (phase of ['observed', 'warm-partial', 'partial', 'denied', 'recovered', 'confirmed-empty']) {
if (phase === 'observed') await page.goto(origin + '/proxmox/backups/coverage');
else await page.reload();
const expand = page.getByRole('button', { name: 'Expand details for guest-102', exact: true });
@@ -99,7 +99,7 @@ let browser, server;
assert(body.includes(`History ${cap(provider.historyCompleteness)}`));
assert(body.includes(`Access ${cap(provider.permissions)}`));
}
- if (phase === 'partial' || phase === 'denied') {
+ if (phase === 'warm-partial' || phase === 'partial' || phase === 'denied') {
assert.notEqual(posture.state, 'protected');
assert.equal(posture.providerStates[0].historyCompleteness, 'partial');
assert.equal(Object.keys(producer.frames[phase].points).length, 2, 'unseen historical point disappeared');