Skip to content

armgit / Pulse

Public Imported from GitHub
0

Real-time monitoring dashboard for Proxmox VE, PBS, Docker, Kubernetes, TrueNAS and vSphere. Self-hosted, with alerts for failed backups, full disks, unhealthy containers and offline nodes.

Retain PVE recovery history across incomplete backup scans

pulse-triage[bot]50abcf51c436
diff --git a/docs/release-control/v6/internal/subsystems/monitoring.md b/docs/release-control/v6/internal/subsystems/monitoring.md
index 7d5e9d3f81..bdd2770a06 100644
--- a/docs/release-control/v6/internal/subsystems/monitoring.md
+++ b/docs/release-control/v6/internal/subsystems/monitoring.md
@@ -17,6 +17,23 @@
## Purpose
+### Partial PVE backup scans cannot delete durable history
+
+Only a successfully built, complete PVE backup-history observation admits
+instance-wide recovery-point reconciliation. Partial or unavailable scans may
+publish accepted rows and their actual provider-quality evidence, but cannot
+delete unseen historical points, including after live inventory is rebuilt.
+Cancellation before publication leaves the previous payload and provider
+observation untouched. Complete empty scans still remove absent PVE backup
+points without crossing connection or evidence-class boundaries.
+
+`pve_backup_reconciliation_test.go` drives ordinary storage polling, asynchronous
+ingest and canonical posture through node/content failures, denied access,
+offline/missing nodes, complete removal, recovery and cancellation. Its opt-in
+producer feeds the production Backups coverage surface in
+`scripts/check-pve-backup-reconciliation.cjs`. Retained history is not a fresh
+provider read, an available archive, guest thaw or filesystem-liveness proof.
+
### Locally built images report an uncheckable update state (21 October 2026)
An image with no RepoDigest was built or loaded on its host, so no registry
diff --git a/internal/monitoring/monitor_backups.go b/internal/monitoring/monitor_backups.go
index 22a3aada1b..01ef0f379a 100644
--- a/internal/monitoring/monitor_backups.go
+++ b/internal/monitoring/monitor_backups.go
@@ -436,6 +436,12 @@ func (m *Monitor) pollStorageBackupsWithNodes(ctx context.Context, instanceName
}
}
+ if ctx.Err() != nil {
+ // A cancelled enumeration cannot publish a successful empty inventory
+ // or renew provider evidence for a payload we did not accept.
+ return
+ }
+
allBackups, preservedStorages := preserveFailedStorageBackups(instanceName, snapshot, storagePreserveNeeded, allBackups)
if len(preservedStorages) > 0 {
log.Warn().
@@ -522,15 +528,19 @@ func (m *Monitor) pollStorageBackupsWithNodes(ctx context.Context, instanceName
if protectionObservationErr == nil {
observations = append(observations, protectionObservation)
}
- m.ingestAndReconcileRecoveryPointsWithObservationsAsync(
- points,
- observations,
- recoveryReconcileScope{
+ batch := recoveryIngestBatch{points: points, observations: observations}
+ // Live inventory can be empty after a monitor replacement even when
+ // durable recovery history exists. A partial scan may add observations,
+ // but cannot establish absence anywhere in the instance-wide scope.
+ if protectionObservationErr == nil &&
+ protectionObservation.HistoryCompleteness == recovery.ProtectionHistoryComplete {
+ batch.reconcile = &recoveryReconcileScope{
provider: string(recovery.ProviderProxmoxPVE),
idPrefix: "pve-backup:",
instance: instanceName,
- },
- )
+ }
+ }
+ m.enqueueRecoveryIngest(batch)
}
// Sync backup times to VMs/Containers and republish them to canonical resources.
diff --git a/scripts/build-pve-backup-reconciliation.mjs b/scripts/build-pve-backup-reconciliation.mjs
new file mode 100644
index 0000000000..9d39bb1ab3
--- /dev/null
+++ b/scripts/build-pve-backup-reconciliation.mjs
@@ -0,0 +1,72 @@
+// Compile the production consumer outside the bounded browser process tree.
+import { createRequire } from 'node:module';
+const require = createRequire(new URL('../frontend-modern/package.json', import.meta.url));
+const { build } = await import(require.resolve('vite'));
+const solid = (await import(require.resolve('vite-plugin-solid'))).default;
+const tailwindcss = (await import(require.resolve('@tailwindcss/vite'))).default;
+const { stop } = await import(require.resolve('esbuild'));
+import fs from 'node:fs';
+import path from 'node:path';
+import crypto from 'node:crypto';
+import { fileURLToPath } from 'node:url';
+
+const root = path.resolve(path.dirname(fileURLToPath(import.meta.url)), '../frontend-modern');
+const outDir = path.resolve(root, '../tmp/pve-reconciliation-static');
+const hash = (bytes) => crypto.createHash('sha256').update(bytes).digest('hex');
+const sources = {};
+const artifacts = {};
+
+try {
+ await build({
+ root,
+ base: '/',
+ configFile: false,
+ publicDir: false,
+ resolve: { alias: { '@': path.join(root, 'src') } },
+ plugins: [
+ {
+ name: 'bind-pve-reconciliation-consumer',
+ generateBundle() {
+ for (const id of this.getModuleIds()) {
+ const file = id.split('?')[0];
+ if (
+ file.startsWith(root + '/') &&
+ !file.includes('/node_modules/') &&
+ fs.existsSync(file) &&
+ fs.statSync(file).isFile()
+ ) {
+ sources[path.relative(root, file)] = hash(fs.readFileSync(file));
+ }
+ }
+ },
+ },
+ tailwindcss(),
+ solid(),
+ ],
+ build: {
+ target: 'esnext',
+ outDir,
+ emptyOutDir: true,
+ rollupOptions: { input: path.join(root, 'browser-tests/backups.html') },
+ },
+ });
+ const walk = (dir) => {
+ for (const entry of fs.readdirSync(dir, { withFileTypes: true })) {
+ const file = path.join(dir, entry.name);
+ if (entry.isDirectory()) walk(file);
+ else artifacts[path.relative(outDir, file)] = hash(fs.readFileSync(file));
+ }
+ };
+ walk(outDir);
+ fs.writeFileSync(
+ path.join(outDir, 'binding.json'),
+ JSON.stringify({
+ graph: hash(fs.readFileSync(path.join(root, 'package-lock.json'))),
+ builder: hash(fs.readFileSync(fileURLToPath(import.meta.url))),
+ sources,
+ artifacts,
+ }, null, 2) + '\n',
+ );
+} finally {
+ stop();
+}
diff --git a/scripts/check-pve-backup-reconciliation.cjs b/scripts/check-pve-backup-reconciliation.cjs
new file mode 100644
index 0000000000..d6573c70aa
--- /dev/null
+++ b/scripts/check-pve-backup-reconciliation.cjs
@@ -0,0 +1,128 @@
+// Actual poller/store/posture frames -> production Backups table and API hooks.
+// Static offline preview; no guest/backup commands or external service access.
+const assert = require('node:assert/strict');
+const fs = require('node:fs');
+const path = require('node:path');
+const http = require('node:http');
+const crypto = require('node:crypto');
+const { chromium, webkit } = require('playwright');
+const root = '/workspace';
+const compiled = path.join(root, 'tmp/pve-reconciliation-static');
+const output = path.join(root, 'tmp/pve-reconciliation-browser');
+const origin = 'http://127.0.0.1:5373';
+const hash = (bytes) => crypto.createHash('sha256').update(bytes).digest('hex');
+const producer = JSON.parse(fs.readFileSync(path.join(root, 'tmp/pve-reconciliation-producer.json')));
+const binding = JSON.parse(fs.readFileSync(path.join(compiled, 'binding.json')));
+const result = {
+ result: 'incomplete',
+ producer_sha256: hash(fs.readFileSync(path.join(root, 'tmp/pve-reconciliation-producer.json'))),
+ consumer: binding,
+ harness: hash(fs.readFileSync(__filename)),
+ playwright: require('playwright/package.json').version,
+ cases: [], requests: [], captures: [], cleanup: { browser: false, server: false },
+ limits: 'Real PVE storage poll, recovery ingest and posture reader through production Backups/API hooks. Synthetic HTTP envelope and workloads. Not full application, installed recovery, archive availability, guest thaw or filesystem liveness.',
+};
+let browser, server;
+
+(async () => {
+ fs.mkdirSync(output, { recursive: true });
+ assert.equal(result.playwright, JSON.parse(fs.readFileSync(path.join(root, 'tests/integration/package-lock.json'))).packages['node_modules/@playwright/test'].version);
+ assert.equal(producer.runtime_sha256, hash(fs.readFileSync(path.join(root, 'internal/monitoring/monitor_backups.go'))));
+ assert.equal(binding.graph, hash(fs.readFileSync(path.join(root, 'frontend-modern/package-lock.json'))));
+ assert.equal(binding.builder, hash(fs.readFileSync(path.join(root, 'scripts/build-pve-backup-reconciliation.mjs'))));
+ for (const [file, digest] of Object.entries(binding.sources)) {
+ assert.equal(hash(fs.readFileSync(path.join(root, 'frontend-modern', file))), digest, file);
+ }
+ for (const file of ['src/hooks/useProtectionPostures.ts', 'src/features/proxmox/ProxmoxBackupsTable.tsx', 'src/features/proxmox/ProxmoxCoverageTable.tsx']) {
+ assert(binding.sources[file], `production consumer not compiled: ${file}`);
+ }
+ const assets = new Map();
+ for (const [file, digest] of Object.entries(binding.artifacts)) {
+ const bytes = fs.readFileSync(path.join(compiled, file));
+ assert.equal(hash(bytes), digest, file);
+ assets.set('/' + file, bytes);
+ }
+ server = http.createServer((request, response) => {
+ const pathname = new URL(request.url, origin).pathname;
+ const isEntry = pathname === '/proxmox/backups/coverage';
+ const bytes = assets.get(isEntry ? '/browser-tests/backups.html' : pathname);
+ if (!bytes || request.method !== 'GET') return response.writeHead(404).end();
+ response.writeHead(200, { 'Content-Type': isEntry ? 'text/html' : ({ '.js': 'text/javascript', '.css': 'text/css' }[path.extname(pathname)] || 'application/octet-stream') });
+ response.end(bytes);
+ });
+ await new Promise((resolve, reject) => { server.once('error', reject); server.listen(5373, '127.0.0.1', resolve); });
+ for (const [name, engine, width, height] of [['desktop', chromium, 1280, 900], ['narrow', webkit, 320, 812]]) {
+ browser = await engine.launch(engine === chromium ? { headless: true, channel: 'chromium', args: ['--no-sandbox'] } : { headless: true });
+ const page = await browser.newPage({ viewport: { width, height }, isMobile: width < 768, hasTouch: width < 768, locale: 'en-GB', timezoneId: 'UTC' });
+ page.setDefaultTimeout(10000);
+ await page.clock.setFixedTime(new Date(producer.observed_at));
+ const record = { name, width, height, browser: browser.version(), errors: [], phases: [] };
+ page.on('pageerror', (error) => record.errors.push(error.message));
+ let phase = 'observed';
+ const responses = [];
+ page.on('response', async (response) => {
+ if (new URL(response.url()).pathname === '/api/recovery/postures') responses.push(await response.json());
+ });
+ await page.route('**/*', async (route) => {
+ const request = route.request(), url = new URL(request.url());
+ assert.equal(url.origin, origin, 'external request');
+ assert.equal(request.method(), 'GET', 'unexpected mutation');
+ const workloads = producer.resource_ids.map((id, index) => ({ id, type: 'vm', name: `guest-${101 + index}`, platformId: 'cluster', platformType: 'proxmox-pve', sourceType: 'api', status: 'running', lastSeen: Date.parse(producer.observed_at), proxmox: { vmid: 101 + index, node: index === 0 ? 'node-a' : 'node-b', instance: 'cluster' } }));
+ let body;
+ if (url.pathname === '/fixture/workloads') body = workloads;
+ else if (url.pathname === '/api/backups/pve') body = producer.frames[phase].pve;
+ else if (url.pathname === '/api/backups/pbs') body = { data: { backups: [] } };
+ else if (url.pathname === '/api/recovery/postures') {
+ assert.deepEqual(url.searchParams.getAll('resourceId').sort(), [...producer.resource_ids].sort());
+ body = producer.frames[phase].postures;
+ } else {
+ assert(!url.pathname.startsWith('/api/'), `unexpected API read: ${url.pathname}`);
+ return route.continue();
+ }
+ result.requests.push({ viewport: name, phase, path: url.pathname });
+ return route.fulfill({ json: body });
+ });
+ for (phase of ['observed', 'partial', 'denied', 'recovered', 'confirmed-empty']) {
+ if (phase === 'observed') await page.goto(origin + '/proxmox/backups/coverage');
+ else await page.reload();
+ const expand = page.getByRole('button', { name: 'Expand details for guest-102', exact: true });
+ await expand.waitFor();
+ await expand.focus();
+ await expand.press('Enter');
+ const panel = page.getByTestId('proxmox-coverage-detail');
+ const posture = producer.frames[phase].postures.data.find((item) => item.subjectResourceId === producer.resource_ids[1]);
+ await panel.getByText(posture.explanation, { exact: true }).waitFor();
+ await page.waitForFunction(() => document.fonts.status === 'loaded');
+ const body = await panel.innerText();
+ for (const provider of posture.providerStates) {
+ const cap = (s) => s[0].toUpperCase() + s.slice(1);
+ assert(body.includes(`History ${cap(provider.historyCompleteness)}`));
+ assert(body.includes(`Access ${cap(provider.permissions)}`));
+ }
+ if (phase === 'partial' || phase === 'denied') {
+ assert.notEqual(posture.state, 'protected');
+ assert.equal(posture.providerStates[0].historyCompleteness, 'partial');
+ assert.equal(Object.keys(producer.frames[phase].points).length, 2, 'unseen historical point disappeared');
+ }
+ assert.deepEqual(responses.at(-1), producer.frames[phase].postures, 'actual hook response changed producer evidence');
+ const overflow = await page.evaluate(() => document.documentElement.scrollWidth > innerWidth);
+ assert.equal(overflow, false, `${name}/${phase}: horizontal overflow`);
+ const file = path.join(output, `${name}-${phase}.png`);
+ await page.screenshot({ path: file, fullPage: true });
+ result.captures.push({ name, phase, file: path.relative(root, file), sha256: hash(fs.readFileSync(file)) });
+ record.phases.push({ phase, state: posture.state, body, overflow });
+ await page.getByRole('button', { name: 'Collapse details for guest-102', exact: true }).press('Enter');
+ await panel.waitFor({ state: 'hidden' });
+ }
+ assert.deepEqual(record.errors, []);
+ result.cases.push(record);
+ await browser.close(); browser = null;
+ }
+ result.result = 'passed';
+})().catch((error) => { result.error = error.stack; process.exitCode = 1; }).finally(async () => {
+ if (browser) await browser.close();
+ result.cleanup.browser = true;
+ if (server) await new Promise((resolve) => server.close(resolve));
+ result.cleanup.server = true;
+ fs.writeFileSync(path.join(output, 'result.json'), JSON.stringify(result, null, 2) + '\n');
+});