Skip to content

armgit / Pulse

Public Imported from GitHub
0

Real-time monitoring dashboard for Proxmox VE, PBS, Docker, Kubernetes, TrueNAS and vSphere. Self-hosted, with alerts for failed backups, full disks, unhealthy containers and offline nodes.

Keep known LXC snapshots when their listing is unavailable

pulse-triage[bot]850a63104460
diff --git a/docs/release-control/v6/internal/subsystems/monitoring.md b/docs/release-control/v6/internal/subsystems/monitoring.md
index 7d5e9d3f81..1b158fe28b 100644
--- a/docs/release-control/v6/internal/subsystems/monitoring.md
+++ b/docs/release-control/v6/internal/subsystems/monitoring.md
@@ -17,6 +17,25 @@
## Purpose
+### Failed guest snapshot reads do not establish removal
+
+A snapshot listing is accepted only when its API call succeeds. In particular,
+LXC errors containing `596` or `not available` retain the last accepted listing
+just like other VM/container read errors; they are not successful empty replies.
+Any partial result returned with an error is discarded. Other guests in the
+same cycle still accept fresh or empty listings normally, and another instance's
+inventory is untouched. A later successful empty reply still establishes removal.
+
+`TestSnapshotCollectionFailureIsNotRemoval` exercises both guest kinds, repeated
+errors, partial replies, independent guest/instance membership and warning
+occurrence/acknowledgement continuity through the actual poller and served state.
+The first-read and explicit-alert-disable controls prevent fabricated evidence
+or retention overriding policy. `TestSnapshotCollectionBrowserProducer` emits
+synthetic poller/manager frames for the real AlertsAPI and Overview consumer.
+No guest-agent command, backup operation, provider retry, schema, identity or
+policy change is introduced. Retained membership is not proof of a fresh read,
+artifact availability, guest thaw, filesystem liveness or installed recovery.
+
### Locally built images report an uncheckable update state (21 October 2026)
An image with no RepoDigest was built or loaded on its host, so no registry
diff --git a/internal/monitoring/monitor_backups.go b/internal/monitoring/monitor_backups.go
index 22a3aada1b..6ea99da588 100644
--- a/internal/monitoring/monitor_backups.go
+++ b/internal/monitoring/monitor_backups.go
@@ -1176,11 +1176,9 @@ func (m *Monitor) pollGuestSnapshots(ctx context.Context, instanceName string, c
for _, snap := range snapshots {
result.snapshots = append(result.snapshots, buildSnapshot(target, snap))
}
- } else if target.guestType == "lxc" {
- errStr := err.Error()
- if strings.Contains(errStr, "596") || strings.Contains(errStr, "not available") {
- result.polled = true
- }
+ // Only an accepted listing establishes membership, including an
+ // empty one. LXC 596/unavailable errors are not deletion evidence;
+ // leave failed guests unpolled so their last listing is retained.
}
results <- result
diff --git a/internal/monitoring/snapshot_collection_observation_test.go b/internal/monitoring/snapshot_collection_observation_test.go
index a6dae138c3..6e0e9aea8d 100644
--- a/internal/monitoring/snapshot_collection_observation_test.go
+++ b/internal/monitoring/snapshot_collection_observation_test.go
@@ -59,11 +59,15 @@ func snapshotCollectionMonitor(t *testing.T, kind string, withPeers bool) *Monit
if node == "node-b" && (!withPeers || site == "other-site") {
continue
}
- id := alerts.BuildGuestKey(site, node, 101)
+ vmid := 101
+ if node == "node-b" {
+ vmid = 202
+ }
+ id := alerts.BuildGuestKey(site, node, vmid)
if kind == "qemu" {
- vms = append(vms, models.VM{ID: id, VMID: 101, Name: "database", Node: node, Instance: site, Type: kind, Status: "running"})
+ vms = append(vms, models.VM{ID: id, VMID: vmid, Name: "database", Node: node, Instance: site, Type: kind, Status: "running"})
} else {
- containers = append(containers, models.Container{ID: id, VMID: 101, Name: "database", Node: node, Instance: site, Type: kind, Status: "running"})
+ containers = append(containers, models.Container{ID: id, VMID: vmid, Name: "database", Node: node, Instance: site, Type: kind, Status: "running"})
}
}
m.state.UpdateGuestsForInstance(site, vms, containers)
@@ -97,7 +101,7 @@ func TestSnapshotCollectionFailureIsNotRemoval(t *testing.T) {
m := snapshotCollectionMonitor(t, kind, true)
old := proxmox.Snapshot{Name: "weekly", SnapTime: time.Now().Add(-16 * 24 * time.Hour).Unix(), Description: "last accepted listing", Parent: "base"}
key := kind + "/node-a/101"
- peerKey := kind + "/node-b/101"
+ peerKey := kind + "/node-b/202"
client := &snapshotCollectionClient{replies: map[string]snapshotCollectionReply{key: {snapshots: []proxmox.Snapshot{old}}, peerKey: {snapshots: []proxmox.Snapshot{old}}}}
m.pollGuestSnapshots(context.Background(), "site", client)
m.pollGuestSnapshots(context.Background(), "other-site", client)
@@ -120,7 +124,7 @@ func TestSnapshotCollectionFailureIsNotRemoval(t *testing.T) {
if !reflect.DeepEqual(after[targetID], before[targetID]) {
t.Fatalf("failed %s listing removed or changed accepted snapshot: before=%+v after=%+v", kind, before[targetID], after[targetID])
}
- if len(after) != 3 || !reflect.DeepEqual(after[otherID], before[otherID]) || after["site-node-b-101-fresh-peer"].Name != "fresh-peer" {
+ if len(after) != 3 || !reflect.DeepEqual(after[otherID], before[otherID]) || after["site-node-b-202-fresh-peer"].Name != "fresh-peer" {
t.Fatalf("failed listing leaked across node/instance or blocked accepted peer: %+v", after)
}
current := snapshotCollectionAlert(t, m, resourceID)
diff --git a/scripts/check-snapshot-collection-observation.cjs b/scripts/check-snapshot-collection-observation.cjs
new file mode 100644
index 0000000000..10b9f1a02c
--- /dev/null
+++ b/scripts/check-snapshot-collection-observation.cjs
@@ -0,0 +1,150 @@
+// Run from the assigned root with pulse-worker-browser. No external services.
+const assert = require('node:assert/strict');
+const fs = require('node:fs');
+const path = require('node:path');
+const http = require('node:http');
+const crypto = require('node:crypto');
+const { chromium, webkit } = require('playwright');
+
+const root = '/workspace';
+const compiled = path.join(root, 'tmp/backup-date-static');
+const output = path.join(root, 'tmp/snapshot-collection-browser');
+const origin = 'http://127.0.0.1:5371';
+const hash = (bytes) => crypto.createHash('sha256').update(bytes).digest('hex');
+const binding = JSON.parse(fs.readFileSync(path.join(compiled, 'binding.json')));
+const producer = JSON.parse(fs.readFileSync(path.join(root, 'tmp/snapshot-collection-producer.json')));
+const frames = producer.frames;
+const result = {
+ result: 'incomplete',
+ source: producer.source_sha,
+ base: producer.base_sha,
+ producer,
+ consumer: binding,
+ harness: hash(fs.readFileSync(__filename)),
+ playwright: require('playwright/package.json').version,
+ cases: [],
+ requests: [],
+ captures: [],
+ cleanup: { browser: false, server: false },
+ limits: 'Real pollGuestSnapshots/GetState.ActiveAlerts -> AlertsAPI -> Overview/card; synthetic PVE listings only. Inventory retention is checked by the producer, not rendered by this alert fixture. No installed recovery, guest action, backup execution, notification delivery or reporter acceptance.',
+};
+let server;
+let browser;
+
+(async () => {
+ fs.mkdirSync(output, { recursive: true });
+ assert.equal(result.playwright, JSON.parse(fs.readFileSync(path.join(root, 'tests/integration/package-lock.json'))).packages['node_modules/@playwright/test'].version);
+ assert.equal(binding.graph, hash(fs.readFileSync(path.join(root, 'frontend-modern/package-lock.json'))));
+ assert.equal(binding.builder, hash(fs.readFileSync(path.join(root, 'frontend-modern/browser-tests/build-backup-date-observation.mjs'))));
+ for (const [file, digest] of Object.entries(producer.sources)) {
+ assert.equal(hash(fs.readFileSync(path.join(root, file))), digest, file);
+ }
+ const phases = ['observed', 'failed-596', 'failed-unavailable', 'confirmed-empty'];
+ for (const phase of phases.slice(0, -1)) {
+ assert.deepEqual(frames[phase].snapshots, frames.observed.snapshots, phase);
+ assert.equal(frames[phase].alerts.length, 1, phase);
+ assert.equal(frames[phase].alerts[0].id, frames.observed.alerts[0].id, phase);
+ assert.equal(frames[phase].alerts[0].startTime, frames.observed.alerts[0].startTime, phase);
+ assert.equal(frames[phase].alerts[0].level, 'critical', phase);
+ }
+ assert.equal(frames['confirmed-empty'].snapshots.length, 0);
+ assert.deepEqual(frames['confirmed-empty'].alerts, []);
+ for (const [file, digest] of Object.entries(binding.sources)) {
+ assert.equal(hash(fs.readFileSync(path.join(root, 'frontend-modern', file))), digest, file);
+ }
+ for (const file of ['src/api/alerts.ts', 'src/features/alerts/OverviewTab.tsx', 'src/features/alerts/AlertOverviewAlertCard.tsx']) {
+ assert(binding.sources[file], `real consumer not compiled: ${file}`);
+ }
+ const assets = new Map();
+ for (const [file, digest] of Object.entries(binding.artifacts)) {
+ const bytes = fs.readFileSync(path.join(compiled, file));
+ assert.equal(hash(bytes), digest, file);
+ assets.set('/' + file, bytes);
+ }
+ server = http.createServer((request, response) => {
+ const pathname = new URL(request.url, origin).pathname;
+ const bytes = assets.get(pathname);
+ if (!bytes || request.method !== 'GET') {
+ response.writeHead(404).end();
+ return;
+ }
+ response.writeHead(200, { 'Content-Type': { '.html': 'text/html', '.js': 'text/javascript', '.css': 'text/css' }[path.extname(pathname)] || 'application/octet-stream' });
+ response.end(bytes);
+ });
+ await new Promise((resolve, reject) => {
+ server.once('error', reject);
+ server.listen(5371, '127.0.0.1', resolve);
+ });
+ for (const [name, engine, width, height] of [
+ ['desktop', chromium, 1280, 900],
+ ['narrow', webkit, 320, 812],
+ ]) {
+ browser = await engine.launch(engine === chromium ? { headless: true, channel: 'chromium', args: ['--no-sandbox'] } : { headless: true });
+ const page = await browser.newPage({ viewport: { width, height }, isMobile: width < 768, hasTouch: width < 768, locale: 'en-GB', timezoneId: 'UTC' });
+ page.setDefaultTimeout(10000);
+ const record = { name, width, height, browser: browser.version(), errors: [], phases: [] };
+ page.on('pageerror', (error) => record.errors.push(error.message));
+ let phase = 'observed';
+ await page.route('**/*', async (route) => {
+ const request = route.request();
+ const url = new URL(request.url());
+ assert.equal(url.origin, origin, 'external request');
+ assert.equal(request.method(), 'GET', 'unexpected mutation');
+ if (!url.pathname.startsWith('/api/')) return route.continue();
+ result.requests.push({ viewport: name, phase, path: url.pathname });
+ let body;
+ if (url.pathname === '/api/alerts/active') body = frames[phase].alerts;
+ else if (url.pathname === '/api/alerts/delivery-diagnosis') body = [];
+ else if (url.pathname === '/api/notifications/health') body = { queue: { status: 'unavailable' } };
+ else if (url.pathname === '/api/alerts/incidents') body = null;
+ else throw new Error(`unexpected API read: ${url.pathname}`);
+ return route.fulfill({ json: body });
+ });
+ await page.goto(origin + '/browser-tests/backup-date-observation.html');
+ await page.waitForFunction(() => window.__backupDate?.generation() === 1);
+ for (phase of phases) {
+ if (phase !== 'observed') {
+ const previous = await page.evaluate(() => window.__backupDate.generation());
+ await page.getByRole('button', { name: 'Refresh monitoring evidence', exact: true }).click();
+ await page.waitForFunction((generation) => window.__backupDate.generation() > generation, previous);
+ }
+ const active = await page.evaluate(() => Object.values(window.__backupDate.active()));
+ assert.deepEqual(active, frames[phase].alerts, `producer/consumer mismatch in ${name}/${phase}`);
+ if (phase !== 'confirmed-empty') {
+ await page.getByText(frames[phase].alerts[0].message, { exact: true }).waitFor();
+ const more = page.getByRole('button', { name: 'More', exact: true });
+ if (await more.count()) {
+ await more.click();
+ await page.getByRole('button', { name: 'Less', exact: true }).click();
+ }
+ } else {
+ assert.equal(await page.getByText(frames.observed.alerts[0].message, { exact: true }).count(), 0);
+ }
+ await page.evaluate(async () => {
+ await document.fonts.ready;
+ await new Promise((resolve) => requestAnimationFrame(() => requestAnimationFrame(resolve)));
+ });
+ const overflow = await page.evaluate(() => document.documentElement.scrollWidth > window.innerWidth);
+ assert.equal(overflow, false, `${name}/${phase} horizontal overflow`);
+ const capture = `${name}-${phase}.png`;
+ await page.screenshot({ path: path.join(output, capture), fullPage: true });
+ result.captures.push({ file: capture, sha256: hash(fs.readFileSync(path.join(output, capture))) });
+ record.phases.push({ phase, active: active.length, overflow });
+ }
+ assert.deepEqual(record.errors, []);
+ result.cases.push(record);
+ await browser.close();
+ browser = undefined;
+ }
+ result.result = 'passed';
+})().catch((error) => {
+ result.result = 'failed';
+ result.error = String(error.stack || error);
+ process.exitCode = 1;
+}).finally(async () => {
+ if (browser) await browser.close();
+ result.cleanup.browser = true;
+ if (server) await new Promise((resolve) => server.close(resolve));
+ result.cleanup.server = true;
+ fs.writeFileSync(path.join(output, 'result.json'), JSON.stringify(result, null, 2) + '\n');
+});