Skip to content

Self-hosted, Self-Improving, Federated Everything App — dev management, agent orchestration, persistent mind, digital twin, second brain, and creative pipeline from a single dashboard over Tailscale to port the user into the OS

test: retain late gallery database wait evidence

Adam Eivyf4ef70a73a4e
diff --git a/server/test/dbPhaseDiagnostics.js b/server/test/dbPhaseDiagnostics.js
index 3031b93898..daf9506b4c 100644
--- a/server/test/dbPhaseDiagnostics.js
+++ b/server/test/dbPhaseDiagnostics.js
@@ -14,6 +14,8 @@ const SNAPSHOT_SQL = `SELECT a.pid = $1 AS fixture, count(*) OVER ()::int AS ses
AND a.backend_type = 'client backend' AND a.pid <> pg_backend_pid()
ORDER BY fixture DESC, run_lock DESC, a.pid LIMIT 16`;
const fields = ['fixture', 'session_count', 'state', 'wait_event_type', 'wait_event', 'transaction_ms', 'blockers', 'waiting_locks', 'run_lock'];
+// Cover the runner's longest (Windows) deadline without an unbounded observer loop.
+const MAX_PHASE_SNAPSHOTS = 32;
/** Opt-in observation; after its test bodies settle, the caller drains before closing the observer. */
export function createDbPhaseDiagnostics({ onTestFailed, observer, fixturePid, enabled = false,
@@ -24,36 +26,52 @@ export function createDbPhaseDiagnostics({ onTestFailed, observer, fixturePid, e
let active;
let pending = Promise.resolve();
let sampling = false;
+ let ended = false;
const record = event => {
if (evidence.length === 64) evidence.shift();
evidence.push(event);
write(`DB phases: ${JSON.stringify(event)}`);
};
const metrics = () => ({ elapsedMs: Math.round(performance.now() - active.started),
- cpuMs: Math.round(Object.values(process.cpuUsage(active.cpu)).reduce((sum, value) => sum + value, 0) / 1000) });
+ cpuMs: Math.round(Object.values(process.cpuUsage(active.cpu)).reduce((sum, value) => sum + value, 0) / 1000),
+ timerDelayMs: Math.round(Math.max(active.timerDelayMs, performance.now() - active.nextSampleAt, 0)) });
onTestFailed(({ task }) => {
+ ended = true;
+ active?.stop();
const summary = `DB phase evidence: ${JSON.stringify({ active: active && { phase: active.name, ...metrics() }, recent: evidence })}`;
for (const error of task.result.errors) {
error.message += `\n${summary}`;
if (error.stack) error.stack += `\n${summary}`;
}
});
- const sample = (name, count) => {
- if (sampling || count.value === 3) return;
+ const sample = (phase, count) => {
+ if (sampling || count.value === MAX_PHASE_SNAPSHOTS) return;
count.value++;
sampling = true;
+ const name = phase.name;
+ const sampleStartedMs = Math.round(performance.now() - phase.started);
pending = Promise.resolve().then(() => observer.query(SNAPSHOT_SQL, [fixturePid, DB_RUN_ADVISORY_LOCK_KEY])).then(({ rows }) => {
- record({ phase: name, sessions: rows.map(row => Object.fromEntries(fields.map(key => [key, row[key]]))) });
- }, () => record({ phase: name, snapshot: 'unavailable' })).finally(() => { sampling = false; });
+ record({ phase: name, sampleStartedMs, elapsedMs: Math.round(performance.now() - phase.started),
+ sessions: rows.map(row => Object.fromEntries(fields.map(key => [key, row[key]]))) });
+ }, () => record({ phase: name, sampleStartedMs, snapshot: 'unavailable' })).finally(() => { sampling = false; });
};
return {
phase: (name, operation) => track(name, async () => {
- active = { name, started: performance.now(), cpu: process.cpuUsage() };
+ if (ended) return operation();
+ const started = performance.now();
+ active = { name, started, cpu: process.cpuUsage(), timerDelayMs: 0, nextSampleAt: started + intervalMs };
record({ phase: name, state: 'start' });
const count = { value: 0 };
- sample(name, count);
- const timer = setInterval(() => sample(name, count), intervalMs);
+ // Fast phases need no SQL observation. Keep sampling throughout slow
+ // phases: the old three immediate snapshots missed waits after ~2 s.
+ const timer = setInterval(() => {
+ const now = performance.now();
+ active.timerDelayMs = Math.max(active.timerDelayMs, now - active.nextSampleAt);
+ active.nextSampleAt = now + intervalMs;
+ sample(active, count);
+ }, intervalMs);
timer.unref?.();
+ active.stop = () => clearInterval(timer);
try { return await operation(); }
finally {
clearInterval(timer);
diff --git a/server/test/dbPhaseDiagnostics.test.js b/server/test/dbPhaseDiagnostics.test.js
index 1f36f84fab..6ca51ee4a8 100644
--- a/server/test/dbPhaseDiagnostics.test.js
+++ b/server/test/dbPhaseDiagnostics.test.js
@@ -2,7 +2,7 @@ import { afterEach, expect, it, vi } from 'vitest';
import { createDbPhaseDiagnostics } from './dbPhaseDiagnostics.js';
import { ownTestBodies } from '../lib/mockPathsDataRoot.js';
-afterEach(() => vi.useRealTimers());
+afterEach(() => { vi.restoreAllMocks(); vi.useRealTimers(); });
// A slow observer must not spend a later phase's test budget. Teardown
// still owns its completion before closing the diagnostic connection.
@@ -52,22 +52,56 @@ it('settles the operation independently and drains bounded private observations
expect(lines.join('\n')).not.toContain('private');
});
-it('retains timeout phase and CPU evidence with at most three snapshots per phase', async () => {
+// A wait that starts after the old three-snapshot window must survive a timeout.
+it('retains late waits and timer delay at failure without sampling cancelled work', async () => {
vi.useFakeTimers();
const hooks = [];
- const observer = { query: vi.fn(async () => ({ rows: [] })) };
+ let now = 0;
+ vi.spyOn(performance, 'now').mockImplementation(() => now);
+ const observer = { query: vi.fn(async () => ({ rows: [{ fixture: true,
+ wait_event_type: now >= 10000 ? 'Lock' : null, wait_event: now >= 10000 ? 'relation' : null }] })) };
const operation = Promise.withResolvers();
const diagnostics = createDbPhaseDiagnostics({ onTestFailed: hook => hooks.push(hook), observer,
fixturePid: 42, enabled: true, write: () => {} });
+ expect(await diagnostics.phase('fast-fixture', () => 'original result')).toBe('original result');
+ expect(observer.query).not.toHaveBeenCalled();
const run = diagnostics.phase('compact-explain', () => operation.promise);
- await vi.advanceTimersByTimeAsync(10000);
- expect(observer.query).toHaveBeenCalledTimes(3);
+ for (let second = 1; second <= 3; second++) {
+ now = second * 1000;
+ await vi.advanceTimersByTimeAsync(1000);
+ }
+ // The scheduler resumes late; CPU counters alone cannot distinguish this
+ // from a responsive process waiting on PostgreSQL.
+ now = 14000;
+ await vi.advanceTimersByTimeAsync(1000);
const error = { message: 'original timeout', stack: 'original stack' };
for (const hook of hooks) hook({ task: { result: { errors: [error] } } });
expect(error.message).toContain('original timeout');
expect(error.message).toContain('active=compact-explain');
expect(error.message).toContain('"cpuMs":');
+ expect(error.message).toContain('"timerDelayMs":10000');
+ expect(error.message).toContain('"wait_event":"relation"');
expect(error.message).toContain('"phase":"compact-explain"');
+ await vi.advanceTimersByTimeAsync(50000);
+ expect(observer.query).toHaveBeenCalledTimes(4);
+ operation.resolve();
+ await run;
+ await diagnostics.phase('after-cancellation', () => undefined);
+ await diagnostics.drain();
+ expect(observer.query).toHaveBeenCalledTimes(4);
+});
+
+it('observes waits beyond the first two seconds with a bounded query budget', async () => {
+ vi.useFakeTimers();
+ const observer = { query: vi.fn(async () => ({ rows: [] })) };
+ const operation = Promise.withResolvers();
+ const diagnostics = createDbPhaseDiagnostics({ onTestFailed: () => {}, observer,
+ fixturePid: 42, enabled: true, write: () => {} });
+ const run = diagnostics.phase('full-explain', () => operation.promise);
+ await vi.advanceTimersByTimeAsync(14000);
+ expect(observer.query).toHaveBeenCalledTimes(14);
+ await vi.advanceTimersByTimeAsync(100000);
+ expect(observer.query).toHaveBeenCalledTimes(32);
operation.resolve();
await run;
await diagnostics.drain();