Skip to content

Self-hosted, Self-Improving, Federated Everything App — dev management, agent orchestration, persistent mind, digital twin, second brain, and creative pipeline from a single dashboard over Tailscale to port the user into the OS

test: retain gallery timeout phase and database wait evidence (#13369)

Adam Eivy893784e3a4fc
diff --git a/docs/CONTRIBUTING.md b/docs/CONTRIBUTING.md
index 24e4ac3dd0..8d3d863c34 100644
--- a/docs/CONTRIBUTING.md
+++ b/docs/CONTRIBUTING.md
@@ -311,6 +311,19 @@ the database restore suite drains a timed-out case before the next one touches
unloaded machine, so a concurrent run can time out cases a sequential run
passes.
+For the intermittent synthetic gallery-page timeout investigation (#13369),
+`PORTOS_DB_PHASE_DIAGNOSTICS=1 npm run pregate` adds wall/CPU timings for fixture
+construction, seeding, public calls, EXPLAINs and result serialization. To obtain
+the complete serial DB workload regardless of test-memo reuse, run
+`PORTOS_DB_PHASE_DIAGNOSTICS=1 npm run test:db` after the earlier validation
+stages finish. The observer reads only `portos_test` session state, waits,
+transaction ages and run-lock ownership: no query text, network addresses or
+record contents. It samples at most three times per phase, caps each snapshot
+at 16 sessions, and uses a separate connection with a 500 ms statement timeout.
+Opt-in diagnostics add observation work; original test deadlines stay intact.
+Failure messages retain bounded phase evidence even with quiet reporting.
+A passing run without a reproduced slow phase does not resolve the timeout.
+
For server watch mode, run this alternative from the repository root:
```bash
diff --git a/server/services/mediaAssetIndex/galleryCollections.db.test.js b/server/services/mediaAssetIndex/galleryCollections.db.test.js
index be3d050da2..824a903408 100644
--- a/server/services/mediaAssetIndex/galleryCollections.db.test.js
+++ b/server/services/mediaAssetIndex/galleryCollections.db.test.js
@@ -1,10 +1,14 @@
-import { afterAll, beforeAll, expect, it, vi } from 'vitest';
+import { afterAll, beforeAll, beforeEach, expect, it as vitestIt, vi } from 'vitest';
import pg from 'pg';
import { mediaDdl } from '../../lib/db/schema/media.js';
import { requireDbOrSkip } from '../../lib/dbTestGate.js';
import { mediaFixture } from '../../../scripts/perf/collectionFixtureData.js';
import { listGalleryCollectionSummaries, listGalleryFacets, listGalleryPage } from './gallery.js';
import { compactGalleryRecord } from './logic.js';
+import { ownTestBodies } from '../../lib/mockPathsDataRoot.js';
+import { createDbPhaseDiagnostics } from '../../test/dbPhaseDiagnostics.js';
+
+const { it, drain } = ownTestBodies(vitestIt);
const state = vi.hoisted(() => ({ file: false, images: [], videos: [], collections: [], annotations: {}, query: vi.fn() }));
vi.mock('../../lib/runtimeEnv.js', () => ({ isTestRunner: () => state.file }));
@@ -18,6 +22,8 @@ vi.mock('../mediaCollections.js', () => ({ listCollections: async () => state.co
vi.mock('../peerHostedMedia.js', () => ({ resolveMediaLocations: async items => items.map(() => ({ location: 'local' })) }));
vi.mock('../mediaAnnotations.js', () => ({ listAnnotations: async () => state.annotations }));
let client;
+let observer;
+let diagnostics;
let available = false;
beforeAll(async () => {
vi.stubEnv('MEMORY_BACKEND', 'db');
@@ -29,6 +35,12 @@ beforeAll(async () => {
port: Number(process.env.PGPORT || 5432), user: process.env.PGUSER || 'portos',
password: process.env.PGPASSWORD || 'portos', options: '', connectionTimeoutMillis: 2000 });
await client.connect();
+ if (process.env.PORTOS_DB_PHASE_DIAGNOSTICS === '1') {
+ observer = new pg.Client({ ...client.connectionParameters, options: '', application_name: 'portos-test-diagnostics',
+ connectionTimeoutMillis: 2000, statement_timeout: 500, query_timeout: 750 });
+ observer.on('error', () => {});
+ await observer.connect();
+ }
// A session-private table shadows the index, including for EXPLAIN: no live rows.
const ddl = mediaDdl.filter(sql => /(?:TABLE IF NOT EXISTS media_assets|INDEX IF NOT EXISTS idx_media_assets_)/.test(sql));
for (const sql of ddl) await client.query(sql.replace('CREATE TABLE IF NOT EXISTS', 'CREATE TEMP TABLE'));
@@ -42,16 +54,25 @@ beforeAll(async () => {
});
available = true;
});
-afterAll(async () => { await client?.end(); vi.unstubAllEnvs(); });
+beforeEach(() => drain());
+afterAll(async () => {
+ try { await drain(); }
+ finally {
+ await diagnostics?.drain();
+ await observer?.end();
+ await client?.end();
+ vi.unstubAllEnvs();
+ }
+});
-async function seed(images, indexedVideos = []) {
+async function seed(images, indexedVideos = [], phase = (_name, operation) => operation()) {
state.images = images;
- await client.query('TRUNCATE pg_temp.media_assets');
- const rows = [...images.map(data => ({ kind: 'image', ref: data.filename, data })),
- ...indexedVideos.map(data => ({ kind: 'video', ref: data.id, data }))];
- await client.query(`INSERT INTO pg_temp.media_assets (media_key, kind, ref, data, created_at)
+ await phase('seed-truncate', () => client.query('TRUNCATE pg_temp.media_assets'));
+ const rows = await phase('seed-serialize', () => JSON.stringify([...images.map(data => ({ kind: 'image', ref: data.filename, data })),
+ ...indexedVideos.map(data => ({ kind: 'video', ref: data.id, data }))]));
+ await phase('seed-insert', () => client.query(`INSERT INTO pg_temp.media_assets (media_key, kind, ref, data, created_at)
SELECT kind || ':' || ref, kind, ref, data, COALESCE((data->>'createdAt')::timestamptz, 'epoch')
- FROM jsonb_to_recordset($1::jsonb) AS r(kind text, ref text, data jsonb)`, [JSON.stringify(rows)]);
+ FROM jsonb_to_recordset($1::jsonb) AS r(kind text, ref text, data jsonb)`, [rows]));
}
const member = (kind, ref, addedAt = '2026-01-01') => ({ kind, ref, addedAt });
const disk = async () => state.images;
@@ -208,24 +229,28 @@ it('projects compact no-search pages before SQL with identical rows, previews an
it('sends no full video detail to SQL and does not spill for the synthetic 2400-image/1200-video page', async context => {
if (!available) return context.skip();
- await seed(Array.from({ length: 2400 }, (_, i) => mediaFixture('image', i)));
- state.videos = Array.from({ length: 1200 }, (_, i) => mediaFixture('video', i));
+ diagnostics = createDbPhaseDiagnostics({ onTestFailed: callback => context.onTestFailed(callback), observer, fixturePid: client.processID });
+ const { phase } = diagnostics;
+ const images = await phase('fixture-images', () => Array.from({ length: 2400 }, (_, i) => mediaFixture('image', i)));
+ await seed(images, [], phase);
+ state.videos = await phase('fixture-videos', () => Array.from({ length: 1200 }, (_, i) => mediaFixture('video', i)));
state.collections = [];
- await client.query('RESET work_mem');
+ await phase('reset-work-mem', () => client.query('RESET work_mem'));
const measure = async compact => {
+ const label = compact ? 'compact' : 'full';
state.query.mockClear();
- const page = await listGalleryPage({ media: true, kind: 'all', summary: true, hidden: false, compact, limit: 60 });
+ const page = await phase(`${label}-public-query`, () => listGalleryPage({ media: true, kind: 'all', summary: true, hidden: false, compact, limit: 60 }));
expect(state.query).toHaveBeenCalledTimes(1);
const [sql, params] = state.query.mock.calls[0];
// Measure the actual public-boundary result without executing the same query again.
const result = await state.query.mock.results[0].value;
- const explained = await client.query(`EXPLAIN (ANALYZE, BUFFERS, FORMAT JSON) ${sql}`, params);
+ const explained = await phase(`${label}-explain`, () => client.query(`EXPLAIN (ANALYZE, BUFFERS, FORMAT JSON) ${sql}`, params));
const nodes = planNodes(explained.rows[0]['QUERY PLAN'][0].Plan);
- return { page, params, videoBytes: Buffer.byteLength(params[0]), resultBytes: Buffer.byteLength(JSON.stringify(result.rows)),
+ return phase(`${label}-result-sizes`, () => ({ page, params, videoBytes: Buffer.byteLength(params[0]), resultBytes: Buffer.byteLength(JSON.stringify(result.rows)),
responseBytes: Buffer.byteLength(JSON.stringify(page)),
tempReadBlocks: Math.max(...nodes.map(node => node['Temp Read Blocks'] || 0)),
tempWrittenBlocks: Math.max(...nodes.map(node => node['Temp Written Blocks'] || 0)),
- executionMs: explained.rows[0]['QUERY PLAN'][0]['Execution Time'] };
+ executionMs: explained.rows[0]['QUERY PLAN'][0]['Execution Time'] }));
};
const before = await measure(false);
const after = await measure(true);
diff --git a/server/test/dbPhaseDiagnostics.js b/server/test/dbPhaseDiagnostics.js
new file mode 100644
index 0000000000..9d604d6249
--- /dev/null
+++ b/server/test/dbPhaseDiagnostics.js
@@ -0,0 +1,68 @@
+import { createTestPhaseDiagnostics } from './testPhaseDiagnostics.js';
+import { DB_RUN_ADVISORY_LOCK_KEY } from './dbRunExclusivity.js';
+
+// No SQL text, identities, addresses or record values leave the test database.
+const SNAPSHOT_SQL = `SELECT a.pid = $1 AS fixture, count(*) OVER ()::int AS session_count,
+ a.state, a.wait_event_type, a.wait_event,
+ floor(extract(epoch FROM (clock_timestamp() - a.xact_start)) * 1000)::float8 AS transaction_ms,
+ cardinality(pg_blocking_pids(a.pid)) AS blockers,
+ (SELECT count(*)::int FROM pg_locks l WHERE l.pid = a.pid AND NOT l.granted) AS waiting_locks,
+ EXISTS (SELECT 1 FROM pg_locks l WHERE l.pid = a.pid AND l.locktype = 'advisory'
+ AND l.classid = 0 AND l.objid = $2 AND l.granted) AS run_lock
+ FROM pg_stat_activity a
+ WHERE current_database() = 'portos_test' AND a.datname = 'portos_test'
+ AND a.backend_type = 'client backend' AND a.pid <> pg_backend_pid()
+ ORDER BY fixture DESC, run_lock DESC, a.pid LIMIT 16`;
+const fields = ['fixture', 'session_count', 'state', 'wait_event_type', 'wait_event', 'transaction_ms', 'blockers', 'waiting_locks', 'run_lock'];
+
+/** Opt-in observation; the caller owns the observer connection and its query deadline. */
+export function createDbPhaseDiagnostics({ onTestFailed, observer, fixturePid,
+ enabled = process.env.PORTOS_DB_PHASE_DIAGNOSTICS === '1',
+ write = line => process.stdout.write(`${line}\n`), intervalMs = 1000 } = {}) {
+ const track = createTestPhaseDiagnostics(onTestFailed);
+ if (!enabled) return { phase: track, drain: async () => {} };
+ const evidence = [];
+ let active;
+ let pending = Promise.resolve();
+ let sampling = false;
+ const record = event => {
+ if (evidence.length === 64) evidence.shift();
+ evidence.push(event);
+ write(`DB phases: ${JSON.stringify(event)}`);
+ };
+ const metrics = () => ({ elapsedMs: Math.round(performance.now() - active.started),
+ cpuMs: Math.round(Object.values(process.cpuUsage(active.cpu)).reduce((sum, value) => sum + value, 0) / 1000) });
+ onTestFailed(({ task }) => {
+ const summary = `DB phase evidence: ${JSON.stringify({ active: active && { phase: active.name, ...metrics() }, recent: evidence })}`;
+ for (const error of task.result.errors) {
+ error.message += `\n${summary}`;
+ if (error.stack) error.stack += `\n${summary}`;
+ }
+ });
+ const sample = (name, count) => {
+ if (sampling || count.value === 3) return;
+ count.value++;
+ sampling = true;
+ pending = Promise.resolve().then(() => observer.query(SNAPSHOT_SQL, [fixturePid, DB_RUN_ADVISORY_LOCK_KEY])).then(({ rows }) => {
+ record({ phase: name, sessions: rows.map(row => Object.fromEntries(fields.map(key => [key, row[key]]))) });
+ }, () => record({ phase: name, snapshot: 'unavailable' })).finally(() => { sampling = false; });
+ };
+ return {
+ phase: (name, operation) => track(name, async () => {
+ active = { name, started: performance.now(), cpu: process.cpuUsage() };
+ record({ phase: name, state: 'start' });
+ const count = { value: 0 };
+ sample(name, count);
+ const timer = setInterval(() => sample(name, count), intervalMs);
+ timer.unref?.();
+ try { return await operation(); }
+ finally {
+ clearInterval(timer);
+ record({ phase: name, state: 'settled', ...metrics() });
+ active = undefined;
+ await pending;
+ }
+ }),
+ drain: () => pending,
+ };
+}
diff --git a/server/test/dbPhaseDiagnostics.test.js b/server/test/dbPhaseDiagnostics.test.js
new file mode 100644
index 0000000000..8b477596b5
--- /dev/null
+++ b/server/test/dbPhaseDiagnostics.test.js
@@ -0,0 +1,66 @@
+import { afterEach, expect, it, vi } from 'vitest';
+import { createDbPhaseDiagnostics } from './dbPhaseDiagnostics.js';
+
+afterEach(() => vi.useRealTimers());
+
+// Pins the observer's privacy boundary and prevents a slow observer from
+// spawning overlapping work or surviving the operation's settlement.
+it('bounds test-only snapshots and drains the observer without exposing query text', async () => {
+ vi.useFakeTimers();
+ const snapshot = Promise.withResolvers();
+ const operation = Promise.withResolvers();
+ const observer = { query: vi.fn(() => snapshot.promise) };
+ const lines = [];
+ const diagnostics = createDbPhaseDiagnostics({ onTestFailed: () => {}, observer, fixturePid: 42,
+ enabled: true, write: line => lines.push(line) });
+ const run = diagnostics.phase('seed-insert', () => operation.promise);
+ await vi.advanceTimersByTimeAsync(5000);
+ expect(observer.query).toHaveBeenCalledOnce();
+ const [sql, params] = observer.query.mock.calls[0];
+ expect(sql).toContain("current_database() = 'portos_test' AND a.datname = 'portos_test'");
+ expect(sql).toContain('LIMIT 16');
+ expect(params).toEqual([42, 5977002]);
+ operation.resolve('original result');
+ let settled = false;
+ void run.then(() => { settled = true; });
+ await vi.advanceTimersByTimeAsync(5000);
+ expect(settled).toBe(false);
+ snapshot.resolve({ rows: [{ fixture: true, state: 'active', wait_event: 'DataFileRead',
+ query: 'private query', client_addr: 'private address' }] });
+ expect(await run).toBe('original result');
+ await diagnostics.drain();
+ await vi.advanceTimersByTimeAsync(5000);
+ expect(observer.query).toHaveBeenCalledOnce();
+ expect(lines.join('\n')).toContain('DataFileRead');
+ expect(lines.join('\n')).not.toContain('private');
+});
+
+it('retains timeout phase and CPU evidence with at most three snapshots per phase', async () => {
+ vi.useFakeTimers();
+ const hooks = [];
+ const observer = { query: vi.fn(async () => ({ rows: [] })) };
+ const operation = Promise.withResolvers();
+ const diagnostics = createDbPhaseDiagnostics({ onTestFailed: hook => hooks.push(hook), observer,
+ fixturePid: 42, enabled: true, write: () => {} });
+ const run = diagnostics.phase('compact-explain', () => operation.promise);
+ await vi.advanceTimersByTimeAsync(10000);
+ expect(observer.query).toHaveBeenCalledTimes(3);
+ const error = { message: 'original timeout', stack: 'original stack' };
+ for (const hook of hooks) hook({ task: { result: { errors: [error] } } });
+ expect(error.message).toContain('original timeout');
+ expect(error.message).toContain('active=compact-explain');
+ expect(error.message).toContain('"cpuMs":');
+ expect(error.message).toContain('"phase":"compact-explain"');
+ operation.resolve();
+ await run;
+ await diagnostics.drain();
+});
+
+it('keeps observation opt-in and preserves operation failures', async () => {
+ const observer = { query: vi.fn() };
+ const diagnostics = createDbPhaseDiagnostics({ onTestFailed: () => {}, observer, enabled: false });
+ const error = new Error('synthetic operation failure');
+ await expect(diagnostics.phase('full-public-query', () => { throw error; })).rejects.toBe(error);
+ await diagnostics.drain();
+ expect(observer.query).not.toHaveBeenCalled();
+});