Self-hosted, Self-Improving, Federated Everything App — dev management, agent orchestration, persistent mind, digital twin, second brain, and creative pipeline from a single dashboard over Tailscale to port the user into the OS
JavaScript ai-agentsdashboarddeveloper-toolsdigital-twinquantified-selfsecond-brainself-hostedtailscale
test: retain gallery timeout phase and database wait evidence (#13369)
Adam Eivy893784e3a4fc
diff --git a/docs/CONTRIBUTING.md b/docs/CONTRIBUTING.mdindex 24e4ac3dd0..8d3d863c34 100644--- a/docs/CONTRIBUTING.md+++ b/docs/CONTRIBUTING.md@@ -311,6 +311,19 @@ the database restore suite drains a timed-out case before the next one touchesunloaded machine, so a concurrent run can time out cases a sequential runpasses.+For the intermittent synthetic gallery-page timeout investigation (#13369),+`PORTOS_DB_PHASE_DIAGNOSTICS=1 npm run pregate` adds wall/CPU timings for fixture+construction, seeding, public calls, EXPLAINs and result serialization. To obtain+the complete serial DB workload regardless of test-memo reuse, run+`PORTOS_DB_PHASE_DIAGNOSTICS=1 npm run test:db` after the earlier validation+stages finish. The observer reads only `portos_test` session state, waits,+transaction ages and run-lock ownership: no query text, network addresses or+record contents. It samples at most three times per phase, caps each snapshot+at 16 sessions, and uses a separate connection with a 500 ms statement timeout.+Opt-in diagnostics add observation work; original test deadlines stay intact.+Failure messages retain bounded phase evidence even with quiet reporting.+A passing run without a reproduced slow phase does not resolve the timeout.+For server watch mode, run this alternative from the repository root:```bashdiff --git a/server/services/mediaAssetIndex/galleryCollections.db.test.js b/server/services/mediaAssetIndex/galleryCollections.db.test.jsindex be3d050da2..824a903408 100644--- a/server/services/mediaAssetIndex/galleryCollections.db.test.js+++ b/server/services/mediaAssetIndex/galleryCollections.db.test.js@@ -1,10 +1,14 @@-import { afterAll, beforeAll, expect, it, vi } from 'vitest';+import { afterAll, beforeAll, beforeEach, expect, it as vitestIt, vi } from 'vitest';import pg from 'pg';import { mediaDdl } from '../../lib/db/schema/media.js';import { requireDbOrSkip } from '../../lib/dbTestGate.js';import { mediaFixture } from '../../../scripts/perf/collectionFixtureData.js';import { listGalleryCollectionSummaries, listGalleryFacets, listGalleryPage } from './gallery.js';import { compactGalleryRecord } from './logic.js';+import { ownTestBodies } from '../../lib/mockPathsDataRoot.js';+import { createDbPhaseDiagnostics } from '../../test/dbPhaseDiagnostics.js';++const { it, drain } = ownTestBodies(vitestIt);const state = vi.hoisted(() => ({ file: false, images: [], videos: [], collections: [], annotations: {}, query: vi.fn() }));vi.mock('../../lib/runtimeEnv.js', () => ({ isTestRunner: () => state.file }));@@ -18,6 +22,8 @@ vi.mock('../mediaCollections.js', () => ({ listCollections: async () => state.covi.mock('../peerHostedMedia.js', () => ({ resolveMediaLocations: async items => items.map(() => ({ location: 'local' })) }));vi.mock('../mediaAnnotations.js', () => ({ listAnnotations: async () => state.annotations }));let client;+let observer;+let diagnostics;let available = false;beforeAll(async () => {vi.stubEnv('MEMORY_BACKEND', 'db');@@ -29,6 +35,12 @@ beforeAll(async () => {port: Number(process.env.PGPORT || 5432), user: process.env.PGUSER || 'portos',password: process.env.PGPASSWORD || 'portos', options: '', connectionTimeoutMillis: 2000 });await client.connect();+ if (process.env.PORTOS_DB_PHASE_DIAGNOSTICS === '1') {+ observer = new pg.Client({ ...client.connectionParameters, options: '', application_name: 'portos-test-diagnostics',+ connectionTimeoutMillis: 2000, statement_timeout: 500, query_timeout: 750 });+ observer.on('error', () => {});+ await observer.connect();+ }// A session-private table shadows the index, including for EXPLAIN: no live rows.const ddl = mediaDdl.filter(sql => /(?:TABLE IF NOT EXISTS media_assets|INDEX IF NOT EXISTS idx_media_assets_)/.test(sql));for (const sql of ddl) await client.query(sql.replace('CREATE TABLE IF NOT EXISTS', 'CREATE TEMP TABLE'));@@ -42,16 +54,25 @@ beforeAll(async () => {});available = true;});-afterAll(async () => { await client?.end(); vi.unstubAllEnvs(); });+beforeEach(() => drain());+afterAll(async () => {+ try { await drain(); }+ finally {+ await diagnostics?.drain();+ await observer?.end();+ await client?.end();+ vi.unstubAllEnvs();+ }+});-async function seed(images, indexedVideos = []) {+async function seed(images, indexedVideos = [], phase = (_name, operation) => operation()) {state.images = images;- await client.query('TRUNCATE pg_temp.media_assets');- const rows = [...images.map(data => ({ kind: 'image', ref: data.filename, data })),- ...indexedVideos.map(data => ({ kind: 'video', ref: data.id, data }))];- await client.query(`INSERT INTO pg_temp.media_assets (media_key, kind, ref, data, created_at)+ await phase('seed-truncate', () => client.query('TRUNCATE pg_temp.media_assets'));+ const rows = await phase('seed-serialize', () => JSON.stringify([...images.map(data => ({ kind: 'image', ref: data.filename, data })),+ ...indexedVideos.map(data => ({ kind: 'video', ref: data.id, data }))]));+ await phase('seed-insert', () => client.query(`INSERT INTO pg_temp.media_assets (media_key, kind, ref, data, created_at)SELECT kind || ':' || ref, kind, ref, data, COALESCE((data->>'createdAt')::timestamptz, 'epoch')- FROM jsonb_to_recordset($1::jsonb) AS r(kind text, ref text, data jsonb)`, [JSON.stringify(rows)]);+ FROM jsonb_to_recordset($1::jsonb) AS r(kind text, ref text, data jsonb)`, [rows]));}const member = (kind, ref, addedAt = '2026-01-01') => ({ kind, ref, addedAt });const disk = async () => state.images;@@ -208,24 +229,28 @@ it('projects compact no-search pages before SQL with identical rows, previews anit('sends no full video detail to SQL and does not spill for the synthetic 2400-image/1200-video page', async context => {if (!available) return context.skip();- await seed(Array.from({ length: 2400 }, (_, i) => mediaFixture('image', i)));- state.videos = Array.from({ length: 1200 }, (_, i) => mediaFixture('video', i));+ diagnostics = createDbPhaseDiagnostics({ onTestFailed: callback => context.onTestFailed(callback), observer, fixturePid: client.processID });+ const { phase } = diagnostics;+ const images = await phase('fixture-images', () => Array.from({ length: 2400 }, (_, i) => mediaFixture('image', i)));+ await seed(images, [], phase);+ state.videos = await phase('fixture-videos', () => Array.from({ length: 1200 }, (_, i) => mediaFixture('video', i)));state.collections = [];- await client.query('RESET work_mem');+ await phase('reset-work-mem', () => client.query('RESET work_mem'));const measure = async compact => {+ const label = compact ? 'compact' : 'full';state.query.mockClear();- const page = await listGalleryPage({ media: true, kind: 'all', summary: true, hidden: false, compact, limit: 60 });+ const page = await phase(`${label}-public-query`, () => listGalleryPage({ media: true, kind: 'all', summary: true, hidden: false, compact, limit: 60 }));expect(state.query).toHaveBeenCalledTimes(1);const [sql, params] = state.query.mock.calls[0];// Measure the actual public-boundary result without executing the same query again.const result = await state.query.mock.results[0].value;- const explained = await client.query(`EXPLAIN (ANALYZE, BUFFERS, FORMAT JSON) ${sql}`, params);+ const explained = await phase(`${label}-explain`, () => client.query(`EXPLAIN (ANALYZE, BUFFERS, FORMAT JSON) ${sql}`, params));const nodes = planNodes(explained.rows[0]['QUERY PLAN'][0].Plan);- return { page, params, videoBytes: Buffer.byteLength(params[0]), resultBytes: Buffer.byteLength(JSON.stringify(result.rows)),+ return phase(`${label}-result-sizes`, () => ({ page, params, videoBytes: Buffer.byteLength(params[0]), resultBytes: Buffer.byteLength(JSON.stringify(result.rows)),responseBytes: Buffer.byteLength(JSON.stringify(page)),tempReadBlocks: Math.max(...nodes.map(node => node['Temp Read Blocks'] || 0)),tempWrittenBlocks: Math.max(...nodes.map(node => node['Temp Written Blocks'] || 0)),- executionMs: explained.rows[0]['QUERY PLAN'][0]['Execution Time'] };+ executionMs: explained.rows[0]['QUERY PLAN'][0]['Execution Time'] }));};const before = await measure(false);const after = await measure(true);diff --git a/server/test/dbPhaseDiagnostics.js b/server/test/dbPhaseDiagnostics.jsnew file mode 100644index 0000000000..9d604d6249--- /dev/null+++ b/server/test/dbPhaseDiagnostics.js@@ -0,0 +1,68 @@+import { createTestPhaseDiagnostics } from './testPhaseDiagnostics.js';+import { DB_RUN_ADVISORY_LOCK_KEY } from './dbRunExclusivity.js';++// No SQL text, identities, addresses or record values leave the test database.+const SNAPSHOT_SQL = `SELECT a.pid = $1 AS fixture, count(*) OVER ()::int AS session_count,+ a.state, a.wait_event_type, a.wait_event,+ floor(extract(epoch FROM (clock_timestamp() - a.xact_start)) * 1000)::float8 AS transaction_ms,+ cardinality(pg_blocking_pids(a.pid)) AS blockers,+ (SELECT count(*)::int FROM pg_locks l WHERE l.pid = a.pid AND NOT l.granted) AS waiting_locks,+ EXISTS (SELECT 1 FROM pg_locks l WHERE l.pid = a.pid AND l.locktype = 'advisory'+ AND l.classid = 0 AND l.objid = $2 AND l.granted) AS run_lock+ FROM pg_stat_activity a+ WHERE current_database() = 'portos_test' AND a.datname = 'portos_test'+ AND a.backend_type = 'client backend' AND a.pid <> pg_backend_pid()+ ORDER BY fixture DESC, run_lock DESC, a.pid LIMIT 16`;+const fields = ['fixture', 'session_count', 'state', 'wait_event_type', 'wait_event', 'transaction_ms', 'blockers', 'waiting_locks', 'run_lock'];++/** Opt-in observation; the caller owns the observer connection and its query deadline. */+export function createDbPhaseDiagnostics({ onTestFailed, observer, fixturePid,+ enabled = process.env.PORTOS_DB_PHASE_DIAGNOSTICS === '1',+ write = line => process.stdout.write(`${line}\n`), intervalMs = 1000 } = {}) {+ const track = createTestPhaseDiagnostics(onTestFailed);+ if (!enabled) return { phase: track, drain: async () => {} };+ const evidence = [];+ let active;+ let pending = Promise.resolve();+ let sampling = false;+ const record = event => {+ if (evidence.length === 64) evidence.shift();+ evidence.push(event);+ write(`DB phases: ${JSON.stringify(event)}`);+ };+ const metrics = () => ({ elapsedMs: Math.round(performance.now() - active.started),+ cpuMs: Math.round(Object.values(process.cpuUsage(active.cpu)).reduce((sum, value) => sum + value, 0) / 1000) });+ onTestFailed(({ task }) => {+ const summary = `DB phase evidence: ${JSON.stringify({ active: active && { phase: active.name, ...metrics() }, recent: evidence })}`;+ for (const error of task.result.errors) {+ error.message += `\n${summary}`;+ if (error.stack) error.stack += `\n${summary}`;+ }+ });+ const sample = (name, count) => {+ if (sampling || count.value === 3) return;+ count.value++;+ sampling = true;+ pending = Promise.resolve().then(() => observer.query(SNAPSHOT_SQL, [fixturePid, DB_RUN_ADVISORY_LOCK_KEY])).then(({ rows }) => {+ record({ phase: name, sessions: rows.map(row => Object.fromEntries(fields.map(key => [key, row[key]]))) });+ }, () => record({ phase: name, snapshot: 'unavailable' })).finally(() => { sampling = false; });+ };+ return {+ phase: (name, operation) => track(name, async () => {+ active = { name, started: performance.now(), cpu: process.cpuUsage() };+ record({ phase: name, state: 'start' });+ const count = { value: 0 };+ sample(name, count);+ const timer = setInterval(() => sample(name, count), intervalMs);+ timer.unref?.();+ try { return await operation(); }+ finally {+ clearInterval(timer);+ record({ phase: name, state: 'settled', ...metrics() });+ active = undefined;+ await pending;+ }+ }),+ drain: () => pending,+ };+}diff --git a/server/test/dbPhaseDiagnostics.test.js b/server/test/dbPhaseDiagnostics.test.jsnew file mode 100644index 0000000000..8b477596b5--- /dev/null+++ b/server/test/dbPhaseDiagnostics.test.js@@ -0,0 +1,66 @@+import { afterEach, expect, it, vi } from 'vitest';+import { createDbPhaseDiagnostics } from './dbPhaseDiagnostics.js';++afterEach(() => vi.useRealTimers());++// Pins the observer's privacy boundary and prevents a slow observer from+// spawning overlapping work or surviving the operation's settlement.+it('bounds test-only snapshots and drains the observer without exposing query text', async () => {+ vi.useFakeTimers();+ const snapshot = Promise.withResolvers();+ const operation = Promise.withResolvers();+ const observer = { query: vi.fn(() => snapshot.promise) };+ const lines = [];+ const diagnostics = createDbPhaseDiagnostics({ onTestFailed: () => {}, observer, fixturePid: 42,+ enabled: true, write: line => lines.push(line) });+ const run = diagnostics.phase('seed-insert', () => operation.promise);+ await vi.advanceTimersByTimeAsync(5000);+ expect(observer.query).toHaveBeenCalledOnce();+ const [sql, params] = observer.query.mock.calls[0];+ expect(sql).toContain("current_database() = 'portos_test' AND a.datname = 'portos_test'");+ expect(sql).toContain('LIMIT 16');+ expect(params).toEqual([42, 5977002]);+ operation.resolve('original result');+ let settled = false;+ void run.then(() => { settled = true; });+ await vi.advanceTimersByTimeAsync(5000);+ expect(settled).toBe(false);+ snapshot.resolve({ rows: [{ fixture: true, state: 'active', wait_event: 'DataFileRead',+ query: 'private query', client_addr: 'private address' }] });+ expect(await run).toBe('original result');+ await diagnostics.drain();+ await vi.advanceTimersByTimeAsync(5000);+ expect(observer.query).toHaveBeenCalledOnce();+ expect(lines.join('\n')).toContain('DataFileRead');+ expect(lines.join('\n')).not.toContain('private');+});++it('retains timeout phase and CPU evidence with at most three snapshots per phase', async () => {+ vi.useFakeTimers();+ const hooks = [];+ const observer = { query: vi.fn(async () => ({ rows: [] })) };+ const operation = Promise.withResolvers();+ const diagnostics = createDbPhaseDiagnostics({ onTestFailed: hook => hooks.push(hook), observer,+ fixturePid: 42, enabled: true, write: () => {} });+ const run = diagnostics.phase('compact-explain', () => operation.promise);+ await vi.advanceTimersByTimeAsync(10000);+ expect(observer.query).toHaveBeenCalledTimes(3);+ const error = { message: 'original timeout', stack: 'original stack' };+ for (const hook of hooks) hook({ task: { result: { errors: [error] } } });+ expect(error.message).toContain('original timeout');+ expect(error.message).toContain('active=compact-explain');+ expect(error.message).toContain('"cpuMs":');+ expect(error.message).toContain('"phase":"compact-explain"');+ operation.resolve();+ await run;+ await diagnostics.drain();+});++it('keeps observation opt-in and preserves operation failures', async () => {+ const observer = { query: vi.fn() };+ const diagnostics = createDbPhaseDiagnostics({ onTestFailed: () => {}, observer, enabled: false });+ const error = new Error('synthetic operation failure');+ await expect(diagnostics.phase('full-public-query', () => { throw error; })).rejects.toBe(error);+ await diagnostics.drain();+ expect(observer.query).not.toHaveBeenCalled();+});