Skip to content

Self-hosted, Self-Improving, Federated Everything App — dev management, agent orchestration, persistent mind, digital twin, second brain, and creative pipeline from a single dashboard over Tailscale to port the user into the OS

test: serialize DB suite runs across worktrees with an advisory lock (#12371)

[._.]/ Adam Eivyabfeaa3b1602
diff --git a/docs/CONTRIBUTING.md b/docs/CONTRIBUTING.md
index 30dcdf157c..6edaa839a4 100644
--- a/docs/CONTRIBUTING.md
+++ b/docs/CONTRIBUTING.md
@@ -243,6 +243,11 @@ npm run setup:db:test
npm run test:db
```
+Every worktree shares one `portos_test`, and some DB suites restore a whole-database
+dump, so `test:db` takes a session-level Postgres advisory lock for the run
+(`server/test/dbRunExclusivity.js`). A second run from another worktree prints
+`Another DB test run holds portos_test` and waits for the first to finish.
+
Supported full local validation uses the CI worker budget. An unbounded
`npm test` can oversubscribe a loaded machine: real media, prompt-catalog,
and nested-worker cases then hit their existing budgets. A green focused
diff --git a/server/test/dbRunExclusivity.js b/server/test/dbRunExclusivity.js
new file mode 100644
index 0000000000..38c03ec6fa
--- /dev/null
+++ b/server/test/dbRunExclusivity.js
@@ -0,0 +1,72 @@
+/**
+ * Global setup for `vitest.config.db.js`: one DB suite run at a time per `portos_test`.
+ *
+ * There is ONE Postgres per install and every git worktree shares its
+ * `portos_test`. `fileParallelism: false` only serializes files inside one
+ * Vitest process; a second run from another worktree (parallel claim agents)
+ * overlaps it. Suites like `backup.db.test.js` restore a whole-database dump
+ * (`--clean`), which discards every other run's fixture rows mid-case — the
+ * `Production project not found` flake in `stages.db.test.js` (#12371).
+ *
+ * A session-level advisory lock on the test database is held for the whole
+ * run. It is connection-scoped, so a killed run releases it automatically.
+ */
+import pg from 'pg';
+
+// Distinct from SCHEMA_DDL_ADVISORY_LOCK_KEY (5977001, lib/db.js) — ensureSchema
+// takes that one inside a run and would otherwise deadlock against this holder.
+export const DB_RUN_ADVISORY_LOCK_KEY = 5977002;
+
+const WAIT_LOG_INTERVAL_MS = 30_000;
+
+/** Connection target for the test database, mirroring lib/db.js defaults. */
+function testDatabaseConfig(env) {
+ return {
+ host: env.PGHOST || 'localhost',
+ port: parseInt(env.PGPORT || '5432', 10),
+ database: env.PGTESTDATABASE || 'portos_test',
+ user: env.PGUSER || 'portos',
+ password: env.PGPASSWORD || 'portos',
+ };
+}
+
+/**
+ * Acquire the run lock. Resolves to a release function; resolves to a no-op
+ * when the database is unreachable or not a `_test` database (the suites then
+ * skip themselves via requireDbOrSkip, so there is nothing to protect).
+ */
+export async function acquireDbRunLock({
+ env = process.env,
+ createClient = config => new pg.Client(config),
+ log = console.log,
+ waitLogIntervalMs = WAIT_LOG_INTERVAL_MS,
+} = {}) {
+ const config = testDatabaseConfig(env);
+ if (!/_test$/.test(config.database)) return async () => {};
+
+ const client = createClient(config);
+ client.on?.('error', () => {});
+ try {
+ await client.connect();
+ } catch {
+ await client.end().catch(() => {});
+ return async () => {};
+ }
+
+ const { rows: [{ locked }] } = await client.query('SELECT pg_try_advisory_lock($1) AS locked', [DB_RUN_ADVISORY_LOCK_KEY]);
+ if (!locked) {
+ log(`⏳ Another DB test run holds ${config.database} — waiting for it to finish…`);
+ const timer = setInterval(() => log(`⏳ Still waiting for the DB test run holding ${config.database}…`), waitLogIntervalMs);
+ timer.unref?.();
+ await client.query('SELECT pg_advisory_lock($1)', [DB_RUN_ADVISORY_LOCK_KEY]).finally(() => clearInterval(timer));
+ }
+
+ return async () => {
+ await client.query('SELECT pg_advisory_unlock($1)', [DB_RUN_ADVISORY_LOCK_KEY]).catch(() => {});
+ await client.end().catch(() => {});
+ };
+}
+
+export async function setup() {
+ return acquireDbRunLock();
+}
diff --git a/server/test/dbRunExclusivity.test.js b/server/test/dbRunExclusivity.test.js
new file mode 100644
index 0000000000..450ae741ef
--- /dev/null
+++ b/server/test/dbRunExclusivity.test.js
@@ -0,0 +1,76 @@
+import { describe, expect, it, vi } from 'vitest';
+import { DB_RUN_ADVISORY_LOCK_KEY, acquireDbRunLock } from './dbRunExclusivity.js';
+import dbConfig from '../vitest.config.db.js';
+
+// A lock that is free the first time it is tried; `held` makes try-lock fail and
+// the blocking call wait until `grant()` — the shape of a peer run in progress.
+function fakeClient({ held = false, connectError = null } = {}) {
+ const grant = Promise.withResolvers();
+ const calls = [];
+ const client = {
+ on: vi.fn(),
+ connect: vi.fn(async () => { if (connectError) throw connectError; }),
+ end: vi.fn(async () => {}),
+ query: vi.fn(async sql => {
+ calls.push(sql);
+ if (sql.includes('pg_try_advisory_lock')) return { rows: [{ locked: !held }] };
+ if (sql.includes('pg_advisory_lock')) await grant.promise;
+ return { rows: [] };
+ }),
+ };
+ return { client, calls, grant: () => grant.resolve() };
+}
+
+describe('acquireDbRunLock', () => {
+ it('holds the lock for the whole run and unlocks on release', async () => {
+ const { client, calls } = fakeClient();
+ const release = await acquireDbRunLock({ env: {}, createClient: () => client, log: vi.fn() });
+ expect(calls).toEqual(['SELECT pg_try_advisory_lock($1) AS locked']);
+ expect(client.end).not.toHaveBeenCalled();
+ await release();
+ expect(calls.at(-1)).toBe('SELECT pg_advisory_unlock($1)');
+ expect(client.query).toHaveBeenLastCalledWith('SELECT pg_advisory_unlock($1)', [DB_RUN_ADVISORY_LOCK_KEY]);
+ expect(client.end).toHaveBeenCalledOnce();
+ });
+
+ it('queues behind a run that holds the lock instead of overlapping it', async () => {
+ const { client, calls, grant } = fakeClient({ held: true });
+ const log = vi.fn();
+ let acquired = false;
+ const pending = acquireDbRunLock({ env: {}, createClient: () => client, log }).then(release => { acquired = true; return release; });
+ await vi.waitFor(() => expect(calls).toContain('SELECT pg_advisory_lock($1)'));
+ expect(acquired).toBe(false);
+ expect(log).toHaveBeenCalledWith(expect.stringContaining('waiting'));
+ grant();
+ await (await pending)();
+ expect(acquired).toBe(true);
+ });
+
+ it('targets PGTESTDATABASE and refuses to lock a non-test database', async () => {
+ const config = vi.fn(() => fakeClient().client);
+ await acquireDbRunLock({ env: { PGTESTDATABASE: 'custom_test' }, createClient: config, log: vi.fn() });
+ expect(config).toHaveBeenCalledWith(expect.objectContaining({ database: 'custom_test' }));
+
+ const refused = vi.fn();
+ const release = await acquireDbRunLock({ env: { PGTESTDATABASE: 'portos' }, createClient: refused, log: vi.fn() });
+ expect(refused).not.toHaveBeenCalled();
+ await release();
+ });
+
+ it('does not block the run when Postgres is unreachable (the suites skip themselves)', async () => {
+ const { client } = fakeClient({ connectError: new Error('ECONNREFUSED') });
+ const release = await acquireDbRunLock({ env: {}, createClient: () => client, log: vi.fn() });
+ expect(client.query).not.toHaveBeenCalled();
+ await release();
+ });
+});
+
+describe('vitest.config.db.js', () => {
+ it('takes the run lock before the temp-root setup', () => {
+ const setups = dbConfig.test.globalSetup.map(path => path.replaceAll('\\', '/'));
+ const lock = setups.findIndex(path => path.endsWith('/test/dbRunExclusivity.js'));
+ const tempRoot = setups.findIndex(path => path.endsWith('/scripts/vitestTempRootSetup.js'));
+ expect(lock).toBeGreaterThanOrEqual(0);
+ expect(lock).toBeLessThan(tempRoot);
+ });
+});
diff --git a/server/vitest.config.db.js b/server/vitest.config.db.js
index 04087494f4..28abacc338 100644
--- a/server/vitest.config.db.js
+++ b/server/vitest.config.db.js
@@ -33,6 +33,11 @@ export { DB_TEST_INCLUDE } from '../scripts/lib/dbTestFiles.js';
* The default `vitest.config.js` excludes the same inventory, so `npm test`
* never runs them; `npm run test:db` does, after `npm run setup:db:test`.
*
+ * One run at a time: every worktree shares the single `portos_test`, and a
+ * suite such as backup.db.test.js restores a whole-database dump, wiping another
+ * run's fixture rows mid-case (#12371). test/dbRunExclusivity.js holds a
+ * session-level advisory lock on the test database for the run's lifetime.
+ *
* Adding a new DB-backed test? Add it to `DB_TEST_INCLUDE` in scripts/lib/dbTestFiles.js — the
* `*.db.test.js` naming convention is documentation, not a glob match, so a new
* suite silently never runs until it's listed. db.guards.test.js fails if a
@@ -50,7 +55,11 @@ const config = defineConfig({
hookTimeout: 30000,
// Explicit 'vitest' imports only, matching vitest.config.js (#9049).
globals: false,
- globalSetup: [fileURLToPath(new URL('../scripts/vitestTempRootSetup.js', import.meta.url))],
+ // The lock comes first so a queued run waits before touching the shared database or temp root.
+ globalSetup: [
+ fileURLToPath(new URL('./test/dbRunExclusivity.js', import.meta.url)),
+ fileURLToPath(new URL('../scripts/vitestTempRootSetup.js', import.meta.url)),
+ ],
setupFiles: ['./vitest.setup.js'],
// One file at a time — these suites assume exclusive access to their tables.
fileParallelism: false,