Where agent teams gather. Cross-platform team chat where AI agents are first-class teammates — with cloud or bring-your-own (Claude Code / Codex) brains.
TypeScript
fix(db): run the frozen baseline one statement at a time (#178)
yetone7232e4d9dc76
diff --git a/docs/decisions/0003-versioned-schema-migration-ownership.md b/docs/decisions/0003-versioned-schema-migration-ownership.mdindex 7cff40a..4cc7ce1 100644--- a/docs/decisions/0003-versioned-schema-migration-ownership.md+++ b/docs/decisions/0003-versioned-schema-migration-ownership.md@@ -45,6 +45,12 @@ application replica has no reason to create or alter schema while starting.version identity and required-index health.- A first adoption run executes the frozen legacy baseline once and records it.Subsequent runs read the ledger and execute only newly appended versions.+ The baseline is sent one statement at a time with per-statement lock+ retries: as a single implicit transaction it holds AccessExclusiveLocks on+ ~30 tables until commit and deadlocks against live traffic on every attempt+ (the first v0.14.0 adoption Job failed that way). Every baseline statement+ is idempotent, so a rerun after a partial batch resumes safely; versions+ after the baseline keep their single-transaction atomicity.- Local development and manual Kubernetes installation must run`npm run migrate` before starting the server; `npm run dev:all` performs thatone explicit step automatically.diff --git a/server/src/__tests__/baseline-statement-runner.test.ts b/server/src/__tests__/baseline-statement-runner.test.tsnew file mode 100644index 0000000..7977aeb--- /dev/null+++ b/server/src/__tests__/baseline-statement-runner.test.ts@@ -0,0 +1,103 @@+import assert from 'node:assert/strict'+import { test } from 'node:test'++process.env.CUMORA_RUNTIME_CLIENT = 'http'+process.env.OPENAI_API_KEY ??= 'test-key'++const { frozenBaselineSql, runBaselineStatements, splitSqlStatements } = await import('../db/migrate.js')++test('splitSqlStatements only splits on top-level semicolons', () => {+ const sql = `+ -- a comment; with a semicolon+ CREATE TABLE IF NOT EXISTS a (x TEXT DEFAULT 'semi;colon', y TEXT DEFAULT 'it''s; quoted');+ /* block; comment */+ DO $$ BEGIN+ IF EXISTS (SELECT 1) THEN EXECUTE 'ALTER TABLE a ADD COLUMN IF NOT EXISTS z TEXT'; END IF;+ END $$;+ CREATE FUNCTION f() RETURNS TEXT LANGUAGE plpgsql AS $fn$ BEGIN RETURN 'x;y'; END $fn$;+ INSERT INTO a (x) VALUES ('last') -- trailing; comment+ `+ const statements = splitSqlStatements(sql)+ assert.equal(statements.length, 4)+ assert.match(statements[0], /^-- a comment; with a semicolon\s+CREATE TABLE IF NOT EXISTS a/)+ assert.match(statements[0], /'it''s; quoted'\)$/)+ assert.match(statements[1], /^\/\* block; comment \*\/\s+DO \$\$ BEGIN[\s\S]*END \$\$$/)+ assert.match(statements[2], /^CREATE FUNCTION f\(\)[\s\S]*END \$fn\$$/)+ assert.match(statements[3], /^INSERT INTO a \(x\) VALUES \('last'\) -- trailing; comment$/)+})++test('splitSqlStatements drops comment-only chunks and preserves the frozen baseline verbatim', () => {+ const sql = frozenBaselineSql()+ const statements = splitSqlStatements(sql)+ assert.ok(statements.length > 250, `expected a few hundred statements, got ${statements.length}`)++ // Nothing but whitespace and comment text may be lost by the split.+ const strip = (text: string) => text.replace(/--[^\n]*/g, '').replace(/\/\*[\s\S]*?\*\//g, '').replace(/\s+/g, '')+ assert.equal(strip(statements.join(';')), strip(sql).replace(/;$/, ''))++ for (const statement of statements) {+ assert.doesNotMatch(statement, /^\s*$/)+ assert.notEqual(strip(statement), '', 'a comment-only chunk leaked through')+ const dollarTags = statement.match(/\$[A-Za-z_]*\$/g) ?? []+ assert.equal(dollarTags.length % 2, 0, `unbalanced dollar quoting in: ${statement.slice(0, 80)}`)+ }+ assert.ok(statements.some((statement) => /^DO \$\$/.test(statement)), 'DO blocks must survive as single statements')+})++test('runBaselineStatements retries only lock contention, one statement at a time, in order', async () => {+ const executed: string[] = []+ const sleeps: number[] = []+ let deadlocks = 2+ const client = {+ async query(sql: string) {+ executed.push(sql)+ if (sql.startsWith('ALTER TABLE hot') && deadlocks > 0) {+ deadlocks--+ throw Object.assign(new Error('deadlock detected'), { code: '40P01' })+ }+ },+ }+ const count = await runBaselineStatements(+ client,+ 'CREATE TABLE IF NOT EXISTS a (id TEXT); ALTER TABLE hot ADD COLUMN IF NOT EXISTS x TEXT; INSERT INTO a VALUES (\'z\');',+ { sleep: async (ms) => { sleeps.push(ms) } },+ )+ assert.equal(count, 3)+ assert.deepEqual(executed, [+ 'CREATE TABLE IF NOT EXISTS a (id TEXT)',+ 'ALTER TABLE hot ADD COLUMN IF NOT EXISTS x TEXT',+ 'ALTER TABLE hot ADD COLUMN IF NOT EXISTS x TEXT',+ 'ALTER TABLE hot ADD COLUMN IF NOT EXISTS x TEXT',+ "INSERT INTO a VALUES ('z')",+ ])+ assert.equal(sleeps.length, 2)+ assert.ok(sleeps[1] > sleeps[0], 'backoff must grow')+})++test('runBaselineStatements propagates real errors immediately and gives up after maxAttempts', async () => {+ const executed: string[] = []+ const failing = {+ async query(sql: string) {+ executed.push(sql)+ if (sql.startsWith('ALTER')) throw Object.assign(new Error('relation "nope" does not exist'), { code: '42P01' })+ },+ }+ await assert.rejects(+ runBaselineStatements(failing, 'SELECT 1; ALTER TABLE nope ADD COLUMN x TEXT; SELECT 2', { sleep: async () => {} }),+ (err: unknown) => (err as { code?: string }).code === '42P01',+ )+ assert.deepEqual(executed, ['SELECT 1', 'ALTER TABLE nope ADD COLUMN x TEXT'])++ let attempts = 0+ const alwaysLocked = {+ async query() {+ attempts+++ throw Object.assign(new Error('canceling statement due to lock timeout'), { code: '55P03' })+ },+ }+ await assert.rejects(+ runBaselineStatements(alwaysLocked, 'ALTER TABLE hot ADD COLUMN x TEXT', { sleep: async () => {}, maxAttempts: 3 }),+ (err: unknown) => (err as { code?: string }).code === '55P03',+ )+ assert.equal(attempts, 3)+})diff --git a/server/src/db/migrate.ts b/server/src/db/migrate.tsindex 4e9b47c..bd79f52 100644--- a/server/src/db/migrate.ts+++ b/server/src/db/migrate.ts@@ -2049,6 +2049,141 @@ interface VersionedMigration extends MigrationMetadata {up(client: import('pg').PoolClient): Promise<void>}+/**+ * Split a SQL batch into its top-level statements.+ *+ * Honors single-quoted strings (with `''` escapes), dollar-quoted bodies+ * (`$$ … $$` and `$tag$ … $tag$`), `--` line comments and block comments, so a+ * `;` inside a DO block, a string, or a comment never ends a statement.+ * Comment-only chunks are dropped. Exported for the unit test.+ */+export function splitSqlStatements(sql: string): string[] {+ const statements: string[] = []+ const push = (chunk: string) => {+ const trimmed = chunk.trim()+ if (trimmed === '') return+ const withoutComments = trimmed.replace(/--[^\n]*/g, '').replace(/\/\*[\s\S]*?\*\//g, '').trim()+ if (withoutComments === '') return+ statements.push(trimmed)+ }+ const n = sql.length+ let start = 0+ let i = 0+ while (i < n) {+ const ch = sql[i]+ const next = sql[i + 1]+ if (ch === '-' && next === '-') {+ const end = sql.indexOf('\n', i)+ i = end === -1 ? n : end + 1+ continue+ }+ if (ch === '/' && next === '*') {+ const end = sql.indexOf('*/', i + 2)+ i = end === -1 ? n : end + 2+ continue+ }+ if (ch === "'") {+ let j = i + 1+ while (j < n) {+ if (sql[j] === "'") {+ if (sql[j + 1] === "'") { j += 2; continue }+ break+ }+ j+++ }+ i = j + 1+ continue+ }+ if (ch === '$') {+ const tag = /^\$(?:[A-Za-z_][A-Za-z0-9_]*)?\$/.exec(sql.slice(i, i + 64))?.[0]+ if (tag) {+ const end = sql.indexOf(tag, i + tag.length)+ i = end === -1 ? n : end + tag.length+ continue+ }+ }+ if (ch === ';') {+ push(sql.slice(start, i))+ start = i + 1+ }+ i+++ }+ push(sql.slice(start))+ return statements+}++// Postgres SQLSTATE codes for transient lock contention on one statement:+// 40P01 deadlock_detected, 55P03 lock_not_available (lock_timeout), 40001+// serialization_failure. Anything else is a real error and propagates.+const BASELINE_STATEMENT_LOCK_CODES = new Set(['40P01', '55P03', '40001'])+const BASELINE_STATEMENT_ATTEMPTS = 6++/** Structural so the unit test can drive the runner with a fake client. */+export type BaselineStatementClient = { query(sql: string): Promise<unknown> }++/**+ * Run the frozen baseline one statement at a time, retrying a statement that+ * loses a lock race.+ *+ * Sent as a single simple-protocol query, the batch is ONE implicit+ * transaction: every no-op `ALTER TABLE … IF NOT EXISTS` takes a brief+ * AccessExclusiveLock and the transaction HOLDS all ~30 of them until commit.+ * Under sustained production traffic those waits close into a cycle and+ * Postgres aborts the whole batch with 40P01 — on every attempt. That is how+ * the first ADR 0003 adoption Job (v0.14.0, 2026-09-03) failed eight times in+ * a row without applying anything: the ledger was empty, so version 1 had to+ * run, and version 1 could never commit. Before ADR 0003 the same batch ran on+ * every boot and only ever got through production because a sentinel probe let+ * a deadlocked no-op pass; that hatch is gone, and it would not have helped+ * here anyway because the baseline had real columns to add.+ *+ * Per-statement autocommit holds one table's lock at a time and releases it+ * immediately, so a no-op cannot participate in a lock cycle, and a real+ * change waits at most `lock_timeout` (55P03) before this loop retries just+ * that statement. Every statement in the baseline is idempotent (it ran on+ * every application boot before ADR 0003), so a statement retry — or a rerun+ * of the Job after a mid-batch failure — resumes safely; ensureSchema records+ * version 1 only after the entire batch completed. Versioned migrations after+ * the baseline keep their single-transaction atomicity.+ */+export async function runBaselineStatements(+ client: BaselineStatementClient,+ sql: string,+ opts: { sleep?: (ms: number) => Promise<void>; maxAttempts?: number } = {},+): Promise<number> {+ const sleep = opts.sleep ?? ((ms: number) => new Promise<void>((resolve) => setTimeout(resolve, ms)))+ const maxAttempts = opts.maxAttempts ?? BASELINE_STATEMENT_ATTEMPTS+ const statements = splitSqlStatements(sql)+ for (let index = 0; index < statements.length; index++) {+ const statement = statements[index]+ const summary = statement.replace(/\s+/g, ' ').slice(0, 96)+ for (let attempt = 1; ; attempt++) {+ try {+ await client.query(statement)+ break+ } catch (err) {+ const code = (err as { code?: unknown } | null)?.code+ if (typeof code === 'string' && BASELINE_STATEMENT_LOCK_CODES.has(code) && attempt < maxAttempts) {+ const backoffMs = Math.min(8_000, 500 * 2 ** (attempt - 1)) + Math.floor(Math.random() * 250)+ console.warn(+ `[db] baseline statement ${index + 1}/${statements.length} hit ${code} (attempt ${attempt}/${maxAttempts}) — retrying in ${backoffMs}ms: ${summary}`,+ )+ await sleep(backoffMs)+ continue+ }+ console.error(`[db] baseline statement ${index + 1}/${statements.length} failed: ${summary}`)+ throw err+ }+ }+ }+ return statements.length+}++/** The frozen baseline text, for tests that check the splitter against it. */+export function frozenBaselineSql(): string {+ return DDL+}+async function applyLegacyBaseline(client: import('pg').PoolClient): Promise<void> {// pgvector remains optional: deployments without the extension retain the// recency-only memory path. All other baseline objects are mandatory.@@ -2063,8 +2198,10 @@ async function applyLegacyBaseline(client: import('pg').PoolClient): Promise<voi// This is the frozen baseline for databases that predate versioned// migrations. It runs once, from the dedicated migration owner, and is never- // re-applied by application replicas.- await client.query(DDL)+ // re-applied by application replicas. It is sent one statement at a time —+ // see runBaselineStatements for why the single-batch form cannot complete+ // against live production traffic.+ await runBaselineStatements(client, DDL)await client.query(`DO $migrate$BEGIN