Skip to content

Where agent teams gather. Cross-platform team chat where AI agents are first-class teammates — with cloud or bring-your-own (Claude Code / Codex) brains.

TypeScript

fix(db): run the frozen baseline one statement at a time (#178)

yetone7232e4d9dc76
diff --git a/docs/decisions/0003-versioned-schema-migration-ownership.md b/docs/decisions/0003-versioned-schema-migration-ownership.md
index 7cff40a..4cc7ce1 100644
--- a/docs/decisions/0003-versioned-schema-migration-ownership.md
+++ b/docs/decisions/0003-versioned-schema-migration-ownership.md
@@ -45,6 +45,12 @@ application replica has no reason to create or alter schema while starting.
version identity and required-index health.
- A first adoption run executes the frozen legacy baseline once and records it.
Subsequent runs read the ledger and execute only newly appended versions.
+ The baseline is sent one statement at a time with per-statement lock
+ retries: as a single implicit transaction it holds AccessExclusiveLocks on
+ ~30 tables until commit and deadlocks against live traffic on every attempt
+ (the first v0.14.0 adoption Job failed that way). Every baseline statement
+ is idempotent, so a rerun after a partial batch resumes safely; versions
+ after the baseline keep their single-transaction atomicity.
- Local development and manual Kubernetes installation must run
`npm run migrate` before starting the server; `npm run dev:all` performs that
one explicit step automatically.
diff --git a/server/src/__tests__/baseline-statement-runner.test.ts b/server/src/__tests__/baseline-statement-runner.test.ts
new file mode 100644
index 0000000..7977aeb
--- /dev/null
+++ b/server/src/__tests__/baseline-statement-runner.test.ts
@@ -0,0 +1,103 @@
+import assert from 'node:assert/strict'
+import { test } from 'node:test'
+
+process.env.CUMORA_RUNTIME_CLIENT = 'http'
+process.env.OPENAI_API_KEY ??= 'test-key'
+
+const { frozenBaselineSql, runBaselineStatements, splitSqlStatements } = await import('../db/migrate.js')
+
+test('splitSqlStatements only splits on top-level semicolons', () => {
+ const sql = `
+ -- a comment; with a semicolon
+ CREATE TABLE IF NOT EXISTS a (x TEXT DEFAULT 'semi;colon', y TEXT DEFAULT 'it''s; quoted');
+ /* block; comment */
+ DO $$ BEGIN
+ IF EXISTS (SELECT 1) THEN EXECUTE 'ALTER TABLE a ADD COLUMN IF NOT EXISTS z TEXT'; END IF;
+ END $$;
+ CREATE FUNCTION f() RETURNS TEXT LANGUAGE plpgsql AS $fn$ BEGIN RETURN 'x;y'; END $fn$;
+ INSERT INTO a (x) VALUES ('last') -- trailing; comment
+ `
+ const statements = splitSqlStatements(sql)
+ assert.equal(statements.length, 4)
+ assert.match(statements[0], /^-- a comment; with a semicolon\s+CREATE TABLE IF NOT EXISTS a/)
+ assert.match(statements[0], /'it''s; quoted'\)$/)
+ assert.match(statements[1], /^\/\* block; comment \*\/\s+DO \$\$ BEGIN[\s\S]*END \$\$$/)
+ assert.match(statements[2], /^CREATE FUNCTION f\(\)[\s\S]*END \$fn\$$/)
+ assert.match(statements[3], /^INSERT INTO a \(x\) VALUES \('last'\) -- trailing; comment$/)
+})
+
+test('splitSqlStatements drops comment-only chunks and preserves the frozen baseline verbatim', () => {
+ const sql = frozenBaselineSql()
+ const statements = splitSqlStatements(sql)
+ assert.ok(statements.length > 250, `expected a few hundred statements, got ${statements.length}`)
+
+ // Nothing but whitespace and comment text may be lost by the split.
+ const strip = (text: string) => text.replace(/--[^\n]*/g, '').replace(/\/\*[\s\S]*?\*\//g, '').replace(/\s+/g, '')
+ assert.equal(strip(statements.join(';')), strip(sql).replace(/;$/, ''))
+
+ for (const statement of statements) {
+ assert.doesNotMatch(statement, /^\s*$/)
+ assert.notEqual(strip(statement), '', 'a comment-only chunk leaked through')
+ const dollarTags = statement.match(/\$[A-Za-z_]*\$/g) ?? []
+ assert.equal(dollarTags.length % 2, 0, `unbalanced dollar quoting in: ${statement.slice(0, 80)}`)
+ }
+ assert.ok(statements.some((statement) => /^DO \$\$/.test(statement)), 'DO blocks must survive as single statements')
+})
+
+test('runBaselineStatements retries only lock contention, one statement at a time, in order', async () => {
+ const executed: string[] = []
+ const sleeps: number[] = []
+ let deadlocks = 2
+ const client = {
+ async query(sql: string) {
+ executed.push(sql)
+ if (sql.startsWith('ALTER TABLE hot') && deadlocks > 0) {
+ deadlocks--
+ throw Object.assign(new Error('deadlock detected'), { code: '40P01' })
+ }
+ },
+ }
+ const count = await runBaselineStatements(
+ client,
+ 'CREATE TABLE IF NOT EXISTS a (id TEXT); ALTER TABLE hot ADD COLUMN IF NOT EXISTS x TEXT; INSERT INTO a VALUES (\'z\');',
+ { sleep: async (ms) => { sleeps.push(ms) } },
+ )
+ assert.equal(count, 3)
+ assert.deepEqual(executed, [
+ 'CREATE TABLE IF NOT EXISTS a (id TEXT)',
+ 'ALTER TABLE hot ADD COLUMN IF NOT EXISTS x TEXT',
+ 'ALTER TABLE hot ADD COLUMN IF NOT EXISTS x TEXT',
+ 'ALTER TABLE hot ADD COLUMN IF NOT EXISTS x TEXT',
+ "INSERT INTO a VALUES ('z')",
+ ])
+ assert.equal(sleeps.length, 2)
+ assert.ok(sleeps[1] > sleeps[0], 'backoff must grow')
+})
+
+test('runBaselineStatements propagates real errors immediately and gives up after maxAttempts', async () => {
+ const executed: string[] = []
+ const failing = {
+ async query(sql: string) {
+ executed.push(sql)
+ if (sql.startsWith('ALTER')) throw Object.assign(new Error('relation "nope" does not exist'), { code: '42P01' })
+ },
+ }
+ await assert.rejects(
+ runBaselineStatements(failing, 'SELECT 1; ALTER TABLE nope ADD COLUMN x TEXT; SELECT 2', { sleep: async () => {} }),
+ (err: unknown) => (err as { code?: string }).code === '42P01',
+ )
+ assert.deepEqual(executed, ['SELECT 1', 'ALTER TABLE nope ADD COLUMN x TEXT'])
+
+ let attempts = 0
+ const alwaysLocked = {
+ async query() {
+ attempts++
+ throw Object.assign(new Error('canceling statement due to lock timeout'), { code: '55P03' })
+ },
+ }
+ await assert.rejects(
+ runBaselineStatements(alwaysLocked, 'ALTER TABLE hot ADD COLUMN x TEXT', { sleep: async () => {}, maxAttempts: 3 }),
+ (err: unknown) => (err as { code?: string }).code === '55P03',
+ )
+ assert.equal(attempts, 3)
+})
diff --git a/server/src/db/migrate.ts b/server/src/db/migrate.ts
index 4e9b47c..bd79f52 100644
--- a/server/src/db/migrate.ts
+++ b/server/src/db/migrate.ts
@@ -2049,6 +2049,141 @@ interface VersionedMigration extends MigrationMetadata {
up(client: import('pg').PoolClient): Promise<void>
}
+/**
+ * Split a SQL batch into its top-level statements.
+ *
+ * Honors single-quoted strings (with `''` escapes), dollar-quoted bodies
+ * (`$$ … $$` and `$tag$ … $tag$`), `--` line comments and block comments, so a
+ * `;` inside a DO block, a string, or a comment never ends a statement.
+ * Comment-only chunks are dropped. Exported for the unit test.
+ */
+export function splitSqlStatements(sql: string): string[] {
+ const statements: string[] = []
+ const push = (chunk: string) => {
+ const trimmed = chunk.trim()
+ if (trimmed === '') return
+ const withoutComments = trimmed.replace(/--[^\n]*/g, '').replace(/\/\*[\s\S]*?\*\//g, '').trim()
+ if (withoutComments === '') return
+ statements.push(trimmed)
+ }
+ const n = sql.length
+ let start = 0
+ let i = 0
+ while (i < n) {
+ const ch = sql[i]
+ const next = sql[i + 1]
+ if (ch === '-' && next === '-') {
+ const end = sql.indexOf('\n', i)
+ i = end === -1 ? n : end + 1
+ continue
+ }
+ if (ch === '/' && next === '*') {
+ const end = sql.indexOf('*/', i + 2)
+ i = end === -1 ? n : end + 2
+ continue
+ }
+ if (ch === "'") {
+ let j = i + 1
+ while (j < n) {
+ if (sql[j] === "'") {
+ if (sql[j + 1] === "'") { j += 2; continue }
+ break
+ }
+ j++
+ }
+ i = j + 1
+ continue
+ }
+ if (ch === '$') {
+ const tag = /^\$(?:[A-Za-z_][A-Za-z0-9_]*)?\$/.exec(sql.slice(i, i + 64))?.[0]
+ if (tag) {
+ const end = sql.indexOf(tag, i + tag.length)
+ i = end === -1 ? n : end + tag.length
+ continue
+ }
+ }
+ if (ch === ';') {
+ push(sql.slice(start, i))
+ start = i + 1
+ }
+ i++
+ }
+ push(sql.slice(start))
+ return statements
+}
+
+// Postgres SQLSTATE codes for transient lock contention on one statement:
+// 40P01 deadlock_detected, 55P03 lock_not_available (lock_timeout), 40001
+// serialization_failure. Anything else is a real error and propagates.
+const BASELINE_STATEMENT_LOCK_CODES = new Set(['40P01', '55P03', '40001'])
+const BASELINE_STATEMENT_ATTEMPTS = 6
+
+/** Structural so the unit test can drive the runner with a fake client. */
+export type BaselineStatementClient = { query(sql: string): Promise<unknown> }
+
+/**
+ * Run the frozen baseline one statement at a time, retrying a statement that
+ * loses a lock race.
+ *
+ * Sent as a single simple-protocol query, the batch is ONE implicit
+ * transaction: every no-op `ALTER TABLE … IF NOT EXISTS` takes a brief
+ * AccessExclusiveLock and the transaction HOLDS all ~30 of them until commit.
+ * Under sustained production traffic those waits close into a cycle and
+ * Postgres aborts the whole batch with 40P01 — on every attempt. That is how
+ * the first ADR 0003 adoption Job (v0.14.0, 2026-09-03) failed eight times in
+ * a row without applying anything: the ledger was empty, so version 1 had to
+ * run, and version 1 could never commit. Before ADR 0003 the same batch ran on
+ * every boot and only ever got through production because a sentinel probe let
+ * a deadlocked no-op pass; that hatch is gone, and it would not have helped
+ * here anyway because the baseline had real columns to add.
+ *
+ * Per-statement autocommit holds one table's lock at a time and releases it
+ * immediately, so a no-op cannot participate in a lock cycle, and a real
+ * change waits at most `lock_timeout` (55P03) before this loop retries just
+ * that statement. Every statement in the baseline is idempotent (it ran on
+ * every application boot before ADR 0003), so a statement retry — or a rerun
+ * of the Job after a mid-batch failure — resumes safely; ensureSchema records
+ * version 1 only after the entire batch completed. Versioned migrations after
+ * the baseline keep their single-transaction atomicity.
+ */
+export async function runBaselineStatements(
+ client: BaselineStatementClient,
+ sql: string,
+ opts: { sleep?: (ms: number) => Promise<void>; maxAttempts?: number } = {},
+): Promise<number> {
+ const sleep = opts.sleep ?? ((ms: number) => new Promise<void>((resolve) => setTimeout(resolve, ms)))
+ const maxAttempts = opts.maxAttempts ?? BASELINE_STATEMENT_ATTEMPTS
+ const statements = splitSqlStatements(sql)
+ for (let index = 0; index < statements.length; index++) {
+ const statement = statements[index]
+ const summary = statement.replace(/\s+/g, ' ').slice(0, 96)
+ for (let attempt = 1; ; attempt++) {
+ try {
+ await client.query(statement)
+ break
+ } catch (err) {
+ const code = (err as { code?: unknown } | null)?.code
+ if (typeof code === 'string' && BASELINE_STATEMENT_LOCK_CODES.has(code) && attempt < maxAttempts) {
+ const backoffMs = Math.min(8_000, 500 * 2 ** (attempt - 1)) + Math.floor(Math.random() * 250)
+ console.warn(
+ `[db] baseline statement ${index + 1}/${statements.length} hit ${code} (attempt ${attempt}/${maxAttempts}) — retrying in ${backoffMs}ms: ${summary}`,
+ )
+ await sleep(backoffMs)
+ continue
+ }
+ console.error(`[db] baseline statement ${index + 1}/${statements.length} failed: ${summary}`)
+ throw err
+ }
+ }
+ }
+ return statements.length
+}
+
+/** The frozen baseline text, for tests that check the splitter against it. */
+export function frozenBaselineSql(): string {
+ return DDL
+}
+
async function applyLegacyBaseline(client: import('pg').PoolClient): Promise<void> {
// pgvector remains optional: deployments without the extension retain the
// recency-only memory path. All other baseline objects are mandatory.
@@ -2063,8 +2198,10 @@ async function applyLegacyBaseline(client: import('pg').PoolClient): Promise<voi
// This is the frozen baseline for databases that predate versioned
// migrations. It runs once, from the dedicated migration owner, and is never
- // re-applied by application replicas.
- await client.query(DDL)
+ // re-applied by application replicas. It is sent one statement at a time —
+ // see runBaselineStatements for why the single-batch form cannot complete
+ // against live production traffic.
+ await runBaselineStatements(client, DDL)
await client.query(`
DO $migrate$
BEGIN