Skip to content

Where agent teams gather. Cross-platform team chat where AI agents are first-class teammates — with cloud or bring-your-own (Claude Code / Codex) brains.

TypeScript

fix(db): derive the schema-current check from the DDL instead of a hand-kept list

yetoneec7d557de5b2
diff --git a/server/src/__tests__/db-schema-sentinels.test.ts b/server/src/__tests__/db-schema-sentinels.test.ts
new file mode 100644
index 0000000..6103345
--- /dev/null
+++ b/server/src/__tests__/db-schema-sentinels.test.ts
@@ -0,0 +1,51 @@
+/**
+ * The lock-contention shortcut must never declare a schema "current" while a
+ * column the DDL promises is absent.
+ *
+ * On 2026-08-31 it did exactly that: #133 added `messages.delivery_recipient_id`
+ * to the DDL but not to the hand-kept sentinel list, a loaded production lost
+ * the lock race on `messages` (55P03), the shortcut answered "current", migrate
+ * exited 0, and every BYOA daemon's /inbox 500'd on the missing column. The
+ * expectations are now derived from the DDL text, so these tests pin that the
+ * derivation actually sees what the DDL declares.
+ *
+ * Run: node --import tsx --test server/src/__tests__/db-schema-sentinels.test.ts
+ */
+import { test } from 'node:test'
+import assert from 'node:assert/strict'
+
+process.env.CUMORA_RUNTIME_CLIENT = 'http'
+process.env.OPENAI_API_KEY ??= 'test-key'
+
+const { ddlExpectations } = await import('../db/migrate.js')
+
+test('every ADD COLUMN IF NOT EXISTS in the DDL becomes an expectation', () => {
+ const { columns } = ddlExpectations()
+ const has = (t: string, c: string) => columns.some((x) => x.table === t && x.column === c)
+
+ // The column whose omission caused the outage.
+ assert.ok(has('messages', 'delivery_recipient_id'), 'messages.delivery_recipient_id must be expected')
+ // Its neighbours, which were present in prod and masked the gap.
+ assert.ok(has('messages', 'client_id'))
+ assert.ok(has('messages', 'quoted_message_id'))
+
+ assert.ok(columns.length > 20, `expected many derived columns, got ${columns.length}`)
+})
+
+test('CREATE TABLE IF NOT EXISTS names become table expectations', () => {
+ const { tables } = ddlExpectations()
+ for (const t of ['messages', 'conversations', 'participants', 'llm_calls', 'shipping_features']) {
+ assert.ok(tables.includes(t), `${t} must be expected`)
+ }
+ assert.equal(new Set(tables).size, tables.length, 'table list must be deduplicated')
+})
+
+test('commented-out DDL does not become a phantom expectation', () => {
+ const { columns } = ddlExpectations()
+ // A phantom would make schemaAlreadyCurrent permanently false and disable the
+ // shortcut entirely, turning every loaded deploy into a crash-loop.
+ for (const c of columns) {
+ assert.match(c.table, /^[a-z_][a-z0-9_]*$/)
+ assert.match(c.column, /^[a-z_][a-z0-9_]*$/)
+ }
+})
diff --git a/server/src/db/migrate.ts b/server/src/db/migrate.ts
index b02e35b..aae4223 100644
--- a/server/src/db/migrate.ts
+++ b/server/src/db/migrate.ts
@@ -2187,51 +2187,75 @@ export async function releaseMigrationClient(client: MigrationClient): Promise<v
* rather than crash-loop. Read-only, so it's fine to run on the same connection
* right after the failed (auto-rolled-back) DDL.
*
- * Keep this list pointing at the LATEST schema additions. The cost of forgetting
- * to update it is benign: a loaded prod simply won't take the skip shortcut and
- * falls back to today's retry behavior — it can never skip a migration that
- * hasn't actually been applied.
+ * The expectations are derived FROM `DDL` itself rather than hand-listed. A
+ * hand-kept list is a whitelist of things that must be present, so an entry
+ * nobody remembered to add does not block the shortcut — its absence is simply
+ * never checked. That is not benign, and this comment used to claim it was: on
+ * 2026-08-31 #133 added `messages.delivery_recipient_id` without a sentinel, a
+ * loaded prod lost the lock race on the hot `messages` table (55P03), this
+ * function answered "current", migrate exited 0, and every daemon's /inbox 500'd
+ * on the missing column until production was rolled back. The same shape had
+ * already been caught twice before (llm_calls, llm_calls_rollup) — hence
+ * deriving the list instead of remembering it.
*/
+/** Every table and column `DDL` promises, read out of the DDL text so this can
+ * never fall behind it. Comments are stripped first so a commented-out example
+ * cannot become a phantom expectation. */
+export function ddlExpectations(): { tables: string[]; columns: Array<{ table: string; column: string }> } {
+ const sql = DDL.replace(/--[^\n]*/g, '')
+ const tables = [...sql.matchAll(/CREATE\s+TABLE\s+IF\s+NOT\s+EXISTS\s+([a-z_][a-z0-9_]*)/gi)]
+ .map((m) => m[1] as string)
+ const columns = [...sql.matchAll(/ALTER\s+TABLE\s+([a-z_][a-z0-9_]*)\s+ADD\s+COLUMN\s+IF\s+NOT\s+EXISTS\s+([a-z_][a-z0-9_]*)/gi)]
+ .map((m) => ({ table: m[1] as string, column: m[2] as string }))
+ return { tables: [...new Set(tables)], columns }
+}
+
async function schemaAlreadyCurrent(client: import('pg').PoolClient): Promise<boolean> {
try {
- const { rows } = await client.query<{ ok: boolean }>(`
- SELECT
- (SELECT count(*) FROM information_schema.columns
- WHERE table_name = 'conversations' AND column_name = 'topic') > 0
- AND (SELECT count(*) FROM information_schema.columns
- WHERE table_name = 'participants' AND column_name = 'status_updated_at') > 0
- AND (SELECT count(*) FROM information_schema.columns
- WHERE table_name = 'participants' AND column_name = 'company_id') > 0
- AND (SELECT count(*) FROM pg_class WHERE relname = 'participants_agent_id_unique') > 0
- AND (SELECT count(*) FROM information_schema.columns
- WHERE table_name = 'messages' AND column_name = 'client_id') > 0
- AND EXISTS (
- SELECT 1 FROM pg_class c
- JOIN pg_index i ON i.indexrelid = c.oid
- WHERE c.relname = 'uniq_messages_client_id' AND i.indisvalid
- )
- -- llm_calls is the universal sub2api ledger added in the observability
- -- rollout (29155c5). Without this sentinel, a 40P01 deadlock on the big
- -- DDL batch would take the "already current" shortcut and silently
- -- skip CREATE TABLE — every Observability page request then 500s with
- -- relation "llm_calls" does not exist. Caught exactly that on prod
- -- the day this shipped; sentinel added so the next new table can't
- -- repeat it.
- AND (SELECT count(*) FROM pg_class WHERE relname = 'llm_calls') > 0
- -- llm_calls.daemon_version added so 40P01-fallback doesn't skip the
- -- ALTER. Keep updating this list whenever a new column lands.
- AND (SELECT count(*) FROM information_schema.columns
- WHERE table_name = 'llm_calls' AND column_name = 'daemon_version') > 0
- -- llm_calls_rollup: the Observability pre-aggregation table. Without
- -- this sentinel a 40P01 fallback would skip CREATE TABLE and every
- -- dashboard query would 500 on "relation llm_calls_rollup does not exist".
- AND (SELECT count(*) FROM pg_class WHERE relname = 'llm_calls_rollup') > 0
- -- Shipping is the latest product-domain addition. Never take the
- -- lock-contention shortcut on a pod that has not created its core table.
- AND (SELECT count(*) FROM pg_class WHERE relname = 'shipping_features') > 0
- AS ok
- `)
- return rows[0]?.ok === true
+ const { tables, columns } = ddlExpectations()
+ // Indexes are not derivable the same way (partial/concurrent/renamed), so
+ // the few that gate correctness stay explicit.
+ const indexes = ['participants_agent_id_unique', 'uniq_messages_client_id']
+ const { rows } = await client.query<{
+ missing_tables: string[]; missing_columns: string[]; missing_indexes: string[]
+ }>(
+ `WITH want_t(name) AS (SELECT unnest($1::text[])),
+ want_c(t, c) AS (SELECT unnest($2::text[]), unnest($3::text[])),
+ want_i(name) AS (SELECT unnest($4::text[]))
+ SELECT
+ COALESCE(ARRAY(
+ SELECT w.name FROM want_t w
+ WHERE NOT EXISTS (SELECT 1 FROM pg_class WHERE relname = w.name)
+ ), '{}') AS missing_tables,
+ COALESCE(ARRAY(
+ SELECT w.t || '.' || w.c FROM want_c w
+ WHERE NOT EXISTS (
+ SELECT 1 FROM information_schema.columns
+ WHERE table_name = w.t AND column_name = w.c
+ )
+ ), '{}') AS missing_columns,
+ COALESCE(ARRAY(
+ SELECT w.name FROM want_i w
+ WHERE NOT EXISTS (
+ SELECT 1 FROM pg_class c JOIN pg_index i ON i.indexrelid = c.oid
+ WHERE c.relname = w.name AND i.indisvalid
+ )
+ ), '{}') AS missing_indexes`,
+ [tables, columns.map((c) => c.table), columns.map((c) => c.column), indexes],
+ )
+ const missing = [
+ ...(rows[0]?.missing_tables ?? []),
+ ...(rows[0]?.missing_columns ?? []),
+ ...(rows[0]?.missing_indexes ?? []),
+ ]
+ if (missing.length > 0) {
+ // Say what is missing. The failure this replaces was invisible: migrate
+ // logged "schema ensured" and exited 0 while a column the app queries on
+ // every request did not exist.
+ console.warn(`[db] schema is NOT current — missing: ${missing.slice(0, 20).join(', ')}${missing.length > 20 ? ` (+${missing.length - 20} more)` : ''}`)
+ return false
+ }
+ return true
} catch {
return false
}