Garry's Opinionated OpenClaw/Hermes Agent Brain
TypeScript
import type { BrainEngine } from './engine.ts';
import type { Migration } from './schema-migrations/types.ts';
import { isLockTimeoutError } from './retry-matcher.ts';
import { resolveSchemaLockTimeoutMs, schemaLockBlockedError } from './postgres-engine/schema-lock-timeout.ts';
import { SCHEMA_MIGRATION_LOCK_APPLICATION_NAME } from './postgres-engine/init-schema-lock.ts';
import { assertSchemaMigrationLockHeld, withSchemaMigrationLock } from './schema-migration-lock.ts';
import { MIGRATIONS } from './schema-migrations/registry.generated.ts';
import { hasPendingMigrations, noteSchemaVersion } from './migrate-pending.ts';
import { assertOwnedTableShapes } from './owned-table-shapes.ts';
import { runMigrationStatement, setQuietMigrationNotices } from './schema-migrations/helpers.ts';
import { MIGRATION_PRELUDES, runMigrationPrelude } from './migration-preludes.ts';
import { clearMigrationTimeout, migrationStatementTimeoutMs, recordMigrationTimeout } from './migration-timeout.ts';
// runMigrations executes while an initialized engine is live. Keep its helper
// modules in the static graph rather than importing them from async handlers.
import {
isStatementTimeoutError,
isRetryableConnError,
} from './retry-matcher.ts';
import { repairTimelineDedupIndex } from './timeline-dedup-repair.ts';
import { hardenFailureLine, hardenFunctionSearchPaths } from './function-search-path.ts';
import { resumePageRevisionBackfill } from './page-state/revision-backfill-schema.ts';
import { repairPagesUpsertArbiter } from './pages-upsert-arbiter.ts';
import { repairLinkSourceCheck, LINK_SOURCE_GATE_MIGRATION_VERSION } from './link-source-check-repair.ts';
export type { Migration };
/**
* Resolve idempotent classification with the v0.30.1 default. Used by the
* migration runner's verify path and by the twice-run safety test
* (test/migrate-idempotent-classify.test.ts).
*/
export function isMigrationIdempotent(m: Migration): boolean {
// Default true: existing migrations were authored as idempotent (every
// CREATE/ALTER uses IF NOT EXISTS guards). Explicit false opts out.
return m.idempotent !== false;
}
/**
* Migration drift error — verify hook failed and migration is non-idempotent.
* Caller surfaces the column/table names that diverged and requires
* `--skip-verify` to force re-run.
*/
export class MigrationDriftError extends Error {
constructor(
public readonly version: number,
public readonly migrationName: string,
public readonly hint: string,
) {
super(`Migration v${version} (${migrationName}) verify failed: ${hint}`);
this.name = 'MigrationDriftError';
}
}
/**
* Retry-exhausted envelope (v0.30.1 / Finding F2). Surface the most recent
* idle blockers we observed so the user has a paste-ready
* pg_terminate_backend(<pid>) command.
*/
export class MigrationRetryExhausted extends Error {
constructor(
public readonly version: number,
public readonly migrationName: string,
public readonly attempts: number,
public readonly lastBlockers: IdleBlocker[],
public readonly lastError: Error,
) {
const lastB = lastBlockers[0];
const hint = lastB
? `PID ${lastB.pid} idle since ${lastB.query_start} likely holds the lock; run: psql ... -c "SELECT pg_terminate_backend(${lastB.pid})"`
: 'No idle-in-transaction blockers detected; check pg_locks for active waiters and ~/.gbrain/audit/connection-events-*.jsonl';
super(
`Migration v${version} (${migrationName}) failed after ${attempts} attempts. ${hint}. Original: ${lastError.message}`
);
this.name = 'MigrationRetryExhausted';
}
}
// Schema migrations live one per file in src/core/schema-migrations/v<NNN>-<name>.ts;
// the generated static-import registry (bun run build:schema-migrations) supplies
// MIGRATIONS in its historical array order. Exported for tests that structurally
// assert migration contents. Read-only contract.
export { MIGRATIONS };
export const LATEST_VERSION = MIGRATIONS.length > 0
? Math.max(...MIGRATIONS.map(m => m.version))
: 1;
/** The newest migration runMigrations would apply now, read from the live registry (LATEST_VERSION is fixed at load). */
export const registeredLatestVersion = (): number => Math.max(1, ...MIGRATIONS.map(m => m.version));
/**
* Row returned by `getIdleBlockers`. The shape is the public contract
* for both `gbrain doctor --locks` output and the internal DDL pre-flight.
*/
export interface IdleBlocker {
pid: number;
state: string;
query_start: string;
query: string;
}
/**
* Find idle-in-transaction connections older than 5 minutes that might
* block DDL. Postgres-only. Returns `[]` on PGLite, query failure, or
* no blockers. The query-failure path is intentionally silent because
* some managed Postgres configs restrict `pg_stat_activity` — a partial
* view of the server is still useful for doctor/pre-flight.
*
* Single source of truth shared by:
* - `checkForBlockingConnections` (DDL pre-flight warning)
* - `gbrain doctor --locks` (CLI diagnostic)
* - any future `--exclusive` drain-wait logic
*/
export async function getIdleBlockers(engine: BrainEngine): Promise<IdleBlocker[]> {
if (engine.kind !== 'postgres') return [];
try {
return await engine.executeRaw<IdleBlocker>(
`SELECT pid, state, query_start::text, substring(query, 1, 120) as query
FROM pg_stat_activity
WHERE state = 'idle in transaction'
AND query_start < NOW() - INTERVAL '5 minutes'
AND pid != pg_backend_pid()
AND application_name IS DISTINCT FROM '${SCHEMA_MIGRATION_LOCK_APPLICATION_NAME}'`
);
} catch {
return [];
}
}
/**
* Check for idle-in-transaction connections that might block DDL.
* Returns true if blockers were found (logged as warnings).
*/
async function checkForBlockingConnections(engine: BrainEngine): Promise<boolean> {
const rows = await getIdleBlockers(engine);
if (rows.length > 0) {
console.warn(`\n⚠️ Found ${rows.length} idle-in-transaction connection(s) older than 5 minutes:`);
for (const r of rows) {
console.warn(` PID ${r.pid} — idle since ${r.query_start}`);
console.warn(` Query: ${r.query}`);
}
console.warn(` These may block ALTER TABLE DDL. To kill: SELECT pg_terminate_backend(<pid>);\n`);
return true;
}
return false;
}
/**
* v0.30.1 (Cherry D3 / Finding F2): wrap a migration attempt in 3-attempt
* retry+backoff (5s/15s/45s). Retry only on statement_timeout (57014) or
* connection-reset patterns; other errors fail loud immediately.
*
* Before each retry: log idle-in-transaction blockers so the user knows
* which PID is holding the lock. After exhaustion: throw
* `MigrationRetryExhausted` with the named PID + suggested
* pg_terminate_backend command.
*/
async function runMigrationSQLWithRetry(
engine: BrainEngine,
m: Migration,
sql: string,
): Promise<void> {
// GBRAIN_MIGRATE_BACKOFF_MS lets tests skip the 5s/15s/45s backoff. In
// production the env var is unset and the default cadence applies.
const fastBackoff = process.env.GBRAIN_MIGRATE_BACKOFF_MS;
const backoffs = fastBackoff !== undefined
? [parseInt(fastBackoff, 10) || 0, parseInt(fastBackoff, 10) || 0, parseInt(fastBackoff, 10) || 0]
: [5000, 15000, 45000];
let lastErr: Error | null = null;
let lastBlockers: IdleBlocker[] = [];
for (let attempt = 0; attempt < 3; attempt++) {
try {
// Pre-attempt diagnostic: if there are idle blockers, log them so
// the operator can see what we're racing against. Cherry D3.
if (attempt > 0) {
lastBlockers = await getIdleBlockers(engine);
if (lastBlockers.length > 0) {
console.warn(` [retry ${attempt}/3] ${lastBlockers.length} idle-in-transaction blocker(s):`);
for (const b of lastBlockers) {
console.warn(` PID ${b.pid} idle since ${b.query_start} — ${b.query.slice(0, 80)}`);
}
}
}
await runMigrationPrelude(engine, m.version);
await runMigrationSQL(engine, m, sql);
return;
} catch (err: unknown) {
lastErr = err instanceof Error ? err : new Error(String(err));
const retryable = isStatementTimeoutError(err) || isRetryableConnError(err);
if (!retryable || attempt === 2) {
// Final failure: capture blockers + throw enriched envelope when
// retry-eligible (named-PID UX from F2). Non-retryable errors fall
// through to the existing 57014 handler in runMigrations.
if (retryable) {
lastBlockers = await getIdleBlockers(engine);
throw new MigrationRetryExhausted(m.version, m.name, attempt + 1, lastBlockers, lastErr);
}
throw err;
}
const delay = backoffs[attempt];
console.warn(` [retry ${attempt + 1}/3] ${m.name} hit ${lastErr.message.slice(0, 80)}; retrying in ${delay}ms`);
await new Promise(resolve => setTimeout(resolve, delay));
}
}
// Defensive: shouldn't reach here.
if (lastErr) throw lastErr;
}
/**
* Wrap migration SQL execution with Supabase-compatible timeout.
* Uses SET LOCAL statement_timeout inside a transaction to override
* server-enforced timeouts (required for Supabase Postgres).
*/
export async function runMigrationSQL(
engine: BrainEngine,
m: Migration,
sql: string,
): Promise<void> {
const useTransaction = m.transaction !== false;
if (useTransaction || engine.kind === 'pglite') {
// Wrap in transaction with extended timeout for Supabase compatibility.
// SET LOCAL scopes the timeout to this transaction only.
try {
await engine.transaction(async (tx) => {
if (engine.kind === 'postgres') {
try {
await tx.runMigration(m.version, `SET LOCAL statement_timeout = '${migrationStatementTimeoutMs()}'`);
// #5227 (W14 P1.6): a real transaction, so SET LOCAL binds the lock wait to this migration only.
await tx.runMigration(m.version, `SET LOCAL lock_timeout = '${resolveSchemaLockTimeoutMs()}ms'`);
} catch {
// Non-fatal: PGLite or older Postgres versions may not support this
}
}
await tx.runMigration(m.version, sql);
});
} catch (error) {
if (engine.kind === 'postgres' && isLockTimeoutError(error)) throw schemaLockBlockedError({ step: `migration v${m.version}`, timeoutMs: resolveSchemaLockTimeoutMs(), sessions: null, sampledDuringWait: false });
throw error;
}
} else {
await runMigrationStatement(engine, m.version, sql);
}
}
export { hasPendingMigrations };
/**
* v0.41.6.0 D4 — race-tolerant CLI-side migration runner.
*
* Wraps `engine.initSchema()` with a deadlock-aware retry + poll loop so
* the common "two CLIs probe schema simultaneously" race doesn't surface
* an alarming `Schema probe/migrate failed: deadlock detected` warning
* on every sync.
*
* Flow:
* 1. Try `engine.initSchema()`.
* 2. On SQLSTATE 40P01 (deadlock_detected) from Postgres: wait 250ms,
* retry once.
* 3. If second attempt still 40P01 (or any persistent lock-busy signal):
* poll `hasPendingMigrations()` every 250ms for up to 5s. If poll
* flips to `false` mid-window, return `{ status: 'race_resolved' }`
* silently (another runner finished — common case the user
* complained about).
* 4. If still pending at deadline: return `{ status: 'persistent', error }`.
* Caller surfaces the revised warning.
* 5. Non-40P01 errors propagate normally (real failures).
*
* The deeper root cause (codex F12 in plan-eng-review: initSchema
* already holds pg_advisory_lock(42), so the deadlock graph likely
* involves OTHER locks like DDL vs application-query contention or
* PgBouncer pool artifacts) is filed as a P2 follow-up TODO. The
* symptom fix here quiets the warning on the COMMON case where the race
* resolves itself, while loud-failing when migration is genuinely stuck.
*
* `deadlineMs` defaults to 5000 (5s polling window). Test-only callers
* pass smaller values for hermeticity; production paths use the default.
*
* `pollIntervalMs` defaults to 250ms — matches the retry-backoff delay
* for a symmetric design (eng-review D11). ~20 polls per deadline window;
* trivial DB load even on a stressed PgBouncer pool.
*/
export type TryRunPendingMigrationsResult =
| { status: 'ok'; attempts: number }
| { status: 'not_needed' }
| { status: 'race_resolved'; attempts: number; pollIterations: number }
| { status: 'persistent'; attempts: number; pollIterations: number; error: Error }
| { status: 'error'; error: Error };
export interface TryRunPendingMigrationsOpts {
deadlineMs?: number;
pollIntervalMs?: number;
retryBackoffMs?: number;
/** Test seam: inject a custom hasPendingMigrations / initSchema pair. */
_hooks?: {
initSchema?: () => Promise<void>;
hasPending?: () => Promise<boolean>;
sleep?: (ms: number) => Promise<void>;
now?: () => number;
};
}
export async function tryRunPendingMigrations(
engine: BrainEngine,
opts: TryRunPendingMigrationsOpts = {},
): Promise<TryRunPendingMigrationsResult> {
const deadlineMs = opts.deadlineMs ?? 5000;
const pollIntervalMs = opts.pollIntervalMs ?? 250;
const retryBackoffMs = opts.retryBackoffMs ?? 250;
const initSchema = opts._hooks?.initSchema ?? (() => engine.initSchema());
const hasPending = opts._hooks?.hasPending ?? (() => hasPendingMigrations(engine));
const sleep = opts._hooks?.sleep ?? ((ms: number) => new Promise(r => setTimeout(r, ms)));
const now = opts._hooks?.now ?? (() => Date.now());
// Quick early-exit: if no migrations are actually pending, skip entirely.
if (!await hasPending()) return { status: 'not_needed' };
let attempts = 0;
let lastErr: Error | null = null;
for (let attempt = 0; attempt < 2; attempt++) {
attempts++;
try {
await initSchema();
return { status: 'ok', attempts };
} catch (err) {
lastErr = err instanceof Error ? err : new Error(String(err));
if (!isDeadlockError(lastErr)) {
// Real failure: propagate to caller's catch.
return { status: 'error', error: lastErr };
}
// Deadlock — backoff before retry.
if (attempt === 0) await sleep(retryBackoffMs);
}
}
// Both attempts deadlocked. Poll hasPendingMigrations until deadline.
const deadline = now() + deadlineMs;
let pollIterations = 0;
while (now() < deadline) {
pollIterations++;
await sleep(pollIntervalMs);
try {
if (!await hasPending()) return { status: 'race_resolved', attempts, pollIterations };
} catch {
// hasPending throws → treat as pending (defensive; matches its own catch).
}
}
return {
status: 'persistent',
attempts,
pollIterations,
error: lastErr ?? new Error('deadlock_persistent'),
};
}
/**
* Detect Postgres SQLSTATE 40P01 (deadlock_detected) from arbitrary
* thrown values. Pattern-matches on:
* - postgres.js `.code === '40P01'`
* - error message containing `40P01` or `deadlock detected`
* The text-fallback covers cases where the driver doesn't expose `.code`.
*/
export function isDeadlockError(err: unknown): boolean {
if (!err) return false;
const maybe = err as { code?: string; sqlState?: string; message?: string };
if (maybe.code === '40P01' || maybe.sqlState === '40P01') return true;
const msg = String(maybe.message ?? err);
return /40P01|deadlock detected/i.test(msg);
}
export { withSchemaMigrationLock };
/** GBRA-467: under the schema migration lock (schema-migration-lock.ts); the version is read once it is held. */
export async function runMigrations(engine: BrainEngine): Promise<{ applied: number; current: number }> {
return withSchemaMigrationLock(engine, () => runMigrationsLocked(engine));
}
async function runMigrationsLocked(engine: BrainEngine): Promise<{ applied: number; current: number }> {
const currentStr = await engine.getConfig('version');
const current = parseInt(currentStr || '1', 10);
noteSchemaVersion(engine, current);
// Sort by version ascending so array insertion order doesn't affect
// correctness. Migrations MUST run in version order; if v16 accidentally
// precedes v15 in MIGRATIONS, setConfig(version, 16) would cause v15 to
// be skipped on the next iteration.
const sorted = [...MIGRATIONS].sort((a, b) => a.version - b.version);
const pending = sorted.filter(m => m.version > current);
if (pending.length > 0) await assertOwnedTableShapes(async sql => (await engine.executeRaw(sql))[0]); // #9190: before any repair or migration
// #2038: schema-drift self-heal. A migration renumbered during a master
// merge (v102 timeline dedup, originally v99) can be recorded-as-applied
// without its DDL ever running — the version counter can't see it. Repair
// the known drift on EVERY pass, including when nothing is pending (the
// affected brains are stamped AHEAD of the missing migration, so they never
// reach the loop below). Best-effort + idempotent: a no-op on a healthy
// index; `doctor` surfaces it independently if this ever fails.
try {
const r = await repairTimelineDedupIndex(engine);
if (r.repaired) {
console.error(
`[migrate] healed idx_timeline_dedup drift (#2038): ${r.before.join(',') || '(absent)'} ` +
`→ page_id,date,md5(summary),source` +
(r.collapsedDuplicates > 0 ? ` (collapsed ${r.collapsedDuplicates} duplicate row(s))` : ''),
);
}
} catch { /* best-effort; doctor reports the drift if this couldn't run */ }
// #550: same drift class for the pages upsert arbiter. When the
// UNIQUE(source_id, slug) constraint vanishes (partial restore, manual DDL,
// name-only migration guards), EVERY putPage fails with "no unique or
// exclusion constraint" and neither re-initSchema nor the version counter
// can see it. ADD-only self-heal; refuses (loudly) on duplicate rows.
try {
const p = await repairPagesUpsertArbiter(engine);
if (p.repaired) {
console.error(`[migrate] restored pages_source_slug_key UNIQUE(source_id, slug) (#550)`);
} else if (p.reason === 'duplicates') {
console.error(
`[migrate] cannot restore pages_source_slug_key: ${p.duplicateGroups} duplicate ` +
`(source_id, slug) group(s) exist — page upserts will keep failing until the ` +
`duplicates are resolved (#550). See \`gbrain doctor\`.`,
);
}
} catch { /* best-effort; doctor reports the drift if this couldn't run */ }
// #4613: same drift class for links_link_source_check. A brain stamped past
// v114 whose CHECK still carries the pre-v114 allowlist rejects every kebab
// provenance write; the version counter can't see it. Refuses loudly on
// violators. Ledger-gated: below v114 the pending loop replays v114 itself
// (the repair would rewrite the constraint twice; pre-v11 has no column).
if (current >= LINK_SOURCE_GATE_MIGRATION_VERSION) {
try {
const l = await repairLinkSourceCheck(engine);
if (l.repaired) {
console.error(`[migrate] restored links_link_source_check to the v114 kebab-case gate (#4613)`);
} else if (l.reason === 'violations') {
console.error(
`[migrate] cannot restore links_link_source_check: ${l.violations} links row(s) have a ` +
`non-kebab link_source — fix or delete them, then re-run (#4613). See \`gbrain doctor\`.`,
);
}
} catch (e) {
console.error(`[migrate] links_link_source_check self-heal could not run (#4613): ${e instanceof Error ? e.message : String(e)}`);
}
}
const backfill = () => resumePageRevisionBackfill(engine).catch(e => console.error(`[migrate] page revision backfill paused (#5216): ${e instanceof Error ? e.message : String(e)}`));
// #7632: a migration that re-creates a gbrain function from a historical body resets its search_path, a failing
// migration can stop the run before v277, and a failed earlier attempt leaves one behind; put pg_temp back on every
// run. At head (nothing pending) it still runs, for a function a failed earlier run left (e2e case 9), but does not
// name functions another role owns: doctor's function_search_path does, so a non-owner role is not told on every
// boot (#7688). Best-effort: doctor names whatever is left.
const harden = async (nameLeftovers: boolean): Promise<void> => {
try {
const left = await hardenFunctionSearchPaths(engine);
if (left === null) {
process.stderr.write(' search_path: pg_temp put back on gbrain functions, but listing any left without it failed; gbrain doctor (function_search_path) names them\n');
} else if (nameLeftovers && left.length > 0) {
process.stderr.write(` search_path: ${left.length} function(s) left without pg_temp on their search_path (${left.join(', ')}): `
+ 'ALTER FUNCTION needs the owner. Run gbrain doctor for the SQL to run as that role.\n');
}
} catch (e) {
process.stderr.write(hardenFailureLine(e));
}
};
if (pending.length === 0) {
await harden(false);
await backfill();
return { applied: 0, current };
}
// Fresh install vs upgrade: a never-migrated brain (schema blob seeds
// version='1'; every migration is >= 2) replays the FULL history — printing
// ~240 lines of internal migration names as the user's first-run experience.
// That wall makes a 2-second init read as complex and fragile ("1 → 125"
// implies the brand-new install was 124 versions stale). Fresh installs get
// one summary line; EXISTING brains keep the full per-migration detail
// (upgrades are where the names carry diagnostic value).
// GBRAIN_MIGRATE_VERBOSE=1 is the incident escape hatch (env-first, matching
// the GBRAIN_SYNC_*/GBRAIN_PACE_* pattern).
const freshInstall = current <= 1 && pending.length === sorted.length;
const quietReplay = freshInstall && process.env.GBRAIN_MIGRATE_VERBOSE !== '1';
// Suppress per-migration explanatory notices during a fresh-install replay
// (they are upgrade diagnostics, noise on a new user's first run). Restored
// in the finally so an in-process upgrade after a fresh init still narrates.
setQuietMigrationNotices(quietReplay);
// Progress messages route to stderr so callers parsing stdout (e.g.
// `gbrain jobs submit --json | jq`) aren't polluted by migration noise.
if (quietReplay) {
process.stderr.write(` Setting up brain schema (v${LATEST_VERSION})...\n`);
} else {
process.stderr.write(` Schema version ${current} → ${LATEST_VERSION} (${pending.length} migration(s) pending)\n`);
}
let applied = 0;
let lockRefused = false;
try {
// Pre-flight: warn about connections that might block DDL
await checkForBlockingConnections(engine);
for (const m of pending) {
await assertSchemaMigrationLockHeld();
if (!quietReplay) process.stderr.write(` [${m.version}] ${m.name}...\n`);
try {
await applyOneMigration(engine, m);
// Update version after both SQL and handler succeed. Inside the same
// catch so a stamp-write failure is also NAMED in quiet mode.
await engine.setConfig('version', String(m.version));
noteSchemaVersion(engine, m.version);
} catch (err) {
// Quiet fresh-install replay: name the failing migration — without the
// per-step lines, the error would otherwise be anonymous.
if (quietReplay) process.stderr.write(` [${m.version}] ${m.name} failed\n`);
// #7108: apply-migrations and doctor read this to name the migration and the step that helps.
const timedOut = err instanceof MigrationRetryExhausted ? isStatementTimeoutError(err.lastError) : isStatementTimeoutError(err);
if (timedOut) await recordMigrationTimeout(engine, m, MIGRATION_PRELUDES.has(m.version), err instanceof MigrationRetryExhausted ? err.lastBlockers.map(b => b.pid) : []);
throw err;
}
if (!quietReplay) process.stderr.write(` [${m.version}] ✓ ${m.name}\n`);
applied++;
}
} catch (err) {
lockRefused = (err as { code?: unknown }).code === 'schema_lock_blocked' || isLockTimeoutError(err);
throw err;
} finally {
// Never leak the fresh-install quiet flag into a later in-process run —
// covers every exit path from here on (incl. the pre-flight probe).
setQuietMigrationNotices(false);
// A run stopped by a lock it could not take would wait a second lock_timeout here before its own error surfaced.
if (lockRefused) process.stderr.write(' search_path: not put back on gbrain functions after the lock refusal; the next migration run does it, and gbrain doctor (function_search_path) names what is left\n');
else await harden(true);
}
await clearMigrationTimeout(engine);
await backfill();
return { applied, current: LATEST_VERSION };
}
/** One migration's full body (SQL + handler + verify), extracted so the
* runMigrations loop can name the failing migration in quiet-replay mode. */
async function applyOneMigration(engine: BrainEngine, m: Migration): Promise<void> {
// Pick SQL: engine-specific `sqlFor` wins over `sql`; Rust has no dialect (D18) and its runMigration refuses raw_sql.
const sql = (m.sqlFor as Partial<Record<string, string>> | undefined)?.[engine.kind] ?? m.sql;
if (sql) {
try {
// v0.30.1: retry wrapper handles statement_timeout + conn-reset
// across 3 attempts (5s/15s/45s). Other errors throw immediately.
await runMigrationSQLWithRetry(engine, m, sql);
} catch (err: unknown) {
// Actionable diagnostics for statement timeout (Postgres error 57014).
// Shape matches the 4-part error standard (what / why / fix / verify).
const code = (err as { code?: string })?.code;
if (code === '57014' || err instanceof MigrationRetryExhausted) {
console.error(`\n❌ Migration ${m.version} (${m.name}) ${err instanceof MigrationRetryExhausted ? 'exhausted retries' : 'hit statement_timeout (SQLSTATE 57014)'}.`);
if (err instanceof MigrationRetryExhausted && err.lastBlockers.length > 0) {
const b = err.lastBlockers[0];
console.error('');
console.error(` Likely blocker: PID ${b.pid}, idle since ${b.query_start}`);
console.error(` Query: ${b.query.slice(0, 120)}`);
console.error('');
console.error(` Recovery: psql ... -c "SELECT pg_terminate_backend(${b.pid})"`);
console.error('');
} else {
console.error('');
console.error(` Cause: no idle-in-transaction session was open, and a lock wait stops at the lock timeout, so its own work`);
console.error(` outlasted its ${Math.round(migrationStatementTimeoutMs() / 60_000)}-minute statement budget on this brain; every attempt rolled back.`);
console.error(' A plain rerun with this release repeats the failure (#7108).');
console.error('');
console.error(' Fix:');
console.error(' 1. gbrain doctor --locks # confirm no session holds a lock on the table');
console.error(' 2. If a newer release exists: gbrain upgrade, then gbrain apply-migrations --yes');
console.error(" 3. Otherwise, with the user's agreement: gbrain apply-migrations --yes --statement-timeout 120, and report it");
console.error('');
}
console.error(' Verify:');
console.error(' gbrain doctor # schema_version should match latest');
console.error('');
}
throw err;
}
}
// Application-level handler (runs outside transaction for flexibility)
if (m.handler) {
await m.handler(engine);
}
// v0.30.1 (D6): post-condition probe. If a verify hook is declared, run
// it before bumping config.version. When verify returns false, check
// idempotent — if true, log + retry the same migration once; if false,
// throw MigrationDriftError so operator runs --skip-verify deliberately.
if (m.verify) {
const verifyOk = await m.verify(engine).catch(() => false);
if (!verifyOk) {
const idempotent = isMigrationIdempotent(m);
if (idempotent) {
console.warn(` [${m.version}] ⚠️ verify failed; re-running idempotent migration once`);
if (sql) await runMigrationSQLWithRetry(engine, m, sql);
if (m.handler) await m.handler(engine);
// Best-effort: don't double-throw if second run still fails verify.
// Operator's next run of doctor will re-detect drift.
} else {
throw new MigrationDriftError(
m.version,
m.name,
`Schema does not match expected post-condition. Run with --skip-verify to force.`,
);
}
}
}
}