Skip to content

Garry's Opinionated OpenClaw/Hermes Agent Brain

TypeScript
14.9 KB
Raw Blame
/**
 * Engine graduation custody state in the database: the `persistence_graduation`
 * singleton (each engine's own row: the source records `quiesced` -> `cutover`,
 * the target records `copying` -> ... -> `authoritative`) and the
 * `gbrain_graduation_fence` statement triggers that refuse every write on the
 * target except from transactions that carry the run's identity
 * (`SET LOCAL gbrain.graduation_run = '<run_id>'`, PgBouncer transaction-mode
 * safe). The GUC and `GBRAIN_GRADUATION_RUN` are safety interlocks, not
 * security boundaries.
 */
import type { BrainEngine } from '../engine.ts';
import {
  assertTransition, SOURCE_ROW_STATES, SOURCE_TRANSITIONS, TARGET_ROW_STATES, TARGET_TRANSITIONS,
  type ReplayProbeResult, type SourceRowState, type TableReceipt, type TargetRowState, type TriggerBypass,
} from './engine-graduation.types.ts';

export const GRADUATION_FENCE_TRIGGER = 'gbrain_graduation_fence';
export const GRADUATION_RUN_SETTING = 'gbrain.graduation_run';

const ROW_STATES = [...new Set<string>([...SOURCE_ROW_STATES, ...TARGET_ROW_STATES])];

/** Shared by the schema migration on both engines. Idempotent. */
export const PERSISTENCE_GRADUATION_SCHEMA_SQL = `
CREATE TABLE IF NOT EXISTS persistence_graduation (
  singleton smallint PRIMARY KEY DEFAULT 1 CHECK (singleton = 1),
  role text NOT NULL CHECK (role IN ('source','target')),
  run_id uuid NOT NULL,
  state text NOT NULL CHECK (state IN (${ROW_STATES.map(s => `'${s}'`).join(',')})),
  source_brain_id uuid,
  source_data_dir text,
  cutover_sequence bigint,
  trigger_bypass text CHECK (trigger_bypass IS NULL OR trigger_bypass IN ('session_replication_role','disable_trigger')),
  table_receipts jsonb,
  replay_probe jsonb,
  timings jsonb NOT NULL DEFAULT '{}'::jsonb,
  doctor jsonb,
  rollback jsonb,
  graduated_at timestamptz,
  created_at timestamptz NOT NULL DEFAULT now(),
  updated_at timestamptz NOT NULL DEFAULT now()
);
CREATE OR REPLACE FUNCTION gbrain_graduation_fence() RETURNS trigger LANGUAGE plpgsql SET search_path = pg_catalog, public, pg_temp AS $fence$
DECLARE fence_run text;
BEGIN
  SELECT run_id::text INTO fence_run FROM persistence_graduation WHERE singleton = 1;
  IF fence_run IS NULL OR fence_run IS DISTINCT FROM NULLIF(current_setting('${GRADUATION_RUN_SETTING}', true), '') THEN
    RAISE EXCEPTION 'graduation_in_progress: this database is fenced by gbrain engine graduation run %; writes are refused until the run grants it authority', COALESCE(fence_run, 'unknown')
      USING ERRCODE = '55000', HINT = 'Run: gbrain migrate --status --json';
  END IF;
  RETURN NULL;
END
$fence$;
`;

export interface GraduationRow {
  role: 'source' | 'target';
  run_id: string;
  state: SourceRowState | TargetRowState;
  source_brain_id: string | null;
  source_data_dir: string | null;
  cutover_sequence: string | null;
  trigger_bypass: TriggerBypass | null;
  table_receipts: readonly TableReceipt[] | null;
  replay_probe: ReplayProbeResult | null;
  timings: Record<string, number>;
  doctor: unknown;
  rollback: unknown;
  graduated_at: string | null;
}

function parseJson<T>(value: unknown): T | null {
  if (value === null || value === undefined) return null;
  if (typeof value === 'string') { try { return JSON.parse(value) as T; } catch { return null; } }
  return value as T;
}

/**
 * Engines (their root, never a transaction clone) whose database is known to have the table; an absence is never
 * kept. Only writer admission reads it (`cached`): a database that admits writes has the persistence schema, while
 * a graduation target engine can be pointed at a database that does not have the table yet.
 */
const tablePresent = new WeakSet<object>();
function rootEngine(engine: object): object {
  let root = engine;
  while (Object.hasOwn(root, '_pageTransaction')) root = Object.getPrototypeOf(root);
  return root;
}
export async function graduationTablePresent(engine: BrainEngine, opts: { cached?: boolean } = {}): Promise<boolean> {
  if (opts.cached && tablePresent.has(rootEngine(engine))) return true;
  const [row] = await engine.executeRaw<{ present: boolean }>(`SELECT to_regclass('persistence_graduation') IS NOT NULL AS present`);
  if (row?.present === true) tablePresent.add(rootEngine(engine));
  return row?.present === true;
}

/** The engine's own custody row, or null on a normal brain (or before the migration). */
export async function readGraduationRow(engine: BrainEngine, opts: { forUpdate?: boolean } = {}): Promise<GraduationRow | null> {
  if (!await graduationTablePresent(engine)) return null;
  const [row] = await engine.executeRaw<Record<string, unknown>>(
    `SELECT role, run_id::text AS run_id, state, source_brain_id::text AS source_brain_id, source_data_dir,
       cutover_sequence::text AS cutover_sequence, trigger_bypass, table_receipts, replay_probe, timings, doctor, rollback,
       graduated_at::text AS graduated_at
     FROM persistence_graduation WHERE singleton = 1${opts.forUpdate ? ' FOR UPDATE' : ''}`);
  if (!row) return null;
  return {
    role: row.role as GraduationRow['role'],
    run_id: String(row.run_id),
    state: row.state as GraduationRow['state'],
    source_brain_id: (row.source_brain_id as string | null) ?? null,
    source_data_dir: (row.source_data_dir as string | null) ?? null,
    cutover_sequence: (row.cutover_sequence as string | null) ?? null,
    trigger_bypass: (row.trigger_bypass as TriggerBypass | null) ?? null,
    table_receipts: parseJson<TableReceipt[]>(row.table_receipts),
    replay_probe: parseJson<ReplayProbeResult>(row.replay_probe),
    timings: parseJson<Record<string, number>>(row.timings) ?? {},
    doctor: parseJson(row.doctor),
    rollback: parseJson(row.rollback),
    graduated_at: (row.graduated_at as string | null) ?? null,
  };
}

/**
 * The connect gate's read of the custody row in one statement (#7147): every CLI call pays this read before its
 * work, so it reads the row directly and treats only an undefined table (42P01, a brain before the migration) as "no
 * row" instead of probing for the table first. Outside a transaction only: a failed statement would abort one.
 */
export async function readGraduationConnectRow(engine: BrainEngine): Promise<Pick<GraduationRow, 'role' | 'run_id' | 'state' | 'source_data_dir'> | null> {
  let row: Record<string, unknown> | undefined;
  try {
    [row] = await engine.executeRaw<Record<string, unknown>>(
      `SELECT role, run_id::text AS run_id, state, source_data_dir FROM persistence_graduation WHERE singleton = 1`);
  } catch (error) {
    if ((error as { code?: unknown } | null)?.code === '42P01') return null;
    throw error;
  }
  tablePresent.add(rootEngine(engine));
  if (!row) return null;
  return {
    role: row.role as GraduationRow['role'],
    run_id: String(row.run_id),
    state: row.state as GraduationRow['state'],
    source_data_dir: (row.source_data_dir as string | null) ?? null,
  };
}

/** Run `fn` in one transaction that carries the run's identity, so the fence admits its writes. */
export async function withGraduationRun<T>(engine: BrainEngine, runId: string, fn: (tx: BrainEngine) => Promise<T>): Promise<T> {
  return engine.transaction(async tx => {
    await tx.executeRaw(`SELECT set_config('${GRADUATION_RUN_SETTING}', $1, true)`, [runId]);
    return fn(tx);
  });
}

export interface GraduationRowPatch {
  sourceBrainId?: string | null;
  sourceDataDir?: string | null;
  cutoverSequence?: string | null;
  triggerBypass?: TriggerBypass | null;
  tableReceipts?: readonly TableReceipt[] | null;
  replayProbe?: ReplayProbeResult | null;
  timings?: Record<string, number>;
  doctor?: unknown;
  rollback?: unknown;
  graduatedAt?: 'now' | null;
}

async function writeRow(tx: BrainEngine, role: 'source' | 'target', runId: string, state: string, patch: GraduationRowPatch, insert: boolean): Promise<void> {
  const json = (v: unknown) => (v === undefined ? undefined : v === null ? null : JSON.stringify(v));
  const fields: Array<[string, unknown, string]> = [];
  const add = (column: string, value: unknown, cast = '') => { if (value !== undefined) fields.push([column, value, cast]); };
  add('source_brain_id', patch.sourceBrainId, '::uuid');
  add('source_data_dir', patch.sourceDataDir);
  add('cutover_sequence', patch.cutoverSequence, '::bigint');
  add('trigger_bypass', patch.triggerBypass);
  add('table_receipts', json(patch.tableReceipts), '::text::jsonb');
  add('replay_probe', json(patch.replayProbe), '::text::jsonb');
  add('timings', json(patch.timings), '::text::jsonb');
  add('doctor', json(patch.doctor), '::text::jsonb');
  add('rollback', json(patch.rollback), '::text::jsonb');
  const params: unknown[] = [role, runId, state, ...fields.map(f => f[1])];
  const graduatedAt = patch.graduatedAt === 'now' ? 'now()' : patch.graduatedAt === null ? 'NULL' : undefined;
  if (insert) {
    const cols = ['role', 'run_id', 'state', ...fields.map(f => f[0]), ...(graduatedAt ? ['graduated_at'] : [])];
    const vals = ['$1', '$2::uuid', '$3', ...fields.map((f, i) => `$${i + 4}${f[2]}`), ...(graduatedAt ? [graduatedAt] : [])];
    await tx.executeRaw(`INSERT INTO persistence_graduation (singleton, ${cols.join(', ')}) VALUES (1, ${vals.join(', ')})
      ON CONFLICT (singleton) DO UPDATE SET ${cols.map(c => `${c} = EXCLUDED.${c}`).join(', ')}, created_at = now(), updated_at = now()`, params);
    return;
  }
  const sets = ['role = $1', 'run_id = $2::uuid', 'state = $3', ...fields.map((f, i) => `${f[0]} = $${i + 4}${f[2]}`),
    ...(graduatedAt ? [`graduated_at = ${graduatedAt}`] : []), 'updated_at = now()'];
  await tx.executeRaw(`UPDATE persistence_graduation SET ${sets.join(', ')} WHERE singleton = 1`, params);
}

/**
 * Move the target row to `to`, driven by TARGET_TRANSITIONS. A missing row may
 * only start as `copying`; an `abandoned` row of another run may be reused
 * (`abandoned` -> `copying`) only when `reuse` is set (`--force`). Call inside
 * `withGraduationRun` so the fence admits the write.
 */
export async function setTargetState(tx: BrainEngine, runId: string, to: TargetRowState, patch: GraduationRowPatch = {}, opts: { reuse?: boolean } = {}): Promise<void> {
  const row = await readGraduationRow(tx, { forUpdate: true });
  if (!row) {
    if (to !== 'copying') throw new Error(`Illegal graduation transition (none) -> ${to}`);
    await writeRow(tx, 'target', runId, to, patch, true);
    return;
  }
  if (row.role !== 'target') throw new Error('This database records a graduation source row; it cannot become a graduation target.');
  if (row.run_id !== runId) {
    if (!(opts.reuse && row.state === 'abandoned' && to === 'copying')) {
      throw new Error(`The target row belongs to graduation run ${row.run_id}, not ${runId}.`);
    }
    // The fence keys on the stored run id: switch the row, then this transaction's identity.
    await writeRow(tx, 'target', runId, to, { ...patch, cutoverSequence: null, tableReceipts: null, replayProbe: null, rollback: null, graduatedAt: null }, false);
    await tx.executeRaw(`SELECT set_config('${GRADUATION_RUN_SETTING}', $1, true)`, [runId]);
    return;
  }
  if (row.state !== to) assertTransition(TARGET_TRANSITIONS, row.state as TargetRowState, to);
  await writeRow(tx, 'target', runId, to, patch, false);
}

/** Move the source row to `to`, driven by SOURCE_TRANSITIONS; a fresh run starts at `quiesced`. */
export async function setSourceState(tx: BrainEngine, runId: string, to: SourceRowState, patch: GraduationRowPatch = {}): Promise<void> {
  const row = await readGraduationRow(tx, { forUpdate: true });
  if (!row) {
    if (to !== 'quiesced') throw new Error(`Illegal graduation transition (none) -> ${to}`);
    await writeRow(tx, 'source', runId, to, patch, true);
    return;
  }
  if (row.role !== 'source') throw new Error('This database records a graduation target row; it cannot become a graduation source.');
  if (row.run_id !== runId) {
    if (row.state !== 'rolled_back' || to !== 'quiesced') throw new Error(`The source row belongs to graduation run ${row.run_id}, not ${runId}.`);
    await writeRow(tx, 'source', runId, to, { cutoverSequence: null, ...patch }, true);
    return;
  }
  if (row.state !== to) assertTransition(SOURCE_TRANSITIONS, row.state as SourceRowState, to);
  await writeRow(tx, 'source', runId, to, patch, false);
}

const FENCED_TABLES_SQL = `SELECT c.oid AS relid, c.relname FROM pg_class c JOIN pg_namespace n ON n.oid = c.relnamespace
  WHERE n.nspname = current_schema() AND c.relkind IN ('r','p') AND NOT c.relispartition`;

/**
 * Install `gbrain_graduation_fence` (statement-level BEFORE INSERT OR UPDATE
 * OR DELETE OR TRUNCATE, `ENABLE ALWAYS` so it also fires under
 * `session_replication_role = replica`) on every table of the target schema.
 * Idempotent: repairs missing triggers and ones not set to ALWAYS.
 */
export async function installGraduationFence(target: BrainEngine, runId: string): Promise<void> {
  const row = await readGraduationRow(target);
  if (!row || row.role !== 'target' || row.run_id !== runId) {
    throw new Error(`The graduation fence needs the target row of run ${runId} first.`);
  }
  await target.executeRaw(`DO $install$ DECLARE r record; BEGIN
    FOR r IN ${FENCED_TABLES_SQL} LOOP
      IF NOT EXISTS (SELECT 1 FROM pg_trigger t WHERE t.tgrelid = r.relid AND t.tgname = '${GRADUATION_FENCE_TRIGGER}') THEN
        EXECUTE format('CREATE TRIGGER ${GRADUATION_FENCE_TRIGGER} BEFORE INSERT OR UPDATE OR DELETE OR TRUNCATE ON %I FOR EACH STATEMENT EXECUTE FUNCTION gbrain_graduation_fence()', r.relname);
      END IF;
      EXECUTE format('ALTER TABLE %I ENABLE ALWAYS TRIGGER ${GRADUATION_FENCE_TRIGGER}', r.relname);
    END LOOP; END $install$`);
}

/** Drop every fence trigger. The authority transaction calls this in the same transaction that grants authority. */
export async function dropGraduationFence(target: BrainEngine): Promise<void> {
  await target.executeRaw(`DO $drop$ DECLARE r record; BEGIN
    FOR r IN SELECT c.relname FROM pg_trigger t JOIN pg_class c ON c.oid = t.tgrelid JOIN pg_namespace n ON n.oid = c.relnamespace
      WHERE n.nspname = current_schema() AND t.tgname = '${GRADUATION_FENCE_TRIGGER}' LOOP
      EXECUTE format('DROP TRIGGER IF EXISTS ${GRADUATION_FENCE_TRIGGER} ON %I', r.relname);
    END LOOP; END $drop$`);
}

export interface FenceStatus {
  tables: number;
  /** Tables with the fence trigger set to ALWAYS. */
  fenced: number;
  /** Tables without a fence trigger, or with one not set to ALWAYS. */
  unfenced: readonly string[];
}

export async function graduationFenceStatus(engine: BrainEngine): Promise<FenceStatus> {
  const rows = await engine.executeRaw<{ relname: string; tgenabled: string | null }>(
    `SELECT c.relname, t.tgenabled::text AS tgenabled FROM pg_class c JOIN pg_namespace n ON n.oid = c.relnamespace
     LEFT JOIN pg_trigger t ON t.tgrelid = c.oid AND t.tgname = '${GRADUATION_FENCE_TRIGGER}'
     WHERE n.nspname = current_schema() AND c.relkind IN ('r','p') AND NOT c.relispartition ORDER BY c.relname COLLATE "C"`);
  const unfenced = rows.filter(r => r.tgenabled !== 'A').map(r => r.relname);
  return { tables: rows.length, fenced: rows.length - unfenced.length, unfenced };
}