Skip to content

Garry's Opinionated OpenClaw/Hermes Agent Brain

TypeScript
20.9 KB
Raw Blame
/**
 * Graduation verify: the gate for cutover. With both engines open and the
 * target still fenced it compares relation sets, column contracts, row counts
 * and canonical digests (source side transformed by the inventory's permitted
 * transforms), sequence positions, every FK, trigger state, the target doctor
 * and a replay probe that resubmits a carried request through admission
 * inside a rolled-back transaction.
 */
import type { BrainEngine } from '../engine.ts';
import { OperationError } from '../ops/contract.ts';
import { sqlEngineKind } from '../sql-engine-kind.ts';
import { admitWriteInTransaction, type WriteAdmission } from './journal.ts';
import type { Principal, WriteAuthority } from './model.ts';
import { readManagedMode } from './managed-mode.ts';
import type {
  GraduationEngines, Inventory, InventoryEntry, ReplayProbeResult, TableReceipt, VerifyFailure, VerifyResult,
} from './engine-graduation.types.ts';
import { assertRelationSet, copyOrder, foreignKeys } from './graduation-inventory.ts';
import {
  DEFAULT_DIGEST_BATCH_ROWS, digestPlan, digestTable, quoteIdent, readDigestRows, tableColumns, withDigestSession,
} from './graduation-digest.ts';

export const GRADUATION_FENCE_TRIGGER = 'gbrain_graduation_fence';
/** Derived side tables the copy must reproduce exactly (they are carried, so the digest covers them). */
export const DERIVED_SIDE_TABLES = ['page_projection_jobs', 'page_generation_clock'] as const;

export interface VerifyContext {
  inventory: Inventory;
  /** Copy-time source snapshot: digestTable(source, entry, { applyTransforms: true }) per carried relation. */
  sourceReceipts: readonly TableReceipt[];
  replayRequestId?: string;
  /** The original caller input of the replay request (persistence_requests stores only the normalized intent). */
  replayCallerIntent?: Record<string, unknown>;
  /** The graduation run id; the replay probe sets gbrain.graduation_run to it so the target fence admits its rolled-back writes. */
  runId?: string;
  /** Verify runs before cutover, so the gbrain_graduation_fence triggers must be present and ENABLE ALWAYS (default true). */
  expectFence?: boolean;
  batchRows?: number;
  runDoctor: () => Promise<readonly string[]>;
}

function message(error: unknown): string {
  return error instanceof Error ? error.message : String(error);
}

async function firstDifference(e: GraduationEngines, entry: InventoryEntry, source: TableReceipt, target: TableReceipt, batchRows: number)
  : Promise<{ firstKey?: string; column?: string }> {
  let index = source.batches.findIndex((b, i) => b.sha256 !== target.batches[i]?.sha256 || b.lastKey !== target.batches[i]?.lastKey);
  if (index < 0) index = source.batches.length;
  const after = index > 0 ? JSON.parse(source.batches[index - 1]!.lastKey) as string[] : null;
  const limit = Math.max(batchRows, source.batches[index]?.rows ?? 0, target.batches[index]?.rows ?? 0);
  const [srcPlan, dstPlan] = await Promise.all([digestPlan(e.source, entry, true), digestPlan(e.target, entry, false)]);
  const [srcRows, dstRows] = await Promise.all([
    withDigestSession(e.source, tx => readDigestRows(tx, srcPlan, after, limit)),
    withDigestSession(e.target, tx => readDigestRows(tx, dstPlan, after, limit)),
  ]);
  for (let i = 0; i < Math.max(srcRows.length, dstRows.length); i++) {
    const s = srcRows[i], d = dstRows[i];
    if (s && d && s.text === d.text && JSON.stringify(s.key) === JSON.stringify(d.key)) continue;
    if (!s || !d || JSON.stringify(s.key) !== JSON.stringify(d.key)) return { firstKey: JSON.stringify((s ?? d)!.key) };
    const sv = JSON.parse(s.text) as unknown[], dv = JSON.parse(d.text) as unknown[];
    const col = srcPlan.columns.findIndex((_, j) => sv[j] !== dv[j]);
    return { firstKey: JSON.stringify(s.key), ...(col >= 0 ? { column: srcPlan.columns[col]!.name } : {}) };
  }
  return {};
}

async function verifyTable(e: GraduationEngines, entry: InventoryEntry, recorded: TableReceipt | undefined, batchRows: number,
  failures: VerifyFailure[]): Promise<TableReceipt | null> {
  const relation = entry.relation;
  const [srcCols, dstCols] = await Promise.all([tableColumns(e.source, relation), tableColumns(e.target, relation)]);
  const dstByName = new Map(dstCols.map(c => [c.name, c]));
  const contract = [
    ...srcCols.filter(c => !dstByName.has(c.name)).map(c => ({ column: c.name, detail: `missing on the target (${c.type})` })),
    ...dstCols.filter(c => !srcCols.some(s => s.name === c.name)).map(c => ({ column: c.name, detail: `exists only on the target (${c.type})` })),
    ...srcCols.filter(c => dstByName.has(c.name) && (dstByName.get(c.name)!.type !== c.type || dstByName.get(c.name)!.generated !== c.generated))
      .map(c => ({ column: c.name, detail: `source ${c.type}${c.generated ? ' generated' : ''}, target ${dstByName.get(c.name)!.type}${dstByName.get(c.name)!.generated ? ' generated' : ''}` })),
  ];
  if (contract.length) {
    for (const c of contract) failures.push({ relation, kind: 'digest', column: c.column, detail: `Column contract: ${c.column} ${c.detail}.` });
    return null;
  }
  const [source, target] = await Promise.all([
    digestTable(e.source, entry, { batchRows, applyTransforms: true }),
    digestTable(e.target, entry, { batchRows }),
  ]);
  if (!recorded) failures.push({ relation, kind: 'digest', detail: 'No copy-time source receipt was recorded for this table.' });
  else if (recorded.rootSha256 !== source.rootSha256 || recorded.rows !== source.rows) {
    failures.push({ relation, kind: 'digest', detail: `The source changed since the copy (rows ${recorded.rows} -> ${source.rows}); re-copy this table and its FK closure.` });
  }
  if (source.rows !== target.rows) failures.push({ relation, kind: 'count', detail: `Source has ${source.rows} rows, target has ${target.rows}.` });
  const countSql = `SELECT count(*)::text AS n FROM ${quoteIdent(relation)}${entry.rowFilter ? ` WHERE (${entry.rowFilter})` : ''}`;
  const [[srcCount], [dstCount]] = await Promise.all([
    e.source.executeRaw<{ n: string }>(countSql), e.target.executeRaw<{ n: string }>(countSql)]);
  for (const [side, counted, digested] of [['source', Number(srcCount?.n), source.rows], ['target', Number(dstCount?.n), target.rows]] as const) {
    if (counted !== digested) failures.push({ relation, kind: 'count', detail: `The ${side} has ${counted} rows by count(*) but the keyset digest read ${digested}; batching skipped rows.` });
  }
  if (source.rootSha256 !== target.rootSha256) {
    const diff = await firstDifference(e, entry, source, target, batchRows);
    failures.push({ relation, kind: 'digest', ...diff, detail: `Canonical content differs${diff.firstKey ? ` first at key ${diff.firstKey}` : ''}${diff.column ? ` in column ${diff.column}` : ''}.` });
  }
  return target;
}

interface SequenceRow { seq: string; tbl: string | null; col: string | null }

async function sequences(engine: BrainEngine): Promise<Map<string, SequenceRow>> {
  const rows = await engine.executeRaw<SequenceRow>(`SELECT s.relname AS seq, t.relname AS tbl, a.attname AS col
    FROM pg_class s JOIN pg_namespace n ON n.oid=s.relnamespace
    LEFT JOIN pg_depend d ON d.objid=s.oid AND d.classid='pg_class'::regclass AND d.refclassid='pg_class'::regclass AND d.deptype IN ('a','i')
    LEFT JOIN pg_class t ON t.oid=d.refobjid
    LEFT JOIN pg_attribute a ON a.attrelid=t.oid AND a.attnum=d.refobjsubid
    WHERE s.relkind='S' AND n.nspname=current_schema() ORDER BY s.relname COLLATE "C"`);
  return new Map(rows.map(r => [r.seq, r]));
}

/** The highest value the sequence has handed out (or would hand out first, minus one). */
async function sequencePosition(engine: BrainEngine, seq: string): Promise<bigint> {
  const [row] = await engine.executeRaw<{ last_value: string; is_called: unknown }>(`SELECT last_value::text AS last_value, is_called FROM ${quoteIdent(seq)}`);
  const called = row!.is_called === true || row!.is_called === 't';
  return BigInt(row!.last_value) - (called ? 0n : 1n);
}

async function verifySequences(e: GraduationEngines, inventory: Inventory, failures: VerifyFailure[]): Promise<void> {
  const copied = new Set(inventory.entries.filter(x => x.class === 'carry' || x.class === 'rebind').map(x => x.relation));
  const [src, dst] = await Promise.all([sequences(e.source), sequences(e.target)]);
  for (const [name, row] of src) {
    if (row.tbl !== null && !copied.has(row.tbl)) continue;
    if (!dst.has(name)) { failures.push({ relation: name, kind: 'sequence', detail: 'Sequence is missing on the target.' }); continue; }
    const [sp, tp] = await Promise.all([sequencePosition(e.source, name), sequencePosition(e.target, name)]);
    if (tp < sp) failures.push({ relation: name, kind: 'sequence', detail: `Target position ${tp} is below the source position ${sp}.` });
    if (row.tbl && row.col) {
      const [max] = await e.target.executeRaw<{ m: string | null }>(`SELECT max(${quoteIdent(row.col)})::text AS m FROM ${quoteIdent(row.tbl)}`);
      if (max?.m != null && tp < BigInt(max.m)) failures.push({ relation: name, kind: 'sequence', detail: `Target position ${tp} is below ${row.tbl}.${row.col} maximum ${max.m}.` });
    }
  }
}

async function verifyForeignKeys(target: BrainEngine, failures: VerifyFailure[]): Promise<void> {
  for (const fk of await foreignKeys(target)) {
    const notNull = fk.childColumns.map(c => `c.${quoteIdent(c)} IS NOT NULL`).join(' AND ');
    const match = fk.childColumns.map((c, i) => `p.${quoteIdent(fk.parentColumns[i]!)} = c.${quoteIdent(c)}`).join(' AND ');
    const rows = await target.executeRaw<Record<string, string>>(`SELECT ${fk.childColumns.map((c, i) => `c.${quoteIdent(c)}::text AS ${quoteIdent(`v${i}`)}`).join(', ')}
      FROM ${quoteIdent(fk.child)} c WHERE ${notNull} AND NOT EXISTS (SELECT 1 FROM ${quoteIdent(fk.parent)} p WHERE ${match}) LIMIT 1`);
    if (rows.length) {
      const key = JSON.stringify(fk.childColumns.map((_, i) => rows[0]![`v${i}`]));
      failures.push({ relation: fk.child, kind: 'fk', firstKey: key, column: fk.childColumns.join(','),
        detail: `${fk.name}: ${fk.child}(${fk.childColumns.join(',')}) = ${key} has no row in ${fk.parent}.` });
    }
  }
}

async function verifyTriggers(target: BrainEngine, inventory: Inventory, expectFence: boolean, failures: VerifyFailure[]): Promise<void> {
  const rows = await target.executeRaw<{ rel: string; name: string; enabled: string; internal: unknown }>(`SELECT c.relname AS rel, t.tgname AS name,
      t.tgenabled::text AS enabled, t.tgisinternal AS internal
    FROM pg_trigger t JOIN pg_class c ON c.oid=t.tgrelid JOIN pg_namespace n ON n.oid=c.relnamespace
    WHERE n.nspname=current_schema() ORDER BY c.relname COLLATE "C", t.tgname COLLATE "C"`);
  const fenced = new Set<string>();
  for (const t of rows) {
    if (t.name === GRADUATION_FENCE_TRIGGER) {
      if (!expectFence) failures.push({ relation: t.rel, kind: 'trigger', detail: `${t.name} is still installed after cutover.` });
      else if (t.enabled !== 'A') failures.push({ relation: t.rel, kind: 'trigger', detail: `${t.name} is '${t.enabled}', expected ENABLE ALWAYS ('A').` });
      else fenced.add(t.rel);
      continue;
    }
    if (t.enabled !== 'O') failures.push({ relation: t.rel, kind: 'trigger', detail: `Trigger ${t.name} is '${t.enabled}', expected enabled ('O').` });
  }
  if (!expectFence) return;
  const live = new Set(rows.map(r => r.rel));
  const tables = new Set((await target.executeRaw<{ relname: string }>(`SELECT c.relname FROM pg_class c JOIN pg_namespace n ON n.oid=c.relnamespace
    WHERE n.nspname=current_schema() AND c.relkind IN ('r','p')`)).map(r => r.relname));
  for (const entry of inventory.entries) {
    if ((entry.class === 'carry' || entry.class === 'rebind') && entry.engines.postgres && tables.has(entry.relation) && !fenced.has(entry.relation)) {
      failures.push({ relation: entry.relation, kind: 'trigger', detail: `${GRADUATION_FENCE_TRIGGER} is missing${live.has(entry.relation) ? '' : ' (no triggers at all)'}; the target is not fenced.` });
    }
  }
}

interface StoredRequest {
  id: string; principal_kind: Principal['kind']; principal_id: string; request_id: string; operation: string; source_id: string;
  source_incarnation: string; page_id: number | null; slug: string; worktree_id: string | null; topology_generation: string | number | null;
  intent: unknown; authority: unknown; state: string; outcome: unknown; compacted: unknown; target_kind: string | null;
  protocol_version: number | null; terminal_reservation: string | number;
}

function parsedJson(value: unknown): unknown {
  return typeof value === 'string' ? JSON.parse(value) : value;
}

function stable(value: unknown): string {
  if (value === null || typeof value !== 'object') return JSON.stringify(value ?? null);
  if (Array.isArray(value)) return `[${value.map(stable).join(',')}]`;
  const obj = value as Record<string, unknown>;
  return `{${Object.keys(obj).sort().map(k => `${JSON.stringify(k)}:${stable(obj[k])}`).join(',')}}`;
}

class ReplayRollback extends Error {
  constructor(readonly result: ReplayProbeResult) { super('replay probe rollback'); }
}

const TERMINAL_STATES = ['committed', 'conflict', 'failed', 'cancelled'];

/**
 * Resubmits a carried request through admitWriteInTransaction on the target
 * inside a transaction that is always rolled back, with persistence_brain.enabled
 * set to the source value so the managed admission path runs. Passes when the
 * stored id and outcome come back with no new request row and unchanged counters.
 */
export async function replayProbe(target: BrainEngine, requestId: string, callerIntent?: unknown,
  opts: { sourceEnabled?: boolean; runId?: string } = {}): Promise<ReplayProbeResult> {
  const stored = await target.executeRaw<StoredRequest>('SELECT * FROM persistence_requests WHERE request_id=$1::uuid', [requestId]);
  if (!stored.length) return { status: 'failed', requestId, detail: 'The request is not on the target; it was not carried.' };
  if (stored.length > 1) return { status: 'failed', requestId, detail: `${stored.length} principals hold this request id; the probe needs one.` };
  const row = stored[0]!;
  if (row.compacted === true || row.compacted === 't' || row.intent == null) return { status: 'not_available', reason: 'no_uncompacted_request' };
  if (callerIntent === undefined || callerIntent === null || typeof callerIntent !== 'object') return { status: 'not_available', reason: 'no_caller_input' };
  const [source] = await target.executeRaw<{ incarnation: string; archived: unknown }>('SELECT incarnation, archived FROM sources WHERE id=$1', [row.source_id]);
  if (!source || source.archived === true || source.archived === 't' || source.incarnation !== row.source_incarnation) {
    return { status: 'not_available', reason: 'archived_source' };
  }
  const admission: WriteAdmission = {
    principal: { kind: row.principal_kind, id: row.principal_id },
    operation: row.operation,
    targetKind: (row.target_kind ?? 'page') as WriteAdmission['targetKind'],
    protocolVersion: (row.protocol_version ?? 1) as WriteAdmission['protocolVersion'],
    sourceId: row.source_id,
    sourceIncarnation: row.source_incarnation,
    slug: row.slug,
    pageId: row.page_id,
    worktreeId: row.worktree_id,
    topologyGeneration: row.topology_generation,
    requestId: row.request_id,
    callerIntent: callerIntent as Record<string, unknown>,
    intent: parsedJson(row.intent) as Record<string, unknown>,
    authority: parsedJson(row.authority) as WriteAuthority,
    terminalReservation: Number(row.terminal_reservation),
  };
  try {
    await target.transaction(async tx => {
      if (opts.runId) await tx.executeRaw("SELECT set_config('gbrain.graduation_run', $1, true)", [opts.runId]);
      if (opts.sourceEnabled !== undefined) await tx.executeRaw('UPDATE persistence_brain SET enabled=$1 WHERE singleton=1', [opts.sourceEnabled]);
      const snapshot = async () => {
        const [count] = await tx.executeRaw<{ n: string }>('SELECT count(*)::text AS n FROM persistence_requests');
        const counters = await tx.executeRaw<Record<string, unknown>>('SELECT row_to_json(c)::text AS r FROM persistence_counters c ORDER BY key COLLATE "C"');
        return `${count!.n}|${counters.map(c => c.r).join('\n')}`;
      };
      const before = await snapshot();
      const replayed = await admitWriteInTransaction(tx, admission);
      const after = await snapshot();
      const problems = [
        String(replayed.id) !== String(row.id) ? `returned row ${replayed.id}, stored ${row.id}` : '',
        stable(parsedJson(replayed.outcome)) !== stable(parsedJson(row.outcome)) ? 'returned outcome differs from the stored outcome' : '',
        before !== after ? 'request rows or counters changed' : '',
        TERMINAL_STATES.includes(row.state) ? '' : `stored request is ${row.state}, not terminal`,
      ].filter(Boolean);
      throw new ReplayRollback(problems.length ? { status: 'failed', requestId, detail: problems.join('; ') } : { status: 'passed', requestId });
    });
  } catch (error) {
    if (error instanceof ReplayRollback) return error.result;
    if (error instanceof OperationError && error.code === 'permission_denied') return { status: 'not_available', reason: 'revoked_principal' };
    if (error instanceof OperationError && error.code === 'source_changed') return { status: 'not_available', reason: 'source_changed' };
    return { status: 'failed', requestId, detail: `Admission refused the replay: ${message(error)}` };
  }
  return { status: 'failed', requestId, detail: 'The probe transaction committed; it must always roll back.' };
}

async function chooseReplay(e: GraduationEngines, ctx: VerifyContext): Promise<ReplayProbeResult> {
  let requestId = ctx.replayRequestId;
  if (!requestId) {
    const [newest] = await e.target.executeRaw<{ request_id: string }>(`SELECT request_id FROM persistence_requests
      WHERE NOT compacted AND intent IS NOT NULL AND state = ANY($1::text[]) ORDER BY sequence DESC LIMIT 1`, [TERMINAL_STATES]);
    if (!newest) return { status: 'not_available', reason: 'no_uncompacted_request' };
    if (!ctx.replayCallerIntent) return { status: 'not_available', reason: 'no_caller_input' };
    requestId = newest.request_id;
  }
  const sourceEnabled = await readManagedMode(e.source);
  return replayProbe(e.target, requestId, ctx.replayCallerIntent, { sourceEnabled, runId: ctx.runId });
}

export async function verifyGraduation(e: GraduationEngines, ctx: VerifyContext): Promise<VerifyResult> {
  const failures: VerifyFailure[] = [];
  const batchRows = ctx.batchRows ?? DEFAULT_DIGEST_BATCH_ROWS;
  for (const [engine, kind] of [[e.source, sqlEngineKind(e.source.kind, 'graduation verify')], [e.target, sqlEngineKind(e.target.kind, 'graduation verify')]] as const) {
    try { await assertRelationSet(engine, kind, ctx.inventory); }
    catch (error) { failures.push({ relation: `(${kind} schema)`, kind: 'relation_set', detail: message(error) }); }
  }
  const carried = new Set(ctx.inventory.entries.filter(x => x.class === 'carry').map(x => x.relation));
  for (const side of DERIVED_SIDE_TABLES) {
    if (!carried.has(side)) failures.push({ relation: side, kind: 'digest', detail: 'Derived side table is not carried, so verify cannot prove it equals the source.' });
  }
  const recorded = new Map(ctx.sourceReceipts.map(r => [r.relation, r]));
  const targetTables = new Set((await copyOrder(e.target, ctx.inventory)).map(x => x.relation));
  const tables: TableReceipt[] = [];
  for (const entry of await copyOrder(e.source, ctx.inventory)) {
    if (!targetTables.has(entry.relation)) {
      failures.push({ relation: entry.relation, kind: 'relation_set', detail: 'Copied relation is missing on the target.' });
      continue;
    }
    try {
      const receipt = await verifyTable(e, entry, recorded.get(entry.relation), batchRows, failures);
      if (receipt) tables.push(receipt);
    } catch (error) {
      failures.push({ relation: entry.relation, kind: 'digest', detail: `Digest failed: ${message(error)}` });
    }
  }
  const checks: Array<[VerifyFailure['kind'], () => Promise<void>]> = [
    ['sequence', () => verifySequences(e, ctx.inventory, failures)],
    ['fk', () => verifyForeignKeys(e.target, failures)],
    ['trigger', () => verifyTriggers(e.target, ctx.inventory, ctx.expectFence ?? true, failures)],
  ];
  for (const [kind, run] of checks) {
    try { await run(); } catch (error) { failures.push({ relation: '(target)', kind, detail: `Check failed: ${message(error)}` }); }
  }
  let replay: ReplayProbeResult;
  try { replay = await chooseReplay(e, ctx); }
  catch (error) { replay = { status: 'failed', requestId: ctx.replayRequestId ?? '', detail: message(error) }; }
  if (replay.status === 'failed') failures.push({ relation: 'persistence_requests', kind: 'replay', firstKey: replay.requestId, detail: replay.detail });
  let doctorFailingChecks: readonly string[] = [];
  try { doctorFailingChecks = await ctx.runDoctor(); }
  catch (error) { doctorFailingChecks = [`doctor_unavailable: ${message(error)}`]; }
  for (const check of doctorFailingChecks) failures.push({ relation: '(target doctor)', kind: 'doctor', detail: `Doctor check ${check} fails on the target.` });
  return { ok: failures.length === 0, tables, failures, replay, doctorFailingChecks };
}