Skip to content

Garry's Opinionated OpenClaw/Hermes Agent Brain

TypeScript
19.1 KB
Raw Blame
/**
 * Graduation verify (src/core/persistence/graduation-verify.ts).
 *
 * Protects: a faithful copy of a brain with real persistence history (Lane A's
 * history fixture plus an active job, an orphaned running effect, a vector
 * and text keys that differ by case, '-' versus '_' and non-ASCII) verifies
 * green against a fenced target, with the inventory's transforms applied to
 * the source side; each kind of damage is reported with its table and, for
 * content, the first differing key and column: an edited row, a lost row, an
 * orphan FK, a disabled trigger, a missing fence, a lowered sequence, a source
 * written after the copy, an extra relation and a failing doctor check; the
 * replay probe resubmits the drained request through admission inside a
 * rolled-back transaction and reports passed, failed or not_available
 * (no caller input, revoked principal, archived source, changed topology).
 * Fails when: verify misses one of those, or the replay probe leaves a row,
 * a counter or the enabled flag behind.
 * The copy here is a minimal test copier (row-by-row, triggers bypassed with
 * session_replication_role); the production copier is graduation-copy.ts.
 * Runs PGLite -> PGLite, and PGLite -> Postgres when DATABASE_URL is set.
 */
import { afterAll, beforeAll, describe, expect, test } from 'bun:test';
import { randomUUID } from 'node:crypto';
import { mkdtempSync, rmSync } from 'node:fs';
import { tmpdir } from 'node:os';
import { join } from 'node:path';
import { PGLiteEngine } from '../src/core/pglite-engine.ts';
import type { BrainEngine } from '../src/core/engine.ts';
import { declarePersistenceProtocol } from '../src/core/persistence/protocol.ts';
import type { TableReceipt } from '../src/core/persistence/engine-graduation.types.ts';
import { copyOrder, GRADUATION_INVENTORY } from '../src/core/persistence/graduation-inventory.ts';
import { DIGEST_SESSION_SETTINGS, digestTable, quoteIdent, tableColumns } from '../src/core/persistence/graduation-digest.ts';
import { GRADUATION_FENCE_TRIGGER, replayProbe, verifyGraduation, type VerifyContext } from '../src/core/persistence/graduation-verify.ts';
import { buildHistoryFixture, type HistoryFixture } from '../scripts/persistence/history-fixture.ts';
import { isolatedPersistencePostgres } from './helpers/persistence-postgres.ts';
import { withEnv } from './helpers/with-env.ts';

const RUN_ID = randomUUID();
const home = mkdtempSync(join(tmpdir(), 'gbrain-graduation-verify-'));
let source: PGLiteEngine;
let liteTarget: PGLiteEngine;
let fixture: HistoryFixture;
let callerIntent: Record<string, unknown>;
let receipts: TableReceipt[];
const targets: Array<{ name: string; engine: BrainEngine }> = [];
let closePostgres: (() => Promise<void>) | undefined;

const CONFIG_KEYS = ['graduation-Key', 'graduation-key', 'graduation_key', 'graduation key', 'graduation-ünï', 'Graduation-Ω'];

async function withProtocol(engine: BrainEngine, fn: (tx: BrainEngine) => Promise<void>): Promise<void> {
  await engine.transaction(async tx => { await declarePersistenceProtocol(tx); await fn(tx); });
}

/** Writes on the fenced target: the run's GUC, the persistence protocol and user triggers bypassed. */
async function targetWrite(target: BrainEngine, sql: string, params: unknown[] = []): Promise<void> {
  await target.transaction(async tx => {
    await tx.executeRaw("SELECT set_config('gbrain.graduation_run',$1,true), set_config('session_replication_role','replica',true), set_config('gbrain.persistence_protocol','2',true)", [RUN_ID]);
    await tx.executeRaw(sql, params);
  });
}

async function installFence(target: BrainEngine): Promise<void> {
  await target.executeRaw(`CREATE OR REPLACE FUNCTION gbrain_graduation_fence_probe() RETURNS trigger LANGUAGE plpgsql AS $$ BEGIN
    IF current_setting('gbrain.graduation_run', true) IS DISTINCT FROM '${RUN_ID}' THEN RAISE EXCEPTION 'graduation_in_progress: fenced'; END IF;
    RETURN NULL; END $$`);
  for (const entry of await copyOrder(target)) await fenceTable(target, entry.relation);
}
async function fenceTable(target: BrainEngine, relation: string): Promise<void> {
  await target.executeRaw(`CREATE TRIGGER ${GRADUATION_FENCE_TRIGGER} BEFORE INSERT OR UPDATE OR DELETE OR TRUNCATE ON ${quoteIdent(relation)}
    FOR EACH STATEMENT EXECUTE FUNCTION gbrain_graduation_fence_probe()`);
  await target.executeRaw(`ALTER TABLE ${quoteIdent(relation)} ENABLE ALWAYS TRIGGER ${GRADUATION_FENCE_TRIGGER}`);
}

/** Minimal copier: every carry/rebind row as text, transforms applied, triggers bypassed, then sequence positions. */
async function testCopy(target: BrainEngine): Promise<void> {
  const order = await copyOrder(source);
  const plans = await Promise.all(order.map(async entry => ({ entry, columns: (await tableColumns(source, entry.relation)).filter(c => !c.generated) })));
  const data = await source.transaction(async tx => {
    await tx.executeRaw(DIGEST_SESSION_SETTINGS);
    const out: Record<string, Array<Record<string, string | null>>> = {};
    for (const { entry, columns } of plans) {
      const transforms = new Map(entry.transforms.map(t => [t.column, t.expression!]));
      out[entry.relation] = await tx.executeRaw(`SELECT ${columns.map((c, i) => `(${transforms.get(c.name) ?? quoteIdent(c.name)})::text AS c${i}`).join(', ')}
        FROM ${quoteIdent(entry.relation)}${entry.rowFilter ? ` WHERE ${entry.rowFilter}` : ''}`);
    }
    return out;
  });
  await target.transaction(async tx => {
    await tx.executeRaw("SELECT set_config('gbrain.graduation_run',$1,true), set_config('session_replication_role','replica',true)", [RUN_ID]);
    for (const { entry } of [...plans].reverse()) await tx.executeRaw(`DELETE FROM ${quoteIdent(entry.relation)}${entry.rowFilter ? ` WHERE ${entry.rowFilter}` : ''}`);
    for (const { entry, columns } of plans) {
      const insert = `INSERT INTO ${quoteIdent(entry.relation)} (${columns.map(c => quoteIdent(c.name)).join(', ')})
        VALUES (${columns.map((c, i) => `$${i + 1}::text::${c.type}`).join(', ')})`;
      for (const row of data[entry.relation]!) await tx.executeRaw(insert, columns.map((_, i) => row[`c${i}`]));
    }
  });
  const seqs = await source.executeRaw<{ seq: string }>(`SELECT c.relname AS seq FROM pg_class c JOIN pg_namespace n ON n.oid=c.relnamespace
    WHERE c.relkind='S' AND n.nspname=current_schema() AND c.relname <> 'planner_stats_deltas_id_seq'`);
  for (const { seq } of seqs) {
    const [pos] = await source.executeRaw<{ v: string; called: unknown }>(`SELECT last_value::text AS v, is_called AS called FROM ${quoteIdent(seq)}`);
    await target.executeRaw('SELECT setval($1::regclass, $2::bigint, $3::boolean)', [quoteIdent(seq), pos!.v, pos!.called === true || pos!.called === 't']);
  }
}

function context(overrides: Partial<VerifyContext> = {}): VerifyContext {
  return { inventory: GRADUATION_INVENTORY, sourceReceipts: receipts, replayRequestId: fixture.queuedRequestId, replayCallerIntent: callerIntent,
    runId: RUN_ID, runDoctor: async () => [], ...overrides };
}

beforeAll(async () => {
  source = new PGLiteEngine();
  await source.connect({});
  await source.initSchema();
  liteTarget = new PGLiteEngine();
  await liteTarget.connect({});
  await liteTarget.initSchema();
  const databaseUrl = process.env.DATABASE_URL;
  await withEnv({ GBRAIN_HOME: home, DATABASE_URL: undefined, GBRAIN_DATABASE_URL: undefined }, async () => {
    fixture = await buildHistoryFixture(source, { pages: 12, seed: 5, sources: 2, worktrees: 1, root: join(home, 'checkouts') });
  });
  const [queued] = await source.executeRaw<{ intent: unknown }>('SELECT intent FROM persistence_requests WHERE request_id=$1::uuid', [fixture.queuedRequestId]);
  callerIntent = (typeof queued!.intent === 'string' ? JSON.parse(queued!.intent) : queued!.intent) as Record<string, unknown>;
  await withProtocol(source, async tx => {
    await tx.executeRaw(`UPDATE persistence_requests SET state='committed', outcome='{"status":"committed","drained":true}'::jsonb, completed_at=now()
      WHERE request_id=$1::uuid`, [fixture.queuedRequestId]);
    await tx.executeRaw(`UPDATE persistence_effects SET state='running', execution_token=gen_random_uuid(), claim_expires_at=now() - interval '1 minute'
      WHERE id = (SELECT min(id) FROM persistence_effects WHERE state='committed')`);
  });
  await source.executeRaw(`INSERT INTO minion_jobs (name, status, submission_authority) VALUES ('graduation-verify-probe', 'waiting', '{}'::jsonb)`);
  await source.executeRaw(`UPDATE minion_jobs SET status='active', claim_generation=1, lock_token='lease-token', lock_until=now() + interval '1 minute',
    started_at=now(), timeout_at=now() + interval '1 hour', attempts_started=1 WHERE name='graduation-verify-probe'`);
  for (const key of CONFIG_KEYS) await source.executeRaw('INSERT INTO config(key, value) VALUES ($1, $2)', [key, `value of ${key}`]);
  await source.executeRaw('UPDATE persistence_brain SET enabled=false WHERE singleton=1');
  const [dims] = await source.executeRaw<{ n: number }>("SELECT atttypmod AS n FROM pg_attribute WHERE attrelid='content_chunks'::regclass AND attname='embedding'");
  await source.executeRaw(`UPDATE content_chunks SET embedding = (SELECT array_agg(sin(g * id)::real)::vector FROM generate_series(1, $1::int) g)
    WHERE id IN (SELECT id FROM content_chunks ORDER BY id LIMIT 3)`, [dims!.n]);
  await source.executeRaw('UPDATE persistence_brain SET enabled=true WHERE singleton=1');
  receipts = [];
  for (const entry of await copyOrder(source)) receipts.push(await digestTable(source, entry, { applyTransforms: true }));

  targets.push({ name: 'pglite', engine: liteTarget });
  if (databaseUrl) {
    const pg = await isolatedPersistencePostgres(databaseUrl);
    targets.push({ name: 'postgres', engine: pg.engine });
    closePostgres = pg.close;
  }
  for (const { engine } of targets) { await installFence(engine); await testCopy(engine); }
}, 300_000);
afterAll(async () => {
  await source.disconnect();
  await liteTarget.disconnect();
  await closePostgres?.();
  rmSync(home, { recursive: true, force: true });
});

describe('verifyGraduation', () => {
  test('a faithful, fenced copy of a brain with history verifies green, with transforms applied to the source side', async () => {
    for (const { name, engine } of targets) {
      const [before] = await engine.executeRaw<{ n: number }>('SELECT count(*)::int AS n FROM persistence_requests');
      const result = await verifyGraduation({ source, target: engine }, context());
      expect({ name, failures: result.failures }).toEqual({ name, failures: [] });
      expect(result.ok).toBe(true);
      expect(result.replay).toEqual({ status: 'passed', requestId: fixture.queuedRequestId });
      expect(result.tables.map(t => t.relation)).toEqual((await copyOrder(source)).map(e => e.relation));
      expect(result.tables.find(t => t.relation === 'config')!.rows).toBeGreaterThan(CONFIG_KEYS.length);
      const [job] = await engine.executeRaw<{ status: string; lock_token: string | null }>("SELECT status, lock_token FROM minion_jobs WHERE name='graduation-verify-probe'");
      expect(job).toEqual({ status: 'waiting', lock_token: null });
      expect((await engine.executeRaw("SELECT 1 FROM persistence_effects WHERE state='running'")).length).toBe(0);
      expect(await engine.executeRaw<{ n: number }>('SELECT count(*)::int AS n FROM content_chunks WHERE embedding IS NOT NULL')).toEqual([{ n: 3 }]);
      expect((await engine.executeRaw<{ key: string }>('SELECT key FROM config WHERE key = ANY($1::text[]) ORDER BY key COLLATE "C"', [CONFIG_KEYS])).map(r => r.key))
        .toEqual([...CONFIG_KEYS].sort((a, b) => Buffer.compare(Buffer.from(a), Buffer.from(b))));
      expect((await engine.executeRaw<{ value: string }>("SELECT value FROM config WHERE key='engine'")).every(r => r.value === engine.kind)).toBe(true);
      const [after] = await engine.executeRaw<{ n: number }>('SELECT count(*)::int AS n FROM persistence_requests');
      expect(after!.n).toBe(before!.n);
      const [brain] = await engine.executeRaw<{ enabled: boolean }>('SELECT enabled FROM persistence_brain');
      expect(brain!.enabled).toBe(false);
    }
  }, 120_000);

  test('each kind of damage is reported with its table, first key and column', async () => {
    for (const { name, engine } of targets) {
      const [page] = await engine.executeRaw<{ id: number; title: string }>('SELECT id, title FROM pages ORDER BY id LIMIT 1');
      await targetWrite(engine, "UPDATE pages SET title = title || ' (edited)' WHERE id=$1", [page!.id]);
      await targetWrite(engine, 'DELETE FROM timeline_entries WHERE id = (SELECT max(id) FROM timeline_entries)');
      await targetWrite(engine, "INSERT INTO tags (page_id, tag) VALUES (2147483000, 'orphan')");
      await engine.executeRaw('ALTER TABLE pages DISABLE TRIGGER bump_page_generation_trg');
      await engine.executeRaw(`DROP TRIGGER ${GRADUATION_FENCE_TRIGGER} ON links`);
      await engine.executeRaw("SELECT setval('pages_id_seq', 1)");
      await engine.executeRaw('CREATE TABLE graduation_verify_stray (id int PRIMARY KEY)');
      await source.executeRaw("UPDATE config SET value='changed after copy' WHERE key='graduation-key'");
      try {
        const result = await verifyGraduation({ source, target: engine }, context({ runDoctor: async () => ['embedding_coverage'] }));
        expect(result.ok).toBe(false);
        const has = (match: Record<string, unknown>) => expect({ name, failures: result.failures }).toMatchObject({ name, failures: expect.arrayContaining([expect.objectContaining(match)]) });
        has({ relation: 'pages', kind: 'digest', firstKey: JSON.stringify([String(page!.id)]), column: 'title' });
        has({ relation: 'timeline_entries', kind: 'count' });
        has({ relation: 'timeline_entries', kind: 'digest' });
        has({ relation: 'tags', kind: 'fk', column: 'page_id', firstKey: '["2147483000"]' });
        has({ relation: 'pages', kind: 'trigger', detail: expect.stringContaining('bump_page_generation_trg') });
        has({ relation: 'links', kind: 'trigger', detail: expect.stringContaining('missing') });
        has({ relation: 'pages_id_seq', kind: 'sequence' });
        has({ relation: `(${engine.kind} schema)`, kind: 'relation_set', detail: expect.stringContaining('graduation_verify_stray') });
        has({ relation: 'config', kind: 'digest', detail: expect.stringContaining('source changed since the copy') });
        has({ relation: 'config', kind: 'digest', firstKey: '["graduation-key"]', column: 'value' });
        has({ relation: '(target doctor)', kind: 'doctor', detail: expect.stringContaining('embedding_coverage') });
        expect(result.doctorFailingChecks).toEqual(['embedding_coverage']);
      } finally {
        await source.executeRaw("UPDATE config SET value='value of graduation-key' WHERE key='graduation-key'");
        await engine.executeRaw('DROP TABLE graduation_verify_stray');
        await engine.executeRaw('ALTER TABLE pages ENABLE TRIGGER bump_page_generation_trg');
        await fenceTable(engine, 'links');
        await testCopy(engine);
      }
      expect((await verifyGraduation({ source, target: engine }, context())).failures).toEqual([]);
    }
  }, 240_000);

  test('after cutover the fence must be gone; missing receipts are failures', async () => {
    for (const { name, engine } of targets) {
      const result = await verifyGraduation({ source, target: engine }, context({ expectFence: false, sourceReceipts: receipts.filter(r => r.relation !== 'pages') }));
      expect({ name, fenceFailures: result.failures.filter(f => f.kind === 'trigger' && f.detail.includes('still installed')).length > 0 }).toEqual({ name, fenceFailures: true });
      expect(result.failures).toContainEqual(expect.objectContaining({ relation: 'pages', kind: 'digest', detail: expect.stringContaining('No copy-time source receipt') }));
    }
  }, 120_000);
});

describe('replayProbe', () => {
  test('resubmits the drained request with the original caller input and leaves nothing behind', async () => {
    for (const { engine } of targets) {
      const reordered = Object.fromEntries(Object.entries(callerIntent).reverse());
      expect(await replayProbe(engine, fixture.queuedRequestId, reordered, { sourceEnabled: true, runId: RUN_ID }))
        .toEqual({ status: 'passed', requestId: fixture.queuedRequestId });
      const changed = await replayProbe(engine, fixture.queuedRequestId, { ...callerIntent, content: 'different' }, { sourceEnabled: true, runId: RUN_ID });
      expect(changed).toMatchObject({ status: 'failed', detail: expect.stringContaining('used for a different write') });
      const unfenced = await replayProbe(engine, fixture.queuedRequestId, callerIntent, { sourceEnabled: true });
      expect(unfenced).toMatchObject({ status: 'failed', detail: expect.stringContaining('graduation_in_progress') });
      expect(await replayProbe(engine, fixture.queuedRequestId, undefined)).toEqual({ status: 'not_available', reason: 'no_caller_input' });
      expect(await replayProbe(engine, randomUUID(), callerIntent)).toMatchObject({ status: 'failed', detail: expect.stringContaining('not on the target') });
      const [brain] = await engine.executeRaw<{ enabled: boolean }>('SELECT enabled FROM persistence_brain');
      expect(brain!.enabled).toBe(false);
    }
  }, 120_000);

  test('a revoked principal, an archived source and a changed topology report not_available', async () => {
    for (const { engine } of targets) {
      const [row] = await engine.executeRaw<{ principal_id: string; source_id: string }>('SELECT principal_id, source_id FROM persistence_requests WHERE request_id=$1::uuid', [fixture.queuedRequestId]);
      const probe = () => replayProbe(engine, fixture.queuedRequestId, callerIntent, { sourceEnabled: true, runId: RUN_ID });
      await targetWrite(engine, 'UPDATE persistence_local_writers SET revoked_at=now() WHERE id=$1::uuid', [row!.principal_id]);
      try { expect(await probe()).toEqual({ status: 'not_available', reason: 'revoked_principal' }); }
      finally { await targetWrite(engine, 'UPDATE persistence_local_writers SET revoked_at=NULL WHERE id=$1::uuid', [row!.principal_id]); }
      await targetWrite(engine, 'UPDATE sources SET archived=true WHERE id=$1', [row!.source_id]);
      try { expect(await probe()).toEqual({ status: 'not_available', reason: 'archived_source' }); }
      finally { await targetWrite(engine, 'UPDATE sources SET archived=false WHERE id=$1', [row!.source_id]); }
      await targetWrite(engine, 'UPDATE persistence_source_bindings SET topology_generation=topology_generation+1 WHERE source_id=$1', [row!.source_id]);
      try { expect(await probe()).toEqual({ status: 'not_available', reason: 'source_changed' }); }
      finally { await targetWrite(engine, 'UPDATE persistence_source_bindings SET topology_generation=topology_generation-1 WHERE source_id=$1', [row!.source_id]); }
      expect(await probe()).toEqual({ status: 'passed', requestId: fixture.queuedRequestId });
    }
  }, 120_000);

  test('without a named request verify picks the newest uncompacted terminal request and needs its caller input', async () => {
    const { engine } = targets[0]!;
    const result = await verifyGraduation({ source, target: engine }, context({ replayRequestId: undefined, replayCallerIntent: undefined }));
    expect(result.replay).toEqual({ status: 'not_available', reason: 'no_caller_input' });
    expect(result.failures).toEqual([]);
  }, 120_000);
});