Garry's Opinionated OpenClaw/Hermes Agent Brain
TypeScript
/**
* Engine graduation state machine and run custody.
*
* Protects: the transition tables drive every manifest/row change (an illegal
* transition throws); manifest v3 is 0600 and never leaks the password; the
* run reaches exactly one authoritative engine from a fresh start and after a
* crash at every custody seam; the target fence refuses every writer but the
* run; a source change during a lock gap reaches the target; a stray
* datastore at the old path is split brain with the target withheld;
* refusals before the fence leave the source writable; --status mutates
* nothing. Regressions that fail it: a resume that trusts current routing, a
* cutover that grants authority before the tombstone, a fence that admits
* unidentified sessions, a lock gap that skips the re-check.
*/
import { afterAll, beforeAll, describe, expect, test } from 'bun:test';
import { existsSync, lstatSync, mkdirSync, readFileSync, statSync } from 'node:fs';
import { join } from 'node:path';
import {
assertTransition, GRADUATION_RUN_BOUNDARIES, MANIFEST_STATES, MANIFEST_TRANSITIONS, SOURCE_ROW_STATES, SOURCE_TRANSITIONS, TARGET_ROW_STATES, TARGET_TRANSITIONS,
type GraduationManifest, type ManifestState,
} from '../src/core/persistence/engine-graduation.types.ts';
import {
graduationStatus, planGraduation, readGraduationManifest, redactManifest, resumeGraduation, runGraduation, transitionManifest,
writeGraduationManifest, type GraduationOptions,
} from '../src/core/persistence/engine-graduation.ts';
import { assertGraduationAdmission, assertGraduationConnectAllowed, graduatedPath, readIntentMarker, readTombstone } from '../src/core/persistence/graduation-custody.ts';
import { graduationFenceStatus, readGraduationRow, setSourceState, setTargetState, withGraduationRun } from '../src/core/persistence/graduation-schema.ts';
import { LiveServeLockError } from '../src/core/pglite-lock.ts';
import { assertSafeE2eDatabaseUrl } from './helpers/db-guard.ts';
import { crashAt, makeHarness, openPglite, probeRows, TARGET_URL, type Harness } from './helpers/graduation-harness.ts';
/** Another process opening the datastore (a respawned serve): its refusal code, or 'opened'. */
function openInChild(dataDir: string): string {
const script = `import { PGLiteEngine } from ${JSON.stringify(join(import.meta.dir, '../src/core/pglite-engine.ts'))};
const Engine = PGLiteEngine;
const e = new Engine();
try { await e.connect({ engine: 'pglite', database_path: ${JSON.stringify(dataDir)} }); console.log('opened'); await e.disconnect(); }
catch (err) { console.log(err?.code ?? String(err)); }`;
const child = Bun.spawnSync([process.execPath, '-e', script], { stdout: 'pipe', stderr: 'pipe', timeout: 60_000 });
return child.stdout.toString().trim().split('\n').pop() ?? '';
}
function codeOf(error: unknown): string | undefined { return (error as { code?: string })?.code; }
async function refusal(promise: Promise<unknown>): Promise<{ code?: string; fix?: { argv?: string[]; plan_hash?: string }; message: string }> {
try { await promise; } catch (error) { return { code: codeOf(error), fix: (error as { fix?: { argv?: string[] } }).fix, message: (error as Error).message }; }
throw new Error('expected a refusal');
}
describe('transition tables', () => {
const tables = [
['manifest', MANIFEST_STATES, MANIFEST_TRANSITIONS],
['target', TARGET_ROW_STATES, TARGET_TRANSITIONS],
['source', SOURCE_ROW_STATES, SOURCE_TRANSITIONS],
] as const;
for (const [name, states, table] of tables) {
test(`${name}: assertTransition admits exactly the listed transitions`, () => {
for (const from of states) {
for (const to of states) {
const allowed = (table as Record<string, readonly string[]>)[from]!.includes(to);
const attempt = () => assertTransition(table as Record<string, readonly string[]>, from, to);
if (allowed) expect(attempt).not.toThrow(); else expect(attempt).toThrow(/Illegal graduation transition/);
}
}
});
}
test('every manifest state is reachable from planned, and terminal states have no exit', () => {
const seen = new Set<string>(['planned']);
const queue: string[] = ['planned'];
while (queue.length) for (const next of MANIFEST_TRANSITIONS[queue.shift() as ManifestState]) if (!seen.has(next)) { seen.add(next); queue.push(next); }
expect([...seen].sort()).toEqual([...MANIFEST_STATES].sort());
expect(MANIFEST_TRANSITIONS.rolled_back).toEqual([]);
expect(MANIFEST_TRANSITIONS.abandoned).toEqual([]);
expect(MANIFEST_TRANSITIONS.rollback_approved).toEqual(['source_restoring']);
});
});
describe('manifest v3', () => {
let h: Harness;
beforeAll(async () => { h = await makeHarness(); });
afterAll(async () => { await h.close(); });
test('written 0600 atomically, round-trips, redaction drops the URLs, illegal transitions throw', () => {
const path = join(h.root, 'manifest-check.json');
const m: GraduationManifest = {
version: 3, runId: 'r', state: 'planned', source: { dataDir: h.dataDir, brainId: '', hostId: '' },
target: { id: 'x', host: 'h', port: 5432, database: 'd', user: 'u' }, routes: { main: 'postgres://u@h:5432/d', ddl: 'postgres://u@h:5432/d' },
inventoryVersion: 1, schemaVersion: 1, planHash: 'p', triggerBypass: 'session_replication_role', tables: [], timings: {},
startedAt: '', updatedAt: '', targetUrls: { main: TARGET_URL, ddl: TARGET_URL },
};
writeGraduationManifest(m, path);
expect(statSync(path).mode & 0o777).toBe(0o600);
expect(readGraduationManifest(path)).toEqual(m);
expect(JSON.stringify(redactManifest(m))).not.toContain('secret-pw');
expect(() => transitionManifest(m, 'cutover')).toThrow(/planned -> cutover/);
transitionManifest(m, 'quiesced');
expect(m.state).toBe('quiesced');
});
});
describe('row state machine, connect and admission checks', () => {
let h: Harness;
beforeAll(async () => { h = await makeHarness(); await h.target.initSchema(); });
afterAll(async () => { await h.close(); });
test('target row transitions follow TARGET_TRANSITIONS; connect and admission refuse until authority', async () => {
const run = crypto.randomUUID();
const t = h.target;
await expect(withGraduationRun(t, run, tx => setTargetState(tx, run, 'verified'))).rejects.toThrow(/\(none\) -> verified/);
await withGraduationRun(t, run, tx => setTargetState(tx, run, 'copying'));
await expect(withGraduationRun(t, run, tx => setTargetState(tx, run, 'authoritative'))).rejects.toThrow(/copying -> authoritative/);
expect(codeOf(await assertGraduationConnectAllowed(t, {}).catch(e => e))).toBe('graduation_in_progress');
await expect(assertGraduationConnectAllowed(t, { GBRAIN_GRADUATION_RUN: run })).resolves.toBeUndefined();
expect(codeOf(await t.transaction(tx => assertGraduationAdmission(tx)).catch(e => e))).toBe('graduation_in_progress');
await expect(withGraduationRun(t, run, tx => assertGraduationAdmission(tx))).resolves.toBeUndefined();
await expect(withGraduationRun(t, crypto.randomUUID(), tx => setTargetState(tx, crypto.randomUUID(), 'copying'))).rejects.toThrow(/belongs to graduation run/);
await withGraduationRun(t, run, async tx => { await setTargetState(tx, run, 'verifying'); await setTargetState(tx, run, 'verified'); await setTargetState(tx, run, 'authoritative'); });
await expect(assertGraduationConnectAllowed(t, {})).resolves.toBeUndefined();
await expect(t.transaction(tx => assertGraduationAdmission(tx))).resolves.toBeUndefined();
}, 60_000);
test('a source row at cutover refuses: interrupted before the tombstone exists', async () => {
const source = await openPglite(join(h.root, 'cutover-source.pglite'));
try {
await source.initSchema();
const run = crypto.randomUUID();
await withGraduationRun(source, run, tx => setSourceState(tx, run, 'quiesced', { sourceDataDir: join(h.root, 'cutover-source.pglite') }));
await expect(assertGraduationConnectAllowed(source, {})).resolves.toBeUndefined();
await withGraduationRun(source, run, tx => setSourceState(tx, run, 'cutover'));
expect(codeOf(await assertGraduationConnectAllowed(source, {}).catch(e => e))).toBe('graduation_interrupted');
expect(codeOf(await source.transaction(tx => assertGraduationAdmission(tx)).catch(e => e))).toBe('graduation_interrupted');
await expect(withGraduationRun(source, run, tx => setSourceState(tx, run, 'quiesced'))).resolves.toBeUndefined();
} finally { await source.disconnect(); }
}, 60_000);
});
const POSTGRES_URL = process.env.DATABASE_URL;
const targets: Array<[string, string | undefined]> = [['pglite target', undefined]];
if (POSTGRES_URL) { assertSafeE2eDatabaseUrl(POSTGRES_URL); targets.push(['postgres target', POSTGRES_URL]); }
for (const [label, postgresUrl] of targets) {
describe(`graduation run (${label})`, () => {
let h: Harness;
beforeAll(async () => { h = await makeHarness({ postgresUrl }); });
afterAll(async () => { await h.close(); });
async function fresh(): Promise<void> { await h.close(); h = await makeHarness({ postgresUrl }); }
function runOpts(expect: string, extra: Partial<GraduationOptions> = {}): GraduationOptions {
return { config: { engine: 'pglite', database_path: h.dataDir } as GraduationOptions['config'], to: 'postgres', url: TARGET_URL, env: {}, drainTimeoutMs: 60_000,
force: false, yes: true, expectPlanHash: expect, deps: h.deps, handoffTimeoutMs: 1_000, ...extra };
}
async function plan(extra: Partial<GraduationOptions> = {}): Promise<string> {
return (await planGraduation(runOpts('', { yes: false, ...extra }))).planHash;
}
async function assertGraduated(): Promise<void> {
const config = JSON.parse(readFileSync(join(h.gbrainDir, 'config.json'), 'utf8'));
expect(config.engine).toBe('postgres');
expect(config.database_url).toBe(TARGET_URL);
expect(config.database_path).toBeUndefined();
expect(lstatSync(h.dataDir).isFile()).toBe(true);
const tombstone = readTombstone(h.dataDir)!;
expect(JSON.stringify(tombstone)).not.toContain('secret-pw');
expect(existsSync(tombstone.movedTo)).toBe(true);
const row = await readGraduationRow(h.target);
expect(row?.state).toBe('authoritative');
expect((await graduationFenceStatus(h.target)).fenced).toBe(0);
expect((await probeRows(h.target)).map(r => r.v)).toEqual(['alpha', 'beta', 'gamma']);
const manifest = readGraduationManifest(join(h.gbrainDir, 'graduation-manifest.json'))!;
expect(manifest.state).toBe('graduated');
expect(statSync(join(h.gbrainDir, 'graduation-manifest.json')).mode & 0o777).toBe(0o600);
const mounts = JSON.parse(readFileSync(h.mountsPath, 'utf8'));
expect(mounts.mounts[0]).toMatchObject({ id: 'team-example', engine: 'postgres', database_url: TARGET_URL });
expect(mounts.mounts[0].database_path).toBeUndefined();
const reopen = await openPglite(h.dataDir).catch(e => e);
expect(codeOf(reopen)).toBe('engine_graduated');
expect(lstatSync(h.dataDir).isFile()).toBe(true);
}
test('plan -> run reaches one authoritative engine; the source path is a tombstone', async () => {
await h.inHome(async () => {
const hash = await plan();
const receipt = await runGraduation(runOpts(hash));
expect(receipt.state).toBe('authoritative');
expect(receipt.triggerBypass).toBe('session_replication_role');
await assertGraduated();
const moved = readTombstone(h.dataDir)!.movedTo;
const retained = await openPglite(moved);
try { expect((await readGraduationRow(retained))?.state).toBe('cutover'); } finally { await retained.disconnect(); }
expect(h.calls.pause_released).toBe(1);
});
}, 60_000);
test('the fence refuses other writers while the copy runs; the run itself writes', async () => {
await fresh();
await h.inHome(async () => {
let observed: { write?: string; exempt?: boolean; respawn?: string } = {};
const hash = await plan();
await runGraduation(runOpts(hash, { pauseAt: 'table_copied', pauseHook: async () => {
const run = readGraduationManifest(join(h.gbrainDir, 'graduation-manifest.json'))!.runId;
observed = {
...observed,
write: await h.target.executeRaw(`INSERT INTO config (key, value) VALUES ('stray', 'x')`).then(() => 'accepted', (e: Error) => e.message.split(':')[0]),
exempt: await assertGraduationConnectAllowed(h.target, { GBRAIN_GRADUATION_RUN: run }).then(() => true),
};
const marker = readIntentMarker(h.dataDir)!;
expect(marker.state).toBe('copying');
observed.respawn = openInChild(h.dataDir);
} }));
expect(observed).toEqual({ write: 'graduation_in_progress', exempt: true, respawn: 'graduation_in_progress' });
await assertGraduated();
});
}, 60_000);
const seams = GRADUATION_RUN_BOUNDARIES.filter(b => b !== 'batch_copied' && b !== 'graduated');
for (const seam of seams) {
test(`a crash at ${seam} resumes to exactly one authoritative engine`, async () => {
await fresh();
await h.inHome(async () => {
const hash = await plan();
await expect(runGraduation(runOpts(hash, { pauseAt: seam, pauseHook: crashAt(seam) }))).rejects.toThrow(`simulated crash at ${seam}`);
const status = await graduationStatus({ deps: h.deps });
expect(status.liveRun).toBeNull();
expect(status.nextArgv).toEqual(['gbrain', 'migrate', '--resume']);
const row = await readGraduationRow(h.target);
if (row) expect(row.state === 'authoritative' ? seam : 'non-authoritative').toBe(['authoritative', 'config_flipped', 'registry_rewritten'].includes(seam) ? seam : 'non-authoritative');
await resumeGraduation({ env: {}, deps: h.deps, handoffTimeoutMs: 1_000 });
await assertGraduated();
});
}, 60_000);
}
test('a source write during a lock gap after verify reaches the target', async () => {
await fresh();
await h.inHome(async () => {
const hash = await plan();
await expect(runGraduation(runOpts(hash, { pauseAt: 'verified', pauseHook: crashAt('verified') }))).rejects.toThrow();
const copiesBefore = h.calls.copy_probe ?? 0;
const source = await openPglite(h.dataDir);
try { await source.executeRaw(`INSERT INTO grad_probe VALUES (4, 'written-in-gap')`); } finally { await source.disconnect(); }
await resumeGraduation({ env: {}, deps: h.deps, handoffTimeoutMs: 1_000 });
expect(h.calls.copy_probe).toBe(copiesBefore + 1);
expect((await probeRows(h.target)).map(r => r.v)).toContain('written-in-gap');
});
}, 60_000);
test('a stray datastore at the old path is split brain; the target stays non-authoritative', async () => {
await fresh();
await h.inHome(async () => {
const hash = await plan();
await expect(runGraduation(runOpts(hash, { pauseAt: 'moved_aside', pauseHook: crashAt('moved_aside') }))).rejects.toThrow();
mkdirSync(h.dataDir);
const refused = await refusal(resumeGraduation({ env: {}, deps: h.deps, handoffTimeoutMs: 1_000 }));
expect(refused.code).toBe('graduation_split_brain');
expect(refused.fix?.argv).toEqual(['gbrain', 'migrate', '--resume', '--yes']);
expect((await readGraduationRow(h.target))?.state).toBe('verified');
const status = await graduationStatus({ deps: h.deps });
expect(status.sourcePath?.state).toBe('split_brain');
expect(status.splitBrain?.map(side => side.path)).toEqual([h.dataDir, `${h.dataDir}.graduated-${status.runId}`]);
await resumeGraduation({ env: {}, deps: h.deps, handoffTimeoutMs: 1_000, yes: true });
expect(existsSync(`${h.dataDir}.stray-${status.runId}`)).toBe(true);
await assertGraduated();
});
}, 60_000);
test('a path occupied when the live run creates the tombstone refuses as split brain', async () => {
await fresh();
await h.inHome(async () => {
const hash = await plan();
const refused = await refusal(runGraduation(runOpts(hash, { pauseAt: 'moved_aside', pauseHook: async () => { mkdirSync(h.dataDir); } })));
expect(refused.code).toBe('graduation_split_brain');
expect((await readGraduationRow(h.target))?.state).toBe('verified');
expect((await graduationFenceStatus(h.target)).unfenced).toEqual([]);
});
}, 60_000);
test('a changed plan refuses before anything is fenced and leaves the source writable', async () => {
await fresh();
await h.inHome(async () => {
const refused = await refusal(runGraduation(runOpts('0000000000000000')));
expect(refused.code).toBe('preview_changed');
expect(readIntentMarker(h.dataDir)).toBeNull();
expect(readGraduationManifest(join(h.gbrainDir, 'graduation-manifest.json'))?.state).toBe('abandoned');
const source = await openPglite(h.dataDir);
try { await source.executeRaw(`INSERT INTO grad_probe VALUES (9, 'still-writable')`); } finally { await source.disconnect(); }
expect(await readGraduationRow(h.target)).toBeNull();
});
}, 60_000);
test('a live serve that never hands off refuses with the two-step writer-held fix; one that does is waited for', async () => {
await fresh();
await h.inHome(async () => {
const hash = await plan();
const held = { ...h.deps, openSource: async () => { throw new LiveServeLockError('held by serve', { pid: 424242, transport: 'stdio' }); } };
const refused = await refusal(runGraduation(runOpts(hash, { deps: held, handoffTimeoutMs: 300 })));
expect(refused.code).toBe('graduation_source_writer_held');
expect(refused.fix?.argv).toEqual(['kill', '424242']);
expect(readIntentMarker(h.dataDir)).toBeNull();
let refusals = 2;
const { defaultGraduationDeps } = await import('../src/core/persistence/engine-graduation.ts');
const real = defaultGraduationDeps();
const handOff = { ...h.deps, openSource: async (dir: string, o: { migrate: boolean }) => {
if (refusals-- > 0) throw new LiveServeLockError('held by serve', { pid: 424242, transport: 'stdio' });
return real.openSource.call(real, dir, o);
} };
await runGraduation(runOpts(hash, { deps: handOff, handoffTimeoutMs: 5_000 }));
await assertGraduated();
});
}, 60_000);
test('a drain timeout and a verify failure are resumable; the source stays writable in between', async () => {
await fresh();
await h.inHome(async () => {
const hash = await plan();
const stuck = { ...h.deps, drainForGraduation: async () => ({ drained: [], blockers: [{ kind: 'request' as const, id: 'req-1', detail: 'running', needsUser: false }] }) };
const timeout = await refusal(runGraduation(runOpts(hash, { deps: stuck, drainTimeoutMs: 60_000 })));
expect(timeout.code).toBe('graduation_drain_timeout');
expect(timeout.fix?.argv).toEqual(['gbrain', 'migrate', '--resume', '--drain-timeout', '120']);
expect(readGraduationManifest(join(h.gbrainDir, 'graduation-manifest.json'))?.state).toBe('draining');
const failing = { ...h.deps, verifyGraduation: async () => ({ ok: false, tables: [], failures: [{ relation: 'grad_probe', kind: 'digest' as const, detail: 'mismatch' }], replay: { status: 'not_available' as const, reason: 'no_caller_input' as const }, doctorFailingChecks: [] }) };
const failed = await refusal(resumeGraduation({ env: {}, deps: failing, handoffTimeoutMs: 1_000 }));
expect(failed.code).toBe('graduation_verify_failed');
expect((await readGraduationRow(h.target))?.state).toBe('verify_failed');
const source = await openPglite(h.dataDir);
try { await source.executeRaw(`INSERT INTO grad_probe VALUES (5, 'after-verify-failure')`); } finally { await source.disconnect(); }
await resumeGraduation({ env: {}, deps: h.deps, handoffTimeoutMs: 1_000 });
expect((await probeRows(h.target)).map(r => r.v)).toContain('after-verify-failure');
expect(lstatSync(h.dataDir).isFile()).toBe(true);
});
}, 60_000);
test('--status performs zero mutations at a crash point', async () => {
await fresh();
await h.inHome(async () => {
const hash = await plan();
await expect(runGraduation(runOpts(hash, { pauseAt: 'source_cutover', pauseHook: crashAt('source_cutover') }))).rejects.toThrow();
const files = [join(h.gbrainDir, 'graduation-manifest.json'), `${h.dataDir}.gbrain-graduation.json`, join(h.gbrainDir, 'config.json')];
const before = files.map(f => [readFileSync(f, 'utf8'), statSync(f).mtimeMs]);
const status = await graduationStatus({ deps: h.deps });
expect(status.state).toBe('cutover');
expect(status.sourcePath?.state).toBe('interrupted');
expect(status.target?.row).toBe('verified');
expect(JSON.stringify(status)).not.toContain('secret-pw');
expect(files.map(f => [readFileSync(f, 'utf8'), statSync(f).mtimeMs])).toEqual(before);
expect(existsSync(graduatedPath(h.dataDir, status.runId!))).toBe(false);
});
}, 60_000);
});
}