Garry's Opinionated OpenClaw/Hermes Agent Brain
TypeScript
/**
* Engine graduation filesystem custody: intent marker, tombstone, lock-free
* path inspection, lock-retaining close, held-lock move-aside and the PGLite
* open-path refusals.
*
* Protects: the kernel lock never gaps between close, rename and tombstone
* (no opener acquires it and none creates a data dir); the tombstone is
* exclusive and 0600; a tombstoned path refuses with engine_graduated and is
* never re-initialised; a live marker refuses opens before the lock while a
* dead pre-cutover marker lets the source open; a dead marker past cutover is
* graduation_interrupted with the lock released. Regressions that fail it: a
* close that releases the lock, a tombstone write that overwrites, an opener
* that mkdirs over the tombstone, a liveness check fooled by PID reuse.
*/
import { afterAll, beforeAll, describe, expect, test } from 'bun:test';
import { existsSync, lstatSync, mkdirSync, mkdtempSync, readFileSync, statSync, writeFileSync } from 'node:fs';
import { tmpdir } from 'node:os';
import { join } from 'node:path';
import {
assertPgliteGraduationOpenable, currentProcessIdentity, graduatedPath, graduationHandOffRequested, inspectGraduationPath,
intentMarkerPath, moveAsideHeld, processStartTime, readIntentMarker, readTombstone, registerGraduationRunInProcess, removeTombstone,
TombstonePathOccupiedError, writeIntentMarker, writeTombstone,
} from '../src/core/persistence/graduation-custody.ts';
import type { IntentMarker, ManifestState, Tombstone } from '../src/core/persistence/engine-graduation.types.ts';
import { acquireKernelLockOnly, acquireLock, PgliteBusyError, releaseLock, type LockHandle } from '../src/core/pglite-lock.ts';
import { moveHeldPglite } from '../src/core/persistence/maintenance.ts';
import { newPgliteEngine, openPglite } from './helpers/graduation-harness.ts';
const target = { id: 'target-id', host: 'db.example.test', port: 5432, database: 'brain', user: 'alice-example' };
function marker(runId: string, state: ManifestState, identity: Partial<IntentMarker> = {}): IntentMarker {
return { runId, state, ...currentProcessIdentity(), target, updatedAt: new Date().toISOString(), ...identity };
}
function tombstone(runId: string, movedTo: string): Tombstone {
return { kind: 'gbrain-engine-graduated', runId, brainId: 'brain', movedTo, target, targetDisplayUrl: 'postgres://[email protected]:5432/brain',
graduatedAt: new Date().toISOString(), fixArgv: ['gbrain', 'config', 'set', 'database_url', '<GBRAIN_TARGET_URL>'] };
}
function codeOf(error: unknown): string | undefined { return (error as { code?: string })?.code; }
function deadPid(): number {
const child = Bun.spawnSync(['sh', '-c', 'echo $$']);
return Number(child.stdout.toString().trim());
}
let root: string;
beforeAll(() => { root = mkdtempSync(join(tmpdir(), 'gbrain-custody-')); });
afterAll(() => {});
describe('intent marker and tombstone files', () => {
test('the marker is replaced atomically with mode 0600; an unreadable one is never read as "no graduation"', () => {
const dir = join(root, 'marker.pglite');
mkdirSync(dir);
expect(readIntentMarker(dir)).toBeNull();
writeIntentMarker(dir, marker('run-a', 'planned'));
writeIntentMarker(dir, marker('run-a', 'quiesced'));
expect(readIntentMarker(dir)?.state).toBe('quiesced');
expect(statSync(intentMarkerPath(dir)).mode & 0o777).toBe(0o600);
writeFileSync(intentMarkerPath(dir), '{not json');
expect(() => readIntentMarker(dir)).toThrow();
expect(inspectGraduationPath(dir).state).toBe('interrupted');
});
test('the tombstone is created exclusively, 0600, and only its own run removes it', () => {
const path = join(root, 'tomb.pglite');
writeTombstone(path, tombstone('run-t', `${path}.graduated-run-t`));
expect(statSync(path).mode & 0o777).toBe(0o600);
expect(() => writeTombstone(path, tombstone('run-u', 'x'))).toThrow(TombstonePathOccupiedError);
expect(readTombstone(path)?.runId).toBe('run-t');
expect(() => removeTombstone(path, 'run-u')).toThrow(/belongs to graduation run run-t/);
removeTombstone(path, 'run-t');
expect(existsSync(path)).toBe(false);
mkdirSync(path);
expect(() => writeTombstone(path, tombstone('run-t', 'x'))).toThrow(TombstonePathOccupiedError);
expect(readTombstone(path)).toBeNull();
});
});
describe('inspectGraduationPath and marker liveness', () => {
test('none, in_progress, interrupted, graduated and split_brain', () => {
const dir = join(root, 'inspect.pglite');
mkdirSync(dir);
expect(inspectGraduationPath(dir).state).toBe('none');
const unregister = registerGraduationRunInProcess('run-live');
writeIntentMarker(dir, marker('run-live', 'copying'));
expect(inspectGraduationPath(dir)).toMatchObject({ state: 'in_progress', liveness: 'alive' });
unregister();
expect(inspectGraduationPath(dir)).toMatchObject({ state: 'interrupted', liveness: 'dead' });
writeIntentMarker(dir, marker('run-dead', 'copying', { pid: deadPid(), processStart: null }));
expect(inspectGraduationPath(dir)).toMatchObject({ state: 'interrupted', liveness: 'dead' });
writeIntentMarker(dir, marker('run-reused', 'copying', { pid: process.ppid, processStart: '1' }));
if (processStartTime(process.ppid)) expect(inspectGraduationPath(dir).liveness).toBe('dead');
writeIntentMarker(dir, marker('run-other-host', 'copying', { pid: process.ppid, bootId: 'another-boot' }));
expect(inspectGraduationPath(dir).liveness).toBe('unknown');
writeIntentMarker(dir, marker('run-split', 'tombstoned', { pid: deadPid() }));
mkdirSync(graduatedPath(dir, 'run-split'));
expect(inspectGraduationPath(dir)).toMatchObject({ state: 'split_brain', movedTo: graduatedPath(dir, 'run-split') });
writeIntentMarker(dir, marker('run-split', 'rolled_back'));
expect(inspectGraduationPath(dir).state).toBe('none');
const tombPath = join(root, 'inspect-tomb.pglite');
writeTombstone(tombPath, tombstone('run-g', `${tombPath}.graduated-run-g`));
expect(inspectGraduationPath(tombPath)).toMatchObject({ state: 'graduated', tombstone: { runId: 'run-g' } });
});
test('a serve hands off only to a live requester on another process', () => {
const dir = join(root, 'handoff.pglite');
mkdirSync(dir);
writeIntentMarker(dir, marker('run-self', 'planned'));
expect(graduationHandOffRequested(dir)).toBeNull();
writeIntentMarker(dir, marker('run-dead', 'planned', { pid: deadPid(), processStart: null }));
expect(graduationHandOffRequested(dir)).toBeNull();
writeIntentMarker(dir, marker('run-parent', 'planned', { pid: process.ppid, processStart: processStartTime(process.ppid) }));
expect(graduationHandOffRequested(dir)?.runId).toBe('run-parent');
});
});
describe('lock-retaining close and held-lock move-aside', () => {
test('the kernel lock never gaps from close through rename and tombstone; the tombstone then refuses every opener', async () => {
const dataDir = join(root, 'custody.pglite');
const engine = await openPglite(dataDir);
let lock: LockHandle | null = null;
const attempts: string[] = [];
let polling = true;
const poller = (async () => {
while (polling) {
try { const stolen = await acquireLock(dataDir, { timeoutMs: 0 }); attempts.push('acquired'); await releaseLock(stolen); }
catch (error) { attempts.push(error instanceof PgliteBusyError ? 'busy' : codeOf(error) ?? (error as Error).message); }
await Bun.sleep(5);
}
})();
try {
await engine.executeRaw('CREATE TABLE kept (id int primary key)');
await engine.executeRaw('INSERT INTO kept VALUES (1)');
lock = await engine.closeRetainingLock();
await engine.disconnect();
expect(lock.acquired).toBe(true);
await Bun.sleep(30);
const movedTo = await moveAsideHeld(dataDir, lock, 'run-c');
expect(lock.lockDir).toBe(join(movedTo, '.gbrain-lock'));
expect(existsSync(dataDir)).toBe(false);
await Bun.sleep(30);
writeTombstone(dataDir, tombstone('run-c', movedTo));
await Bun.sleep(30);
} finally {
polling = false;
await poller;
if (lock) await releaseLock(lock);
}
expect(attempts.length).toBeGreaterThan(3);
expect(attempts).not.toContain('acquired');
expect(lstatSync(dataDir).isFile()).toBe(true);
await expect(acquireLock(dataDir, { timeoutMs: 0 })).rejects.toThrow(/EEXIST|ENOTDIR|not a directory|file already exists/i);
expect(lstatSync(dataDir).isFile()).toBe(true);
expect(codeOf(await openPglite(dataDir).catch(e => e))).toBe('engine_graduated');
const retained = await openPglite(graduatedPath(dataDir, 'run-c'));
try { expect(await retained.executeRaw('SELECT id FROM kept')).toEqual([{ id: 1 }]); } finally { await retained.disconnect(); }
}, 30_000);
test('a held lock moves only its own datastore, and a held-lock open adopts it without a gap', async () => {
const dataDir = join(root, 'adopt.pglite');
const other = join(root, 'adopt-other.pglite');
mkdirSync(other);
const engine = await openPglite(dataDir);
const lock = await engine.closeRetainingLock();
await engine.disconnect();
expect(() => moveHeldPglite(other, `${other}.moved`, lock)).toThrow(/does not belong/);
const moved = `${dataDir}.graduated-run-adopt`;
moveHeldPglite(dataDir, moved, lock);
moveHeldPglite(moved, dataDir, lock);
await expect(acquireLock(dataDir, { timeoutMs: 0 })).rejects.toBeInstanceOf(PgliteBusyError);
const adopted = newPgliteEngine();
await adopted.connectWithHeldLock({ engine: 'pglite', database_path: dataDir }, lock);
try { expect(await adopted.executeRaw('SELECT 1 AS one')).toEqual([{ one: 1 }]); } finally { await adopted.disconnect(); }
const after = await openPglite(dataDir);
await after.disconnect();
}, 30_000);
test('a kernel-only lock on a tombstoned path creates nothing and excludes openers', async () => {
const path = join(root, 'kernel-only.pglite');
writeTombstone(path, tombstone('run-k', `${path}.graduated-run-k`));
const lock = await acquireKernelLockOnly(path, { lockDir: join(`${path}.graduated-run-k`, '.gbrain-lock'), timeoutMs: 0 });
try {
await expect(acquireKernelLockOnly(path, { lockDir: 'x', timeoutMs: 0 })).rejects.toBeInstanceOf(PgliteBusyError);
expect(lstatSync(path).isFile()).toBe(true);
} finally { await releaseLock(lock); }
}, 30_000);
});
describe('PGLite open-path refusals', () => {
test('a dead run before cutover lets the source open; past cutover it is interrupted and the lock is released', async () => {
const dataDir = join(root, 'open.pglite');
const first = await openPglite(dataDir);
await first.disconnect();
writeIntentMarker(dataDir, marker('run-o', 'verified', { pid: deadPid(), processStart: null }));
const opened = await openPglite(dataDir);
await opened.disconnect();
writeIntentMarker(dataDir, marker('run-o', 'cutover', { pid: deadPid(), processStart: null }));
const refused = await openPglite(dataDir).catch(e => e);
expect(codeOf(refused)).toBe('graduation_interrupted');
expect(refused.fix?.argv).toEqual(['gbrain', 'migrate', '--resume']);
const lock = await acquireLock(dataDir, { timeoutMs: 0 });
await releaseLock(lock);
}, 30_000);
test('a live run on another process refuses before the lock, and the refusal names the status command', () => {
const dataDir = join(root, 'live.pglite');
mkdirSync(dataDir);
writeIntentMarker(dataDir, marker('run-p', 'copying', { pid: process.ppid, processStart: processStartTime(process.ppid) }));
let error: unknown;
try { assertPgliteGraduationOpenable(dataDir, 'pre_lock'); } catch (e) { error = e; }
expect(codeOf(error)).toBe('graduation_in_progress');
expect((error as { fix?: { argv?: string[]; actor?: string } }).fix).toMatchObject({ argv: ['gbrain', 'migrate', '--status', '--json'], actor: 'provider' });
expect(readFileSync(intentMarkerPath(dataDir), 'utf8')).toContain('run-p');
});
});