Garry's Opinionated OpenClaw/Hermes Agent Brain
TypeScript
import { describe, expect, test, spyOn } from 'bun:test';
import { existsSync, mkdtempSync, mkdirSync, readFileSync, readdirSync, rmSync, writeFileSync } from 'node:fs';
import * as fs from 'node:fs';
import { tmpdir } from 'node:os';
import { join } from 'node:path';
import { pathToFileURL } from 'node:url';
import { spawn, type ChildProcess } from 'node:child_process';
import { ChildWorkerSupervisor } from '../src/core/minions/child-worker-supervisor.ts';
import { OwnerProcessingState, readAutopilotProcessingStatus, readOwnerProcessingStatus, writeWorkerProcessingStatus } from '../src/core/minions/processing-state.ts';
import { MinionSupervisor } from '../src/core/minions/supervisor.ts';
import { guardAutopilotEngine } from '../src/commands/autopilot.ts';
import { autopilotLockPath } from '../src/core/autopilot-paths.ts';
import { killProcessGroup } from '../src/core/minions/job-isolation.ts';
import type { BrainEngine } from '../src/core/engine.ts';
import { withEnv } from './helpers/with-env.ts';
async function isolated(fn: (root: string) => Promise<void>): Promise<void> {
const root = mkdtempSync(join(tmpdir(), 'gbrain-blocked-owner-'));
try { await withEnv({ GBRAIN_HOME: root, HOME: root }, () => fn(root)); }
finally { rmSync(root, { recursive: true, force: true }); }
}
describe('configuration-blocked worker ownership', () => {
test('real spawn keeps the owner handoff through hostile-cwd quarantine without restoring untrusted values', async () => isolated(async (root) => {
const quarantineModule = pathToFileURL(join(import.meta.dir, '../src/core/env-trust.ts')).href;
const preflightModule = pathToFileURL(join(import.meta.dir, '../src/core/cli-preflight.ts')).href;
const statusModule = pathToFileURL(join(import.meta.dir, '../src/core/minions/processing-state.ts')).href;
const supervisorModule = pathToFileURL(join(import.meta.dir, '../src/core/minions/child-worker-supervisor.ts')).href;
const poisonedPath = join(root, 'untrusted-status');
writeFileSync(join(root, '.env'), `GBRAIN_WORKER_STATUS_PATH=${poisonedPath}\nGBRAIN_WORKER_STATUS_TOKEN=untrusted\n`);
const worker = join(root, 'worker.ts');
writeFileSync(worker, `
import {writeFileSync} from 'node:fs';
import {runCliPreflight,CWD_ENV_QUARANTINED_MARKER} from ${JSON.stringify(preflightModule)};
import {writeWorkerProcessingStatus} from ${JSON.stringify(statusModule)};
const neutral=JSON.parse(process.env[CWD_ENV_QUARANTINED_MARKER]!).neutral;
await runCliPreflight();
const published=writeWorkerProcessingStatus({state:'ready'});
writeFileSync('worker-observation.json',JSON.stringify({cwd:process.cwd(),neutral,marker:process.env[CWD_ENV_QUARANTINED_MARKER]??null,published,supervised:process.env.GBRAIN_SUPERVISED,path:process.env.GBRAIN_WORKER_STATUS_PATH}));
process.exit(1);
`);
const parent = join(root, 'parent.ts');
writeFileSync(parent, `
import {quarantineCwdDotenv} from ${JSON.stringify(quarantineModule)};
import {OwnerProcessingState} from ${JSON.stringify(statusModule)};
import {ChildWorkerSupervisor} from ${JSON.stringify(supervisorModule)};
quarantineCwdDotenv();
const state=new OwnerProcessingState('supervisor','fixture');
await new ChildWorkerSupervisor({cliPath:process.execPath,args:['./worker.ts'],env:{...process.env,GBRAIN_SUPERVISED:undefined},processingState:state,maxCrashes:1,hardStopMaxCrashes:1,isStopping:()=>false,onMaxCrashesExceeded:()=>{},onEvent:()=>{}}).run();
state.close();
`);
const child = Bun.spawn([process.execPath, '--no-env-file', parent], {
cwd: root, env: { ...process.env, GBRAIN_WORKER_STATUS_PATH: poisonedPath, GBRAIN_WORKER_STATUS_TOKEN: 'untrusted' }, stdout: 'pipe', stderr: 'pipe',
});
const stdout = new Response(child.stdout).text();
const stderr = new Response(child.stderr).text();
const timer = setTimeout(() => child.kill('SIGKILL'), 10_000);
try {
expect(await child.exited, `${await stdout}\n${await stderr}`).toBe(0);
const observation = JSON.parse(readFileSync(join(root, 'worker-observation.json'), 'utf8'));
expect(observation.published).toBe(true);
expect(observation.supervised).toBeUndefined();
expect(observation.marker).toBeNull();
expect(observation.cwd).toBe(root);
expect(observation.path).not.toBe(poisonedPath);
expect(existsSync(poisonedPath)).toBe(false);
expect(existsSync(observation.neutral)).toBe(false);
} finally { clearTimeout(timer); }
}));
test('unpublished readiness is bounded independently of health, retries softly, and stops at hard equality', async () => isolated(async (root) => {
const state = new OwnerProcessingState('supervisor', 'fixture');
const worker = join(root, 'unready.ts');
writeFileSync(worker, `import {appendFileSync} from 'node:fs';
appendFileSync(${JSON.stringify(join(root, 'worker-pids'))},process.pid+'\\n');
process.on('SIGTERM',()=>{}); setInterval(()=>{},1000);`);
let timeouts = 0;
let hard = 0;
let degraded = 0;
const retrySnapshots: any[] = [];
const sup = new ChildWorkerSupervisor({
cliPath: process.execPath, args: [worker], processingState: state,
maxCrashes: 1, hardStopMaxCrashes: 2, _startupReadyTimeoutMs: 500, _startupKillGraceMs: 20, _backoffFloorMs: 50,
isStopping: () => false, onMaxCrashesExceeded: (count, ceiling) => { expect(count).toBe(ceiling); hard++; },
onEvent: e => {
if (e.kind === 'worker_startup_timeout') { timeouts++; expect(state.snapshot.processing_stage).toBe('startup_timeout'); }
if (e.kind === 'health_warn' && e.reason === 'crash_budget_degraded') degraded++;
if (e.kind === 'backoff') queueMicrotask(() => retrySnapshots.push(state.snapshot));
},
});
await sup.run();
expect({ timeouts, hard, degraded }).toEqual({ timeouts: 2, hard: 1, degraded: 1 });
expect(sup.crashCount).toBe(2);
expect(retrySnapshots).toHaveLength(1);
expect(retrySnapshots[0].processing_stage).toBe('retry_backoff');
expect(Number.isFinite(Date.parse(retrySnapshots[0].retry_at))).toBe(true);
expect(state.snapshot.processing_stage).toBe('stopped');
expect(state.snapshot.retry_at).toBeNull();
const pids = readFileSync(join(root, 'worker-pids'), 'utf8').trim().split('\n').map(Number);
expect(pids).toHaveLength(2);
for (const pid of pids) expect(() => process.kill(pid, 0)).toThrow();
state.close();
}), 10_000);
test('a confirmed ready worker survives the startup deadline and blocked state has no retry timestamp', async () => isolated(async (root) => {
const state = new OwnerProcessingState('autopilot', 'default');
const worker = join(root, 'ready.ts');
const statusModule = pathToFileURL(join(import.meta.dir, '../src/core/minions/processing-state.ts')).href;
writeFileSync(worker, `import {writeWorkerProcessingStatus} from ${JSON.stringify(statusModule)};
writeWorkerProcessingStatus({state:'ready'});
setTimeout(()=>{writeWorkerProcessingStatus({state:'configuration_blocked',reason_code:'child_protocol_incompatible'});process.exit(16);},800);`);
let timeouts = 0;
const sup = new ChildWorkerSupervisor({
cliPath: process.execPath, args: [worker], processingState: state,
maxCrashes: 1, _startupReadyTimeoutMs: 500, _startupKillGraceMs: 20,
isStopping: () => false, onMaxCrashesExceeded: () => {},
onEvent: e => { if (e.kind === 'worker_startup_timeout') timeouts++; },
});
await sup.run();
expect(timeouts).toBe(0);
expect(state.snapshot.processing_stage).toBe('configuration_blocked');
expect(state.snapshot.retry_at).toBeNull();
expect(sup.crashCount).toBe(0);
state.close();
}));
test('autopilot status binds processing state to the live lock holder PID', async () => isolated(async () => {
const state = new OwnerProcessingState('autopilot', 'default');
const other = spawn('sleep', ['30'], { stdio: 'ignore' });
try {
const otherPid = other.pid!;
const autopilot = { readProcessCommand: () => 'gbrain autopilot' };
expect(readAutopilotProcessingStatus()).toBeNull();
writeFileSync(autopilotLockPath(), String(process.pid));
expect(readAutopilotProcessingStatus()?.owner_pid).toBe(process.pid);
writeFileSync(autopilotLockPath(), String(otherPid));
expect(readAutopilotProcessingStatus(autopilot)).toBeNull();
const runtime = join(autopilotLockPath(), '..', 'runtime');
const ownerFile = join(runtime, readdirSync(runtime).find(f => f.startsWith('autopilot-') && f.endsWith('.json'))!);
writeFileSync(ownerFile, JSON.stringify({ ...JSON.parse(readFileSync(ownerFile, 'utf8')), owner_pid: otherPid, processing_state: 'ready', processing_stage: 'ready' }));
expect(readAutopilotProcessingStatus({ readProcessCommand: () => 'sleep 30' })).toBeNull();
expect(readAutopilotProcessingStatus({ isPidAlive: () => false })).toBeNull();
expect(readAutopilotProcessingStatus(autopilot)?.processing_ready).toBe(true);
} finally {
other.kill('SIGKILL');
state.close();
}
}));
test('startup timeout also reaps an unready direct worker without tini', async () => withEnv({ PATH: '' }, () => isolated(async () => {
const state = new OwnerProcessingState('supervisor', 'fixture');
let pid = 0;
let timeouts = 0;
const sup = new ChildWorkerSupervisor({
cliPath: process.execPath, args: ['-e', "process.on('SIGTERM',()=>{});setInterval(()=>{},1000)"], processingState: state,
maxCrashes: 1, hardStopMaxCrashes: 1, _startupReadyTimeoutMs: 300, _startupKillGraceMs: 20,
isStopping: () => false, onMaxCrashesExceeded: () => {},
onEvent: e => {
if (e.kind === 'worker_spawned') pid = e.pid;
if (e.kind === 'worker_startup_timeout') timeouts++;
},
});
expect(sup.isTiniDetected).toBe(false);
await sup.run();
expect(timeouts).toBe(1);
expect(sup.crashCount).toBe(1);
expect(pid).toBeGreaterThan(0);
expect(() => process.kill(pid, 0)).toThrow();
state.close();
})));
test('managed spawns leave GBRAIN_SUPERVISED to the owner and always hand off the status channel', async () => withEnv({ GBRAIN_SUPERVISED: '1' }, () => isolated(async (root) => {
const statusModule = pathToFileURL(join(import.meta.dir, '../src/core/minions/processing-state.ts')).href;
const observations = join(root, 'observations');
const worker = join(root, 'env-observer.ts');
writeFileSync(worker, `import {appendFileSync} from 'node:fs';
import {writeWorkerProcessingStatus} from ${JSON.stringify(statusModule)};
appendFileSync(${JSON.stringify(observations)},JSON.stringify({supervised:process.env.GBRAIN_SUPERVISED??null,path:Boolean(process.env.GBRAIN_WORKER_STATUS_PATH),token:Boolean(process.env.GBRAIN_WORKER_STATUS_TOKEN),published:writeWorkerProcessingStatus({state:'ready'})})+'\\n');
process.exit(1);`);
const observe = async (kind: 'autopilot' | 'supervisor', env: NodeJS.ProcessEnv) => {
const state = new OwnerProcessingState(kind, 'fixture');
await new ChildWorkerSupervisor({
cliPath: process.execPath, args: [worker], env, processingState: state, maxCrashes: 1, hardStopMaxCrashes: 1,
isStopping: () => false, onMaxCrashesExceeded: () => {}, onEvent: () => {},
}).run();
state.close();
};
await observe('autopilot', { ...process.env, GBRAIN_SUPERVISED: undefined });
await observe('supervisor', { ...process.env, GBRAIN_SUPERVISED: '1' });
const [autopilot, supervisor] = readFileSync(observations, 'utf8').trim().split('\n').map(line => JSON.parse(line));
expect(autopilot).toEqual({ supervised: null, path: true, token: true, published: true });
expect(supervisor).toEqual({ supervised: '1', path: true, token: true, published: true });
})));
test.skipIf(process.platform === 'win32')('without tini, killing the owner process group also kills its managed worker', async () => isolated(async (root) => {
const statusModule = pathToFileURL(join(import.meta.dir, '../src/core/minions/processing-state.ts')).href;
const supervisorModule = pathToFileURL(join(import.meta.dir, '../src/core/minions/child-worker-supervisor.ts')).href;
const workerPidFile = join(root, 'worker.pid');
const worker = join(root, 'worker.ts');
writeFileSync(worker, `import {writeFileSync} from 'node:fs';
writeFileSync(${JSON.stringify(workerPidFile)},String(process.pid));
setInterval(()=>{},1000);`);
const ownerScript = join(root, 'owner.ts');
writeFileSync(ownerScript, `import {OwnerProcessingState} from ${JSON.stringify(statusModule)};
import {ChildWorkerSupervisor} from ${JSON.stringify(supervisorModule)};
const state=new OwnerProcessingState('autopilot','default');
const sup=new ChildWorkerSupervisor({cliPath:process.execPath,args:[${JSON.stringify(worker)}],env:{...process.env,GBRAIN_SUPERVISED:undefined},processingState:state,maxCrashes:5,isStopping:()=>false,onMaxCrashesExceeded:()=>{},onEvent:()=>{}});
if (sup.isTiniDetected) process.exit(3);
void sup.run();
setInterval(()=>{},1000);`);
const alive = (pid: number) => {
try { process.kill(pid, 0); } catch { return false; }
if (process.platform !== 'linux') return true;
try {
const stat = readFileSync(`/proc/${pid}/stat`, 'utf8');
return stat.slice(stat.lastIndexOf(')') + 2, stat.lastIndexOf(')') + 3) !== 'Z';
} catch { return false; }
};
const owner = spawn(process.execPath, [ownerScript], {
cwd: root, detached: true, stdio: 'ignore', env: { ...process.env, PATH: '', TMPDIR: root },
});
let workerPid = 0;
try {
const started = Date.now() + 10_000;
while (!existsSync(workerPidFile) && owner.exitCode === null && Date.now() < started) await Bun.sleep(20);
expect(owner.exitCode).toBeNull();
workerPid = Number(readFileSync(workerPidFile, 'utf8'));
expect(workerPid).toBeGreaterThan(0);
expect(killProcessGroup(owner.pid!, 'SIGKILL')).toBe(true);
const reaped = Date.now() + 5_000;
while (alive(workerPid) && Date.now() < reaped) await Bun.sleep(20);
expect(alive(workerPid)).toBe(false);
} finally {
killProcessGroup(owner.pid!, 'SIGKILL');
if (workerPid) {
killProcessGroup(workerPid, 'SIGKILL');
try { process.kill(workerPid, 'SIGKILL'); } catch {}
}
}
}), 20_000);
test('unwritable managed status fails publication before admission and follows ordinary bounded retry', async () => isolated(async (root) => {
const state = new OwnerProcessingState('autopilot', 'default');
const reporter = pathToFileURL(join(import.meta.dir, '../src/commands/jobs-readiness.ts')).href;
const worker = join(root, 'unwritable.ts');
writeFileSync(worker, `import {mkdirSync,appendFileSync,writeFileSync,unlinkSync} from 'node:fs';
import {reportWorkerReady} from ${JSON.stringify(reporter)};
unlinkSync(process.env.GBRAIN_WORKER_STATUS_PATH!);
mkdirSync(process.env.GBRAIN_WORKER_STATUS_PATH!);
try {reportWorkerReady();writeFileSync(${JSON.stringify(join(root, 'admitted'))},'unexpected');}
catch(error){appendFileSync(${JSON.stringify(join(root, 'publication-failures'))},String(error)+'\\n');process.exit(1);}`);
let spawns = 0;
let hardStops = 0;
let timeouts = 0;
const sup = new ChildWorkerSupervisor({
cliPath: process.execPath, args: [worker], processingState: state,
env: { ...process.env, GBRAIN_SUPERVISED: undefined }, maxCrashes: 1, hardStopMaxCrashes: 2,
_startupReadyTimeoutMs: 10_000, _backoffFloorMs: 1, isStopping: () => false,
onMaxCrashesExceeded: () => { hardStops++; },
onEvent: e => {
if (e.kind === 'worker_spawned') spawns++;
if (e.kind === 'worker_startup_timeout') timeouts++;
},
});
await sup.run();
expect({ spawns, hardStops, timeouts }).toEqual({ spawns: 2, hardStops: 1, timeouts: 0 });
expect(existsSync(join(root, 'admitted'))).toBe(false);
const failures = readFileSync(join(root, 'publication-failures'), 'utf8').trim().split('\n');
expect(failures).toHaveLength(2);
expect(failures.every(line => line.includes('no jobs were admitted'))).toBe(true);
expect(state.snapshot.processing_ready).toBe(false);
expect(state.snapshot.processing_stage).toBe('stopped');
state.close();
}));
test('authenticated startup phases and bounded retry timestamps survive offline status reads', async () => isolated(async () => {
const state = new OwnerProcessingState('autopilot', 'default');
const env = state.prepareChild();
for (const stage of ['database_readiness', 'child_readiness', 'worker_startup'] as const) {
writeWorkerProcessingStatus({ state: 'starting', stage }, env);
expect(state.snapshot.processing_stage).toBe(stage);
expect(readOwnerProcessingStatus('autopilot', 'default')?.processing_stage).toBe(stage);
}
state.workerExited();
state.waiting('retry_backoff', Date.now() + 1000);
const offline = readOwnerProcessingStatus('autopilot', 'default');
expect(offline?.processing_stage).toBe('retry_backoff');
expect(Date.parse(offline!.retry_at!)).toBeGreaterThan(Date.now());
state.close();
}));
test('publication rejects absent channels, retired generations, and another bound worker PID', async () => isolated(async (root) => {
const missing = { GBRAIN_WORKER_STATUS_PATH: join(root, 'missing', 'status.json'), GBRAIN_WORKER_STATUS_TOKEN: 'fixture-token' };
expect(writeWorkerProcessingStatus({ state: 'ready' }, missing)).toBe(false);
expect(existsSync(join(root, 'missing'))).toBe(false);
const state = new OwnerProcessingState('supervisor', 'fixture');
const first = state.prepareChild();
expect(writeWorkerProcessingStatus({ state: 'ready' }, { ...first, GBRAIN_WORKER_STATUS_TOKEN: 'wrong-token' })).toBe(false);
const status = JSON.parse(readFileSync(first.GBRAIN_WORKER_STATUS_PATH!, 'utf8'));
writeFileSync(first.GBRAIN_WORKER_STATUS_PATH!, JSON.stringify({ ...status, worker_pid: process.pid + 1 }));
expect(writeWorkerProcessingStatus({ state: 'ready' }, first)).toBe(false);
const current = state.prepareChild();
expect(writeWorkerProcessingStatus({ state: 'ready' }, first)).toBe(false);
expect(existsSync(first.GBRAIN_WORKER_STATUS_PATH!)).toBe(false);
expect(writeWorkerProcessingStatus({ state: 'starting', stage: 'worker_startup' }, current)).toBe(true);
state.waiting('startup_timeout');
expect(writeWorkerProcessingStatus({ state: 'ready' }, current)).toBe(false);
expect(existsSync(current.GBRAIN_WORKER_STATUS_PATH!)).toBe(false);
state.close();
}));
test('retirement after atomic publication but before its return fails the generation recheck', async () => isolated(async () => {
const state = new OwnerProcessingState('supervisor', 'fixture');
const previous = state.prepareChild();
const originalRename = fs.renameSync;
let intercepted = false;
const rename = spyOn(fs, 'renameSync').mockImplementation((from, to) => {
originalRename(from, to);
if (to === previous.GBRAIN_WORKER_STATUS_PATH && !intercepted) {
intercepted = true;
state.prepareChild();
}
});
try {
expect(writeWorkerProcessingStatus({ state: 'ready' }, previous)).toBe(false);
expect(intercepted).toBe(true);
expect(state.snapshot.processing_ready).toBe(false);
expect(existsSync(previous.GBRAIN_WORKER_STATUS_PATH!)).toBe(false);
} finally {
rename.mockRestore();
state.close();
}
}));
test('an old worker resumed by the timeout signal cannot publish ready or admit during the kill grace', async () => isolated(async (root) => {
const state = new OwnerProcessingState('supervisor', 'fixture');
const reporter = pathToFileURL(join(import.meta.dir, '../src/commands/jobs-readiness.ts')).href;
const worker = join(root, 'late-ready.ts');
writeFileSync(worker, `import {writeFileSync} from 'node:fs';
import {reportWorkerReady} from ${JSON.stringify(reporter)};
process.on('SIGTERM',()=>{
try {reportWorkerReady();writeFileSync(${JSON.stringify(join(root, 'late-admission'))},'unexpected');}
catch(error){writeFileSync(${JSON.stringify(join(root, 'late-refusal'))},String(error));}
process.exit(1);
});
setInterval(()=>{},1000);`);
let timeouts = 0;
const sup = new ChildWorkerSupervisor({
cliPath: process.execPath, args: [worker], processingState: state,
maxCrashes: 1, hardStopMaxCrashes: 1, _startupReadyTimeoutMs: 1000, _startupKillGraceMs: 1000,
isStopping: () => false, onMaxCrashesExceeded: () => {},
onEvent: event => { if (event.kind === 'worker_startup_timeout') timeouts++; },
});
await sup.run();
expect(timeouts).toBe(1);
expect(existsSync(join(root, 'late-admission'))).toBe(false);
expect(readFileSync(join(root, 'late-refusal'), 'utf8')).toContain('no jobs were admitted');
expect(state.snapshot.processing_ready).toBe(false);
state.close();
}));
for (const kind of ['supervisor', 'autopilot'] as const) {
test.skipIf(process.platform === 'win32')(`${kind} survives Restart=always, retains singleton and recovers only after explicit restart`, async () => isolated(async (root) => {
mkdirSync(join(root, '.gbrain'), { recursive: true });
writeFileSync(join(root, '.gbrain', 'config.json'), JSON.stringify({ engine: 'postgres' }));
const fixture = join(root, 'owner.ts');
const worker = join(root, 'cli.ts');
writeFileSync(join(root, 'gbrain'), `#!${process.execPath}\nprocess.exit(99);\n`, { mode: 0o755 });
const statusModule = pathToFileURL(join(import.meta.dir, '../src/core/minions/processing-state.ts')).href;
const ownerModule = pathToFileURL(join(import.meta.dir, kind === 'supervisor' ? '../src/core/minions/supervisor.ts' : '../src/commands/autopilot.ts')).href;
const pidFile = join(root, 'supervisor.pid');
const key = kind === 'supervisor' ? pidFile : 'default';
writeFileSync(worker, `#!${process.execPath}
import {existsSync,appendFileSync} from 'node:fs';
import {writeWorkerProcessingStatus} from ${JSON.stringify(statusModule)};
appendFileSync(${JSON.stringify(join(root, 'spawns'))}, 'spawn\\n');
if (!existsSync(${JSON.stringify(join(root, 'repaired'))})) {
writeWorkerProcessingStatus({state:'configuration_blocked',reason_code:'postgres_cancellation_unavailable'});
process.exit(16);
}
writeWorkerProcessingStatus({state:'ready',worker_identity:{executable:'/fixture/gbrain',version:'1.0.0'}});
appendFileSync(${JSON.stringify(join(root, 'handlers'))}, 'completed\\n');
setInterval(()=>{}, 1000);
process.on('SIGTERM',()=>process.exit(0));
`, { mode: 0o755 });
writeFileSync(fixture, `
import {${kind === 'supervisor' ? 'MinionSupervisor' : 'runAutopilot'}} from ${JSON.stringify(ownerModule)};
const engine = {
kind:'postgres', sql:async()=>[{id:'fixture',fence:'1'}],
getConfig:async()=>null, executeRaw:async()=>[], executeRawDirect:async()=>[{id:'fixture'}],
getHealth:async()=>({brain_score:100}), disconnect:async()=>{}, reconnect:async()=>{}
};
${kind === 'supervisor'
? `await new MinionSupervisor(engine as any,{cliPath:process.execPath,cliArgsPrefix:[${JSON.stringify(worker)}],pidFile:${JSON.stringify(pidFile)},healthInterval:10,_backoffFloorMs:1}).start();`
: `process.argv[1]=${JSON.stringify(worker)}; await runAutopilot(engine as any,['--repo',${JSON.stringify(root)},'--interval','1']);`}
`);
let owner: ChildProcess;
let logs = '';
let outerRestarts = 0;
const launch = () => {
owner = spawn(process.execPath, [fixture], {
env: { ...process.env, HOME: root, GBRAIN_HOME: root, PATH: `${root}:${process.env.PATH}`, OPENAI_API_KEY: '', ANTHROPIC_API_KEY: '', GBRAIN_SKIP_STARTUP_HOOKS: '1' },
stdio: ['ignore', 'pipe', 'pipe'],
});
owner.stdout?.on('data', data => { logs += data; });
owner.stderr?.on('data', data => { logs += data; });
};
const until = async (predicate: () => boolean) => {
const deadline = Date.now() + 15_000;
while (!predicate() && Date.now() < deadline) await new Promise(r => setTimeout(r, 20));
if (!predicate()) throw new Error(`Owner fixture timed out: ${logs}`);
};
const stop = async () => {
owner.kill('SIGTERM');
await until(() => owner.exitCode !== null || owner.signalCode !== null);
};
launch();
try {
await until(() => readOwnerProcessingStatus(kind, key)?.processing_state === 'configuration_blocked');
const firstPid = owner!.pid;
for (let tick = 0; tick < 8; tick++) {
await new Promise(r => setTimeout(r, 30));
if (owner!.exitCode !== null || owner!.signalCode !== null) { outerRestarts++; launch(); }
}
expect(outerRestarts).toBe(0);
expect(owner!.pid).toBe(firstPid);
expect(readFileSync(join(root, 'spawns'), 'utf8').trim().split('\n')).toHaveLength(1);
expect(readdirSync(root)).not.toContain('handlers');
const lock = kind === 'supervisor' ? pidFile : join(root, '.gbrain', 'autopilot.lock');
expect(readFileSync(lock, 'utf8').trim().split('\n')[0]).toBe(String(firstPid));
writeFileSync(join(root, 'repaired'), 'yes');
await new Promise(r => setTimeout(r, 150));
expect(readOwnerProcessingStatus(kind, key)?.processing_state).toBe('configuration_blocked');
await stop();
launch();
await until(() => readOwnerProcessingStatus(kind, key)?.processing_ready === true);
expect(readFileSync(join(root, 'handlers'), 'utf8').trim()).toBe('completed');
await stop();
expect(owner!.exitCode).toBe(0);
} finally {
if (owner!.exitCode === null && owner!.signalCode === null) {
owner!.kill('SIGTERM');
await until(() => owner!.exitCode !== null || owner!.signalCode !== null).catch(() => { owner!.kill('SIGKILL'); });
}
}
}), 40_000);
}
test('exit 16 blocks once before hard threshold, recovery and backoff', async () => isolated(async (root) => {
const script = join(root, 'worker.mjs');
writeFileSync(script, 'process.exit(16)');
const state = new OwnerProcessingState('supervisor', 'fixture');
let spawns = 0;
let recoveries = 0;
let blocked = 0;
let hardStops = 0;
let backoffs = 0;
const supervisor = new ChildWorkerSupervisor({
cliPath: process.execPath, args: [script], processingState: state,
maxCrashes: 1, hardStopMaxCrashes: 1, _backoffFloorMs: 0,
isStopping: () => false, beforeSpawn: async () => { recoveries++; },
onConfigurationBlocked: () => { blocked++; },
onMaxCrashesExceeded: () => { hardStops++; },
onEvent: (e) => { if (e.kind === 'worker_spawned') spawns++; if (e.kind === 'backoff') backoffs++; },
});
await supervisor.run();
await supervisor.run();
await supervisor.restartCurrentChild(0);
expect({ spawns, recoveries, blocked, hardStops, backoffs }).toEqual({ spawns: 1, recoveries: 1, blocked: 1, hardStops: 0, backoffs: 0 });
expect(supervisor.crashCount).toBe(0);
expect(readOwnerProcessingStatus('supervisor', 'fixture', process.pid)).toMatchObject({ processing_ready: false, processing_state: 'configuration_blocked', reason_code: null });
state.close();
}));
test('runtime handoff latches during drain and wins over a later clean exit', async () => isolated(async (root) => {
const script = join(root, 'worker.mjs');
const statusModule = pathToFileURL(join(import.meta.dir, '../src/core/minions/processing-state.ts')).href;
writeFileSync(script, `import {writeWorkerProcessingStatus} from ${JSON.stringify(statusModule)};
writeWorkerProcessingStatus({state:'configuration_blocked',reason_code:'postgres_cancellation_unavailable'});
setTimeout(()=>process.exit(0), 350);`);
const state = new OwnerProcessingState('autopilot', 'default');
let observedAlive = false;
let spawns = 0;
const supervisor = new ChildWorkerSupervisor({
cliPath: process.execPath, args: [script], processingState: state, maxCrashes: 1,
isStopping: () => false, onMaxCrashesExceeded: () => {},
onConfigurationBlocked: () => { observedAlive = supervisor.childAlive; },
onEvent: (event) => { if (event.kind === 'worker_spawned') spawns++; },
});
await supervisor.run();
expect(observedAlive).toBe(true);
expect(spawns).toBe(1);
expect(state.snapshot.reason_code).toBe('postgres_cancellation_unavailable');
state.close();
}));
test('stale child token cannot make a new runtime ready or blocked; restart resets observation', async () => isolated(async () => {
const state = new OwnerProcessingState('autopilot', 'default');
const oldEnv = state.prepareChild();
const freshEnv = state.prepareChild();
expect(writeWorkerProcessingStatus({ state: 'configuration_blocked', reason_code: 'child_protocol_incompatible' }, oldEnv)).toBe(false);
expect(state.snapshot.processing_state).toBe('starting');
writeWorkerProcessingStatus({ state: 'ready', worker_identity: { executable: '/fixture/gbrain', version: '1.2.3' }, child_identity: { executable: '/fixture/child', version: '1.2.2', protocol_version: 1 } }, freshEnv);
expect(state.snapshot.processing_ready).toBe(true);
writeWorkerProcessingStatus({ state: 'configuration_blocked', reason_code: 'child_protocol_incompatible' }, freshEnv);
expect(state.blocked).toBe(true);
expect(state.snapshot.child_identity).toEqual({ executable: '/fixture/child', version: '1.2.2', protocol_version: 1 });
writeWorkerProcessingStatus({ state: 'ready' }, freshEnv);
expect(state.blocked).toBe(true);
const restarted = new OwnerProcessingState('autopilot', 'default');
expect(restarted.snapshot.processing_state).toBe('starting');
state.close();
expect(readOwnerProcessingStatus('autopilot', 'default')?.runtime_id).toBe(restarted.snapshot.runtime_id);
expect(readOwnerProcessingStatus('autopilot', 'default', process.pid + 1)).toBeNull();
restarted.close();
}));
test('block while beforeSpawn is suspended prevents spawn on its continuation', async () => isolated(async () => {
const state = new OwnerProcessingState('supervisor', 'fixture');
let resume!: () => void;
let spawns = 0;
const supervisor = new ChildWorkerSupervisor({
cliPath: process.execPath, args: ['-e', 'process.exit(0)'], processingState: state,
maxCrashes: 1, isStopping: () => false,
beforeSpawn: () => new Promise<void>(r => { resume = r; }),
onMaxCrashesExceeded: () => {}, onEvent: (e) => { if (e.kind === 'worker_spawned') spawns++; },
});
const running = supervisor.run();
state.block('child_executable_invalid');
resume();
await running;
expect(spawns).toBe(0);
state.close();
}));
test('configuration block cancels an already pending watchdog backoff', async () => isolated(async () => {
const state = new OwnerProcessingState('supervisor', 'fixture');
let spawns = 0;
let blockTimer: ReturnType<typeof setTimeout> | undefined;
const supervisor = new ChildWorkerSupervisor({
cliPath: process.execPath, args: ['-e', 'process.exit(12)'], processingState: state,
maxCrashes: 1, watchdogBackoffMs: 60_000, isStopping: () => false,
onMaxCrashesExceeded: () => {},
onEvent: (e) => {
if (e.kind === 'worker_spawned') spawns++;
if (e.kind === 'backoff') blockTimer = setTimeout(() => state.block('child_protocol_incompatible'), 10);
},
});
try {
await supervisor.run();
expect(spawns).toBe(1);
expect(supervisor.inBackoff).toBe(false);
expect(supervisor.configurationBlocked).toBe(true);
} finally {
if (blockTimer) clearTimeout(blockTimer);
state.close();
}
}));
test('autopilot blocks dispatch continuations and reconnect but still permits engine shutdown', async () => isolated(async () => {
const state = new OwnerProcessingState('autopilot', 'default');
let resume!: () => void;
let writes = 0;
let disconnects = 0;
const engine = guardAutopilotEngine({
getConfig: () => new Promise<void>(r => { resume = r; }),
executeRaw: async () => { writes++; return []; },
reconnect: async () => { writes++; },
disconnect: async () => { disconnects++; },
} as unknown as BrainEngine, state);
const dispatch = (async () => { await engine.getConfig('version'); await engine.executeRaw('fixture'); })();
state.block('postgres_cancellation_unavailable');
resume();
await expect(dispatch).rejects.toThrow('configuration-blocked');
expect(() => engine.reconnect()).toThrow('configuration-blocked');
await engine.disconnect();
expect({ writes, disconnects }).toEqual({ writes: 0, disconnects: 1 });
state.close();
}));
test('standalone suppresses an in-flight health failure and failed lock refresh while blocked', async () => isolated(async (root) => {
const state = new OwnerProcessingState('supervisor', 'fixture');
let rejectHealth!: (error: Error) => void;
let reconnects = 0;
const engine = {
executeRaw: () => new Promise((_r, reject) => { rejectHealth = reject; }),
reconnect: async () => { reconnects++; },
} as unknown as BrainEngine;
const supervisor = new MinionSupervisor(engine, { cliPath: process.execPath, pidFile: join(root, 'owner.pid') });
Object.assign(supervisor, { processingState: state, consecutiveHealthFailures: 2 });
writeWorkerProcessingStatus({ state: 'ready' }, state.prepareChild());
const health = supervisor._healthCheckOnceForTests();
state.block('postgres_cancellation_unavailable');
rejectHealth(new Error('temporary outage'));
await health;
supervisor._setDbLockForTests({ refresh: async () => false } as any);
await supervisor._refreshDbLockForTests();
supervisor._setDbLockForTests({ refresh: async () => { throw new Error('offline'); } } as any);
for (let i = 0; i < 4; i++) await supervisor._refreshDbLockForTests();
expect(reconnects).toBe(0);
expect(supervisor.processingStatus?.processing_state).toBe('configuration_blocked');
state.close();
}));
test('configuration reported during maintenance shutdown keeps the owner and singleton alive', async () => isolated(async (root) => {
const state = new OwnerProcessingState('supervisor', 'fixture');
let released = 0;
const supervisor = new MinionSupervisor({} as BrainEngine, { cliPath: process.execPath, pidFile: join(root, 'owner.pid') });
Object.assign(supervisor, {
processingState: state,
childSupervisor: {
killChild: () => {}, childAlive: false,
awaitChildExit: async () => { state.block('postgres_cancellation_unavailable'); },
},
});
supervisor._setDbLockForTests({ release: async () => { released++; } } as any);
const exit = spyOn(process, 'exit').mockImplementation(() => { throw new Error('unexpected owner exit'); });
try {
await Reflect.get(supervisor, 'shutdown').call(supervisor, 'supervisor_lock_lost', 4);
expect(exit).not.toHaveBeenCalled();
expect(released).toBe(0);
expect(Reflect.get(supervisor, 'stopping')).toBe(false);
} finally {
exit.mockRestore();
state.close();
}
}));
});