Garry's Opinionated OpenClaw/Hermes Agent Brain
TypeScript
import { afterAll, beforeAll, beforeEach, describe, expect, test } from 'bun:test';
import { PGLiteEngine } from '../src/core/pglite-engine.ts';
import { MinionQueue } from '../src/core/minions/queue.ts';
import { MinionWorker, type UnhealthyReason } from '../src/core/minions/worker.ts';
import { LocalConfigurationError } from '../src/core/minions/configuration-error.ts';
import type { BrainEngine } from '../src/core/engine.ts';
import { ChildWorkerShutdownError } from '../src/core/minions/child-job-runner.ts';
import { assertWorkerDbReadiness } from '../src/core/minions/db-probe.ts';
let engine: PGLiteEngine;
let queue: MinionQueue;
beforeAll(async () => {
engine = new PGLiteEngine();
await engine.connect({});
await engine.initSchema();
queue = new MinionQueue(engine);
});
afterAll(async () => {
await engine.disconnect();
});
beforeEach(async () => {
await engine.executeRaw('DELETE FROM minion_jobs');
});
function deferred<T>() {
let resolve!: (value: T) => void;
const promise = new Promise<T>(r => { resolve = r; });
return { promise, resolve };
}
function makeWorker() {
return new MinionWorker(engine, { pollInterval: 5, healthCheckInterval: 0, stalledInterval: 60_000 });
}
function fault() {
return new LocalConfigurationError('postgres_cancellation_unavailable', 'fixture cancellation unavailable');
}
describe('configuration release ownership and accounting', () => {
test('confirmed release preserves attempts and stalls and clears ownership', async () => {
const job = await queue.add('fixture', {});
await queue.claim('fixture-token', 30_000, 'default', ['fixture']);
await engine.executeRaw('UPDATE minion_jobs SET stalled_counter = 2 WHERE id = $1', [job.id]);
const before = await queue.getJob(job.id);
expect(await queue.releaseConfigurationJob(job.id, 'fixture-token', new AbortController().signal)).toBe('released');
const after = await queue.getJob(job.id);
expect(after?.status).toBe('delayed');
expect(after?.attempts_made).toBe(before?.attempts_made);
expect(after?.stalled_counter).toBe(2);
expect(after?.lock_token).toBeNull();
expect(after?.lock_until).toBeNull();
expect(after?.delay_until!.getTime()).toBeGreaterThan(Date.now());
expect(after?.delay_until!.getTime()).toBeLessThanOrEqual(Date.now() + 30_000);
expect(await queue.releaseConfigurationJob(job.id, 'fixture-token', new AbortController().signal)).toBe('no_op');
});
test('stale tokens and terminal statuses are fenced no-ops', async () => {
for (const status of ['active', 'cancelled', 'completed'] as const) {
const job = await queue.add('fixture', {});
const token = `token-${job.id}`;
await queue.claim(token, 30_000, 'default', ['fixture']);
if (status !== 'active') await engine.executeRaw('UPDATE minion_jobs SET status = $1 WHERE id = $2', [status, job.id]);
const before = await queue.getJob(job.id);
expect(await queue.releaseConfigurationJob(job.id, status === 'active' ? 'stale-token' : token, new AbortController().signal)).toBe('no_op');
expect(await queue.getJob(job.id)).toEqual(before);
}
});
test('cancelled SQL and late responses remain explicitly unconfirmed', async () => {
const response = deferred<Array<{ id: number }>>();
let seenSignal: AbortSignal | undefined;
const isolated = new MinionQueue({
executeRaw: (_sql: string, _params: unknown[], opts: { signal: AbortSignal }) => {
seenSignal = opts.signal;
return response.promise;
},
} as unknown as BrainEngine);
const controller = new AbortController();
const release = isolated.releaseConfigurationJob(1, 'token', controller.signal);
controller.abort();
expect(await release).toBe('unconfirmed');
expect(seenSignal?.aborted).toBe(true);
response.resolve([{ id: 1 }]);
expect(await release).toBe('unconfirmed');
});
test('database loss reports fallback rather than protected release', async () => {
const isolated = new MinionQueue({ executeRaw: async () => { throw new Error('fixture unavailable'); } } as unknown as BrainEngine);
expect(await isolated.releaseConfigurationJob(1, 'token', new AbortController().signal)).toBe('unconfirmed');
});
});
describe('worker configuration admission and actual execution', () => {
test('ready fires once only after the final probe, schema and authority checks', async () => {
const probeEntered = deferred<void>();
const probeReady = deferred<void>();
const schemaEntered = deferred<void>();
const schemaReady = deferred<void>();
const events: string[] = [];
let probes = 0;
const engine = {
kind: 'postgres',
executeRaw: async (sql: string) => {
if (sql.trim() === 'SELECT 1') {
probes++;
if (probes === 2) throw new Error('fixture transient second probe');
if (probes === 3) { probeEntered.resolve(); await probeReady.promise; }
} else { events.push('authority'); }
return [];
},
} as unknown as BrainEngine;
await assertWorkerDbReadiness(engine);
const worker = new MinionWorker(engine, { pollInterval: 1, healthCheckInterval: 0 });
(worker as unknown as { queue: unknown }).queue = {
ensureSchema: async () => { schemaEntered.resolve(); await schemaReady.promise; events.push('schema'); },
promoteDelayed: async () => { events.push('promote'); },
claim: async () => { events.push('claim'); worker.stop(); return null; },
hasOverdueWatchdogSuspect: async () => false,
};
worker.register('fixture', async () => {});
worker.on('ready', () => { events.push('ready'); });
const run = worker.start();
await probeEntered.promise;
expect(events).toEqual([]);
probeReady.resolve();
await schemaEntered.promise;
expect(events).toEqual([]);
schemaReady.resolve();
await run;
expect(events).toEqual(['schema', 'authority', 'ready', 'promote', 'claim']);
expect(probes).toBe(3);
});
test('failed ready publication aborts startup before timers and claims', async () => {
const statements: string[] = [];
const engine = {
kind: 'postgres',
executeRaw: async (sql: string) => { statements.push(sql); return []; },
} as unknown as BrainEngine;
const worker = new MinionWorker(engine, { pollInterval: 1, healthCheckInterval: 1, stalledInterval: 1 });
const events: string[] = [];
(worker as unknown as { queue: unknown }).queue = {
ensureSchema: async () => { events.push('schema'); },
promoteDelayed: async () => { events.push('promote'); },
claim: async () => { events.push('claim'); return null; },
hasOverdueWatchdogSuspect: async () => false,
};
worker.register('fixture', async () => {});
worker.on('ready', () => { throw new Error('fixture status publication failed'); });
await expect(worker.start()).rejects.toThrow('fixture status publication failed');
await Bun.sleep(10);
expect(events).toEqual(['schema']);
expect(statements).toHaveLength(2);
expect((worker as unknown as { running: boolean }).running).toBe(false);
});
test('Postgres startup faults block before schema recovery, claims, and timers', async () => {
const error = fault();
const statements: string[] = [];
const worker = new MinionWorker({
kind: 'postgres',
executeRaw: async (sql: string) => { statements.push(sql); throw error; },
} as unknown as BrainEngine, { healthCheckInterval: 1, stalledInterval: 1 });
worker.register('fixture', async () => { throw new Error('handler must not run'); });
await worker.start();
await new Promise(resolve => setTimeout(resolve, 10));
expect(worker.configurationError).toBe(error);
expect(statements).toEqual(['SELECT 1']);
});
test('Postgres readiness waits before any schema recovery or claims', async () => {
const ready = deferred<void>();
const entered = deferred<void>();
const statements: string[] = [];
const worker = new MinionWorker({
kind: 'postgres',
executeRaw: async (sql: string) => {
statements.push(sql);
entered.resolve();
await ready.promise;
return [];
},
} as unknown as BrainEngine, { healthCheckInterval: 1, stalledInterval: 1 });
worker.register('fixture', async () => {});
const run = worker.start();
await entered.promise;
await new Promise(resolve => setTimeout(resolve, 10));
expect(statements).toEqual(['SELECT 1']);
worker.stop();
ready.resolve();
await run;
expect(statements).toEqual(['SELECT 1']);
});
test('transient startup failure retries readiness before admitting work', async () => {
const events: string[] = [];
let probes = 0;
const worker = new MinionWorker({
kind: 'postgres',
executeRaw: async (sql: string) => {
if (sql.trim() === 'SELECT 1') {
events.push('probe');
if (++probes === 1) throw new Error('fixture connection refused');
}
return [];
},
} as unknown as BrainEngine, { pollInterval: 1, healthCheckInterval: 0 });
(worker as unknown as { queue: unknown }).queue = {
ensureSchema: async () => { events.push('schema'); },
promoteDelayed: async () => { events.push('promote'); },
claim: async () => { events.push('claim'); worker.stop(); return null; },
hasOverdueWatchdogSuspect: async () => false,
};
worker.register('fixture', async () => {});
await worker.start();
expect(events).toEqual(['probe', 'probe', 'schema', 'promote', 'claim']);
expect(worker.configurationError).toBeNull();
});
test('periodic typed faults bypass the ordinary database failure counter', async () => {
const executeRaw = engine.executeRaw;
const error = fault();
let probes = 0;
engine.executeRaw = (async (sql: string, ...args: unknown[]) => {
if (sql.trim() === 'SELECT 1') { probes++; throw error; }
return executeRaw.call(engine, sql, ...args as [unknown[]]);
}) as typeof engine.executeRaw;
try {
const worker = new MinionWorker(engine, { pollInterval: 5, healthCheckInterval: 5, dbFailExitAfter: 100 });
worker.register('fixture', async () => {});
await worker.start();
expect(worker.configurationError).toBe(error);
expect(probes).toBe(1);
} finally {
engine.executeRaw = executeRaw;
}
});
test('a late ordinary health failure cannot override a latched configuration block', async () => {
const original = engine.executeRaw;
const entered = deferred<void>();
const resume = deferred<void>();
engine.executeRaw = (async (sql: string, ...args: unknown[]) => {
if (sql.trim() === 'SELECT 1') {
entered.resolve();
await resume.promise;
throw new Error('fixture network failure');
}
return original.call(engine, sql, ...args as [unknown[]]);
}) as typeof engine.executeRaw;
try {
const worker = new MinionWorker(engine, { pollInterval: 5, healthCheckInterval: 5, dbFailExitAfter: 1 });
const events: UnhealthyReason[] = [];
worker.on('unhealthy', event => events.push(event));
worker.register('fixture', async () => {});
const run = worker.start();
await entered.promise;
worker.blockForConfiguration(fault());
await run;
resume.resolve();
await new Promise(resolve => setTimeout(resolve, 20));
expect(events.map(event => event.reason)).toEqual(['client_misconfigured']);
} finally {
resume.resolve();
engine.executeRaw = original;
}
});
test('typed handler fault blocks before accounting and preserves the claimed job', async () => {
const job = await queue.add('fixture', {}, { max_attempts: 1 });
const second = await queue.add('fixture', {});
const worker = makeWorker();
const error = fault();
const events: UnhealthyReason[] = [];
let starts = 0;
worker.register('fixture', async () => { starts++; throw error; });
worker.on('unhealthy', info => {
expect(worker.configurationError).toBe(error);
events.push(info);
});
await worker.start();
worker.blockForConfiguration(fault());
expect(events).toEqual([{ reason: 'client_misconfigured', error }]);
expect(starts).toBe(1);
expect((await queue.getJob(job.id))?.status).toBe('delayed');
expect((await queue.getJob(job.id))?.attempts_made).toBe(0);
expect((await queue.getJob(job.id))?.stalled_counter).toBe(0);
expect((await queue.getJob(second.id))?.status).toBe('waiting');
expect(worker.configurationReleaseResults.map(result => result.outcome)).toEqual(['released']);
await worker.start();
expect(starts).toBe(1);
});
test('same error text without the type remains an ordinary job failure', async () => {
const job = await queue.add('fixture', {}, { max_attempts: 1 });
const worker = makeWorker();
worker.register('fixture', async () => { worker.stop(); throw new Error(fault().message); });
await worker.start();
expect(worker.configurationError).toBeNull();
expect((await queue.getJob(job.id))?.status).toBe('dead');
expect((await queue.getJob(job.id))?.attempts_made).toBe(1);
});
test('a typed recording fault blocks before execution tracking is discarded', async () => {
const job = await queue.add('fixture', {});
const worker = makeWorker();
const error = fault();
(worker as unknown as { queue: MinionQueue }).queue.failJob = async () => { throw error; };
worker.register('fixture', async () => { throw new Error('fixture ordinary handler error'); });
await worker.start();
expect(worker.configurationError).toBe(error);
expect((await queue.getJob(job.id))?.status).toBe('delayed');
expect((await queue.getJob(job.id))?.attempts_made).toBe(0);
expect(worker.configurationReleaseResults.map(result => result.outcome)).toEqual(['released']);
});
test('a paused late claim is registered and released without executing', async () => {
const job = await queue.add('fixture', {});
const worker = makeWorker();
const claimed = deferred<void>();
const resume = deferred<void>();
const internals = worker as unknown as { queue: MinionQueue };
const claim = internals.queue.claim.bind(internals.queue);
internals.queue.claim = async (...args) => {
const result = await claim(...args);
claimed.resolve();
await resume.promise;
return result;
};
let starts = 0;
worker.register('fixture', async () => { starts++; });
const run = worker.start();
await claimed.promise;
worker.blockForConfiguration(fault());
resume.resolve();
await run;
expect(starts).toBe(0);
expect((await queue.getJob(job.id))?.status).toBe('delayed');
expect((await queue.getJob(job.id))?.attempts_made).toBe(0);
expect(worker.configurationReleaseResults.map(result => result.outcome)).toEqual(['released']);
});
test('scheduling eviction cannot authorize release of a live inline handler', async () => {
const job = await queue.add('fixture', {});
const worker = makeWorker();
const started = deferred<void>();
const settle = deferred<void>();
const internals = worker as unknown as { inFlight: Map<number, unknown>; configurationDeadline: number };
worker.register('fixture', async () => { started.resolve(); await settle.promise; throw fault(); });
const run = worker.start();
await started.promise;
internals.inFlight.clear();
worker.blockForConfiguration(fault());
internals.configurationDeadline = performance.now() + 30;
await run;
expect((await queue.getJob(job.id))?.status).toBe('active');
expect((await queue.getJob(job.id))?.attempts_made).toBe(0);
expect(worker.configurationReleaseResults.map(result => result.outcome)).toEqual(['unconfirmed']);
settle.resolve();
await new Promise(resolve => setTimeout(resolve, 20));
expect((await queue.getJob(job.id))?.status).toBe('active');
});
test('cooperative inline shutdown releases only after execution has settled', async () => {
const job = await queue.add('fixture', {});
const worker = makeWorker();
const started = deferred<void>();
const settle = deferred<void>();
worker.register('fixture', async () => { started.resolve(); await settle.promise; throw new Error('fixture stopped'); });
const run = worker.start();
await started.promise;
worker.blockForConfiguration(fault());
await new Promise(resolve => setTimeout(resolve, 20));
expect((await queue.getJob(job.id))?.status).toBe('active');
settle.resolve();
await run;
expect((await queue.getJob(job.id))?.status).toBe('delayed');
expect((await queue.getJob(job.id))?.attempts_made).toBe(0);
expect(worker.configurationReleaseResults.map(result => result.outcome)).toEqual(['released']);
});
test('unconfirmed child cleanup cannot authorize delayed release', async () => {
const job = await queue.add('fixture', {});
const worker = makeWorker();
const started = deferred<void>();
const settle = deferred<void>();
worker.register('fixture', async () => { started.resolve(); await settle.promise; throw new ChildWorkerShutdownError('fixture cleanup unconfirmed', false); });
const run = worker.start();
await started.promise;
worker.blockForConfiguration(fault());
(worker as unknown as { configurationDeadline: number }).configurationDeadline = performance.now() + 30;
settle.resolve();
await run;
expect((await queue.getJob(job.id))?.status).toBe('active');
expect((await queue.getJob(job.id))?.attempts_made).toBe(0);
expect(worker.configurationReleaseResults.map(result => result.outcome)).toEqual(['unconfirmed']);
});
test('the 30-second eviction deadline cannot dead-letter or release live inline execution', async () => {
const job = await queue.add('fixture', {}, { max_attempts: 1 });
const worker = makeWorker();
const started = deferred<void>();
const settle = deferred<void>();
worker.register('fixture', async () => { started.resolve(); await settle.promise; throw new Error('fixture stopped late'); });
const run = worker.start();
await started.promise;
const internals = worker as unknown as { configurationDeadline: number };
const blockedAt = performance.now();
worker.blockForConfiguration(fault());
expect(internals.configurationDeadline - blockedAt).toBeGreaterThanOrEqual(29_990);
expect(internals.configurationDeadline - blockedAt).toBeLessThan(30_100);
const began = performance.now();
internals.configurationDeadline = began + 300;
await run;
expect(performance.now() - began).toBeGreaterThanOrEqual(290);
expect(performance.now() - began).toBeLessThan(2_300);
expect((await queue.getJob(job.id))?.status).toBe('active');
expect((await queue.getJob(job.id))?.attempts_made).toBe(0);
expect(worker.configurationReleaseResults.map(result => result.outcome)).toEqual(['unconfirmed']);
settle.resolve();
await new Promise(resolve => setTimeout(resolve, 20));
expect((await queue.getJob(job.id))?.status).toBe('active');
}, 15_000);
test('a claim still unknown at the global deadline reports lease-expiry fallback', async () => {
const job = await queue.add('fixture', {});
const worker = makeWorker();
const entered = deferred<void>();
const resume = deferred<void>();
const internals = worker as unknown as { queue: MinionQueue; configurationDeadline: number };
const claim = internals.queue.claim.bind(internals.queue);
internals.queue.claim = async (...args) => {
const result = await claim(...args);
entered.resolve();
await resume.promise;
return result;
};
let starts = 0;
worker.register('fixture', async () => { starts++; });
const run = worker.start();
await entered.promise;
worker.blockForConfiguration(fault());
internals.configurationDeadline = performance.now() + 30;
await run;
expect(worker.configurationReleaseResults).toHaveLength(1);
expect(worker.configurationReleaseResults[0]).toMatchObject({ jobId: null, outcome: 'unconfirmed' });
resume.resolve();
await new Promise(resolve => setTimeout(resolve, 20));
expect(starts).toBe(0);
expect(worker.configurationReleaseResults[0]).toMatchObject({ jobId: job.id, outcome: 'unconfirmed' });
expect((await queue.getJob(job.id))?.status).toBe('active');
});
test('40 owned tokens share one deadline and at most four release queries', async () => {
const worker = makeWorker();
const internals = worker as unknown as {
queue: MinionQueue;
executions: Map<string, { job: { id: number }; lockToken: string; stopped: boolean }>;
configurationDeadline: number;
drainConfiguration(): Promise<void>;
};
for (let id = 1; id <= 40; id++) internals.executions.set(`token-${id}`, { job: { id }, lockToken: `token-${id}`, stopped: true });
let active = 0;
let peak = 0;
internals.queue.releaseConfigurationJob = async (id, _token, signal) => {
active++;
peak = Math.max(peak, active);
if (id === 1) await new Promise<void>(resolve => signal.addEventListener('abort', () => resolve(), { once: true }));
else await new Promise(resolve => setTimeout(resolve, 1));
active--;
return signal.aborted ? 'unconfirmed' : 'released';
};
worker.blockForConfiguration(fault());
internals.configurationDeadline = performance.now() + 250;
const began = performance.now();
await internals.drainConfiguration();
expect(performance.now() - began).toBeLessThan(1000);
expect(peak).toBeLessThanOrEqual(4);
expect(worker.configurationReleaseResults).toHaveLength(40);
expect(worker.configurationReleaseResults.filter(result => result.outcome === 'released')).toHaveLength(39);
expect(worker.configurationReleaseResults.filter(result => result.outcome === 'unconfirmed')).toHaveLength(1);
});
});