Garry's Opinionated OpenClaw/Hermes Agent Brain
TypeScript
/**
* #1849: queue-scoped DB supervisor singleton.
*
* The pidfile guard is mutually exclusive only per pidfile PATH; the DB lock
* makes the (database, queue) pair the mutex domain so two supervisors with
* different $HOME / --pid-file can't both run on one queue. These tests pin:
* - the lock id keys on DB identity + queue (T2)
* - a second acquire of the same (db, queue) lock is refused (the singleton)
* - different queues don't collide
* - refresh-failure past the threshold fails SAFE (exits non-zero) (F1A)
*/
import { describe, test, expect, beforeAll, afterAll, beforeEach, spyOn } from 'bun:test';
import { existsSync, unlinkSync } from 'fs';
import { join } from 'path';
import { tmpdir } from 'os';
import { PGLiteEngine } from '../src/core/pglite-engine.ts';
import { tryAcquireDbLock, inspectLock, isLockHolderLive } from '../src/core/db-lock.ts';
import { MinionSupervisor, ExitCodes, supervisorLockId, classifySupervisorSingleton, SUPERVISOR_LOCK_TTL_MIN } from '../src/core/minions/supervisor.ts';
import type { DbLockHandle, LockSnapshot } from '../src/core/db-lock.ts';
// Build a LockSnapshot fixture for the isLockHolderLive matrix. Only ttl_expired
// and ms_since_last_refresh are consulted; the rest are filled for shape.
function snap(over: Partial<LockSnapshot>): LockSnapshot {
return {
id: 'gbrain-supervisor:default',
holder_pid: 4242,
holder_host: 'box',
acquisition_token: '00000000-0000-4000-8000-000000000001',
acquired_at: new Date(),
ttl_expires_at: new Date(),
age_ms: 1000,
ttl_expired: false,
last_refreshed_at: new Date(),
ms_since_last_refresh: 0,
db_now: new Date(),
...over,
};
}
let engine: PGLiteEngine;
beforeAll(async () => {
engine = new PGLiteEngine();
await engine.connect({});
await engine.initSchema();
}, 30000);
afterAll(async () => {
await engine.disconnect();
});
beforeEach(async () => {
await engine.executeRaw(`DELETE FROM gbrain_cycle_locks WHERE id LIKE 'gbrain-supervisor:%'`);
});
describe('#1849 supervisorLockId', () => {
test('keys on queue ONLY (DB scoping is physical — the lock row lives in the DB)', () => {
expect(supervisorLockId('default')).toBe('gbrain-supervisor:default');
expect(supervisorLockId('shell')).toBe('gbrain-supervisor:shell');
// Different queues → different locks.
expect(supervisorLockId('default')).not.toBe(supervisorLockId('shell'));
// Regression (the bug this fixes): the id must NOT depend on how the same
// physical DB was addressed. Two supervisors on one DB via different URLs
// must compute the SAME id so they collide on the one shared locks table.
// The function takes no DB-identity arg precisely so it can't diverge.
expect(supervisorLockId.length).toBe(1);
});
});
describe('#1849 classifySupervisorSingleton (doctor)', () => {
test('no live lock → no_lock', () => {
expect(classifySupervisorSingleton({
lockLive: false, lockHolderHost: 'h', lockHolderPid: 1, localHost: 'h', localPid: 1,
})).toBe('no_lock');
});
test('live lock held by the local (host,pid) → single', () => {
expect(classifySupervisorSingleton({
lockLive: true, lockHolderHost: 'box', lockHolderPid: 42, localHost: 'box', localPid: 42,
})).toBe('single');
});
test('live lock held by a DIFFERENT pid → mismatch (rogue second supervisor)', () => {
expect(classifySupervisorSingleton({
lockLive: true, lockHolderHost: 'box', lockHolderPid: 99, localHost: 'box', localPid: 42,
})).toBe('mismatch');
});
test('same pid but DIFFERENT host → mismatch (bare pid is meaningless cross-host)', () => {
expect(classifySupervisorSingleton({
lockLive: true, lockHolderHost: 'other', lockHolderPid: 42, localHost: 'box', localPid: 42,
})).toBe('mismatch');
});
test('live lock but no local pidfile → mismatch', () => {
expect(classifySupervisorSingleton({
lockLive: true, lockHolderHost: 'box', lockHolderPid: 42, localHost: 'box', localPid: null,
})).toBe('mismatch');
});
});
describe('#1849 DB lock is the real singleton', () => {
test('second acquire of the same (db, queue) lock is refused', async () => {
const id = supervisorLockId('default');
const first = await tryAcquireDbLock(engine, id, 5);
expect(first).not.toBeNull();
// A second supervisor (different pidfile, same db+queue) gets null → exit 2.
const second = await tryAcquireDbLock(engine, id, 5);
expect(second).toBeNull();
// After release, a fresh supervisor can take over.
await first!.release();
const third = await tryAcquireDbLock(engine, id, 5);
expect(third).not.toBeNull();
await third!.release();
});
test('different queues on the same DB do not collide', async () => {
const a = await tryAcquireDbLock(engine, supervisorLockId('default'), 5);
const b = await tryAcquireDbLock(engine, supervisorLockId('shell'), 5);
expect(a).not.toBeNull();
expect(b).not.toBeNull();
await a!.release();
await b!.release();
});
});
describe('#1849 LOCK_HELD path does not strand the pidfile', () => {
test('the pidfile-cleanup exit listener is installed BEFORE the DB-lock acquire', async () => {
// Supervisor A already holds the queue lock.
const holderA = await tryAcquireDbLock(engine, supervisorLockId('default'), 5);
expect(holderA).not.toBeNull();
const pidFile = join(tmpdir(), `gbrain-sup-stranded-${process.pid}-${Math.random().toString(36).slice(2)}.pid`);
const sup = new MinionSupervisor(engine, { cliPath: '/bin/sh', healthInterval: 0, json: true, pidFile });
// Capture the 'exit' listener start() registers (if any) and stop execution
// at the first process.exit (the LOCK_HELD path) the way the real exit would.
let exitListener: ((...a: unknown[]) => void) | null = null;
const onSpy = spyOn(process, 'on').mockImplementation(((event: string, cb: (...a: unknown[]) => void) => {
if (event === 'exit') exitListener = cb;
return process;
}) as never);
const exitSpy = spyOn(process, 'exit').mockImplementation(((code?: number) => {
throw new Error(`exit:${code}`);
}) as never);
try {
try { await sup.start(); } catch { /* exit stub throws at LOCK_HELD */ }
expect(exitSpy).toHaveBeenCalledWith(ExitCodes.LOCK_HELD);
// The bug: the exit listener was registered AFTER the DB-lock exit, so
// start() threw before reaching it and the pidfile this process created
// is stranded. The fix installs it first → it's captured here.
expect(exitListener).not.toBeNull();
// And it actually cleans up the pidfile we created (contents match our pid).
expect(existsSync(pidFile)).toBe(true);
exitListener!();
expect(existsSync(pidFile)).toBe(false);
} finally {
onSpy.mockRestore();
exitSpy.mockRestore();
if (existsSync(pidFile)) unlinkSync(pidFile);
await holderA!.release();
}
});
});
describe('#2227 isLockHolderLive — PID-reuse-safe supervisor liveness', () => {
test('fresh TTL → live (the normal running case)', () => {
expect(isLockHolderLive(snap({ ttl_expired: false }), SUPERVISOR_LOCK_TTL_MIN)).toBe(true);
});
test('expired TTL but refreshed within the steal grace → live (starved-but-alive #1794)', () => {
// ttl lapsed but the holder heartbeat is recent → it is alive, just starved.
expect(isLockHolderLive(snap({ ttl_expired: true, ms_since_last_refresh: 5_000 }), SUPERVISOR_LOCK_TTL_MIN)).toBe(true);
});
test('expired TTL and stale heartbeat → dead (a gone supervisor stops refreshing)', () => {
expect(isLockHolderLive(snap({ ttl_expired: true, ms_since_last_refresh: 36_000_000 }), SUPERVISOR_LOCK_TTL_MIN)).toBe(false);
});
test('expired TTL and no heartbeat column → dead', () => {
expect(isLockHolderLive(snap({ ttl_expired: true, ms_since_last_refresh: null }), SUPERVISOR_LOCK_TTL_MIN)).toBe(false);
});
test('liveness NEVER consults process.kill (PID reuse cannot false-positive)', () => {
// A row whose holder_pid happens to be a live, unrelated process (PID reuse)
// but whose lock is stale must read as NOT live — proving freshness, not the
// PID probe, is the signal. holder_pid=1 (init, always alive) + expired/stale.
expect(isLockHolderLive(snap({ holder_pid: 1, ttl_expired: true, ms_since_last_refresh: 36_000_000 }), SUPERVISOR_LOCK_TTL_MIN)).toBe(false);
});
});
describe('#2227 status detects a live supervisor via the DB lock (split-$HOME)', () => {
test('a live queue lock with no local pidfile reads as running via inspectLock', async () => {
// Simulate the keeper holding the queue lock under a different $HOME: there
// is a live lock row but the local pidfile path is empty.
const holder = await tryAcquireDbLock(engine, supervisorLockId('default'), SUPERVISOR_LOCK_TTL_MIN);
expect(holder).not.toBeNull();
const live = await inspectLock(engine, supervisorLockId('default'));
expect(live).not.toBeNull();
expect(isLockHolderLive(live!, SUPERVISOR_LOCK_TTL_MIN)).toBe(true);
await holder!.release();
// After release the row is gone → not running.
const gone = await inspectLock(engine, supervisorLockId('default'));
expect(gone).toBeNull();
});
});
describe('#1849 refresh-failure fails safe (F1A)', () => {
test('exits LOCK_LOST after the failure threshold; tolerates a single blip', async () => {
const sup = new MinionSupervisor(engine, { cliPath: '/bin/sh', healthInterval: 0, json: true });
const exitSpy = spyOn(process, 'exit').mockImplementation(((_code?: number) => {
throw new Error(`exit:${_code}`); // stop execution like the real exit would
}) as never);
let refreshCalls = 0;
const failingLock: DbLockHandle = {
id: 'x',
acquiredAt: 'test-fence',
acquisitionToken: '00000000-0000-4000-8000-000000000001',
refresh: async () => { refreshCalls++; throw new Error('pooler down'); },
release: async () => {},
};
sup._setDbLockForTests(failingLock);
try {
// First two failures: tolerated (counter climbs, no exit).
await sup._refreshDbLockForTests();
await sup._refreshDbLockForTests();
expect(exitSpy).not.toHaveBeenCalled();
// Third failure crosses the threshold → shutdown → process.exit(LOCK_LOST).
try { await sup._refreshDbLockForTests(); } catch { /* exit stub throws */ }
expect(exitSpy).toHaveBeenCalledWith(ExitCodes.LOCK_LOST);
expect(refreshCalls).toBe(3);
} finally {
exitSpy.mockRestore();
}
});
test('a successful refresh resets the failure counter', async () => {
const sup = new MinionSupervisor(engine, { cliPath: '/bin/sh', healthInterval: 0, json: true });
const exitSpy = spyOn(process, 'exit').mockImplementation(((_code?: number) => {
throw new Error(`exit:${_code}`);
}) as never);
let mode: 'fail' | 'ok' = 'fail';
const flakyLock: DbLockHandle = {
id: 'x',
acquiredAt: 'test-fence',
acquisitionToken: '00000000-0000-4000-8000-000000000001',
refresh: async () => { if (mode === 'fail') throw new Error('blip'); return true; },
release: async () => {},
};
sup._setDbLockForTests(flakyLock);
try {
await sup._refreshDbLockForTests(); // fail 1
await sup._refreshDbLockForTests(); // fail 2
mode = 'ok';
await sup._refreshDbLockForTests(); // success → reset
mode = 'fail';
await sup._refreshDbLockForTests(); // fail 1 again
await sup._refreshDbLockForTests(); // fail 2
// Counter was reset, so we are NOT past threshold yet.
expect(exitSpy).not.toHaveBeenCalled();
} finally {
exitSpy.mockRestore();
}
});
});
test('W0 (D5.10): fenced refresh returning false is CERTAIN loss — immediate LOCK_LOST, no threshold', async () => {
const sup = new MinionSupervisor(engine, { cliPath: '/bin/sh', healthInterval: 0, json: true });
const exitSpy = spyOn(process, 'exit').mockImplementation(((_code?: number) => {
throw new Error(`exit:${_code}`);
}) as never);
let refreshCalls = 0;
const stolenLock: DbLockHandle = {
id: 'x',
acquiredAt: 'test-fence',
acquisitionToken: '00000000-0000-4000-8000-000000000001',
refresh: async () => { refreshCalls++; return false; },
release: async () => {},
};
sup._setDbLockForTests(stolenLock);
try {
// FIRST tick exits — a fenced miss is proof of loss, not a blip to
// count toward SUPERVISOR_LOCK_REFRESH_MAX_FAILURES (pre-fix it was
// treated as SUCCESS and reset the failure counter while a second
// supervisor drained the same queue).
try { await sup._refreshDbLockForTests(); } catch { /* exit stub throws */ }
expect(exitSpy).toHaveBeenCalledWith(ExitCodes.LOCK_LOST);
expect(refreshCalls).toBe(1);
} finally {
exitSpy.mockRestore();
}
});