Garry's Opinionated OpenClaw/Hermes Agent Brain
TypeScript
/**
* issue #1678 — Minion hot-path lock recovery contract.
*
* - promoteDelayed (idempotent) self-heals: a reaped-socket CONNECTION_ENDED
* triggers a reconnect + retry against a fresh pool.
* - claim does NOT retry inline (Codex #1): blind-retrying a claim whose
* UPDATE...RETURNING may have committed could double-claim a job. The error
* propagates; the worker poll loop reconnects + re-claims on the next tick.
*
* Hermetic: a fake BrainEngine whose executeRaw is scripted; no real DB.
*/
import { describe, it, expect } from 'bun:test';
import { tmpdir } from 'os';
import { join } from 'path';
import { MinionQueue } from '../src/core/minions/queue.ts';
import { withEnv } from './helpers/with-env.ts';
function connEndedError(): Error & { code: string } {
const e = new Error('write CONNECTION_ENDED localhost:6543') as Error & { code: string };
e.code = 'CONNECTION_ENDED';
return e;
}
function isAuthorityInventory(sql: string): boolean {
return sql.includes('SELECT id, submission_authority FROM minion_jobs') &&
sql.includes('submission_authority IS DISTINCT FROM');
}
const AUDIT_DIR = join(tmpdir(), `gbrain-queue-lock-retry-${process.pid}-${Date.now()}`);
// Fast retry + isolated audit dir so the test doesn't sleep ~1s or pollute ~/.gbrain.
const FAST_ENV = {
GBRAIN_BULK_RETRY_BASE_MS: '1',
GBRAIN_BULK_RETRY_MAX_MS: '2',
GBRAIN_AUDIT_DIR: AUDIT_DIR,
};
describe('MinionQueue lock-path recovery (issue #1678)', () => {
it('promoteDelayed reconnects + retries on a reaped-socket error', async () => {
await withEnv(FAST_ENV, async () => {
let calls = 0;
let reconnects = 0;
const engine = {
kind: 'postgres',
executeRaw: async (sql: string) => {
if (isAuthorityInventory(sql)) return [];
if (!sql.startsWith('UPDATE minion_jobs SET') ||
!sql.includes("WHERE status = 'delayed' AND delay_until <= now()")) {
throw new Error('Unexpected SQL in promotion retry fixture');
}
calls++;
if (calls === 1) throw connEndedError();
return [];
},
reconnect: async () => { reconnects++; },
} as unknown as ConstructorParameters<typeof MinionQueue>[0];
const q = new MinionQueue(engine);
const out = await q.promoteDelayed();
expect(out).toEqual([]);
expect(calls).toBe(2); // first attempt threw, retry succeeded
expect(reconnects).toBe(1); // reconnect fired between attempts
});
});
it('claim does NOT retry inline on a reaped-socket error (Codex #1 double-claim guard)', async () => {
await withEnv(FAST_ENV, async () => {
let calls = 0;
const engine = {
kind: 'postgres',
// claim() routes through executeRawDirect (direct session pool) as of
// the lock-hot-path fix. Only the read-only authority inventory may
// use executeRaw; the claim mutation must stay on the direct pool.
executeRawDirect: async (sql: string) => {
if (!sql.includes("status = 'active'") || !sql.includes('FOR UPDATE SKIP LOCKED')) {
throw new Error('Unexpected SQL in claim retry fixture');
}
calls++;
throw connEndedError();
},
executeRaw: async (sql: string) => {
if (isAuthorityInventory(sql)) return [];
throw new Error('claim mutation must not use executeRaw');
},
reconnect: async () => {},
} as unknown as ConstructorParameters<typeof MinionQueue>[0];
const q = new MinionQueue(engine);
await expect(q.claim('tok', 1000, 'default', ['sync'])).rejects.toThrow('CONNECTION_ENDED');
expect(calls).toBe(1); // exactly one attempt — no inline retry
});
});
it('an unavailable authority inventory prevents both claim and promotion mutations', async () => {
await withEnv(FAST_ENV, async () => {
let mutations = 0;
const engine = {
kind: 'postgres',
executeRaw: async (sql: string) => {
if (isAuthorityInventory(sql)) throw connEndedError();
mutations++;
return [];
},
executeRawDirect: async () => { mutations++; return []; },
reconnect: async () => {},
} as unknown as ConstructorParameters<typeof MinionQueue>[0];
const q = new MinionQueue(engine);
await expect(q.promoteDelayed()).rejects.toThrow('CONNECTION_ENDED');
await expect(q.claim('tok', 1000, 'default', ['sync'])).rejects.toThrow('CONNECTION_ENDED');
expect(mutations).toBe(0);
});
});
});