Garry's Opinionated OpenClaw/Hermes Agent Brain
TypeScript
// #1685 GAP A — worker_oom_loop doctor check.
//
// Hermetic: writes synthetic supervisor audit JSONL into a GBRAIN_AUDIT_DIR
// tempdir and drives computeWorkerOomLoopCheck with engine=null (supervised
// half + cap logic + thresholds; the minion_jobs bare-worker branch is
// Postgres-only and mirrors the queue_health subcheck-3 query covered by E2E).
// Also pins the cross-week supervisor reader (CODEX #7).
import { describe, expect, test, beforeEach, afterEach } from 'bun:test';
import * as fs from 'fs';
import * as os from 'os';
import * as path from 'path';
import { withEnv } from './helpers/with-env.ts';
import { computeWorkerOomLoopCheck } from '../src/commands/doctor.ts';
import {
computeSupervisorAuditFilename,
readRecentSupervisorEvents,
} from '../src/core/minions/handlers/supervisor-audit.ts';
let tmpDir: string;
beforeEach(() => {
tmpDir = fs.mkdtempSync(path.join(os.tmpdir(), 'worker-oom-loop-'));
});
afterEach(() => {
try { fs.rmSync(tmpDir, { recursive: true, force: true }); } catch { /* best-effort */ }
});
function writeSupervisorRows(rows: object[], fileDate = new Date()): void {
const file = path.join(tmpDir, computeSupervisorAuditFilename(fileDate));
fs.writeFileSync(file, rows.map((r) => JSON.stringify(r)).join('\n') + '\n', 'utf8');
}
const nowIso = () => new Date().toISOString();
const rssKill = () => ({ event: 'worker_exited', ts: nowIso(), likely_cause: 'rss_watchdog', code: 12 });
const breaker = (cap: number) => ({ event: 'health_warn', ts: nowIso(), reason: 'rss_watchdog_loop', max_rss_mb: cap });
describe('computeWorkerOomLoopCheck', () => {
test('fail with breaker-stamped cap when the breaker tripped', async () => {
await withEnv({ GBRAIN_AUDIT_DIR: tmpDir }, async () => {
writeSupervisorRows([rssKill(), rssKill(), rssKill(), rssKill(), rssKill(), rssKill(), breaker(2048)]);
const c = await computeWorkerOomLoopCheck(null);
expect(c?.status).toBe('fail');
expect(c?.name).toBe('worker_oom_loop');
expect(c?.message).toContain('cap=2048MB');
expect(c?.message).toContain('raise --max-rss');
expect(c?.details?.oom_kills).toBe(6);
expect(c?.details?.supervisor_kills).toBe(6);
expect(c?.details?.bare_worker_kills).toBe(0);
expect(c?.details?.cap_source).toBe('breaker');
expect(c?.details?.breaker_tripped).toBe(true);
});
});
test('warn with auto-sized cap fallback when no breaker event stamped a cap (CODEX #6)', async () => {
await withEnv({ GBRAIN_AUDIT_DIR: tmpDir }, async () => {
writeSupervisorRows([rssKill(), rssKill()]);
const c = await computeWorkerOomLoopCheck(null);
expect(c?.status).toBe('warn');
expect(c?.details?.cap_source).toBe('default');
expect(c?.message).toContain('auto-sized default');
expect(c?.details?.oom_kills).toBe(2);
});
});
test('fail at >=5 kills even without a breaker event', async () => {
await withEnv({ GBRAIN_AUDIT_DIR: tmpDir }, async () => {
writeSupervisorRows([rssKill(), rssKill(), rssKill(), rssKill(), rssKill()]);
const c = await computeWorkerOomLoopCheck(null);
expect(c?.status).toBe('fail');
expect(c?.details?.breaker_tripped).toBe(false);
});
});
test('null when the worker never OOM-looped', async () => {
await withEnv({ GBRAIN_AUDIT_DIR: tmpDir }, async () => {
writeSupervisorRows([{ event: 'started', ts: nowIso() }]);
expect(await computeWorkerOomLoopCheck(null)).toBeNull();
});
});
test('null on an empty audit dir', async () => {
await withEnv({ GBRAIN_AUDIT_DIR: tmpDir }, async () => {
expect(await computeWorkerOomLoopCheck(null)).toBeNull();
});
});
});
describe('readRecentSupervisorEvents — cross-week (CODEX #7)', () => {
test('reads a within-24h event from the PREVIOUS ISO-week file', async () => {
await withEnv({ GBRAIN_AUDIT_DIR: tmpDir }, async () => {
// Event timestamped within the 24h window but written into last week's
// file (the Monday-reads-Sunday case). The single-file reader would miss
// it; the cross-week reader must find it.
const prevWeekFileDate = new Date(Date.now() - 7 * 86400000);
writeSupervisorRows([rssKill()], prevWeekFileDate);
const events = readRecentSupervisorEvents(24);
expect(events.length).toBe(1);
expect(events[0].event).toBe('worker_exited');
});
});
});