Garry's Opinionated OpenClaw/Hermes Agent Brain
TypeScript
import { afterAll, afterEach, beforeAll, expect, test } from 'bun:test';
import { randomUUID } from 'node:crypto';
import { execFileSync } from 'node:child_process';
import { mkdirSync, mkdtempSync, rmSync, writeFileSync } from 'node:fs';
import { tmpdir } from 'node:os';
import { join } from 'node:path';
import { PGLiteEngine } from '../src/core/pglite-engine.ts';
import type { BrainEngine } from '../src/core/engine.ts';
import type { SyncOpts, SyncResult } from '../src/commands/sync.ts';
import { performSync } from '../src/commands/sync/perform.ts';
import { claimWorktree } from '../src/core/persistence/ownership.ts';
import { performManagedSync } from '../src/core/persistence/sync-run.ts';
import { disposePersistenceConsumer } from '../src/core/persistence/service.ts';
import { DRAIN_LEASE_TTL_MINUTES, HOLDER_STALL_MS, LEASE_BEACON_QUEUE, followStopAt, holderStalled, readDrainHolder, withDrainLease } from '../src/core/persistence/sync-lease.ts';
import { writeGitHold } from '../src/core/persistence/sync-holds.ts';
import { drainNext } from '../src/core/persistence/sync-drain.ts';
import { liveSyncJob } from '../src/core/persistence/sync-movement.ts';
import { installFaultHook } from '../src/core/persistence/fault-points.ts';
import { syncLockId, tryAcquireDbLock } from '../src/core/db-lock.ts';
import { SyncLockBusyError } from '../src/core/sync-lock.ts';
import { noteForwardProgress, registerRunDeadline } from '../src/core/forward-progress.ts';
import { withEnv } from './helpers/with-env.ts';
import { isolatedPersistencePostgres } from './helpers/persistence-postgres.ts';
import type { MinionJobContext } from '../src/core/minions/types.ts';
import { makeSyncHandler } from '../src/core/minions/handlers/sync.ts';
// #6712 PR 1: the drain lease. One managed drainer per source, whoever started it, fenced on the lock's acquisition
// token; a plain CLI sync follows a live holder within its budget instead of draining beside it. Each test here fails on
// the base (no lease: both drainers run, nothing is fenced, nothing is followed).
const home = mkdtempSync(join(tmpdir(), 'gbrain-sync-lease-'));
const engines: BrainEngine[] = [];
let closePostgres: (() => Promise<void>) | undefined;
const git = (root: string, ...args: string[]) => execFileSync('git', ['-C', root, ...args], { encoding: 'utf8', stdio: ['ignore', 'pipe', 'pipe'] }).trim();
beforeAll(async () => {
const lite = new PGLiteEngine(); await lite.connect({}); await lite.initSchema(); engines.push(lite);
if (process.env.DATABASE_URL) { const pg = await isolatedPersistencePostgres(process.env.DATABASE_URL); engines.push(pg.engine); closePostgres = pg.close; }
}, 120_000);
afterEach(() => { installFaultHook(undefined); registerRunDeadline(null); });
afterAll(async () => {
await withEnv({ GBRAIN_HOME: home }, async () => { for (const engine of engines) { await disposePersistenceConsumer(engine); await engine.disconnect(); } });
await closePostgres?.(); rmSync(home, { recursive: true, force: true });
});
async function fixture(engine: BrainEngine) {
const id = `lease-${randomUUID().slice(0, 12)}`, root = join(home, id);
mkdirSync(root); git(root, 'init', '-q');
for (const name of ['a', 'b', 'c']) writeFileSync(join(root, `${name}.md`), `---\ntitle: ${name}\n---\nA durable observation named ${name} at acme-example.\n`);
git(root, 'add', '.'); git(root, '-c', 'user.name=Example', '-c', '[email protected]', 'commit', '-qm', 'seed');
await disposePersistenceConsumer(engine);
await engine.executeRaw('UPDATE persistence_brain SET enabled=false WHERE singleton=1');
await engine.executeRaw('INSERT INTO sources(id,name,local_path,config) VALUES($1,$1,$2,\'{}\')', [id, root]);
await claimWorktree(engine, id, root);
await engine.executeRaw('UPDATE persistence_brain SET enabled=true WHERE singleton=1');
return id;
}
const isolated = (fn: (engine: BrainEngine) => Promise<void>) => withEnv({ GBRAIN_HOME: home, OPENAI_API_KEY: undefined, ANTHROPIC_API_KEY: undefined }, async () => {
for (const engine of engines) await fn(engine);
});
const plain = (sourceId: string, extra: Partial<SyncOpts> = {}): SyncOpts => ({ sourceId, noPull: true, noEmbed: true, noExtract: true, ...extra });
const done = (): SyncResult => ({ status: 'up_to_date', fromCommit: null, toCommit: '', added: 0, modified: 0, deleted: 0, renamed: 0, chunksCreated: 0, embedded: 0, pagesAffected: [] });
/** A holder that keeps the lease until `release()`; `started` resolves once it holds it. */
function hold(engine: BrainEngine, sourceId: string, kind: 'serve' | 'cli' | 'job' = 'serve') {
let release!: () => void, started!: () => void;
const gate = new Promise<void>(resolve => { release = resolve; });
const ready = new Promise<void>(resolve => { started = resolve; });
const result = withDrainLease(engine, sourceId, { holder: { kind, id: kind === 'job' ? '4242' : `${kind}-example` } }, async () => { started(); await gate; return done(); });
return { ready, release: () => { release(); return result; } };
}
const managedRows = async (engine: BrainEngine, id: string) => ({
cursors: (await engine.executeRaw("SELECT 1 FROM op_checkpoints WHERE op='managed-sync' AND completed_keys->0->>'sourceId'=$1", [id])).length,
requests: (await engine.executeRaw('SELECT 1 FROM persistence_requests WHERE source_id=$1', [id])).length,
});
test('the lease is the per-source sync lock with a short TTL a dead holder loses inside one cron tick', async () => isolated(async engine => {
const id = await fixture(engine);
const holder = hold(engine, id);
await holder.ready;
const [row] = await engine.executeRaw<{ ttl_seconds: number }>('SELECT extract(epoch from ttl_expires_at - acquired_at)::int AS ttl_seconds FROM gbrain_cycle_locks WHERE id=$1', [syncLockId(id)]);
expect(row!.ttl_seconds).toBe(DRAIN_LEASE_TTL_MINUTES * 60);
expect(DRAIN_LEASE_TTL_MINUTES * 60_000 + 60_000).toBeLessThan(15 * 60_000);
expect(await readDrainHolder(engine, id)).toMatchObject({ kind: 'serve', id: 'serve-example', pid: process.pid, version: expect.any(String), progress: { committed: 0 } });
await holder.release();
expect(await engine.executeRaw('SELECT 1 FROM gbrain_cycle_locks WHERE id=$1', [syncLockId(id)])).toHaveLength(0);
}), 120_000);
test('a job, the cycle and sync_brain skip a busy lease with SyncLockBusyError; a nested pass inside the lease runs under it', async () => isolated(async engine => {
const id = await fixture(engine);
const holder = hold(engine, id);
await holder.ready;
await expect(performSync(engine, plain(id, { drainHolder: { kind: 'job', id: '7' } }))).rejects.toBeInstanceOf(SyncLockBusyError);
await holder.release();
// Reentrancy: a pass inside a held lease (a drain's passes, staged stages) never follows itself.
const nested = await withDrainLease(engine, id, { holder: { kind: 'cli' } }, () => performManagedSync(engine, plain(id)));
expect(nested.status).toBe('first_sync');
}), 120_000);
test('a plain sync follows a live holder, takes the lease when it releases and drains', async () => isolated(async engine => {
const id = await fixture(engine);
const holder = hold(engine, id);
await holder.ready;
const lines: string[] = [];
const origError = console.error;
console.error = (...args: unknown[]) => { lines.push(args.map(String).join(' ')); };
let followed: Promise<SyncResult>;
try {
followed = performSync(engine, plain(id, { followDrainLease: true, drainHolder: { kind: 'cli', id: null } }));
await new Promise(resolve => setTimeout(resolve, 3000));
expect(await managedRows(engine, id)).toEqual({ cursors: 0, requests: 0 });
await holder.release();
expect((await followed).status).toBe('first_sync');
} finally { console.error = origError; }
expect(lines.join('')).toContain('is being drained by a serve drainer');
expect(lines.join('')).toContain('Ctrl-C stops following and leaves it running');
expect(await engine.getPage('c', { sourceId: id })).not.toBeNull();
}), 120_000);
test('out of budget the follower is resumable with sync_drain_lease_held, the holder, and a wait fix', async () => isolated(async engine => {
const id = await fixture(engine);
const holder = hold(engine, id);
await holder.ready;
const result = await performSync(engine, plain(id, { followDrainLease: true, deadlineAtMs: Date.now() + 3000 }));
expect(result).toMatchObject({ status: 'partial', reason: 'drain_lease_held', drain: { outcome: 'resumable', stop_reason: 'drain_lease_held' },
leaseHeld: { code: 'sync_drain_lease_held', safe_to_loop: true, retry_after_ms: 60_000,
holder: { kind: 'serve', id: 'serve-example', pid: process.pid, host: expect.any(String), started_at: expect.any(String), last_heartbeat_at: expect.any(String),
version: expect.any(String), progress: { committed: 0 } } } });
expect(result.leaseHeld!.fix.why).toContain('--break-lock');
const next = drainNext(result, `gbrain sync --source ${id} --no-pull`, id);
expect(next).toMatchObject({ code: 'sync_drain_lease_held', safe_to_loop: true, retry_after_ms: 60_000, fix: { next: 'wait' } });
expect(await managedRows(engine, id)).toEqual({ cursors: 0, requests: 0 });
await holder.release();
}), 120_000);
test('Ctrl-C stops following at once and leaves the holder holding the lease', async () => isolated(async engine => {
const id = await fixture(engine);
const holder = hold(engine, id);
await holder.ready;
const token = async () => (await engine.executeRaw<{ t: string }>('SELECT acquisition_token::text AS t FROM gbrain_cycle_locks WHERE id=$1', [syncLockId(id)]))[0]?.t;
const before = await token();
const interrupt = new AbortController();
setTimeout(() => interrupt.abort(new Error('SIGINT')), 500);
const started = Date.now();
const result = await performSync(engine, plain(id, { followDrainLease: true, signal: interrupt.signal }));
expect(Date.now() - started).toBeLessThan(10_000);
expect(result.leaseHeld?.code).toBe('sync_drain_lease_held');
expect(await token()).toBe(before!);
await holder.release();
}), 120_000);
test('a holder that heartbeats but makes no progress for the stall window stops the follower blocked, not resumable', async () => isolated(async engine => {
const id = await fixture(engine);
const holder = hold(engine, id, 'cli');
await holder.ready;
await engine.executeRaw(`UPDATE gbrain_cycle_locks SET acquired_at=now()-make_interval(secs => $2) WHERE id=$1`, [syncLockId(id), HOLDER_STALL_MS / 1000 + 60]);
const result = await performSync(engine, plain(id, { followDrainLease: true, deadlineAtMs: Date.now() + 30_000 }));
expect(result.drain).toMatchObject({ outcome: 'blocked', stop_reason: 'drain_stalled', stall: { cause: 'no_progress', owner_kind: 'cli', lease_holder: { kind: 'cli' } } });
const next = drainNext(result, `gbrain sync --source ${id} --no-pull`, id);
expect(next).toMatchObject({ code: 'drain_stalled', cause: 'no_progress', safe_to_loop: false });
expect(next?.why).toContain('--break-lock');
await holder.release();
}), 120_000);
test('#7327: a holder writing only holds is progressing; the follower waits instead of stopping drain_stalled', async () => isolated(async engine => {
const id = await fixture(engine);
const holder = hold(engine, id, 'job');
await holder.ready;
await engine.executeRaw(`UPDATE gbrain_cycle_locks SET acquired_at=now()-make_interval(secs => $2) WHERE id=$1`, [syncLockId(id), HOLDER_STALL_MS / 1000 + 60]);
const [binding] = await engine.executeRaw<{ incarnation: string }>('SELECT source_incarnation::text AS incarnation FROM persistence_source_bindings WHERE source_id=$1', [id]);
await engine.transaction(tx => writeGitHold(tx, { source_id: id, incarnation: binding!.incarnation, path: 'linked/a.md', source_path: 'linked/a.md', slug: 'linked/a', page_id: null,
code: 'file_unreadable', message: 'Sync did not read linked/a.md: it is a symlink.', upstream_version: null, observed_at: new Date().toISOString(), run_id: 'run-1', mode: 'managed',
meta: { reason: 'symlink_dangling', recovery_version: 1 } } as Parameters<typeof writeGitHold>[1]));
const seen = await readDrainHolder(engine, id);
expect(seen?.progress?.committed).toBe(0);
expect(Date.now() - Date.parse(seen!.progress!.last_progress_at!)).toBeLessThan(HOLDER_STALL_MS);
expect(holderStalled(seen)).toBe(false);
const result = await performSync(engine, plain(id, { followDrainLease: true, deadlineAtMs: Date.now() + 3_000 }));
expect(result.drain).toMatchObject({ outcome: 'resumable', stop_reason: 'drain_lease_held' });
await holder.release();
}), 120_000);
test('fencing: a pass whose lease was taken over after its TTL admits nothing and advances no checkpoint', async () => isolated(async engine => {
const id = await fixture(engine);
let thief: Awaited<ReturnType<typeof tryAcquireDbLock>> = null;
installFaultHook(async (point, detail) => {
if (point !== 'sync:lease_acquired' || detail.sourceId !== id || thief) return;
// The holder pauses past its TTL (no heartbeat inside the steal grace) and another process takes the lease over.
await engine.executeRaw(`UPDATE gbrain_cycle_locks SET ttl_expires_at=now()-interval '1 minute', last_refreshed_at=now()-interval '1 hour' WHERE id=$1`, [syncLockId(id)]);
thief = await tryAcquireDbLock(engine, syncLockId(id), DRAIN_LEASE_TTL_MINUTES);
expect(thief).not.toBeNull();
});
const stale = await performSync(engine, plain(id));
installFaultHook(undefined);
expect(stale).toMatchObject({ reason: 'drain_lease_held', leaseHeld: { lost: true } });
expect(await managedRows(engine, id)).toEqual({ cursors: 0, requests: 0 });
await thief!.release();
expect((await performSync(engine, plain(id))).status).toBe('first_sync');
}), 120_000);
// Outside-voice review: a lease that lapsed with nobody taking it over is lost too; the token alone does not admit.
test('fencing: a pass whose lease expired untaken admits nothing and advances no checkpoint', async () => isolated(async engine => {
const id = await fixture(engine);
let lapsed = false;
installFaultHook(async (point, detail) => {
if (point !== 'sync:lease_acquired' || detail.sourceId !== id || lapsed) return;
lapsed = true;
await engine.executeRaw(`UPDATE gbrain_cycle_locks SET ttl_expires_at=now()-interval '1 second' WHERE id=$1`, [syncLockId(id)]);
});
const stale = await performSync(engine, plain(id));
installFaultHook(undefined);
expect(lapsed).toBe(true);
expect(stale).toMatchObject({ reason: 'drain_lease_held', leaseHeld: { lost: true } });
expect(await managedRows(engine, id)).toEqual({ cursors: 0, requests: 0 });
expect((await performSync(engine, plain(id))).status).toBe('first_sync');
}), 120_000);
// Outside-voice review: a job result is readable remotely (get_job), so the skip names the holder's kind, never its host, pid or id.
test('a sync job that finds the lease held skips with the public holder only', async () => isolated(async engine => {
const id = await fixture(engine);
const holder = hold(engine, id, 'cli');
await holder.ready;
try {
const ctx = { id: 9101, name: 'sync', data: { sourceId: id, pull: false }, attempts_made: 0, signal: new AbortController().signal, shutdownSignal: new AbortController().signal,
deadlineAtMs: Date.now() + 60_000, updateProgress: async () => undefined, updateTokens: async () => undefined, log: async () => undefined, isActive: async () => true,
readInbox: async () => [] } as unknown as MinionJobContext;
const skipped = await makeSyncHandler(engine)(ctx) as Record<string, unknown>;
expect(skipped).toMatchObject({ skipped: true, reason: 'sync_in_progress', holder: { kind: 'cli', host: null, pid: null, id: null } });
} finally { await holder.release(); }
}), 120_000);
test('the follower stops before the process watchdog: the lesser of its deadline and the hard deadline, less a margin', () => {
const now = Date.now();
registerRunDeadline({ atMs: now + 13 * 60_000, strict: true });
expect(followStopAt(now)).toBe(now + 13 * 60_000 - 30_000);
expect(followStopAt(now, now + 60_000)).toBe(now + 60_000 - 30_000);
expect(followStopAt(now, now + 20_000)).toBe(now + 10_000);
// The non-TTY window extends while progress arrives (a holder's commits count as this run's progress).
registerRunDeadline({ atMs: now + 60_000, strict: false, progressWindowMs: 10 * 60_000 });
noteForwardProgress();
expect(followStopAt(now)).toBeGreaterThan(now + 9 * 60_000);
});
// The cron recipe doctor prints (`gbrain sync --all --no-pull --hard-deadline 13m`) against a live serve holder: it follows the
// serve instead of refusing or draining beside it, and finishes the source once the serve's drain ends.
test('doctor\'s cron recipe follows a live serve holder and finishes the source; --no-delegate does not bypass the lease', async () => isolated(async engine => {
const id = await fixture(engine);
await engine.executeRaw("UPDATE sources SET config=jsonb_set(config,'{syncEnabled}','false'::jsonb) WHERE id<>$1", [id]);
const holder = hold(engine, id, 'serve');
await holder.ready;
const { runSync } = await import('../src/commands/sync.ts');
const lines: string[] = [], exits: number[] = [];
const origError = console.error, origLog = console.log, origExit = process.exit;
console.error = (...args: unknown[]) => { lines.push(args.map(String).join(' ')); };
console.log = (...args: unknown[]) => { lines.push(args.map(String).join(' ')); };
process.exit = ((code?: number) => { exits.push(code ?? 0); throw new Error(`process.exit(${code})`); }) as typeof process.exit;
try {
// The recipe as doctor prints it, plus --no-embed (this test brain has no embedding provider key) and --no-delegate.
const cron = runSync(engine, ['--all', '--no-pull', '--hard-deadline', '13m', '--no-embed', '--no-delegate']).catch((error: unknown) => error);
await new Promise(resolve => setTimeout(resolve, 3000));
expect(await managedRows(engine, id)).toEqual({ cursors: 0, requests: 0 });
await holder.release();
await cron;
} finally { console.error = origError; console.log = origLog; process.exit = origExit; }
expect({ exits, lines: exits.length ? lines : [] }).toEqual({ exits: [], lines: [] });
expect(await engine.getPage('c', { sourceId: id })).not.toBeNull();
}), 180_000);
// Older binaries take no lease, but their autopilot and sync jobs back off from a live `sync` job row (#6713). The
// query below is master's `liveSyncJob` before this change, verbatim.
test('a holder that is not a sync job is visible to an older binary\'s liveSyncJob, and only while it holds the lease', async () => isolated(async engine => {
const id = await fixture(engine);
const legacyLiveSyncJob = async () => (await engine.executeRaw<{ id: number }>(`SELECT id FROM minion_jobs WHERE name='sync' AND status='active' AND lock_until > now()
AND (COALESCE(data->>'sourceId', data->>'source_id')=$1 OR (COALESCE(data->>'sourceId', data->>'source_id') IS NULL
AND (data->>'repoPath' IS NULL AND $1='default' OR data->>'repoPath'=(SELECT local_path FROM sources WHERE id=$1)))) ORDER BY started_at LIMIT 1`, [id]))[0]?.id ?? null;
expect(await legacyLiveSyncJob()).toBeNull();
const holder = hold(engine, id, 'cli');
await holder.ready;
expect(await legacyLiveSyncJob()).not.toBeNull();
// This build reads the lease itself; the beacon is not a job it reports.
expect(await liveSyncJob(engine, id)).toBeNull();
await holder.release();
expect(await legacyLiveSyncJob()).toBeNull();
expect(await engine.executeRaw('SELECT 1 FROM minion_jobs WHERE queue=$1', [LEASE_BEACON_QUEUE])).toHaveLength(0);
// A sync job holder's record names the job, and liveSyncJob reports the job itself, not the record row.
const job = hold(engine, id, 'job');
await job.ready;
expect(await engine.executeRaw("SELECT data->'drain_lease_beacon'->>'id' AS id FROM minion_jobs WHERE queue=$1", [LEASE_BEACON_QUEUE])).toEqual([{ id: '4242' }]);
expect(await liveSyncJob(engine, id)).toBe(4242);
await job.release();
expect(await engine.executeRaw('SELECT 1 FROM minion_jobs WHERE queue=$1', [LEASE_BEACON_QUEUE])).toHaveLength(0);
}), 120_000);