Garry's Opinionated OpenClaw/Hermes Agent Brain
TypeScript
/**
* #5984 Phase 4.5 (Postgres only): an agent's page write from another process goes ahead of the queued sync
* groups of a draining managed sync that do not name its page, waits in order behind one that does, and a
* stream of writes still lets the drain publish a group after each one. `GBRAIN_SYNC_FOREGROUND_PRIORITY=0`
* restores the FIFO (and fails the first case).
*/
import { afterAll, beforeAll, expect, test } from 'bun:test';
import { randomUUID } from 'node:crypto';
import { existsSync, mkdtempSync, mkdirSync, readFileSync, rmSync, writeFileSync } from 'node:fs';
import { tmpdir } from 'node:os';
import { join } from 'node:path';
import { execFileSync } from 'node:child_process';
import { isolatedPersistencePostgres } from './helpers/persistence-postgres.ts';
import type { BrainEngine } from '../src/core/engine.ts';
import { claimWorktree } from '../src/core/persistence/ownership.ts';
import { performSync } from '../src/commands/sync/perform.ts';
import { disposePersistenceConsumer } from '../src/core/persistence/service.ts';
import { resetWriteSwitches } from '../src/core/persistence/switches.ts';
import { withEnv } from './helpers/with-env.ts';
import { installFaultHook } from '../src/core/persistence/fault-points.ts';
import { setClaimOwnerForTest } from '../src/core/persistence/claim-phase.ts';
import { laneClaimsInFlight } from '../src/core/persistence/sync-lanes.ts';
const home = mkdtempSync(join(tmpdir(), 'gbrain-fg-priority-'));
let engine: BrainEngine | undefined;
let databaseUrl = '';
let closePostgres: (() => Promise<void>) | undefined;
const sources: string[] = [];
function git(root: string, ...args: string[]): string { return execFileSync('git', ['-C', root, ...args], { encoding: 'utf8', stdio: ['ignore', 'pipe', 'pipe'] }).trim(); }
const pad = (i: number) => String(i).padStart(3, '0');
async function fixture(e: BrainEngine, count: number) {
const id = `fgp-${randomUUID().replace(/-/g, '').slice(0, 20)}`; sources.push(id);
const root = join(home, id); mkdirSync(join(root, 'notes'), { recursive: true }); git(root, 'init', '-q');
for (let i = 0; i < count; i++) writeFileSync(join(root, 'notes', `n${pad(i)}.md`), `---\ntitle: Note ${i}\n---\nA durable observation number ${i}.\n`);
git(root, 'add', '.'); git(root, '-c', 'user.name=Example', '-c', '[email protected]', 'commit', '-qm', 'notes');
await e.executeRaw('UPDATE persistence_brain SET enabled=false WHERE singleton=1');
await e.executeRaw('INSERT INTO sources(id,name,local_path,config) VALUES($1,$1,$2,\'{}\')', [id, root]);
await claimWorktree(e, id, root);
await e.executeRaw('UPDATE persistence_brain SET enabled=true WHERE singleton=1');
return id;
}
type Row = { id: string; request_id: string; created: number; grp: string | null; slug: string; sequence: string; state: string; kind: string | null; completed: number | null };
const rows = (e: BrainEngine, source: string) => e.executeRaw<Row>(`SELECT id::text,request_id::text,intent->>'group' AS grp,slug,sequence::text,state,intent->>'kind' AS kind,
(extract(epoch FROM created_at)*1000)::float8 AS created,
(extract(epoch FROM completed_at)*1000)::float8 AS completed FROM persistence_requests WHERE source_id=$1 ORDER BY sequence`, [source]);
const until = async (check: () => Promise<boolean>, ms = 60_000) => {
const deadline = Date.now() + ms;
while (!await check()) { if (Date.now() > deadline) throw new Error('timed out'); await Bun.sleep(10); }
};
/** Records every request the drain's process claims (FIFO, lane head or group follower), with the time. */
function recordClaims(e: BrainEngine): { claims: Map<string, number>; restore: () => void } {
const claims = new Map<string, number>();
const execute = e.executeRaw;
// When each claim transaction chose its row: a claim already deciding when the write was admitted is not an overtake.
const chose = new WeakMap<object, number>();
e.executeRaw = async function (this: BrainEngine, sql: string, params?: unknown[], opts?: unknown) {
const sent = Date.now();
if (/FOR UPDATE OF r SKIP LOCKED/.test(sql)) chose.set(this, sent);
const out = await execute.call(this, sql, params, opts as never) as Array<{ id?: string; state?: string }>;
if (/^UPDATE persistence_requests( r)? SET state='running'/.test(sql.trim())) for (const row of out) if (row.id && row.state === 'running') claims.set(String(row.id), chose.get(this) ?? sent);
return out as never;
} as BrainEngine['executeRaw'];
return { claims, restore: () => { e.executeRaw = execute; } };
}
function writer(source: string, slugs: string[], intervalMs = 0, go?: string, preparingHoldMs = 0, ready?: string) {
const out = join(home, `worker-${randomUUID()}.json`);
const child = Bun.spawn([process.execPath, join(import.meta.dir, 'fixtures', 'foreground-put-page-worker.ts')], { stdin: 'ignore', stdout: 'pipe', stderr: 'pipe',
env: { ...process.env, GBRAIN_HOME: home, WORKER_DATABASE_URL: databaseUrl, WORKER_SOURCE: source, WORKER_SLUGS: slugs.join(',') || '-', WORKER_INTERVAL_MS: String(intervalMs), WORKER_OUT: out, ...(go ? { WORKER_GO: go } : {}), ...(ready ? { WORKER_READY: ready } : {}),
...(preparingHoldMs ? { WORKER_PREPARING_HOLD_MS: String(preparingHoldMs) } : {}) } });
return async () => {
const [stderr, code] = await Promise.all([new Response(child.stderr).text(), child.exited]);
if (code !== 0) throw new Error(`writer exited ${code}: ${stderr.slice(-2000)}`);
return JSON.parse(readFileSync(out, 'utf8')) as Array<{ slug: string; state: string; error?: string; submitted: number; returned: number; admitted?: number; held?: [number, number] }>;
};
}
beforeAll(async () => {
if (!process.env.DATABASE_URL) return;
const pg = await isolatedPersistencePostgres(process.env.DATABASE_URL, 'instance', 12);
engine = pg.engine; closePostgres = pg.close; databaseUrl = pg.databaseUrl;
}, 120_000);
afterAll(async () => {
if (engine) await withEnv({ GBRAIN_HOME: home }, async () => {
await disposePersistenceConsumer(engine!); await engine!.executeRaw('UPDATE persistence_brain SET enabled=false WHERE singleton=1');
for (const id of sources) await engine!.executeRaw('DELETE FROM sources WHERE id=$1', [id]);
});
await closePostgres?.();
rmSync(home, { recursive: true, force: true });
});
/**
* Starts a drain, waits until it has sync groups queued that nobody claimed, then writes `slug` from another
* process. Returns the queued sync rows admitted before the write, the drain's claims and the final rows.
*/
async function writeDuringDrain(lanes: number, slug: (queued: Row[]) => string, laneHold = 0, preparingHoldMs = 0, force = false, inFlight = false) {
const e = engine!, source = await fixture(e, 64);
const recorder = recordClaims(e);
// Each sync page holds its group open a little before commit, so groups stay queued while the writer starts. With
// `laneHold`, once the write is sent each lane group that has applied its pages holds before its commit turn (no
// counter lock held) until the write commits or `laneHold` ms pass, and `beside` records, for each foreground
// publication in this process, whether a lane group was then open.
let laneOpen = 0, armed = false, holdStarts = false;
const beside: boolean[] = [];
const written = Promise.withResolvers<void>();
// Without `laneHold`, from the moment an unstarted sync group is seen until `before` is read, the drain's process claims
// nothing new (`consumer:claiming`): no lane takes another group, while the groups the lanes hold run on and commit, so the
// groups queued ahead of the write stay queued however long the write takes to be admitted. A claim already past that
// point when the test parks can still take a queued group (at the first commit only one is queued), so the test waits
// for those claims to finish, counts again and, if no group is left, lets the claims go and waits for the next.
// (Parking a lane at `lane:applied` of its next group instead let each lane past that point claim one more group
// first: with as many groups queued as lanes, both could start before the write was admitted.)
let gate = Promise.withResolvers<void>(), parkedClaims = 0;
// `force` (forced probe of that race): the write is admitted only after every group running when the test parked has
// committed and the lanes have had their chance to claim the queued ones, and the drain admits no new group meanwhile
// (`sync:after_freeze`), so only the parking keeps groups queued ahead of the write.
const runningAtReady = new Set<string>();
// With `preparingHoldMs`, whichever process claims the write holds its preparation that long, and this process stamps
// its claims with another process's owner, so either way the write is claimed elsewhere and unpublished meanwhile.
let heldHere: [number, number] | undefined;
if (preparingHoldMs) setClaimOwnerForTest({ kind: 'mcp', pid: 1, version: 'test', nonce: 'another-process' });
// `inFlight` (forced probe of that claim): once this process has claimed a group, each lane head claim waits inside its
// claim transaction, past the parking point, until the test parks, and the test parks only while one waits.
let parked = Promise.withResolvers<void>(), pending = 0;
const recorded = e.executeRaw;
if (inFlight) e.executeRaw = async function (this: BrainEngine, sql: string, params?: unknown[], opts?: unknown) {
if (!holdStarts && recorder.claims.size > 0 && sql.includes("r.intent->>'lane'=$3")) {
pending++;
try { await parked.promise; } finally { pending--; }
}
return recorded.call(this, sql, params, opts as never) as never;
} as BrainEngine['executeRaw'];
installFaultHook(async (point, detail) => {
if (!laneHold && holdStarts && point === 'consumer:claiming') {
parkedClaims++;
try { await Promise.race([gate.promise, Bun.sleep(30_000)]); } finally { parkedClaims--; }
}
if (preparingHoldMs && point === 'consumer:preparing' && detail.operation === 'put_page' && !heldHere) {
const start = Date.now();
await Bun.sleep(preparingHoldMs);
heldHere = [start, Date.now()];
}
if (detail.sourceId !== source) return;
if (point === 'publication:before_commit' && detail.operation !== 'submit_job') beside.push(laneOpen > 0);
if (point === 'publication:after_commit' && detail.operation !== 'submit_job') written.resolve();
if (laneHold && armed && point === 'lane:applied') {
laneOpen++;
try { await Promise.race([written.promise, Bun.sleep(laneHold)]); } finally { laneOpen--; }
}
if (force && holdStarts && point === 'sync:after_freeze') await Promise.race([gate.promise, Bun.sleep(30_000)]);
if (!laneHold && point === 'publication:before_commit' && detail.operation === 'submit_job') await Bun.sleep(150);
});
try {
const drain = performSync(e, { sourceId: source, noPull: true, noEmbed: true, noExtract: true, drain: true, lanes });
let queued: Row[] = [];
// The writer process starts now and writes (the slug the go file names) once a sync group is queued that nobody has started.
const go = join(home, `go-${randomUUID()}`);
const finish = writer(source, [], 0, go, preparingHoldMs);
const unstarted = (all: Row[]) => {
const begun = new Set(all.filter(row => row.state !== 'queued').map(row => row.request_id));
return all.filter(row => row.state === 'queued' && row.kind?.startsWith('managed_sync_') && !(row.grp && begun.has(row.grp)) && !begun.has(row.request_id));
};
await until(async () => {
let all = await rows(e, source);
queued = unstarted(all);
// With lanes the drain admits groups ahead; without them only the publishing group is out, so the write comes while one runs.
const ready = all.some(row => row.state === 'committed') && (!inFlight || pending > 0) && (lanes > 1 ? queued.length > 0 : all.some(row => row.state === 'running' && row.kind?.startsWith('managed_sync_')));
if (!ready || lanes === 1) return ready;
holdStarts = true;
parked.resolve();
if (!laneHold) {
await until(async () => laneClaimsInFlight() === parkedClaims);
all = await rows(e, source);
queued = unstarted(all);
if (!queued.length) {
holdStarts = false;
gate.resolve();
gate = Promise.withResolvers<void>();
parked = Promise.withResolvers<void>();
return false;
}
}
for (const row of all) if (row.state === 'running' && row.kind?.startsWith('managed_sync_')) runningAtReady.add(row.grp ?? row.request_id);
return true;
});
const target = slug(queued.length ? queued : await rows(e, source).then(all => all.filter(row => row.state === 'queued')));
armed = true;
// With `laneHold`, the write is sent while a lane group holds the worktree lease, so only the drain's process can publish it.
if (laneHold) await until(async () => laneOpen > 0);
if (force) {
const ids = new Set(queued.map(row => row.grp ?? row.request_id));
await until(async () => (await rows(e, source)).every(row => !runningAtReady.has(row.grp ?? row.request_id) || !['queued', 'running'].includes(row.state)));
const deadline = Date.now() + 1000;
while (Date.now() < deadline && (await rows(e, source)).some(row => ids.has(row.request_id) && row.state === 'queued')) await Bun.sleep(10);
}
writeFileSync(go, target);
let before: Row[] = [];
await until(async () => {
const all = await rows(e, source);
const put = all.find(row => row.slug === target && !row.kind?.startsWith('managed_sync_'));
if (!put) return false;
// Queued sync rows admitted before the write whose group has not started (a started group's followers go with it).
const started = new Set(all.filter(row => row.state !== 'queued').map(row => row.request_id));
before = all.filter(row => row.kind?.startsWith('managed_sync_') && row.state === 'queued' && BigInt(row.sequence) < BigInt(put.sequence)
&& !(row.grp && started.has(row.grp)));
return true;
});
gate.resolve();
const [written] = await finish();
const result = await drain;
return { source, target, before, written: written!, result, claims: recorder.claims, final: await rows(e, source), beside, held: heldHere ?? written!.held };
} finally { gate.resolve(); e.executeRaw = recorded; recorder.restore(); installFaultHook(undefined); setClaimOwnerForTest(undefined); }
}
for (const [lanes, force, inFlight] of [[2, false, false], [1, false, false], [2, true, false], [2, true, true]] as const) {
test(`a write to a page no queued group names commits before another sync group starts (lanes ${lanes})${force ? ', forced probe: the lanes finish their groups before the write is admitted' : ''}${inFlight ? ', and a claim is past the parking point when the test parks' : ''}`, async () => withEnv({ GBRAIN_HOME: home, GBRAIN_SYNC_BULK_SIZE: '4' }, async () => {
if (!engine) return;
resetWriteSwitches();
const run = await writeDuringDrain(lanes, () => `notes/foreground-${lanes}${force ? '-forced' : ''}${inFlight ? '-in-flight' : ''}`, 0, 0, force, inFlight);
expect(run.written.state).toBe('committed');
expect(run.result.status).toBe('first_sync');
// With lanes, groups admitted ahead were queued when the write arrived; it still went first.
if (lanes > 1) expect(run.before.length).toBeGreaterThan(0);
const put = run.final.find(row => row.slug === run.target)!;
// No sync group started (its head chosen) after the write's admission committed and before the write was claimed
// (with lanes, a claimed write publishes beside the lane groups, so they may start again) or, without lanes,
// committed. `created` is the admission transaction's start: a claim choosing its row before the commit cannot see the write.
expect(run.written.admitted).toBeGreaterThanOrEqual(put.created - 5);
const windowEnd = lanes > 1 ? run.claims.get(put.id) ?? put.completed! : put.completed!;
const heads = run.final.filter(row => row.kind?.startsWith('managed_sync_') && (row.grp ?? row.request_id) === row.request_id);
const early = heads.filter(row => { const claimed = run.claims.get(row.id) ?? Infinity; return claimed >= run.written.admitted! && claimed < windowEnd; }).map(row => row.slug);
expect(early).toEqual([]);
expect(run.final.filter(row => row.kind?.startsWith('managed_sync_import')).every(row => row.state === 'committed')).toBe(true);
}), 300_000);
}
test('a write the writer\'s own process claimed but cannot publish beside the lanes still holds back new lane heads', async () => withEnv({ GBRAIN_HOME: home, GBRAIN_SYNC_BULK_SIZE: '4' }, async () => {
if (!engine) return;
resetWriteSwitches();
// The write is claimed by a process other than the drain's (the writer's own, or the drain's stamped as another's) and
// held in preparation for 1.5 s. No lane head may be chosen while that claim is held.
const run = await writeDuringDrain(2, () => 'notes/claimed-elsewhere', 0, 1500);
expect(run.written.state).toBe('committed');
expect(run.result.status).toBe('first_sync');
expect(run.before.length).toBeGreaterThan(0);
const [start, end] = run.held!;
const heads = run.final.filter(row => row.kind?.startsWith('managed_sync_') && (row.grp ?? row.request_id) === row.request_id);
expect(heads.filter(row => { const claimed = run.claims.get(row.id) ?? Infinity; return claimed >= start && claimed < end; }).map(row => row.slug)).toEqual([]);
}), 300_000);
test('a write publishes beside the running lane groups instead of waiting for them, and the lanes keep starting groups', async () => withEnv({ GBRAIN_HOME: home, GBRAIN_SYNC_BULK_SIZE: '4' }, async () => {
if (!engine) return;
resetWriteSwitches();
const run = await writeDuringDrain(2, () => 'notes/beside-lanes', 20_000);
expect(run.written.state).toBe('committed');
expect(run.result.status).toBe('first_sync');
const put = run.final.find(row => row.slug === run.target)!;
// The drain's process published the write while a lane group's transaction was open in it.
expect(run.beside).toEqual([true]);
expect(run.final.filter(row => row.kind?.startsWith('managed_sync_import')).every(row => row.state === 'committed')).toBe(true);
}), 300_000);
test('a write to a page a queued group names waits for that group and settles after it, in order', async () => withEnv({ GBRAIN_HOME: home, GBRAIN_SYNC_BULK_SIZE: '4' }, async () => {
if (!engine) return;
resetWriteSwitches();
const run = await writeDuringDrain(2, queued => queued.at(-1)!.slug);
expect(run.result.status).toBe('first_sync');
const sync = run.final.find(row => row.slug === run.target && row.kind === 'managed_sync_import')!;
const put = run.final.find(row => row.slug === run.target && row.kind !== 'managed_sync_import')!;
expect(sync.state).toBe('committed');
// The write was claimed after the import it names committed, so it met the imported page: a write that creates
// the page is refused because the page now exists, exactly as if it had arrived after the sync.
expect(run.written).toMatchObject({ state: 'error', error: expect.stringContaining('already exists') });
expect(['failed', 'conflict']).toContain(put.state);
expect(sync.completed!).toBeLessThan(put.completed!);
expect(run.claims.has(put.id) ? run.claims.get(put.id)! : put.completed!).toBeGreaterThanOrEqual(sync.completed! - 5);
const lastSync = Math.max(...run.final.filter(row => row.kind === 'managed_sync_import').map(row => row.completed!));
expect(put.completed!).toBeLessThanOrEqual(lastSync + 5_000);
}), 300_000);
test('a stream of writes from another process still lets the drain publish a group after each write', async () => withEnv({ GBRAIN_HOME: home, GBRAIN_SYNC_BULK_SIZE: '4' }, async () => {
if (!engine) return;
resetWriteSwitches();
const e = engine, source = await fixture(e, 64);
// The writer process boots and connects before the drain starts, so the stream begins right after the first group
// commits instead of whenever a cold process gets going (by then a fast drain can have committed most of its groups).
const go = join(home, `go-${randomUUID()}`), ready = join(home, `ready-${randomUUID()}`);
const finish = writer(source, [], 0, go, 0, ready);
await until(async () => existsSync(ready));
const drain = performSync(e, { sourceId: source, noPull: true, noEmbed: true, noExtract: true, drain: true, lanes: 2 });
await until(async () => (await rows(e, source)).some(row => row.state === 'committed' && row.kind === 'managed_sync_import'));
writeFileSync(go, Array.from({ length: 12 }, (_, i) => `notes/stream-${i}`).join(','));
const written = await finish();
const result = await drain;
expect(written.map(write => write.state)).toEqual(written.map(() => 'committed'));
expect(result.status).toBe('first_sync');
const final = await rows(e, source);
const writes = final.filter(row => row.slug.startsWith('notes/stream-')).map(row => row.completed!).sort((a, b) => a - b);
const groupCommits = final.filter(row => row.kind === 'managed_sync_import' && (row.grp ?? row.request_id) === row.request_id).map(row => row.completed!);
// The drain kept publishing while the writes kept coming: sync groups committed throughout the stream, about one
// per write (each foreground commit lets the drain start a group before the next write goes ahead).
const during = groupCommits.filter(at => at > writes[0]! && at < writes.at(-1)!).length;
expect(during).toBeGreaterThanOrEqual(Math.floor((writes.length - 1) / 2));
}), 300_000);