Garry's Opinionated OpenClaw/Hermes Agent Brain
TypeScript
/**
* GBRA-130 A-2: one refused or still-pending transcript part never aborts the
* transcripts ingest, and through it the ChatGPT/Claude connector batch.
*
* 1. Protects: a session the coordinator refuses is reported per session with
* its typed code while the other sessions land; an accepted write that
* outlasts the wait is a pending session (never a failure, never synced);
* a rerun waits on that write instead of admitting a duplicate; a pending
* write that ends without committing is re-queued; a page deleted and
* retention-purged since its write is created again instead of answered
* with the old committed receipt (an identical rerun then admits nothing,
* even before that write's recovery record is cleared), while an owner
* purge still refuses it purged_content; the connector watermark never
* passes a pending or failed conversation.
* 2. Regression: the per-part catch treating anything but /too large|invalid
* byte sequence/ as a run abort, a random request id per run, or a
* committed receipt replayed for a page that no longer exists.
* 3. Existing coverage stubs the ingest result (connectors-ingest-failure) or
* pins the regex seam (transcripts-nul); none drives a real coordinator
* refusal or a real accepted write that commits after the wait.
* 4. No production seam: the bad states are a canonical file edited outside
* coordination and a paused owner (worktree draining).
*/
import { describe, expect, test } from 'bun:test';
import { appendFileSync, readFileSync, writeFileSync } from 'node:fs';
import { join } from 'node:path';
import type { BrainEngine } from '../../src/core/engine.ts';
import type { OperationContext } from '../../src/core/ops/contract.ts';
import { runTranscriptsIngest, type TranscriptsIngestResult } from '../../src/core/transcripts/ingest.ts';
import { runConnectorSync } from '../../src/core/connectors/sync.ts';
import { saveCredential } from '../../src/core/connectors/credentials.ts';
import { connectorSourceKey } from '../../src/core/connectors/config-keys.ts';
import { CHATGPT_BASE_URL } from '../../src/core/connectors/providers/chatgpt.ts';
import { waitForWrites } from '../../src/core/persistence/service.ts';
import { purgeDeletedPagesCoordinated } from '../../src/core/persistence/purge-deleted.ts';
import { installFaultHook } from '../../src/core/persistence/fault-points.ts';
import { operations } from '../../src/core/operations.ts';
import type { WriteRequest } from '../../src/core/persistence/model.ts';
import { chatgptHandler, newFixtureState, startFixture, type FixtureConversation, type FixtureState } from '../fixtures/connectors/fixture-server.ts';
import { expectItemFailure, expectOthersUnchanged, expectUnitNotAborted, managedBrain, pathologicalArms, runScopedErrors, snapshotItems } from './harness.ts';
const T0 = 1_786_080_000;
const message = (role: string, t: number, text: string) => ({ author: { role }, create_time: t, content: { content_type: 'text', parts: [text] } });
const exportConversation = (id: string, t: number, text: string) => ({ title: `Conversation ${id}`, create_time: t, update_time: t + 15,
conversation_id: id, current_node: 'n2', mapping: { root: { id: 'root', parent: null, children: ['n1'], message: null },
n1: { id: 'n1', parent: 'root', children: ['n2'], message: message('user', t + 5, text) },
n2: { id: 'n2', parent: 'n1', children: [], message: message('assistant', t + 15, `An answer to ${text}.`) } } });
function exportFile(root: string, texts: Record<string, string>): string {
const path = join(root, '..', 'export.json');
writeFileSync(path, JSON.stringify(Object.entries(texts).map(([id, text], i) => exportConversation(id, T0 + i * 1000, text))));
return path;
}
const ingest = (engine: BrainEngine, ctx: OperationContext, path: string, writeWaitMs?: number) =>
runTranscriptsIngest(engine, { paths: [path], sourceId: 'default', format: 'chatgpt', context: writeWaitMs === undefined ? ctx : { ...ctx, writeWaitMs } });
const sessions = (r: TranscriptsIngestResult) => new Map(r.files.flatMap(f => f.sessions).map(s => [s.sessionId, s]));
const slugOf = async (engine: BrainEngine, id: string) => (await engine.executeRaw<{ slug: string; source_path: string }>(
"SELECT slug, source_path FROM pages WHERE source_id='default' AND frontmatter->'transcript_import'->>'session_id' = $1 AND deleted_at IS NULL ORDER BY slug", [id]))[0]!;
const putPageRequests = async (engine: BrainEngine) => engine.executeRaw<WriteRequest>(
"SELECT * FROM persistence_requests WHERE source_id='default' AND operation='put_page' ORDER BY sequence");
/** Pauses or resumes the canonical owner: a draining worktree admits writes and publishes none. */
async function owner(engine: BrainEngine, state: 'draining' | 'active'): Promise<void> {
await engine.executeRaw('UPDATE persistence_brain SET enabled=false WHERE singleton=1');
await engine.executeRaw("UPDATE persistence_worktrees SET state=$1 WHERE id IN (SELECT worktree_id FROM persistence_source_bindings WHERE source_id='default')", [state]);
await engine.executeRaw('UPDATE persistence_brain SET enabled=true WHERE singleton=1');
}
async function settle(engine: BrainEngine): Promise<WriteRequest[]> {
const rows = await engine.executeRaw<WriteRequest>("SELECT * FROM persistence_requests WHERE source_id='default' ORDER BY sequence");
return waitForWrites(engine, rows, { engine: engine.kind }, 30_000);
}
/** The bad state: a conversation page whose canonical file carries an edit made outside coordinated publication. */
async function editOutsideCoordination(engine: BrainEngine, root: string, id: string): Promise<() => void> {
const file = join(root, (await slugOf(engine, id)).source_path);
const original = readFileSync(file, 'utf8');
appendFileSync(file, '\nAn edit made outside coordinated publication.\n');
return () => writeFileSync(file, original);
}
for (const { backend, databaseUrl } of pathologicalArms()) {
describe(`A-2 transcripts ingest (${backend})`, () => {
test('item: a session the coordinator refuses is reported with its code and the other sessions land', async () => managedBrain(async ({ engine, ctx, root }) => {
await ingest(engine, ctx, exportFile(root, { 'c-1': 'one', 'c-2': 'two', 'c-3': 'three' }));
const restore = await editOutsideCoordination(engine, root, 'c-2');
const before = await snapshotItems(engine, 'default');
const path = exportFile(root, { 'c-1': 'one edited', 'c-2': 'two edited', 'c-3': 'three edited' });
const result = await expectUnitNotAborted(() => ingest(engine, ctx, path));
const c2 = sessions(result).get('c-2')!;
expectItemFailure({ key: c2.sessionId, code: c2.code }, { key: 'c-2', code: 'source_changed' });
expect(result).toMatchObject({ sessionsErrored: 1, sessionsImported: 2, cleanScan: false });
const [one, three] = [await slugOf(engine, 'c-1'), await slugOf(engine, 'c-3')];
expectOthersUnchanged(before, await snapshotItems(engine, 'default'), { expectProgress: [one.slug, three.slug], progressFields: ['content_hash'] });
// Recovery: once the canonical file is back to its committed bytes, the same rerun lands the session.
restore();
const again = await ingest(engine, ctx, path);
expect(again).toMatchObject({ sessionsErrored: 0, sessionsImported: 3, cleanScan: true });
}, { databaseUrl }), 180_000);
test('pending: an accepted write that outlasts the wait stays pending under one request id and lands later without a duplicate admission', async () => managedBrain(async ({ engine, ctx, root }) => {
const path = exportFile(root, { 'c-1': 'one', 'c-2': 'two', 'c-3': 'three' });
await owner(engine, 'draining');
const first = await expectUnitNotAborted(() => ingest(engine, ctx, path, 50));
expect(first).toMatchObject({ sessionsPending: 3, sessionsErrored: 0, sessionsImported: 0, cleanScan: false });
const ids = [...sessions(first).values()].map(s => s.pending);
expect(ids.every(id => id?.length === 1)).toBe(true);
const second = await expectUnitNotAborted(() => ingest(engine, ctx, path, 50));
expect([...sessions(second).values()].map(s => s.pending)).toEqual(ids);
expect((await putPageRequests(engine)).length).toBe(3);
await owner(engine, 'active');
expect((await settle(engine)).map(row => row.state)).toEqual(['committed', 'committed', 'committed']);
const third = await ingest(engine, ctx, path);
expect(third).toMatchObject({ sessionsPending: 0, sessionsImported: 3, cleanScan: true, pages: { imported: 0, skipped: 3 } });
expect((await putPageRequests(engine)).length).toBe(3);
}, { databaseUrl }), 180_000);
test('recovery: a pending write that ends without committing is re-queued under a new request and lands', async () => managedBrain(async ({ engine, ctx, root }) => {
await ingest(engine, ctx, exportFile(root, { 'c-1': 'one' }));
const path = exportFile(root, { 'c-1': 'one edited' });
await owner(engine, 'draining');
const pending = await ingest(engine, ctx, path, 50);
expect(pending.sessionsPending).toBe(1);
const restore = await editOutsideCoordination(engine, root, 'c-1');
await owner(engine, 'active');
expect((await settle(engine)).map(row => row.state)).toEqual(['committed', 'conflict']);
restore();
const retried = await expectUnitNotAborted(() => ingest(engine, ctx, path));
expect(retried).toMatchObject({ sessionsPending: 0, sessionsErrored: 0, sessionsImported: 1, cleanScan: true });
expect((await putPageRequests(engine)).map(row => row.state)).toEqual(['committed', 'conflict', 'committed']);
}, { databaseUrl }), 180_000);
test('recovery: an identical re-ingest after the page was deleted and retention-purged admits a new creation instead of replaying the old receipt', async () => managedBrain(async ({ engine, ctx, root }) => {
const path = exportFile(root, { 'c-1': 'one' });
expect(await ingest(engine, ctx, path)).toMatchObject({ sessionsImported: 1, sessionsErrored: 0 });
const { slug } = await slugOf(engine, 'c-1');
const snapshot = await engine.readPageSnapshot(slug, { sourceId: 'default' });
await operations.find(op => op.name === 'delete_page')!.handler(ctx, { slug, expected_revision: snapshot!.revision, request_id: crypto.randomUUID() });
expect((await purgeDeletedPagesCoordinated(engine, 0)).slugs).toEqual([slug]);
expect(await engine.readPageSnapshot(slug, { sourceId: 'default', includeDeleted: true })).toBeNull();
const again = await expectUnitNotAborted(() => ingest(engine, ctx, path));
expect(again).toMatchObject({ sessionsImported: 1, sessionsErrored: 0, sessionsPending: 0, cleanScan: true });
expect((await slugOf(engine, 'c-1')).slug).toBe(slug);
expect((await putPageRequests(engine)).map(row => row.state)).toEqual(['committed', 'committed']);
const rerun = await ingest(engine, ctx, path);
expect(rerun).toMatchObject({ sessionsImported: 1, sessionsErrored: 0, pages: { imported: 0, skipped: 1 } });
expect((await putPageRequests(engine)).length).toBe(2);
}, { databaseUrl }), 180_000);
// Postgres publishes a single write as a group of one and hands the waiter its commit before clearing the recovery record.
if (backend === 'postgres') test('recovery: a rerun while the re-created write is committed but its recovery record is not yet cleared still admits nothing', async () => managedBrain(async ({ engine, ctx, root }) => {
const path = exportFile(root, { 'c-1': 'one' });
await ingest(engine, ctx, path);
const { slug } = await slugOf(engine, 'c-1');
const snapshot = await engine.readPageSnapshot(slug, { sourceId: 'default' });
await operations.find(op => op.name === 'delete_page')!.handler(ctx, { slug, expected_revision: snapshot!.revision, request_id: crypto.randomUUID() });
expect((await purgeDeletedPagesCoordinated(engine, 0)).slugs).toEqual([slug]);
// The re-creation's request row is held from its commit, so the post-commit recovery clear waits until the rerun has screened.
let release!: () => void;
const released = new Promise<void>(resolve => { release = resolve; });
let held: Promise<unknown> | undefined;
const executeRaw = engine.executeRaw;
try {
installFaultHook(async (point, detail) => {
if (point !== 'publication:after_commit' || detail.operation !== 'put_page' || held) return;
let locked!: () => void;
const acquired = new Promise<void>(resolve => { locked = resolve; });
held = engine.transaction(async tx => {
await tx.executeRaw('SELECT 1 FROM persistence_requests WHERE request_id=$1::uuid FOR UPDATE', [detail.requestId]);
locked();
await released;
});
await acquired;
});
await ingest(engine, ctx, path);
expect((await putPageRequests(engine)).map(row => [row.state, row.recovery !== null])).toEqual([['committed', false], ['committed', true]]);
engine.executeRaw = (async function (this: BrainEngine, ...args: Parameters<BrainEngine['executeRaw']>) {
const rows = await executeRaw.apply(this, args);
if (args[0].includes('intent IS NOT NULL AND NOT compacted')) release();
return rows;
}) as BrainEngine['executeRaw'];
const rerun = await ingest(engine, ctx, path);
expect(rerun).toMatchObject({ sessionsImported: 1, sessionsErrored: 0, pages: { imported: 0, skipped: 1 } });
expect((await putPageRequests(engine)).length).toBe(2);
} finally {
engine.executeRaw = executeRaw;
installFaultHook(undefined);
release();
await held;
}
}, { databaseUrl }), 180_000);
test('item: an identical re-ingest after the owner purged the page is refused purged_content, never replayed or re-created', async () => managedBrain(async ({ engine, ctx, root }) => {
const path = exportFile(root, { 'c-1': 'one' });
await ingest(engine, ctx, path);
const { slug } = await slugOf(engine, 'c-1');
const snapshot = await engine.readPageSnapshot(slug, { sourceId: 'default' });
await operations.find(op => op.name === 'delete_page')!.handler(ctx, { slug, purge: true, expected_revision: snapshot!.revision, request_id: crypto.randomUUID() });
await purgeDeletedPagesCoordinated(engine, 0);
const again = await expectUnitNotAborted(() => ingest(engine, ctx, path));
const c1 = sessions(again).get('c-1')!;
expectItemFailure({ key: c1.sessionId, code: c1.code }, { key: 'c-1', code: 'purged_content' });
expect(again).toMatchObject({ sessionsErrored: 1, sessionsImported: 0 });
expect((await putPageRequests(engine)).map(row => [row.state, row.error_code])).toEqual([['committed', null], ['failed', 'purged_content']]);
expect(await engine.readPageSnapshot(slug, { sourceId: 'default', includeDeleted: true })).toBeNull();
const rerun = await ingest(engine, ctx, path);
expect(sessions(rerun).get('c-1')?.code).toBe('purged_content');
expect(await engine.readPageSnapshot(slug, { sourceId: 'default', includeDeleted: true })).toBeNull();
}, { databaseUrl }), 180_000);
for (const injected of runScopedErrors()) {
test(`run: a run-scoped failure inside a session stops the ingest (${injected.name})`, async () => managedBrain(async ({ engine, ctx, root }) => {
const path = exportFile(root, { 'c-1': 'one', 'c-2': 'two', 'c-3': 'three' });
const error = injected.error();
let calls = 0;
const failing = new Proxy(engine, { get(target, key) {
const value = (target as unknown as Record<string | symbol, unknown>)[key];
if (key === 'readPageSnapshot') return async (...args: unknown[]) => { if (++calls === 2) throw error; return (value as (...a: unknown[]) => unknown).apply(target, args); };
return typeof value === 'function' ? (value as (...a: unknown[]) => unknown).bind(target) : value;
} });
await expect(ingest(failing, ctx, path)).rejects.toBe(error);
}, { databaseUrl }), 120_000);
}
test('breaker: when most sessions are refused the ingest stops item_failures_systemic', async () => managedBrain(async ({ engine, ctx, root }) => {
const ids = ['c-1', 'c-2', 'c-3', 'c-4', 'c-5', 'c-6'];
await ingest(engine, ctx, exportFile(root, Object.fromEntries(ids.map(id => [id, id]))));
for (const id of ids.slice(0, 5)) await editOutsideCoordination(engine, root, id);
await expect(ingest(engine, ctx, exportFile(root, Object.fromEntries(ids.map(id => [id, `${id} edited`])))))
.rejects.toMatchObject({ code: 'item_failures_systemic' });
}, { databaseUrl }), 180_000);
});
describe(`A-2 connector sync (${backend})`, () => {
const NOW_MS = (T0 + 100_000) * 1000;
const conv = (id: string, updateTime: number, text = `question ${id}`): FixtureConversation => ({ id, title: `Title ${id}`,
createTime: updateTime - 100, updateTime, turns: [{ role: 'user', text }, { role: 'assistant', text: `answer ${id}` }] });
const sync = async (engine: BrainEngine, ctx: OperationContext, state: FixtureState, writeWaitMs?: number,
runIngest: typeof runTranscriptsIngest = (eng, opts) => runTranscriptsIngest(eng, { ...opts, context: writeWaitMs === undefined ? ctx : { ...ctx, writeWaitMs } })) => {
saveCredential({ provider: 'chatgpt', strategy: 'browser-session', cookie: 'sessionKey=fixture', savedAt: new Date(0).toISOString() });
const srv = startFixture(chatgptHandler(state));
try {
return await runConnectorSync(engine, { provider: 'chatgpt', sourceId: 'default', deps: {
fetchImpl: (url, init) => fetch(url.replace(CHATGPT_BASE_URL, srv.baseUrl), init), now: () => NOW_MS, sleep: () => Promise.resolve(), runIngest } });
} finally { srv.stop(); }
};
const ledger = async (engine: BrainEngine, which: 'synced' | 'failed') => JSON.parse(await engine.getConfig(connectorSourceKey('chatgpt', 'default', which)) ?? '{}');
test('item: a conversation whose ingest is refused is counted toward quarantine and the batch siblings sync', async () => managedBrain(async ({ engine, ctx, root }) => {
const state = newFixtureState([conv('c-1', T0 + 10), conv('c-2', T0 + 20), conv('c-3', T0 + 30)]);
await sync(engine, ctx, state);
const restore = await editOutsideCoordination(engine, root, 'c-2');
state.conversations = [conv('c-1', T0 + 40, 'one edited'), conv('c-2', T0 + 50, 'two edited'), conv('c-3', T0 + 60, 'three edited')];
const before = await snapshotItems(engine, 'default');
const results = [];
for (let run = 0; run < 3; run++) results.push(await expectUnitNotAborted(() => sync(engine, ctx, state)));
const failed = await ledger(engine, 'failed');
expectItemFailure({ key: Object.keys(failed)[0], code: failed['c-2']?.code }, { key: 'c-2', code: 'source_changed' });
expect(results.map(r => r.watermarkAdvancedTo)).toEqual([undefined, undefined, new Date((T0 + 60) * 1000).toISOString()]);
expect(results[2].quarantined).toEqual(['c-2']);
const iso = (t: number) => new Date(t * 1000).toISOString();
expect(await ledger(engine, 'synced')).toEqual({ 'c-1': iso(T0 + 40), 'c-2': iso(T0 + 20), 'c-3': iso(T0 + 60) });
const [one, three] = [await slugOf(engine, 'c-1'), await slugOf(engine, 'c-3')];
expectOthersUnchanged(before, await snapshotItems(engine, 'default'), { expectProgress: [one.slug, three.slug], progressFields: ['content_hash'] });
restore();
state.conversations[1] = conv('c-2', T0 + 70, 'two edited again');
const recovered = await sync(engine, ctx, state);
expect(recovered.unresolved).toBe(0);
expect(await ledger(engine, 'failed')).toEqual({});
}, { databaseUrl }), 240_000);
test('pending: a conversation whose write is still pending holds the watermark, is never marked synced, and lands later without a duplicate admission', async () => managedBrain(async ({ engine, ctx }) => {
const state = newFixtureState([conv('c-1', T0 + 10), conv('c-2', T0 + 20)]);
await owner(engine, 'draining');
const first = await expectUnitNotAborted(() => sync(engine, ctx, state, 50));
expect(first.status).toBe('partial');
expect([...first.pending].sort()).toEqual(['c-1', 'c-2']);
expect(first.watermarkAdvancedTo).toBeUndefined();
expect(await ledger(engine, 'synced')).toEqual({});
expect(await ledger(engine, 'failed')).toEqual({});
const second = await expectUnitNotAborted(() => sync(engine, ctx, state, 50));
expect(second.watermarkAdvancedTo).toBeUndefined();
expect((await putPageRequests(engine)).length).toBe(2);
await owner(engine, 'active');
await settle(engine);
const third = await sync(engine, ctx, state);
expect(third.pending).toEqual([]);
expect(third.watermarkAdvancedTo).toBe(new Date((T0 + 20) * 1000).toISOString());
expect(Object.keys(await ledger(engine, 'synced')).sort()).toEqual(['c-1', 'c-2']);
expect((await putPageRequests(engine)).map(row => row.state)).toEqual(['committed', 'committed']);
}, { databaseUrl }), 240_000);
test('recovery: a pending conversation write that ends in conflict is re-queued, never marked synced, and lands on the next run', async () => managedBrain(async ({ engine, ctx, root }) => {
const state = newFixtureState([conv('c-1', T0 + 10)]);
await sync(engine, ctx, state);
state.conversations = [conv('c-1', T0 + 20, 'one edited')];
await owner(engine, 'draining');
expect((await sync(engine, ctx, state, 50)).pending).toEqual(['c-1']);
const restore = await editOutsideCoordination(engine, root, 'c-1');
await owner(engine, 'active');
await settle(engine);
expect((await ledger(engine, 'synced'))['c-1']).toBe(new Date((T0 + 10) * 1000).toISOString());
restore();
const retried = await sync(engine, ctx, state);
expect(retried.watermarkAdvancedTo).toBe(new Date((T0 + 20) * 1000).toISOString());
expect((await ledger(engine, 'synced'))['c-1']).toBe(new Date((T0 + 20) * 1000).toISOString());
expect((await putPageRequests(engine)).map(row => row.state)).toEqual(['committed', 'conflict', 'committed']);
}, { databaseUrl }), 240_000);
for (const injected of runScopedErrors()) {
test(`run: a run-scoped failure inside the connector ingest stops the sync and records no failure (${injected.name})`, async () => managedBrain(async ({ engine, ctx }) => {
const state = newFixtureState([conv('c-1', T0 + 10), conv('c-2', T0 + 20)]);
const error = injected.error();
await expect(sync(engine, ctx, state, undefined, async () => { throw error; })).rejects.toBe(error);
expect(await ledger(engine, 'failed')).toEqual({});
expect(await ledger(engine, 'synced')).toEqual({});
expect(await engine.getConfig(connectorSourceKey('chatgpt', 'default', 'watermark_iso'))).toBeNull();
}, { databaseUrl }), 120_000);
}
});
}