Garry's Opinionated OpenClaw/Hermes Agent Brain
TypeScript
import { afterAll, beforeAll, expect, test } from 'bun:test';
import { withGoogleAccount } from './helpers/connector-fixture.ts';
import { chmodSync, chownSync, mkdtempSync, readFileSync, rmSync, writeFileSync } from 'node:fs';
import { tmpdir } from 'node:os';
import { dirname, join } from 'node:path';
import type { BrainEngine } from '../src/core/engine.ts';
import { parseGitHubSourceConfig, runGitHubSync } from '../src/core/github-source.ts';
import { parseGoogleSourceConfig, runGoogleSync } from '../src/core/google/google-source.ts';
import { disposePersistenceConsumer, waitForWrite } from '../src/core/persistence/service.ts';
import type { WriteRequest } from '../src/core/persistence/model.ts';
import { beginConnectorSync } from '../src/core/persistence/connector-sync.ts';
import { compactWriteReceipts } from '../src/core/persistence/journal.ts';
import { purgeStaleCheckpoints } from '../src/core/op-checkpoint.ts';
import { withEnv } from './helpers/with-env.ts';
import { createConnectorFixture, options, json, googleConfig, githubConfig, contact, issueFixture, githubFetch, sourceCheckpoint, sourceCursor, cursorOf, connectorPendingSet, completeBaselineSweep, withPausedOwnerBudget } from './helpers/connector-fixture.ts';
const { engines, env, backends, source, boundSource, standaloneConnector, setup, teardown } = createConnectorFixture();
beforeAll(setup, 120_000);
afterAll(teardown);
test('explicit connector retry replaces a real storage failure without changing ordinary replay or the cursor', async () => {
if (process.env.GBRAIN_TEST_CONNECTOR_UNPRIVILEGED === '1') {
expect(process.getuid?.()).toBe(65534);
process.stdout.write('CONNECTOR_PERMISSION_UID=65534\n');
}
if (process.getuid?.() === 0) {
const childHome = mkdtempSync(join(tmpdir(), 'gbrain-connector-unprivileged-'));
chownSync(childHome, 65534, 65534);
try {
const child = Bun.spawn(['setpriv', '--reuid=65534', '--regid=65534', '--clear-groups', process.execPath,
'test', import.meta.path, '--test-name-pattern', '^explicit connector retry replaces a real storage failure without changing ordinary replay or the cursor$'], {
env: { PATH: process.env.PATH, HOME: childHome, GBRAIN_HOME: childHome, TMPDIR: childHome,
DATABASE_URL: process.env.DATABASE_URL, GBRAIN_TEST_ALLOW_DATABASE_URL: '1', GBRAIN_CI_DISABLE_TEST_ENV_FILE: '1',
GBRAIN_TEST_BACKEND: backends.includes('pglite') ? undefined : 'postgres',
GBRAIN_PGLITE_SNAPSHOT: process.env.GBRAIN_PGLITE_SNAPSHOT, GBRAIN_TEST_CONNECTOR_UNPRIVILEGED: '1' },
stdout: 'pipe', stderr: 'pipe',
});
const timer = setTimeout(() => child.kill('SIGKILL'), 90_000);
try {
const [stdout, stderr, exitCode] = await Promise.all([new Response(child.stdout).text(), new Response(child.stderr).text(), child.exited]);
expect({ stdout, stderr, exitCode }).toMatchObject({ exitCode: 0 });
expect(stdout).toContain('CONNECTOR_PERMISSION_UID=65534');
expect(stderr).toContain('1 pass');
expect(stderr).toContain('0 fail');
process.stdout.write('CONNECTOR_PERMISSION_UID=65534 child passed the complete EACCES case\n');
} finally { clearTimeout(timer); if (child.exitCode === null) child.kill('SIGKILL'); await child.exited; }
} finally { rmSync(childHome, { recursive: true, force: true }); }
return;
}
await withEnv(env, async () => {
for (const engine of engines) for (const compact of [false, true]) {
const f = await boundSource(engine, githubConfig);
let body = 'Initial retry fixture';
const run = (retryFailed = false) => runGitHubSync(engine, f.id, parseGitHubSourceConfig(githubConfig, f.dir),
{ ...options, retryFailed, githubItem: { repo: 'acme-example/app', number: 1, kind: 'issue' } },
async url => new URL(url).pathname.endsWith('/issues/1') ? json({ ...issueFixture, body }) : githubFetch()(url));
await completeBaselineSweep(engine, f.id, run);
await disposePersistenceConsumer(engine);
const checkpoint = await sourceCheckpoint(engine, f.id);
expect(checkpoint).toHaveLength(1);
const path = join(f.dir, 'gh/acme-example/app/1.md');
const initial = readFileSync(path, 'utf8');
body = 'Updated organization retry fixture';
chmodSync(dirname(path), 0o555);
try { await expect(run()).rejects.toMatchObject({ code: 'storage_error' }); }
finally { await disposePersistenceConsumer(engine); chmodSync(dirname(path), 0o755); }
const [failed] = await engine.executeRaw<WriteRequest>("SELECT * FROM persistence_requests WHERE source_id=$1 AND state='failed'", [f.id]);
expect(failed).toMatchObject({ error_code: 'storage_error', recovery: null });
expect(readFileSync(path, 'utf8')).toBe(initial);
expect(await sourceCursor(engine, f.id)).toEqual(cursorOf(checkpoint));
if (compact) {
await engine.executeRaw("UPDATE persistence_requests SET completed_at=now()-interval '31 days' WHERE id=$1::uuid", [failed.id]);
expect(await compactWriteReceipts(engine)).toBeGreaterThanOrEqual(1);
expect((await engine.executeRaw<WriteRequest>('SELECT * FROM persistence_requests WHERE id=$1::uuid', [failed.id]))[0]).toMatchObject({ compacted: true, intent: null });
}
const [immutable] = await engine.executeRaw<WriteRequest>('SELECT * FROM persistence_requests WHERE id=$1::uuid', [failed.id]);
await expect(run()).rejects.toMatchObject({ code: 'storage_error', writeRequest: { request_id: failed.request_id } });
expect((await run(true)).modified).toBe(1);
expect(readFileSync(path, 'utf8')).toContain(body);
expect((await engine.executeRaw<WriteRequest>('SELECT * FROM persistence_requests WHERE id=$1::uuid', [failed.id]))[0]).toEqual(immutable);
const retries = await engine.executeRaw<WriteRequest>("SELECT * FROM persistence_requests WHERE source_id=$1 AND intent->>'retryOf'=$2", [f.id, failed.request_id]);
expect(retries).toHaveLength(1);
expect(retries[0].state).toBe('committed');
expect(retries[0].request_id).not.toBe(failed.request_id);
const effects = await engine.executeRaw('SELECT id FROM persistence_effects WHERE source_id=$1 ORDER BY id', [f.id]);
expect((await run(true)).modified).toBe(0);
expect(await engine.executeRaw('SELECT id FROM persistence_effects WHERE source_id=$1 ORDER BY id', [f.id])).toEqual(effects);
}
});
}, 120_000);
const faultEngines = new WeakSet<BrainEngine>();
async function retryFixture(engine: BrainEngine, connector: 'google' | 'github', bound: boolean) {
if (!faultEngines.has(engine)) {
await engine.executeRaw('CREATE TABLE connector_retry_faults(source_id text PRIMARY KEY, checkpoint_key text, pages boolean DEFAULT false, checkpoints boolean DEFAULT false, retries boolean DEFAULT false)');
await engine.executeRaw(`CREATE FUNCTION connector_retry_storage_fault() RETURNS trigger LANGUAGE plpgsql AS $$ BEGIN
IF TG_TABLE_NAME='pages' THEN
IF EXISTS(SELECT 1 FROM connector_retry_faults f WHERE f.source_id=NEW.source_id AND f.pages) THEN
RAISE EXCEPTION 'Synthetic connector page storage failure' USING ERRCODE='58030';
END IF;
ELSIF NEW.op='managed-connector' AND EXISTS(SELECT 1 FROM connector_retry_faults f WHERE f.checkpoint_key=NEW.fingerprint AND f.checkpoints) THEN
RAISE EXCEPTION 'Synthetic connector checkpoint storage failure' USING ERRCODE='58030';
ELSIF NEW.op='managed-connector-retry' AND EXISTS(SELECT 1 FROM connector_retry_faults f WHERE f.checkpoint_key=NEW.completed_keys->0->>'checkpointKey' AND f.retries) THEN
RAISE EXCEPTION 'Synthetic connector approval storage failure' USING ERRCODE='58030';
END IF;
RETURN NEW;
END $$`);
await engine.executeRaw('CREATE TRIGGER connector_retry_page_fault BEFORE INSERT OR UPDATE ON pages FOR EACH ROW EXECUTE FUNCTION connector_retry_storage_fault()');
await engine.executeRaw('CREATE TRIGGER connector_retry_checkpoint_fault BEFORE INSERT OR UPDATE ON op_checkpoints FOR EACH ROW EXECUTE FUNCTION connector_retry_storage_fault()');
faultEngines.add(engine);
}
const config = connector === 'google' ? googleConfig : githubConfig;
const f = bound ? await boundSource(engine, config) : await source(engine, config);
let updated = false;
const calls: string[] = [];
const fetcher = async (url: string) => {
calls.push(url);
if (connector === 'google') return json(url.includes('/settings/sendAs') ? { sendAs: [] } : {
connections: [{ ...contact('first', 'First Example'), organizations: [{ name: updated ? 'Updated retry organization' : 'Initial retry organization' }] }],
nextSyncToken: updated ? 'contacts-after-retry' : 'contacts-before-retry',
});
const path = new URL(url).pathname;
const issue = { ...issueFixture, body: updated ? 'Updated retry organization' : 'Initial retry organization',
updated_at: updated ? '2026-01-03T00:00:00Z' : issueFixture.updated_at };
if (path.endsWith('/issues')) return json([issue]);
if (path.endsWith('/issues/1')) return json(issue);
return githubFetch()(url);
};
const cfg = connector === 'google' ? parseGoogleSourceConfig(config, f.dir) : parseGitHubSourceConfig(config, f.dir);
const run = (retryFailed = false) => connector === 'google'
? runGoogleSync(engine, f.id, cfg as ReturnType<typeof parseGoogleSourceConfig>, { ...options, retryFailed }, withGoogleAccount(fetcher))
: runGitHubSync(engine, f.id, cfg as ReturnType<typeof parseGitHubSourceConfig>, { ...options, retryFailed }, fetcher);
await completeBaselineSweep(engine, f.id, run);
await disposePersistenceConsumer(engine);
const checkpoint = await sourceCheckpoint(engine, f.id);
expect(checkpoint).toHaveLength(1);
const session = (await beginConnectorSync(engine, f.id, connector, cfg, options))!;
await engine.executeRaw('INSERT INTO connector_retry_faults(source_id,checkpoint_key) VALUES($1,$2)', [f.id, session.checkpointKey]);
updated = true;
return { ...f, config, cfg, run, calls, checkpoint, checkpointKey: session.checkpointKey };
}
test('explicit retries survive compaction for bound and database-only Google and GitHub imports without resetting API state', async () => withEnv(env, async () => {
for (const engine of engines) for (const connector of ['google', 'github'] as const) for (const bound of [false, true]) {
const f = await retryFixture(engine, connector, bound);
await engine.executeRaw('UPDATE connector_retry_faults SET pages=true WHERE source_id=$1', [f.id]);
await expect(f.run()).rejects.toMatchObject({ code: 'storage_error' });
await disposePersistenceConsumer(engine);
const [failed] = await engine.executeRaw<WriteRequest>("SELECT * FROM persistence_requests WHERE source_id=$1 AND state='failed'", [f.id]);
expect(failed.recovery).toBeNull();
expect(await sourceCursor(engine, f.id)).toEqual(cursorOf(f.checkpoint));
await engine.executeRaw("UPDATE persistence_requests SET completed_at=now()-interval '31 days' WHERE id=$1::uuid", [failed.id]);
await compactWriteReceipts(engine);
const [immutable] = await engine.executeRaw<WriteRequest>('SELECT * FROM persistence_requests WHERE id=$1::uuid', [failed.id]);
expect(immutable).toMatchObject({ compacted: true, intent: null });
await engine.executeRaw('UPDATE connector_retry_faults SET pages=false WHERE source_id=$1', [f.id]);
await expect(f.run()).rejects.toMatchObject({ code: 'storage_error', writeRequest: { request_id: failed.request_id } });
f.calls.length = 0;
expect((await f.run(true)).status).not.toBe('partial');
expect(f.calls.some(url => connector === 'google' ? url.includes('syncToken=contacts-before-retry')
: new URL(url).pathname.endsWith('/issues') && new URL(url).searchParams.has('since'))).toBe(true);
expect((await engine.executeRaw<WriteRequest>('SELECT * FROM persistence_requests WHERE id=$1::uuid', [failed.id]))[0]).toEqual(immutable);
const retries = await engine.executeRaw<WriteRequest>("SELECT * FROM persistence_requests WHERE source_id=$1 AND intent->>'retryOf'=$2", [f.id, failed.request_id]);
expect(retries).toHaveLength(1);
expect(retries[0].state).toBe('committed');
const before = await engine.executeRaw('SELECT id FROM persistence_effects WHERE source_id=$1 ORDER BY id', [f.id]);
expect((await f.run(true)).modified).toBe(0);
expect(await engine.executeRaw('SELECT id FROM persistence_effects WHERE source_id=$1 ORDER BY id', [f.id])).toEqual(before);
}
}), 120_000);
test('checkpoint storage failures require explicit retry after pages have already committed', async () => withEnv(env, async () => {
for (const engine of engines) for (const connector of ['google', 'github'] as const) for (const bound of [false, true]) {
const f = await retryFixture(engine, connector, bound);
await engine.executeRaw('UPDATE connector_retry_faults SET checkpoints=true WHERE source_id=$1', [f.id]);
await expect(f.run()).rejects.toMatchObject({ code: 'storage_error' });
await disposePersistenceConsumer(engine);
const [failed] = await engine.executeRaw<WriteRequest>("SELECT * FROM persistence_requests WHERE source_id=$1 AND state='failed'", [f.id]);
expect(failed.intent?.kind).toBe('connector_v2_checkpoint');
expect(await sourceCursor(engine, f.id)).toEqual(cursorOf(f.checkpoint));
await engine.executeRaw("UPDATE persistence_requests SET completed_at=now()-interval '31 days' WHERE id=$1::uuid", [failed.id]);
await compactWriteReceipts(engine);
const [immutable] = await engine.executeRaw<WriteRequest>('SELECT * FROM persistence_requests WHERE id=$1::uuid', [failed.id]);
expect(immutable).toMatchObject({ compacted: true, intent: null });
await engine.executeRaw('UPDATE connector_retry_faults SET checkpoints=false WHERE source_id=$1', [f.id]);
await expect(f.run()).rejects.toMatchObject({ code: 'storage_error', writeRequest: { request_id: failed.request_id, compacted: true } });
const effects = await engine.executeRaw('SELECT id FROM persistence_effects WHERE source_id=$1 ORDER BY id', [f.id]);
expect((await f.run(true)).status).not.toBe('partial');
expect(await sourceCheckpoint(engine, f.id)).not.toEqual(f.checkpoint);
expect(await engine.executeRaw('SELECT id FROM persistence_effects WHERE source_id=$1 ORDER BY id', [f.id])).toEqual(effects);
expect((await engine.executeRaw<WriteRequest>('SELECT * FROM persistence_requests WHERE id=$1::uuid', [failed.id]))[0]).toEqual(immutable);
expect(await engine.executeRaw("SELECT id FROM persistence_requests WHERE source_id=$1 AND state='committed' AND intent->>'retryOf'=$2", [f.id, failed.request_id])).toHaveLength(1);
}
}), 120_000);
test('checkpoint replay keeps its originally admitted page dependencies when another sweep has different receipts', async () => withEnv(env, async () => {
for (const engine of engines) {
const f = await boundSource(engine, githubConfig);
const cfg = parseGitHubSourceConfig(githubConfig, f.dir);
const first = (await beginConnectorSync(engine, f.id, 'github', cfg, options))!;
const second = (await beginConnectorSync(engine, f.id, 'github', cfg, options))!;
await first.importMarkdown('notes/first.md', '---\ntitle: First fixture\n---\nFirst frozen dependency.\n');
await second.importMarkdown('notes/second.md', '---\ntitle: Second fixture\n---\nSecond frozen dependency.\n');
await disposePersistenceConsumer(engine);
await engine.executeRaw("UPDATE persistence_worktrees SET state='draining' WHERE id=$1::uuid", [f.binding.worktree_id]);
const state = { cursor: 'same-logical-checkpoint' };
await expect(withPausedOwnerBudget(() => first.saveState(state, true))).rejects.toMatchObject({ code: 'write_pending' });
const [accepted] = await engine.executeRaw<WriteRequest>("SELECT * FROM persistence_requests WHERE source_id=$1 AND intent->>'kind'='connector_v2_checkpoint'", [f.id]);
const [dependency] = await engine.executeRaw<WriteRequest>("SELECT * FROM persistence_requests WHERE source_id=$1 AND slug='notes/first'", [f.id]);
expect(accepted.intent?.receipts).toEqual([dependency.id]);
await expect(withPausedOwnerBudget(() => second.saveState(state, true))).rejects.toMatchObject({ code: 'write_pending', writeRequest: { request_id: accepted.request_id } });
expect(await engine.executeRaw("SELECT id FROM persistence_requests WHERE source_id=$1 AND intent->>'kind'='connector_v2_checkpoint'", [f.id])).toHaveLength(1);
expect((await engine.executeRaw<WriteRequest>('SELECT * FROM persistence_requests WHERE id=$1::uuid', [accepted.id]))[0].intent?.receipts).toEqual([dependency.id]);
await disposePersistenceConsumer(engine);
await engine.executeRaw("UPDATE persistence_worktrees SET state='active' WHERE id=$1::uuid", [f.binding.worktree_id]);
expect((await waitForWrite(engine, accepted, { engine: engine.kind })).state).toBe('committed');
await second.saveState(state, true);
expect((await engine.executeRaw<WriteRequest>('SELECT * FROM persistence_requests WHERE id=$1::uuid', [accepted.id]))[0].intent?.receipts).toEqual([dependency.id]);
expect((await beginConnectorSync(engine, f.id, 'github', cfg, options))!.state({})).toEqual(state);
}
}), 120_000);
test('a retry pointer survives real purge and compaction without silently renewing another failed attempt', async () => withEnv(env, async () => {
for (const engine of engines) {
const f = await retryFixture(engine, 'google', false);
await engine.executeRaw('UPDATE connector_retry_faults SET pages=true WHERE source_id=$1', [f.id]);
await expect(f.run()).rejects.toMatchObject({ code: 'storage_error' });
await disposePersistenceConsumer(engine);
const [original] = await engine.executeRaw<WriteRequest>("SELECT * FROM persistence_requests WHERE source_id=$1 AND state='failed'", [f.id]);
await expect(f.run(true)).rejects.toMatchObject({ code: 'storage_error' });
await disposePersistenceConsumer(engine);
const failures = await engine.executeRaw<WriteRequest>("SELECT * FROM persistence_requests WHERE source_id=$1 AND state='failed' ORDER BY sequence", [f.id]);
expect(failures).toHaveLength(2);
expect(failures[1].intent?.retryOf).toBe(original.request_id);
await engine.executeRaw("UPDATE persistence_requests SET completed_at=now()-interval '31 days' WHERE source_id=$1 AND state='failed'", [f.id]);
await compactWriteReceipts(engine);
const immutable = await engine.executeRaw<WriteRequest>("SELECT * FROM persistence_requests WHERE source_id=$1 AND state='failed' ORDER BY sequence", [f.id]);
expect(immutable.every(row => row.compacted && row.intent === null)).toBe(true);
const pointers = await engine.executeRaw("SELECT * FROM op_checkpoints WHERE op='managed-connector-retry' AND completed_keys->0->>'checkpointKey'=$1", [f.checkpointKey]);
expect(pointers).toHaveLength(1);
await engine.executeRaw("UPDATE op_checkpoints SET updated_at=now()-interval '8 days' WHERE op='managed-connector-retry'");
await purgeStaleCheckpoints(engine);
await expect(f.run()).rejects.toMatchObject({ code: 'storage_error', writeRequest: { request_id: failures[1].request_id, compacted: true } });
expect(await engine.executeRaw("SELECT id FROM persistence_requests WHERE source_id=$1 AND state='failed'", [f.id])).toHaveLength(2);
await engine.executeRaw('UPDATE connector_retry_faults SET pages=false WHERE source_id=$1', [f.id]);
expect((await f.run(true)).modified).toBe(1);
expect(await engine.executeRaw<WriteRequest>("SELECT * FROM persistence_requests WHERE source_id=$1 AND state='failed' ORDER BY sequence", [f.id])).toEqual(immutable);
expect(await engine.executeRaw("SELECT id FROM persistence_requests WHERE source_id=$1 AND state='committed' AND intent->>'retryOf'=$2", [f.id, failures[1].request_id])).toHaveLength(1);
}
}), 120_000);
test('a pooled connection whose last statement failed after streaming a row does not stop the retry run before its sweep', async () => withEnv(env, async () => {
for (const engine of engines) {
if (engine.kind !== 'postgres') continue;
const f = await retryFixture(engine, 'google', false);
await engine.executeRaw('UPDATE connector_retry_faults SET pages=true WHERE source_id=$1', [f.id]);
await expect(f.run()).rejects.toMatchObject({ code: 'storage_error' });
await disposePersistenceConsumer(engine);
// Every pooled connection's last statement streams one row and then fails (as a cancel at consumer stop can).
await Promise.allSettled(Array.from({ length: 16 }, () =>
engine.executeRaw('SELECT g, 1 / (g - 2) AS v FROM (SELECT g FROM generate_series(1, 2) g, pg_sleep(0.2)) s')));
await expect(f.run(true)).rejects.toMatchObject({ code: 'storage_error', writeRequest: { state: 'failed' } });
await disposePersistenceConsumer(engine);
const failures = await engine.executeRaw<WriteRequest>("SELECT * FROM persistence_requests WHERE source_id=$1 AND state='failed' ORDER BY sequence", [f.id]);
expect(failures).toHaveLength(2);
expect(failures[1].intent?.retryOf).toBe(failures[0].request_id);
}
}), 120_000);
test('concurrent connector approvals and a restarted paused owner retain one pending replacement receipt', async () => withEnv(env, async () => {
for (const engine of engines) {
const f = await retryFixture(engine, 'github', true);
await engine.executeRaw('UPDATE connector_retry_faults SET pages=true WHERE source_id=$1', [f.id]);
await expect(f.run()).rejects.toMatchObject({ code: 'storage_error' });
await disposePersistenceConsumer(engine);
await engine.executeRaw('UPDATE connector_retry_faults SET pages=false WHERE source_id=$1', [f.id]);
const [original] = await engine.executeRaw<WriteRequest>("SELECT * FROM persistence_requests WHERE source_id=$1 AND state='failed'", [f.id]);
await engine.executeRaw("UPDATE persistence_worktrees SET state='draining' WHERE id=$1::uuid", [original.worktree_id]);
const sessions = await Promise.all([1, 2].map(() => beginConnectorSync(engine, f.id, 'github', f.cfg, { ...options, retryFailed: true })));
const results = await withPausedOwnerBudget(() => Promise.allSettled(sessions.map(session => session!.importMarkdown(original.intent!.sourcePath as string, original.intent!.content as string))));
const [retry] = await engine.executeRaw<WriteRequest>("SELECT * FROM persistence_requests WHERE source_id=$1 AND intent->>'retryOf'=$2", [f.id, original.request_id]);
expect(retry).toMatchObject({ state: 'queued' });
// #5600: a session that admitted or replayed the replacement reports accepted-pending progress; a racing approval refuses write_pending on the same receipt.
for (const result of results) {
if (result.status === 'fulfilled') expect(result.value).toMatchObject({ status: 'imported' });
else expect(result.reason).toMatchObject({ code: 'write_pending', writeRequest: { request_id: retry.request_id } });
}
await disposePersistenceConsumer(engine);
expect(await sourceCursor(engine, f.id)).toEqual(cursorOf(f.checkpoint));
const restarted = await standaloneConnector(engine, f, githubConfig, false, true);
expect(restarted.exitCode).toBe(1);
expect(restarted.stdout).not.toContain('CONNECTOR_FIXTURE_FETCH');
expect(JSON.parse(restarted.stdout.split('CONNECTOR_ERROR ')[1].trim())).toMatchObject({ code: 'write_pending', receipt: { request_id: retry.request_id } });
const count = await engine.executeRaw('SELECT id FROM persistence_requests WHERE source_id=$1 ORDER BY sequence', [f.id]);
expect(await withPausedOwnerBudget(() => f.run())).toMatchObject({ status: 'partial', reason: 'writer_pending' });
expect((await connectorPendingSet(engine, f.id)).map(entry => entry.requestId)).toEqual([retry.request_id]);
expect(await engine.executeRaw('SELECT id FROM persistence_requests WHERE source_id=$1 ORDER BY sequence', [f.id])).toEqual(count);
await disposePersistenceConsumer(engine);
await engine.executeRaw("UPDATE persistence_worktrees SET state='active' WHERE id=$1::uuid", [original.worktree_id]);
expect((await f.run(true)).status).not.toBe('partial');
expect((await engine.executeRaw<WriteRequest>('SELECT * FROM persistence_requests WHERE id=$1::uuid', [retry.id]))[0].state).toBe('committed');
expect((await engine.executeRaw<WriteRequest>('SELECT * FROM persistence_requests WHERE id=$1::uuid', [original.id]))[0]).toEqual(original);
expect(await engine.executeRaw("SELECT id FROM persistence_requests WHERE source_id=$1 AND intent->>'retryOf'=$2", [f.id, original.request_id])).toHaveLength(1);
}
}), 120_000);
test('concurrent database-only connector retry approvals admit one replacement on both engines', async () => withEnv(env, async () => {
for (const engine of engines) for (const connector of ['google', 'github'] as const) {
const f = await retryFixture(engine, connector, false);
await engine.executeRaw('UPDATE connector_retry_faults SET pages=true WHERE source_id=$1', [f.id]);
await expect(f.run()).rejects.toMatchObject({ code: 'storage_error' });
await disposePersistenceConsumer(engine);
await engine.executeRaw('UPDATE connector_retry_faults SET pages=false WHERE source_id=$1', [f.id]);
const [original] = await engine.executeRaw<WriteRequest>("SELECT * FROM persistence_requests WHERE source_id=$1 AND state='failed'", [f.id]);
const sessions = await Promise.all([1, 2].map(() => beginConnectorSync(engine, f.id, connector, f.cfg, { ...options, retryFailed: true })));
await Promise.all(sessions.map(session => session!.importMarkdown(original.intent!.sourcePath as string, original.intent!.content as string)));
const retries = await engine.executeRaw<WriteRequest>("SELECT * FROM persistence_requests WHERE source_id=$1 AND intent->>'retryOf'=$2", [f.id, original.request_id]);
expect(retries).toHaveLength(1);
expect(retries[0]).toMatchObject({ state: 'committed', worktree_id: null });
expect((await engine.executeRaw<WriteRequest>('SELECT * FROM persistence_requests WHERE id=$1::uuid', [original.id]))[0]).toEqual(original);
expect(await sourceCursor(engine, f.id)).toEqual(cursorOf(f.checkpoint));
expect(await engine.executeRaw("SELECT fingerprint FROM op_checkpoints WHERE op='managed-connector-retry' AND completed_keys->0->>'checkpointKey'=$1", [f.checkpointKey])).toHaveLength(1);
}
}), 120_000);
test('connector retry approval revalidates current and accepted grants plus source and owner changes', async () => withEnv(env, async () => {
for (const engine of engines) for (const change of ['current-grant', 'accepted-grant', 'config', 'source', 'owner', 'file'] as const) {
const f = await retryFixture(engine, 'github', true);
await engine.executeRaw('UPDATE connector_retry_faults SET pages=true WHERE source_id=$1', [f.id]);
await expect(f.run()).rejects.toMatchObject({ code: 'storage_error' });
await disposePersistenceConsumer(engine);
await engine.executeRaw('UPDATE connector_retry_faults SET pages=false WHERE source_id=$1', [f.id]);
const [original] = await engine.executeRaw<WriteRequest>("SELECT * FROM persistence_requests WHERE source_id=$1 AND state='failed'", [f.id]);
const [writer] = await engine.executeRaw<{ grant_ceiling: unknown }>('SELECT grant_ceiling FROM persistence_local_writers WHERE id=$1::uuid', [original.principal_id]);
const session = (await beginConnectorSync(engine, f.id, 'github', f.cfg, { ...options, retryFailed: true }))!;
const requests = await engine.executeRaw('SELECT id FROM persistence_requests WHERE source_id=$1 ORDER BY sequence', [f.id]);
await engine.executeRaw('UPDATE persistence_brain SET enabled=false WHERE singleton=1');
if (change === 'current-grant') await engine.executeRaw("UPDATE persistence_local_writers SET grant_ceiling=jsonb_set(grant_ceiling,'{scopes}','[]'::jsonb) WHERE id=$1::uuid", [original.principal_id]);
else if (change === 'accepted-grant') await engine.executeRaw("UPDATE persistence_requests SET authority=jsonb_set(authority,'{scopes}','[]'::jsonb) WHERE id=$1::uuid", [original.id]);
else if (change === 'config') await engine.executeRaw('UPDATE sources SET config=$2::text::jsonb WHERE id=$1', [f.id, JSON.stringify({ ...githubConfig, gh_repos: 'foreign-example/app' })]);
else if (change === 'source') {
await engine.executeRaw('DELETE FROM sources WHERE id=$1', [f.id]);
await engine.executeRaw('INSERT INTO sources(id,name,local_path,config) VALUES($1,$1,$2,$3::text::jsonb)', [f.id, f.dir, JSON.stringify(githubConfig)]);
} else if (change === 'owner') await engine.executeRaw('UPDATE persistence_worktrees SET owner_epoch=owner_epoch+1 WHERE id=$1::uuid', [original.worktree_id]);
else {
const path = join(f.dir, original.intent!.sourcePath as string);
writeFileSync(path, `${readFileSync(path, 'utf8')}\nOperator edit must not be overwritten by retry.\n`);
}
await engine.executeRaw('UPDATE persistence_brain SET enabled=true WHERE singleton=1');
try {
await expect(session.importMarkdown(original.intent!.sourcePath as string, original.intent!.content as string)).rejects.toMatchObject({
code: change.endsWith('grant') ? 'permission_denied' : 'source_changed',
});
if (change === 'current-grant' || change === 'config') {
f.calls.length = 0;
await expect(f.run(true)).rejects.toBeInstanceOf(Error);
expect(f.calls).toHaveLength(0);
}
expect(await engine.executeRaw('SELECT id FROM persistence_requests WHERE source_id=$1 ORDER BY sequence', [f.id])).toEqual(requests);
expect(await engine.executeRaw("SELECT fingerprint FROM op_checkpoints WHERE op='managed-connector-retry' AND completed_keys->0->>'checkpointKey'=$1", [f.checkpointKey])).toHaveLength(0);
} finally {
await disposePersistenceConsumer(engine);
await engine.executeRaw('UPDATE persistence_local_writers SET grant_ceiling=$2::text::jsonb WHERE id=$1::uuid', [original.principal_id, JSON.stringify(writer.grant_ceiling)]);
if (change === 'accepted-grant') await engine.executeRaw('UPDATE persistence_requests SET authority=$2::text::jsonb WHERE id=$1::uuid', [original.id, JSON.stringify(original.authority)]);
}
}
}), 120_000);
test('connector retry approval is atomic and a lost acknowledgement reuses the committed pointer', async () => withEnv(env, async () => {
for (const engine of engines) for (const bound of [false, true]) {
const f = await retryFixture(engine, 'google', bound);
await engine.executeRaw('UPDATE connector_retry_faults SET pages=true WHERE source_id=$1', [f.id]);
await expect(f.run()).rejects.toMatchObject({ code: 'storage_error' });
await disposePersistenceConsumer(engine);
const [original] = await engine.executeRaw<WriteRequest>("SELECT * FROM persistence_requests WHERE source_id=$1 AND state='failed'", [f.id]);
const requests = await engine.executeRaw('SELECT id FROM persistence_requests WHERE source_id=$1 ORDER BY sequence', [f.id]);
await engine.executeRaw('UPDATE connector_retry_faults SET pages=false,retries=true WHERE source_id=$1', [f.id]);
await expect(f.run(true)).rejects.toBeInstanceOf(Error);
expect(await engine.executeRaw('SELECT id FROM persistence_requests WHERE source_id=$1 ORDER BY sequence', [f.id])).toEqual(requests);
expect(await engine.executeRaw("SELECT fingerprint FROM op_checkpoints WHERE op='managed-connector-retry' AND completed_keys->0->>'checkpointKey'=$1", [f.checkpointKey])).toHaveLength(0);
await expect(f.run()).rejects.toMatchObject({ code: 'storage_error', writeRequest: { request_id: original.request_id } });
await engine.executeRaw('UPDATE connector_retry_faults SET retries=false WHERE source_id=$1', [f.id]);
const transaction = engine.transaction;
let lostAck = false;
engine.transaction = async function <T>(this: BrainEngine, run: (tx: BrainEngine) => Promise<T>): Promise<T> {
const result = await transaction.call(this, run) as T;
const row = result as Partial<WriteRequest> | undefined;
if (!lostAck && row?.intent?.retryOf === original.request_id && row.state === 'queued') {
lostAck = true;
throw new Error('Synthetic lost connector approval acknowledgement');
}
return result;
};
try { await expect(f.run(true)).rejects.toMatchObject({ code: 'storage_error' }); }
finally { engine.transaction = transaction; }
expect(lostAck).toBe(true);
const [retry] = await engine.executeRaw<WriteRequest>("SELECT * FROM persistence_requests WHERE source_id=$1 AND intent->>'retryOf'=$2", [f.id, original.request_id]);
expect(retry.state).toBe('queued');
expect(await sourceCursor(engine, f.id)).toEqual(cursorOf(f.checkpoint));
expect((await f.run(true)).status).not.toBe('partial');
expect((await engine.executeRaw<WriteRequest>('SELECT * FROM persistence_requests WHERE id=$1::uuid', [retry.id]))[0].state).toBe('committed');
expect(await engine.executeRaw("SELECT id FROM persistence_requests WHERE source_id=$1 AND intent->>'retryOf'=$2", [f.id, original.request_id])).toHaveLength(1);
expect((await engine.executeRaw<WriteRequest>('SELECT * FROM persistence_requests WHERE id=$1::uuid', [original.id]))[0]).toEqual(original);
}
}), 120_000);