Garry's Opinionated OpenClaw/Hermes Agent Brain
TypeScript
import { afterAll, beforeAll, expect, test } from 'bun:test';
import { randomUUID, generateKeyPairSync } from 'node:crypto';
import { existsSync, mkdirSync, readFileSync, rmSync, writeFileSync } from 'node:fs';
import { dirname, join } from 'node:path';
import type { BrainEngine } from '../src/core/engine.ts';
import { parseGoogleSourceConfig, runGoogleSync } from '../src/core/google/google-source.ts';
import { parseGitHubSourceConfig, runGitHubSync } from '../src/core/github-source.ts';
import { disposePersistenceConsumer, startPersistenceConsumer, waitForWrite, writeResponse } from '../src/core/persistence/service.ts';
import { admitWrite, compactWriteReceipts } from '../src/core/persistence/journal.ts';
import { declarePersistenceProtocol } from '../src/core/persistence/protocol.ts';
import { readManagedConnectorState } from '../src/core/persistence/connector-state.ts';
import { readConnectorSourceStatuses } from '../src/core/persistence/connector-status.ts';
import { migrateConnectorCheckpoints, CONNECTOR_MIGRATION_OP } from '../src/core/persistence/connector-checkpoint-migration.ts';
import { connectorCheckpointKey, connectorIdentity } from '../src/core/persistence/connector-identity.ts';
import { preparePersistedMutation } from '../src/core/persistence/service.ts';
import { checkConnectorCheckpoints } from '../src/commands/doctor/checks/connector-checkpoints.ts';
import { connectorCheckpointsRepair } from '../src/core/repair/connector-checkpoints.ts';
import { resolveRepairScope, runRepair } from '../src/core/repair/core.ts';
import { performSync } from '../src/commands/sync.ts';
import { handleToolCall } from '../src/mcp/server.ts';
import { withCoordinatedWrite } from '../src/core/persistence/context.ts';
import { TEST_WRITE_ATTRIBUTION } from './helpers/write-attribution.ts';
import { inspectUnchanged } from '../src/core/persistence/noop-kernel.ts';
import { ALL_SOURCES } from '../src/core/source-id.ts';
import type { WriteRequest } from '../src/core/persistence/model.ts';
import type { OperationContext } from '../src/core/ops/contract.ts';
import { withEnv } from './helpers/with-env.ts';
import { createConnectorFixture, options, json, googleConfig, githubConfig, contact, issueFixture, githubFetch, withGoogleAccount, connectorPendingSet, withPausedOwnerBudget } from './helpers/connector-fixture.ts';
const { home, engines, env, source, boundSource, setup, teardown } = createConnectorFixture();
beforeAll(setup, 120_000);
afterAll(teardown);
const mark = async (engine: BrainEngine, id: string) => (await engine.executeRaw<{ seq: string | null }>('SELECT max(sequence)::text AS seq FROM persistence_requests WHERE source_id=$1', [id]))[0]?.seq ?? '0';
const since = (engine: BrainEngine, id: string, after: string) => engine.executeRaw<WriteRequest>('SELECT * FROM persistence_requests WHERE source_id=$1 AND sequence>$2::bigint ORDER BY sequence', [id, after]);
const incarnation = async (engine: BrainEngine, id: string) => (await engine.executeRaw<{ incarnation: string }>('SELECT incarnation::text FROM sources WHERE id=$1', [id]))[0].incarnation;
/** People fixture: `all` is the full listing a walk without a sync token returns; incremental walks return nothing. */
function people(all: () => Array<ReturnType<typeof contact>>, opts: { fail?: () => boolean; token?: () => string } = {}) {
const calls: string[] = [];
const fetcher = async (url: string) => {
calls.push(url);
const u = new URL(url);
if (url.includes('/settings/sendAs')) return json({ sendAs: [] });
if (opts.fail?.()) return json({ error: { message: 'fixture people outage' } }, 400);
return json({ connections: u.searchParams.has('syncToken') ? [] : all(), nextSyncToken: opts.token?.() ?? 'contacts-stable' });
};
return { fetcher, calls };
}
const google = (engine: BrainEngine, f: { id: string; dir: string }, fetcher: (url: string) => Promise<Response>, account = googleConfig.g_account, extra: object = {}, config: Record<string, unknown> = googleConfig) =>
runGoogleSync(engine, f.id, parseGoogleSourceConfig(config, f.dir), { ...options, ...extra }, withGoogleAccount(fetcher, account));
test('account pin: a contacts-only or calendar-only credential swap refuses before any admission; rotation for the same account resumes', async () => withEnv(env, async () => {
for (const engine of engines) for (const service of ['contacts', 'calendar'] as const) {
const config = { ...googleConfig, g_services: service };
const f = await source(engine, config);
const at = new Date(Date.now() - 3600_000).toISOString();
const base = async (url: string) => {
if (url.includes('/settings/sendAs')) return json({ sendAs: [] });
if (url.includes('/calendar/')) return json({ items: [{ id: 'example-event', summary: 'Example planning', status: 'confirmed', start: { dateTime: at }, end: { dateTime: at } }], nextSyncToken: 'calendar-1' });
return json({ connections: [contact('first', 'First Example')], nextSyncToken: 'contacts-1' });
};
expect((await google(engine, f, base, googleConfig.g_account, {}, config)).added).toBe(1);
const state = await readManagedConnectorState(engine, f.id, await incarnation(engine, f.id));
expect(state.account).toEqual({ kind: 'google', email: googleConfig.g_account });
await disposePersistenceConsumer(engine);
const before = await mark(engine, f.id);
const swapped = google(engine, f, base, '[email protected]', {}, config);
await expect(swapped).rejects.toMatchObject({ code: 'connector_account_changed', detail: 'account_changed', docs: 'docs/guides/write-refusals.md#connector-account-changed' });
const error = await swapped.catch(e => e) as { suggestion: string; message: string };
// Identities stay out of the message (job records persist and serve it remotely) and appear in the local suggestion.
expect(error.message).not.toContain('[email protected]');
expect(error.suggestion).toContain('pinned to [email protected]; the credential resolves to [email protected]');
expect(error.suggestion).toContain('g_token_env (CONNECTOR_TEST_TOKEN)');
expect(error.suggestion).toContain(`gbrain sync --source ${f.id}`);
expect(error.suggestion).toContain(`gbrain sources archive ${f.id}`);
expect(error.suggestion).toContain('gbrain google setup --account [email protected]');
expect(await since(engine, f.id, before)).toHaveLength(0);
// --reset-checkpoint never authorizes an account change.
await expect(google(engine, f, base, '[email protected]', { resetCheckpoint: true }, config)).rejects.toMatchObject({ code: 'connector_account_changed' });
// A rotated token for the same account resumes the checkpoint: no page admission.
expect((await google(engine, f, base, googleConfig.g_account, {}, config)).status).not.toBe('partial');
expect((await since(engine, f.id, before)).filter(row => String(row.intent?.kind).endsWith('_import'))).toHaveLength(0);
}
}), 180_000);
test('account pin: a GitHub App without an install id that resolves a different installation refuses with the new-source hint', async () => withEnv(env, async () => {
const { privateKey } = generateKeyPairSync('rsa', { modulusLength: 2048 });
const pemPath = join(home, `app-${randomUUID().slice(0, 8)}.pem`);
writeFileSync(pemPath, privateKey.export({ type: 'pkcs1', format: 'pem' }));
const config = { kind: 'github', gh_scope: 'repos', gh_repos: 'acme-example/app', gh_app_id: 4242, gh_app_pem_path: pemPath };
for (const engine of engines) {
const f = await source(engine, config);
let installation = 100;
const fetcher = async (url: string) => {
const path = new URL(url).pathname;
if (path === '/app/installations') return json([{ id: installation }]);
if (path === `/app/installations/${installation}/access_tokens`) return json({ token: 'ghs_fixture', expires_at: new Date(Date.now() + 3600_000).toISOString() });
return githubFetch()(url);
};
const run = () => runGitHubSync(engine, f.id, parseGitHubSourceConfig(config, f.dir), options, fetcher);
expect((await run()).status).not.toBe('partial');
expect((await readManagedConnectorState(engine, f.id, await incarnation(engine, f.id))).account).toEqual({ kind: 'github', installationId: 100, login: null });
installation = 200;
const before = await mark(engine, f.id);
const refused = await run().catch(e => e) as { code: string; suggestion: string };
expect(refused.code).toBe('connector_account_changed');
expect(refused.suggestion).toContain('--app-install 200');
expect(refused.suggestion).toContain(`gbrain sources archive ${f.id}`);
expect(await since(engine, f.id, before)).toHaveLength(0);
// A token refresh re-mints for the installation first resolved, never a newly discovered one.
const { AppTokenProvider } = await import('../src/core/github-source.ts');
installation = 100;
const minted: string[] = [];
const provider = new AppTokenProvider(parseGitHubSourceConfig(config, f.dir).app!, async (url: string, init?: RequestInit) => { minted.push(new URL(url).pathname); return fetcher(url); });
await provider.getToken();
installation = 200;
await provider.refresh().catch(() => {});
expect(provider.installationId).toBe(100);
expect(minted.filter(path => path.endsWith('/access_tokens'))).toEqual(['/app/installations/100/access_tokens', '/app/installations/100/access_tokens']);
// Moving the PEM keeps the identity; the pin still decides.
installation = 100;
const moved = join(home, `moved-${randomUUID().slice(0, 8)}.pem`);
writeFileSync(moved, privateKey.export({ type: 'pkcs1', format: 'pem' }));
const movedConfig = { ...config, gh_app_pem_path: moved };
expect(connectorIdentity('github', movedConfig, f.dir).digest).toBe(connectorIdentity('github', config, f.dir).digest);
}
}), 180_000);
test('--reset-checkpoint re-walks once: newly covered items import, unchanged pages take no admission, the pin is kept; non-connector sources refuse', async () => withEnv(env, async () => {
for (const engine of engines) {
const f = await source(engine, googleConfig);
const listing = [contact('first', 'First Example')];
const { fetcher, calls } = people(() => listing);
expect((await google(engine, f, fetcher)).added).toBe(1);
// Provider now has an item only a full walk reveals (like a widened history window).
listing.push(contact('second', 'Second Example'));
await disposePersistenceConsumer(engine);
expect((await google(engine, f, fetcher)).added).toBe(0);
let before = await mark(engine, f.id);
calls.length = 0;
const reset = await google(engine, f, fetcher, googleConfig.g_account, { resetCheckpoint: true });
expect(reset.added).toBe(1);
expect(calls.some(url => url.includes('syncToken'))).toBe(false);
const pages = (await since(engine, f.id, before)).filter(row => String(row.intent?.kind).endsWith('_import'));
expect(pages.map(row => row.slug)).toEqual(['people/second-example']);
const state = await readManagedConnectorState(engine, f.id, await incarnation(engine, f.id));
expect(state.account).toEqual({ kind: 'google', email: googleConfig.g_account });
// The re-walk saved a fresh cursor (it did not replay an older committed checkpoint), so the next run is incremental.
await disposePersistenceConsumer(engine);
calls.length = 0;
await google(engine, f, fetcher);
expect(calls.some(url => url.includes('syncToken=contacts-stable'))).toBe(true);
await disposePersistenceConsumer(engine);
before = await mark(engine, f.id);
await google(engine, f, fetcher, googleConfig.g_account, { resetCheckpoint: true });
expect((await since(engine, f.id, before)).filter(row => String(row.intent?.kind).endsWith('_import'))).toHaveLength(0);
// A Git-backed source is not a connector.
await engine.executeRaw('UPDATE persistence_brain SET enabled=false WHERE singleton=1');
const gitId = `git-${randomUUID().slice(0, 8)}`;
await engine.executeRaw("INSERT INTO sources(id,name,local_path,config) VALUES($1,$1,$2,'{}')", [gitId, join(home, gitId)]);
await engine.executeRaw('UPDATE persistence_brain SET enabled=true WHERE singleton=1');
await expect(performSync(engine, { sourceId: gitId, resetCheckpoint: true, noPull: true })).rejects.toMatchObject({ code: 'invalid_params' });
}
}), 180_000);
test('freshness: a quiet run with an unchanged cursor stamps last_sync_at under the lease so gbrain waiting stays fresh; a failed sweep does not stamp', async () => withEnv(env, async () => {
for (const engine of engines) {
const f = await source(engine, googleConfig);
let fail = false;
const { fetcher } = people(() => [contact('first', 'First Example')], { fail: () => fail });
await google(engine, f, fetcher);
const stamp = async () => (await engine.executeRaw<{ at: string }>('SELECT last_sync_at::text AS at FROM sources WHERE id=$1', [f.id]))[0].at;
await engine.executeRaw("UPDATE sources SET last_sync_at=now()-interval '2 days' WHERE id=$1", [f.id]).catch(async () => {
await engine.executeRaw('UPDATE persistence_brain SET enabled=false WHERE singleton=1');
await engine.executeRaw("UPDATE sources SET last_sync_at=now()-interval '2 days' WHERE id=$1", [f.id]);
await engine.executeRaw('UPDATE persistence_brain SET enabled=true WHERE singleton=1');
});
const old = await stamp();
await disposePersistenceConsumer(engine);
const before = await mark(engine, f.id);
expect((await google(engine, f, fetcher)).status).toBe('up_to_date');
expect(await since(engine, f.id, before)).toHaveLength(0);
const fresh = await stamp();
expect(new Date(fresh).getTime()).toBeGreaterThan(new Date(old).getTime());
const waiting = await handleToolCall(engine, 'open_loops', { group_by: 'counterparty', limit: 3 }, { sourceId: ALL_SOURCES }) as { sources?: Array<{ id: string; stale: boolean }> };
expect(waiting.sources?.find(row => row.id === f.id)?.stale ?? false).toBe(false);
fail = true;
expect((await google(engine, f, fetcher)).status).toBe('partial');
expect(await stamp()).toBe(fresh);
const status = (await readConnectorSourceStatuses(engine)).get(f.id)!;
expect(status.upgrade_recovery).toBe('none');
}
}), 180_000);
test('run counts: sources status reports the second run of the two-cycle fixture as zero admissions and N skipped unchanged', async () => withEnv(env, async () => {
for (const engine of engines) {
const f = await source(engine, googleConfig);
const all = () => [contact('first', 'First Example'), contact('second', 'Second Example'), contact('third', 'Third Example')];
const rewalk = async (url: string) => {
if (url.includes('/settings/sendAs')) return json({ sendAs: [] });
return json({ connections: all(), nextSyncToken: 'contacts-stable' });
};
await google(engine, f, rewalk);
expect((await readConnectorSourceStatuses(engine)).get(f.id)!.last_run).toMatchObject({ page_admissions: 3, skipped_unchanged: 0 });
await disposePersistenceConsumer(engine);
await google(engine, f, rewalk);
expect((await readConnectorSourceStatuses(engine)).get(f.id)!.last_run).toMatchObject({
page_admissions: 0, skipped_unchanged: 3, pending: 0, checkpoint_admissions: 0, stopped_on_wait_budget: false });
}
}), 180_000);
test('pending set: a pending write that later fails is re-fetched under a new request id without --retry-failed; one that commits is not re-fetched', async () => withEnv(env, async () => {
for (const engine of engines) for (const outcome of ['fails', 'commits'] as const) {
const f = await boundSource(engine, githubConfig);
await engine.executeRaw("UPDATE persistence_worktrees SET state='draining' WHERE id=$1::uuid", [f.binding.worktree_id]);
const run = () => runGitHubSync(engine, f.id, parseGitHubSourceConfig(githubConfig, f.dir), { ...options, githubItem: { repo: 'acme-example/app', number: 1, kind: 'issue' } }, githubFetch());
expect(await withPausedOwnerBudget(() => run())).toMatchObject({ status: 'partial', reason: 'writer_pending' });
const [entry] = await connectorPendingSet(engine, f.id);
const [accepted] = await engine.executeRaw<WriteRequest>('SELECT * FROM persistence_requests WHERE source_id=$1 AND request_id=$2::uuid', [f.id, entry.requestId]);
await disposePersistenceConsumer(engine);
const path = join(f.dir, 'gh/acme-example/app/1.md');
if (outcome === 'fails') mkdirSync(dirname(path), { recursive: true });
if (outcome === 'fails') writeFileSync(path, '---\ntitle: Operator file\n---\nAn unindexed operator file occupies the path.\n');
await engine.executeRaw("UPDATE persistence_worktrees SET state='active' WHERE id=$1::uuid", [f.binding.worktree_id]);
startPersistenceConsumer(engine, { engine: engine.kind });
expect((await waitForWrite(engine, accepted, { engine: engine.kind }, 20_000)).state).toBe(outcome === 'fails' ? 'conflict' : 'committed');
if (outcome === 'fails') {
rmSync(path);
// A targeted refresh of a different item enumerates nothing else, so it keeps the failed entry for its own retry.
await disposePersistenceConsumer(engine);
await runGitHubSync(engine, f.id, parseGitHubSourceConfig(githubConfig, f.dir), { ...options, githubItem: { repo: 'acme-example/app', number: 2, kind: 'issue', deleted: true } }, githubFetch());
expect((await connectorPendingSet(engine, f.id)).map(pending => pending.requestId)).toEqual([accepted.request_id]);
}
await disposePersistenceConsumer(engine);
const before = await mark(engine, f.id);
expect((await run()).status).not.toBe('partial');
const pages = (await since(engine, f.id, before)).filter(row => String(row.intent?.kind).endsWith('_import'));
if (outcome === 'fails') {
expect(pages).toHaveLength(1);
expect(pages[0]).toMatchObject({ state: 'committed' });
expect(pages[0].request_id).not.toBe(accepted.request_id);
expect(pages[0].intent?.retryOf).toBe(accepted.request_id);
} else expect(pages).toHaveLength(0);
expect(await connectorPendingSet(engine, f.id)).toEqual([]);
expect(existsSync(path)).toBe(true);
}
}), 180_000);
test('pending set: a committed-then-compacted pending receipt and a crash before the checkpoint cost no page admission on the next run; duplicates in one sweep admit once', async () => withEnv(env, async () => {
for (const engine of engines) {
const f = await boundSource(engine, googleConfig);
await engine.executeRaw("UPDATE persistence_worktrees SET state='draining' WHERE id=$1::uuid", [f.binding.worktree_id]);
const dup = async (url: string) => url.includes('/settings/sendAs') ? json({ sendAs: [] })
: json({ connections: [contact('first', 'First Example'), contact('first', 'First Example')], nextSyncToken: 'contacts-dup' });
expect(await withPausedOwnerBudget(() => google(engine, f, dup))).toMatchObject({ status: 'partial', reason: 'writer_pending' });
const imports = await engine.executeRaw<WriteRequest>("SELECT * FROM persistence_requests WHERE source_id=$1 AND intent->>'kind'='connector_v2_import'", [f.id]);
expect(imports).toHaveLength(1);
await disposePersistenceConsumer(engine);
await engine.executeRaw("UPDATE persistence_worktrees SET state='active' WHERE id=$1::uuid", [f.binding.worktree_id]);
startPersistenceConsumer(engine, { engine: engine.kind });
expect((await waitForWrite(engine, imports[0], { engine: engine.kind }, 20_000)).state).toBe('committed');
await disposePersistenceConsumer(engine);
await engine.executeRaw("UPDATE persistence_requests SET completed_at=now()-interval '31 days' WHERE source_id=$1", [f.id]);
await compactWriteReceipts(engine);
const before = await mark(engine, f.id);
expect((await google(engine, f, dup)).status).not.toBe('partial');
expect((await since(engine, f.id, before)).filter(row => String(row.intent?.kind).endsWith('_import'))).toHaveLength(0);
expect(await connectorPendingSet(engine, f.id)).toEqual([]);
}
}), 180_000);
test('mixed versions: a v2 receipt refused by an old consumer names the consumer upgrade; permission_denied is never relabeled', async () => {
const receipt = (error_code: string, kind: string) => ({ id: randomUUID(), request_id: randomUUID(), operation: 'submit_job', source_id: 'connector-example', state: 'failed',
error_code, error_message: 'refused', intent: { kind }, created_at: new Date().toISOString(), updated_at: new Date().toISOString() }) as unknown as WriteRequest;
const refusal = (row: WriteRequest) => { try { writeResponse(row); } catch (error) { return error as { code: string; suggestion: string; detail?: string; docs?: string }; } throw new Error('expected refusal'); };
const old = refusal(receipt('unsupported_mutation_protocol', 'connector_v2_import'));
expect(old).toMatchObject({ code: 'unsupported_mutation_protocol', detail: 'consumer_upgrade_required', docs: 'docs/guides/write-refusals.md#unsupported-mutation-protocol' });
expect(old.suggestion).toContain('gbrain upgrade on every consumer and worktree-owner host');
for (const kind of ['connector_v2_import', 'managed_sync_import']) {
const denied = refusal(receipt('permission_denied', kind));
expect(denied.code).toBe('permission_denied');
expect(denied.detail).toBeUndefined();
expect(denied.suggestion).toMatch(/^The submit_job write \(request_id [0-9a-f-]{36}\) ended failed with permission_denied; it will not publish\. Read the receipt and the current state before deciding to submit again; a new attempt needs a new request_id\.$/);
}
// A binary that predates the v2 namespace falls through to the same code for any kind it does not know.
await expect(preparePersistedMutation({} as BrainEngine, receipt('', 'connector_v3_import') as WriteRequest & { operation: string }, { engine: 'pglite' }))
.rejects.toMatchObject({ code: 'unsupported_mutation_protocol' });
});
test('mixed versions: a retired managed_connector_* intent fails connector_intent_outdated, pre_upgrade before the cutoff and naming the host upgrade after it', async () => withEnv(env, async () => {
for (const engine of engines) for (const phase of ['pre_upgrade', 'post_cutoff'] as const) {
const f = await source(engine, googleConfig);
const { fetcher } = people(() => [contact('first', 'First Example')]);
await google(engine, f, fetcher);
const [template] = await engine.executeRaw<WriteRequest>("SELECT * FROM persistence_requests WHERE source_id=$1 AND intent->>'kind'='connector_v2_import' LIMIT 1", [f.id]);
await disposePersistenceConsumer(engine);
await engine.executeRaw("DELETE FROM op_checkpoints WHERE op=$1 AND fingerprint='v2-cutoff'", [CONNECTOR_MIGRATION_OP]);
const cutoff = phase === 'pre_upgrade' ? "now()+interval '1 hour'" : "now()-interval '1 hour'";
await engine.executeRaw(`INSERT INTO op_checkpoints(op,fingerprint,completed_keys) VALUES($1,'v2-cutoff',jsonb_build_array(jsonb_build_object('cutoff',(${cutoff})::text)))`, [CONNECTOR_MIGRATION_OP]);
const intent = { ...template.intent, kind: 'managed_connector_import', checkpointKey: 'legacy-raw-config-key' };
const row = await admitWrite(engine, { requestId: randomUUID(), operation: 'submit_job', sourceId: f.id, sourceIncarnation: template.source_incarnation,
slug: template.slug, pageId: template.page_id, worktreeId: template.worktree_id ?? undefined, principal: { kind: template.principal_kind, id: template.principal_id },
authority: template.authority, callerIntent: intent, intent } as never);
startPersistenceConsumer(engine, { engine: engine.kind });
const done = await waitForWrite(engine, row, { engine: engine.kind }, 20_000);
expect(done).toMatchObject({ state: 'failed', error_code: 'connector_intent_outdated' });
let delivered: { detail?: string; suggestion?: string; docs?: string } = {};
try { writeResponse(done); } catch (error) { delivered = error as typeof delivered; }
expect(delivered.docs).toBe('docs/guides/write-refusals.md#connector-intent-outdated');
if (phase === 'pre_upgrade') {
expect(delivered.detail).toBe('pre_upgrade');
expect(delivered.suggestion).toContain('No host upgrade is needed');
} else expect(delivered.suggestion).toContain('Upgrade gbrain on the host that runs connector jobs');
await disposePersistenceConsumer(engine);
}
}), 180_000);
/** A committed legacy checkpoint receipt, as a pre-upgrade connector host left it. */
async function legacyCheckpoint(engine: BrainEngine, f: { id: string }, key: string, state: unknown, compacted = false) {
const [template] = await engine.executeRaw<WriteRequest>("SELECT * FROM persistence_requests WHERE source_id=$1 AND intent->>'kind'='connector_v2_checkpoint' ORDER BY sequence DESC LIMIT 1", [f.id]);
await engine.executeRaw("INSERT INTO op_checkpoints(op,fingerprint,completed_keys) VALUES('managed-connector',$1,$2::text::jsonb)", [key, JSON.stringify([{ generation: 1, state }])]);
const intent = { ...template.intent, kind: 'managed_connector_checkpoint', checkpointKey: key, configHash: 'legacy-raw-config-hash' };
const row = await admitWrite(engine, { requestId: randomUUID(), operation: 'submit_job', sourceId: f.id, sourceIncarnation: template.source_incarnation,
slug: template.slug, pageId: null, worktreeId: template.worktree_id ?? undefined, principal: { kind: template.principal_kind, id: template.principal_id },
authority: template.authority, callerIntent: intent, intent } as never);
await engine.transaction(async tx => {
await declarePersistenceProtocol(tx);
await tx.executeRaw(`UPDATE persistence_requests SET state='committed',completed_at=now(),outcome='{"status":"checkpointed"}'::jsonb${compacted ? ',compacted=true,intent=NULL' : ''} WHERE id=$1::uuid`, [row.id]);
});
return row;
}
test('migration 176: re-keys the newest committed checkpoint, re-walks a compacted one once, removes orphans, keeps referenced and newer rows, and a rerun changes nothing', async () => withEnv(env, async () => {
for (const engine of engines) {
const live = await source(engine, googleConfig), compacted = await source(engine, googleConfig), archived = await source(engine, googleConfig);
const { fetcher, calls } = people(() => [contact('first', 'First Example')]);
await google(engine, live, fetcher); await google(engine, compacted, fetcher); await google(engine, archived, fetcher);
await disposePersistenceConsumer(engine);
// Pretend the sources were last checkpointed by a pre-upgrade binary under raw-config keys.
for (const f of [live, compacted, archived]) {
await engine.executeRaw("DELETE FROM op_checkpoints WHERE op IN ('managed-connector','managed-connector-state') AND fingerprint IN ($1,$2)",
[connectorCheckpointKey(f.id, await incarnation(engine, f.id), connectorIdentity('google', googleConfig, f.dir)), '']);
await engine.executeRaw("DELETE FROM op_checkpoints WHERE op='managed-connector-state'");
}
const liveKey = `legacy-live-${randomUUID()}`;
await legacyCheckpoint(engine, live, `legacy-older-${randomUUID()}`, { contacts_sync_token: 'older-token' });
await legacyCheckpoint(engine, live, liveKey, { contacts_sync_token: 'legacy-token' });
await legacyCheckpoint(engine, compacted, `legacy-compacted-${randomUUID()}`, { contacts_sync_token: 'lost-token' }, true);
await legacyCheckpoint(engine, archived, `legacy-archived-${randomUUID()}`, { contacts_sync_token: 'archived-token' });
await engine.executeRaw('UPDATE persistence_brain SET enabled=false WHERE singleton=1');
await engine.executeRaw('UPDATE sources SET archived=true WHERE id=$1', [archived.id]);
await engine.executeRaw('UPDATE persistence_brain SET enabled=true WHERE singleton=1');
const orphan = `legacy-orphan-${randomUUID()}`, newer = `legacy-newer-${randomUUID()}`, referenced = `legacy-referenced-${randomUUID()}`;
await engine.executeRaw("INSERT INTO op_checkpoints(op,fingerprint,completed_keys,updated_at) VALUES('managed-connector',$1,'[]'::jsonb,now()-interval '9 days'),('managed-connector',$2,'[]'::jsonb,now()+interval '1 hour'),('managed-connector',$3,'[]'::jsonb,now()-interval '9 days')", [orphan, newer, referenced]);
await engine.executeRaw("INSERT INTO op_checkpoints(op,fingerprint,completed_keys,updated_at) VALUES('managed-connector-retry',$1,$2::text::jsonb,now()-interval '9 days')",
[randomUUID(), JSON.stringify([{ checkpointKey: orphan, requestId: randomUUID(), baseRequestId: randomUUID() }])]);
const [template] = await engine.executeRaw<WriteRequest>("SELECT * FROM persistence_requests WHERE source_id=$1 AND intent->>'kind'='connector_v2_import' LIMIT 1", [live.id]);
await admitWrite(engine, { requestId: randomUUID(), operation: 'submit_job', sourceId: live.id, sourceIncarnation: template.source_incarnation, slug: template.slug,
pageId: template.page_id, principal: { kind: template.principal_kind, id: template.principal_id }, authority: template.authority,
callerIntent: { ...template.intent, kind: 'managed_connector_import', checkpointKey: referenced }, intent: { ...template.intent, kind: 'managed_connector_import', checkpointKey: referenced } } as never);
const lines: string[] = [];
const report = await migrateConnectorCheckpoints(engine, line => lines.push(line));
expect(report.rekeyed.map(row => row.source_id).sort()).toEqual([live.id, archived.id].sort());
// An archived source keeps its cursor under the stable key, so a restore resumes it.
const archivedKey = connectorCheckpointKey(archived.id, await incarnation(engine, archived.id), connectorIdentity('google', googleConfig, archived.dir));
expect(await engine.executeRaw("SELECT fingerprint FROM op_checkpoints WHERE op='managed-connector' AND fingerprint=$1", [archivedKey])).toHaveLength(1);
expect(report.rewalking).toEqual([compacted.id]);
expect(lines.join('\n')).toContain(`${live.id}: resumed from pre-upgrade checkpoint of`);
expect(lines.join('\n')).toContain(`to re-walk: gbrain sync --source ${live.id} --reset-checkpoint`);
const fingerprints = (await engine.executeRaw<{ op: string; fingerprint: string }>("SELECT op,fingerprint FROM op_checkpoints WHERE op IN ('managed-connector','managed-connector-retry')")).map(row => row.fingerprint);
expect(fingerprints).not.toContain(orphan);
expect(fingerprints).toContain(newer);
expect(fingerprints).toContain(referenced);
const liveNew = connectorCheckpointKey(live.id, await incarnation(engine, live.id), connectorIdentity('google', googleConfig, live.dir));
const [copied] = await engine.executeRaw<{ completed_keys: Array<{ state: { contacts_sync_token: string } }> }>("SELECT completed_keys FROM op_checkpoints WHERE op='managed-connector' AND fingerprint=$1", [liveNew]);
expect(copied.completed_keys[0].state.contacts_sync_token).toBe('legacy-token');
const statuses = await readConnectorSourceStatuses(engine);
expect(statuses.get(live.id)).toMatchObject({ upgrade_recovery: 'resumed' });
expect(statuses.get(live.id)!.resumed_from).not.toBeNull();
expect(statuses.get(compacted.id)).toMatchObject({ upgrade_recovery: 'rewalking_once' });
const snapshot = await engine.executeRaw("SELECT op,fingerprint,completed_keys FROM op_checkpoints WHERE op LIKE 'managed-connector%' ORDER BY op,fingerprint");
expect(await migrateConnectorCheckpoints(engine, () => {})).toEqual({ rekeyed: [], rewalking: [], removed: { checkpoints: 0, retries: 0 } });
expect(await engine.executeRaw("SELECT op,fingerprint,completed_keys FROM op_checkpoints WHERE op LIKE 'managed-connector%' ORDER BY op,fingerprint")).toEqual(snapshot);
// The first post-upgrade runs: the live source resumes its legacy cursor and is pinned with unverified continuity; the compacted one re-walks once.
// #6324: the resumed cursor carries no `contacts_person_fields`, so it was minted under the old field mask and People would
// reject it; the first sweep lists without it and commits a token under the current mask, which the second sweep reuses.
calls.length = 0;
await google(engine, live, fetcher);
const connections = () => calls.filter(url => url.includes('/people/me/connections'));
expect(connections().length).toBeGreaterThan(0);
expect(connections().some(url => url.includes('syncToken='))).toBe(false);
expect((await readConnectorSourceStatuses(engine)).get(live.id)).toMatchObject({ upgrade_recovery: 'resumed', continuity_unverified: true, account_pinned: true });
calls.length = 0;
await google(engine, live, fetcher);
expect(connections().some(url => url.includes('syncToken=contacts-stable'))).toBe(true);
await google(engine, compacted, fetcher);
expect((await readConnectorSourceStatuses(engine)).get(compacted.id)).toMatchObject({ upgrade_recovery: 'none' });
}
}), 240_000);
test('doctor counts a checkpoint orphaned by a content change once it is 7 days old; repair connector-checkpoints removes it and never copies', async () => withEnv(env, async () => {
for (const engine of engines) {
const f = await source(engine, googleConfig);
const { fetcher } = people(() => [contact('first', 'First Example')]);
await google(engine, f, fetcher);
const oldKey = connectorCheckpointKey(f.id, await incarnation(engine, f.id), connectorIdentity('google', googleConfig, f.dir));
const widened = { ...googleConfig, g_history_days: 365 };
expect(await engine.updateSourceConfig(f.id, { g_history_days: 365 })).toBe(true);
await disposePersistenceConsumer(engine);
await google(engine, f, fetcher, googleConfig.g_account, {}, widened);
const count = async () => ((await checkConnectorCheckpoints(engine)).details as { count: number }).count;
const before = await count();
await engine.executeRaw("UPDATE op_checkpoints SET updated_at=now()-interval '8 days' WHERE op='managed-connector' AND fingerprint=$1", [oldKey]);
expect(await count()).toBe(before + 1);
const aged = await checkConnectorCheckpoints(engine);
expect(aged.status).toBe('warn');
expect(aged.message).toContain('gbrain repair connector-checkpoints');
const ctx = { engine, config: { engine: engine.kind }, remote: false, logger: console } as unknown as OperationContext;
const scope = await resolveRepairScope(engine);
const dry = await runRepair(ctx, connectorCheckpointsRepair, scope, { apply: false });
expect(dry.affected).toBeGreaterThanOrEqual(1);
expect(dry.cost).toMatchObject({ lifetime_ids: 0, embedding_pages: 0 });
const applied = await runRepair(ctx, connectorCheckpointsRepair, scope, { apply: true });
expect(applied.applied).toBe(dry.affected);
expect(await engine.executeRaw("SELECT fingerprint FROM op_checkpoints WHERE op='managed-connector' AND fingerprint=$1", [oldKey])).toHaveLength(0);
expect((await checkConnectorCheckpoints(engine)).status).toBe('ok');
const liveKey = connectorCheckpointKey(f.id, await incarnation(engine, f.id), connectorIdentity('google', widened, f.dir));
expect(await engine.executeRaw("SELECT fingerprint FROM op_checkpoints WHERE op='managed-connector' AND fingerprint=$1", [liveKey])).toHaveLength(1);
}
}), 180_000);
test('#5470 connector kernel: a safe-chunk reseal, projection lag, a pending contextual-mode page, a deleted page and a changed page admit; a carried-forward timeline row does not defeat the skip', async () => withEnv(env, async () => {
for (const engine of engines) {
const f = await boundSource(engine, googleConfig);
let org = 'Example Org', tombstone = false;
const rewalk = async (url: string) => url.includes('/settings/sendAs') ? json({ sendAs: [] })
: json({ connections: [tombstone ? { resourceName: 'people/first', metadata: { deleted: true } } : { ...contact('first', 'First Example'), organizations: [{ name: org }] }], nextSyncToken: 'contacts-stable' });
const slug = 'people/first-example';
const admitted = async (change: () => Promise<unknown>) => {
await change();
await disposePersistenceConsumer(engine);
const before = await mark(engine, f.id);
await google(engine, f, rewalk);
return (await since(engine, f.id, before)).filter(row => row.slug === slug && String(row.intent?.kind).endsWith('_import')).length;
};
await google(engine, f, rewalk);
// A database-only timeline row (as extract --stale records one) is carried forward into the connector
// render (#5567): the first re-walk may materialize it into the page once; it survives, and later re-walks skip.
const summaries = () => engine.executeRaw<{ summary: string }>('SELECT t.summary FROM timeline_entries t JOIN pages p ON p.id=t.page_id WHERE p.source_id=$1 AND p.slug=$2', [f.id, slug]);
expect(await admitted(() => engine.transaction(tx => withCoordinatedWrite(tx, [f.id], () =>
tx.addTimelineEntry(slug, { date: '2026-01-02', summary: 'Met to review the example plan', source: 'meetings/2026-01-02' }, { sourceId: f.id }), TEST_WRITE_ATTRIBUTION)))).toBeLessThanOrEqual(1);
expect(await admitted(async () => {})).toBe(0);
expect(await summaries()).toEqual([{ summary: 'Met to review the example plan' }]);
expect(await admitted(() => engine.executeRaw('UPDATE pages SET text_projection_revision=gen_random_uuid() WHERE source_id=$1 AND slug=$2', [f.id, slug]))).toBe(1);
await engine.executeRaw('UPDATE pages SET text_projection_revision=knowledge_revision WHERE source_id=$1 AND slug=$2', [f.id, slug]);
expect(await admitted(async () => {})).toBe(0);
// Below the safe-chunk fence: asked of the kernel directly, since the owner's projection worker may re-seal it before a re-walk.
await engine.executeRaw('UPDATE pages SET chunker_version=1 WHERE source_id=$1 AND slug=$2', [f.id, slug]);
const snapshot = await engine.readPageSnapshot(slug, { sourceId: f.id, includeDeleted: true });
const file = { root: f.dir, path: join(f.dir, `${slug}.md`), content: existsSync(join(f.dir, `${slug}.md`)) ? readFileSync(join(f.dir, `${slug}.md`), 'utf8') : '' };
const prepared = { noop: true, observedRevision: snapshot!.revision, apply: async () => ({}), file };
expect(await inspectUnchanged(engine, { prepared, snapshot, sourcePath: `${slug}.md`, databaseOnly: false })).toMatchObject({ admitReason: 'projection_work', projectionWorkRequired: true });
await engine.executeRaw('UPDATE pages SET chunker_version=4 WHERE source_id=$1 AND slug=$2', [f.id, slug]);
expect((await inspectUnchanged(engine, { prepared, snapshot, sourcePath: `${slug}.md`, databaseOnly: false })).admitReason).toBeUndefined();
// A publication that would queue embedding is not skipped while chunks lack vectors.
expect(await inspectUnchanged(engine, { prepared, snapshot, sourcePath: `${slug}.md`, databaseOnly: false, embeddingRequested: true })).toMatchObject({ admitReason: 'projection_work' });
// An embedded page without a contextual mode awaits the contextual-mode repair: never skipped.
const vector = `[${Array(1536).fill(0.01).join(',')}]`;
await engine.executeRaw('UPDATE content_chunks SET embedding=$2::text::vector WHERE page_id=(SELECT id FROM pages WHERE source_id=$1 AND slug=$3)', [f.id, vector, slug]);
await engine.executeRaw('UPDATE pages SET contextual_retrieval_mode=NULL WHERE source_id=$1 AND slug=$2', [f.id, slug]);
expect(await inspectUnchanged(engine, { prepared, snapshot, sourcePath: `${slug}.md`, databaseOnly: false })).toMatchObject({ admitReason: 'projection_work' });
await engine.executeRaw('UPDATE content_chunks SET embedding=NULL WHERE page_id=(SELECT id FROM pages WHERE source_id=$1 AND slug=$2)', [f.id, slug]);
// The provider tombstones the contact, then lists it again: resurrecting a deleted page is always admitted.
tombstone = true;
await disposePersistenceConsumer(engine);
await google(engine, f, rewalk);
expect(await engine.getPage(slug, { sourceId: f.id })).toBeNull();
tombstone = false;
expect(await admitted(async () => {})).toBe(1);
expect(await engine.getPage(slug, { sourceId: f.id })).not.toBeNull();
org = 'Renamed Example Org';
expect(await admitted(async () => {})).toBe(1);
}
}), 240_000);
test('connector state row: a lost lease writes nothing; a new source incarnation starts empty', async () => withEnv(env, async () => {
for (const engine of engines) {
const f = await source(engine, googleConfig);
const cfg = parseGoogleSourceConfig(googleConfig, f.dir);
const { beginConnectorSync } = await import('../src/core/persistence/connector-sync.ts');
const lost = { handle: { id: `gbrain-sync:${f.id}`, acquisitionToken: randomUUID(), acquiredAt: '1' }, signal: new AbortController().signal };
await expect(beginConnectorSync(engine, f.id, 'google', cfg, options, lost as never)).rejects.toMatchObject({ name: 'LockStolenError' });
expect(await engine.executeRaw("SELECT fingerprint FROM op_checkpoints WHERE op='managed-connector-state' AND completed_keys->0->'last_run' IS NOT NULL AND completed_keys::text LIKE '%' || $1 || '%'", [f.id])).toHaveLength(0);
expect((await readManagedConnectorState(engine, f.id, await incarnation(engine, f.id))).account).toBeNull();
const { fetcher } = people(() => [contact('first', 'First Example')]);
await google(engine, f, fetcher);
const first = await incarnation(engine, f.id);
expect((await readManagedConnectorState(engine, f.id, first)).account).not.toBeNull();
await disposePersistenceConsumer(engine);
await engine.executeRaw('UPDATE persistence_brain SET enabled=false WHERE singleton=1');
await engine.executeRaw('DELETE FROM sources WHERE id=$1', [f.id]);
await engine.executeRaw('INSERT INTO sources(id,name,local_path,config) VALUES($1,$1,$2,$3::text::jsonb)', [f.id, f.dir, JSON.stringify(googleConfig)]);
await engine.executeRaw('UPDATE persistence_brain SET enabled=true WHERE singleton=1');
const second = await incarnation(engine, f.id);
expect(second).not.toBe(first);
const fresh = await readManagedConnectorState(engine, f.id, second);
expect(fresh).toMatchObject({ account: null, pending: [], upgrade_recovery: 'none', last_run: null });
}
}), 180_000);
test('pending set: a checkpoint still pending at the end of a run is recorded and resolved first next run; a delta run keeps an unreached failure that only a full sweep drops', async () => withEnv(env, async () => {
for (const engine of engines) {
const f = await boundSource(engine, googleConfig);
let token = 0;
const { fetcher } = people(() => [contact('first', 'First Example')], { token: () => `contacts-${++token}` });
await google(engine, f, fetcher);
await engine.executeRaw("UPDATE persistence_worktrees SET state='draining' WHERE id=$1::uuid", [f.binding.worktree_id]);
await disposePersistenceConsumer(engine);
expect(await withPausedOwnerBudget(() => google(engine, f, fetcher))).toMatchObject({ status: 'partial', reason: 'writer_pending' });
const [checkpoint] = await connectorPendingSet(engine, f.id);
expect(checkpoint).toMatchObject({ itemRef: '__managed_connector_checkpoint__' });
await disposePersistenceConsumer(engine);
await engine.executeRaw("UPDATE persistence_worktrees SET state='active' WHERE id=$1::uuid", [f.binding.worktree_id]);
expect((await google(engine, f, fetcher)).status).not.toBe('partial');
expect(await connectorPendingSet(engine, f.id)).toEqual([]);
}
for (const engine of engines) {
const f = await boundSource(engine, githubConfig);
await engine.executeRaw("UPDATE persistence_worktrees SET state='draining' WHERE id=$1::uuid", [f.binding.worktree_id]);
const run = (fetch: ReturnType<typeof githubFetch>, extra: object = {}) => runGitHubSync(engine, f.id, parseGitHubSourceConfig(githubConfig, f.dir), { ...options, ...extra }, fetch);
expect(await withPausedOwnerBudget(() => run(githubFetch()))).toMatchObject({ status: 'partial', reason: 'writer_pending' });
const [entry] = await connectorPendingSet(engine, f.id);
const [accepted] = await engine.executeRaw<WriteRequest>('SELECT * FROM persistence_requests WHERE request_id=$1::uuid', [entry.requestId]);
await disposePersistenceConsumer(engine);
const path = join(f.dir, 'gh/acme-example/app/1.md');
mkdirSync(dirname(path), { recursive: true });
writeFileSync(path, '---\ntitle: Operator file\n---\nAn unindexed operator file occupies the path.\n');
await engine.executeRaw("UPDATE persistence_worktrees SET state='active' WHERE id=$1::uuid", [f.binding.worktree_id]);
startPersistenceConsumer(engine, { engine: engine.kind });
expect((await waitForWrite(engine, accepted, { engine: engine.kind }, 20_000)).state).toBe('conflict');
rmSync(path);
await disposePersistenceConsumer(engine);
// The provider no longer lists the item: a delta run cannot tell deletion from absence, so it keeps the entry.
expect((await run(githubFetch({ deleted: true }))).status).not.toBe('partial');
expect((await connectorPendingSet(engine, f.id)).map(pending => pending.requestId)).toEqual([accepted.request_id]);
await disposePersistenceConsumer(engine);
await run(githubFetch({ deleted: true }), { full: true });
expect(await connectorPendingSet(engine, f.id)).toEqual([]);
expect((await readConnectorSourceStatuses(engine)).get(f.id)!.last_run).toMatchObject({ dropped_upstream: 1 });
}
}), 240_000);