Skip to content

Garry's Opinionated OpenClaw/Hermes Agent Brain

TypeScript
29.1 KB
Raw Blame
/**
 * #5984 lanes (Postgres only): several bulk groups of one draining managed
 * sync publish at once and still commit in manifest order.
 */
import { afterAll, beforeAll, expect, setSystemTime, test } from 'bun:test';
import { randomUUID } from 'node:crypto';
import { mkdtempSync, mkdirSync, rmSync, writeFileSync } from 'node:fs';
import { tmpdir } from 'node:os';
import { join } from 'node:path';
import { execFileSync } from 'node:child_process';
import { isolatedPersistencePostgres } from './helpers/persistence-postgres.ts';
import type { BrainEngine } from '../src/core/engine.ts';
import { claimWorktree } from '../src/core/persistence/ownership.ts';
import { performSync } from '../src/commands/sync/perform.ts';
import { disposePersistenceConsumer } from '../src/core/persistence/service.ts';
import { resolveBulkSettings } from '../src/core/persistence/sync-group.ts';
import { acquireShared, DEFER_WAIT_MS, deferToLease, exclusiveAcquired, joinLease, leaseDraining, leaseWounded, yieldLease } from '../src/core/persistence/worktree-lease.ts';
import type { NativeLockHandle } from '../src/core/persistence/native-lock.ts';
import { withEnv } from './helpers/with-env.ts';
import { installFaultHook } from '../src/core/persistence/fault-points.ts';
import { awaitLaneTurn, closeLaneRun, laneApplyBegin, laneClaim, laneOf, laneRoots, laneTask, openLanes } from '../src/core/persistence/sync-lanes.ts';
import { lanesLimit } from '../src/core/persistence/sync-drain.ts';
import { readSyncStatus } from '../src/core/persistence/sync-status.ts';
import { waitFor } from './helpers/wait-for.ts';

const home = mkdtempSync(join(tmpdir(), 'gbrain-managed-lanes-'));
let engine: BrainEngine | undefined;
let closePostgres: (() => Promise<void>) | undefined;
const sources: string[] = [];
function git(root: string, ...args: string[]): string { return execFileSync('git', ['-C', root, ...args], { encoding: 'utf8', stdio: ['ignore', 'pipe', 'pipe'] }).trim(); }
async function fixture(e: BrainEngine, files: Record<string, string>) {
  const id = `lanes-${randomUUID().replace(/-/g, '').slice(0, 20)}`; sources.push(id);
  const root = join(home, id); mkdirSync(root); git(root, 'init', '-q');
  for (const [path, body] of Object.entries(files)) { const full = join(root, path); mkdirSync(join(full, '..'), { recursive: true }); writeFileSync(full, body); }
  git(root, 'add', '.'); git(root, '-c', 'user.name=Example', '-c', '[email protected]', 'commit', '-qm', 'notes');
  await e.executeRaw('UPDATE persistence_brain SET enabled=false WHERE singleton=1');
  await e.executeRaw('INSERT INTO sources(id,name,local_path,config) VALUES($1,$1,$2,\'{}\')', [id, root]);
  await claimWorktree(e, id, root);
  await e.executeRaw('UPDATE persistence_brain SET enabled=true WHERE singleton=1');
  return { id, root };
}
const pad = (i: number) => String(i).padStart(3, '0');
const notes = (count: number, edit: (i: number) => string | null = () => null) => Object.fromEntries(Array.from({ length: count }, (_, i) =>
  [`notes/n${pad(i)}.md`, edit(i) ?? `---\ntitle: Note ${i}\n---\nA durable observation number ${i}, see [[notes/n${pad((i + 7) % count)}]].\n`]));
const imports = (e: BrainEngine, source: string) => e.executeRaw<{ slug: string; state: string; grp: string | null; lane: string | null; error_message: string | null }>(
  `SELECT slug,state,intent->>'group' AS grp,intent->>'lane' AS lane,error_message FROM persistence_requests WHERE source_id=$1 AND intent->>'kind'='managed_sync_import' ORDER BY sequence`, [source]);

beforeAll(async () => {
  if (!process.env.DATABASE_URL) return;
  const pg = await isolatedPersistencePostgres(process.env.DATABASE_URL, 'instance', 12); engine = pg.engine; closePostgres = pg.close;
}, 120_000);
afterAll(async () => {
  if (engine) await withEnv({ GBRAIN_HOME: home }, async () => {
    await disposePersistenceConsumer(engine!); await engine!.executeRaw('UPDATE persistence_brain SET enabled=false WHERE singleton=1');
    for (const id of sources) await engine!.executeRaw('DELETE FROM sources WHERE id=$1', [id]);
  });
  await closePostgres?.();
  rmSync(home, { recursive: true, force: true });
});

test('lane settings are a maximum: flag > env > config > 16, clamped to the pool, and need bulk groups', async () => {
  if (!engine) return;
  expect(await resolveBulkSettings(engine, false)).toMatchObject({ enabled: true, lanes: 8, lanesMax: 16, lanesCap: 'pool',
    lanesReason: 'the connection pool holds 11 long transactions; set GBRAIN_POOL_SIZE=20 for 16 lanes' });
  expect(await resolveBulkSettings(engine, false, 1)).toMatchObject({ lanes: 1, lanesMax: 1, lanesCap: 'disabled', lanesReason: 'disabled by --no-lanes' });
  expect(await resolveBulkSettings(engine, false, 8)).toMatchObject({ lanes: 8, lanesMax: 8, lanesCap: 'maximum', lanesReason: null });
  expect(await resolveBulkSettings(engine, false, 12)).toMatchObject({ lanes: 8, lanesMax: 12, lanesCap: 'pool' });
  expect(await resolveBulkSettings(engine, true, 4)).toMatchObject({ enabled: false, lanes: 1 });
  await withEnv({ GBRAIN_SYNC_LANES: '2' }, async () => expect(await resolveBulkSettings(engine!, false)).toMatchObject({ lanes: 2, lanesCap: 'maximum' }));
  await expect(resolveBulkSettings(engine, false, 17)).rejects.toMatchObject({ code: 'invalid_params' });
  await withEnv({ GBRAIN_SYNC_LANES: '17' }, async () => expect(resolveBulkSettings(engine!, false)).rejects.toMatchObject({ code: 'invalid_params' }));
});

test('the drain names what limited its lanes and what raises it', () => {
  const base = { maximum: 16, configured: 6, effective: 6, step_down: null, busy: 5.8 };
  expect(lanesLimit({ ...base, configured: 1, effective: 1, maximum: 1 }, 'disabled')).toMatchObject({ kind: 'lanes_off' });
  expect(lanesLimit({ ...base, effective: 5, step_down: 'lock_timeout on a lane group' }, 'pool')).toMatchObject({ kind: 'database_contention' });
  expect(lanesLimit({ ...base, busy: 3.1 }, 'pool')).toMatchObject({ kind: 'feeder', raise: null });
  expect(lanesLimit(base, 'pool')).toMatchObject({ kind: 'pool', raise: expect.stringContaining('GBRAIN_POOL_SIZE=20') });
  expect(lanesLimit({ ...base, maximum: 6 }, 'maximum')).toMatchObject({ kind: 'maximum', raise: expect.stringContaining('sync.lanes 16') });
  expect(lanesLimit({ ...base, maximum: 16, configured: 16, effective: 16, busy: 15 }, 'maximum')).toMatchObject({ kind: 'maximum', raise: null });
});

test('lane timing counts busy lanes, apply time per page and the commit-turn wait share', async () => {
  // The clock is driven by the test, so a late timer on a loaded machine cannot move the measured shares.
  const t0 = Date.now();
  try {
    setSystemTime(new Date(t0));
    openLanes('wt-time', 'run-time', 4, null);
    const state = laneOf({ worktree_id: 'wt-time', intent: { lane: 'run-time' } } as never)!;
    const a = laneApplyBegin(state), b = laneApplyBegin(state);
    setSystemTime(new Date(t0 + 60));
    a.turn(); b.turn();
    setSystemTime(new Date(t0 + 100));
    a.turned(); a.end(4);
    b.turned(); b.end(2);
    const stats = (await closeLaneRun('run-time'))!;
    expect(stats.busy).toBe(2);
    expect(stats.applyMsPerPage).toBe(20);
    expect(stats.turnWaitShare).toBe(0.4);
  } finally {
    setSystemTime();
  }
});

test('a lease is shared by lanes; an exclusive writer drains and wounds it and gets the lock once the lanes leave', async () => {
  let locks = 0, released = 0;
  const native = (): Promise<NativeLockHandle> => { locks++; let done = false; return Promise.resolve({ get released() { return done; }, async release() { done = true; released++; } }); };
  const path = join(home, 'lease-probe');
  const a = (await acquireShared(path, native))!, b = (await acquireShared(path, native))!;
  expect(locks).toBe(1);
  expect(await yieldLease(path, 0)).toBe(false);
  expect(leaseDraining(path) && leaseWounded(path)).toBe(true);
  expect(await acquireShared(path, native)).toBeNull();
  const waiting = yieldLease(path, 5000);
  await a.release(); expect(released).toBe(0);
  await b.release(); expect(released).toBe(1);
  expect(await waiting).toBe(true);
  expect(leaseDraining(path)).toBe(false);
});

test('a background writer never wounds the lanes: the lease drains for it after the defer wait, and no new lease starts until it has the lock', async () => {
  const native = (): Promise<NativeLockHandle> => { let done = false; return Promise.resolve({ get released() { return done; }, async release() { done = true; } }); };
  const path = join(home, 'lease-defer');
  // The clock is pinned before the writer defers, so the defer wait is measured from a known instant.
  const start = Date.now();
  setSystemTime(new Date(start));
  try {
    const lane = (await acquireShared(path, native))!;
    expect(deferToLease(path)).toBe(false);
    expect(leaseWounded(path) || leaseDraining(path)).toBe(false);
    setSystemTime(new Date(start + DEFER_WAIT_MS));
    expect(leaseDraining(path)).toBe(true);
    expect(leaseWounded(path)).toBe(false);
    await lane.release();
    // Between leases the waiting writer goes first: lanes do not open a new lease until it took the lock.
    expect(leaseDraining(path)).toBe(true);
    expect(deferToLease(path)).toBe(true);
    exclusiveAcquired(path);
    expect(leaseDraining(path)).toBe(false);
  } finally { setSystemTime(); }
});

test('a foreground write joins a live lane lease and keeps it open until it leaves; it never takes a wounded or absent one', async () => {
  let released = 0;
  const native = (): Promise<NativeLockHandle> => { let done = false; return Promise.resolve({ get released() { return done; }, async release() { done = true; released++; } }); };
  const path = join(home, 'lease-join');
  expect(joinLease(path)).toBeNull();
  const lane = (await acquireShared(path, native))!;
  const write = joinLease(path)!;
  expect(write).not.toBeNull();
  await lane.release(); expect(released).toBe(0);
  await write.release(); expect(released).toBe(1);
  const again = (await acquireShared(path, native))!;
  expect(await yieldLease(path, 0)).toBe(false);
  expect(joinLease(path)).toBeNull();
  await again.release();
});

test('lanes that start together share one native lock instead of all but one reporting busy', async () => {
  let held = false, locks = 0;
  const native = async (): Promise<NativeLockHandle | null> => {
    await new Promise(resolve => setTimeout(resolve, 20));
    if (held) return null;
    held = true; locks++;
    let done = false;
    return { get released() { return done; }, async release() { held = false; done = true; } };
  };
  const path = join(home, 'lease-start-race');
  const lanes = await Promise.all([acquireShared(path, native), acquireShared(path, native), acquireShared(path, native)]);
  expect(lanes.every(Boolean)).toBe(true);
  expect(locks).toBe(1);
  for (const lane of lanes) await lane!.release();
  expect(held).toBe(false);
});

test('a lane that joins while the last holder is still releasing the native lock waits for it instead of reporting busy', async () => {
  let held = false, locks = 0;
  const native = async (): Promise<NativeLockHandle | null> => {
    if (held) return null;
    held = true; locks++;
    let done = false;
    return { get released() { return done; }, async release() { await new Promise(resolve => setTimeout(resolve, 50)); held = false; done = true; } };
  };
  const path = join(home, 'lease-release-gap');
  const first = (await acquireShared(path, native))!;
  const releasing = first.release();
  const next = await acquireShared(path, native);
  await releasing;
  expect(next).not.toBeNull();
  expect(locks).toBe(2);
  expect(held).toBe(true);
  await next!.release();
  expect(held).toBe(false);
});

test('a lane waits for its predecessor to end, yields to a requeued or unadmitted one and takes its turn after a failed one (#6712)', async () => {
  openLanes('wt-turn', 'run-turn', 4, null);
  const state = laneOf({ worktree_id: 'wt-turn', intent: { lane: 'run-turn' } } as never)!;
  const rows = [{ request_id: 'b', principal_kind: 'local_cli', principal_id: 'p', intent: { after: 'a', lane: 'run-turn' } }] as never;
  const tx = (states: Array<string | null>) => ({ executeRaw: async () => { const next = states.shift(); return next === null || next === undefined ? [] : [{ state: next }]; } }) as never;
  await awaitLaneTurn(tx(['running', 'running', 'committed']), state, rows);
  await expect(awaitLaneTurn(tx(['running', 'queued']), state, rows)).rejects.toMatchObject({ reason: 'predecessor_requeued' });
  await expect(awaitLaneTurn(tx([null]), state, rows)).rejects.toMatchObject({ reason: 'predecessor_requeued' });
  // #6712 (one page, one fate): a predecessor that ended without committing does not stop this group; it commits in its turn.
  await awaitLaneTurn(tx(['running', 'failed']), state, rows);
  await awaitLaneTurn(tx(['cancelled']), state, rows);
  await expect(awaitLaneTurn(tx(['running', 'running', 'running']), state, rows, 60)).rejects.toMatchObject({ reason: 'order_timeout' });
  await closeLaneRun('run-turn');
  expect(laneOf({ worktree_id: 'wt-turn', intent: { lane: 'run-turn' } } as never)).toBeNull();
});

test('closing a lane run stops new lane claims and waits for the lane tasks still running', async () => {
  openLanes('wt-close', 'run-close', 4, null);
  const state = laneOf({ worktree_id: 'wt-close', intent: { lane: 'run-close' } } as never)!;
  const release = laneTask(state);
  let closed = false;
  const closing = closeLaneRun('run-close').then(() => { closed = true; });
  await new Promise(resolve => setTimeout(resolve, 100));
  expect(closed).toBe(false);
  expect(laneRoots().find(root => root.worktreeId === 'wt-close')?.capacity).toBe(0);
  release(); release();
  await closing;
  expect(state.tasks).toBe(0);
  expect(laneOf({ worktree_id: 'wt-close', intent: { lane: 'run-close' } } as never)).toBeNull();
});

test('closing a lane run waits for a claim still in flight to count its lane task', async () => {
  openLanes('wt-claim', 'run-claim', 4, null);
  const state = laneOf({ worktree_id: 'wt-claim', intent: { lane: 'run-claim' } } as never)!;
  const claimed = laneClaim();
  let closed = false;
  const closing = closeLaneRun('run-claim').then(() => { closed = true; });
  await new Promise(resolve => setTimeout(resolve, 100));
  expect(closed).toBe(false);
  const release = laneTask(state);
  claimed();
  await new Promise(resolve => setTimeout(resolve, 100));
  expect(closed).toBe(false);
  release();
  await closing;
  expect(laneOf({ worktree_id: 'wt-claim', intent: { lane: 'run-claim' } } as never)).toBeNull();
});

test('lanes publish several groups at once and every page still commits, attributed to its own request, in manifest order', async () => withEnv({ GBRAIN_HOME: home, GBRAIN_SYNC_BULK_SIZE: '4' }, async () => {
  if (!engine) return;
  const f = await fixture(engine, notes(80));
  const result = await performSync(engine, { sourceId: f.id, noPull: true, noEmbed: true, noExtract: true, drain: true, lanes: 4 });
  expect(result.drain).toMatchObject({ outcome: 'synced', bulk: { enabled: true, lanes: { maximum: 4, configured: 4, limited_by: { kind: expect.any(String) } } } });
  expect(result.drain!.bulk!.lanes.busy).toBeGreaterThan(1);
  expect(result.drain!.bulk!.lanes.apply_ms_per_page).toBeGreaterThan(0);
  expect(result.drain!.bulk!.lanes.overlapped_groups).toBeGreaterThan(0);
  expect(result.drain!.bulk!.lanes.fallbacks).toBe(0);
  const rows = await imports(engine, f.id);
  expect(rows).toHaveLength(80);
  expect(rows.every(row => row.state === 'committed')).toBe(true);
  expect(rows.some(row => row.lane)).toBe(true);
  const attributed = await engine.executeRaw<{ mismatched: boolean }>(`SELECT p.revision_write_request_id IS DISTINCT FROM r.id AS mismatched
    FROM pages p JOIN persistence_requests r ON r.source_id=p.source_id AND r.slug=p.slug AND r.intent->>'kind'='managed_sync_import' WHERE p.source_id=$1`, [f.id]);
  expect(attributed).toHaveLength(80);
  expect(attributed.filter(row => row.mismatched)).toEqual([]);
  const [cursor] = await engine.executeRaw<{ n: number }>("SELECT count(*)::int AS n FROM op_checkpoints WHERE op='managed-sync' AND (completed_keys->0 ? 'window' OR completed_keys->0 ? 'group') AND completed_keys->0->>'sourceId'=$1", [f.id]);
  expect(cursor!.n).toBe(0);
}), 300_000);

test('admit-ahead fits the writer\'s outstanding-request limit instead of stopping when a batch would exceed it', async () => withEnv({ GBRAIN_HOME: home, GBRAIN_SYNC_BULK_SIZE: '4' }, async () => {
  if (!engine) return;
  // Eight lanes keep sixteen groups of four admitted ahead; the principal may hold only 40 outstanding requests.
  await engine.setConfig('persistence.limits.principal_outstanding', '40');
  try {
    const f = await fixture(engine, notes(160));
    const admitted: number[] = []; // the source's queued and running requests at each cursor step
    installFaultHook(async (point, detail) => {
      if (point !== 'sync:mid_checkpoint' || detail.sourceId !== f.id) return;
      const [row] = await engine!.executeRaw<{ n: number }>("SELECT count(*)::int AS n FROM persistence_requests WHERE source_id=$1 AND state IN ('queued','running')", [f.id]);
      admitted.push(row!.n);
    });
    // Every admission transaction the limit refused (an admission over it used to stop admit-ahead until the window drained).
    const refused: string[] = [];
    const transaction = engine.transaction;
    engine.transaction = async function (this: BrainEngine, fn: (tx: BrainEngine) => Promise<unknown>) {
      try { return await transaction.call(this, fn); }
      catch (error) { if ((error as { code?: string }).code === 'queue_capacity') refused.push((error as Error).message); throw error; }
    } as BrainEngine['transaction'];
    let result;
    try { result = await performSync(engine, { sourceId: f.id, noPull: true, noEmbed: true, noExtract: true, drain: true, lanes: 8 }); }
    finally { engine.transaction = transaction; installFaultHook(undefined); }
    expect(result.drain).toMatchObject({ outcome: 'synced' });
    expect((await imports(engine, f.id)).every(row => row.state === 'committed')).toBe(true);
    expect(refused).toEqual([]);
    // Admit-ahead still kept groups admitted while it stayed under the limit.
    expect(Math.max(...admitted)).toBeGreaterThan(8);
    expect(Math.max(...admitted)).toBeLessThanOrEqual(40);
  } finally {
    installFaultHook(undefined);
    await engine.executeRaw("DELETE FROM config WHERE key='persistence.limits.principal_outstanding'");
  }
}), 300_000);

test('a failed page under lanes stops the run and cancels nothing: every other admitted page commits as its own fate (#6712)', async () => withEnv({ GBRAIN_HOME: home, GBRAIN_SYNC_FAILURES_DIR: home, GBRAIN_SYNC_BULK_SIZE: '4' }, async () => {
  if (!engine) return;
  const f = await fixture(engine, notes(60, i => i === 30 ? '---\ntitle: Conflict\nslug: notes/other\n---\nA conflicting identity must not be imported.\n' : null));
  await engine.setConfig('sync.holds', 'fail');
  const result = await performSync(engine, { sourceId: f.id, noPull: true, noEmbed: true, noExtract: true, drain: true, lanes: 4 })
    .finally(() => engine!.unsetConfig('sync.holds'));
  expect(result.drain).toMatchObject({ outcome: 'blocked' });
  expect(result.managedWrite?.slug).toBe('notes/n030');
  // The groups admitted after the failure keep publishing after the run returns.
  await waitFor(async () => (await imports(engine!, f.id)).every(row => row.state !== 'queued' && row.state !== 'running'), { timeoutMs: 60_000, label: 'admitted pages settle' });
  const rows = await imports(engine, f.id);
  const failedAt = rows.findIndex(row => row.slug === 'notes/n030');
  expect(rows[failedAt]!.state).toBe('failed');
  expect(rows.filter((_, i) => i !== failedAt && rows[i]!.state !== 'committed').map(row => `${row.slug} ${row.state} group=${row.grp} lane=${row.lane}`)).toEqual([]);
  expect(rows.slice(failedAt + 1).length).toBeGreaterThan(0);
  for (const row of rows.slice(failedAt + 1)) expect(await engine.getPage(row.slug, { sourceId: f.id })).not.toBeNull();
}), 300_000);

/**
 * #6712 (convergence mode e): a page edited after it was frozen is refused inside its lane group's transaction. That
 * page alone takes its fate (held `worktree_dirty` once its deferrals run out) and the rest of the group still publishes;
 * before, the whole group fell back to the single path, whose exclusive lock waits behind the lanes, and the drain
 * released and re-claimed the group without end.
 */
test('a page edited after its freeze fails alone inside a lane group; the group still publishes and the run ends with the page held', async () => withEnv({ GBRAIN_HOME: home, GBRAIN_SYNC_FAILURES_DIR: home, GBRAIN_SYNC_BULK_SIZE: '8' }, async () => {
  if (!engine) return;
  const f = await fixture(engine, notes(60));
  const opts = { sourceId: f.id, noPull: true, noEmbed: true, noExtract: true, drain: true, lanes: 4 };
  expect((await performSync(engine, opts)).drain).toMatchObject({ outcome: 'synced' });
  const revised = (i: number, word: string) => `---\ntitle: Note ${i}\n---\nA ${word} observation number ${i}.\n`;
  for (let i = 0; i < 40; i++) writeFileSync(join(f.root, `notes/n${pad(i)}.md`), revised(i, 'revised'));
  git(f.root, '-c', 'user.name=Example', '-c', '[email protected]', 'commit', '-qam', 'revise');
  const edited = 'notes/n013.md';
  // The edited page's group is held in preparation until a later lane group has applied and waits for its turn behind it,
  // so the refusal always lands while that group waits (the window CI hit only under load: the group sent back to the
  // queue made every lane group waiting behind it yield, `predecessor_requeued`, and count as a fallback).
  // #6712 PR 4b: a pass's intents carry no index; the zero-padded paths give the manifest order.
  const requestRow = async (requestId: string | undefined) => (await engine!.executeRaw<{ slug: string; idx: number }>(
    "SELECT slug, substring(slug from 'n([0-9]+)$')::int AS idx FROM persistence_requests WHERE request_id=$1::uuid", [requestId ?? null]))[0];
  let behind: () => void = () => {};
  const laterGroupWaiting = new Promise<void>(resolve => { behind = resolve; });
  let waited = false, reached = false;
  installFaultHook(async (point, detail) => {
    if (point === 'sync:after_freeze' && detail.sourceId === f.id && detail.path === edited) writeFileSync(join(f.root, edited), revised(13, 'uncommitted'));
    if (detail.sourceId !== f.id || !detail.requestId) return;
    if (point === 'lane:applied' && ((await requestRow(detail.requestId))?.idx ?? -1) > 13) { reached = true; behind(); }
    if (point === 'consumer:preparing' && !waited && (await requestRow(detail.requestId))?.slug === 'notes/n013') {
      waited = true;
      await Promise.race([laterGroupWaiting, new Promise(resolve => setTimeout(resolve, 20_000))]);
    }
  });
  // Every claim of this source that goes back to the queue, and why.
  await engine.executeRaw(`CREATE TABLE lane_requeues(slug text, reason text);
    CREATE FUNCTION lane_requeue_log() RETURNS trigger LANGUAGE plpgsql AS $$ BEGIN
      INSERT INTO lane_requeues VALUES (NEW.slug, NEW.blocked_reason); RETURN NEW; END $$;
    CREATE TRIGGER lane_requeue_log AFTER UPDATE ON persistence_requests FOR EACH ROW
      WHEN (OLD.state = 'running' AND NEW.state = 'queued' AND NEW.source_id = '${f.id}') EXECUTE FUNCTION lane_requeue_log()`);
  const stop = new AbortController(), timer = setTimeout(() => stop.abort(), 90_000);
  let result, requeues: Array<{ slug: string; reason: string | null }> = [];
  try {
    result = await performSync(engine, { ...opts, signal: stop.signal });
    requeues = await engine.executeRaw('SELECT slug, reason FROM lane_requeues ORDER BY slug');
  } finally {
    clearTimeout(timer); installFaultHook(undefined);
    await engine.executeRaw('DROP TRIGGER lane_requeue_log ON persistence_requests; DROP FUNCTION lane_requeue_log(); DROP TABLE lane_requeues');
  }
  expect(stop.signal.aborted).toBe(false);
  expect(waited && reached).toBe(true);
  // The refused page's group keeps its other members' claims and publishes them; none goes back to the queue.
  expect(requeues).toEqual([]);
  expect(result.drain?.outcome).not.toBe('blocked');
  // The refused page never sends its group to the single path or back to the queue (a lane fallback each).
  expect(result.drain?.bulk?.lanes.fallbacks).toBe(0);
  const status = await readSyncStatus(engine, f.id);
  expect(status.holds.map(hold => `${hold.code}:${hold.path}`)).toEqual([`worktree_dirty:${edited}`]);
  for (let i = 0; i < 40; i++) if (i !== 13) expect((await engine.getPage(`notes/n${pad(i)}`, { sourceId: f.id }))?.compiled_truth).toContain(`revised observation number ${i}`);
}), 300_000);

test('a crash right after a group is admitted and saved skips nothing and admits nothing twice; the first group is small', async () => withEnv({ GBRAIN_HOME: home, GBRAIN_SYNC_BULK_SIZE: '4' }, async () => {
  if (!engine) return;
  const f = await fixture(engine, notes(40));
  let saves = 0, crashed = 0;
  installFaultHook(point => { if (point === 'sync:mid_checkpoint' && [3, 4, 6].includes(++saves)) { crashed++; throw new Error('injected crash after save-and-admit'); } });
  let result;
  try {
    for (let attempt = 0; attempt < 4 && result?.drain?.outcome !== 'synced'; attempt++) {
      result = await performSync(engine, { sourceId: f.id, noPull: true, noEmbed: true, noExtract: true, drain: true, lanes: 4 }).catch(() => null);
    }
  } finally { installFaultHook(undefined); }
  expect(crashed).toBe(3);
  if (result?.drain?.outcome !== 'synced') result = await performSync(engine, { sourceId: f.id, noPull: true, noEmbed: true, noExtract: true, drain: true, lanes: 4 });
  expect(result!.drain).toMatchObject({ outcome: 'synced' });
  const rows = await imports(engine, f.id);
  expect(rows).toHaveLength(40);
  expect(new Set(rows.map(row => row.slug)).size).toBe(40);
  expect(rows.every(row => row.state === 'committed')).toBe(true);
  const groups = new Map<string, number>();
  for (const row of rows) groups.set(row.grp ?? row.slug, (groups.get(row.grp ?? row.slug) ?? 0) + 1);
  expect(groups.get(rows[0]!.grp ?? rows[0]!.slug)).toBeLessThanOrEqual(2);
}), 300_000);

test('a group admission whose cursor another run moved rolls back: no request is admitted that the cursor does not hold', async () => withEnv({ GBRAIN_HOME: home, GBRAIN_SYNC_BULK_SIZE: '4' }, async () => {
  if (!engine) return;
  const f = await fixture(engine, notes(24));
  let moved = false;
  const racing = new Proxy(engine, { get(target, key) {
    if (key === 'transaction') return async (fn: (tx: BrainEngine) => Promise<unknown>) => target.transaction(async tx => {
      const out = await fn(new Proxy(tx, { get(inner, k) {
        if (k === 'executeRaw') return async (sql: string, params?: unknown[], opts?: unknown) => {
          if (!moved && /^INSERT INTO persistence_requests/.test(sql.trim()) && /jsonb_array_elements/.test(sql)) {
            moved = true;
            await target.executeRaw("UPDATE op_checkpoints SET completed_keys=jsonb_set(completed_keys,'{0,progress}','\"moved\"'::jsonb,true) WHERE op='managed-sync' AND completed_keys->0->>'sourceId'=$1", [f.id]);
          }
          return inner.executeRaw(sql, params as never, opts as never);
        };
        const value = Reflect.get(inner, k); return typeof value === 'function' ? value.bind(inner) : value;
      } }));
      return out;
    });
    const value = Reflect.get(target, key); return typeof value === 'function' ? value.bind(target) : value;
  } });
  const first = await performSync(racing, { sourceId: f.id, noPull: true, noEmbed: true, noExtract: true, drain: true, lanes: 4 }).catch(() => null);
  expect(moved).toBe(true);
  const result = first?.drain?.outcome === 'synced' ? first : await performSync(engine, { sourceId: f.id, noPull: true, noEmbed: true, noExtract: true, drain: true, lanes: 4 });
  expect(result.drain).toMatchObject({ outcome: 'synced' });
  const rows = await imports(engine, f.id);
  expect(rows).toHaveLength(24);
  expect(new Set(rows.map(row => row.slug)).size).toBe(24);
  expect(rows.every(row => row.state === 'committed')).toBe(true);
}), 300_000);

test('lanes and --no-lanes build the same pages and the same links from a cross-linked corpus', async () => withEnv({ GBRAIN_HOME: home, GBRAIN_SYNC_BULK_SIZE: '4' }, async () => {
  if (!engine) return;
  const corpus = notes(48);
  const laned = await fixture(engine, corpus), serial = await fixture(engine, corpus);
  const a = await performSync(engine, { sourceId: laned.id, noPull: true, noEmbed: true, drain: true, lanes: 4 });
  const b = await performSync(engine, { sourceId: serial.id, noPull: true, noEmbed: true, drain: true, lanes: 1 });
  expect(a.drain).toMatchObject({ outcome: 'synced', bulk: { lanes: { configured: 4 } } });
  expect(b.drain).toMatchObject({ outcome: 'synced', bulk: { lanes: { configured: 1, reason: 'disabled by --no-lanes' } } });
  const graph = (source: string) => engine!.executeRaw<{ edge: string }>(`SELECT f.slug||'>'||t.slug AS edge FROM links l JOIN pages f ON f.id=l.from_page_id JOIN pages t ON t.id=l.to_page_id
    WHERE f.source_id=$1 ORDER BY 1`, [source]).then(rows => rows.map(row => row.edge));
  const pages = (source: string) => engine!.executeRaw<{ slug: string; hash: string }>('SELECT slug,content_hash AS hash FROM pages WHERE source_id=$1 AND deleted_at IS NULL ORDER BY slug', [source]);
  expect(await graph(laned.id)).toEqual(await graph(serial.id));
  expect((await graph(laned.id)).length).toBeGreaterThan(0);
  expect(await pages(laned.id)).toEqual(await pages(serial.id));
}), 300_000);