Garry's Opinionated OpenClaw/Hermes Agent Brain
TypeScript
test(refresh): share fixtures across the #7679 state-matrix cells (7c, 7d) to cut their cost, every assertion kept (#7782)
garrytan4f51d318c091
diff --git a/test/persistence-worktree-refresh.test.ts b/test/persistence-worktree-refresh.test.tsindex 21dd45753b..b8e551fddd 100644--- a/test/persistence-worktree-refresh.test.ts+++ b/test/persistence-worktree-refresh.test.ts@@ -389,14 +389,28 @@ test('7c. #7679 state matrix: an interrupted refresh of each kind, settled by thconst scanned: Record<Kind, [string, number]> = { draining: ['aborted', 1], fenced_old: ['aborted', 1], fenced_target: ['syncing', 1], fenced_unrelated: ['recovery_required', 1],merged_target: ['syncing', 1], merged_unrelated: ['recovery_required', 1], syncing_at_target: ['completed', 1], syncing_descendant: ['syncing', 0], syncing_unrelated: ['syncing', 0], syncing_unrelated_synced: ['syncing', 0] };const cells = kinds.flatMap(kind => (['owner_scan', 'plain', 'resume'] as Action[]).map(action => [kind, action] as [Kind, Action]));- for (const engine of engines) for (const [kind, action] of cells) await withRefreshFixture(engine, async f => {+ // Test cost (#7782): a cell that leaves the checkout settled (no unfinished refresh, a HEAD the upstream contains, every member synced to it) hands+ // its fixture to the next cell instead of building a brain per cell, so each chain is at most one settled cell and then one unsettled cell.+ const endsSettled = ([kind, action]: [Kind, Action]) => !kind.includes('_unrelated') && kind !== 'syncing_descendant'+ && !(action === 'owner_scan' && (kind === 'fenced_target' || kind === 'merged_target'));+ const settledCells = cells.filter(endsSettled), unsettledCells = cells.filter(cell => !endsSettled(cell));+ const chains = [...unsettledCells.map((cell, i) => settledCells[i] ? [settledCells[i], cell] : [cell]), ...settledCells.slice(unsettledCells.length).map(cell => [cell])];+ expect(chains.flat()).toHaveLength(cells.length);+ for (const engine of engines) for (const chain of chains) await withRefreshFixture(engine, async f => { for (const [kind, action] of chain) {const label = `${kind} x ${action}`;- const target = f.push('alpha/two.md', page('Alpha two', 'Upstream.'));+ const earlier = await f.refreshRows();+ if (earlier.length) {+ expect(earlier.map(r => r.state).filter(state => state !== 'aborted' && state !== 'completed'), `${label}: handoff`).toEqual([]);+ expect(git(f.root, 'merge-base', 'HEAD', 'origin/main'), `${label}: handoff`).toBe(f.head());+ expect([await f.lastCommit(f.alpha), await f.lastCommit(f.beta)], `${label}: handoff`).toEqual([f.head(), f.head()]);+ }+ const cellRows = async () => (await f.refreshRows()).filter(r => !earlier.some(e => e.id === r.id));+ const target = f.push('alpha/two.md', page('Alpha two', `Upstream for ${label}.`));const old = f.head();await expect(refreshWorktree(f.engine, f.alpha, { hooks: { boundary: point => { if (point === crashAt[stateOf(kind)]) throw new Error(`simulated crash at ${point}`); } } })).rejects.toThrow('simulated crash');await disposePersistenceConsumer(f.engine);- const [orphan] = await f.refreshRows();+ const [orphan] = await cellRows();// The resident consumer's owner scan may already have settled it; restore what the crash left.await f.engine.executeRaw(`UPDATE persistence_worktree_refreshes SET state=$2,completed_at=NULL WHERE id=$1::uuid`, [orphan.id, stateOf(kind)]);if (kind === 'fenced_target' || kind === 'merged_target') git(f.root, 'merge', '-q', '--ff-only', target);@@ -405,7 +419,7 @@ test('7c. #7679 state matrix: an interrupted refresh of each kind, settled by thif (kind === 'syncing_at_target' || kind === 'syncing_unrelated_synced') for (const member of [f.alpha, f.beta]) await f.engine.transaction(tx => withCoordinatedWrite(tx, [member],() => tx.executeRaw('UPDATE sources SET last_commit=$2 WHERE id=$1', [member, target]), TEST_WRITE_ATTRIBUTION));const head = f.head();- const rows = async () => (await f.refreshRows()).map(r => r.state);+ const rows = async () => (await cellRows()).map(r => r.state);if (kind === 'draining') {// Ship audit (Step 7). Value: protects=writer status tells an orphaned draining refresh's writers to --resume it, never to retry (only a live// drain finishes on its own); fails_when=the draining branch of next stops checking liveness (master's wording); why_new=test 10 reads a live@@ -419,14 +433,14 @@ test('7c. #7679 state matrix: an interrupted refresh of each kind, settled by thexpect(await resumeWorktreeRefreshes(f.engine), label).toBe(moved);expect(await rows(), label).toEqual([state]);expect(f.head(), label).toBe(head);- return;+ continue;}const run = refreshWorktree(f.engine, f.alpha, action === 'resume' ? { resume: true } : {});if (kind.includes('_unrelated')) {await refusedWith(run, 'refresh_recovery_required');expect(await rows(), label).toEqual(['recovery_required']);expect(f.head(), label).toBe(head);- return;+ continue;}if (kind === 'syncing_descendant' && action === 'plain') {// The adoption completes the orphan at the descendant HEAD; the refresh itself then refuses the unpushed local commit.@@ -435,7 +449,7 @@ test('7c. #7679 state matrix: an interrupted refresh of each kind, settled by thexpect(f.head(), label).toBe(head);expect(await f.lastCommit(f.alpha), label).toBe(head);expect(await f.lastCommit(f.beta), label).toBe(head);- return;+ continue;}const result = await run;const rolledBack = kind === 'draining' || kind === 'fenced_old';@@ -443,7 +457,7 @@ test('7c. #7679 state matrix: an interrupted refresh of each kind, settled by thexpect(result, label).toMatchObject({ status: rolledBack ? 'abandoned' : 'completed', refresh_id: orphan.id });expect(await rows(), label).toEqual([rolledBack ? 'aborted' : 'completed']);expect(f.head(), label).toBe(head);- return;+ continue;}if (rolledBack) {expect(result, label).toMatchObject({ status: 'completed', target_head: target, recovered: { refresh_id: orphan.id, state: 'aborted', status: 'abandoned' } });@@ -456,7 +470,7 @@ test('7c. #7679 state matrix: an interrupted refresh of each kind, settled by thexpect(f.head(), label).toBe(synced);expect(await f.lastCommit(f.alpha), label).toBe(synced);expect(await f.lastCommit(f.beta), label).toBe(synced);- });+ } });}, 900_000);test('7d. #7679 state matrix: an interrupted refresh that the unattended paths cannot verify, or that a live or remote holder owns, is never moved', async () => {@@ -470,12 +484,13 @@ test('7d. #7679 state matrix: an interrupted refresh that the unattended paths ctype Kind = 'draining' | 'fenced_old' | 'fenced_target' | 'merged_target' | 'syncing_lagging' | 'syncing_at_target' | 'recovery_required';type Action = 'plain_head_unreadable' | 'scan_head_unreadable' | 'plain_recovery_pending' | 'plain_worktree_inactive' | 'live_holder' | 'other_host_holder';const kinds: Kind[] = ['draining', 'fenced_old', 'fenced_target', 'merged_target', 'syncing_lagging', 'syncing_at_target', 'recovery_required'];- const actions: Action[] = ['plain_head_unreadable', 'scan_head_unreadable', 'plain_recovery_pending', 'plain_worktree_inactive', 'live_holder', 'other_host_holder'];+ // The scan aborts a dead drain, so scan_head_unreadable runs last.+ const actions: Action[] = ['plain_head_unreadable', 'plain_recovery_pending', 'plain_worktree_inactive', 'live_holder', 'other_host_holder', 'scan_head_unreadable'];const stateOf = (kind: Kind) => kind === 'recovery_required' ? 'recovery_required' : kind.split('_')[0];const crashAt = (kind: Kind) => kind === 'draining' ? 'drained' : kind.startsWith('syncing') ? 'syncing' : 'fenced';- const cells = kinds.flatMap(kind => actions.map(action => [kind, action] as [Kind, Action]));- for (const engine of engines) for (const [kind, action] of cells) await withRefreshFixture(engine, async f => {- const label = `${kind} x ${action}`;+ // Test cost (#7782): each kind builds one brain and one interrupted refresh, and every action runs on it in turn. No action but the last may move+ // the refresh, so each one undoes its seam and hands the next action the journal, HEAD and worktree it started with.+ for (const engine of engines) for (const kind of kinds) await withRefreshFixture(engine, async f => {const target = f.push('alpha/two.md', page('Alpha two', 'Upstream.'));await expect(refreshWorktree(f.engine, f.alpha, { hooks: { boundary: point => { if (point === crashAt(kind)) throw new Error(`simulated crash at ${point}`); } } })).rejects.toThrow('simulated crash');@@ -491,47 +506,61 @@ test('7d. #7679 state matrix: an interrupted refresh that the unattended paths cconst before = await settled();const plainCode = kind === 'recovery_required' ? 'refresh_recovery_required' : 'refresh_in_progress';const writerStatus = async () => (await runPersistenceAdministration(f.engine, 'writer_status', { source_id: f.alpha }) as { worktree_refreshes: Array<Record<string, any>> }).worktree_refreshes[0];- if (action === 'plain_recovery_pending' || action === 'plain_worktree_inactive') {- if (action === 'plain_recovery_pending') await f.engine.executeRaw(`INSERT INTO persistence_topology_changes(principal_id,request_id,digest,operation,source_id,worktree_ids,state,recovery)- VALUES($1::uuid,$2::uuid,'example','reclone',$3,ARRAY[$4::uuid],'recovering',$5::text::jsonb)`,- [randomUUID(), randomUUID(), f.alpha, f.worktreeId, JSON.stringify({ worktreeId: f.worktreeId })]);- else await f.engine.executeRaw(`UPDATE persistence_worktrees SET state='recovering' WHERE id=$1::uuid`, [f.worktreeId]);- await refusedWith(refreshWorktree(f.engine, f.alpha), 'refresh_recovery_required');- // Status does not promise that the next refresh finishes an orphan the recovery gate holds back.- if (kind !== 'recovery_required') expect(await writerStatus(), label).toMatchObject({ liveness: 'orphaned', auto_recovery: 'none',- fix: { argv: ['gbrain', 'sources', 'writer', 'status', '--source', f.alpha, '--json'] } });- } else if (action === 'live_holder') {- const lock = await tryAcquireNativeLock(refreshLockPath(f.worktreeId));- expect(lock, label).not.toBeNull();- try {- expect(await resumeWorktreeRefreshes(f.engine), label).toBe(0);+ const journal = async () => JSON.stringify(await f.refreshRows());+ const worktree = async () => (await f.engine.executeRaw<{ state: string; topology: number }>(`SELECT state,(SELECT count(*)::int FROM persistence_topology_changes+ WHERE $1::uuid = ANY(worktree_ids)) AS topology FROM persistence_worktrees WHERE id=$1::uuid`, [f.worktreeId]))[0];+ const pristine = { journal: await journal(), worktree: await worktree() };+ for (const action of actions) {+ const label = `${kind} x ${action}`;+ if (action === 'plain_recovery_pending' || action === 'plain_worktree_inactive') {+ const topologyRequest = randomUUID();+ if (action === 'plain_recovery_pending') await f.engine.executeRaw(`INSERT INTO persistence_topology_changes(principal_id,request_id,digest,operation,source_id,worktree_ids,state,recovery)+ VALUES($1::uuid,$2::uuid,'example','reclone',$3,ARRAY[$4::uuid],'recovering',$5::text::jsonb)`,+ [randomUUID(), topologyRequest, f.alpha, f.worktreeId, JSON.stringify({ worktreeId: f.worktreeId })]);+ else await f.engine.executeRaw(`UPDATE persistence_worktrees SET state='recovering' WHERE id=$1::uuid`, [f.worktreeId]);+ await refusedWith(refreshWorktree(f.engine, f.alpha), 'refresh_recovery_required');+ // Status does not promise that the next refresh finishes an orphan the recovery gate holds back.+ if (kind !== 'recovery_required') expect(await writerStatus(), label).toMatchObject({ liveness: 'orphaned', auto_recovery: 'none',+ fix: { argv: ['gbrain', 'sources', 'writer', 'status', '--source', f.alpha, '--json'] } });+ if (action === 'plain_recovery_pending') await f.engine.executeRaw('DELETE FROM persistence_topology_changes WHERE request_id=$1::uuid', [topologyRequest]);+ else await f.engine.executeRaw('UPDATE persistence_worktrees SET state=$2 WHERE id=$1::uuid', [f.worktreeId, pristine.worktree.state]);+ } else if (action === 'live_holder') {+ const lock = await tryAcquireNativeLock(refreshLockPath(f.worktreeId));+ expect(lock, label).not.toBeNull();+ try {+ expect(await resumeWorktreeRefreshes(f.engine), label).toBe(0);+ await refusedWith(refreshWorktree(f.engine, f.alpha), plainCode);+ await refusedWith(refreshWorktree(f.engine, f.alpha, { resume: true }), 'refresh_in_progress');+ expect(await writerStatus(), label).toMatchObject({ liveness: 'live', auto_recovery: 'none' });+ } finally { await lock!.release(); }+ } else if (action === 'other_host_holder') {+ const [{ outcome }] = await f.refreshRows();+ await f.engine.executeRaw(`UPDATE persistence_worktree_refreshes SET outcome=jsonb_set(outcome,'{holder,host_id}',to_jsonb($2::text)) WHERE id=$1::uuid`, [orphan.id, randomUUID()]);await refusedWith(refreshWorktree(f.engine, f.alpha), plainCode);- await refusedWith(refreshWorktree(f.engine, f.alpha, { resume: true }), 'refresh_in_progress');- expect(await writerStatus(), label).toMatchObject({ liveness: 'live', auto_recovery: 'none' });- } finally { await lock!.release(); }- } else if (action === 'other_host_holder') {- await f.engine.executeRaw(`UPDATE persistence_worktree_refreshes SET outcome=jsonb_set(outcome,'{holder,host_id}',to_jsonb($2::text)) WHERE id=$1::uuid`, [orphan.id, randomUUID()]);- await refusedWith(refreshWorktree(f.engine, f.alpha), plainCode);- if (kind !== 'recovery_required') expect(await writerStatus(), label).toMatchObject({ liveness: 'unknown', auto_recovery: 'unknown', fix: { actor: 'host_admin' },- next: `gbrain sources writer status --source ${f.alpha} --json` });- } else {- const headFile = join(f.root, '.git', 'HEAD');- const saved = readFileSync(headFile, 'utf8');- writeFileSync(headFile, 'not a ref\n');- try {- if (action === 'plain_head_unreadable') {- const refused = await refusedWith(refreshWorktree(f.engine, f.alpha), plainCode);- if (kind !== 'recovery_required') expect(refused.suggestion, label).toContain(`gbrain sources writer status --source ${f.alpha} --json`);- } else {- // The scan aborts a dead drain (it does not depend on HEAD) and leaves every other kind for a scan that can read HEAD.- expect(await resumeWorktreeRefreshes(f.engine), label).toBe(kind === 'draining' ? 1 : 0);- expect((await f.refreshRows()).map(r => r.state), label).toEqual([kind === 'draining' ? 'aborted' : state]);- }- } finally { writeFileSync(headFile, saved); }- if (action === 'scan_head_unreadable') { expect(f.head(), label).toBe(head); return; }+ if (kind !== 'recovery_required') expect(await writerStatus(), label).toMatchObject({ liveness: 'unknown', auto_recovery: 'unknown', fix: { actor: 'host_admin' },+ next: `gbrain sources writer status --source ${f.alpha} --json` });+ await f.engine.executeRaw('UPDATE persistence_worktree_refreshes SET outcome=$2::text::jsonb WHERE id=$1::uuid', [orphan.id, JSON.stringify(outcome)]);+ } else {+ const headFile = join(f.root, '.git', 'HEAD');+ const saved = readFileSync(headFile, 'utf8');+ writeFileSync(headFile, 'not a ref\n');+ try {+ if (action === 'plain_head_unreadable') {+ const refused = await refusedWith(refreshWorktree(f.engine, f.alpha), plainCode);+ if (kind !== 'recovery_required') expect(refused.suggestion, label).toContain(`gbrain sources writer status --source ${f.alpha} --json`);+ } else {+ // The scan aborts a dead drain (it does not depend on HEAD) and leaves every other kind for a scan that can read HEAD.+ expect(await resumeWorktreeRefreshes(f.engine), label).toBe(kind === 'draining' ? 1 : 0);+ expect((await f.refreshRows()).map(r => r.state), label).toEqual([kind === 'draining' ? 'aborted' : state]);+ }+ } finally { writeFileSync(headFile, saved); }+ if (action === 'scan_head_unreadable') { expect(f.head(), label).toBe(head); continue; }+ }+ expect(await settled(), label).toEqual(before);+ expect(f.head(), label).toBe(head);+ await disposePersistenceConsumer(f.engine);+ expect({ journal: await journal(), worktree: await worktree() }, `${label}: handoff`).toEqual(pristine);}- expect(await settled(), label).toEqual(before);- expect(f.head(), label).toBe(head);});}, 900_000);