Skip to content

Garry's Opinionated OpenClaw/Hermes Agent Brain

TypeScript

test(refresh): share fixtures across the #7679 state-matrix cells (7c, 7d) to cut their cost, every assertion kept (#7782)

garrytan4f51d318c091
diff --git a/test/persistence-worktree-refresh.test.ts b/test/persistence-worktree-refresh.test.ts
index 21dd45753b..b8e551fddd 100644
--- a/test/persistence-worktree-refresh.test.ts
+++ b/test/persistence-worktree-refresh.test.ts
@@ -389,14 +389,28 @@ test('7c. #7679 state matrix: an interrupted refresh of each kind, settled by th
const scanned: Record<Kind, [string, number]> = { draining: ['aborted', 1], fenced_old: ['aborted', 1], fenced_target: ['syncing', 1], fenced_unrelated: ['recovery_required', 1],
merged_target: ['syncing', 1], merged_unrelated: ['recovery_required', 1], syncing_at_target: ['completed', 1], syncing_descendant: ['syncing', 0], syncing_unrelated: ['syncing', 0], syncing_unrelated_synced: ['syncing', 0] };
const cells = kinds.flatMap(kind => (['owner_scan', 'plain', 'resume'] as Action[]).map(action => [kind, action] as [Kind, Action]));
- for (const engine of engines) for (const [kind, action] of cells) await withRefreshFixture(engine, async f => {
+ // Test cost (#7782): a cell that leaves the checkout settled (no unfinished refresh, a HEAD the upstream contains, every member synced to it) hands
+ // its fixture to the next cell instead of building a brain per cell, so each chain is at most one settled cell and then one unsettled cell.
+ const endsSettled = ([kind, action]: [Kind, Action]) => !kind.includes('_unrelated') && kind !== 'syncing_descendant'
+ && !(action === 'owner_scan' && (kind === 'fenced_target' || kind === 'merged_target'));
+ const settledCells = cells.filter(endsSettled), unsettledCells = cells.filter(cell => !endsSettled(cell));
+ const chains = [...unsettledCells.map((cell, i) => settledCells[i] ? [settledCells[i], cell] : [cell]), ...settledCells.slice(unsettledCells.length).map(cell => [cell])];
+ expect(chains.flat()).toHaveLength(cells.length);
+ for (const engine of engines) for (const chain of chains) await withRefreshFixture(engine, async f => { for (const [kind, action] of chain) {
const label = `${kind} x ${action}`;
- const target = f.push('alpha/two.md', page('Alpha two', 'Upstream.'));
+ const earlier = await f.refreshRows();
+ if (earlier.length) {
+ expect(earlier.map(r => r.state).filter(state => state !== 'aborted' && state !== 'completed'), `${label}: handoff`).toEqual([]);
+ expect(git(f.root, 'merge-base', 'HEAD', 'origin/main'), `${label}: handoff`).toBe(f.head());
+ expect([await f.lastCommit(f.alpha), await f.lastCommit(f.beta)], `${label}: handoff`).toEqual([f.head(), f.head()]);
+ }
+ const cellRows = async () => (await f.refreshRows()).filter(r => !earlier.some(e => e.id === r.id));
+ const target = f.push('alpha/two.md', page('Alpha two', `Upstream for ${label}.`));
const old = f.head();
await expect(refreshWorktree(f.engine, f.alpha, { hooks: { boundary: point => { if (point === crashAt[stateOf(kind)]) throw new Error(`simulated crash at ${point}`); } } }))
.rejects.toThrow('simulated crash');
await disposePersistenceConsumer(f.engine);
- const [orphan] = await f.refreshRows();
+ const [orphan] = await cellRows();
// The resident consumer's owner scan may already have settled it; restore what the crash left.
await f.engine.executeRaw(`UPDATE persistence_worktree_refreshes SET state=$2,completed_at=NULL WHERE id=$1::uuid`, [orphan.id, stateOf(kind)]);
if (kind === 'fenced_target' || kind === 'merged_target') git(f.root, 'merge', '-q', '--ff-only', target);
@@ -405,7 +419,7 @@ test('7c. #7679 state matrix: an interrupted refresh of each kind, settled by th
if (kind === 'syncing_at_target' || kind === 'syncing_unrelated_synced') for (const member of [f.alpha, f.beta]) await f.engine.transaction(tx => withCoordinatedWrite(tx, [member],
() => tx.executeRaw('UPDATE sources SET last_commit=$2 WHERE id=$1', [member, target]), TEST_WRITE_ATTRIBUTION));
const head = f.head();
- const rows = async () => (await f.refreshRows()).map(r => r.state);
+ const rows = async () => (await cellRows()).map(r => r.state);
if (kind === 'draining') {
// Ship audit (Step 7). Value: protects=writer status tells an orphaned draining refresh's writers to --resume it, never to retry (only a live
// drain finishes on its own); fails_when=the draining branch of next stops checking liveness (master's wording); why_new=test 10 reads a live
@@ -419,14 +433,14 @@ test('7c. #7679 state matrix: an interrupted refresh of each kind, settled by th
expect(await resumeWorktreeRefreshes(f.engine), label).toBe(moved);
expect(await rows(), label).toEqual([state]);
expect(f.head(), label).toBe(head);
- return;
+ continue;
}
const run = refreshWorktree(f.engine, f.alpha, action === 'resume' ? { resume: true } : {});
if (kind.includes('_unrelated')) {
await refusedWith(run, 'refresh_recovery_required');
expect(await rows(), label).toEqual(['recovery_required']);
expect(f.head(), label).toBe(head);
- return;
+ continue;
}
if (kind === 'syncing_descendant' && action === 'plain') {
// The adoption completes the orphan at the descendant HEAD; the refresh itself then refuses the unpushed local commit.
@@ -435,7 +449,7 @@ test('7c. #7679 state matrix: an interrupted refresh of each kind, settled by th
expect(f.head(), label).toBe(head);
expect(await f.lastCommit(f.alpha), label).toBe(head);
expect(await f.lastCommit(f.beta), label).toBe(head);
- return;
+ continue;
}
const result = await run;
const rolledBack = kind === 'draining' || kind === 'fenced_old';
@@ -443,7 +457,7 @@ test('7c. #7679 state matrix: an interrupted refresh of each kind, settled by th
expect(result, label).toMatchObject({ status: rolledBack ? 'abandoned' : 'completed', refresh_id: orphan.id });
expect(await rows(), label).toEqual([rolledBack ? 'aborted' : 'completed']);
expect(f.head(), label).toBe(head);
- return;
+ continue;
}
if (rolledBack) {
expect(result, label).toMatchObject({ status: 'completed', target_head: target, recovered: { refresh_id: orphan.id, state: 'aborted', status: 'abandoned' } });
@@ -456,7 +470,7 @@ test('7c. #7679 state matrix: an interrupted refresh of each kind, settled by th
expect(f.head(), label).toBe(synced);
expect(await f.lastCommit(f.alpha), label).toBe(synced);
expect(await f.lastCommit(f.beta), label).toBe(synced);
- });
+ } });
}, 900_000);
test('7d. #7679 state matrix: an interrupted refresh that the unattended paths cannot verify, or that a live or remote holder owns, is never moved', async () => {
@@ -470,12 +484,13 @@ test('7d. #7679 state matrix: an interrupted refresh that the unattended paths c
type Kind = 'draining' | 'fenced_old' | 'fenced_target' | 'merged_target' | 'syncing_lagging' | 'syncing_at_target' | 'recovery_required';
type Action = 'plain_head_unreadable' | 'scan_head_unreadable' | 'plain_recovery_pending' | 'plain_worktree_inactive' | 'live_holder' | 'other_host_holder';
const kinds: Kind[] = ['draining', 'fenced_old', 'fenced_target', 'merged_target', 'syncing_lagging', 'syncing_at_target', 'recovery_required'];
- const actions: Action[] = ['plain_head_unreadable', 'scan_head_unreadable', 'plain_recovery_pending', 'plain_worktree_inactive', 'live_holder', 'other_host_holder'];
+ // The scan aborts a dead drain, so scan_head_unreadable runs last.
+ const actions: Action[] = ['plain_head_unreadable', 'plain_recovery_pending', 'plain_worktree_inactive', 'live_holder', 'other_host_holder', 'scan_head_unreadable'];
const stateOf = (kind: Kind) => kind === 'recovery_required' ? 'recovery_required' : kind.split('_')[0];
const crashAt = (kind: Kind) => kind === 'draining' ? 'drained' : kind.startsWith('syncing') ? 'syncing' : 'fenced';
- const cells = kinds.flatMap(kind => actions.map(action => [kind, action] as [Kind, Action]));
- for (const engine of engines) for (const [kind, action] of cells) await withRefreshFixture(engine, async f => {
- const label = `${kind} x ${action}`;
+ // Test cost (#7782): each kind builds one brain and one interrupted refresh, and every action runs on it in turn. No action but the last may move
+ // the refresh, so each one undoes its seam and hands the next action the journal, HEAD and worktree it started with.
+ for (const engine of engines) for (const kind of kinds) await withRefreshFixture(engine, async f => {
const target = f.push('alpha/two.md', page('Alpha two', 'Upstream.'));
await expect(refreshWorktree(f.engine, f.alpha, { hooks: { boundary: point => { if (point === crashAt(kind)) throw new Error(`simulated crash at ${point}`); } } }))
.rejects.toThrow('simulated crash');
@@ -491,47 +506,61 @@ test('7d. #7679 state matrix: an interrupted refresh that the unattended paths c
const before = await settled();
const plainCode = kind === 'recovery_required' ? 'refresh_recovery_required' : 'refresh_in_progress';
const writerStatus = async () => (await runPersistenceAdministration(f.engine, 'writer_status', { source_id: f.alpha }) as { worktree_refreshes: Array<Record<string, any>> }).worktree_refreshes[0];
- if (action === 'plain_recovery_pending' || action === 'plain_worktree_inactive') {
- if (action === 'plain_recovery_pending') await f.engine.executeRaw(`INSERT INTO persistence_topology_changes(principal_id,request_id,digest,operation,source_id,worktree_ids,state,recovery)
- VALUES($1::uuid,$2::uuid,'example','reclone',$3,ARRAY[$4::uuid],'recovering',$5::text::jsonb)`,
- [randomUUID(), randomUUID(), f.alpha, f.worktreeId, JSON.stringify({ worktreeId: f.worktreeId })]);
- else await f.engine.executeRaw(`UPDATE persistence_worktrees SET state='recovering' WHERE id=$1::uuid`, [f.worktreeId]);
- await refusedWith(refreshWorktree(f.engine, f.alpha), 'refresh_recovery_required');
- // Status does not promise that the next refresh finishes an orphan the recovery gate holds back.
- if (kind !== 'recovery_required') expect(await writerStatus(), label).toMatchObject({ liveness: 'orphaned', auto_recovery: 'none',
- fix: { argv: ['gbrain', 'sources', 'writer', 'status', '--source', f.alpha, '--json'] } });
- } else if (action === 'live_holder') {
- const lock = await tryAcquireNativeLock(refreshLockPath(f.worktreeId));
- expect(lock, label).not.toBeNull();
- try {
- expect(await resumeWorktreeRefreshes(f.engine), label).toBe(0);
+ const journal = async () => JSON.stringify(await f.refreshRows());
+ const worktree = async () => (await f.engine.executeRaw<{ state: string; topology: number }>(`SELECT state,(SELECT count(*)::int FROM persistence_topology_changes
+ WHERE $1::uuid = ANY(worktree_ids)) AS topology FROM persistence_worktrees WHERE id=$1::uuid`, [f.worktreeId]))[0];
+ const pristine = { journal: await journal(), worktree: await worktree() };
+ for (const action of actions) {
+ const label = `${kind} x ${action}`;
+ if (action === 'plain_recovery_pending' || action === 'plain_worktree_inactive') {
+ const topologyRequest = randomUUID();
+ if (action === 'plain_recovery_pending') await f.engine.executeRaw(`INSERT INTO persistence_topology_changes(principal_id,request_id,digest,operation,source_id,worktree_ids,state,recovery)
+ VALUES($1::uuid,$2::uuid,'example','reclone',$3,ARRAY[$4::uuid],'recovering',$5::text::jsonb)`,
+ [randomUUID(), topologyRequest, f.alpha, f.worktreeId, JSON.stringify({ worktreeId: f.worktreeId })]);
+ else await f.engine.executeRaw(`UPDATE persistence_worktrees SET state='recovering' WHERE id=$1::uuid`, [f.worktreeId]);
+ await refusedWith(refreshWorktree(f.engine, f.alpha), 'refresh_recovery_required');
+ // Status does not promise that the next refresh finishes an orphan the recovery gate holds back.
+ if (kind !== 'recovery_required') expect(await writerStatus(), label).toMatchObject({ liveness: 'orphaned', auto_recovery: 'none',
+ fix: { argv: ['gbrain', 'sources', 'writer', 'status', '--source', f.alpha, '--json'] } });
+ if (action === 'plain_recovery_pending') await f.engine.executeRaw('DELETE FROM persistence_topology_changes WHERE request_id=$1::uuid', [topologyRequest]);
+ else await f.engine.executeRaw('UPDATE persistence_worktrees SET state=$2 WHERE id=$1::uuid', [f.worktreeId, pristine.worktree.state]);
+ } else if (action === 'live_holder') {
+ const lock = await tryAcquireNativeLock(refreshLockPath(f.worktreeId));
+ expect(lock, label).not.toBeNull();
+ try {
+ expect(await resumeWorktreeRefreshes(f.engine), label).toBe(0);
+ await refusedWith(refreshWorktree(f.engine, f.alpha), plainCode);
+ await refusedWith(refreshWorktree(f.engine, f.alpha, { resume: true }), 'refresh_in_progress');
+ expect(await writerStatus(), label).toMatchObject({ liveness: 'live', auto_recovery: 'none' });
+ } finally { await lock!.release(); }
+ } else if (action === 'other_host_holder') {
+ const [{ outcome }] = await f.refreshRows();
+ await f.engine.executeRaw(`UPDATE persistence_worktree_refreshes SET outcome=jsonb_set(outcome,'{holder,host_id}',to_jsonb($2::text)) WHERE id=$1::uuid`, [orphan.id, randomUUID()]);
await refusedWith(refreshWorktree(f.engine, f.alpha), plainCode);
- await refusedWith(refreshWorktree(f.engine, f.alpha, { resume: true }), 'refresh_in_progress');
- expect(await writerStatus(), label).toMatchObject({ liveness: 'live', auto_recovery: 'none' });
- } finally { await lock!.release(); }
- } else if (action === 'other_host_holder') {
- await f.engine.executeRaw(`UPDATE persistence_worktree_refreshes SET outcome=jsonb_set(outcome,'{holder,host_id}',to_jsonb($2::text)) WHERE id=$1::uuid`, [orphan.id, randomUUID()]);
- await refusedWith(refreshWorktree(f.engine, f.alpha), plainCode);
- if (kind !== 'recovery_required') expect(await writerStatus(), label).toMatchObject({ liveness: 'unknown', auto_recovery: 'unknown', fix: { actor: 'host_admin' },
- next: `gbrain sources writer status --source ${f.alpha} --json` });
- } else {
- const headFile = join(f.root, '.git', 'HEAD');
- const saved = readFileSync(headFile, 'utf8');
- writeFileSync(headFile, 'not a ref\n');
- try {
- if (action === 'plain_head_unreadable') {
- const refused = await refusedWith(refreshWorktree(f.engine, f.alpha), plainCode);
- if (kind !== 'recovery_required') expect(refused.suggestion, label).toContain(`gbrain sources writer status --source ${f.alpha} --json`);
- } else {
- // The scan aborts a dead drain (it does not depend on HEAD) and leaves every other kind for a scan that can read HEAD.
- expect(await resumeWorktreeRefreshes(f.engine), label).toBe(kind === 'draining' ? 1 : 0);
- expect((await f.refreshRows()).map(r => r.state), label).toEqual([kind === 'draining' ? 'aborted' : state]);
- }
- } finally { writeFileSync(headFile, saved); }
- if (action === 'scan_head_unreadable') { expect(f.head(), label).toBe(head); return; }
+ if (kind !== 'recovery_required') expect(await writerStatus(), label).toMatchObject({ liveness: 'unknown', auto_recovery: 'unknown', fix: { actor: 'host_admin' },
+ next: `gbrain sources writer status --source ${f.alpha} --json` });
+ await f.engine.executeRaw('UPDATE persistence_worktree_refreshes SET outcome=$2::text::jsonb WHERE id=$1::uuid', [orphan.id, JSON.stringify(outcome)]);
+ } else {
+ const headFile = join(f.root, '.git', 'HEAD');
+ const saved = readFileSync(headFile, 'utf8');
+ writeFileSync(headFile, 'not a ref\n');
+ try {
+ if (action === 'plain_head_unreadable') {
+ const refused = await refusedWith(refreshWorktree(f.engine, f.alpha), plainCode);
+ if (kind !== 'recovery_required') expect(refused.suggestion, label).toContain(`gbrain sources writer status --source ${f.alpha} --json`);
+ } else {
+ // The scan aborts a dead drain (it does not depend on HEAD) and leaves every other kind for a scan that can read HEAD.
+ expect(await resumeWorktreeRefreshes(f.engine), label).toBe(kind === 'draining' ? 1 : 0);
+ expect((await f.refreshRows()).map(r => r.state), label).toEqual([kind === 'draining' ? 'aborted' : state]);
+ }
+ } finally { writeFileSync(headFile, saved); }
+ if (action === 'scan_head_unreadable') { expect(f.head(), label).toBe(head); continue; }
+ }
+ expect(await settled(), label).toEqual(before);
+ expect(f.head(), label).toBe(head);
+ await disposePersistenceConsumer(f.engine);
+ expect({ journal: await journal(), worktree: await worktree() }, `${label}: handoff`).toEqual(pristine);
}
- expect(await settled(), label).toEqual(before);
- expect(f.head(), label).toBe(head);
});
}, 900_000);