Skip to content

Garry's Opinionated OpenClaw/Hermes Agent Brain

TypeScript

fix: pre-landing review fixes (#7679): state-matrix tests for every orphan kind, holder bookkeeping, unattended owner scan, status fix/next agreement

garrytan57bbfdc8277b
diff --git a/src/commands/sources-refresh.ts b/src/commands/sources-refresh.ts
index a17260ddc0..d5288e3a46 100644
--- a/src/commands/sources-refresh.ts
+++ b/src/commands/sources-refresh.ts
@@ -53,6 +53,7 @@ export function parseRefreshArgs(args: string[]): { sourceId: string; json: bool
function describe(result: WorktreeRefreshResult): string {
const head = (value: string | null) => value ? value.slice(0, 12) : '-';
const lines = [`Refresh ${result.status}: sources ${result.source_ids.join(', ')}; ${head(result.old_head)} -> ${head(result.target_head)}${result.upstream_ref ? ` (${result.upstream_ref})` : ''}.`];
+ if (result.recovered) lines.push(`Recovered orphaned refresh ${result.recovered.refresh_id} (${result.recovered.status}; previous holder: ${result.recovered.previous_holder ? `pid ${result.recovered.previous_holder.pid} on ${result.recovered.previous_holder.hostname}` : 'not recorded'}).`);
if (result.incoming_files !== undefined) lines.push(`Incoming files: ${result.incoming_files}.`);
for (const member of result.synced) lines.push(` ${member.source_id}: sync ${member.status}, last_commit ${head(member.last_commit)}`);
if (result.preserved_uncommitted.length) lines.push(`Preserved uncommitted paths (outside the incoming diff): ${result.preserved_uncommitted.slice(0, 20).join(', ')}${result.preserved_uncommitted.length > 20 ? ' ...' : ''}`);
diff --git a/src/core/persistence/sync-status.ts b/src/core/persistence/sync-status.ts
index 444c74d893..e090b945b6 100644
--- a/src/core/persistence/sync-status.ts
+++ b/src/core/persistence/sync-status.ts
@@ -419,7 +419,7 @@ export async function readSyncStatus(engine: BrainEngine, sourceId: string, opts
const human = holds.find(hold => hold.needs_human) ?? (liveError?.needs_human ? liveError : undefined);
const refreshRow = typeof source.refresh === 'string' ? JSON.parse(source.refresh) as RefreshProbeRow : source.refresh;
const worktreeRefresh = refreshRow ? { refresh_id: refreshRow.id, state: refreshRow.state, updated_at: new Date(refreshRow.updated_at).toISOString(),
- ...await (await import('./worktree-refresh.ts')).refreshHolderReport(refreshRow) } : null;
+ ...await (await import('./worktree-refresh.ts')).refreshHolderReport(engine, refreshRow) } : null;
// An orphaned or recovery_required refresh wedges the worktree, so its fix leads; one whose holder this host cannot probe is only the last fallback.
const refreshFix = worktreeRefresh?.liveness === 'orphaned' || worktreeRefresh?.state === 'recovery_required' ? worktreeRefresh.fix : null;
// #6643: a knock-on or a superseded run is not a failure of its page: the resume re-freezes the entry.
diff --git a/src/core/persistence/worktree-refresh.ts b/src/core/persistence/worktree-refresh.ts
index 04b2f64b8f..565a747216 100644
--- a/src/core/persistence/worktree-refresh.ts
+++ b/src/core/persistence/worktree-refresh.ts
@@ -575,7 +575,7 @@ async function hostRoot(engine: BrainEngine, worktreeId: string, hostId = localH
*/
async function recoverOne(engine: BrainEngine, git: Git, row: WorktreeRefreshRow, root: string, unattended = false): Promise<WorktreeRefreshRow> {
const head = await headOf(git, root);
- if (head === null && unattended) throw inProgress(`Refresh ${row.id} is orphaned, but the checkout HEAD could not be read, so it stays ${row.state}; once git can read HEAD again, the next refresh retries it.`, row.source_ids[0], row.id);
+ if (head === null && unattended && row.state !== 'draining') throw inProgress(`Refresh ${row.id} is orphaned, but the checkout HEAD could not be read, so it stays ${row.state}; once git can read HEAD again, the next refresh retries it.`, row.source_ids[0], row.id);
if (row.state === 'draining') {
await engine.transaction(tx => transition(tx, row, ['draining'], 'aborted', { refusal: { code: 'refresh_interrupted', cause: 'The refreshing process stopped while draining; no merge was started.' } }));
} else if (row.state === 'fenced' && head === row.old_head) {
@@ -596,9 +596,12 @@ async function recoverOne(engine: BrainEngine, git: Git, row: WorktreeRefreshRow
* is free) is converged one step, so the scan finds nothing on the next tick.
* Member syncs are not run here. #7679: an orphaned `syncing` row whose members
* all synced to the target or a descendant of it (a later cycle sync caught
- * them up) is completed here; one whose members lag is finished by the next
+ * them up) and whose HEAD is still at the target is completed here (a lagging one is re-checked only after a member moves); one whose members lag is finished by the next
* plain `gbrain sources refresh`, which runs the `--resume` path, or by `--resume`.
*/
+/** A lagging orphaned `syncing` row, keyed by refresh id, with the member `last_commit`s it lagged at: the scan re-checks it only after a member moves. */
+const laggingSyncing = new Map<string, string>();
+
export async function resumeWorktreeRefreshes(engine: BrainEngine, opts: { hostId?: string } = {}): Promise<number> {
const host = opts.hostId ?? localHostId();
const rows = await engine.executeRaw<WorktreeRefreshRow>(`SELECT f.* FROM persistence_worktree_refreshes f
@@ -609,31 +612,40 @@ export async function resumeWorktreeRefreshes(engine: BrainEngine, opts: { hostI
for (const row of rows) {
const root = await hostRoot(engine, row.worktree_id, host);
if (!root) continue;
+ const commits = row.state === 'syncing' ? (await engine.executeRaw<{ last_commit: string | null }>(
+ 'SELECT last_commit FROM sources WHERE id=ANY($1::text[]) ORDER BY id', [row.source_ids])).map(source => source.last_commit ?? '').join(',') : null;
+ if (commits !== null && laggingSyncing.get(row.id) === commits) continue;
const lock = await tryAcquireNativeLock(refreshLockPath(row.worktree_id));
if (!lock) continue;
try {
await withGit(async git => {
- if (row.state === 'syncing' && !await membersAtTarget(engine, git, row, root)) return;
+ if (commits !== null && !await membersAtTarget(engine, git, row, root)) { laggingSyncing.set(row.id, commits); return; }
const after = await withTopologyLocks(engine, row.source_ids[0], async () => {
const current = await readRefresh(engine, row.id);
- return current.state === 'syncing' ? completeSyncedOrphan(engine, git, current, root) : recoverOne(engine, git, current, root);
+ return current.state === 'syncing' ? completeSyncedOrphan(engine, git, current, root) : recoverOne(engine, git, current, root, true);
}, undefined, 0);
if (after.state !== row.state) moved++;
});
} catch (error) {
// GBRA-130 A-34: one refresh's unexpected failure leaves that row for the next topology scan and never stops the others or the scan.
- if (!(error instanceof OperationError && ['write_pending', 'recovery_required', 'refresh_recovery_required', 'owner_unavailable'].includes(error.code))) {
- console.error(`[gbrain] worktree refresh ${row.id} (${row.source_ids.join(', ')}) did not converge at owner start (${String((error as { code?: unknown } | null)?.code ?? (error as Error)?.name ?? 'error')}); it stays ${row.state} and the owner's next topology scan retries it. Check it with gbrain sources writer status ${row.source_ids[0]}.`);
+ if (!(error instanceof OperationError && ['write_pending', 'recovery_required', 'refresh_recovery_required', 'refresh_in_progress', 'owner_unavailable'].includes(error.code))) {
+ console.error(`[gbrain] worktree refresh ${row.id} (${row.source_ids.join(', ')}) did not converge in the owner's topology scan (${String((error as { code?: unknown } | null)?.code ?? (error as Error)?.name ?? 'error')}); it stays ${row.state} and the owner's next topology scan retries it. Check it with gbrain sources writer status ${row.source_ids[0]}.`);
}
} finally { await lock.release(); }
}
return moved;
}
-/** #7679: an orphaned `syncing` row completes once every member's `last_commit` reached the target (or a descendant); otherwise it is left as it is. */
+/**
+ * #7679: an orphaned `syncing` row completes once every member's `last_commit` reached the target (or a descendant) and the
+ * checkout HEAD is still the target or a descendant of it; otherwise it is left as it is (an unrelated or unreadable HEAD
+ * waits for an attended --resume, which names both ways out).
+ */
async function completeSyncedOrphan(engine: BrainEngine, git: Git, row: WorktreeRefreshRow, root: string): Promise<WorktreeRefreshRow> {
const synced = await membersAtTarget(engine, git, row, root);
if (!synced) return row;
+ const head = await headOf(git, root);
+ if (head === null || (head !== row.target_head && !await isAncestor(git, root, row.target_head, head))) return row;
await engine.transaction(tx => transition(tx, row, ['syncing'], 'completed', { synced, completed_by: 'owner_recovery' }));
return readRefresh(engine, row.id);
}
@@ -685,10 +697,10 @@ async function resumeOrAbandon(engine: BrainEngine, sourceId: string, opts: Refr
/**
* `--resume` under the refresh lock the caller holds: one recovery step, then the member syncs of a `syncing` row.
- * #7679: the caller becomes the row's holder, and the holder it replaced is kept as `previous_holder`.
+ * #7679: the caller becomes the row's holder; `previous_holder` keeps the first holder replaced (the refresher that died), so a refused --resume never hides it.
*/
async function resumeHeld(engine: BrainEngine, git: Git, sourceId: string, refreshId: string, root: string, head: string | null, empty: WorktreeRefreshResult, unattended = false): Promise<WorktreeRefreshResult> {
- await engine.executeRaw(`UPDATE persistence_worktree_refreshes SET outcome=outcome||jsonb_build_object('holder',$2::text::jsonb,'previous_holder',outcome->'holder')
+ await engine.executeRaw(`UPDATE persistence_worktree_refreshes SET outcome=outcome||jsonb_build_object('holder',$2::text::jsonb,'previous_holder',COALESCE(outcome->'previous_holder',outcome->'holder'))
WHERE id=$1::uuid AND state IN ${ACTIVE_REFRESH_STATES_SQL}`, [refreshId, JSON.stringify(refreshHolder())]);
let row = await withTopologyLocks(engine, sourceId, async () => recoverOne(engine, git, await readRefresh(engine, refreshId), root, unattended), undefined, 30_000);
if (row.state === 'recovery_required' && head === row.old_head) throw refusal('refresh_recovery_required',
@@ -776,7 +788,7 @@ export async function refreshLiveness(worktreeId: string, holderHostId: string |
}
/** #7679: the holder of an active refresh, whether it is orphaned, and the step that moves it on (`fix` names `--resume` whenever no process will). */
-export async function refreshHolderReport(row: Pick<WorktreeRefreshRow, 'id' | 'worktree_id' | 'state' | 'source_ids' | 'outcome'> & { owner_host_id: string | null }):
+export async function refreshHolderReport(engine: BrainEngine, row: Pick<WorktreeRefreshRow, 'id' | 'worktree_id' | 'state' | 'source_ids' | 'outcome'> & { owner_host_id: string | null }):
Promise<{ holder: RefreshHolder | null; liveness: RefreshLiveness; orphaned: boolean | null; auto_recovery: 'next_refresh' | 'none' | 'unknown'; fix: Action | null }> {
const holder = holderOf(row);
const holderHost = holder?.host_id ?? row.owner_host_id;
@@ -788,6 +800,9 @@ export async function refreshHolderReport(row: Pick<WorktreeRefreshRow, 'id' | '
if (row.state === 'recovery_required') return { ...base, auto_recovery: 'none', fix: { argv: resume, consent: [], actor: 'agent', requires_exclusive: false, verify,
why: `Refresh ${row.id} could not verify the checkout HEAD, so nothing recovers it on its own. --resume changes nothing here: it prints the two ways out (reset to the start commit and --abandon, or to the upstream commit and --resume); resetting is destructive, so ask the user first.` } };
if (liveness === 'live') return { ...base, auto_recovery: 'none', fix: null };
+ const binding = liveness === 'orphaned' ? await getWorktreeBinding(engine, source, holderHost) : null;
+ if (binding && (binding.state !== 'active' || await worktreeRecoveryPending(engine, binding))) return { ...base, auto_recovery: 'none', fix: { argv: verify.argv, consent: [], actor: 'agent', requires_exclusive: false, verify,
+ why: `No process holds refresh ${row.id}'s lock, but publication or topology recovery is pending on its worktree, so no refresh adopts it until the owner finishes that recovery (this read shows it, read-only). Then the next gbrain sources refresh ${source} finishes the refresh on its own.` } };
if (liveness === 'orphaned') return { ...base, auto_recovery: 'next_refresh', fix: { argv: resume, consent: [], actor: 'agent', requires_exclusive: false, verify,
why: `No process holds refresh ${row.id}'s lock (recorded holder: ${describeHolder(holder)}), so it is orphaned. The next gbrain sources refresh ${source} finishes it on its own through the --resume path`
+ `${row.state === 'syncing' ? ', and the owner completes it once every member has synced to its target' : ''}; --resume finishes it now.` } };
@@ -801,13 +816,13 @@ export async function activeWorktreeRefreshes(engine: BrainEngine, sourceId?: st
LEFT JOIN persistence_worktrees w ON w.id=f.worktree_id WHERE f.state IN ${ACTIVE_REFRESH_STATES_SQL}
AND ($1::text IS NULL OR $1=ANY(f.source_ids)) ORDER BY f.created_at`, [sourceId ?? null]);
return Promise.all(rows.map(async row => {
- const report = await refreshHolderReport(row);
+ const report = await refreshHolderReport(engine, row);
return { refresh_id: row.id, worktree_id: row.worktree_id, state: row.state, source_ids: row.source_ids,
old_head: row.old_head, target_head: row.target_head, upstream_ref: row.upstream_ref, updated_at: new Date(row.updated_at).toISOString(),
...(row.outcome.refusal ? { refusal: row.outcome.refusal } : {}), ...report,
next: report.liveness === 'live' && row.state === 'draining' ? 'Writes to this worktree are refused while queued work drains; retry them after about a second.'
: report.liveness === 'live' ? `Wait for it: ${describeHolder(report.holder)} holds the refresh lock and finishes it.`
- : `gbrain sources refresh ${row.source_ids[0]} --resume` };
+ : report.fix?.argv?.join(' ') ?? `gbrain sources refresh ${row.source_ids[0]} --resume` };
}));
}
diff --git a/test/persistence-worktree-refresh.test.ts b/test/persistence-worktree-refresh.test.ts
index 3823cdcbcd..bcaff7a9fe 100644
--- a/test/persistence-worktree-refresh.test.ts
+++ b/test/persistence-worktree-refresh.test.ts
@@ -34,7 +34,8 @@ import { PersistenceConsumer } from '../src/core/persistence/consumer.ts';
import { runManagedSourceLifecycle } from '../src/core/persistence/source-lifecycle.ts';
import { runPersistenceAdministration } from '../src/core/persistence/administration.ts';
import { performManagedSync } from '../src/core/persistence/sync-run.ts';
-import { refreshWorktree, resumeWorktreeRefreshes } from '../src/core/persistence/worktree-refresh.ts';
+import { readSyncStatus } from '../src/core/persistence/sync-status.ts';
+import { refreshLockPath, refreshWorktree, resumeWorktreeRefreshes } from '../src/core/persistence/worktree-refresh.ts';
import { commitAndPush } from '../src/core/git/commit-push.ts';
import { gitConvergenceCheck } from '../src/commands/doctor/checks/git-convergence.ts';
import { MANAGED_SYNC_PASS_OP, writePassRow, type ManagedSyncPassValue } from '../src/core/persistence/sync-pass-row.ts';
@@ -318,7 +319,13 @@ test('7. a crash leaving HEAD on an unrelated commit is recovery_required; write
expect((await f.refreshRows()).map(r => r.state)).toEqual(['recovery_required']);
const refused = await refusedWith(f.put(f.alpha, 'notes/blocked', page('Blocked', 'refused')), 'refresh_recovery_required');
expect(refused.suggestion).toContain(`gbrain sources refresh ${f.alpha} --resume`);
+ // Ship review: a refused --resume never hides the refresher that died. Value: protects=previous_holder keeps the first holder replaced across
+ // refused resumes; fails_when=each resume shifts its own pid into previous_holder; why_new=no test resumed twice; seam=a holder written with SQL;
+ // no_claim=a real foreign process (the holder pid is a placeholder).
+ await f.engine.executeRaw(`UPDATE persistence_worktree_refreshes SET outcome=outcome||jsonb_build_object('holder',jsonb_build_object('host_id',NULL,'hostname','host-example','pid',424242,'started_at','2026-01-01T00:00:00.000Z')) WHERE state='recovery_required'`);
+ await refusedWith(refreshWorktree(f.engine, f.alpha, { resume: true }), 'refresh_recovery_required');
const resume = await refusedWith(refreshWorktree(f.engine, f.alpha, { resume: true }), 'refresh_recovery_required');
+ expect((await f.refreshRows())[0].outcome).toMatchObject({ holder: { pid: process.pid }, previous_holder: { pid: 424242, hostname: 'host-example' } });
expect(resume.suggestion).toContain(`reset --hard ${old}`);
// Ship audit, #7679. Value: protects=a plain refresh never adopts a recovery_required row, and writer status says nothing recovers it on its
// own (auto_recovery none, ask the user before resetting); fails_when=adoption drops its recovery_required guard, or the holder report treats
@@ -330,6 +337,13 @@ test('7. a crash leaving HEAD on an unrelated commit is recovery_required; write
const [held] = (await runPersistenceAdministration(f.engine, 'writer_status', { source_id: f.alpha }) as { worktree_refreshes: Array<Record<string, any>> }).worktree_refreshes;
expect(held).toMatchObject({ state: 'recovery_required', auto_recovery: 'none', fix: { argv: ['gbrain', 'sources', 'refresh', f.alpha, '--resume'], actor: 'agent' } });
expect(held.fix.why).toContain('ask the user');
+ // Ship audit (Step 7). Value: protects=sync status reports a recovery_required refresh and its next is the ask-the-user --resume step, not a
+ // hold, pass or another read; fails_when=the sync-status probe skips recovery_required rows or another next outranks the refresh's fix;
+ // why_new=writer status is read above; sync status has its own probe and next order; seam=none; no_claim=the reset choice itself.
+ const sync = await readSyncStatus(f.engine, f.alpha, { detail: false });
+ expect(sync.worktree_refresh).toMatchObject({ state: 'recovery_required', auto_recovery: 'none' });
+ expect(sync.next).toMatchObject({ argv: ['gbrain', 'sources', 'refresh', f.alpha, '--resume'], actor: 'agent' });
+ expect(sync.next?.why).toContain('ask the user');
expect(await resumeWorktreeRefreshes(f.engine)).toBe(0);
expect((await f.refreshRows()).map(r => r.state)).toEqual(['recovery_required']);
git(f.root, 'reset', '-q', '--hard', old);
@@ -364,51 +378,64 @@ test('7c. #7679 state matrix: an interrupted refresh of each kind, settled by th
// --resume (resumeHeld) settles it; fails_when=a kind falls through to a sibling's path (an unverified HEAD adopted as the target, a syncing
// row with an unrelated HEAD completed or refreshed over, a dead drain left standing); why_new=the #7679 tests cover syncing orphans with HEAD at
// the target only; seam=none; no_claim=a real process death (the crash is a hook throw; the refresh lock is released as on exit).
- type Kind = 'draining' | 'fenced_old' | 'fenced_target' | 'fenced_unrelated' | 'syncing_at_target' | 'syncing_unrelated';
+ type Kind = 'draining' | 'fenced_old' | 'fenced_target' | 'fenced_unrelated' | 'merged_target' | 'merged_unrelated' | 'syncing_at_target' | 'syncing_unrelated'
+ | 'syncing_unrelated_synced';
type Action = 'owner_scan' | 'plain' | 'resume';
- const crashAt = { draining: 'drained', fenced_old: 'fenced', fenced_target: 'fenced', fenced_unrelated: 'fenced', syncing_at_target: 'syncing', syncing_unrelated: 'syncing' } as const;
- const cells: Array<[Kind, Action]> = [['draining', 'owner_scan'], ['fenced_old', 'owner_scan'], ['syncing_unrelated', 'owner_scan'],
- ['draining', 'plain'], ['fenced_target', 'plain'], ['fenced_unrelated', 'plain'], ['syncing_at_target', 'plain'], ['syncing_unrelated', 'plain'],
- ['draining', 'resume'], ['fenced_old', 'resume'], ['fenced_target', 'resume'], ['fenced_unrelated', 'resume'], ['syncing_unrelated', 'resume']];
+ const kinds: Kind[] = ['draining', 'fenced_old', 'fenced_target', 'fenced_unrelated', 'merged_target', 'merged_unrelated', 'syncing_at_target', 'syncing_unrelated',
+ 'syncing_unrelated_synced'];
+ const stateOf = (kind: Kind) => kind.split('_')[0] as 'draining' | 'fenced' | 'merged' | 'syncing';
+ const crashAt = { draining: 'drained', fenced: 'fenced', merged: 'fenced', syncing: 'syncing' } as const;
+ // The owner scan's row and moved count; a merged row's bookkeeping leaves it syncing, and a syncing row with an unrelated HEAD waits for --resume.
+ const scanned: Record<Kind, [string, number]> = { draining: ['aborted', 1], fenced_old: ['aborted', 1], fenced_target: ['syncing', 1], fenced_unrelated: ['recovery_required', 1],
+ merged_target: ['syncing', 1], merged_unrelated: ['recovery_required', 1], syncing_at_target: ['completed', 1], syncing_unrelated: ['syncing', 0], syncing_unrelated_synced: ['syncing', 0] };
+ const cells = kinds.flatMap(kind => (['owner_scan', 'plain', 'resume'] as Action[]).map(action => [kind, action] as [Kind, Action]));
for (const engine of engines) for (const [kind, action] of cells) await withRefreshFixture(engine, async f => {
const label = `${kind} x ${action}`;
const target = f.push('alpha/two.md', page('Alpha two', 'Upstream.'));
const old = f.head();
- await expect(refreshWorktree(f.engine, f.alpha, { hooks: { boundary: point => { if (point === crashAt[kind]) throw new Error(`simulated crash at ${point}`); } } }))
+ await expect(refreshWorktree(f.engine, f.alpha, { hooks: { boundary: point => { if (point === crashAt[stateOf(kind)]) throw new Error(`simulated crash at ${point}`); } } }))
.rejects.toThrow('simulated crash');
await disposePersistenceConsumer(f.engine);
const [orphan] = await f.refreshRows();
// The resident consumer's owner scan may already have settled it; restore what the crash left.
- const state = kind === 'draining' ? 'draining' : kind.startsWith('fenced') ? 'fenced' : 'syncing';
- await f.engine.executeRaw(`UPDATE persistence_worktree_refreshes SET state=$2,completed_at=NULL WHERE id=$1::uuid`, [orphan.id, state]);
- if (kind === 'fenced_target') git(f.root, 'merge', '-q', '--ff-only', target);
- if (kind.endsWith('_unrelated')) { git(f.root, 'reset', '-q', '--hard', old); commitFile(f.root, 'beta/unrelated.md', page('Unrelated', 'Not the upstream commit.')); }
- if (kind === 'syncing_at_target') for (const member of [f.alpha, f.beta]) await f.engine.transaction(tx => withCoordinatedWrite(tx, [member],
+ await f.engine.executeRaw(`UPDATE persistence_worktree_refreshes SET state=$2,completed_at=NULL WHERE id=$1::uuid`, [orphan.id, stateOf(kind)]);
+ if (kind === 'fenced_target' || kind === 'merged_target') git(f.root, 'merge', '-q', '--ff-only', target);
+ if (kind.includes('_unrelated')) { git(f.root, 'reset', '-q', '--hard', old); commitFile(f.root, 'beta/unrelated.md', page('Unrelated', 'Not the upstream commit.')); }
+ if (kind === 'syncing_at_target' || kind === 'syncing_unrelated_synced') for (const member of [f.alpha, f.beta]) await f.engine.transaction(tx => withCoordinatedWrite(tx, [member],
() => tx.executeRaw('UPDATE sources SET last_commit=$2 WHERE id=$1', [member, target]), TEST_WRITE_ATTRIBUTION));
const head = f.head();
const rows = async () => (await f.refreshRows()).map(r => r.state);
+ if (kind === 'draining') {
+ // Ship audit (Step 7). Value: protects=writer status tells an orphaned draining refresh's writers to --resume it, never to retry (only a live
+ // drain finishes on its own); fails_when=the draining branch of next stops checking liveness (master's wording); why_new=test 10 reads a live
+ // drain only; seam=none; no_claim=a real process death (the crash is a hook throw; the refresh lock is released as on exit).
+ const [status] = (await runPersistenceAdministration(f.engine, 'writer_status', { source_id: f.alpha }) as { worktree_refreshes: Array<Record<string, any>> }).worktree_refreshes;
+ expect(status, label).toMatchObject({ refresh_id: orphan.id, state: 'draining', liveness: 'orphaned', auto_recovery: 'next_refresh',
+ fix: { argv: ['gbrain', 'sources', 'refresh', f.alpha, '--resume'] }, next: `gbrain sources refresh ${f.alpha} --resume` });
+ }
if (action === 'owner_scan') {
- const moved = await resumeWorktreeRefreshes(f.engine);
- if (kind === 'syncing_unrelated') { expect(moved, label).toBe(0); expect(await rows(), label).toEqual(['syncing']); }
- else { expect(moved, label).toBe(1); expect(await rows(), label).toEqual(['aborted']); }
+ const [state, moved] = scanned[kind];
+ expect(await resumeWorktreeRefreshes(f.engine), label).toBe(moved);
+ expect(await rows(), label).toEqual([state]);
expect(f.head(), label).toBe(head);
return;
}
const run = refreshWorktree(f.engine, f.alpha, action === 'resume' ? { resume: true } : {});
- if (kind.endsWith('_unrelated')) {
+ if (kind.includes('_unrelated')) {
await refusedWith(run, 'refresh_recovery_required');
expect(await rows(), label).toEqual(['recovery_required']);
expect(f.head(), label).toBe(head);
return;
}
const result = await run;
+ const rolledBack = kind === 'draining' || kind === 'fenced_old';
if (action === 'resume') {
- expect(result, label).toMatchObject(kind === 'fenced_target' ? { status: 'completed', refresh_id: orphan.id } : { status: 'abandoned', refresh_id: orphan.id });
- expect(await rows(), label).toEqual([kind === 'fenced_target' ? 'completed' : 'aborted']);
+ expect(result, label).toMatchObject({ status: rolledBack ? 'abandoned' : 'completed', refresh_id: orphan.id });
+ expect(await rows(), label).toEqual([rolledBack ? 'aborted' : 'completed']);
expect(f.head(), label).toBe(head);
return;
}
- if (kind === 'draining') {
+ if (rolledBack) {
expect(result, label).toMatchObject({ status: 'completed', target_head: target, recovered: { refresh_id: orphan.id, state: 'aborted', status: 'abandoned' } });
expect(await rows(), label).toEqual(['aborted', 'completed']);
} else {
@@ -419,62 +446,168 @@ test('7c. #7679 state matrix: an interrupted refresh of each kind, settled by th
expect(await f.lastCommit(f.alpha), label).toBe(target);
expect(await f.lastCommit(f.beta), label).toBe(target);
});
-}, 600_000);
+}, 900_000);
-test('7d. #7679 state matrix: an unreadable HEAD and pending worktree recovery never move an interrupted refresh the unattended paths cannot verify', async () => {
+test('7d. #7679 state matrix: an interrupted refresh that the unattended paths cannot verify, or that a live or remote holder owns, is never moved', async () => {
// Value: protects=a plain refresh leaves an orphan untouched (state, holder, previous_holder, HEAD) when git cannot read HEAD (refusing
- // refresh_in_progress) or when publication/topology recovery is pending (refusing refresh_recovery_required, the blocker that must clear first);
- // the owner scan aborts a dead drain whatever HEAD reads, never adopts an unverified HEAD for a fenced row, and leaves a lagging syncing row;
- // fails_when=adoption runs recovery on a null HEAD (escalating to recovery_required) or ahead of the recovery gate, or precheck names the
- // orphan instead of the pending recovery; why_new=7c covers readable HEADs with no pending recovery only; seam=.git/HEAD overwritten with
- // garbage, a recovering topology change row naming the worktree; no_claim=the race where HEAD becomes unreadable between adoption's two reads.
- type Kind = 'draining' | 'fenced_old' | 'fenced_target' | 'syncing_lagging' | 'recovery_required';
- type Action = 'plain_head_unreadable' | 'scan_head_unreadable' | 'plain_recovery_pending';
- const crashAt = { draining: 'drained', fenced_old: 'fenced', fenced_target: 'fenced', syncing_lagging: 'syncing', recovery_required: 'fenced' } as const;
- const kinds: Kind[] = ['draining', 'fenced_old', 'fenced_target', 'syncing_lagging', 'recovery_required'];
- const cells: Array<[Kind, Action]> = [
- ...(['draining', 'fenced_old', 'syncing_lagging'] as Kind[]).map(kind => [kind, 'plain_head_unreadable'] as [Kind, Action]),
- ...(['draining', 'fenced_old', 'syncing_lagging'] as Kind[]).map(kind => [kind, 'scan_head_unreadable'] as [Kind, Action]),
- ...kinds.map(kind => [kind, 'plain_recovery_pending'] as [Kind, Action])];
+ // refresh_in_progress), when publication/topology recovery is pending or the worktree is not active (refusing refresh_recovery_required, the
+ // blocker that must clear first), when a live process holds the refresh lock, or when its holder is on another host; the owner scan aborts a dead
+ // drain whatever HEAD reads and leaves every other kind on an unreadable HEAD; fails_when=adoption runs recovery on a null HEAD, ahead of the
+ // recovery gate, against a live lock or another host's holder, or the scan escalates an unreadable HEAD; why_new=7c covers readable HEADs with
+ // free locks and no pending recovery only; seam=.git/HEAD overwritten with garbage, a recovering topology change row, the worktree state column,
+ // the refresh lock held in this process, a foreign holder host id; no_claim=the race where HEAD becomes unreadable between adoption's two reads.
+ type Kind = 'draining' | 'fenced_old' | 'fenced_target' | 'merged_target' | 'syncing_lagging' | 'syncing_at_target' | 'recovery_required';
+ type Action = 'plain_head_unreadable' | 'scan_head_unreadable' | 'plain_recovery_pending' | 'plain_worktree_inactive' | 'live_holder' | 'other_host_holder';
+ const kinds: Kind[] = ['draining', 'fenced_old', 'fenced_target', 'merged_target', 'syncing_lagging', 'syncing_at_target', 'recovery_required'];
+ const actions: Action[] = ['plain_head_unreadable', 'scan_head_unreadable', 'plain_recovery_pending', 'plain_worktree_inactive', 'live_holder', 'other_host_holder'];
+ const stateOf = (kind: Kind) => kind === 'recovery_required' ? 'recovery_required' : kind.split('_')[0];
+ const crashAt = (kind: Kind) => kind === 'draining' ? 'drained' : kind.startsWith('syncing') ? 'syncing' : 'fenced';
+ const cells = kinds.flatMap(kind => actions.map(action => [kind, action] as [Kind, Action]));
for (const engine of engines) for (const [kind, action] of cells) await withRefreshFixture(engine, async f => {
const label = `${kind} x ${action}`;
const target = f.push('alpha/two.md', page('Alpha two', 'Upstream.'));
- await expect(refreshWorktree(f.engine, f.alpha, { hooks: { boundary: point => { if (point === crashAt[kind]) throw new Error(`simulated crash at ${point}`); } } }))
+ await expect(refreshWorktree(f.engine, f.alpha, { hooks: { boundary: point => { if (point === crashAt(kind)) throw new Error(`simulated crash at ${point}`); } } }))
.rejects.toThrow('simulated crash');
await disposePersistenceConsumer(f.engine);
const [orphan] = await f.refreshRows();
- const state = kind === 'draining' ? 'draining' : kind === 'syncing_lagging' ? 'syncing' : kind === 'recovery_required' ? 'recovery_required' : 'fenced';
+ const state = stateOf(kind);
await f.engine.executeRaw(`UPDATE persistence_worktree_refreshes SET state=$2,completed_at=NULL WHERE id=$1::uuid`, [orphan.id, state]);
- if (kind === 'fenced_target') git(f.root, 'merge', '-q', '--ff-only', target);
+ if (kind === 'fenced_target' || kind === 'merged_target') git(f.root, 'merge', '-q', '--ff-only', target);
+ if (kind === 'syncing_at_target') for (const member of [f.alpha, f.beta]) await f.engine.transaction(tx => withCoordinatedWrite(tx, [member],
+ () => tx.executeRaw('UPDATE sources SET last_commit=$2 WHERE id=$1', [member, target]), TEST_WRITE_ATTRIBUTION));
const head = f.head();
- const settled = async () => (await f.refreshRows()).map(r => [r.state, r.outcome.previous_holder]);
- if (action === 'plain_recovery_pending') {
- await f.engine.executeRaw(`INSERT INTO persistence_topology_changes(principal_id,request_id,digest,operation,source_id,worktree_ids,state,recovery)
+ const settled = async () => (await f.refreshRows()).map(r => [r.state, (r.outcome.holder as { pid?: number } | undefined)?.pid, r.outcome.previous_holder]);
+ const before = await settled();
+ const plainCode = kind === 'recovery_required' ? 'refresh_recovery_required' : 'refresh_in_progress';
+ const writerStatus = async () => (await runPersistenceAdministration(f.engine, 'writer_status', { source_id: f.alpha }) as { worktree_refreshes: Array<Record<string, any>> }).worktree_refreshes[0];
+ if (action === 'plain_recovery_pending' || action === 'plain_worktree_inactive') {
+ if (action === 'plain_recovery_pending') await f.engine.executeRaw(`INSERT INTO persistence_topology_changes(principal_id,request_id,digest,operation,source_id,worktree_ids,state,recovery)
VALUES($1::uuid,$2::uuid,'example','reclone',$3,ARRAY[$4::uuid],'recovering',$5::text::jsonb)`,
[randomUUID(), randomUUID(), f.alpha, f.worktreeId, JSON.stringify({ worktreeId: f.worktreeId })]);
+ else await f.engine.executeRaw(`UPDATE persistence_worktrees SET state='recovering' WHERE id=$1::uuid`, [f.worktreeId]);
await refusedWith(refreshWorktree(f.engine, f.alpha), 'refresh_recovery_required');
- expect(await settled(), label).toEqual([[state, undefined]]);
- expect(f.head(), label).toBe(head);
- return;
+ // Status does not promise that the next refresh finishes an orphan the recovery gate holds back.
+ if (kind !== 'recovery_required') expect(await writerStatus(), label).toMatchObject({ liveness: 'orphaned', auto_recovery: 'none',
+ fix: { argv: ['gbrain', 'sources', 'writer', 'status', '--source', f.alpha, '--json'] } });
+ } else if (action === 'live_holder') {
+ const lock = await tryAcquireNativeLock(refreshLockPath(f.worktreeId));
+ expect(lock, label).not.toBeNull();
+ try {
+ expect(await resumeWorktreeRefreshes(f.engine), label).toBe(0);
+ await refusedWith(refreshWorktree(f.engine, f.alpha), plainCode);
+ await refusedWith(refreshWorktree(f.engine, f.alpha, { resume: true }), 'refresh_in_progress');
+ expect(await writerStatus(), label).toMatchObject({ liveness: 'live', auto_recovery: 'none' });
+ } finally { await lock!.release(); }
+ } else if (action === 'other_host_holder') {
+ await f.engine.executeRaw(`UPDATE persistence_worktree_refreshes SET outcome=jsonb_set(outcome,'{holder,host_id}',to_jsonb($2::text)) WHERE id=$1::uuid`, [orphan.id, randomUUID()]);
+ await refusedWith(refreshWorktree(f.engine, f.alpha), plainCode);
+ if (kind !== 'recovery_required') expect(await writerStatus(), label).toMatchObject({ liveness: 'unknown', auto_recovery: 'unknown', fix: { actor: 'host_admin' },
+ next: `gbrain sources writer status --source ${f.alpha} --json` });
+ } else {
+ const headFile = join(f.root, '.git', 'HEAD');
+ const saved = readFileSync(headFile, 'utf8');
+ writeFileSync(headFile, 'not a ref\n');
+ try {
+ if (action === 'plain_head_unreadable') {
+ const refused = await refusedWith(refreshWorktree(f.engine, f.alpha), plainCode);
+ if (kind !== 'recovery_required') expect(refused.suggestion, label).toContain(`gbrain sources writer status --source ${f.alpha} --json`);
+ } else {
+ // The scan aborts a dead drain (it does not depend on HEAD) and leaves every other kind for a scan that can read HEAD.
+ expect(await resumeWorktreeRefreshes(f.engine), label).toBe(kind === 'draining' ? 1 : 0);
+ expect((await f.refreshRows()).map(r => r.state), label).toEqual([kind === 'draining' ? 'aborted' : state]);
+ }
+ } finally { writeFileSync(headFile, saved); }
+ if (action === 'scan_head_unreadable') { expect(f.head(), label).toBe(head); return; }
}
- const headFile = join(f.root, '.git', 'HEAD');
- const saved = readFileSync(headFile, 'utf8');
- writeFileSync(headFile, 'not a ref\n');
- try {
- if (action === 'plain_head_unreadable') {
- const refused = await refusedWith(refreshWorktree(f.engine, f.alpha), 'refresh_in_progress');
- expect(refused.suggestion, label).toContain(`gbrain sources writer status --source ${f.alpha} --json`);
- expect(await settled(), label).toEqual([[state, undefined]]);
- } else {
- const moved = await resumeWorktreeRefreshes(f.engine);
- const expected = { draining: 'aborted', fenced_old: 'recovery_required', syncing_lagging: 'syncing' }[kind as 'draining' | 'fenced_old' | 'syncing_lagging'];
- expect(moved, label).toBe(expected === state ? 0 : 1);
- expect((await f.refreshRows()).map(r => r.state), label).toEqual([expected]);
- }
- } finally { writeFileSync(headFile, saved); }
+ expect(await settled(), label).toEqual(before);
expect(f.head(), label).toBe(head);
});
-}, 600_000);
+}, 900_000);
+
+test('7e. #7679: the owner scan leaves an orphaned syncing refresh while one member lags behind its target, and completes it once both reach it', () => each(async f => {
+ // Ship audit (Step 7). Value: protects=the owner scan completes an orphaned syncing row only when every member reached the target, even when
+ // the SQL prefilter admits it (a member past old_head but short of the target); fails_when=completeSyncedOrphan or membersAtTarget stops
+ // checking every member; why_new=every scan test leaves members at old_head, which the prefilter alone skips; seam=none; no_claim=a real
+ // process death (the crash is a hook throw) and member syncs (last_commit is set directly, as the admission test does).
+ const first = f.push('alpha/two.md', page('Alpha two', 'Upstream first.'));
+ const target = f.push('alpha/three.md', page('Alpha three', 'Upstream second.'));
+ await expect(refreshWorktree(f.engine, f.alpha, { hooks: { boundary: point => { if (point === 'syncing') throw new Error('simulated crash at syncing'); } } }))
+ .rejects.toThrow('simulated crash');
+ await disposePersistenceConsumer(f.engine);
+ const [orphan] = await f.refreshRows();
+ // The resident consumer's owner scan may already have settled it; restore what the crash left.
+ await f.engine.executeRaw(`UPDATE persistence_worktree_refreshes SET state='syncing',completed_at=NULL WHERE id=$1::uuid`, [orphan.id]);
+ expect(f.head()).toBe(target);
+ const setLastCommit = (member: string, sha: string) => f.engine.transaction(tx => withCoordinatedWrite(tx, [member],
+ () => tx.executeRaw('UPDATE sources SET last_commit=$2 WHERE id=$1', [member, sha]), TEST_WRITE_ATTRIBUTION));
+ await setLastCommit(f.alpha, target);
+ await setLastCommit(f.beta, first);
+ expect(await resumeWorktreeRefreshes(f.engine)).toBe(0);
+ expect((await f.refreshRows()).map(r => r.state)).toEqual(['syncing']);
+ await setLastCommit(f.beta, target);
+ expect(await resumeWorktreeRefreshes(f.engine)).toBe(1);
+ const [row] = await f.refreshRows();
+ expect(row).toMatchObject({ id: orphan.id, state: 'completed', outcome: { completed_by: 'owner_recovery' } });
+}), 180_000);
+
+test('7f. #7679: a plain refresh whose adopted orphan ends sync_blocked returns that result and does not refresh over it', () => each(async f => {
+ // Ship audit (Step 7, pass 2). Value: protects=a plain refresh that adopts an orphan whose member sync blocks returns sync_blocked naming the
+ // member's resume and leaves HEAD at the orphan's target; fails_when=refreshWorktree drops its non-terminal early return and refreshes on;
+ // why_new=no adoption test blocks a member sync (#7706); seam=none; no_claim=a real process death (the crash is a hook throw) and other
+ // non-terminal recovery statuses (sync.holds=fail stands in for any blocked member write).
+ const target = f.push('alpha/broken.md', '---\ntitle: alice-example first line\nalice-example second line\n---\nA synthetic post.\n');
+ await expect(refreshWorktree(f.engine, f.alpha, { hooks: { boundary: point => { if (point === 'syncing') throw new Error('simulated crash at syncing'); } } }))
+ .rejects.toThrow('simulated crash');
+ await disposePersistenceConsumer(f.engine);
+ const [orphan] = await f.refreshRows();
+ await f.engine.executeRaw(`UPDATE persistence_worktree_refreshes SET state='syncing',completed_at=NULL WHERE id=$1::uuid`, [orphan.id]);
+ expect(f.head()).toBe(target);
+ const next = f.push('alpha/three.md', page('Alpha three', 'Landed upstream while the refresh was orphaned.'));
+ await f.engine.setConfig('sync.holds', 'fail');
+ try {
+ const result = await refreshWorktree(f.engine, f.alpha);
+ expect(result).toMatchObject({ status: 'sync_blocked', refresh_id: orphan.id, target_head: target,
+ sync_blocked: [{ source_id: f.alpha, resume: `gbrain sync --source ${f.alpha} --no-pull` }],
+ recovered: { refresh_id: orphan.id, state: 'completed', status: 'sync_blocked', previous_holder: { pid: process.pid } } });
+ expect(result.next).toContain(`gbrain sync --source ${f.alpha} --no-pull`);
+ expect(f.head()).toBe(target);
+ expect(f.head()).not.toBe(next);
+ const rows = await f.refreshRows();
+ expect(rows.map(r => r.state)).toEqual(['completed']);
+ expect(rows[0].outcome.sync_blocked).toEqual([expect.objectContaining({ source_id: f.alpha })]);
+ } finally { await f.engine.unsetConfig('sync.holds'); }
+}), 180_000);
+
+test('7g. #7679: a backlog of 16 orphaned syncing refreshes waiting on their members does not starve the owner scan\'s rollback of a dead drain', () => each(async f => {
+ // Ship audit (Step 7, pass 2). Value: protects=the owner scan settles a pre-merge orphan (its fence refuses every write) even when 16 older
+ // syncing orphans fill its LIMIT; fails_when=the scan's ORDER BY stops putting syncing rows last; why_new=every scan test has one row;
+ // seam=none; no_claim=16 real worktrees (the backlog is worktree and refresh rows inserted with SQL, with no host binding).
+ f.push('alpha/two.md', page('Alpha two', 'Upstream.'));
+ await expect(refreshWorktree(f.engine, f.alpha, { hooks: { boundary: point => { if (point === 'drained') throw new Error('simulated crash at drained'); } } }))
+ .rejects.toThrow('simulated crash');
+ await disposePersistenceConsumer(f.engine);
+ const [orphan] = await f.refreshRows();
+ await f.engine.executeRaw(`UPDATE persistence_worktree_refreshes SET state='draining',completed_at=NULL WHERE id=$1::uuid`, [orphan.id]);
+ const [{ owner_host_id: host, principal_id: principal }] = await f.engine.executeRaw<{ owner_host_id: string; principal_id: string }>(
+ `SELECT w.owner_host_id::text AS owner_host_id, r.principal_id::text AS principal_id FROM persistence_worktrees w
+ JOIN persistence_worktree_refreshes r ON r.worktree_id=w.id WHERE r.id=$1::uuid`, [orphan.id]);
+ const backlog: string[] = [];
+ try {
+ for (let i = 0; i < 16; i++) {
+ const [{ id }] = await f.engine.executeRaw<{ id: string }>('INSERT INTO persistence_worktrees(owner_host_id) VALUES($1::uuid) RETURNING id::text AS id', [host]);
+ backlog.push(id);
+ await f.engine.executeRaw(`INSERT INTO persistence_worktree_refreshes(worktree_id,source_ids,principal_id,owner_epoch,topology_generation,state,old_head,target_head,upstream_ref,created_at)
+ VALUES($1::uuid,ARRAY[$2]::text[],$3::uuid,0,1,'syncing',$4,$4,'origin/main',now()-interval '1 hour')`, [id, f.alpha, principal, 'f'.repeat(40)]);
+ }
+ expect(await resumeWorktreeRefreshes(f.engine)).toBe(1);
+ expect((await f.refreshRows()).map(r => r.state)).toEqual(['aborted']);
+ } finally {
+ if (backlog.length) {
+ await f.engine.executeRaw('DELETE FROM persistence_worktree_refreshes WHERE worktree_id=ANY($1::uuid[])', [backlog]);
+ await f.engine.executeRaw('DELETE FROM persistence_worktrees WHERE id=ANY($1::uuid[])', [backlog]);
+ }
+ }
+}), 180_000);
test('8. a queued git effect drains before fenced, and nothing on the worktree is claimed while fenced', () => each(async f => {
f.push('alpha/two.md', page('Alpha two', 'Upstream.'));
diff --git a/test/worktree-refresh-orphan-7679.test.ts b/test/worktree-refresh-orphan-7679.test.ts
index a4d27f616a..96ee14ac82 100644
--- a/test/worktree-refresh-orphan-7679.test.ts
+++ b/test/worktree-refresh-orphan-7679.test.ts
@@ -26,7 +26,7 @@ import { join } from 'node:path';
import type { BrainEngine } from '../src/core/engine.ts';
import type { GBrainConfig } from '../src/core/config.ts';
import { PGLiteEngine } from '../src/core/pglite-engine.ts';
-import { runSourcesRefresh } from '../src/commands/sources-refresh.ts';
+import { printRefreshOutcome, runSourcesRefresh } from '../src/commands/sources-refresh.ts';
import { _resetCliExitVerdictForTests } from '../src/core/cli-force-exit.ts';
import { runPersistenceAdministration } from '../src/core/persistence/administration.ts';
import { disposePersistenceConsumer } from '../src/core/persistence/service.ts';
@@ -210,6 +210,13 @@ test('#7679: an orphan written before holders were recorded is reported orphaned
expect(status.fix.why).toContain('a process that recorded no holder');
const result = await refreshWorktree(f.engine, f.alpha);
expect(result).toMatchObject({ status: 'already_current', recovered: { refresh_id: orphan.id, state: 'completed', status: 'completed', previous_holder: null } });
+ // Ship review: text mode (the cron default) reports the recovery too. Value: protects=a plain refresh's text output names the orphan it
+ // recovered; fails_when=describe drops the recovered field; why_new=only --json read it; seam=none; no_claim=none beyond protects.
+ const out: string[] = [];
+ const log = console.log;
+ console.log = (line: string) => { out.push(line); };
+ try { printRefreshOutcome({ result }, false); } finally { console.log = log; }
+ expect(out.join('\n')).toContain(`Recovered orphaned refresh ${orphan.id} (completed; previous holder: not recorded).`);
const rows = await f.refreshRows() as Row[];
expect(rows.map(row => row.state)).toEqual(['completed']);
expect(rows[0].outcome.holder.pid).toBe(process.pid);
diff --git a/test/worktree-refresh-recovery-row-gbra130.test.ts b/test/worktree-refresh-recovery-row-gbra130.test.ts
index 696fe84115..f28ea1ce46 100644
--- a/test/worktree-refresh-recovery-row-gbra130.test.ts
+++ b/test/worktree-refresh-recovery-row-gbra130.test.ts
@@ -50,7 +50,7 @@ test('one refresh whose owner-start convergence fails unexpectedly is left for t
await recoverSourceTopologies(f.engine);
} finally { tx.mockRestore(); err.mockRestore(); }
expect(failed).toBe(true);
- expect(logged.some(line => line.includes('did not converge at owner start'))).toBe(true);
+ expect(logged.some(line => line.includes('did not converge in the owner\'s topology scan'))).toBe(true);
expect(((await f.refreshRows()) as Array<{ state: string }>).map(row => row.state)).toEqual(interrupted.map(row => row.state));
await recoverSourceTopologies(f.engine);
expect(((await f.refreshRows()) as Array<{ state: string }>).map(row => row.state)).toEqual(['syncing']);