Garry's Opinionated OpenClaw/Hermes Agent Brain
TypeScript
fix: recover a refresh crash whether or not its merge ran (GST-1674)
garrytan149455d4d867
diff --git a/docs/designs/REFRESH_WRITE_BARRIER.md b/docs/designs/REFRESH_WRITE_BARRIER.mdindex 9029aff262..091169488b 100644--- a/docs/designs/REFRESH_WRITE_BARRIER.md+++ b/docs/designs/REFRESH_WRITE_BARRIER.md@@ -248,6 +248,15 @@ Deferred rows are durable rows; nothing about them lives in process memory.- Admitting process dies after commit: the receipt exists; the caller replays the request id.- Refresh process dies: its row stays active; orphan adoption or `sources refresh --resume`completes or aborts it, and the trigger releases it. `--abandon` aborts, which also releases.+ The row records `outcome.merge_started` (false when it is fenced, true just before+ `git merge --ff-only`), so recovery tells a merge that never ran from one that ran:+ - Killed while `fenced`, before the merge started, with HEAD moved past `old_head` on its own line+ (writes that landed while it drained, or a commit gbrain did not make): recovery aborts the row,+ as the live tick would, and its deferred writes commit (GBEV-44, GST-1674).+ - Killed after the merge, with HEAD at the target or a commit on top of it and no member synced off+ it: recovery records the merge and the next refresh finishes the member syncs, with no `--resume`.+ - A merge that started and a HEAD that no longer holds the target (a reset and an unrelated commit)+ stays `recovery_required` for the operator.- Consumer dies after claiming: the row is `running` with a lease; ordinary expiry and reclaim.- Release and the deferred-to-running claim are single statements.diff --git a/src/core/persistence/worktree-refresh.ts b/src/core/persistence/worktree-refresh.tsindex 1c16443d59..ca425310aa 100644--- a/src/core/persistence/worktree-refresh.ts+++ b/src/core/persistence/worktree-refresh.ts@@ -164,13 +164,15 @@ async function ownCommits(git: Git, root: string, from: string, to: string): Pro}const OWN_COMMITS = "gbrain's own commits: writes admitted before the fence that landed while the refresh drained";/**- * GBEV-44: a `fenced` row whose HEAD moved past old_head only through gbrain's own commits, with nothing of the target merged and no- * member synced off HEAD's line. The live refresh aborts this case before merging, so recovery aborts it too instead of holding the fence.+ * GBEV-44, GST-1674: a `fenced` row whose merge never started (`outcome.merge_started` is still false; it turns true, durably, just before+ * `git merge --ff-only`) and whose HEAD moved past old_head on its own line, by gbrain's commits or anyone's, with no member synced off it.+ * The live refresh aborts this case before merging, so recovery aborts it too instead of holding the fence. A row whose merge started+ * (a reset and an unrelated commit after the merge) is not this case and still needs the operator.*/async function movedBeforeMerge(engine: BrainEngine, git: Git, root: string, row: WorktreeRefreshRow, head: string | null): Promise<boolean> {- return row.state === 'fenced' && head !== null && head !== row.old_head && head !== row.target_head+ return row.state === 'fenced' && row.outcome.merge_started === false && head !== null && head !== row.old_head&& await ancestry(git, root, row.old_head, head) === true && await ancestry(git, root, row.target_head, head) === false- && await ownCommits(git, root, row.old_head, head) && !await memberOutsideHead(engine, git, root, row, head);+ && !await memberOutsideHead(engine, git, root, row, head);}const keepSyncedWayOut = (row: Pick<WorktreeRefreshRow, 'source_ids'>, root: string, head: string) =>`Ask the user first: git -C ${q(root)} merge --ff-only ${head} moves the checkout to the commit its members already synced (it keeps uncommitted edits git can carry; git -C ${q(root)} status --short lists them), then gbrain sources refresh ${row.source_ids[0]} --resume finishes the refresh and keeps the pages its members synced. If git refuses (an uncommitted edit overlaps an incoming file, or HEAD is not an ancestor of ${head}), ask the user before git -C ${q(root)} reset --hard ${head}, which discards local commits and uncommitted edits not in it, then --resume.`;@@ -873,7 +875,7 @@ async function fenceAndMerge(engine: BrainEngine, git: Git, sourceId: string, plif (counts.requests + counts.effects + counts.recovery > 0) throw retryInTick(refusal('refresh_source_changed',`Work reached the worktree after the drain check (${counts.requests} request(s), ${counts.effects} checkout effect(s), ${counts.recovery} recovery record(s)); nothing moved.`,`Retry gbrain sources refresh ${sourceId}`, row.id));- if (!await transition(tx, row, ['draining'], 'fenced')) throw refusal('refresh_source_changed', 'The refresh record was moved by recovery while draining.', `gbrain sources refresh ${sourceId} --resume`, row.id);+ if (!await transition(tx, row, ['draining'], 'fenced', { merge_started: false })) throw refusal('refresh_source_changed', 'The refresh record was moved by recovery while draining.', `gbrain sources refresh ${sourceId} --resume`, row.id);}).catch(error => { throw (error as { code?: string }).code === LOCK_NOT_AVAILABLE ? new FenceLockTimeout() : error; });const current = await readRefresh(engine, row.id);await hooks?.boundary?.('fenced', current);@@ -891,6 +893,10 @@ async function fenceAndMerge(engine: BrainEngine, git: Git, sourceId: string, pl}const dirt = await dirtyOverlap(git, plan.root, row.old_head, row.target_head);if (dirt.overlap.length) await abort(engine, current, ['fenced'], dirtyRefusal(sourceId, dirt.overlap));+ await engine.transaction(async tx => {+ await tx.executeRaw("SELECT set_config('synchronous_commit','on',true)");+ await tx.executeRaw(`UPDATE persistence_worktree_refreshes SET outcome=outcome||'{"merge_started":true}'::jsonb,updated_at=now() WHERE id=$1::uuid AND state='fenced'`, [row.id]);+ });let merge = await git(['-C', plan.root, 'merge', '--ff-only', '--no-edit', '--quiet', row.target_head], 120_000);let after = await headOf(git, plan.root);for (let attempt = 1; attempt < INDEX_LOCK_ATTEMPTS && merge.code !== 0 && after === row.old_head && INDEX_LOCKED.test(merge.stderr); attempt++) {@@ -949,10 +955,11 @@ async function recoverOne(engine: BrainEngine, git: Git, row: WorktreeRefreshRow} else if (row.state === 'fenced' && head === row.old_head) {await engine.transaction(tx => transition(tx, row, ['fenced'], 'aborted', { refusal: { code: 'refresh_interrupted', cause: 'The refreshing process stopped before merging; HEAD is unchanged.' } }));} else if (await movedBeforeMerge(engine, git, root, row, head)) {- const cause = `The refreshing process stopped before merging; HEAD moved from ${row.old_head} to ${head} (${OWN_COMMITS}) and nothing was merged.`;+ const by = await ownCommits(git, root, row.old_head, head!) ? OWN_COMMITS : 'a commit or checkout gbrain did not make';+ const cause = `The refreshing process stopped before merging; HEAD moved from ${row.old_head} to ${head} (${by}) and nothing was merged.`;await engine.transaction(tx => transition(tx, row, ['fenced'], 'aborted', { refusal: { code: 'refresh_interrupted', cause }, observed_head: head }));- } else if (['fenced', 'merged', 'recovery_required'].includes(row.state) && (head === row.target_head || (row.state === 'recovery_required' && head !== null && await isAncestor(git, root, row.target_head, head)))- && await isAncestor(git, root, row.old_head, row.target_head) && (row.state !== 'recovery_required' || !await memberOutsideHead(engine, git, root, row, head!))) {+ } else if (['fenced', 'merged', 'recovery_required'].includes(row.state) && head !== null && (head === row.target_head || await isAncestor(git, root, row.target_head, head))+ && await isAncestor(git, root, row.old_head, row.target_head) && ((row.state !== 'recovery_required' && head === row.target_head) || !await memberOutsideHead(engine, git, root, row, head))) {await bookkeeping(engine, row);} else if (row.state === 'syncing' && head !== null && (head === row.target_head || await isAncestor(git, root, row.target_head, head))) {// stays syncing; the member syncs run from --resume or any later cycle sync.diff --git a/test/refresh-write-barrier.test.ts b/test/refresh-write-barrier.test.tsindex b1a31d7da9..ba26ef2d1d 100644--- a/test/refresh-write-barrier.test.ts+++ b/test/refresh-write-barrier.test.ts@@ -577,3 +577,39 @@ test('GBEV-44: the owner scan aborts an orphaned fenced row whose HEAD moved onlexpect((await settle(f, deferred)).state).toBe('committed');}), 180_000);++test('GST-1674: a refresh killed at fenced after a commit gbrain did not make moved HEAD before the merge resumes: the row aborts and its deferred write commits', () => each(async f => {+ const { deferred, landed } = await crashAtFenced(f, () => commitFile(f.root, 'alpha/outside.md', page('Outside', 'A commit made around gbrain.'), 'outside'));+ const resumed = await refreshWorktree(f.engine, f.alpha, { resume: true });+ expect(resumed.status).toBe('abandoned');+ const [row] = await f.refreshRows();+ expect(row.state).toBe('aborted');+ expect((row.outcome.refusal as { code: string }).code).toBe('refresh_interrupted');+ expect(f.head()).toBe(landed);+ expect((await settle(f, deferred)).state).toBe('committed');+}), 180_000);++test('GST-1674: a refresh killed after its merge, with a commit landed on the target before bookkeeping, recovers on the owner scan and completes without --resume', () => each(async f => {+ const target = f.push('alpha/two.md', page('Alpha two', 'An upstream observation.'));+ let deferred!: WriteRequest; let landed!: string;+ await expect(refreshWorktree(f.engine, f.alpha, { hooks: { boundary: async point => {+ if (point === 'drained') {+ const out = await call(f, 'put_page', { slug: 'notes/held-past-merge', content: page('Held past merge', 'body') });+ deferred = await requestRow(f, out.body.request_id);+ }+ if (point === 'merged') {+ landed = commitFile(f.root, 'alpha/after-merge.md', page('After merge', 'A commit on top of the target.'), 'gbrain: persist canonical memory update');+ throw new Error('simulated crash after merge');+ }+ } } })).rejects.toThrow('simulated crash');+ expect(git(f.root, 'rev-parse', 'HEAD~1')).toBe(target);+ await resumeWorktreeRefreshes(f.engine);+ expect((await f.refreshRows()).map(r => r.state)).toEqual(['syncing']);+ const next = await refreshWorktree(f.engine, f.alpha).catch(error => error);+ // The scheduled refresh finishes the orphan first; its own tick then stops at the unpublished local commit.+ expect(next.code).toBe('refresh_diverged');+ expect((await f.refreshRows()).map(r => r.state)).toEqual(['completed']);+ expect(f.head()).toBe(landed);+ expect((await settle(f, deferred)).state).toBe('committed');+}), 180_000);+diff --git a/test/worktree-refresh-settle-matrix-7679.test.ts b/test/worktree-refresh-settle-matrix-7679.test.tsindex 24e498a3f0..3eaa58b9da 100644--- a/test/worktree-refresh-settle-matrix-7679.test.ts+++ b/test/worktree-refresh-settle-matrix-7679.test.ts@@ -86,7 +86,9 @@ test('7c. #7679 state matrix: an interrupted refresh of each kind, settled by thawait disposePersistenceConsumer(f.engine);const [orphan] = await cellRows();// The resident consumer's owner scan may already have settled it; restore what the crash left.- await f.engine.executeRaw(`UPDATE persistence_worktree_refreshes SET state=$2,completed_at=NULL WHERE id=$1::uuid`, [orphan.id, stateOf(kind)]);+ // GST-1674: an _unrelated kind is a merge that ran and was then reset under an unrelated commit, so its row records the merge as started.+ await f.engine.executeRaw(`UPDATE persistence_worktree_refreshes SET state=$2,completed_at=NULL,outcome=CASE WHEN $3::boolean THEN outcome||'{"merge_started":true}'::jsonb ELSE outcome END WHERE id=$1::uuid`,+ [orphan.id, stateOf(kind), kind.includes('_unrelated')]);if (kind === 'fenced_target' || kind === 'merged_target') git(f.root, 'merge', '-q', '--ff-only', target);if (kind === 'syncing_descendant') commitFile(f.root, 'beta/local.md', page('Local', 'Committed on top of the target.'));if (kind.includes('_unrelated')) { git(f.root, 'reset', '-q', '--hard', old); commitFile(f.root, 'beta/unrelated.md', page('Unrelated', 'Not the upstream commit.')); }