Skip to content

Garry's Opinionated OpenClaw/Hermes Agent Brain

TypeScript

fix: recover a refresh crash whether or not its merge ran (GST-1674)

garrytan149455d4d867
diff --git a/docs/designs/REFRESH_WRITE_BARRIER.md b/docs/designs/REFRESH_WRITE_BARRIER.md
index 9029aff262..091169488b 100644
--- a/docs/designs/REFRESH_WRITE_BARRIER.md
+++ b/docs/designs/REFRESH_WRITE_BARRIER.md
@@ -248,6 +248,15 @@ Deferred rows are durable rows; nothing about them lives in process memory.
- Admitting process dies after commit: the receipt exists; the caller replays the request id.
- Refresh process dies: its row stays active; orphan adoption or `sources refresh --resume`
completes or aborts it, and the trigger releases it. `--abandon` aborts, which also releases.
+ The row records `outcome.merge_started` (false when it is fenced, true just before
+ `git merge --ff-only`), so recovery tells a merge that never ran from one that ran:
+ - Killed while `fenced`, before the merge started, with HEAD moved past `old_head` on its own line
+ (writes that landed while it drained, or a commit gbrain did not make): recovery aborts the row,
+ as the live tick would, and its deferred writes commit (GBEV-44, GST-1674).
+ - Killed after the merge, with HEAD at the target or a commit on top of it and no member synced off
+ it: recovery records the merge and the next refresh finishes the member syncs, with no `--resume`.
+ - A merge that started and a HEAD that no longer holds the target (a reset and an unrelated commit)
+ stays `recovery_required` for the operator.
- Consumer dies after claiming: the row is `running` with a lease; ordinary expiry and reclaim.
- Release and the deferred-to-running claim are single statements.
diff --git a/src/core/persistence/worktree-refresh.ts b/src/core/persistence/worktree-refresh.ts
index 1c16443d59..ca425310aa 100644
--- a/src/core/persistence/worktree-refresh.ts
+++ b/src/core/persistence/worktree-refresh.ts
@@ -164,13 +164,15 @@ async function ownCommits(git: Git, root: string, from: string, to: string): Pro
}
const OWN_COMMITS = "gbrain's own commits: writes admitted before the fence that landed while the refresh drained";
/**
- * GBEV-44: a `fenced` row whose HEAD moved past old_head only through gbrain's own commits, with nothing of the target merged and no
- * member synced off HEAD's line. The live refresh aborts this case before merging, so recovery aborts it too instead of holding the fence.
+ * GBEV-44, GST-1674: a `fenced` row whose merge never started (`outcome.merge_started` is still false; it turns true, durably, just before
+ * `git merge --ff-only`) and whose HEAD moved past old_head on its own line, by gbrain's commits or anyone's, with no member synced off it.
+ * The live refresh aborts this case before merging, so recovery aborts it too instead of holding the fence. A row whose merge started
+ * (a reset and an unrelated commit after the merge) is not this case and still needs the operator.
*/
async function movedBeforeMerge(engine: BrainEngine, git: Git, root: string, row: WorktreeRefreshRow, head: string | null): Promise<boolean> {
- return row.state === 'fenced' && head !== null && head !== row.old_head && head !== row.target_head
+ return row.state === 'fenced' && row.outcome.merge_started === false && head !== null && head !== row.old_head
&& await ancestry(git, root, row.old_head, head) === true && await ancestry(git, root, row.target_head, head) === false
- && await ownCommits(git, root, row.old_head, head) && !await memberOutsideHead(engine, git, root, row, head);
+ && !await memberOutsideHead(engine, git, root, row, head);
}
const keepSyncedWayOut = (row: Pick<WorktreeRefreshRow, 'source_ids'>, root: string, head: string) =>
`Ask the user first: git -C ${q(root)} merge --ff-only ${head} moves the checkout to the commit its members already synced (it keeps uncommitted edits git can carry; git -C ${q(root)} status --short lists them), then gbrain sources refresh ${row.source_ids[0]} --resume finishes the refresh and keeps the pages its members synced. If git refuses (an uncommitted edit overlaps an incoming file, or HEAD is not an ancestor of ${head}), ask the user before git -C ${q(root)} reset --hard ${head}, which discards local commits and uncommitted edits not in it, then --resume.`;
@@ -873,7 +875,7 @@ async function fenceAndMerge(engine: BrainEngine, git: Git, sourceId: string, pl
if (counts.requests + counts.effects + counts.recovery > 0) throw retryInTick(refusal('refresh_source_changed',
`Work reached the worktree after the drain check (${counts.requests} request(s), ${counts.effects} checkout effect(s), ${counts.recovery} recovery record(s)); nothing moved.`,
`Retry gbrain sources refresh ${sourceId}`, row.id));
- if (!await transition(tx, row, ['draining'], 'fenced')) throw refusal('refresh_source_changed', 'The refresh record was moved by recovery while draining.', `gbrain sources refresh ${sourceId} --resume`, row.id);
+ if (!await transition(tx, row, ['draining'], 'fenced', { merge_started: false })) throw refusal('refresh_source_changed', 'The refresh record was moved by recovery while draining.', `gbrain sources refresh ${sourceId} --resume`, row.id);
}).catch(error => { throw (error as { code?: string }).code === LOCK_NOT_AVAILABLE ? new FenceLockTimeout() : error; });
const current = await readRefresh(engine, row.id);
await hooks?.boundary?.('fenced', current);
@@ -891,6 +893,10 @@ async function fenceAndMerge(engine: BrainEngine, git: Git, sourceId: string, pl
}
const dirt = await dirtyOverlap(git, plan.root, row.old_head, row.target_head);
if (dirt.overlap.length) await abort(engine, current, ['fenced'], dirtyRefusal(sourceId, dirt.overlap));
+ await engine.transaction(async tx => {
+ await tx.executeRaw("SELECT set_config('synchronous_commit','on',true)");
+ await tx.executeRaw(`UPDATE persistence_worktree_refreshes SET outcome=outcome||'{"merge_started":true}'::jsonb,updated_at=now() WHERE id=$1::uuid AND state='fenced'`, [row.id]);
+ });
let merge = await git(['-C', plan.root, 'merge', '--ff-only', '--no-edit', '--quiet', row.target_head], 120_000);
let after = await headOf(git, plan.root);
for (let attempt = 1; attempt < INDEX_LOCK_ATTEMPTS && merge.code !== 0 && after === row.old_head && INDEX_LOCKED.test(merge.stderr); attempt++) {
@@ -949,10 +955,11 @@ async function recoverOne(engine: BrainEngine, git: Git, row: WorktreeRefreshRow
} else if (row.state === 'fenced' && head === row.old_head) {
await engine.transaction(tx => transition(tx, row, ['fenced'], 'aborted', { refusal: { code: 'refresh_interrupted', cause: 'The refreshing process stopped before merging; HEAD is unchanged.' } }));
} else if (await movedBeforeMerge(engine, git, root, row, head)) {
- const cause = `The refreshing process stopped before merging; HEAD moved from ${row.old_head} to ${head} (${OWN_COMMITS}) and nothing was merged.`;
+ const by = await ownCommits(git, root, row.old_head, head!) ? OWN_COMMITS : 'a commit or checkout gbrain did not make';
+ const cause = `The refreshing process stopped before merging; HEAD moved from ${row.old_head} to ${head} (${by}) and nothing was merged.`;
await engine.transaction(tx => transition(tx, row, ['fenced'], 'aborted', { refusal: { code: 'refresh_interrupted', cause }, observed_head: head }));
- } else if (['fenced', 'merged', 'recovery_required'].includes(row.state) && (head === row.target_head || (row.state === 'recovery_required' && head !== null && await isAncestor(git, root, row.target_head, head)))
- && await isAncestor(git, root, row.old_head, row.target_head) && (row.state !== 'recovery_required' || !await memberOutsideHead(engine, git, root, row, head!))) {
+ } else if (['fenced', 'merged', 'recovery_required'].includes(row.state) && head !== null && (head === row.target_head || await isAncestor(git, root, row.target_head, head))
+ && await isAncestor(git, root, row.old_head, row.target_head) && ((row.state !== 'recovery_required' && head === row.target_head) || !await memberOutsideHead(engine, git, root, row, head))) {
await bookkeeping(engine, row);
} else if (row.state === 'syncing' && head !== null && (head === row.target_head || await isAncestor(git, root, row.target_head, head))) {
// stays syncing; the member syncs run from --resume or any later cycle sync.
diff --git a/test/refresh-write-barrier.test.ts b/test/refresh-write-barrier.test.ts
index b1a31d7da9..ba26ef2d1d 100644
--- a/test/refresh-write-barrier.test.ts
+++ b/test/refresh-write-barrier.test.ts
@@ -577,3 +577,39 @@ test('GBEV-44: the owner scan aborts an orphaned fenced row whose HEAD moved onl
expect((await settle(f, deferred)).state).toBe('committed');
}), 180_000);
+
+test('GST-1674: a refresh killed at fenced after a commit gbrain did not make moved HEAD before the merge resumes: the row aborts and its deferred write commits', () => each(async f => {
+ const { deferred, landed } = await crashAtFenced(f, () => commitFile(f.root, 'alpha/outside.md', page('Outside', 'A commit made around gbrain.'), 'outside'));
+ const resumed = await refreshWorktree(f.engine, f.alpha, { resume: true });
+ expect(resumed.status).toBe('abandoned');
+ const [row] = await f.refreshRows();
+ expect(row.state).toBe('aborted');
+ expect((row.outcome.refusal as { code: string }).code).toBe('refresh_interrupted');
+ expect(f.head()).toBe(landed);
+ expect((await settle(f, deferred)).state).toBe('committed');
+}), 180_000);
+
+test('GST-1674: a refresh killed after its merge, with a commit landed on the target before bookkeeping, recovers on the owner scan and completes without --resume', () => each(async f => {
+ const target = f.push('alpha/two.md', page('Alpha two', 'An upstream observation.'));
+ let deferred!: WriteRequest; let landed!: string;
+ await expect(refreshWorktree(f.engine, f.alpha, { hooks: { boundary: async point => {
+ if (point === 'drained') {
+ const out = await call(f, 'put_page', { slug: 'notes/held-past-merge', content: page('Held past merge', 'body') });
+ deferred = await requestRow(f, out.body.request_id);
+ }
+ if (point === 'merged') {
+ landed = commitFile(f.root, 'alpha/after-merge.md', page('After merge', 'A commit on top of the target.'), 'gbrain: persist canonical memory update');
+ throw new Error('simulated crash after merge');
+ }
+ } } })).rejects.toThrow('simulated crash');
+ expect(git(f.root, 'rev-parse', 'HEAD~1')).toBe(target);
+ await resumeWorktreeRefreshes(f.engine);
+ expect((await f.refreshRows()).map(r => r.state)).toEqual(['syncing']);
+ const next = await refreshWorktree(f.engine, f.alpha).catch(error => error);
+ // The scheduled refresh finishes the orphan first; its own tick then stops at the unpublished local commit.
+ expect(next.code).toBe('refresh_diverged');
+ expect((await f.refreshRows()).map(r => r.state)).toEqual(['completed']);
+ expect(f.head()).toBe(landed);
+ expect((await settle(f, deferred)).state).toBe('committed');
+}), 180_000);
+
diff --git a/test/worktree-refresh-settle-matrix-7679.test.ts b/test/worktree-refresh-settle-matrix-7679.test.ts
index 24e498a3f0..3eaa58b9da 100644
--- a/test/worktree-refresh-settle-matrix-7679.test.ts
+++ b/test/worktree-refresh-settle-matrix-7679.test.ts
@@ -86,7 +86,9 @@ test('7c. #7679 state matrix: an interrupted refresh of each kind, settled by th
await disposePersistenceConsumer(f.engine);
const [orphan] = await cellRows();
// The resident consumer's owner scan may already have settled it; restore what the crash left.
- await f.engine.executeRaw(`UPDATE persistence_worktree_refreshes SET state=$2,completed_at=NULL WHERE id=$1::uuid`, [orphan.id, stateOf(kind)]);
+ // GST-1674: an _unrelated kind is a merge that ran and was then reset under an unrelated commit, so its row records the merge as started.
+ await f.engine.executeRaw(`UPDATE persistence_worktree_refreshes SET state=$2,completed_at=NULL,outcome=CASE WHEN $3::boolean THEN outcome||'{"merge_started":true}'::jsonb ELSE outcome END WHERE id=$1::uuid`,
+ [orphan.id, stateOf(kind), kind.includes('_unrelated')]);
if (kind === 'fenced_target' || kind === 'merged_target') git(f.root, 'merge', '-q', '--ff-only', target);
if (kind === 'syncing_descendant') commitFile(f.root, 'beta/local.md', page('Local', 'Committed on top of the target.'));
if (kind.includes('_unrelated')) { git(f.root, 'reset', '-q', '--hard', old); commitFile(f.root, 'beta/unrelated.md', page('Unrelated', 'Not the upstream commit.')); }