Skip to content

Garry's Opinionated OpenClaw/Hermes Agent Brain

TypeScript

fix: owner scan re-checks a cached lagging orphan after a minute, so a git failure is never a permanent verdict (#7679)

garrytan2ed6e3c9ba9d
diff --git a/src/core/persistence/worktree-refresh.ts b/src/core/persistence/worktree-refresh.ts
index 565a747216..0a8a2c838d 100644
--- a/src/core/persistence/worktree-refresh.ts
+++ b/src/core/persistence/worktree-refresh.ts
@@ -596,11 +596,15 @@ async function recoverOne(engine: BrainEngine, git: Git, row: WorktreeRefreshRow
* is free) is converged one step, so the scan finds nothing on the next tick.
* Member syncs are not run here. #7679: an orphaned `syncing` row whose members
* all synced to the target or a descendant of it (a later cycle sync caught
- * them up) and whose HEAD is still at the target is completed here (a lagging one is re-checked only after a member moves); one whose members lag is finished by the next
+ * them up) and whose HEAD is still at the target is completed here (a lagging one is re-checked after a member moves or a minute passes); one whose members lag is finished by the next
* plain `gbrain sources refresh`, which runs the `--resume` path, or by `--resume`.
*/
-/** A lagging orphaned `syncing` row, keyed by refresh id, with the member `last_commit`s it lagged at: the scan re-checks it only after a member moves. */
-const laggingSyncing = new Map<string, string>();
+/**
+ * A lagging orphaned `syncing` row, keyed by refresh id, with the member `last_commit`s it lagged at and when: the scan re-checks it once a
+ * member moves, or after LAGGING_RECHECK_MS (a git failure reads as lagging, so a cached verdict never outlives a transient one).
+ */
+const laggingSyncing = new Map<string, { commits: string; at: number }>();
+const LAGGING_RECHECK_MS = 60_000;
export async function resumeWorktreeRefreshes(engine: BrainEngine, opts: { hostId?: string } = {}): Promise<number> {
const host = opts.hostId ?? localHostId();
@@ -614,12 +618,13 @@ export async function resumeWorktreeRefreshes(engine: BrainEngine, opts: { hostI
if (!root) continue;
const commits = row.state === 'syncing' ? (await engine.executeRaw<{ last_commit: string | null }>(
'SELECT last_commit FROM sources WHERE id=ANY($1::text[]) ORDER BY id', [row.source_ids])).map(source => source.last_commit ?? '').join(',') : null;
- if (commits !== null && laggingSyncing.get(row.id) === commits) continue;
+ const lagging = commits === null ? undefined : laggingSyncing.get(row.id);
+ if (lagging?.commits === commits && Date.now() - lagging.at < LAGGING_RECHECK_MS) continue;
const lock = await tryAcquireNativeLock(refreshLockPath(row.worktree_id));
if (!lock) continue;
try {
await withGit(async git => {
- if (commits !== null && !await membersAtTarget(engine, git, row, root)) { laggingSyncing.set(row.id, commits); return; }
+ if (commits !== null && !await membersAtTarget(engine, git, row, root)) { laggingSyncing.set(row.id, { commits, at: Date.now() }); return; }
const after = await withTopologyLocks(engine, row.source_ids[0], async () => {
const current = await readRefresh(engine, row.id);
return current.state === 'syncing' ? completeSyncedOrphan(engine, git, current, root) : recoverOne(engine, git, current, root, true);
diff --git a/test/persistence-worktree-refresh.test.ts b/test/persistence-worktree-refresh.test.ts
index 2101a22880..21dd45753b 100644
--- a/test/persistence-worktree-refresh.test.ts
+++ b/test/persistence-worktree-refresh.test.ts
@@ -15,10 +15,10 @@
* Seams: the `hooks.boundary` callback pauses the real state machine at its
* drained, fenced and merged points; no production behavior is replaced.
*/
-import { afterAll, beforeAll, expect, test } from 'bun:test';
+import { afterAll, beforeAll, expect, setSystemTime, test } from 'bun:test';
import { execFileSync } from 'node:child_process';
import { randomUUID } from 'node:crypto';
-import { existsSync, mkdirSync, mkdtempSync, readFileSync, renameSync, rmSync, writeFileSync } from 'node:fs';
+import { chmodSync, existsSync, mkdirSync, mkdtempSync, readFileSync, renameSync, rmSync, writeFileSync } from 'node:fs';
import { tmpdir } from 'node:os';
import { join } from 'node:path';
import type { BrainEngine } from '../src/core/engine.ts';
@@ -561,6 +561,34 @@ test('7e. #7679: the owner scan leaves an orphaned syncing refresh while one mem
expect(row).toMatchObject({ id: orphan.id, state: 'completed', outcome: { completed_by: 'owner_recovery' } });
}), 180_000);
+test('7h. #7679: a git failure while the owner scan checks an orphaned syncing refresh is re-checked after a minute, not cached for the life of the process', () => each(async f => {
+ // Ship adversarial (Step 11). Value: protects=the owner scan's lagging cache never turns one failed `git merge-base --is-ancestor` (exit 128,
+ // a timeout) into a permanent lagging verdict for a row whose members already synced past its target; fails_when=the cache drops its
+ // re-check window and keys only on member last_commits; why_new=7e and 7g re-check only after a member moves; seam=the git object store is
+ // made unreadable for one scan, and the clock is moved with setSystemTime; no_claim=a real git timeout.
+ const target = f.push('alpha/two.md', page('Alpha two', 'Upstream.'));
+ await expect(refreshWorktree(f.engine, f.alpha, { hooks: { boundary: point => { if (point === 'syncing') throw new Error('simulated crash at syncing'); } } }))
+ .rejects.toThrow('simulated crash');
+ await disposePersistenceConsumer(f.engine);
+ const [orphan] = await f.refreshRows();
+ await f.engine.executeRaw(`UPDATE persistence_worktree_refreshes SET state='syncing',completed_at=NULL WHERE id=$1::uuid`, [orphan.id]);
+ const descendant = commitFile(f.root, 'beta/local.md', page('Local', 'Committed on top of the target.'));
+ for (const member of [f.alpha, f.beta]) await f.engine.transaction(tx => withCoordinatedWrite(tx, [member],
+ () => tx.executeRaw('UPDATE sources SET last_commit=$2 WHERE id=$1', [member, descendant]), TEST_WRITE_ATTRIBUTION));
+ expect(git(f.root, 'merge-base', '--is-ancestor', target, descendant)).toBe('');
+ const objects = join(f.root, '.git', 'objects');
+ chmodSync(objects, 0o000);
+ try { expect(await resumeWorktreeRefreshes(f.engine)).toBe(0); } finally { chmodSync(objects, 0o755); }
+ expect((await f.refreshRows()).map(r => r.state)).toEqual(['syncing']);
+ // Within the minute the cached verdict stands even though git reads again.
+ expect(await resumeWorktreeRefreshes(f.engine)).toBe(0);
+ expect((await f.refreshRows()).map(r => r.state)).toEqual(['syncing']);
+ setSystemTime(new Date(Date.now() + 61_000));
+ try { expect(await resumeWorktreeRefreshes(f.engine)).toBe(1); } finally { setSystemTime(); }
+ expect(await f.refreshRows()).toMatchObject([{ id: orphan.id, state: 'completed', outcome: { completed_by: 'owner_recovery' } }]);
+ expect(f.head()).toBe(descendant);
+}), 180_000);
+
test('7f. #7679: a plain refresh whose adopted orphan ends sync_blocked returns that result and does not refresh over it', () => each(async f => {
// Ship audit (Step 7, pass 2). Value: protects=a plain refresh that adopts an orphan whose member sync blocks returns sync_blocked naming the
// member's resume and leaves HEAD at the orphan's target; fails_when=refreshWorktree drops its non-terminal early return and refreshes on;