Garry's Opinionated OpenClaw/Hermes Agent Brain
TypeScript
crash robot: a sync op's user commit must land before its sync counts (#8043)
garrytance562f22d4f8
diff --git a/CHANGELOG.md b/CHANGELOG.mdindex 375c769249..0c47bc543c 100644--- a/CHANGELOG.md+++ b/CHANGELOG.md@@ -10,6 +10,19 @@ credits are retained; no result has been reassigned to another provider. Originaidentifiers and attribution are available in the pre-removal Git revision`6040075c6cb95be5881cc2e1b76ef7d71f4e5d29` (retained on 2026-09-23).+## [Unreleased]++**The crash robot no longer reports a lost write when the user's own Git commit in a sync race never landed (#8043).**++In `sync_and_connector_race_direct_write`, the robot plays a user who edits a page file, commits it and syncs while an agent edits the same page. It ran `git add` and `git commit` without reading their exit status. On one Postgres run (seed 5105, `sync:mid_checkpoint#13`) the commit did not land, but the sync ran anyway and was counted as a committed write of the user's marker. Sync correctly imports nothing from an uncommitted file, so the group check called the page lost. The same missing commit explains why the agent's edit was refused as an "uncoordinated local edit" rather than `pending_import`, and why the planned fault was never reached: there were no imports to checkpoint. No write a caller was told succeeded went missing.++The user step now checks that its commit landed. It retries while another Git process holds a lock, for up to 20 s. Any other failure throws with git's stderr and fails the run as a harness error, so the model never sees a sync whose commit does not exist. The reference model is unchanged.++### Itemized changes++- `scripts/persistence/ops.ts`: `userEdit` writes and commits the user's files and verifies them clean at HEAD (retrying a held lock, throwing otherwise) before `executeOp`'s try, and `runSync` now only runs `sync_brain`.+- Tests: `test/persistence-crash-robot.test.ts`, under "a sync op whose user commit did not land". The forced probe holds `index.lock` across the user's commit: on the previous code it reproduces the CI result (sy2 `uncoordinated local edit`, sy1 committed, `lost_write`), and now the commit lands and the race leaves no violation. A refused commit fails the op instead of reporting it committed, and a committed sync whose marker the page lacks is still `lost_write`.+## [0.60.512.0] - 2026-10-30**After an automatic self-upgrade, the autopilot now restarts the serve your installed service runs when it is still on the old build, so one auto-upgrade no longer leaves two builds running until someone runs `gbrain upgrade --restart-only` by hand.**diff --git a/scripts/persistence/ops.ts b/scripts/persistence/ops.tsindex d6921104c8..e1c32b4005 100644--- a/scripts/persistence/ops.ts+++ b/scripts/persistence/ops.ts@@ -167,8 +167,17 @@ export const CONNECTOR_SLUG = 'gh/acme-example/app/1';const CONNECTOR_CONFIG = { kind: 'github', gh_scope: 'repos', gh_repos: 'acme-example/app', gh_token_env: 'CRASH_ROBOT_CONNECTOR_TOKEN' };export function connectorSourceConfig(): Record<string, string> { return { ...CONNECTOR_CONFIG }; }-/** A user edits a page file in the checkout and commits it, then the owner syncs through the real `sync_brain` handler. */-async function runSync(world: World, d: OpDescriptor): Promise<Record<string, unknown>> {+/** Git's refusal while another Git process (a gbrain Git effect committing a page) holds the checkout's index or a ref lock. */+const GIT_LOCK_BUSY = /\.lock'?: File exists|cannot lock ref/;+const USER_COMMIT_RETRY_MS = 20_000;++/**+ * A user edits a page file in the checkout and commits it with their own `git add` and `git commit`. The commit has+ * landed only when Git reports every file clean at HEAD (a resubmission after a crash finds it already committed). A+ * lock another Git process holds is retried, as a person retries; any other failure, or a lock held past the bound,+ * throws out of `executeOp` and fails the run, so the model is never told of a sync whose user commit does not exist.+ */+async function userEdit(world: World, d: OpDescriptor): Promise<void> {const root = world.roots?.[d.source];if (!root) throw new Error(`op descriptor ${d.id}: source ${d.source} has no checkout`);const slug = String(d.args.slug);@@ -182,8 +191,22 @@ async function runSync(world: World, d: OpDescriptor): Promise<Record<string, unwriteFileSync(join(root, sibling), `---\ntype: note\ntitle: batch ${i}\n---\n\nBatch page ${i} of ${slug}.\n`);}const git = (...args: string[]) => Bun.spawnSync(['git', '-C', root, ...args], { stdout: 'pipe', stderr: 'pipe' });- git('add', '--', ...files);- git('-c', 'user.name=Crash Robot', '-c', '[email protected]', 'commit', '-q', '-m', `edit ${slug}`, '--', ...files);+ const deadline = Date.now() + USER_COMMIT_RETRY_MS;+ for (;;) {+ const add = git('add', '--', ...files);+ const commit = add.exitCode === 0 ? git('-c', 'user.name=Crash Robot', '-c', '[email protected]', 'commit', '-q', '-m', `edit ${slug}`, '--', ...files) : add;+ const status = git('--no-optional-locks', 'status', '--porcelain', '--untracked-files=all', '--', ...files);+ if (status.exitCode === 0 && status.stdout.length === 0) return;+ const stderr = commit.stderr.toString().trim();+ if (!GIT_LOCK_BUSY.test(stderr) || Date.now() > deadline) {+ throw new Error(`op descriptor ${d.id}: the user's commit of ${slug} did not land (git ${commit === add ? 'add' : 'commit'} exit ${commit.exitCode}): ${stderr || status.stdout.toString().trim()}`);+ }+ await Bun.sleep(100);+ }+}++/** The owner syncs the user's commit through the real `sync_brain` handler. */+async function runSync(world: World, d: OpDescriptor): Promise<Record<string, unknown>> {return await operationsByName.sync_brain.handler(contextFor(world, 'local', d.source),{ source_id: d.source, no_pull: true, no_embed: true }) as Record<string, unknown>;}@@ -346,9 +369,10 @@ export async function executeOp(world: World, d: OpDescriptor): Promise<OpObservconst { params } = await paramsFor(world, d);(world.submitted ??= new Map()).set(d.id, params);world.onSubmit?.(d, params);+ const run = { ...d, args: params as OpDescriptor['args'] };+ if (d.kind === 'sync') await userEdit(world, run);let observation: OpObservation;try {- const run = { ...d, args: params as OpDescriptor['args'] };const result = d.kind === 'sync' ? await runSync(world, run) : await runConnector(world, run);const status = String(result.status);observation = { ...base, status: ['synced', 'first_sync', 'up_to_date'].includes(status) ? 'committed' : status === 'partial' ? 'pending' : 'refused',diff --git a/test/persistence-crash-robot.test.ts b/test/persistence-crash-robot.test.tsindex 494e71aac8..add9d30c92 100644--- a/test/persistence-crash-robot.test.ts+++ b/test/persistence-crash-robot.test.ts@@ -14,10 +14,10 @@* Seams: none (the fault hook stays uninstalled); PGLite.*/import { describe, expect, test } from 'bun:test';-import { mkdtempSync, readFileSync, rmSync, writeFileSync } from 'node:fs';+import { chmodSync, mkdtempSync, readFileSync, rmSync, writeFileSync } from 'node:fs';import { tmpdir } from 'node:os';import { join } from 'node:path';-import { descriptor, executeOp, parseDescriptor, SessionDrops } from '../scripts/persistence/ops.ts';+import { descriptor, executeOp, parseDescriptor, SessionDrops, type OpObservation } from '../scripts/persistence/ops.ts';import { ReferenceModel } from '../scripts/persistence/model.ts';import { crossBoundarySequences, pageBody, randomSchedule } from '../scripts/persistence/generator.ts';import { ROBOT_RANDOM_SEEDS, ROBOT_ROTATION_SEEDS, ROBOT_TOPOLOGY, explorationSeeds, restrict, runRobotPhase, type RobotRun } from '../scripts/persistence/robot-driver.ts';@@ -134,6 +134,57 @@ describe('concurrent connector publish and direct write', () => {}, 120_000);});+describe('a sync op whose user commit did not land (run 41092788952)', () => {+ // The robot ignored git's exit status: a user commit that never landed still ran the sync and was folded as a+ // committed write of the user's marker, so the group check called the page lost. Another Git process holding the+ // index lock is the stand-in for whatever refused the commit (CI kept no stderr).+ const raceOps = () => {+ const schedule = crossBoundarySequences(ROBOT_TOPOLOGY).find(s => s.label === 'sync_and_connector_race_direct_write')!;+ const keep = new Set(['sy0', 'sy1', 'sy2']);+ return { ...schedule, ops: schedule.ops.filter(d => keep.has(d.id)), groups: schedule.groups.filter(g => g.includes('sy1')) };+ };+ const gitIn = (root: string) => (...args: string[]) => Bun.spawnSync(['git', '-C', root, ...args], { stdout: 'pipe', stderr: 'pipe' });+ const gitPath = (root: string, name: string) => gitIn(root)('rev-parse', '--path-format=absolute', '--git-path', name).stdout.toString().trim();++ test('a commit held off by an index lock lands before the sync runs, and the race leaves no violation', async () => {+ await robotBrain(async ({ world }) => {+ const schedule = raceOps(); const sync = schedule.ops.find(d => d.id === 'sy1')!;+ const root = world.roots![sync.source]; const lock = gitPath(root, 'index.lock');+ world.onSubmit = d => { if (d.id === 'sy1') { writeFileSync(lock, ''); setTimeout(() => rmSync(lock, { force: true }), 300); } };+ const model = new ReferenceModel(world);+ world.descriptors = new Map(schedule.ops.map(d => [d.id, d]));+ await runSteps(world, model, schedule);+ expect(model.violations).toEqual([]);+ expect(world.observations.get('sy1')!.status).toBe('committed');+ expect(gitIn(root)('show', 'HEAD:./notes/alpha.md').stdout.toString()).toContain('Marker: mk-sy1');+ });+ }, 120_000);++ test('a commit that cannot land fails the run; the sync is never reported committed', async () => {+ await robotBrain(async ({ world }) => {+ const schedule = raceOps(); const sync = schedule.ops.find(d => d.id === 'sy1')!;+ const root = world.roots![sync.source]; const hook = join(gitPath(root, 'hooks'), 'pre-commit');+ writeFileSync(hook, '#!/bin/sh\necho "pre-commit refused" >&2\nexit 1\n'); chmodSync(hook, 0o755);+ await expect(executeOp(world, sync)).rejects.toThrow(/sy1: the user's commit of notes\/alpha did not land \(git commit exit 1\): pre-commit refused/);+ expect(world.observations.has('sy1')).toBe(false);+ });+ }, 120_000);++ test('a committed sync whose marker the page lacks is still a lost write', async () => {+ await robotBrain(async ({ world }) => {+ const schedule = raceOps(); const [put, sync] = schedule.ops;+ const model = new ReferenceModel(world);+ world.descriptors = new Map(schedule.ops.map(d => [d.id, d]));+ await runSteps(world, model, { ...schedule, ops: [put], groups: [] });+ const claimed: OpObservation = { id: sync.id, kind: sync.kind, actor: sync.actor, source: sync.source, requestId: sync.requestId, status: 'committed', values: {}, raw: { status: 'synced' } };+ model.beginStep(true);+ await model.observe(sync, claimed);+ await model.settleGroup([sync], [claimed]);+ expect(model.violations.map(v => [v.class, v.detail])).toEqual([['lost_write', 'no committed whole-page write of the group is visible in robot-0/notes/alpha']]);+ });+ }, 120_000);+});+describe('injected session drops', () => {// CI jobs 111669020132 and 111702893848: a pooler_disconnect drop landed on the robot's own `$current`// revision read, which had no retry, and the worker died with `write CONNECTION_CLOSED 127.0.0.1:55433`.