Garry's Opinionated OpenClaw/Hermes Agent Brain
TypeScript
spec/tla: tie the import-hold and recovery guard mutants to code edits and tests
garrytan1f002d5f6758
diff --git a/.github/workflows/tla-spec.yml b/.github/workflows/tla-spec.ymlindex 7062f89a0c..087197ff5e 100644--- a/.github/workflows/tla-spec.yml+++ b/.github/workflows/tla-spec.yml@@ -40,6 +40,7 @@ on:- 'src/core/schema-migrations/v292-refresh-write-barrier.ts'- 'test/spec-refresh-barrier.test.ts'- 'test/refresh-write-barrier.test.ts'+ - 'test/import-write-barrier.test.ts'- 'src/core/persistence/coordinator.ts'- 'src/core/persistence/group-publish.ts'- 'src/core/persistence/page-prepare.ts'@@ -79,6 +80,7 @@ on:- 'src/core/schema-migrations/v292-refresh-write-barrier.ts'- 'test/spec-refresh-barrier.test.ts'- 'test/refresh-write-barrier.test.ts'+ - 'test/import-write-barrier.test.ts'- 'src/core/persistence/coordinator.ts'- 'src/core/persistence/group-publish.ts'- 'src/core/persistence/page-prepare.ts'diff --git a/spec/tla/README.md b/spec/tla/README.mdindex f1942e3d02..271a6c69a9 100644--- a/spec/tla/README.md+++ b/spec/tla/README.md@@ -229,13 +229,13 @@ every config killing the refresh at least once, and the workflow's path filter c| `claim-ignores-fence` | survivor | — | under the fence nothing is claimable at these bounds || `claim-skips-protocol` | survivor | — | the drain count and the request-row lock already order a claim against the fence || `replay-skips-reauth` | survivor | — | publication re-authorizes every write |-| `hold-beside-held-off` | guard | `CommitsInAdmissionOrder` (Import) | #8705's compareFileTarget held-first read; model only until #8705 is on master |-| `import-expiry-locks-source-first` | guard | `NoDeadlock` (Import) | #8705's import expiry; model only until #8705 is on master |-| `import-expiry-skips-source-lock` | guard | `ExpiryOnlyWhileWaiting` (Import) | the same |-| `resume-marks-unmerged-recovery` | guard | `RecoveryOnlyByPolicy` | `recoverOne` as #8682 coded it: a fenced orphan aborts only with HEAD at old_head (the finding below); model only until the fix is on master |-| `resume-needs-exact-target` | guard | `RecoveryOnlyByPolicy` | `recoverOne` as #8682 coded it: bookkeeping only with HEAD exactly the target, so a commit between the merge and bookkeeping strands the row; the same |+| `hold-beside-held-off` | guard | `CommitsInAdmissionOrder` (Import) | compareFileTarget's held-first read; the later write that must not pass a held one (import-write-barrier.test.ts) |+| `import-expiry-locks-source-first` | guard | `NoDeadlock` (Import) | the import expiry's lock order; a Postgres expiry overlapping a fence (import-write-barrier.test.ts) |+| `import-expiry-skips-source-lock` | guard | `ExpiryOnlyWhileWaiting` (Import) | the import expiry's `FOR SHARE OF s`; a Postgres import landing while the expiry waits (import-write-barrier.test.ts) |+| `resume-marks-unmerged-recovery` | guard | `RecoveryOnlyByPolicy` | `movedBeforeMerge` (the finding below): the code edit aborts only a HEAD moved by gbrain's own commits; the refresh killed at fenced after a foreign commit, aborted by the owner scan |+| `resume-needs-exact-target` | guard | `RecoveryOnlyByPolicy` | `recoverOne`'s bookkeeping branch: with HEAD only exactly at the target, a commit between the merge and bookkeeping strands the row; the refresh killed after its merge |-#8705's and #8708's guards get code edits and tests (#8705's repro tests in test/import-write-barrier.test.ts) once their code is on master; until then they are checked in the model only.+#8708's guards get code edits and tests once its code is on master; until then they are checked in the model only.### Found in #8682: a refresh killed at fenced after HEAD moveddiff --git a/spec/tla/refresh-barrier/mutants.json b/spec/tla/refresh-barrier/mutants.jsonindex fe317b0d4c..f8bbea8f7b 100644--- a/spec/tla/refresh-barrier/mutants.json+++ b/spec/tla/refresh-barrier/mutants.json@@ -336,59 +336,136 @@{"id": "hold-beside-held-off","kind": "guard",- "title": "a later write to a page with a write held for an import is not held beside it (#8705's first race, before 03b89121)",+ "title": "a later write to a page with a write held for an import is not held beside it, so it can commit ahead of the held one","expect": "CommitsInAdmissionOrder","config": "RefreshBarrierImport.cfg","spec": {"file": "RefreshBarrier.tla","before": " ELSE IF ImportHolds /\\ HoldBehindHeld /\\ HeldAhead(w) # {} THEN \"pending\"\n","after": " ELSE IF FALSE THEN \"pending\"\n"+ },+ "code": {+ "edits": [+ {+ "file": "src/core/persistence/page-prepare.ts",+ "before": " if (heldFirst) throw pendingImportRefusal(",+ "after": " if (heldFirst && false) throw pendingImportRefusal("+ }+ ],+ "tests": [+ {+ "test": "test/import-write-barrier.test.ts",+ "filter": "a write held for an import is never passed by a later write to the same page"+ }+ ]}},{"id": "import-expiry-locks-source-first","kind": "guard",- "title": "#8705's import expiry locks the source row before it declares the protocol",+ "title": "the import expiry locks the source row before it declares the protocol","expect": "NoDeadlock","config": "RefreshBarrierImport.cfg","spec": {"file": "RefreshBarrier.tla","before": "ImportExpiryLocks == <<L(BRAIN, \"S\"), L(SRC, \"S\")>>","after": "ImportExpiryLocks == <<L(SRC, \"S\"), L(BRAIN, \"S\")>>"+ },+ "code": {+ "edits": [+ {+ "file": "src/core/persistence/refresh-barrier.ts",+ "before": " await declarePersistenceProtocol(tx);\n const [held] = await tx.executeRaw(`SELECT 1 FROM persistence_requests r JOIN sources s ON s.id=r.source_id\n WHERE r.id=$1::uuid AND r.state='deferred' AND COALESCE(s.last_commit,'')=r.deferred_import_from FOR SHARE OF s`, [row.id]);\n",+ "after": " const [held] = await tx.executeRaw(`SELECT 1 FROM persistence_requests r JOIN sources s ON s.id=r.source_id\n WHERE r.id=$1::uuid AND r.state='deferred' AND COALESCE(s.last_commit,'')=r.deferred_import_from FOR SHARE OF s`, [row.id]);\n await declarePersistenceProtocol(tx);\n"+ }+ ],+ "tests": [+ {+ "test": "test/import-write-barrier.test.ts",+ "filter": "an import expiry that overlaps a fence waits on the brain row",+ "postgres": true+ }+ ]}},{"id": "import-expiry-skips-source-lock","kind": "guard",- "title": "#8705's import expiry re-checks last_commit without locking the source row (#8705's second race, before 03b89121)",+ "title": "the import expiry re-checks last_commit without locking the source row","expect": "ExpiryOnlyWhileWaiting","config": "RefreshBarrierImport.cfg","spec": {"file": "RefreshBarrier.tla","before": "Need(EXP, IF ImportExpiryLocksSource THEN ImportExpiryLocks ELSE <<>>)","after": "Need(EXP, <<>>)"+ },+ "code": {+ "edits": [+ {+ "file": "src/core/persistence/refresh-barrier.ts",+ "before": "=r.deferred_import_from FOR SHARE OF s`, [row.id]);",+ "after": "=r.deferred_import_from`, [row.id]);"+ }+ ],+ "tests": [+ {+ "test": "test/import-write-barrier.test.ts",+ "filter": "a held write whose import lands while its expiry waits on the request lock is not expired",+ "postgres": true+ }+ ]}},{"id": "resume-marks-unmerged-recovery","kind": "guard",- "title": "recoverOne marks an orphaned fenced refresh recovery_required when HEAD moved past old_head before any merge (#8682 as coded)",+ "title": "recovery marks an orphaned fenced refresh recovery_required when a commit gbrain did not make moved HEAD past old_head before any merge","expect": "RecoveryOnlyByPolicy","spec": {"file": "RefreshBarrier.tla","before": "FencedAbortable(i) == ~rmstart[i]","after": "FencedAbortable(i) == head = rhead[i]"+ },+ "code": {+ "edits": [+ {+ "file": "src/core/persistence/worktree-refresh.ts",+ "before": " && !await memberOutsideHead(engine, git, root, row, head);\n}",+ "after": " && await ownCommits(git, root, row.old_head, head) && !await memberOutsideHead(engine, git, root, row, head);\n}"+ }+ ],+ "tests": [+ {+ "test": "test/refresh-write-barrier.test.ts",+ "filter": "a refresh killed at fenced after a commit gbrain did not make moved HEAD before the merge aborts on the owner scan"+ }+ ]}},{"id": "resume-needs-exact-target","kind": "guard",- "title": "recoverOne runs bookkeeping for an orphaned fenced refresh only when HEAD is exactly the target, not a descendant (#8682 as coded)",+ "title": "recovery runs bookkeeping for an orphaned fenced or merged refresh only when HEAD is exactly the target, not a descendant","expect": "RecoveryOnlyByPolicy","spec": {"file": "RefreshBarrier.tla","before": "ResumeAtTarget(i) == HoldsTarget(i)","after": "ResumeAtTarget(i) == rtgt[i] # 0 /\\ head = rtgt[i] /\\ ~lost[i]"+ },+ "code": {+ "edits": [+ {+ "file": "src/core/persistence/worktree-refresh.ts",+ "before": "&& head !== null && (head === row.target_head || await isAncestor(git, root, row.target_head, head))\n && await isAncestor(git, root, row.old_head, row.target_head)",+ "after": "&& head !== null && (head === row.target_head || (row.state === 'recovery_required' && await isAncestor(git, root, row.target_head, head)))\n && await isAncestor(git, root, row.old_head, row.target_head)"+ }+ ],+ "tests": [+ {+ "test": "test/refresh-write-barrier.test.ts",+ "filter": "a refresh killed after its merge, with a commit landed on the target before bookkeeping, recovers on the owner scan"+ }+ ]}}]diff --git a/test/import-write-barrier.test.ts b/test/import-write-barrier.test.tsindex 1fdbcee37e..0f87f7560b 100644--- a/test/import-write-barrier.test.ts+++ b/test/import-write-barrier.test.ts@@ -320,6 +320,44 @@ test('a write while a refresh is in recovery_required waits behind it and landsexpect(file(f, 'notes/during-recovery.md')).toContain('Accepted while the refresh awaited recovery.');}), 180_000);+test.skipIf(!backends.includes('postgres'))('an import expiry that overlaps a fence waits on the brain row instead of deadlocking with it', () => each(async f => {+ if (f.engine.kind !== 'postgres') return;+ foreignCommit(f, 'one.md', page('Alpha one', IMPORTED));+ const out = await local(f, 'put_page', { slug: 'one', content: page('Alpha one', 'Held past its limit.') });+ const held = await requestRow(f, out.receipt.request_id);+ expect(held.state).toBe('deferred');+ await disposePersistenceConsumer(f.engine);+ await maxWait(f, 1);+ try {+ await new Promise(resolve => setTimeout(resolve, 20));+ const owner = await ownerHost(f);+ let holding!: () => void; const held1 = new Promise<void>(resolve => { holding = resolve; });+ let waiting!: () => void; const expiryWaits = new Promise<void>(resolve => { waiting = resolve; });+ // fenceDraining's order (lockTopologyRows): the brain row FOR UPDATE, then the worktree, then the sources.+ const fence = f.engine.transaction(async tx => {+ await tx.executeRaw("SELECT set_config('lock_timeout','10s',true)");+ await tx.executeRaw('SELECT singleton FROM persistence_brain WHERE singleton=1 FOR UPDATE');+ holding();+ await expiryWaits;+ await tx.executeRaw('SELECT 1 FROM sources WHERE id=$1 FOR UPDATE', [f.alpha]);+ });+ await held1;+ const expiry = expireDeferredWrites(f.engine, owner);+ for (let i = 0; i < 400; i++) {+ const [w] = await f.engine.executeRaw<{ n: number }>("SELECT count(*)::int AS n FROM pg_stat_activity WHERE datname=current_database() AND wait_event_type='Lock'");+ if (Number(w!.n) > 0) break;+ await new Promise(resolve => setTimeout(resolve, 25));+ }+ waiting();+ const [fenced, expired] = await Promise.allSettled([fence, expiry]);+ expect(fenced.status === 'rejected' ? String(fenced.reason) : 'fence committed').toBe('fence committed');+ expect(expired.status === 'fulfilled' ? expired.value.map(row => row.id) : String(expired.reason)).toEqual([held.id]);+ } finally {+ await f.engine.executeRaw("DELETE FROM config WHERE key='persistence.refresh_barrier_max_wait_ms'");+ }+ expect(await getWriteRequestById(f.engine, held.id)).toMatchObject({ state: 'failed', error_code: 'pending_import_expired' });+}), 180_000);+test('a write held for an import is never passed by a later write to the same page', () => each(async f => {await durable(f);const before = await f.lastCommit(f.alpha);diff --git a/test/refresh-write-barrier.test.ts b/test/refresh-write-barrier.test.tsindex 72955e0bcf..557b2b0014 100644--- a/test/refresh-write-barrier.test.ts+++ b/test/refresh-write-barrier.test.ts@@ -671,6 +671,16 @@ test('GST-1674: a refresh killed at fenced after a commit gbrain did not make moexpect((await settle(f, deferred)).state).toBe('committed');}), 180_000);+test('a refresh killed at fenced after a commit gbrain did not make moved HEAD before the merge aborts on the owner scan, without --resume', () => each(async f => {+ const { deferred, landed } = await crashAtFenced(f, () => commitFile(f.root, 'alpha/outside.md', page('Outside', 'A commit made around gbrain.'), 'outside'));+ await resumeWorktreeRefreshes(f.engine);+ const [row] = await f.refreshRows();+ expect(row.state).toBe('aborted');+ expect((row.outcome.refusal as { code: string }).code).toBe('refresh_interrupted');+ expect(f.head()).toBe(landed);+ expect((await settle(f, deferred)).state).toBe('committed');+}), 180_000);+test('GST-1674: a refresh killed after its merge, with a commit landed on the target before bookkeeping, recovers on the owner scan and completes without --resume', () => each(async f => {const target = f.push('alpha/two.md', page('Alpha two', 'An upstream observation.'));let deferred!: WriteRequest; let landed!: string;