Garry's Opinionated OpenClaw/Hermes Agent Brain
TypeScript
spec: TLA+ model of the refresh write barrier and its deferred-write queue, guard mutants tied to real tests, CI job per spec
garrytanf91fc63ba4c4
diff --git a/.github/workflows/tla-spec.yml b/.github/workflows/tla-spec.ymlindex c055e9e97e..7062f89a0c 100644--- a/.github/workflows/tla-spec.yml+++ b/.github/workflows/tla-spec.yml@@ -31,6 +31,20 @@ on:- 'test/helpers/managed-brain.ts'- 'test/helpers/test-backends.ts'- 'test/helpers/persistence-postgres.ts'+ - 'src/core/persistence/refresh-barrier.ts'+ - 'src/core/persistence/journal.ts'+ - 'src/core/persistence/worktree-refresh.ts'+ - 'src/core/persistence/worktree-refresh-schema.ts'+ - 'src/core/persistence/protocol.ts'+ - 'src/core/shared-skills/restore.ts'+ - 'src/core/schema-migrations/v292-refresh-write-barrier.ts'+ - 'test/spec-refresh-barrier.test.ts'+ - 'test/refresh-write-barrier.test.ts'+ - 'src/core/persistence/coordinator.ts'+ - 'src/core/persistence/group-publish.ts'+ - 'src/core/persistence/page-prepare.ts'+ - 'test/persistence-worktree-refresh.test.ts'+ - 'test/pending-import-async-git-7436.test.ts'push:branches: [master]paths: # the same list as pull_request (actionlint does not accept YAML aliases)@@ -56,6 +70,20 @@ on:- 'test/helpers/managed-brain.ts'- 'test/helpers/test-backends.ts'- 'test/helpers/persistence-postgres.ts'+ - 'src/core/persistence/refresh-barrier.ts'+ - 'src/core/persistence/journal.ts'+ - 'src/core/persistence/worktree-refresh.ts'+ - 'src/core/persistence/worktree-refresh-schema.ts'+ - 'src/core/persistence/protocol.ts'+ - 'src/core/shared-skills/restore.ts'+ - 'src/core/schema-migrations/v292-refresh-write-barrier.ts'+ - 'test/spec-refresh-barrier.test.ts'+ - 'test/refresh-write-barrier.test.ts'+ - 'src/core/persistence/coordinator.ts'+ - 'src/core/persistence/group-publish.ts'+ - 'src/core/persistence/page-prepare.ts'+ - 'test/persistence-worktree-refresh.test.ts'+ - 'test/pending-import-async-git-7436.test.ts'schedule:- cron: '40 9 * * *' # nightly, 09:40 UTCworkflow_dispatch:@@ -69,9 +97,13 @@ concurrency:jobs:tla:- name: TLC and guard mutants+ name: TLC and guard mutants (${{ matrix.spec }})runs-on: ubicloud-standard-4-ubuntu-2404- timeout-minutes: 40+ timeout-minutes: 60+ strategy:+ fail-fast: false+ matrix:+ spec: [drain-lease, refresh-barrier]services:postgres:image: pgvector/pgvector:pg16@@ -106,23 +138,23 @@ jobs:jar="$(bash scripts/spec/tla-jar.sh)" || exit "$?"printf 'TLA2TOOLS=%s\n' "$jar" >> "$GITHUB_ENV"- name: The configs match the code- run: bun test --timeout=60000 test/spec-drain-lease-constants.test.ts+ run: bun test --timeout=60000 test/spec-${{ matrix.spec }}*.test.ts- name: Model-check every config (the *Nightly.cfg ones on the schedule and on dispatch)env:TLC_NIGHTLY: ${{ (github.event_name == 'schedule' || github.event_name == 'workflow_dispatch') && '1' || '0' }}TLC_TIMEOUT_S: '900'- run: bash scripts/spec/tlc.sh+ run: bash scripts/spec/tlc.sh ${{ matrix.spec }}# Only after the model check passed: a mutant means something only against a spec whose properties hold.- name: Every guard mutant is caught, in the model and in the codeenv:DATABASE_URL: postgres://gbrain_test:[email protected]:5432/gbrain_testGBRAIN_TEST_ALLOW_DATABASE_URL: '1'- run: bun scripts/spec/mutants.ts --require-postgres+ run: bun scripts/spec/mutants.ts ${{ matrix.spec }} --require-postgres- name: Keep the logs and counterexample tracesif: always()uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4with:- name: tla-spec-logs+ name: tla-spec-logs-${{ matrix.spec }}path: ${{ runner.temp }}/tlcif-no-files-found: ignoreretention-days: 14diff --git a/docs/architecture/key-files/ci-health-and-test-guards.md b/docs/architecture/key-files/ci-health-and-test-guards.mdindex 72c1785390..00c4e0a8d8 100644--- a/docs/architecture/key-files/ci-health-and-test-guards.md+++ b/docs/architecture/key-files/ci-health-and-test-guards.md@@ -37,4 +37,4 @@ Current behavior and load-bearing invariants; history belongs in Git and CHANGEL- `scripts/check-skill-refs.mjs` + `scripts/portable-skill-links.ts` + `scripts/skills-commit-gate.sh` — skills-tree integrity gate (`bun run check:skill-refs`, in verify and the skills commit gate). Fails on dangling backtick `skills/` refs, `./`/`../` links, `composes:` and RESOLVER dispatcher slugs; donor-workspace paths outside the shrink-only allowlist; any `gbrain <cmd>` in a fenced line or inline code span (command position only: line/span start, after a shell separator, `nohup`/env prefixes) that is not a real command (union of `--tools-json`, `CLI_FLAG_REGISTRY`, `src/cli.ts` dispatch, the command table and op `cliHints`), plus curated known-bad subcommands (`embed refresh`); and a relative link that resolves outside the skills root, in `skills/` and in the generated `plugin/skills` and `plugin-variants/*/skills` trees (`skills/migrations/**` exempt). W4.7: a tools-json load failure FAILS the CLI lane (`--no-cli-refs` skips it deliberately); links are read in every form (`](./x)`, bare `](x.md)` unless a placeholder, `](<x>)`, titled, `[id]: x` definitions); a `[paid-consent]` lane, migrations included, fails a paid `gbrain` command carrying `--yes`/`--max-usd`/`--max-cost[-usd]` with no ask/agree/approve/consent/confirm/permission wording on its line or the 8 before. `portable-skill-links.ts` is that rule's fix: it points `docs/protocol/AGENT_OPERATOR_v1.md` links at the bundled `skills/conventions/agent-operator-protocol.md` and other repo files at absolute blob URLs derived from `LLMS_REPO_BASE` (raw.githubusercontent base → github.com blob base; default `garrytan/gbrain/blob/master`; a non-https or credentialed value throws via `repoBaseOrThrow`, as in `scripts/llms-config.ts`), retargeting default-base links for forks; `--check` reports pending rewrites. Tests: `test/check-skill-refs.test.ts`, `test/skills-portable-links.test.ts` (scaffold `--all` into a tmp host: 0 broken local links).- `scripts/check-file-url-pathname.sh` — `check:file-url-pathname` (in verify, #6845): no `.pathname` of a file URL (`import.meta.url).pathname`, `pathToFileURL(...).pathname`) in test/, which keeps `%20` and breaks a checkout under a path with a space; use `fileURLToPath` or `import.meta.dir`. Self-tested via `GBRAIN_GUARD_ROOT` fixtures; skips `test/fixtures/guards/`.- `scripts/check-dynamic-ci.ts` — `check:dynamic-ci` (in verify): wiring guard for the Trunk Dynamic CI filter (`dynamic-ci` job in `test.yml` and `scale-tier.yml`; rules in [docs/ci-event-parity.md](../../ci-event-parity.md#trunk-dynamic-ci)). `checkWorkflow()` (pure, over parsed YAML) flags a filter job that can run on an event other than `pull_request`, sits in a workflow without that trigger, is not SHA-pinned, lacks `continue-on-error: true` or a step timeout below the job's, or whose `job-keys` differ from its `outputs`; a gated job whose `if` does not start `!cancelled() && needs.<filter>.outputs.<key> != 'false' &&`, lacks `needs.<other>.result == 'success'` for another need, or has a top-level `||` after the prefix; and a gated merge or release aggregate. `scripts/ci-cheap-tier.sh` accepts a gated lane that skipped only on an exact `false` verdict. Self-tested via `GBRAIN_GUARD_ROOT` fixtures in `test/fixtures/guards/check-dynamic-ci.ts/` (`good`, `bad`, `bad-every-event`, `bad-other-needs`, `bad-top-level-or`).-- `spec/tla/` + `scripts/spec/{tlc.sh,tla-jar.sh,tla-tools.env,mutants.ts}` + `.github/workflows/tla-spec.yml` — TLA+ models checked exhaustively by TLC within small bounds; `spec/tla/drain-lease` models the managed-sync drain lease (`sync-lease.ts` over `db-lock.ts`, plus the pass stamp's fences in `sync-pass-fold.ts` and `sync-prepare.ts`). `tlc.sh` runs every config on a private copy and fails `TLC_VIOLATION`, `TLC_TIMEOUT` or `TLC_ERROR`; `tla-jar.sh` fetches the pinned tla2tools.jar and refuses a sha256 mismatch. `mutants.ts` checks `mutants.json`: each guard mutant's model edit must break its named property, and its code edits (every engine branch), applied in a scratch copy of the tree, must fail each named test on its own assertions; assumption mutants document what liveness rests on, survivors stay defense in depth. `bun run test:tla` runs both; the workflow runs them nightly and on changes to anything a mutant names (`test/spec-drain-lease-constants.test.ts` pins that path filter and the configs' timing to the code's constants). Guard tests: `test/drain-lease-guards.test.ts`, `test/drain-lease-stamp-fence.test.ts` (Postgres twins in `test/e2e/`), `test/e2e/drain-lease-fence-postgres.test.ts`. Docs: `spec/tla/README.md`.+- `spec/tla/` + `scripts/spec/{tlc.sh,tla-jar.sh,tla-tools.env,mutants.ts}` + `.github/workflows/tla-spec.yml` — TLA+ models checked exhaustively by TLC within small bounds; `spec/tla/drain-lease` models the managed-sync drain lease (`sync-lease.ts` over `db-lock.ts`, plus the pass stamp's fences in `sync-pass-fold.ts` and `sync-prepare.ts`). `spec/tla/refresh-barrier` models the refresh write barrier and its deferred-write queue (admission, claim, replay, expiry and restore in `journal.ts`, `refresh-barrier.ts`, `worktree-refresh.ts`, `shared-skills/restore.ts` and the v292 triggers, with Postgres row locks and a deadlock invariant), plus #8705/#8708 behind constants. `tlc.sh` runs every config on a private copy and fails `TLC_VIOLATION`, `TLC_TIMEOUT` or `TLC_ERROR`; `tla-jar.sh` fetches the pinned tla2tools.jar and refuses a sha256 mismatch. `mutants.ts` checks `mutants.json`: each guard mutant's model edit must break its named property, and its code edits (every engine branch), applied in a scratch copy of the tree, must fail each named test on its own assertions; assumption mutants document what liveness rests on, survivors stay defense in depth. `bun run test:tla` runs both; the workflow runs them nightly and on changes to anything a mutant names (`test/spec-drain-lease-constants.test.ts` and `test/spec-refresh-barrier.test.ts` pin that path filter; the first also pins the drain-lease configs' timing to the code's constants). Guard tests: `test/drain-lease-guards.test.ts`, `test/drain-lease-stamp-fence.test.ts` (Postgres twins in `test/e2e/`), `test/e2e/drain-lease-fence-postgres.test.ts`. Docs: `spec/tla/README.md`.diff --git a/package.json b/package.jsonindex 07132be579..ae0b10b695 100644--- a/package.json+++ b/package.json@@ -102,7 +102,7 @@"test:profile": "bash scripts/profile-tests.sh","test:serial": "bash scripts/run-serial-tests.sh","test:convergence": "GBRAIN_TEST_CONVERGENCE=1 bun --no-env-file test --timeout=3600000 test/sync-convergence.serial.test.ts",- "test:tla": "bash scripts/spec/tlc.sh && bun scripts/spec/mutants.ts",+ "test:tla": "bash scripts/spec/tlc.sh && bun scripts/spec/mutants.ts drain-lease && bun scripts/spec/mutants.ts refresh-barrier","ci:local": "bash scripts/ci-local.sh","ci:local:diff": "bash scripts/ci-local.sh --diff","ci:ubicloud": "bun run scripts/ci-ubicloud.ts",diff --git a/scripts/spec/tlc.sh b/scripts/spec/tlc.shindex 4eb207f276..de8cf35b07 100755--- a/scripts/spec/tlc.sh+++ b/scripts/spec/tlc.sh@@ -11,9 +11,6 @@# spec is wrong and say why in the PR# TLC_TIMEOUT TLC did not finish within TLC_TIMEOUT_S (default 600): the state space grew; shrink the config# TLC_ERROR TLC stopped (parse or semantic error, out of memory): the log names the line-# TLC_FINDING_CLOSED a config whose `\* OPEN_FINDING: <Property>` line documents a known violation of the code now-# passes: the finding is fixed; flip the config to the fixed code and drop the line-# A config with an OPEN_FINDING line passes (`open`) only when TLC reports exactly that property violated.# Env: TLA2TOOLS (a jar to use; otherwise scripts/spec/tla-jar.sh downloads the pinned one), TLC_OUT (logs; default# .context/tlc), TLC_WORKERS (default: every CPU; configs run one at a time), TLC_TIMEOUT_S, TLC_JAVA_OPTS, TLC_NIGHTLY (1 also runs# the *Nightly.cfg configs, which take too long for every pull request).@@ -58,17 +55,11 @@ status=0for job in "${jobs[@]}"; doIFS='|' read -r d m c <<< "$job"work="$out/$(basename "$d")-${c%.cfg}"; log="$work/tlc.log"; rc="$(cat "$work/rc")"; name="$(basename "$d")/$c"- open="$(sed -nE 's/^\\\* OPEN_FINDING: ([A-Za-z0-9_]+).*/\1/p' "$d/$c" | head -1)"- v="$(grep -oE '(Invariant|Action property|Temporal properties|Property) [A-Za-z0-9_]* ?(is|were) violated' "$log" | head -1 || true)"- if [ -n "$open" ] && [ "$(echo "$v" | awk '{print $2}')" = "$open" ]; then- echo "open $name: $open is violated, as the config documents (an open finding; spec/tla/README.md)"- elif [ -n "$open" ] && grep -q "Model checking completed. No error has been found." "$log"; then- echo "FAIL TLC_FINDING_CLOSED $name: $open now holds, so the open finding this config documents is fixed. Drop its OPEN_FINDING line, set its constants to the fixed code and move it to the README's passing configs." >&2; status=1- elif grep -q "Model checking completed. No error has been found." "$log"; then+ if grep -q "Model checking completed. No error has been found." "$log"; thenecho "ok $name: $(grep -oE '[0-9]+ distinct states found' "$log" | tail -1) ($(grep -oE 'Finished in [^ ]+' "$log" | head -1))"elif [ "$rc" = 124 ] || [ "$rc" = 137 ]; thenecho "FAIL TLC_TIMEOUT $name: TLC did not finish within ${limit}s. Log: $log" >&2; status=1- elif [ -n "$v" ]; then+ elif v="$(grep -oE '(Invariant|Action property|Temporal properties|Property) [A-Za-z0-9_]* ?(is|were) violated' "$log" | head -1)" && [ -n "$v" ]; thenecho "FAIL TLC_VIOLATION $name: $v. The counterexample trace is in $log; spec/tla/README.md maps each action to its code." >&2; status=1elseecho "FAIL TLC_ERROR $name: TLC exited $rc without a result. Log: $log" >&2; grep -m3 -iE 'error|exception' "$log" >&2 || tail -5 "$log" >&2; status=1diff --git a/spec/tla/README.md b/spec/tla/README.mdindex 75fc1a00a5..49b35cd53f 100644--- a/spec/tla/README.md+++ b/spec/tla/README.md@@ -12,6 +12,7 @@ interleaving of the modelled actions inside their bounds. The approach follows Pbun run test:tla # every config of every spec, then every mutant (model and code)bash scripts/spec/tlc.sh drain-lease DrainLease.cfg # one configbun scripts/spec/mutants.ts --only renewal-ignores-token+bun scripts/spec/mutants.ts refresh-barrier # one spec's mutants (the default is drain-lease)bun scripts/spec/mutants.ts --spec-only # model mutants only (no Bun tests)```@@ -22,7 +23,7 @@ that need Postgres run when `DATABASE_URL` points at a test database (with `GBRAotherwise they print `skip`, and CI passes `--require-postgres`. Logs and counterexample traces land in`.context/tlc/` (`TLC_OUT`), and CI uploads them as the `tla-spec-logs` artifact.-`.github/workflows/tla-spec.yml` runs all of it nightly, on dispatch, and on pull requests and master pushes that touch a+`.github/workflows/tla-spec.yml` runs all of it, one job per spec, nightly, on dispatch, and on pull requests and master pushes that touch aspec, its runner, the code a spec models, or a test a mutant names.### When it fails@@ -143,3 +144,113 @@ length of their transaction, so they count against the renewal timeout like a loNone in the lease protocol at these bounds. The two gaps this model first found (#7442: fenced transactions not boundedunder the renewal timeout, and the pass stamp's share lock overlapping the holder's) are closed by the `FOR KEY SHARE`fence and the key-writing takeover upsert (#7450); `DrainLease.cfg` now checks `NoSpuriousLapse` with both conditions on.++## refresh-barrier: the refresh write barrier and its deferred-write queue++With `persistence.refresh_barrier` on, a page write admitted while a managed refresh runs is journaled `deferred`+behind that refresh and published by itself once the refresh is released (docs/designs/REFRESH_WRITE_BARRIER.md). The+model holds one managed worktree with one page P whose canonical file is F, single-page writes to P (one blind+put_page, one with expected_revision), one refresh at a time, another agent's plain-git commit to F, managed sync, the+owner's expiry, and Postgres's four row-lock modes per transaction (`KS`, `S`, `NKU`, `U`) over the brain row,+the worktree, the source row, the counters, each refresh row and each request row. A transaction holds its locks until+it commits; a waits-for cycle among them is a deadlock (Postgres aborts one side with 40P01), so `NoDeadlock` is an+invariant. Behind constants it also models the stacked follow-ups: writes held for an import and deferral behind+`recovery_required` (#8705), and local-edit adoption (#8708).++| Config | What it checks |+| --- | --- |+| `RefreshBarrier.cfg` | #8682 as coded: two writes (one blind), one refresh, one foreign commit: every safety property. |+| `RefreshBarrierEnv.cfg` | The same with a shared-skill restore and a credential revocation that may each run once, no foreign commit. |+| `RefreshBarrierLiveness.cfg` | `FairSpec`, one write: every deferred write leaves deferral and every admitted write ends. `RefreshBarrierLivenessNightly.cfg` checks two writes (nightly). |+| `RefreshBarrierAdopt.cfg` | #8708 on #8705 with its fixes: an uncommitted local edit adopted by a drift refusal, no foreign commit. `RefreshBarrierAdoptNightly.cfg` adds the foreign commit (nightly). |+| `RefreshBarrierImport.cfg` | #8705 with its fixes (03b89121): writes held for an import, a later write held beside them, the import expiry, deferral behind `recovery_required`: every property. |++`test/spec-refresh-barrier.test.ts` keeps the default config on #8682 alone, every safety config on every invariant,+and the workflow's path filter covering every file a mutant names.++### Properties++| Property | Holds that |+| --- | --- |+| `NoDeadlock` | no waits-for cycle among open transactions' row locks |+| `AppliedAtMostOnce`, `ReceiptMatchesApply` | no write is applied twice, and a receipt says committed exactly when it was applied |+| `EveryEndNamed` | every write that ends has a named outcome (nothing is dropped silently) |+| `ImportedRevisionNeverHidden` | a write never replaces a committed version of F the page never had, so a blind put_page after an import never hides the imported revision from page history |+| `FenceNeverMidPrepare` | the fence never comes up, and the checkout never moves, while a write is claimed |+| `DeferredWaitsForRelease`, `RefreshChangeNeverOverwritten` | a deferred write commits only after its refresh is released, and never over a file that refresh changed |+| `NoDeferToReleased`, `ReleasedWhenEnded` | admission never defers to a released refresh; a completed or aborted refresh is released |+| `ExpiryOnlyWhileWaiting` | an expiry ends only a write that still waits (its refresh unreleased, or its import not landed) |+| `RestoreLeavesNoOpenWrite` | a restore leaves no write admitted before it open (finding C) |+| `NoApplyAfterRevoke` | no write commits after its credential was revoked |+| `CommitsInAdmissionOrder` | writes to one page commit in admission order |+| `DeferredLeaves`, `EveryWriteEnds` | liveness under `Fairness`: a deferred write leaves deferral; an admitted write ends |++### Actions and code++| Action | Code |+| --- | --- |+| `AdmBegin`, `AdmRef`, `AdmRead`, `AdmCommit` | `admitWrite` (journal.ts): the protocol declaration, the worktree `FOR SHARE`, the active refresh row `FOR NO KEY UPDATE`, the source, `lockCounters`, the INSERT (queued, or deferred with `deferred_refresh_id`) |+| `ConsBegin`, `ConsSelect`, `ConsClaim` | `claimNextWrite`: `claimableWriteSql` (EARLIER_HEAD, EARLIER_DEFERRED, RELEASED_DEFERRED, `refreshFenceClear`) then the v292 guard trigger |+| `ConsPrep` | preparation: `compareFileTarget` (page-prepare.ts) and `deferredReplayRefusal` (refresh-barrier.ts) |+| `ConsPub` | `publishMutation` / `publishGroup`: `authorizeStoredRequest`, the request row, the revision and file rechecks |+| `ConsHold` | #8705 `holdForImport` |+| `ConsAdopt`, `ConsRelease` | #8708 `adoptLocalEdit`, then the claim's release |+| `ConsDone` | `completeWrite` of a refusal |+| `RefBegin` … `CompleteCommit` | worktree-refresh.ts: `fenceDraining`, the drain poll, `fenceAndMerge` (re-counts under `lockTopologyRows`), the ff-only merge under the commit lock, `bookkeeping`, `syncMembers`, `resumeOrAbandon`, and the v292 release trigger on completed or aborted |+| `ExpRefBegin`, `ExpRefCommit` | `expireDeferredWrites`: the protocol, the refresh row re-read `released_at IS NULL FOR NO KEY UPDATE`, `completeWrite` |+| `ExpImpBegin`, `ExpImpRecheck`, `ExpImpCommit` | #8705's import expiry in `expireDeferredWrites` |+| `SyncBegin` … `SyncStamp` | a managed sync pass, whose rows go through the claim like writes: no write passes the queued row importing its page or a running sync row, and a sync row waits behind earlier writes and is never claimed while a write runs; the page row, then its checkpoint row's `UPDATE sources SET last_commit` (sync-prepare.ts) |+| `Foreign`, `LocalEdit`, `Age`, `Revoke`, `Restore` | the environment: another agent's plain-git commit, an uncommitted edit, the wait limit passing, a revoked token, `quarantineSharedSkillRestore` (PGLite-only, one transaction, no worker) |++### Mutants++`mutants.json`. Each guard mutant has a code edit (every engine branch) and the tests that must fail under it.++| Mutant | Kind | Caught by | Code and test |+| --- | --- | --- | --- |+| `expiry-locks-refresh-first` | guard | `NoDeadlock` | finding D reverted: the expiry locks the refresh row before the brain row; the Postgres test where an expiry overlaps a fence |+| `restore-keeps-deferred` | guard | `RestoreLeavesNoOpenWrite` (Env) | finding C reverted in restore.ts; the restore test (PGLite) |+| `replay-ignores-refresh-change` | guard | `RefreshChangeNeverOverwritten` | `deferredReplayRefusal`'s diff check; the deferred blind write over a refreshed page |+| `release-only-on-completed` | guard | `ReleasedWhenEnded` | the v292 release trigger (both engines); the write deferred behind an aborted refresh |+| `expiry-skips-release-recheck` | guard | `ExpiryOnlyWhileWaiting` | the expiry's `released_at IS NULL` under the lock; a Postgres race with a refresh ending |+| `admission-skips-refresh-lock` | guard | `NoDeferToReleased` | admission's `FOR NO KEY UPDATE` on the refresh row; a Postgres race with a refresh ending |+| `publication-skips-reauth` | guard | `NoApplyAfterRevoke` (Env) | `authorizeStoredRequest` in publishMutation and publishGroup; a token revoked after the claim |+| `preparation-ignores-drift` | guard | `ImportedRevisionNeverHidden` | `compareFileTarget`; the #7436 pending-import test |+| `drain-ignores-running` | guard | `FenceNeverMidPrepare` | `drainCounts`; test 13e in persistence-worktree-refresh.test.ts |+| `trigger-allows-unreleased` | survivor | — | the model's only consumer claims through the predicate; the trigger stops an older binary's claim, and the code test still pins it |+| `fence-skips-recount` | survivor | — | every admission after the refresh row exists is deferred, so nothing queues between the drain poll and the fence |+| `claim-ignores-fence` | survivor | — | under the fence nothing is claimable at these bounds |+| `claim-skips-protocol` | survivor | — | the drain count and the request-row lock already order a claim against the fence |+| `replay-skips-reauth` | survivor | — | publication re-authorizes every write |+| `hold-beside-held-off` | guard | `CommitsInAdmissionOrder` (Import) | #8705's compareFileTarget held-first read; model only until #8705 is on master |+| `import-expiry-locks-source-first` | guard | `NoDeadlock` (Import) | #8705's import expiry; model only until #8705 is on master |+| `import-expiry-skips-source-lock` | guard | `ExpiryOnlyWhileWaiting` (Import) | the same |++#8705's and #8708's guards get code edits and tests (#8705's repro tests in test/import-write-barrier.test.ts) once their code is on master; until then they are checked in the model only.++### Races found in #8705++This model found two races in #8705's first version; 03b89121 fixed both, with the repro tests in+test/import-write-barrier.test.ts. The mutants below put each one back.++- **A held write landed on top of a later one** (`hold-beside-held-off`, `CommitsInAdmissionOrder`). A write held for+ an import holds back nothing in the claim's FIFO rules, so after the sync's page row imported F (the file now matching+ the page) and before its checkpoint stamped `last_commit`, a later write to P was claimed, passed the queued+ checkpoint, prepared cleanly and committed; the stamp then released the held write, which landed over it, its+ `supersedes` naming the later write's commit as the import. The fix holds the later write beside the held one+ (compareFileTarget refuses it `pending_import` with the held write's `deferred_import_from`; constant+ `HoldBehindHeld`). A later write claimed while the held one waits cannot be prepared after the stamp: a sync row is+ never claimed while a foreground write runs. Blocking the later write behind the held one in the claim FIFO instead+ stalls the sync: a pass stopped after a slice queues its checkpoint behind the blocked write.+- **The import expiry failed a write whose import had landed** (`import-expiry-skips-source-lock`,+ `ExpiryOnlyWhileWaiting`; Postgres only). The re-check read `last_commit` without a lock on the source row, so a+ checkpoint that committed between it and `completeWrite` left the write `pending_import_expired`. The fix declares the+ protocol and then reads the source row `FOR SHARE` (`ImportExpiryLocksSource`); taking the source row first deadlocks+ with `fenceDraining`'s brain, worktree, source order (`import-expiry-locks-source-first`).++### Not modelled++The deferred-write cap (`persistence.refresh_barrier_max_deferred`), consumers without the capability, repo-less+sources, `gbrain upgrade --check-downgrade`, edit_page and forget replays, and source lifecycle changes, each covered+by test/refresh-write-barrier.test.ts; more than one worktree, page or refresh at a time; older binaries' claims+(the v292 guard trigger, pinned by its test); effects other than page writes; side effects outside the database.diff --git a/spec/tla/refresh-barrier/RefreshBarrier.cfg b/spec/tla/refresh-barrier/RefreshBarrier.cfgindex 24f588cb2e..d959bf5d7d 100644--- a/spec/tla/refresh-barrier/RefreshBarrier.cfg+++ b/spec/tla/refresh-barrier/RefreshBarrier.cfg@@ -8,7 +8,7 @@ CONSTANTSMaxForeign = 1MaxLocalEdits = 0ImportHolds = FALSE- HeldImportBlocks = FALSE+ HoldBehindHeld = FALSEImportExpiryLocksSource = FALSERecoveryDefers = FALSEAdoptLocalEdits = FALSEdiff --git a/spec/tla/refresh-barrier/RefreshBarrier.tla b/spec/tla/refresh-barrier/RefreshBarrier.tlaindex d51297442d..d463e7e80c 100644--- a/spec/tla/refresh-barrier/RefreshBarrier.tla+++ b/spec/tla/refresh-barrier/RefreshBarrier.tla@@ -41,7 +41,7 @@ CONSTANTSMaxForeign, \* commits to F by another agent with plain git (a pending import until sync imports them)MaxLocalEdits, \* uncommitted edits to F in the worktreeImportHolds, \* #8705: a claimed write refused pending_import is moved back to deferred until the import lands- HeldImportBlocks, \* a write held for an import blocks later writes of its worktree (the claim's FIFO rule); FALSE as coded+ HoldBehindHeld, \* #8705's fix: a write to a page with a write held for an import is held too (refused pending_import)ImportExpiryLocksSource, \* the import expiry re-checks last_commit under a lock on the source row; FALSE as codedRecoveryDefers, \* #8705: admission defers behind a refresh in recovery_required instead of refusingAdoptLocalEdits, \* #8708: a drift refusal on a one-sided local edit adopts the edit and prepares the write again@@ -124,7 +124,8 @@ VARIABLES\* locksheld, want,\* process locals- pc, cw, cprep, cfin, ew, aref, aread, passHead+ pc, cw, cprep, cfin, ew, aref, aread, passHead,+ sseq \* the admission sequence of the sync pass's queued rows (0 when no pass runs)reqv == <<st, code, seq, nseq, dref, dimp, applied, cpos, ncommit, expRev, overdue, adopted>>refv == <<rstate, rel, rchg, rhead, rmoved, cur, nref>>@@ -132,7 +133,7 @@ gitv == <<head, verAt, fileV, dbV, hist, lastCommit, local, nver, nforeign, nlenvv == <<restored, revoked, preRestore>>ghostv == <<hidden, badExpiry, badRelease, badDefer, badRevoke>>lockv == <<held, want>>-procv == <<pc, cw, cprep, cfin, ew, aref, aread, passHead>>+procv == <<pc, cw, cprep, cfin, ew, aref, aread, passHead, sseq>>vars == <<reqv, refv, gitv, envv, ghostv, lockv, procv>>NoLocks == [r \in Rows |-> "-"]@@ -158,7 +159,10 @@ FenceClear == ~RefIs({"fenced", "merged", "recovery"})\* claimableWriteSql: queued, or deferred with every hold released.ClaimState(w) == st[w] = "queued" \/ Released(w)\* EARLIER_HEAD (queued, running, recovering) and EARLIER_DEFERRED (released deferred): a write never passes an earlier one.-Blocks(e) == st[e] \in {"queued", "running"} \/ Released(e) \/ (HeldImportBlocks /\ st[e] = "deferred" /\ dimp[e] # None)+Blocks(e) == st[e] \in {"queued", "running"} \/ Released(e)+\* With HoldBehindHeld: compareFileTarget's held-first read, the writes to the page still held for an import that has+\* not landed.+HeldAhead(w) == {e \in Writes : e # w /\ st[e] = "deferred" /\ dimp[e] # None /\ ~Released(e)}\* The v292 trigger: a deferred row moves to running only when its refresh (and, since v291, its import) is released.TriggerAllows(w) == st[w] # "deferred" \/ Released(w)\* deferredReplayRefusal: a refresh that changed the write's file ends it revision_conflict.@@ -194,6 +198,11 @@ AdoptOneSided == fileV = dbVClaimable(w) == /\ ClaimState(w)/\ FenceClear/\ ~\E e \in Writes : seq[e] # 0 /\ seq[e] < seq[w] /\ Blocks(e)+ /\ pc[SYN] \notin {"s_import", "s_stamp"} \* no write passes a running sync row (it is never a lane group)+ /\ ~(pc[SYN] = "s_queued" /\ sseq < seq[w]) \* NAMES_PAGE: no write passes the queued row importing its page+\* A sync row is claimed only with no earlier write ahead of it (EARLIER_HEAD) and no write claimed at all (a sync row never+\* runs beside a later running foreground write).+SyncBehind == (\E w \in Writes : seq[w] # 0 /\ seq[w] < sseq /\ Blocks(w)) \/ cw # 0DrainZero == /\ \A w \in Writes : ~DrainCounts(w) /\ ~Released(w)/\ pc[SYN] = "idle" \* a sync pass in flight is queued work too@@ -252,6 +261,7 @@ Init ==/\ aref = [w \in Writes |-> None]/\ aread = [w \in Writes |-> "none"]/\ passHead = 0+ /\ sseq = 0\* A step that only acquires the next lock a process waits for.Acquire(p) ==@@ -283,14 +293,14 @@ AdmBegin(w) ==/\ Need(ADM(w), <<L(BRAIN, "S"), L(WT, "S")>>)/\ Go(ADM(w), "a_ref")/\ UNCHANGED <<st, code, seq, nseq, dref, dimp, applied, cpos, ncommit, overdue, adopted, refv, gitv, envv, ghostv, held,- cw, cprep, cfin, ew, aref, aread, passHead>>+ cw, cprep, cfin, ew, aref, aread, passHead, sseq>>AdmRef(w) ==/\ Ready(ADM(w), "a_ref")/\ aref' = [aref EXCEPT ![w] = cur]/\ Need(ADM(w), IF cur # None THEN AdmRefLock(cur) ELSE <<>>)/\ Go(ADM(w), "a_read")- /\ UNCHANGED <<reqv, refv, gitv, envv, ghostv, held, cw, cprep, cfin, ew, aread, passHead>>+ /\ UNCHANGED <<reqv, refv, gitv, envv, ghostv, held, cw, cprep, cfin, ew, aread, passHead, sseq>>\* The SELECT ... FOR NO KEY UPDATE reads the refresh row once it holds it (an EvalPlanQual recheck drops a row that\* went terminal meanwhile).@@ -299,7 +309,7 @@ AdmRead(w) ==/\ aread' = [aread EXCEPT ![w] = IF aref[w] # None /\ rstate[aref[w]] \in RefActive THEN rstate[aref[w]] ELSE "none"]/\ Need(ADM(w), <<L(SRC, "S"), L(CTR, "U")>>)/\ Go(ADM(w), "a_commit")- /\ UNCHANGED <<reqv, refv, gitv, envv, ghostv, held, cw, cprep, cfin, ew, aref, passHead>>+ /\ UNCHANGED <<reqv, refv, gitv, envv, ghostv, held, cw, cprep, cfin, ew, aref, passHead, sseq>>AdmCommit(w) ==/\ Ready(ADM(w), "a_commit")@@ -322,7 +332,7 @@ AdmCommit(w) ==/\ Commit(ADM(w)) /\ Go(ADM(w), "idle") /\ Need(ADM(w), <<>>)/\ aref' = [aref EXCEPT ![w] = None] /\ aread' = [aread EXCEPT ![w] = "none"]/\ UNCHANGED <<code, dimp, applied, cpos, ncommit, expRev, overdue, adopted, rchg, rhead, rmoved, nref, gitv, envv,- hidden, badExpiry, badRelease, badRevoke, cw, cprep, cfin, ew, passHead>>+ hidden, badExpiry, badRelease, badRevoke, cw, cprep, cfin, ew, passHead, sseq>>\* -------------------------------- consumer --------------------------------\* claimNextWrite: the protocol declaration and the head read, SELECT ... ORDER BY sequence LIMIT 1 FOR UPDATE SKIP@@ -330,7 +340,7 @@ AdmCommit(w) ==ConsBegin ==/\ pc[CONS] = "idle" /\ \E w \in Writes : Claimable(w)/\ Need(CONS, ClaimLocks) /\ Go(CONS, "c_select")- /\ UNCHANGED <<reqv, refv, gitv, envv, ghostv, held, cw, cprep, cfin, ew, aref, aread, passHead>>+ /\ UNCHANGED <<reqv, refv, gitv, envv, ghostv, held, cw, cprep, cfin, ew, aref, aread, passHead, sseq>>ConsSelect ==/\ Ready(CONS, "c_select")@@ -342,7 +352,7 @@ ConsSelect ==/\ cw' = w/\ held' = [held EXCEPT ![CONS][REQ(w)] = "U"]/\ Go(CONS, "c_claim")- /\ UNCHANGED <<reqv, refv, gitv, envv, ghostv, want, cprep, cfin, ew, aref, aread, passHead>>+ /\ UNCHANGED <<reqv, refv, gitv, envv, ghostv, want, cprep, cfin, ew, aref, aread, passHead, sseq>>ConsClaim ==/\ Ready(CONS, "c_claim")@@ -352,13 +362,14 @@ ConsClaim ==/\ Commit(CONS)/\ cw' = IF TriggerAllows(cw) THEN cw ELSE 0/\ UNCHANGED <<code, seq, nseq, dref, dimp, applied, cpos, ncommit, expRev, overdue, adopted, refv, gitv, envv, ghostv,- want, cprep, cfin, ew, aref, aread, passHead>>+ want, cprep, cfin, ew, aref, aread, passHead, sseq>>\* Preparation reads the page and the file (no database lock), then the deferred replay check.Outcome(w) ==LET drift == Worktree # dbVpending == drift /\ local = None /\ fileV # verAt[lastCommit]IN IF drift /\ PrepRefusesDrift THEN (IF pending THEN "pending" ELSE IF local # None THEN "local" ELSE "drift")+ ELSE IF ImportHolds /\ HoldBehindHeld /\ HeldAhead(w) # {} THEN "pending"ELSE IF w \notin Blind /\ dbV # expRev[w] THEN "conflict"ELSE IF dref[w] # None /\ ReplayDiff(dref[w]) THEN "conflict"ELSE IF dref[w] # None /\ revoked /\ ReplayAuth THEN "denied"@@ -371,7 +382,8 @@ Finish(s, c) == /\ cfin' = <<s, c>>ConsPrep ==/\ Ready(CONS, "c_prep")/\ LET o == Outcome(cw)- IN /\ cprep' = [o |-> o, db |-> dbV, wt |-> Worktree, lc |-> lastCommit]+ lc == IF ImportHolds /\ HoldBehindHeld /\ HeldAhead(cw) # {} THEN dimp[CHOOSE e \in HeldAhead(cw) : TRUE] ELSE lastCommit+ IN /\ cprep' = [o |-> o, db |-> dbV, wt |-> Worktree, lc |-> lc]/\ CASE o = "ok" ->/\ Need(CONS, <<L(BRAIN, "S"), L(WT, "S"), L(SRC, "S"), L(CTR, "U"), L(REQ(cw), "U")>>)/\ Go(CONS, "c_pub") /\ UNCHANGED cfin@@ -383,7 +395,7 @@ ConsPrep ==[] o \in {"local", "drift"} -> Finish("failed", "source_changed")[] o = "conflict" -> Finish("conflict", "revision_conflict")[] o = "denied" -> Finish("failed", "permission_denied")- /\ UNCHANGED <<reqv, refv, gitv, envv, ghostv, held, cw, ew, aref, aread, passHead>>+ /\ UNCHANGED <<reqv, refv, gitv, envv, ghostv, held, cw, ew, aref, aread, passHead, sseq>>\* publishMutation's transaction: declarePublication, guardOwnership, authorizeStoredRequest (the source FOR SHARE),\* lockCounters, the request FOR UPDATE (claim and state rechecked), the page revision and file rechecked.@@ -406,7 +418,7 @@ ConsPub ==/\ UNCHANGED <<head, verAt, fileV, nver, dbV, hist, local, applied, ncommit, cpos, hidden, badRelease, badRevoke>>/\ Commit(CONS) /\ ConsIdle/\ UNCHANGED <<seq, nseq, dref, dimp, expRev, overdue, adopted, refv, lastCommit, nforeign, nlocal, envv, badExpiry, badDefer,- want, ew, aref, aread, passHead>>+ want, ew, aref, aread, passHead, sseq>>\* holdForImport: UPDATE ... SET state='deferred', deferred_import_from=<last_commit read before the check>\* WHERE execution_token=<ours> AND state='running' AND publication_started=false.@@ -417,7 +429,7 @@ ConsHold ==ELSE UNCHANGED <<st, dimp>>/\ Commit(CONS) /\ ConsIdle/\ UNCHANGED <<code, seq, nseq, dref, applied, cpos, ncommit, expRev, overdue, adopted, refv, gitv, envv, ghostv,- want, ew, aref, aread, passHead>>+ want, ew, aref, aread, passHead, sseq>>\* adoptLocalEdit: the owner publishes the edit as its own managed_file_repair write (one commit of the file), then\* releases the caller's claim so it is prepared again.@@ -428,7 +440,7 @@ ConsAdopt ==/\ adopted' = [adopted EXCEPT ![cw] = TRUE]/\ Need(CONS, <<L(REQ(cw), "U")>>) /\ Go(CONS, "c_release")/\ UNCHANGED <<st, code, seq, nseq, dref, dimp, applied, cpos, ncommit, expRev, overdue, refv, lastCommit, nver, nforeign,- nlocal, envv, badExpiry, badRelease, badDefer, badRevoke, held, cw, cprep, cfin, ew, aref, aread, passHead>>+ nlocal, envv, badExpiry, badRelease, badDefer, badRevoke, held, cw, cprep, cfin, ew, aref, aread, passHead, sseq>>\* releaseUnpublishedClaim: back to queued, prepared again by the next claim.ConsRelease ==@@ -436,7 +448,7 @@ ConsRelease ==/\ st' = [st EXCEPT ![cw] = IF @ = "running" THEN "queued" ELSE @]/\ Commit(CONS) /\ ConsIdle/\ UNCHANGED <<code, seq, nseq, dref, dimp, applied, cpos, ncommit, expRev, overdue, adopted, refv, gitv, envv, ghostv,- want, ew, aref, aread, passHead>>+ want, ew, aref, aread, passHead, sseq>>\* completeWrite of a refusal: declarePersistenceProtocol, lockCounters, the request FOR UPDATE.ConsDone ==@@ -445,14 +457,14 @@ ConsDone ==/\ code' = [code EXCEPT ![cw] = IF st[cw] = "running" THEN cfin[2] ELSE @]/\ Commit(CONS) /\ ConsIdle/\ UNCHANGED <<seq, nseq, dref, dimp, applied, cpos, ncommit, expRev, overdue, adopted, refv, gitv, envv, ghostv,- want, ew, aref, aread, passHead>>+ want, ew, aref, aread, passHead, sseq>>\* -------------------------------- refresher --------------------------------\* fenceDraining: lockTopologyRows (brain FOR UPDATE, worktree FOR UPDATE, sources FOR UPDATE), INSERT draining.RefBegin ==/\ pc[RFR] = "idle" /\ cur = None /\ nref < MaxRefreshes/\ Need(RFR, <<L(BRAIN, "U"), L(WT, "U"), L(SRC, "U")>>) /\ Go(RFR, "r_fd")- /\ UNCHANGED <<reqv, refv, gitv, envv, ghostv, held, cw, cprep, cfin, ew, aref, aread, passHead>>+ /\ UNCHANGED <<reqv, refv, gitv, envv, ghostv, held, cw, cprep, cfin, ew, aref, aread, passHead, sseq>>FdCommit ==/\ Ready(RFR, "r_fd")@@ -463,19 +475,19 @@ FdCommit ==/\ rhead' = [rhead EXCEPT ![nref + 1] = head]/\ cur' = nref + 1 /\ nref' = nref + 1/\ Commit(RFR) /\ Go(RFR, "r_drain")- /\ UNCHANGED <<reqv, rel, rmoved, gitv, envv, ghostv, want, cw, cprep, cfin, ew, aref, aread, passHead>>+ /\ UNCHANGED <<reqv, rel, rmoved, gitv, envv, ghostv, want, cw, cprep, cfin, ew, aref, aread, passHead, sseq>>\* The drain poll (no locks), then fenceAndMerge's transaction: lockTopologyRows, drainCounts, transition to fenced.RefDrained ==/\ pc[RFR] = "r_drain" /\ want[RFR] = <<>> /\ DrainZero/\ Need(RFR, <<L(BRAIN, "U"), L(WT, "U"), L(SRC, "U"), L(REF(nref), "NKU")>>) /\ Go(RFR, "r_fm")- /\ UNCHANGED <<reqv, refv, gitv, envv, ghostv, held, cw, cprep, cfin, ew, aref, aread, passHead>>+ /\ UNCHANGED <<reqv, refv, gitv, envv, ghostv, held, cw, cprep, cfin, ew, aref, aread, passHead, sseq>>\* --wait-drain ran out: transition draining -> aborted.RefDrainTimeout ==/\ pc[RFR] = "r_drain" /\ ~DrainZero/\ Need(RFR, <<L(REF(nref), "NKU")>>) /\ Go(RFR, "r_abort")- /\ UNCHANGED <<reqv, refv, gitv, envv, ghostv, held, cw, cprep, cfin, ew, aref, aread, passHead>>+ /\ UNCHANGED <<reqv, refv, gitv, envv, ghostv, held, cw, cprep, cfin, ew, aref, aread, passHead, sseq>>FmCommit ==/\ Ready(RFR, "r_fm")@@ -483,7 +495,7 @@ FmCommit ==THEN /\ rstate' = [rstate EXCEPT ![nref] = "fenced"] /\ Go(RFR, "r_merge") /\ Need(RFR, <<>>)ELSE /\ UNCHANGED rstate /\ Go(RFR, "r_abort") /\ Need(RFR, <<L(REF(nref), "NKU")>>) \* refresh_source_changed/\ Commit(RFR)- /\ UNCHANGED <<reqv, rel, rchg, rhead, rmoved, cur, nref, gitv, envv, ghostv, cw, cprep, cfin, ew, aref, aread, passHead>>+ /\ UNCHANGED <<reqv, rel, rchg, rhead, rmoved, cur, nref, gitv, envv, ghostv, cw, cprep, cfin, ew, aref, aread, passHead, sseq>>\* transition(..., 'aborted'): the release trigger stamps released_at.AbortCommit ==@@ -493,7 +505,7 @@ AbortCommit ==ELSE UNCHANGED <<rstate, rel>>/\ cur' = None/\ Commit(RFR) /\ Go(RFR, "idle")- /\ UNCHANGED <<reqv, rchg, rhead, rmoved, nref, gitv, envv, ghostv, want, cw, cprep, cfin, ew, aref, aread, passHead>>+ /\ UNCHANGED <<reqv, rchg, rhead, rmoved, nref, gitv, envv, ghostv, want, cw, cprep, cfin, ew, aref, aread, passHead, sseq>>\* Under the commit lock: HEAD must still be old_head (another agent's plain-git commit is not), then the ff-only merge.RefMerge ==@@ -505,13 +517,13 @@ RefMerge ==ELSE NewCommit(fileV) /\ UNCHANGED <<nver, rmoved>>/\ Need(RFR, <<L(REF(nref), "NKU"), L(SRC, "NKU")>>) /\ Go(RFR, "r_bk")/\ UNCHANGED <<reqv, rstate, rel, rchg, rhead, cur, nref, dbV, hist, lastCommit, local, nforeign, nlocal, envv, ghostv,- held, cw, cprep, cfin, ew, aref, aread, passHead>>+ held, cw, cprep, cfin, ew, aref, aread, passHead, sseq>>MarkCommit ==/\ Ready(RFR, "r_mark")/\ rstate' = [rstate EXCEPT ![nref] = "recovery"]/\ Commit(RFR) /\ Go(RFR, "r_recovery")- /\ UNCHANGED <<reqv, rel, rchg, rhead, rmoved, cur, nref, gitv, envv, ghostv, want, cw, cprep, cfin, ew, aref, aread, passHead>>+ /\ UNCHANGED <<reqv, rel, rchg, rhead, rmoved, cur, nref, gitv, envv, ghostv, want, cw, cprep, cfin, ew, aref, aread, passHead, sseq>>\* A person resolves recovery_required: reset to old_head and --abandon, or reset to the target and --resume.PersonAbandon ==@@ -519,7 +531,7 @@ PersonAbandon ==/\ head' = rhead[nref] /\ fileV' = verAt[rhead[nref]] /\ local' = None/\ Need(RFR, <<L(REF(nref), "NKU")>>) /\ Go(RFR, "r_abort")/\ UNCHANGED <<reqv, refv, verAt, dbV, hist, lastCommit, nver, nforeign, nlocal, envv, ghostv, held,- cw, cprep, cfin, ew, aref, aread, passHead>>+ cw, cprep, cfin, ew, aref, aread, passHead, sseq>>PersonResume ==/\ pc[RFR] = "r_recovery"@@ -531,20 +543,20 @@ PersonResume ==/\ local' = None/\ Need(RFR, <<L(REF(nref), "NKU"), L(SRC, "NKU")>>) /\ Go(RFR, "r_bk")/\ UNCHANGED <<reqv, rstate, rel, rchg, rhead, cur, nref, dbV, hist, lastCommit, nforeign, nlocal, envv, ghostv, held,- cw, cprep, cfin, ew, aref, aread, passHead>>+ cw, cprep, cfin, ew, aref, aread, passHead, sseq>>\* bookkeeping: transition to merged (the refresh row), the sources' upstream observation, then syncing.BkCommit ==/\ Ready(RFR, "r_bk")/\ rstate' = [rstate EXCEPT ![nref] = IF @ \in {"fenced", "merged", "recovery"} THEN "syncing" ELSE @]/\ Commit(RFR) /\ Go(RFR, "r_syncing")- /\ UNCHANGED <<reqv, rel, rchg, rhead, rmoved, cur, nref, gitv, envv, ghostv, want, cw, cprep, cfin, ew, aref, aread, passHead>>+ /\ UNCHANGED <<reqv, rel, rchg, rhead, rmoved, cur, nref, gitv, envv, ghostv, want, cw, cprep, cfin, ew, aref, aread, passHead, sseq>>\* syncMembers ends: completed once members reach the target, or (#8072) when a member's sync is blocked or deferred.RefComplete ==/\ pc[RFR] = "r_syncing"/\ Need(RFR, <<L(REF(nref), "NKU")>>) /\ Go(RFR, "r_complete")- /\ UNCHANGED <<reqv, refv, gitv, envv, ghostv, held, cw, cprep, cfin, ew, aref, aread, passHead>>+ /\ UNCHANGED <<reqv, refv, gitv, envv, ghostv, held, cw, cprep, cfin, ew, aref, aread, passHead, sseq>>CompleteCommit ==/\ Ready(RFR, "r_complete")@@ -553,14 +565,14 @@ CompleteCommit ==/\ cur' = NoneELSE UNCHANGED <<rstate, rel, cur>>/\ Commit(RFR) /\ Go(RFR, "idle")- /\ UNCHANGED <<reqv, rchg, rhead, rmoved, nref, gitv, envv, ghostv, want, cw, cprep, cfin, ew, aref, aread, passHead>>+ /\ UNCHANGED <<reqv, rchg, rhead, rmoved, nref, gitv, envv, ghostv, want, cw, cprep, cfin, ew, aref, aread, passHead, sseq>>\* --------------------------------- expiry ---------------------------------\* The candidate query reads without locks; then one transaction per row.ExpRefBegin(w) ==/\ pc[EXP] = "idle" /\ st[w] = "deferred" /\ dref[w] # None /\ ~rel[dref[w]] /\ overdue[w]/\ ew' = w /\ Need(EXP, ExpiryLocks(dref[w], w)) /\ Go(EXP, "e_ref")- /\ UNCHANGED <<reqv, refv, gitv, envv, ghostv, held, cw, cprep, cfin, aref, aread, passHead>>+ /\ UNCHANGED <<reqv, refv, gitv, envv, ghostv, held, cw, cprep, cfin, aref, aread, passHead, sseq>>\* completeWrite: a terminal row stays as it is; a row someone claimed (a live execution token) is write_claim_lost.ExpRefCommit ==@@ -571,7 +583,7 @@ ExpRefCommit ==/\ badExpiry' = (badExpiry \/ (fire /\ ~(st[ew] = "deferred" /\ ~rel[dref[ew]])))/\ Commit(EXP) /\ Go(EXP, "idle") /\ ew' = 0/\ UNCHANGED <<seq, nseq, dref, dimp, applied, cpos, ncommit, expRev, overdue, adopted, refv, gitv, envv,- hidden, badRelease, badDefer, badRevoke, want, cw, cprep, cfin, aref, aread, passHead>>+ hidden, badRelease, badDefer, badRevoke, want, cw, cprep, cfin, aref, aread, passHead, sseq>>\* #8705: a write held for an import that has not landed past the limit ends pending_import_expired.ExpImpBegin(w) ==@@ -581,7 +593,7 @@ ExpImpBegin(w) ==/\ overdue[w]/\ ew' = w /\ Go(EXP, "e_recheck")/\ Need(EXP, IF ImportExpiryLocksSource THEN ImportExpiryLocks ELSE <<>>)- /\ UNCHANGED <<reqv, refv, gitv, envv, ghostv, held, cw, cprep, cfin, aref, aread, passHead>>+ /\ UNCHANGED <<reqv, refv, gitv, envv, ghostv, held, cw, cprep, cfin, aref, aread, passHead, sseq>>ExpImpRecheck ==/\ Ready(EXP, "e_recheck")@@ -590,7 +602,7 @@ ExpImpRecheck ==ELSE UNCHANGED want /\ Go(EXP, "idle")/\ held' = IF ImportExpiryRecheck(ew) THEN held ELSE [held EXCEPT ![EXP] = NoLocks]/\ ew' = IF ImportExpiryRecheck(ew) THEN ew ELSE 0- /\ UNCHANGED <<reqv, refv, gitv, envv, ghostv, cw, cprep, cfin, aref, aread, passHead>>+ /\ UNCHANGED <<reqv, refv, gitv, envv, ghostv, cw, cprep, cfin, aref, aread, passHead, sseq>>ExpImpCommit ==/\ Ready(EXP, "e_imp")@@ -600,7 +612,7 @@ ExpImpCommit ==/\ badExpiry' = (badExpiry \/ (fire /\ ~(st[ew] = "deferred" /\ dimp[ew] # None /\ lastCommit = dimp[ew])))/\ Commit(EXP) /\ Go(EXP, "idle") /\ ew' = 0/\ UNCHANGED <<seq, nseq, dref, dimp, applied, cpos, ncommit, expRev, overdue, adopted, refv, gitv, envv,- hidden, badRelease, badDefer, badRevoke, want, cw, cprep, cfin, aref, aread, passHead>>+ hidden, badRelease, badDefer, badRevoke, want, cw, cprep, cfin, aref, aread, passHead, sseq>>\* ------------------------------ managed sync ------------------------------\* A pass starts only with no refresh, or as the refresh's own member sync (assertManagedSyncAllowed). It imports F's@@ -608,23 +620,36 @@ ExpImpCommit ==SyncBegin ==/\ pc[SYN] = "idle" /\ (cur = None \/ RefIs({"syncing"})) /\ lastCommit # head/\ passHead' = head+ /\ sseq' = nseq + 1 /\ nseq' = nseq + 1+ /\ Go(SYN, "s_queued")+ /\ UNCHANGED <<st, code, seq, dref, dimp, applied, cpos, ncommit, expRev, overdue, adopted, refv, gitv, envv, ghostv,+ lockv, cw, cprep, cfin, ew, aref, aread>>++\* The page row is claimed once no earlier write is ahead of it (EARLIER_HEAD), then imported.+SyncImportBegin ==+ /\ pc[SYN] = "s_queued" /\ ~SyncBehind/\ Need(SYN, <<L(BRAIN, "S")>>) /\ Go(SYN, "s_import")- /\ UNCHANGED <<reqv, refv, gitv, envv, ghostv, held, cw, cprep, cfin, ew, aref, aread>>+ /\ UNCHANGED <<reqv, refv, gitv, envv, ghostv, held, cw, cprep, cfin, ew, aref, aread, passHead, sseq>>SyncImport ==/\ Ready(SYN, "s_import")/\ IF local = None /\ fileV = verAt[passHead] /\ fileV # dbVTHEN dbV' = fileV /\ hist' = hist \cup {fileV}ELSE UNCHANGED <<dbV, hist>> \* nothing to import, or a dirty file sync holds- /\ Commit(SYN)- /\ Need(SYN, <<L(BRAIN, "S"), L(SRC, "NKU")>>) /\ Go(SYN, "s_stamp")+ /\ Commit(SYN) /\ Go(SYN, "s_ckpt")/\ UNCHANGED <<reqv, refv, head, verAt, fileV, lastCommit, local, nver, nforeign, nlocal, envv, ghostv,- cw, cprep, cfin, ew, aref, aread, passHead>>+ want, cw, cprep, cfin, ew, aref, aread, passHead, sseq>>++\* The checkpoint row is claimed once no earlier write is ahead of it; the pass stamp moves last_commit (UPDATE sources).+SyncStampBegin ==+ /\ pc[SYN] = "s_ckpt" /\ ~SyncBehind+ /\ Need(SYN, <<L(BRAIN, "S"), L(SRC, "NKU")>>) /\ Go(SYN, "s_stamp")+ /\ UNCHANGED <<reqv, refv, gitv, envv, ghostv, held, cw, cprep, cfin, ew, aref, aread, passHead, sseq>>SyncStamp ==/\ Ready(SYN, "s_stamp")/\ lastCommit' = passHead- /\ Commit(SYN) /\ Go(SYN, "idle") /\ passHead' = 0+ /\ Commit(SYN) /\ Go(SYN, "idle") /\ passHead' = 0 /\ sseq' = 0/\ UNCHANGED <<reqv, refv, head, verAt, fileV, dbV, hist, local, nver, nforeign, nlocal, envv, ghostv,want, cw, cprep, cfin, ew, aref, aread>>@@ -666,7 +691,7 @@ ConsStep == ConsBegin \/ ConsSelect \/ ConsClaim \/ ConsPrep \/ ConsPub \/ ConsHRefStep == FdCommit \/ RefDrained \/ RefDrainTimeout \/ FmCommit \/ AbortCommit \/ RefMerge \/ MarkCommit\/ BkCommit \/ RefComplete \/ CompleteCommitExpStep == (\E w \in Writes : ExpRefBegin(w) \/ ExpImpBegin(w)) \/ ExpRefCommit \/ ExpImpRecheck \/ ExpImpCommit-SyncStep == SyncBegin \/ SyncImport \/ SyncStamp+SyncStep == SyncBegin \/ SyncImportBegin \/ SyncImport \/ SyncStampBegin \/ SyncStampAdmStep(w) == AdmRef(w) \/ AdmRead(w) \/ AdmCommit(w)Next ==diff --git a/spec/tla/refresh-barrier/RefreshBarrierAdopt.cfg b/spec/tla/refresh-barrier/RefreshBarrierAdopt.cfgindex 6c96b708ac..ce7c5cd1df 100644--- a/spec/tla/refresh-barrier/RefreshBarrierAdopt.cfg+++ b/spec/tla/refresh-barrier/RefreshBarrierAdopt.cfg@@ -1,5 +1,5 @@-\* #8708 as coded on #8705: one uncommitted local edit adopted by a drift refusal, no foreign commit. The two #8705-\* open findings' invariants are left to RefreshBarrierImport*.cfg. RefreshBarrierAdoptNightly.cfg adds a foreign commit.+\* #8708 on #8705 with its fixes: one uncommitted local edit adopted by a drift refusal, no foreign commit.+\* RefreshBarrierAdoptNightly.cfg adds a foreign commit.SPECIFICATION SpecCHECK_DEADLOCK FALSECONSTANTS@@ -9,8 +9,8 @@ CONSTANTSMaxForeign = 0MaxLocalEdits = 1ImportHolds = TRUE- HeldImportBlocks = FALSE- ImportExpiryLocksSource = FALSE+ HoldBehindHeld = TRUE+ ImportExpiryLocksSource = TRUERecoveryDefers = TRUEAdoptLocalEdits = TRUEMayRestore = FALSE@@ -26,5 +26,7 @@ INVARIANT DeferredWaitsForReleaseINVARIANT RefreshChangeNeverOverwrittenINVARIANT NoDeferToReleasedINVARIANT ReleasedWhenEnded+INVARIANT ExpiryOnlyWhileWaitingINVARIANT RestoreLeavesNoOpenWriteINVARIANT NoApplyAfterRevoke+INVARIANT CommitsInAdmissionOrderdiff --git a/spec/tla/refresh-barrier/RefreshBarrierImportOrder.cfg b/spec/tla/refresh-barrier/RefreshBarrierAdoptNightly.cfgsimilarity index 50%rename from spec/tla/refresh-barrier/RefreshBarrierImportOrder.cfgrename to spec/tla/refresh-barrier/RefreshBarrierAdoptNightly.cfgindex 795bc9f068..74d72ac1cd 100644--- a/spec/tla/refresh-barrier/RefreshBarrierImportOrder.cfg+++ b/spec/tla/refresh-barrier/RefreshBarrierAdoptNightly.cfg@@ -1,8 +1,4 @@-\* #8705 as coded (writes held for an import, deferral behind recovery_required). Open finding: a write held for an-\* import holds back no later write, so a later write to the same page commits between the sync's page import and its-\* last_commit stamp, and the held write then lands on top of it. Flip HeldImportBlocks to TRUE once the claim's FIFO-\* rule counts held rows. ExpiryOnlyWhileWaiting is RefreshBarrierImportExpiry.cfg's finding.-\* OPEN_FINDING: CommitsInAdmissionOrder+\* Nightly only: RefreshBarrierAdopt.cfg with another agent's commit to the file.SPECIFICATION SpecCHECK_DEADLOCK FALSECONSTANTS@@ -10,12 +6,12 @@ CONSTANTSBlind = {1}MaxRefreshes = 1MaxForeign = 1- MaxLocalEdits = 0+ MaxLocalEdits = 1ImportHolds = TRUE- HeldImportBlocks = FALSE- ImportExpiryLocksSource = FALSE+ HoldBehindHeld = TRUE+ ImportExpiryLocksSource = TRUERecoveryDefers = TRUE- AdoptLocalEdits = FALSE+ AdoptLocalEdits = TRUEMayRestore = FALSEMayRevoke = FALSEINVARIANT TypeOK@@ -29,6 +25,7 @@ INVARIANT DeferredWaitsForReleaseINVARIANT RefreshChangeNeverOverwrittenINVARIANT NoDeferToReleasedINVARIANT ReleasedWhenEnded+INVARIANT ExpiryOnlyWhileWaitingINVARIANT RestoreLeavesNoOpenWriteINVARIANT NoApplyAfterRevokeINVARIANT CommitsInAdmissionOrderdiff --git a/spec/tla/refresh-barrier/RefreshBarrierEnv.cfg b/spec/tla/refresh-barrier/RefreshBarrierEnv.cfgindex 31127209c8..7de5fe94d5 100644--- a/spec/tla/refresh-barrier/RefreshBarrierEnv.cfg+++ b/spec/tla/refresh-barrier/RefreshBarrierEnv.cfg@@ -8,7 +8,7 @@ CONSTANTSMaxForeign = 0MaxLocalEdits = 0ImportHolds = FALSE- HeldImportBlocks = FALSE+ HoldBehindHeld = FALSEImportExpiryLocksSource = FALSERecoveryDefers = FALSEAdoptLocalEdits = FALSEdiff --git a/spec/tla/refresh-barrier/RefreshBarrierImportExpiry.cfg b/spec/tla/refresh-barrier/RefreshBarrierImport.cfgsimilarity index 55%rename from spec/tla/refresh-barrier/RefreshBarrierImportExpiry.cfgrename to spec/tla/refresh-barrier/RefreshBarrierImport.cfgindex edb7514ed0..5277ebc2c0 100644--- a/spec/tla/refresh-barrier/RefreshBarrierImportExpiry.cfg+++ b/spec/tla/refresh-barrier/RefreshBarrierImport.cfg@@ -1,8 +1,6 @@-\* #8705 as coded. Open finding: the import expiry re-checks last_commit without a lock on the source row, so a sync-\* stamp that commits between the re-check and completeWrite leaves a write failed pending_import_expired after its import-\* landed. Flip ImportExpiryLocksSource to TRUE once the re-check declares the protocol and locks the source row FOR SHARE.-\* CommitsInAdmissionOrder is RefreshBarrierImportOrder.cfg's finding.-\* OPEN_FINDING: ExpiryOnlyWhileWaiting+\* #8705 with its fixes (03b89121): writes held for an import, a later write to the page held beside a held one, the+\* import expiry re-checking under the source row lock, deferral behind recovery_required. Mutants hold-beside-held-off and+\* import-expiry-skips-source-lock put back the two races this model found before the fixes.SPECIFICATION SpecCHECK_DEADLOCK FALSECONSTANTS@@ -12,8 +10,8 @@ CONSTANTSMaxForeign = 1MaxLocalEdits = 0ImportHolds = TRUE- HeldImportBlocks = FALSE- ImportExpiryLocksSource = FALSE+ HoldBehindHeld = TRUE+ ImportExpiryLocksSource = TRUERecoveryDefers = TRUEAdoptLocalEdits = FALSEMayRestore = FALSE@@ -32,3 +30,4 @@ INVARIANT ReleasedWhenEndedINVARIANT ExpiryOnlyWhileWaitingINVARIANT RestoreLeavesNoOpenWriteINVARIANT NoApplyAfterRevoke+INVARIANT CommitsInAdmissionOrderdiff --git a/spec/tla/refresh-barrier/RefreshBarrierLiveness.cfg b/spec/tla/refresh-barrier/RefreshBarrierLiveness.cfgindex 768d2ca99b..178ced9844 100644--- a/spec/tla/refresh-barrier/RefreshBarrierLiveness.cfg+++ b/spec/tla/refresh-barrier/RefreshBarrierLiveness.cfg@@ -1,14 +1,15 @@-\* Liveness under fairness: every deferred write leaves deferral, and every admitted write ends.+\* Liveness under fairness, one write: every deferred write leaves deferral and every admitted write ends.+\* RefreshBarrierLivenessNightly.cfg checks two writes.SPECIFICATION FairSpecCHECK_DEADLOCK FALSECONSTANTS- NW = 2+ NW = 1Blind = {1}MaxRefreshes = 1MaxForeign = 0MaxLocalEdits = 0ImportHolds = FALSE- HeldImportBlocks = FALSE+ HoldBehindHeld = FALSEImportExpiryLocksSource = FALSERecoveryDefers = FALSEAdoptLocalEdits = FALSEdiff --git a/spec/tla/refresh-barrier/RefreshBarrierLivenessNightly.cfg b/spec/tla/refresh-barrier/RefreshBarrierLivenessNightly.cfgnew file mode 100644index 0000000000..64acca86bc--- /dev/null+++ b/spec/tla/refresh-barrier/RefreshBarrierLivenessNightly.cfg@@ -0,0 +1,18 @@+\* Nightly only: RefreshBarrierLiveness.cfg with two writes (one blind).+SPECIFICATION FairSpec+CHECK_DEADLOCK FALSE+CONSTANTS+ NW = 2+ Blind = {1}+ MaxRefreshes = 1+ MaxForeign = 0+ MaxLocalEdits = 0+ ImportHolds = FALSE+ HoldBehindHeld = FALSE+ ImportExpiryLocksSource = FALSE+ RecoveryDefers = FALSE+ AdoptLocalEdits = FALSE+ MayRestore = FALSE+ MayRevoke = FALSE+PROPERTY DeferredLeaves+PROPERTY EveryWriteEndsdiff --git a/spec/tla/refresh-barrier/mutants.json b/spec/tla/refresh-barrier/mutants.jsonnew file mode 100644index 0000000000..7195af7336--- /dev/null+++ b/spec/tla/refresh-barrier/mutants.json@@ -0,0 +1,373 @@+{+ "$comment": "Each mutant removes one guard of the refresh write barrier (kind guard) or a guard that is defense in depth at these bounds (kind survivor, expect survives). spec edits a copy of the model and TLC must report expect; code makes the same edit to the implementation, and each named test must fail under it and pass without it. Run: bun scripts/spec/mutants.ts refresh-barrier [--spec-only|--code-only] [--only <id>]. See spec/tla/README.md.",+ "module": "RefreshBarrier",+ "config": "RefreshBarrier.cfg",+ "mutants": [+ {+ "id": "expiry-locks-refresh-first",+ "kind": "guard",+ "title": "the refresh-barrier expiry locks the refresh row before it declares the protocol (finding D reverted)",+ "expect": "NoDeadlock",+ "spec": {+ "file": "RefreshBarrier.tla",+ "before": "ExpiryLocks(i, w) == <<L(BRAIN, \"S\"), L(REF(i), \"NKU\"), L(CTR, \"U\"), L(REQ(w), \"U\")>>",+ "after": "ExpiryLocks(i, w) == <<L(REF(i), \"NKU\"), L(BRAIN, \"S\"), L(CTR, \"U\"), L(REQ(w), \"U\")>>"+ },+ "code": {+ "edits": [+ {+ "file": "src/core/persistence/refresh-barrier.ts",+ "before": " await declarePersistenceProtocol(tx);\n const [held] = await tx.executeRaw('SELECT 1 FROM persistence_worktree_refreshes WHERE id=$1::uuid AND released_at IS NULL FOR NO KEY UPDATE', [row.deferred_refresh_id]);\n",+ "after": " const [held] = await tx.executeRaw('SELECT 1 FROM persistence_worktree_refreshes WHERE id=$1::uuid AND released_at IS NULL FOR NO KEY UPDATE', [row.deferred_refresh_id]);\n await declarePersistenceProtocol(tx);\n"+ }+ ],+ "tests": [+ {+ "test": "test/refresh-write-barrier.test.ts",+ "filter": "an expiry that overlaps a fence waits on the brain row",+ "postgres": true+ }+ ]+ }+ },+ {+ "id": "restore-keeps-deferred",+ "kind": "guard",+ "title": "a shared-skill restore leaves deferred writes open (finding C reverted)",+ "expect": "RestoreLeavesNoOpenWrite",+ "config": "RefreshBarrierEnv.cfg",+ "spec": {+ "file": "RefreshBarrier.tla",+ "before": "RestoreCancels == {\"queued\", \"deferred\", \"running\"}",+ "after": "RestoreCancels == {\"queued\", \"running\"}"+ },+ "code": {+ "edits": [+ {+ "file": "src/core/shared-skills/restore.ts",+ "before": "state IN ('queued','deferred','running') ORDER BY sequence FOR UPDATE",+ "after": "state IN ('queued','running') ORDER BY sequence FOR UPDATE"+ }+ ],+ "tests": [+ {+ "test": "test/refresh-write-barrier.test.ts",+ "filter": "a shared-skill restore cancels a deferred write too"+ }+ ]+ }+ },+ {+ "id": "replay-ignores-refresh-change",+ "kind": "guard",+ "title": "a deferred write's replay ignores what its refresh changed",+ "expect": "RefreshChangeNeverOverwritten",+ "spec": {+ "file": "RefreshBarrier.tla",+ "before": "ReplayDiff(i) == rmoved[i]",+ "after": "ReplayDiff(i) == FALSE"+ },+ "code": {+ "edits": [+ {+ "file": "src/core/persistence/refresh-barrier.ts",+ "before": "if (paths === null || paths.has(target)) touched.push(target);",+ "after": "if (paths === null) touched.push(target);"+ }+ ],+ "tests": [+ {+ "test": "test/refresh-write-barrier.test.ts",+ "filter": "a deferred blind write to a page the refresh changed ends revision_conflict"+ }+ ]+ }+ },+ {+ "id": "release-only-on-completed",+ "kind": "guard",+ "title": "the release trigger stamps released_at only when a refresh completes, not when it aborts",+ "expect": "ReleasedWhenEnded",+ "spec": {+ "file": "RefreshBarrier.tla",+ "before": "ReleaseOn(s) == s \\in {\"completed\", \"aborted\"}",+ "after": "ReleaseOn(s) == s = \"completed\""+ },+ "code": {+ "edits": [+ {+ "file": "src/core/schema-migrations/v292-refresh-write-barrier.ts",+ "before": " END $$;\n ALTER TABLE persistence_consumers ADD COLUMN IF NOT EXISTS capabilities text[] NOT NULL DEFAULT '{}';\n CREATE OR REPLACE FUNCTION gbrain_release_refresh_barrier() RETURNS trigger LANGUAGE plpgsql SET search_path = pg_catalog, public, pg_temp AS $$\n BEGIN\n NEW.released_at := COALESCE(NEW.released_at, now());\n RETURN NEW;\n END $$;\n DROP TRIGGER IF EXISTS gbrain_refresh_release ON persistence_worktree_refreshes;\n CREATE TRIGGER gbrain_refresh_release BEFORE UPDATE OF state ON persistence_worktree_refreshes FOR EACH ROW\n WHEN (NEW.state IN ('completed','aborted') AND OLD.state NOT IN ('completed','aborted'))",+ "after": " END $$;\n ALTER TABLE persistence_consumers ADD COLUMN IF NOT EXISTS capabilities text[] NOT NULL DEFAULT '{}';\n CREATE OR REPLACE FUNCTION gbrain_release_refresh_barrier() RETURNS trigger LANGUAGE plpgsql SET search_path = pg_catalog, public, pg_temp AS $$\n BEGIN\n NEW.released_at := COALESCE(NEW.released_at, now());\n RETURN NEW;\n END $$;\n DROP TRIGGER IF EXISTS gbrain_refresh_release ON persistence_worktree_refreshes;\n CREATE TRIGGER gbrain_refresh_release BEFORE UPDATE OF state ON persistence_worktree_refreshes FOR EACH ROW\n WHEN (NEW.state='completed' AND OLD.state NOT IN ('completed','aborted'))"+ },+ {+ "file": "src/core/schema-migrations/v292-refresh-write-barrier.ts",+ "before": " ALTER TABLE persistence_requests ADD COLUMN IF NOT EXISTS deferred_refresh_id uuid REFERENCES persistence_worktree_refreshes(id);\n ALTER TABLE persistence_consumers ADD COLUMN IF NOT EXISTS capabilities text[] NOT NULL DEFAULT '{}';\n CREATE OR REPLACE FUNCTION gbrain_release_refresh_barrier() RETURNS trigger LANGUAGE plpgsql SET search_path = pg_catalog, public, pg_temp AS $$\n BEGIN\n NEW.released_at := COALESCE(NEW.released_at, now());\n RETURN NEW;\n END $$;\n DROP TRIGGER IF EXISTS gbrain_refresh_release ON persistence_worktree_refreshes;\n CREATE TRIGGER gbrain_refresh_release BEFORE UPDATE OF state ON persistence_worktree_refreshes FOR EACH ROW\n WHEN (NEW.state IN ('completed','aborted') AND OLD.state NOT IN ('completed','aborted'))",+ "after": " ALTER TABLE persistence_requests ADD COLUMN IF NOT EXISTS deferred_refresh_id uuid REFERENCES persistence_worktree_refreshes(id);\n ALTER TABLE persistence_consumers ADD COLUMN IF NOT EXISTS capabilities text[] NOT NULL DEFAULT '{}';\n CREATE OR REPLACE FUNCTION gbrain_release_refresh_barrier() RETURNS trigger LANGUAGE plpgsql SET search_path = pg_catalog, public, pg_temp AS $$\n BEGIN\n NEW.released_at := COALESCE(NEW.released_at, now());\n RETURN NEW;\n END $$;\n DROP TRIGGER IF EXISTS gbrain_refresh_release ON persistence_worktree_refreshes;\n CREATE TRIGGER gbrain_refresh_release BEFORE UPDATE OF state ON persistence_worktree_refreshes FOR EACH ROW\n WHEN (NEW.state='completed' AND OLD.state NOT IN ('completed','aborted'))"+ }+ ],+ "tests": [+ {+ "test": "test/refresh-write-barrier.test.ts",+ "filter": "a write deferred behind a refresh that aborts commits once the refresh is released"+ }+ ]+ }+ },+ {+ "id": "expiry-skips-release-recheck",+ "kind": "guard",+ "title": "the refresh-barrier expiry does not re-read released_at under the refresh row lock",+ "expect": "ExpiryOnlyWhileWaiting",+ "spec": {+ "file": "RefreshBarrier.tla",+ "before": "ExpiryRecheck(i) == ~rel[i]",+ "after": "ExpiryRecheck(i) == TRUE"+ },+ "code": {+ "edits": [+ {+ "file": "src/core/persistence/refresh-barrier.ts",+ "before": "'SELECT 1 FROM persistence_worktree_refreshes WHERE id=$1::uuid AND released_at IS NULL FOR NO KEY UPDATE'",+ "after": "'SELECT 1 FROM persistence_worktree_refreshes WHERE id=$1::uuid FOR NO KEY UPDATE'"+ }+ ],+ "tests": [+ {+ "test": "test/refresh-write-barrier.test.ts",+ "filter": "racing the end",+ "postgres": true+ }+ ]+ }+ },+ {+ "id": "admission-skips-refresh-lock",+ "kind": "guard",+ "title": "admission reads the active refresh without locking its row",+ "expect": "NoDeferToReleased",+ "spec": {+ "file": "RefreshBarrier.tla",+ "before": "AdmRefLock(i) == <<L(REF(i), \"NKU\")>>",+ "after": "AdmRefLock(i) == <<>>"+ },+ "code": {+ "edits": [+ {+ "file": "src/core/persistence/journal.ts",+ "before": "LIMIT 1${barrier ? ' FOR NO KEY UPDATE' : ''}) f) AS refresh",+ "after": "LIMIT 1) f) AS refresh"+ }+ ],+ "tests": [+ {+ "test": "test/refresh-write-barrier.test.ts",+ "filter": "while its refresh ends",+ "postgres": true+ }+ ]+ }+ },+ {+ "id": "publication-skips-reauth",+ "kind": "guard",+ "title": "publication does not re-authorize the stored request",+ "expect": "NoApplyAfterRevoke",+ "config": "RefreshBarrierEnv.cfg",+ "spec": {+ "file": "RefreshBarrier.tla",+ "before": "PublishAuth == TRUE",+ "after": "PublishAuth == FALSE"+ },+ "code": {+ "edits": [+ {+ "file": "src/core/persistence/coordinator.ts",+ "before": "await authorizeStoredRequest(tx, row, true, { pageVisibility: skill });",+ "after": "if (skill) await authorizeStoredRequest(tx, row, true, { pageVisibility: skill });"+ },+ {+ "file": "src/core/persistence/group-publish.ts",+ "before": "() => authorizeStoredRequest(tx, rows[i]!, true, { pageVisibility: false }),",+ "after": "async () => undefined,"+ }+ ],+ "tests": [+ {+ "test": "test/refresh-write-barrier.test.ts",+ "filter": "revoked after it was claimed"+ }+ ]+ }+ },+ {+ "id": "preparation-ignores-drift",+ "kind": "guard",+ "title": "preparation writes over a file that differs from the page",+ "expect": "ImportedRevisionNeverHidden",+ "spec": {+ "file": "RefreshBarrier.tla",+ "before": "PrepRefusesDrift == TRUE",+ "after": "PrepRefusesDrift == FALSE"+ },+ "code": {+ "edits": [+ {+ "file": "src/core/persistence/page-prepare.ts",+ "before": "if (!matches && !staleCache) {",+ "after": "if (!matches && !staleCache && false) {"+ }+ ],+ "tests": [+ {+ "test": "test/pending-import-async-git-7436.test.ts",+ "filter": "keeps the event loop running"+ }+ ]+ }+ },+ {+ "id": "drain-ignores-running",+ "kind": "guard",+ "title": "the drain count leaves out claimed (running) writes",+ "expect": "FenceNeverMidPrepare",+ "spec": {+ "file": "RefreshBarrier.tla",+ "before": "DrainCounts(w) == st[w] \\in {\"queued\", \"running\"}",+ "after": "DrainCounts(w) == st[w] = \"queued\""+ },+ "code": {+ "edits": [+ {+ "file": "src/core/persistence/worktree-refresh.ts",+ "before": "r.state IN ('queued','running','recovering')",+ "after": "r.state IN ('queued','recovering')"+ }+ ],+ "tests": [+ {+ "test": "test/persistence-worktree-refresh.test.ts",+ "filter": "13e."+ }+ ]+ }+ },+ {+ "id": "trigger-allows-unreleased",+ "kind": "survivor",+ "title": "the v292 guard trigger lets any claim move a deferred row",+ "expect": "survives",+ "reason": "defense in depth in the model, whose only consumer claims through claimableWriteSql; the trigger stops an older binary's claim, which the model leaves out, and the named test pins it",+ "spec": {+ "file": "RefreshBarrier.tla",+ "before": "TriggerAllows(w) == st[w] # \"deferred\" \\/ Released(w)",+ "after": "TriggerAllows(w) == TRUE"+ },+ "code": {+ "edits": [+ {+ "file": "src/core/schema-migrations/v292-refresh-write-barrier.ts",+ "before": "WHEN (OLD.state='deferred' OR NEW.state='deferred')\n EXECUTE FUNCTION gbrain_guard_deferred_request();\n ALTER TABLE persistence_requests DROP CONSTRAINT IF EXISTS persistence_requests_state_check;\n ALTER TABLE persistence_requests ADD CONSTRAINT persistence_requests_state_check\n CHECK (state IN ('queued','deferred','running','recovering','committed','conflict','failed','cancelled')) NOT VALID;",+ "after": "WHEN (false)\n EXECUTE FUNCTION gbrain_guard_deferred_request();\n ALTER TABLE persistence_requests DROP CONSTRAINT IF EXISTS persistence_requests_state_check;\n ALTER TABLE persistence_requests ADD CONSTRAINT persistence_requests_state_check\n CHECK (state IN ('queued','deferred','running','recovering','committed','conflict','failed','cancelled')) NOT VALID;"+ },+ {+ "file": "src/core/schema-migrations/v292-refresh-write-barrier.ts",+ "before": "WHEN (OLD.state='deferred' OR NEW.state='deferred')\n EXECUTE FUNCTION gbrain_guard_deferred_request();\n ALTER TABLE persistence_requests DROP CONSTRAINT IF EXISTS persistence_requests_state_check;\n ALTER TABLE persistence_requests ADD CONSTRAINT persistence_requests_state_check\n CHECK (state IN ('queued','deferred','running','recovering','committed','conflict','failed','cancelled'));",+ "after": "WHEN (false)\n EXECUTE FUNCTION gbrain_guard_deferred_request();\n ALTER TABLE persistence_requests DROP CONSTRAINT IF EXISTS persistence_requests_state_check;\n ALTER TABLE persistence_requests ADD CONSTRAINT persistence_requests_state_check\n CHECK (state IN ('queued','deferred','running','recovering','committed','conflict','failed','cancelled'));"+ }+ ],+ "tests": [+ {+ "test": "test/refresh-write-barrier.test.ts",+ "filter": "a claim without the barrier declaration cannot move a deferred row"+ }+ ]+ }+ },+ {+ "id": "fence-skips-recount",+ "kind": "survivor",+ "title": "fenceAndMerge does not re-count the drain under its locks",+ "expect": "survives",+ "reason": "defense in depth at these bounds: every admission after the refresh row exists is deferred, so no queued work can appear between the drain poll and the fence",+ "spec": {+ "file": "RefreshBarrier.tla",+ "before": "FenceRecount == TRUE",+ "after": "FenceRecount == FALSE"+ }+ },+ {+ "id": "claim-ignores-fence",+ "kind": "survivor",+ "title": "a claim ignores the refresh fence",+ "expect": "survives",+ "reason": "defense in depth at these bounds: under the fence every queued row has drained and every new admission is deferred to the unreleased refresh, so nothing is claimable",+ "spec": {+ "file": "RefreshBarrier.tla",+ "before": "FenceClear == ~RefIs({\"fenced\", \"merged\", \"recovery\"})",+ "after": "FenceClear == TRUE"+ }+ },+ {+ "id": "claim-skips-protocol",+ "kind": "survivor",+ "title": "a claim does not declare the protocol (brain row FOR SHARE)",+ "expect": "survives",+ "reason": "defense in depth at these bounds: the fence's drain count and the claim's own request-row lock already order a claim against the fence",+ "spec": {+ "file": "RefreshBarrier.tla",+ "before": "ClaimLocks == <<L(BRAIN, \"S\")>>",+ "after": "ClaimLocks == <<>>"+ }+ },+ {+ "id": "replay-skips-reauth",+ "kind": "survivor",+ "title": "a deferred write's replay does not re-authorize it",+ "expect": "survives",+ "reason": "defense in depth: publication re-authorizes every write (publication-skips-reauth)",+ "config": "RefreshBarrierEnv.cfg",+ "spec": {+ "file": "RefreshBarrier.tla",+ "before": "ReplayAuth == TRUE",+ "after": "ReplayAuth == FALSE"+ }+ },+ {+ "id": "hold-beside-held-off",+ "kind": "guard",+ "title": "a later write to a page with a write held for an import is not held beside it (#8705's first race, before 03b89121)",+ "expect": "CommitsInAdmissionOrder",+ "config": "RefreshBarrierImport.cfg",+ "spec": {+ "file": "RefreshBarrier.tla",+ "before": " ELSE IF ImportHolds /\\ HoldBehindHeld /\\ HeldAhead(w) # {} THEN \"pending\"\n",+ "after": " ELSE IF FALSE THEN \"pending\"\n"+ }+ },+ {+ "id": "import-expiry-locks-source-first",+ "kind": "guard",+ "title": "#8705's import expiry locks the source row before it declares the protocol",+ "expect": "NoDeadlock",+ "config": "RefreshBarrierImport.cfg",+ "spec": {+ "file": "RefreshBarrier.tla",+ "before": "ImportExpiryLocks == <<L(BRAIN, \"S\"), L(SRC, \"S\")>>",+ "after": "ImportExpiryLocks == <<L(SRC, \"S\"), L(BRAIN, \"S\")>>"+ }+ },+ {+ "id": "import-expiry-skips-source-lock",+ "kind": "guard",+ "title": "#8705's import expiry re-checks last_commit without locking the source row (#8705's second race, before 03b89121)",+ "expect": "ExpiryOnlyWhileWaiting",+ "config": "RefreshBarrierImport.cfg",+ "spec": {+ "file": "RefreshBarrier.tla",+ "before": "Need(EXP, IF ImportExpiryLocksSource THEN ImportExpiryLocks ELSE <<>>)",+ "after": "Need(EXP, <<>>)"+ }+ }+ ]+}diff --git a/test/spec-refresh-barrier.test.ts b/test/spec-refresh-barrier.test.tsnew file mode 100644index 0000000000..60e110a659--- /dev/null+++ b/test/spec-refresh-barrier.test.ts@@ -0,0 +1,52 @@+/**+ * spec/tla/refresh-barrier models the refresh write barrier (spec/tla/README.md). This test keeps the model's configs+ * honest about what they check: the default config is #8682 alone, every config checks a property, and the TLA++ * workflow runs when anything a mutant names changes.+ */+import { describe, expect, test } from 'bun:test';+import { readdirSync, readFileSync } from 'node:fs';+import { join } from 'node:path';++const ROOT = join(import.meta.dir, '..');+const SPEC = join(ROOT, 'spec/tla/refresh-barrier');+const CONFIGS = readdirSync(SPEC).filter(f => f.endsWith('.cfg')).sort();+const read = (cfg: string) => readFileSync(join(SPEC, cfg), 'utf8');+const SAFETY = [...read('RefreshBarrier.cfg').matchAll(/^INVARIANT (\w+)$/gm)].map(m => m[1]!);++function constants(cfg: string): Record<string, string> {+ const out: Record<string, string> = {};+ for (const line of read(cfg).split('\n')) {+ const m = line.match(/^\s+(\w+)\s*=\s*(.+?)\s*$/);+ if (m) out[m[1]!] = m[2]!;+ }+ return out;+}++describe('spec/tla/refresh-barrier configs', () => {+ test('RefreshBarrier.cfg checks #8682 alone, as coded', () => {+ const c = constants('RefreshBarrier.cfg');+ expect({ ImportHolds: c.ImportHolds, RecoveryDefers: c.RecoveryDefers, AdoptLocalEdits: c.AdoptLocalEdits })+ .toEqual({ ImportHolds: 'FALSE', RecoveryDefers: 'FALSE', AdoptLocalEdits: 'FALSE' });+ });++ for (const cfg of CONFIGS) {+ test(cfg + ': checks every safety invariant, or the liveness properties', () => {+ const text = read(cfg);+ const checked = [...text.matchAll(/^(?:INVARIANT|PROPERTY) (\w+)$/gm)].map(m => m[1]!);+ if (text.includes('SPECIFICATION FairSpec')) expect(checked).toEqual(['DeferredLeaves', 'EveryWriteEnds']);+ else expect(checked).toEqual(SAFETY);+ });+ }+});++describe('the TLA+ workflow runs when anything a refresh-barrier mutant names changes', () => {+ test('every code file and test in mutants.json is in tla-spec.yml\'s path filter', () => {+ const workflow = readFileSync(join(ROOT, '.github/workflows/tla-spec.yml'), 'utf8');+ const list = (from: string, to: string) => [...workflow.slice(workflow.indexOf(from), workflow.indexOf(to)).matchAll(/^\s+- '([^']+)'$/gm)].map(m => m[1]!);+ const { mutants } = JSON.parse(read('../refresh-barrier/mutants.json')) as { mutants: Array<{ code?: { edits: Array<{ file: string }>; tests: Array<{ test: string }> } }> };+ const named = new Set(mutants.flatMap(m => m.code ? [...m.code.edits.map(e => e.file), ...m.code.tests.map(t => t.test)] : []));+ const paths = new Set(list(' pull_request:', ' push:'));+ expect([...named].filter(file => !paths.has(file))).toEqual([]);+ expect(workflow).toContain('spec: [drain-lease, refresh-barrier]');+ });+});