Skip to content

Garry's Opinionated OpenClaw/Hermes Agent Brain

TypeScript

spec: TLA+ model of the refresh write barrier and its deferred-write queue, guard mutants tied to real tests, CI job per spec

garrytanf91fc63ba4c4
diff --git a/.github/workflows/tla-spec.yml b/.github/workflows/tla-spec.yml
index c055e9e97e..7062f89a0c 100644
--- a/.github/workflows/tla-spec.yml
+++ b/.github/workflows/tla-spec.yml
@@ -31,6 +31,20 @@ on:
- 'test/helpers/managed-brain.ts'
- 'test/helpers/test-backends.ts'
- 'test/helpers/persistence-postgres.ts'
+ - 'src/core/persistence/refresh-barrier.ts'
+ - 'src/core/persistence/journal.ts'
+ - 'src/core/persistence/worktree-refresh.ts'
+ - 'src/core/persistence/worktree-refresh-schema.ts'
+ - 'src/core/persistence/protocol.ts'
+ - 'src/core/shared-skills/restore.ts'
+ - 'src/core/schema-migrations/v292-refresh-write-barrier.ts'
+ - 'test/spec-refresh-barrier.test.ts'
+ - 'test/refresh-write-barrier.test.ts'
+ - 'src/core/persistence/coordinator.ts'
+ - 'src/core/persistence/group-publish.ts'
+ - 'src/core/persistence/page-prepare.ts'
+ - 'test/persistence-worktree-refresh.test.ts'
+ - 'test/pending-import-async-git-7436.test.ts'
push:
branches: [master]
paths: # the same list as pull_request (actionlint does not accept YAML aliases)
@@ -56,6 +70,20 @@ on:
- 'test/helpers/managed-brain.ts'
- 'test/helpers/test-backends.ts'
- 'test/helpers/persistence-postgres.ts'
+ - 'src/core/persistence/refresh-barrier.ts'
+ - 'src/core/persistence/journal.ts'
+ - 'src/core/persistence/worktree-refresh.ts'
+ - 'src/core/persistence/worktree-refresh-schema.ts'
+ - 'src/core/persistence/protocol.ts'
+ - 'src/core/shared-skills/restore.ts'
+ - 'src/core/schema-migrations/v292-refresh-write-barrier.ts'
+ - 'test/spec-refresh-barrier.test.ts'
+ - 'test/refresh-write-barrier.test.ts'
+ - 'src/core/persistence/coordinator.ts'
+ - 'src/core/persistence/group-publish.ts'
+ - 'src/core/persistence/page-prepare.ts'
+ - 'test/persistence-worktree-refresh.test.ts'
+ - 'test/pending-import-async-git-7436.test.ts'
schedule:
- cron: '40 9 * * *' # nightly, 09:40 UTC
workflow_dispatch:
@@ -69,9 +97,13 @@ concurrency:
jobs:
tla:
- name: TLC and guard mutants
+ name: TLC and guard mutants (${{ matrix.spec }})
runs-on: ubicloud-standard-4-ubuntu-2404
- timeout-minutes: 40
+ timeout-minutes: 60
+ strategy:
+ fail-fast: false
+ matrix:
+ spec: [drain-lease, refresh-barrier]
services:
postgres:
image: pgvector/pgvector:pg16
@@ -106,23 +138,23 @@ jobs:
jar="$(bash scripts/spec/tla-jar.sh)" || exit "$?"
printf 'TLA2TOOLS=%s\n' "$jar" >> "$GITHUB_ENV"
- name: The configs match the code
- run: bun test --timeout=60000 test/spec-drain-lease-constants.test.ts
+ run: bun test --timeout=60000 test/spec-${{ matrix.spec }}*.test.ts
- name: Model-check every config (the *Nightly.cfg ones on the schedule and on dispatch)
env:
TLC_NIGHTLY: ${{ (github.event_name == 'schedule' || github.event_name == 'workflow_dispatch') && '1' || '0' }}
TLC_TIMEOUT_S: '900'
- run: bash scripts/spec/tlc.sh
+ run: bash scripts/spec/tlc.sh ${{ matrix.spec }}
# Only after the model check passed: a mutant means something only against a spec whose properties hold.
- name: Every guard mutant is caught, in the model and in the code
env:
DATABASE_URL: postgres://gbrain_test:[email protected]:5432/gbrain_test
GBRAIN_TEST_ALLOW_DATABASE_URL: '1'
- run: bun scripts/spec/mutants.ts --require-postgres
+ run: bun scripts/spec/mutants.ts ${{ matrix.spec }} --require-postgres
- name: Keep the logs and counterexample traces
if: always()
uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4
with:
- name: tla-spec-logs
+ name: tla-spec-logs-${{ matrix.spec }}
path: ${{ runner.temp }}/tlc
if-no-files-found: ignore
retention-days: 14
diff --git a/docs/architecture/key-files/ci-health-and-test-guards.md b/docs/architecture/key-files/ci-health-and-test-guards.md
index 72c1785390..00c4e0a8d8 100644
--- a/docs/architecture/key-files/ci-health-and-test-guards.md
+++ b/docs/architecture/key-files/ci-health-and-test-guards.md
@@ -37,4 +37,4 @@ Current behavior and load-bearing invariants; history belongs in Git and CHANGEL
- `scripts/check-skill-refs.mjs` + `scripts/portable-skill-links.ts` + `scripts/skills-commit-gate.sh` — skills-tree integrity gate (`bun run check:skill-refs`, in verify and the skills commit gate). Fails on dangling backtick `skills/` refs, `./`/`../` links, `composes:` and RESOLVER dispatcher slugs; donor-workspace paths outside the shrink-only allowlist; any `gbrain <cmd>` in a fenced line or inline code span (command position only: line/span start, after a shell separator, `nohup`/env prefixes) that is not a real command (union of `--tools-json`, `CLI_FLAG_REGISTRY`, `src/cli.ts` dispatch, the command table and op `cliHints`), plus curated known-bad subcommands (`embed refresh`); and a relative link that resolves outside the skills root, in `skills/` and in the generated `plugin/skills` and `plugin-variants/*/skills` trees (`skills/migrations/**` exempt). W4.7: a tools-json load failure FAILS the CLI lane (`--no-cli-refs` skips it deliberately); links are read in every form (`](./x)`, bare `](x.md)` unless a placeholder, `](<x>)`, titled, `[id]: x` definitions); a `[paid-consent]` lane, migrations included, fails a paid `gbrain` command carrying `--yes`/`--max-usd`/`--max-cost[-usd]` with no ask/agree/approve/consent/confirm/permission wording on its line or the 8 before. `portable-skill-links.ts` is that rule's fix: it points `docs/protocol/AGENT_OPERATOR_v1.md` links at the bundled `skills/conventions/agent-operator-protocol.md` and other repo files at absolute blob URLs derived from `LLMS_REPO_BASE` (raw.githubusercontent base → github.com blob base; default `garrytan/gbrain/blob/master`; a non-https or credentialed value throws via `repoBaseOrThrow`, as in `scripts/llms-config.ts`), retargeting default-base links for forks; `--check` reports pending rewrites. Tests: `test/check-skill-refs.test.ts`, `test/skills-portable-links.test.ts` (scaffold `--all` into a tmp host: 0 broken local links).
- `scripts/check-file-url-pathname.sh` — `check:file-url-pathname` (in verify, #6845): no `.pathname` of a file URL (`import.meta.url).pathname`, `pathToFileURL(...).pathname`) in test/, which keeps `%20` and breaks a checkout under a path with a space; use `fileURLToPath` or `import.meta.dir`. Self-tested via `GBRAIN_GUARD_ROOT` fixtures; skips `test/fixtures/guards/`.
- `scripts/check-dynamic-ci.ts` — `check:dynamic-ci` (in verify): wiring guard for the Trunk Dynamic CI filter (`dynamic-ci` job in `test.yml` and `scale-tier.yml`; rules in [docs/ci-event-parity.md](../../ci-event-parity.md#trunk-dynamic-ci)). `checkWorkflow()` (pure, over parsed YAML) flags a filter job that can run on an event other than `pull_request`, sits in a workflow without that trigger, is not SHA-pinned, lacks `continue-on-error: true` or a step timeout below the job's, or whose `job-keys` differ from its `outputs`; a gated job whose `if` does not start `!cancelled() && needs.<filter>.outputs.<key> != 'false' &&`, lacks `needs.<other>.result == 'success'` for another need, or has a top-level `||` after the prefix; and a gated merge or release aggregate. `scripts/ci-cheap-tier.sh` accepts a gated lane that skipped only on an exact `false` verdict. Self-tested via `GBRAIN_GUARD_ROOT` fixtures in `test/fixtures/guards/check-dynamic-ci.ts/` (`good`, `bad`, `bad-every-event`, `bad-other-needs`, `bad-top-level-or`).
-- `spec/tla/` + `scripts/spec/{tlc.sh,tla-jar.sh,tla-tools.env,mutants.ts}` + `.github/workflows/tla-spec.yml` — TLA+ models checked exhaustively by TLC within small bounds; `spec/tla/drain-lease` models the managed-sync drain lease (`sync-lease.ts` over `db-lock.ts`, plus the pass stamp's fences in `sync-pass-fold.ts` and `sync-prepare.ts`). `tlc.sh` runs every config on a private copy and fails `TLC_VIOLATION`, `TLC_TIMEOUT` or `TLC_ERROR`; `tla-jar.sh` fetches the pinned tla2tools.jar and refuses a sha256 mismatch. `mutants.ts` checks `mutants.json`: each guard mutant's model edit must break its named property, and its code edits (every engine branch), applied in a scratch copy of the tree, must fail each named test on its own assertions; assumption mutants document what liveness rests on, survivors stay defense in depth. `bun run test:tla` runs both; the workflow runs them nightly and on changes to anything a mutant names (`test/spec-drain-lease-constants.test.ts` pins that path filter and the configs' timing to the code's constants). Guard tests: `test/drain-lease-guards.test.ts`, `test/drain-lease-stamp-fence.test.ts` (Postgres twins in `test/e2e/`), `test/e2e/drain-lease-fence-postgres.test.ts`. Docs: `spec/tla/README.md`.
+- `spec/tla/` + `scripts/spec/{tlc.sh,tla-jar.sh,tla-tools.env,mutants.ts}` + `.github/workflows/tla-spec.yml` — TLA+ models checked exhaustively by TLC within small bounds; `spec/tla/drain-lease` models the managed-sync drain lease (`sync-lease.ts` over `db-lock.ts`, plus the pass stamp's fences in `sync-pass-fold.ts` and `sync-prepare.ts`). `spec/tla/refresh-barrier` models the refresh write barrier and its deferred-write queue (admission, claim, replay, expiry and restore in `journal.ts`, `refresh-barrier.ts`, `worktree-refresh.ts`, `shared-skills/restore.ts` and the v292 triggers, with Postgres row locks and a deadlock invariant), plus #8705/#8708 behind constants. `tlc.sh` runs every config on a private copy and fails `TLC_VIOLATION`, `TLC_TIMEOUT` or `TLC_ERROR`; `tla-jar.sh` fetches the pinned tla2tools.jar and refuses a sha256 mismatch. `mutants.ts` checks `mutants.json`: each guard mutant's model edit must break its named property, and its code edits (every engine branch), applied in a scratch copy of the tree, must fail each named test on its own assertions; assumption mutants document what liveness rests on, survivors stay defense in depth. `bun run test:tla` runs both; the workflow runs them nightly and on changes to anything a mutant names (`test/spec-drain-lease-constants.test.ts` and `test/spec-refresh-barrier.test.ts` pin that path filter; the first also pins the drain-lease configs' timing to the code's constants). Guard tests: `test/drain-lease-guards.test.ts`, `test/drain-lease-stamp-fence.test.ts` (Postgres twins in `test/e2e/`), `test/e2e/drain-lease-fence-postgres.test.ts`. Docs: `spec/tla/README.md`.
diff --git a/package.json b/package.json
index 07132be579..ae0b10b695 100644
--- a/package.json
+++ b/package.json
@@ -102,7 +102,7 @@
"test:profile": "bash scripts/profile-tests.sh",
"test:serial": "bash scripts/run-serial-tests.sh",
"test:convergence": "GBRAIN_TEST_CONVERGENCE=1 bun --no-env-file test --timeout=3600000 test/sync-convergence.serial.test.ts",
- "test:tla": "bash scripts/spec/tlc.sh && bun scripts/spec/mutants.ts",
+ "test:tla": "bash scripts/spec/tlc.sh && bun scripts/spec/mutants.ts drain-lease && bun scripts/spec/mutants.ts refresh-barrier",
"ci:local": "bash scripts/ci-local.sh",
"ci:local:diff": "bash scripts/ci-local.sh --diff",
"ci:ubicloud": "bun run scripts/ci-ubicloud.ts",
diff --git a/scripts/spec/tlc.sh b/scripts/spec/tlc.sh
index 4eb207f276..de8cf35b07 100755
--- a/scripts/spec/tlc.sh
+++ b/scripts/spec/tlc.sh
@@ -11,9 +11,6 @@
# spec is wrong and say why in the PR
# TLC_TIMEOUT TLC did not finish within TLC_TIMEOUT_S (default 600): the state space grew; shrink the config
# TLC_ERROR TLC stopped (parse or semantic error, out of memory): the log names the line
-# TLC_FINDING_CLOSED a config whose `\* OPEN_FINDING: <Property>` line documents a known violation of the code now
-# passes: the finding is fixed; flip the config to the fixed code and drop the line
-# A config with an OPEN_FINDING line passes (`open`) only when TLC reports exactly that property violated.
# Env: TLA2TOOLS (a jar to use; otherwise scripts/spec/tla-jar.sh downloads the pinned one), TLC_OUT (logs; default
# .context/tlc), TLC_WORKERS (default: every CPU; configs run one at a time), TLC_TIMEOUT_S, TLC_JAVA_OPTS, TLC_NIGHTLY (1 also runs
# the *Nightly.cfg configs, which take too long for every pull request).
@@ -58,17 +55,11 @@ status=0
for job in "${jobs[@]}"; do
IFS='|' read -r d m c <<< "$job"
work="$out/$(basename "$d")-${c%.cfg}"; log="$work/tlc.log"; rc="$(cat "$work/rc")"; name="$(basename "$d")/$c"
- open="$(sed -nE 's/^\\\* OPEN_FINDING: ([A-Za-z0-9_]+).*/\1/p' "$d/$c" | head -1)"
- v="$(grep -oE '(Invariant|Action property|Temporal properties|Property) [A-Za-z0-9_]* ?(is|were) violated' "$log" | head -1 || true)"
- if [ -n "$open" ] && [ "$(echo "$v" | awk '{print $2}')" = "$open" ]; then
- echo "open $name: $open is violated, as the config documents (an open finding; spec/tla/README.md)"
- elif [ -n "$open" ] && grep -q "Model checking completed. No error has been found." "$log"; then
- echo "FAIL TLC_FINDING_CLOSED $name: $open now holds, so the open finding this config documents is fixed. Drop its OPEN_FINDING line, set its constants to the fixed code and move it to the README's passing configs." >&2; status=1
- elif grep -q "Model checking completed. No error has been found." "$log"; then
+ if grep -q "Model checking completed. No error has been found." "$log"; then
echo "ok $name: $(grep -oE '[0-9]+ distinct states found' "$log" | tail -1) ($(grep -oE 'Finished in [^ ]+' "$log" | head -1))"
elif [ "$rc" = 124 ] || [ "$rc" = 137 ]; then
echo "FAIL TLC_TIMEOUT $name: TLC did not finish within ${limit}s. Log: $log" >&2; status=1
- elif [ -n "$v" ]; then
+ elif v="$(grep -oE '(Invariant|Action property|Temporal properties|Property) [A-Za-z0-9_]* ?(is|were) violated' "$log" | head -1)" && [ -n "$v" ]; then
echo "FAIL TLC_VIOLATION $name: $v. The counterexample trace is in $log; spec/tla/README.md maps each action to its code." >&2; status=1
else
echo "FAIL TLC_ERROR $name: TLC exited $rc without a result. Log: $log" >&2; grep -m3 -iE 'error|exception' "$log" >&2 || tail -5 "$log" >&2; status=1
diff --git a/spec/tla/README.md b/spec/tla/README.md
index 75fc1a00a5..49b35cd53f 100644
--- a/spec/tla/README.md
+++ b/spec/tla/README.md
@@ -12,6 +12,7 @@ interleaving of the modelled actions inside their bounds. The approach follows P
bun run test:tla # every config of every spec, then every mutant (model and code)
bash scripts/spec/tlc.sh drain-lease DrainLease.cfg # one config
bun scripts/spec/mutants.ts --only renewal-ignores-token
+bun scripts/spec/mutants.ts refresh-barrier # one spec's mutants (the default is drain-lease)
bun scripts/spec/mutants.ts --spec-only # model mutants only (no Bun tests)
```
@@ -22,7 +23,7 @@ that need Postgres run when `DATABASE_URL` points at a test database (with `GBRA
otherwise they print `skip`, and CI passes `--require-postgres`. Logs and counterexample traces land in
`.context/tlc/` (`TLC_OUT`), and CI uploads them as the `tla-spec-logs` artifact.
-`.github/workflows/tla-spec.yml` runs all of it nightly, on dispatch, and on pull requests and master pushes that touch a
+`.github/workflows/tla-spec.yml` runs all of it, one job per spec, nightly, on dispatch, and on pull requests and master pushes that touch a
spec, its runner, the code a spec models, or a test a mutant names.
### When it fails
@@ -143,3 +144,113 @@ length of their transaction, so they count against the renewal timeout like a lo
None in the lease protocol at these bounds. The two gaps this model first found (#7442: fenced transactions not bounded
under the renewal timeout, and the pass stamp's share lock overlapping the holder's) are closed by the `FOR KEY SHARE`
fence and the key-writing takeover upsert (#7450); `DrainLease.cfg` now checks `NoSpuriousLapse` with both conditions on.
+
+## refresh-barrier: the refresh write barrier and its deferred-write queue
+
+With `persistence.refresh_barrier` on, a page write admitted while a managed refresh runs is journaled `deferred`
+behind that refresh and published by itself once the refresh is released (docs/designs/REFRESH_WRITE_BARRIER.md). The
+model holds one managed worktree with one page P whose canonical file is F, single-page writes to P (one blind
+put_page, one with expected_revision), one refresh at a time, another agent's plain-git commit to F, managed sync, the
+owner's expiry, and Postgres's four row-lock modes per transaction (`KS`, `S`, `NKU`, `U`) over the brain row,
+the worktree, the source row, the counters, each refresh row and each request row. A transaction holds its locks until
+it commits; a waits-for cycle among them is a deadlock (Postgres aborts one side with 40P01), so `NoDeadlock` is an
+invariant. Behind constants it also models the stacked follow-ups: writes held for an import and deferral behind
+`recovery_required` (#8705), and local-edit adoption (#8708).
+
+| Config | What it checks |
+| --- | --- |
+| `RefreshBarrier.cfg` | #8682 as coded: two writes (one blind), one refresh, one foreign commit: every safety property. |
+| `RefreshBarrierEnv.cfg` | The same with a shared-skill restore and a credential revocation that may each run once, no foreign commit. |
+| `RefreshBarrierLiveness.cfg` | `FairSpec`, one write: every deferred write leaves deferral and every admitted write ends. `RefreshBarrierLivenessNightly.cfg` checks two writes (nightly). |
+| `RefreshBarrierAdopt.cfg` | #8708 on #8705 with its fixes: an uncommitted local edit adopted by a drift refusal, no foreign commit. `RefreshBarrierAdoptNightly.cfg` adds the foreign commit (nightly). |
+| `RefreshBarrierImport.cfg` | #8705 with its fixes (03b89121): writes held for an import, a later write held beside them, the import expiry, deferral behind `recovery_required`: every property. |
+
+`test/spec-refresh-barrier.test.ts` keeps the default config on #8682 alone, every safety config on every invariant,
+and the workflow's path filter covering every file a mutant names.
+
+### Properties
+
+| Property | Holds that |
+| --- | --- |
+| `NoDeadlock` | no waits-for cycle among open transactions' row locks |
+| `AppliedAtMostOnce`, `ReceiptMatchesApply` | no write is applied twice, and a receipt says committed exactly when it was applied |
+| `EveryEndNamed` | every write that ends has a named outcome (nothing is dropped silently) |
+| `ImportedRevisionNeverHidden` | a write never replaces a committed version of F the page never had, so a blind put_page after an import never hides the imported revision from page history |
+| `FenceNeverMidPrepare` | the fence never comes up, and the checkout never moves, while a write is claimed |
+| `DeferredWaitsForRelease`, `RefreshChangeNeverOverwritten` | a deferred write commits only after its refresh is released, and never over a file that refresh changed |
+| `NoDeferToReleased`, `ReleasedWhenEnded` | admission never defers to a released refresh; a completed or aborted refresh is released |
+| `ExpiryOnlyWhileWaiting` | an expiry ends only a write that still waits (its refresh unreleased, or its import not landed) |
+| `RestoreLeavesNoOpenWrite` | a restore leaves no write admitted before it open (finding C) |
+| `NoApplyAfterRevoke` | no write commits after its credential was revoked |
+| `CommitsInAdmissionOrder` | writes to one page commit in admission order |
+| `DeferredLeaves`, `EveryWriteEnds` | liveness under `Fairness`: a deferred write leaves deferral; an admitted write ends |
+
+### Actions and code
+
+| Action | Code |
+| --- | --- |
+| `AdmBegin`, `AdmRef`, `AdmRead`, `AdmCommit` | `admitWrite` (journal.ts): the protocol declaration, the worktree `FOR SHARE`, the active refresh row `FOR NO KEY UPDATE`, the source, `lockCounters`, the INSERT (queued, or deferred with `deferred_refresh_id`) |
+| `ConsBegin`, `ConsSelect`, `ConsClaim` | `claimNextWrite`: `claimableWriteSql` (EARLIER_HEAD, EARLIER_DEFERRED, RELEASED_DEFERRED, `refreshFenceClear`) then the v292 guard trigger |
+| `ConsPrep` | preparation: `compareFileTarget` (page-prepare.ts) and `deferredReplayRefusal` (refresh-barrier.ts) |
+| `ConsPub` | `publishMutation` / `publishGroup`: `authorizeStoredRequest`, the request row, the revision and file rechecks |
+| `ConsHold` | #8705 `holdForImport` |
+| `ConsAdopt`, `ConsRelease` | #8708 `adoptLocalEdit`, then the claim's release |
+| `ConsDone` | `completeWrite` of a refusal |
+| `RefBegin` … `CompleteCommit` | worktree-refresh.ts: `fenceDraining`, the drain poll, `fenceAndMerge` (re-counts under `lockTopologyRows`), the ff-only merge under the commit lock, `bookkeeping`, `syncMembers`, `resumeOrAbandon`, and the v292 release trigger on completed or aborted |
+| `ExpRefBegin`, `ExpRefCommit` | `expireDeferredWrites`: the protocol, the refresh row re-read `released_at IS NULL FOR NO KEY UPDATE`, `completeWrite` |
+| `ExpImpBegin`, `ExpImpRecheck`, `ExpImpCommit` | #8705's import expiry in `expireDeferredWrites` |
+| `SyncBegin` … `SyncStamp` | a managed sync pass, whose rows go through the claim like writes: no write passes the queued row importing its page or a running sync row, and a sync row waits behind earlier writes and is never claimed while a write runs; the page row, then its checkpoint row's `UPDATE sources SET last_commit` (sync-prepare.ts) |
+| `Foreign`, `LocalEdit`, `Age`, `Revoke`, `Restore` | the environment: another agent's plain-git commit, an uncommitted edit, the wait limit passing, a revoked token, `quarantineSharedSkillRestore` (PGLite-only, one transaction, no worker) |
+
+### Mutants
+
+`mutants.json`. Each guard mutant has a code edit (every engine branch) and the tests that must fail under it.
+
+| Mutant | Kind | Caught by | Code and test |
+| --- | --- | --- | --- |
+| `expiry-locks-refresh-first` | guard | `NoDeadlock` | finding D reverted: the expiry locks the refresh row before the brain row; the Postgres test where an expiry overlaps a fence |
+| `restore-keeps-deferred` | guard | `RestoreLeavesNoOpenWrite` (Env) | finding C reverted in restore.ts; the restore test (PGLite) |
+| `replay-ignores-refresh-change` | guard | `RefreshChangeNeverOverwritten` | `deferredReplayRefusal`'s diff check; the deferred blind write over a refreshed page |
+| `release-only-on-completed` | guard | `ReleasedWhenEnded` | the v292 release trigger (both engines); the write deferred behind an aborted refresh |
+| `expiry-skips-release-recheck` | guard | `ExpiryOnlyWhileWaiting` | the expiry's `released_at IS NULL` under the lock; a Postgres race with a refresh ending |
+| `admission-skips-refresh-lock` | guard | `NoDeferToReleased` | admission's `FOR NO KEY UPDATE` on the refresh row; a Postgres race with a refresh ending |
+| `publication-skips-reauth` | guard | `NoApplyAfterRevoke` (Env) | `authorizeStoredRequest` in publishMutation and publishGroup; a token revoked after the claim |
+| `preparation-ignores-drift` | guard | `ImportedRevisionNeverHidden` | `compareFileTarget`; the #7436 pending-import test |
+| `drain-ignores-running` | guard | `FenceNeverMidPrepare` | `drainCounts`; test 13e in persistence-worktree-refresh.test.ts |
+| `trigger-allows-unreleased` | survivor | — | the model's only consumer claims through the predicate; the trigger stops an older binary's claim, and the code test still pins it |
+| `fence-skips-recount` | survivor | — | every admission after the refresh row exists is deferred, so nothing queues between the drain poll and the fence |
+| `claim-ignores-fence` | survivor | — | under the fence nothing is claimable at these bounds |
+| `claim-skips-protocol` | survivor | — | the drain count and the request-row lock already order a claim against the fence |
+| `replay-skips-reauth` | survivor | — | publication re-authorizes every write |
+| `hold-beside-held-off` | guard | `CommitsInAdmissionOrder` (Import) | #8705's compareFileTarget held-first read; model only until #8705 is on master |
+| `import-expiry-locks-source-first` | guard | `NoDeadlock` (Import) | #8705's import expiry; model only until #8705 is on master |
+| `import-expiry-skips-source-lock` | guard | `ExpiryOnlyWhileWaiting` (Import) | the same |
+
+#8705's and #8708's guards get code edits and tests (#8705's repro tests in test/import-write-barrier.test.ts) once their code is on master; until then they are checked in the model only.
+
+### Races found in #8705
+
+This model found two races in #8705's first version; 03b89121 fixed both, with the repro tests in
+test/import-write-barrier.test.ts. The mutants below put each one back.
+
+- **A held write landed on top of a later one** (`hold-beside-held-off`, `CommitsInAdmissionOrder`). A write held for
+ an import holds back nothing in the claim's FIFO rules, so after the sync's page row imported F (the file now matching
+ the page) and before its checkpoint stamped `last_commit`, a later write to P was claimed, passed the queued
+ checkpoint, prepared cleanly and committed; the stamp then released the held write, which landed over it, its
+ `supersedes` naming the later write's commit as the import. The fix holds the later write beside the held one
+ (compareFileTarget refuses it `pending_import` with the held write's `deferred_import_from`; constant
+ `HoldBehindHeld`). A later write claimed while the held one waits cannot be prepared after the stamp: a sync row is
+ never claimed while a foreground write runs. Blocking the later write behind the held one in the claim FIFO instead
+ stalls the sync: a pass stopped after a slice queues its checkpoint behind the blocked write.
+- **The import expiry failed a write whose import had landed** (`import-expiry-skips-source-lock`,
+ `ExpiryOnlyWhileWaiting`; Postgres only). The re-check read `last_commit` without a lock on the source row, so a
+ checkpoint that committed between it and `completeWrite` left the write `pending_import_expired`. The fix declares the
+ protocol and then reads the source row `FOR SHARE` (`ImportExpiryLocksSource`); taking the source row first deadlocks
+ with `fenceDraining`'s brain, worktree, source order (`import-expiry-locks-source-first`).
+
+### Not modelled
+
+The deferred-write cap (`persistence.refresh_barrier_max_deferred`), consumers without the capability, repo-less
+sources, `gbrain upgrade --check-downgrade`, edit_page and forget replays, and source lifecycle changes, each covered
+by test/refresh-write-barrier.test.ts; more than one worktree, page or refresh at a time; older binaries' claims
+(the v292 guard trigger, pinned by its test); effects other than page writes; side effects outside the database.
diff --git a/spec/tla/refresh-barrier/RefreshBarrier.cfg b/spec/tla/refresh-barrier/RefreshBarrier.cfg
index 24f588cb2e..d959bf5d7d 100644
--- a/spec/tla/refresh-barrier/RefreshBarrier.cfg
+++ b/spec/tla/refresh-barrier/RefreshBarrier.cfg
@@ -8,7 +8,7 @@ CONSTANTS
MaxForeign = 1
MaxLocalEdits = 0
ImportHolds = FALSE
- HeldImportBlocks = FALSE
+ HoldBehindHeld = FALSE
ImportExpiryLocksSource = FALSE
RecoveryDefers = FALSE
AdoptLocalEdits = FALSE
diff --git a/spec/tla/refresh-barrier/RefreshBarrier.tla b/spec/tla/refresh-barrier/RefreshBarrier.tla
index d51297442d..d463e7e80c 100644
--- a/spec/tla/refresh-barrier/RefreshBarrier.tla
+++ b/spec/tla/refresh-barrier/RefreshBarrier.tla
@@ -41,7 +41,7 @@ CONSTANTS
MaxForeign, \* commits to F by another agent with plain git (a pending import until sync imports them)
MaxLocalEdits, \* uncommitted edits to F in the worktree
ImportHolds, \* #8705: a claimed write refused pending_import is moved back to deferred until the import lands
- HeldImportBlocks, \* a write held for an import blocks later writes of its worktree (the claim's FIFO rule); FALSE as coded
+ HoldBehindHeld, \* #8705's fix: a write to a page with a write held for an import is held too (refused pending_import)
ImportExpiryLocksSource, \* the import expiry re-checks last_commit under a lock on the source row; FALSE as coded
RecoveryDefers, \* #8705: admission defers behind a refresh in recovery_required instead of refusing
AdoptLocalEdits, \* #8708: a drift refusal on a one-sided local edit adopts the edit and prepares the write again
@@ -124,7 +124,8 @@ VARIABLES
\* locks
held, want,
\* process locals
- pc, cw, cprep, cfin, ew, aref, aread, passHead
+ pc, cw, cprep, cfin, ew, aref, aread, passHead,
+ sseq \* the admission sequence of the sync pass's queued rows (0 when no pass runs)
reqv == <<st, code, seq, nseq, dref, dimp, applied, cpos, ncommit, expRev, overdue, adopted>>
refv == <<rstate, rel, rchg, rhead, rmoved, cur, nref>>
@@ -132,7 +133,7 @@ gitv == <<head, verAt, fileV, dbV, hist, lastCommit, local, nver, nforeign, nl
envv == <<restored, revoked, preRestore>>
ghostv == <<hidden, badExpiry, badRelease, badDefer, badRevoke>>
lockv == <<held, want>>
-procv == <<pc, cw, cprep, cfin, ew, aref, aread, passHead>>
+procv == <<pc, cw, cprep, cfin, ew, aref, aread, passHead, sseq>>
vars == <<reqv, refv, gitv, envv, ghostv, lockv, procv>>
NoLocks == [r \in Rows |-> "-"]
@@ -158,7 +159,10 @@ FenceClear == ~RefIs({"fenced", "merged", "recovery"})
\* claimableWriteSql: queued, or deferred with every hold released.
ClaimState(w) == st[w] = "queued" \/ Released(w)
\* EARLIER_HEAD (queued, running, recovering) and EARLIER_DEFERRED (released deferred): a write never passes an earlier one.
-Blocks(e) == st[e] \in {"queued", "running"} \/ Released(e) \/ (HeldImportBlocks /\ st[e] = "deferred" /\ dimp[e] # None)
+Blocks(e) == st[e] \in {"queued", "running"} \/ Released(e)
+\* With HoldBehindHeld: compareFileTarget's held-first read, the writes to the page still held for an import that has
+\* not landed.
+HeldAhead(w) == {e \in Writes : e # w /\ st[e] = "deferred" /\ dimp[e] # None /\ ~Released(e)}
\* The v292 trigger: a deferred row moves to running only when its refresh (and, since v291, its import) is released.
TriggerAllows(w) == st[w] # "deferred" \/ Released(w)
\* deferredReplayRefusal: a refresh that changed the write's file ends it revision_conflict.
@@ -194,6 +198,11 @@ AdoptOneSided == fileV = dbV
Claimable(w) == /\ ClaimState(w)
/\ FenceClear
/\ ~\E e \in Writes : seq[e] # 0 /\ seq[e] < seq[w] /\ Blocks(e)
+ /\ pc[SYN] \notin {"s_import", "s_stamp"} \* no write passes a running sync row (it is never a lane group)
+ /\ ~(pc[SYN] = "s_queued" /\ sseq < seq[w]) \* NAMES_PAGE: no write passes the queued row importing its page
+\* A sync row is claimed only with no earlier write ahead of it (EARLIER_HEAD) and no write claimed at all (a sync row never
+\* runs beside a later running foreground write).
+SyncBehind == (\E w \in Writes : seq[w] # 0 /\ seq[w] < sseq /\ Blocks(w)) \/ cw # 0
DrainZero == /\ \A w \in Writes : ~DrainCounts(w) /\ ~Released(w)
/\ pc[SYN] = "idle" \* a sync pass in flight is queued work too
@@ -252,6 +261,7 @@ Init ==
/\ aref = [w \in Writes |-> None]
/\ aread = [w \in Writes |-> "none"]
/\ passHead = 0
+ /\ sseq = 0
\* A step that only acquires the next lock a process waits for.
Acquire(p) ==
@@ -283,14 +293,14 @@ AdmBegin(w) ==
/\ Need(ADM(w), <<L(BRAIN, "S"), L(WT, "S")>>)
/\ Go(ADM(w), "a_ref")
/\ UNCHANGED <<st, code, seq, nseq, dref, dimp, applied, cpos, ncommit, overdue, adopted, refv, gitv, envv, ghostv, held,
- cw, cprep, cfin, ew, aref, aread, passHead>>
+ cw, cprep, cfin, ew, aref, aread, passHead, sseq>>
AdmRef(w) ==
/\ Ready(ADM(w), "a_ref")
/\ aref' = [aref EXCEPT ![w] = cur]
/\ Need(ADM(w), IF cur # None THEN AdmRefLock(cur) ELSE <<>>)
/\ Go(ADM(w), "a_read")
- /\ UNCHANGED <<reqv, refv, gitv, envv, ghostv, held, cw, cprep, cfin, ew, aread, passHead>>
+ /\ UNCHANGED <<reqv, refv, gitv, envv, ghostv, held, cw, cprep, cfin, ew, aread, passHead, sseq>>
\* The SELECT ... FOR NO KEY UPDATE reads the refresh row once it holds it (an EvalPlanQual recheck drops a row that
\* went terminal meanwhile).
@@ -299,7 +309,7 @@ AdmRead(w) ==
/\ aread' = [aread EXCEPT ![w] = IF aref[w] # None /\ rstate[aref[w]] \in RefActive THEN rstate[aref[w]] ELSE "none"]
/\ Need(ADM(w), <<L(SRC, "S"), L(CTR, "U")>>)
/\ Go(ADM(w), "a_commit")
- /\ UNCHANGED <<reqv, refv, gitv, envv, ghostv, held, cw, cprep, cfin, ew, aref, passHead>>
+ /\ UNCHANGED <<reqv, refv, gitv, envv, ghostv, held, cw, cprep, cfin, ew, aref, passHead, sseq>>
AdmCommit(w) ==
/\ Ready(ADM(w), "a_commit")
@@ -322,7 +332,7 @@ AdmCommit(w) ==
/\ Commit(ADM(w)) /\ Go(ADM(w), "idle") /\ Need(ADM(w), <<>>)
/\ aref' = [aref EXCEPT ![w] = None] /\ aread' = [aread EXCEPT ![w] = "none"]
/\ UNCHANGED <<code, dimp, applied, cpos, ncommit, expRev, overdue, adopted, rchg, rhead, rmoved, nref, gitv, envv,
- hidden, badExpiry, badRelease, badRevoke, cw, cprep, cfin, ew, passHead>>
+ hidden, badExpiry, badRelease, badRevoke, cw, cprep, cfin, ew, passHead, sseq>>
\* -------------------------------- consumer --------------------------------
\* claimNextWrite: the protocol declaration and the head read, SELECT ... ORDER BY sequence LIMIT 1 FOR UPDATE SKIP
@@ -330,7 +340,7 @@ AdmCommit(w) ==
ConsBegin ==
/\ pc[CONS] = "idle" /\ \E w \in Writes : Claimable(w)
/\ Need(CONS, ClaimLocks) /\ Go(CONS, "c_select")
- /\ UNCHANGED <<reqv, refv, gitv, envv, ghostv, held, cw, cprep, cfin, ew, aref, aread, passHead>>
+ /\ UNCHANGED <<reqv, refv, gitv, envv, ghostv, held, cw, cprep, cfin, ew, aref, aread, passHead, sseq>>
ConsSelect ==
/\ Ready(CONS, "c_select")
@@ -342,7 +352,7 @@ ConsSelect ==
/\ cw' = w
/\ held' = [held EXCEPT ![CONS][REQ(w)] = "U"]
/\ Go(CONS, "c_claim")
- /\ UNCHANGED <<reqv, refv, gitv, envv, ghostv, want, cprep, cfin, ew, aref, aread, passHead>>
+ /\ UNCHANGED <<reqv, refv, gitv, envv, ghostv, want, cprep, cfin, ew, aref, aread, passHead, sseq>>
ConsClaim ==
/\ Ready(CONS, "c_claim")
@@ -352,13 +362,14 @@ ConsClaim ==
/\ Commit(CONS)
/\ cw' = IF TriggerAllows(cw) THEN cw ELSE 0
/\ UNCHANGED <<code, seq, nseq, dref, dimp, applied, cpos, ncommit, expRev, overdue, adopted, refv, gitv, envv, ghostv,
- want, cprep, cfin, ew, aref, aread, passHead>>
+ want, cprep, cfin, ew, aref, aread, passHead, sseq>>
\* Preparation reads the page and the file (no database lock), then the deferred replay check.
Outcome(w) ==
LET drift == Worktree # dbV
pending == drift /\ local = None /\ fileV # verAt[lastCommit]
IN IF drift /\ PrepRefusesDrift THEN (IF pending THEN "pending" ELSE IF local # None THEN "local" ELSE "drift")
+ ELSE IF ImportHolds /\ HoldBehindHeld /\ HeldAhead(w) # {} THEN "pending"
ELSE IF w \notin Blind /\ dbV # expRev[w] THEN "conflict"
ELSE IF dref[w] # None /\ ReplayDiff(dref[w]) THEN "conflict"
ELSE IF dref[w] # None /\ revoked /\ ReplayAuth THEN "denied"
@@ -371,7 +382,8 @@ Finish(s, c) == /\ cfin' = <<s, c>>
ConsPrep ==
/\ Ready(CONS, "c_prep")
/\ LET o == Outcome(cw)
- IN /\ cprep' = [o |-> o, db |-> dbV, wt |-> Worktree, lc |-> lastCommit]
+ lc == IF ImportHolds /\ HoldBehindHeld /\ HeldAhead(cw) # {} THEN dimp[CHOOSE e \in HeldAhead(cw) : TRUE] ELSE lastCommit
+ IN /\ cprep' = [o |-> o, db |-> dbV, wt |-> Worktree, lc |-> lc]
/\ CASE o = "ok" ->
/\ Need(CONS, <<L(BRAIN, "S"), L(WT, "S"), L(SRC, "S"), L(CTR, "U"), L(REQ(cw), "U")>>)
/\ Go(CONS, "c_pub") /\ UNCHANGED cfin
@@ -383,7 +395,7 @@ ConsPrep ==
[] o \in {"local", "drift"} -> Finish("failed", "source_changed")
[] o = "conflict" -> Finish("conflict", "revision_conflict")
[] o = "denied" -> Finish("failed", "permission_denied")
- /\ UNCHANGED <<reqv, refv, gitv, envv, ghostv, held, cw, ew, aref, aread, passHead>>
+ /\ UNCHANGED <<reqv, refv, gitv, envv, ghostv, held, cw, ew, aref, aread, passHead, sseq>>
\* publishMutation's transaction: declarePublication, guardOwnership, authorizeStoredRequest (the source FOR SHARE),
\* lockCounters, the request FOR UPDATE (claim and state rechecked), the page revision and file rechecked.
@@ -406,7 +418,7 @@ ConsPub ==
/\ UNCHANGED <<head, verAt, fileV, nver, dbV, hist, local, applied, ncommit, cpos, hidden, badRelease, badRevoke>>
/\ Commit(CONS) /\ ConsIdle
/\ UNCHANGED <<seq, nseq, dref, dimp, expRev, overdue, adopted, refv, lastCommit, nforeign, nlocal, envv, badExpiry, badDefer,
- want, ew, aref, aread, passHead>>
+ want, ew, aref, aread, passHead, sseq>>
\* holdForImport: UPDATE ... SET state='deferred', deferred_import_from=<last_commit read before the check>
\* WHERE execution_token=<ours> AND state='running' AND publication_started=false.
@@ -417,7 +429,7 @@ ConsHold ==
ELSE UNCHANGED <<st, dimp>>
/\ Commit(CONS) /\ ConsIdle
/\ UNCHANGED <<code, seq, nseq, dref, applied, cpos, ncommit, expRev, overdue, adopted, refv, gitv, envv, ghostv,
- want, ew, aref, aread, passHead>>
+ want, ew, aref, aread, passHead, sseq>>
\* adoptLocalEdit: the owner publishes the edit as its own managed_file_repair write (one commit of the file), then
\* releases the caller's claim so it is prepared again.
@@ -428,7 +440,7 @@ ConsAdopt ==
/\ adopted' = [adopted EXCEPT ![cw] = TRUE]
/\ Need(CONS, <<L(REQ(cw), "U")>>) /\ Go(CONS, "c_release")
/\ UNCHANGED <<st, code, seq, nseq, dref, dimp, applied, cpos, ncommit, expRev, overdue, refv, lastCommit, nver, nforeign,
- nlocal, envv, badExpiry, badRelease, badDefer, badRevoke, held, cw, cprep, cfin, ew, aref, aread, passHead>>
+ nlocal, envv, badExpiry, badRelease, badDefer, badRevoke, held, cw, cprep, cfin, ew, aref, aread, passHead, sseq>>
\* releaseUnpublishedClaim: back to queued, prepared again by the next claim.
ConsRelease ==
@@ -436,7 +448,7 @@ ConsRelease ==
/\ st' = [st EXCEPT ![cw] = IF @ = "running" THEN "queued" ELSE @]
/\ Commit(CONS) /\ ConsIdle
/\ UNCHANGED <<code, seq, nseq, dref, dimp, applied, cpos, ncommit, expRev, overdue, adopted, refv, gitv, envv, ghostv,
- want, ew, aref, aread, passHead>>
+ want, ew, aref, aread, passHead, sseq>>
\* completeWrite of a refusal: declarePersistenceProtocol, lockCounters, the request FOR UPDATE.
ConsDone ==
@@ -445,14 +457,14 @@ ConsDone ==
/\ code' = [code EXCEPT ![cw] = IF st[cw] = "running" THEN cfin[2] ELSE @]
/\ Commit(CONS) /\ ConsIdle
/\ UNCHANGED <<seq, nseq, dref, dimp, applied, cpos, ncommit, expRev, overdue, adopted, refv, gitv, envv, ghostv,
- want, ew, aref, aread, passHead>>
+ want, ew, aref, aread, passHead, sseq>>
\* -------------------------------- refresher --------------------------------
\* fenceDraining: lockTopologyRows (brain FOR UPDATE, worktree FOR UPDATE, sources FOR UPDATE), INSERT draining.
RefBegin ==
/\ pc[RFR] = "idle" /\ cur = None /\ nref < MaxRefreshes
/\ Need(RFR, <<L(BRAIN, "U"), L(WT, "U"), L(SRC, "U")>>) /\ Go(RFR, "r_fd")
- /\ UNCHANGED <<reqv, refv, gitv, envv, ghostv, held, cw, cprep, cfin, ew, aref, aread, passHead>>
+ /\ UNCHANGED <<reqv, refv, gitv, envv, ghostv, held, cw, cprep, cfin, ew, aref, aread, passHead, sseq>>
FdCommit ==
/\ Ready(RFR, "r_fd")
@@ -463,19 +475,19 @@ FdCommit ==
/\ rhead' = [rhead EXCEPT ![nref + 1] = head]
/\ cur' = nref + 1 /\ nref' = nref + 1
/\ Commit(RFR) /\ Go(RFR, "r_drain")
- /\ UNCHANGED <<reqv, rel, rmoved, gitv, envv, ghostv, want, cw, cprep, cfin, ew, aref, aread, passHead>>
+ /\ UNCHANGED <<reqv, rel, rmoved, gitv, envv, ghostv, want, cw, cprep, cfin, ew, aref, aread, passHead, sseq>>
\* The drain poll (no locks), then fenceAndMerge's transaction: lockTopologyRows, drainCounts, transition to fenced.
RefDrained ==
/\ pc[RFR] = "r_drain" /\ want[RFR] = <<>> /\ DrainZero
/\ Need(RFR, <<L(BRAIN, "U"), L(WT, "U"), L(SRC, "U"), L(REF(nref), "NKU")>>) /\ Go(RFR, "r_fm")
- /\ UNCHANGED <<reqv, refv, gitv, envv, ghostv, held, cw, cprep, cfin, ew, aref, aread, passHead>>
+ /\ UNCHANGED <<reqv, refv, gitv, envv, ghostv, held, cw, cprep, cfin, ew, aref, aread, passHead, sseq>>
\* --wait-drain ran out: transition draining -> aborted.
RefDrainTimeout ==
/\ pc[RFR] = "r_drain" /\ ~DrainZero
/\ Need(RFR, <<L(REF(nref), "NKU")>>) /\ Go(RFR, "r_abort")
- /\ UNCHANGED <<reqv, refv, gitv, envv, ghostv, held, cw, cprep, cfin, ew, aref, aread, passHead>>
+ /\ UNCHANGED <<reqv, refv, gitv, envv, ghostv, held, cw, cprep, cfin, ew, aref, aread, passHead, sseq>>
FmCommit ==
/\ Ready(RFR, "r_fm")
@@ -483,7 +495,7 @@ FmCommit ==
THEN /\ rstate' = [rstate EXCEPT ![nref] = "fenced"] /\ Go(RFR, "r_merge") /\ Need(RFR, <<>>)
ELSE /\ UNCHANGED rstate /\ Go(RFR, "r_abort") /\ Need(RFR, <<L(REF(nref), "NKU")>>) \* refresh_source_changed
/\ Commit(RFR)
- /\ UNCHANGED <<reqv, rel, rchg, rhead, rmoved, cur, nref, gitv, envv, ghostv, cw, cprep, cfin, ew, aref, aread, passHead>>
+ /\ UNCHANGED <<reqv, rel, rchg, rhead, rmoved, cur, nref, gitv, envv, ghostv, cw, cprep, cfin, ew, aref, aread, passHead, sseq>>
\* transition(..., 'aborted'): the release trigger stamps released_at.
AbortCommit ==
@@ -493,7 +505,7 @@ AbortCommit ==
ELSE UNCHANGED <<rstate, rel>>
/\ cur' = None
/\ Commit(RFR) /\ Go(RFR, "idle")
- /\ UNCHANGED <<reqv, rchg, rhead, rmoved, nref, gitv, envv, ghostv, want, cw, cprep, cfin, ew, aref, aread, passHead>>
+ /\ UNCHANGED <<reqv, rchg, rhead, rmoved, nref, gitv, envv, ghostv, want, cw, cprep, cfin, ew, aref, aread, passHead, sseq>>
\* Under the commit lock: HEAD must still be old_head (another agent's plain-git commit is not), then the ff-only merge.
RefMerge ==
@@ -505,13 +517,13 @@ RefMerge ==
ELSE NewCommit(fileV) /\ UNCHANGED <<nver, rmoved>>
/\ Need(RFR, <<L(REF(nref), "NKU"), L(SRC, "NKU")>>) /\ Go(RFR, "r_bk")
/\ UNCHANGED <<reqv, rstate, rel, rchg, rhead, cur, nref, dbV, hist, lastCommit, local, nforeign, nlocal, envv, ghostv,
- held, cw, cprep, cfin, ew, aref, aread, passHead>>
+ held, cw, cprep, cfin, ew, aref, aread, passHead, sseq>>
MarkCommit ==
/\ Ready(RFR, "r_mark")
/\ rstate' = [rstate EXCEPT ![nref] = "recovery"]
/\ Commit(RFR) /\ Go(RFR, "r_recovery")
- /\ UNCHANGED <<reqv, rel, rchg, rhead, rmoved, cur, nref, gitv, envv, ghostv, want, cw, cprep, cfin, ew, aref, aread, passHead>>
+ /\ UNCHANGED <<reqv, rel, rchg, rhead, rmoved, cur, nref, gitv, envv, ghostv, want, cw, cprep, cfin, ew, aref, aread, passHead, sseq>>
\* A person resolves recovery_required: reset to old_head and --abandon, or reset to the target and --resume.
PersonAbandon ==
@@ -519,7 +531,7 @@ PersonAbandon ==
/\ head' = rhead[nref] /\ fileV' = verAt[rhead[nref]] /\ local' = None
/\ Need(RFR, <<L(REF(nref), "NKU")>>) /\ Go(RFR, "r_abort")
/\ UNCHANGED <<reqv, refv, verAt, dbV, hist, lastCommit, nver, nforeign, nlocal, envv, ghostv, held,
- cw, cprep, cfin, ew, aref, aread, passHead>>
+ cw, cprep, cfin, ew, aref, aread, passHead, sseq>>
PersonResume ==
/\ pc[RFR] = "r_recovery"
@@ -531,20 +543,20 @@ PersonResume ==
/\ local' = None
/\ Need(RFR, <<L(REF(nref), "NKU"), L(SRC, "NKU")>>) /\ Go(RFR, "r_bk")
/\ UNCHANGED <<reqv, rstate, rel, rchg, rhead, cur, nref, dbV, hist, lastCommit, nforeign, nlocal, envv, ghostv, held,
- cw, cprep, cfin, ew, aref, aread, passHead>>
+ cw, cprep, cfin, ew, aref, aread, passHead, sseq>>
\* bookkeeping: transition to merged (the refresh row), the sources' upstream observation, then syncing.
BkCommit ==
/\ Ready(RFR, "r_bk")
/\ rstate' = [rstate EXCEPT ![nref] = IF @ \in {"fenced", "merged", "recovery"} THEN "syncing" ELSE @]
/\ Commit(RFR) /\ Go(RFR, "r_syncing")
- /\ UNCHANGED <<reqv, rel, rchg, rhead, rmoved, cur, nref, gitv, envv, ghostv, want, cw, cprep, cfin, ew, aref, aread, passHead>>
+ /\ UNCHANGED <<reqv, rel, rchg, rhead, rmoved, cur, nref, gitv, envv, ghostv, want, cw, cprep, cfin, ew, aref, aread, passHead, sseq>>
\* syncMembers ends: completed once members reach the target, or (#8072) when a member's sync is blocked or deferred.
RefComplete ==
/\ pc[RFR] = "r_syncing"
/\ Need(RFR, <<L(REF(nref), "NKU")>>) /\ Go(RFR, "r_complete")
- /\ UNCHANGED <<reqv, refv, gitv, envv, ghostv, held, cw, cprep, cfin, ew, aref, aread, passHead>>
+ /\ UNCHANGED <<reqv, refv, gitv, envv, ghostv, held, cw, cprep, cfin, ew, aref, aread, passHead, sseq>>
CompleteCommit ==
/\ Ready(RFR, "r_complete")
@@ -553,14 +565,14 @@ CompleteCommit ==
/\ cur' = None
ELSE UNCHANGED <<rstate, rel, cur>>
/\ Commit(RFR) /\ Go(RFR, "idle")
- /\ UNCHANGED <<reqv, rchg, rhead, rmoved, nref, gitv, envv, ghostv, want, cw, cprep, cfin, ew, aref, aread, passHead>>
+ /\ UNCHANGED <<reqv, rchg, rhead, rmoved, nref, gitv, envv, ghostv, want, cw, cprep, cfin, ew, aref, aread, passHead, sseq>>
\* --------------------------------- expiry ---------------------------------
\* The candidate query reads without locks; then one transaction per row.
ExpRefBegin(w) ==
/\ pc[EXP] = "idle" /\ st[w] = "deferred" /\ dref[w] # None /\ ~rel[dref[w]] /\ overdue[w]
/\ ew' = w /\ Need(EXP, ExpiryLocks(dref[w], w)) /\ Go(EXP, "e_ref")
- /\ UNCHANGED <<reqv, refv, gitv, envv, ghostv, held, cw, cprep, cfin, aref, aread, passHead>>
+ /\ UNCHANGED <<reqv, refv, gitv, envv, ghostv, held, cw, cprep, cfin, aref, aread, passHead, sseq>>
\* completeWrite: a terminal row stays as it is; a row someone claimed (a live execution token) is write_claim_lost.
ExpRefCommit ==
@@ -571,7 +583,7 @@ ExpRefCommit ==
/\ badExpiry' = (badExpiry \/ (fire /\ ~(st[ew] = "deferred" /\ ~rel[dref[ew]])))
/\ Commit(EXP) /\ Go(EXP, "idle") /\ ew' = 0
/\ UNCHANGED <<seq, nseq, dref, dimp, applied, cpos, ncommit, expRev, overdue, adopted, refv, gitv, envv,
- hidden, badRelease, badDefer, badRevoke, want, cw, cprep, cfin, aref, aread, passHead>>
+ hidden, badRelease, badDefer, badRevoke, want, cw, cprep, cfin, aref, aread, passHead, sseq>>
\* #8705: a write held for an import that has not landed past the limit ends pending_import_expired.
ExpImpBegin(w) ==
@@ -581,7 +593,7 @@ ExpImpBegin(w) ==
/\ overdue[w]
/\ ew' = w /\ Go(EXP, "e_recheck")
/\ Need(EXP, IF ImportExpiryLocksSource THEN ImportExpiryLocks ELSE <<>>)
- /\ UNCHANGED <<reqv, refv, gitv, envv, ghostv, held, cw, cprep, cfin, aref, aread, passHead>>
+ /\ UNCHANGED <<reqv, refv, gitv, envv, ghostv, held, cw, cprep, cfin, aref, aread, passHead, sseq>>
ExpImpRecheck ==
/\ Ready(EXP, "e_recheck")
@@ -590,7 +602,7 @@ ExpImpRecheck ==
ELSE UNCHANGED want /\ Go(EXP, "idle")
/\ held' = IF ImportExpiryRecheck(ew) THEN held ELSE [held EXCEPT ![EXP] = NoLocks]
/\ ew' = IF ImportExpiryRecheck(ew) THEN ew ELSE 0
- /\ UNCHANGED <<reqv, refv, gitv, envv, ghostv, cw, cprep, cfin, aref, aread, passHead>>
+ /\ UNCHANGED <<reqv, refv, gitv, envv, ghostv, cw, cprep, cfin, aref, aread, passHead, sseq>>
ExpImpCommit ==
/\ Ready(EXP, "e_imp")
@@ -600,7 +612,7 @@ ExpImpCommit ==
/\ badExpiry' = (badExpiry \/ (fire /\ ~(st[ew] = "deferred" /\ dimp[ew] # None /\ lastCommit = dimp[ew])))
/\ Commit(EXP) /\ Go(EXP, "idle") /\ ew' = 0
/\ UNCHANGED <<seq, nseq, dref, dimp, applied, cpos, ncommit, expRev, overdue, adopted, refv, gitv, envv,
- hidden, badRelease, badDefer, badRevoke, want, cw, cprep, cfin, aref, aread, passHead>>
+ hidden, badRelease, badDefer, badRevoke, want, cw, cprep, cfin, aref, aread, passHead, sseq>>
\* ------------------------------ managed sync ------------------------------
\* A pass starts only with no refresh, or as the refresh's own member sync (assertManagedSyncAllowed). It imports F's
@@ -608,23 +620,36 @@ ExpImpCommit ==
SyncBegin ==
/\ pc[SYN] = "idle" /\ (cur = None \/ RefIs({"syncing"})) /\ lastCommit # head
/\ passHead' = head
+ /\ sseq' = nseq + 1 /\ nseq' = nseq + 1
+ /\ Go(SYN, "s_queued")
+ /\ UNCHANGED <<st, code, seq, dref, dimp, applied, cpos, ncommit, expRev, overdue, adopted, refv, gitv, envv, ghostv,
+ lockv, cw, cprep, cfin, ew, aref, aread>>
+
+\* The page row is claimed once no earlier write is ahead of it (EARLIER_HEAD), then imported.
+SyncImportBegin ==
+ /\ pc[SYN] = "s_queued" /\ ~SyncBehind
/\ Need(SYN, <<L(BRAIN, "S")>>) /\ Go(SYN, "s_import")
- /\ UNCHANGED <<reqv, refv, gitv, envv, ghostv, held, cw, cprep, cfin, ew, aref, aread>>
+ /\ UNCHANGED <<reqv, refv, gitv, envv, ghostv, held, cw, cprep, cfin, ew, aref, aread, passHead, sseq>>
SyncImport ==
/\ Ready(SYN, "s_import")
/\ IF local = None /\ fileV = verAt[passHead] /\ fileV # dbV
THEN dbV' = fileV /\ hist' = hist \cup {fileV}
ELSE UNCHANGED <<dbV, hist>> \* nothing to import, or a dirty file sync holds
- /\ Commit(SYN)
- /\ Need(SYN, <<L(BRAIN, "S"), L(SRC, "NKU")>>) /\ Go(SYN, "s_stamp")
+ /\ Commit(SYN) /\ Go(SYN, "s_ckpt")
/\ UNCHANGED <<reqv, refv, head, verAt, fileV, lastCommit, local, nver, nforeign, nlocal, envv, ghostv,
- cw, cprep, cfin, ew, aref, aread, passHead>>
+ want, cw, cprep, cfin, ew, aref, aread, passHead, sseq>>
+
+\* The checkpoint row is claimed once no earlier write is ahead of it; the pass stamp moves last_commit (UPDATE sources).
+SyncStampBegin ==
+ /\ pc[SYN] = "s_ckpt" /\ ~SyncBehind
+ /\ Need(SYN, <<L(BRAIN, "S"), L(SRC, "NKU")>>) /\ Go(SYN, "s_stamp")
+ /\ UNCHANGED <<reqv, refv, gitv, envv, ghostv, held, cw, cprep, cfin, ew, aref, aread, passHead, sseq>>
SyncStamp ==
/\ Ready(SYN, "s_stamp")
/\ lastCommit' = passHead
- /\ Commit(SYN) /\ Go(SYN, "idle") /\ passHead' = 0
+ /\ Commit(SYN) /\ Go(SYN, "idle") /\ passHead' = 0 /\ sseq' = 0
/\ UNCHANGED <<reqv, refv, head, verAt, fileV, dbV, hist, local, nver, nforeign, nlocal, envv, ghostv,
want, cw, cprep, cfin, ew, aref, aread>>
@@ -666,7 +691,7 @@ ConsStep == ConsBegin \/ ConsSelect \/ ConsClaim \/ ConsPrep \/ ConsPub \/ ConsH
RefStep == FdCommit \/ RefDrained \/ RefDrainTimeout \/ FmCommit \/ AbortCommit \/ RefMerge \/ MarkCommit
\/ BkCommit \/ RefComplete \/ CompleteCommit
ExpStep == (\E w \in Writes : ExpRefBegin(w) \/ ExpImpBegin(w)) \/ ExpRefCommit \/ ExpImpRecheck \/ ExpImpCommit
-SyncStep == SyncBegin \/ SyncImport \/ SyncStamp
+SyncStep == SyncBegin \/ SyncImportBegin \/ SyncImport \/ SyncStampBegin \/ SyncStamp
AdmStep(w) == AdmRef(w) \/ AdmRead(w) \/ AdmCommit(w)
Next ==
diff --git a/spec/tla/refresh-barrier/RefreshBarrierAdopt.cfg b/spec/tla/refresh-barrier/RefreshBarrierAdopt.cfg
index 6c96b708ac..ce7c5cd1df 100644
--- a/spec/tla/refresh-barrier/RefreshBarrierAdopt.cfg
+++ b/spec/tla/refresh-barrier/RefreshBarrierAdopt.cfg
@@ -1,5 +1,5 @@
-\* #8708 as coded on #8705: one uncommitted local edit adopted by a drift refusal, no foreign commit. The two #8705
-\* open findings' invariants are left to RefreshBarrierImport*.cfg. RefreshBarrierAdoptNightly.cfg adds a foreign commit.
+\* #8708 on #8705 with its fixes: one uncommitted local edit adopted by a drift refusal, no foreign commit.
+\* RefreshBarrierAdoptNightly.cfg adds a foreign commit.
SPECIFICATION Spec
CHECK_DEADLOCK FALSE
CONSTANTS
@@ -9,8 +9,8 @@ CONSTANTS
MaxForeign = 0
MaxLocalEdits = 1
ImportHolds = TRUE
- HeldImportBlocks = FALSE
- ImportExpiryLocksSource = FALSE
+ HoldBehindHeld = TRUE
+ ImportExpiryLocksSource = TRUE
RecoveryDefers = TRUE
AdoptLocalEdits = TRUE
MayRestore = FALSE
@@ -26,5 +26,7 @@ INVARIANT DeferredWaitsForRelease
INVARIANT RefreshChangeNeverOverwritten
INVARIANT NoDeferToReleased
INVARIANT ReleasedWhenEnded
+INVARIANT ExpiryOnlyWhileWaiting
INVARIANT RestoreLeavesNoOpenWrite
INVARIANT NoApplyAfterRevoke
+INVARIANT CommitsInAdmissionOrder
diff --git a/spec/tla/refresh-barrier/RefreshBarrierImportOrder.cfg b/spec/tla/refresh-barrier/RefreshBarrierAdoptNightly.cfg
similarity index 50%
rename from spec/tla/refresh-barrier/RefreshBarrierImportOrder.cfg
rename to spec/tla/refresh-barrier/RefreshBarrierAdoptNightly.cfg
index 795bc9f068..74d72ac1cd 100644
--- a/spec/tla/refresh-barrier/RefreshBarrierImportOrder.cfg
+++ b/spec/tla/refresh-barrier/RefreshBarrierAdoptNightly.cfg
@@ -1,8 +1,4 @@
-\* #8705 as coded (writes held for an import, deferral behind recovery_required). Open finding: a write held for an
-\* import holds back no later write, so a later write to the same page commits between the sync's page import and its
-\* last_commit stamp, and the held write then lands on top of it. Flip HeldImportBlocks to TRUE once the claim's FIFO
-\* rule counts held rows. ExpiryOnlyWhileWaiting is RefreshBarrierImportExpiry.cfg's finding.
-\* OPEN_FINDING: CommitsInAdmissionOrder
+\* Nightly only: RefreshBarrierAdopt.cfg with another agent's commit to the file.
SPECIFICATION Spec
CHECK_DEADLOCK FALSE
CONSTANTS
@@ -10,12 +6,12 @@ CONSTANTS
Blind = {1}
MaxRefreshes = 1
MaxForeign = 1
- MaxLocalEdits = 0
+ MaxLocalEdits = 1
ImportHolds = TRUE
- HeldImportBlocks = FALSE
- ImportExpiryLocksSource = FALSE
+ HoldBehindHeld = TRUE
+ ImportExpiryLocksSource = TRUE
RecoveryDefers = TRUE
- AdoptLocalEdits = FALSE
+ AdoptLocalEdits = TRUE
MayRestore = FALSE
MayRevoke = FALSE
INVARIANT TypeOK
@@ -29,6 +25,7 @@ INVARIANT DeferredWaitsForRelease
INVARIANT RefreshChangeNeverOverwritten
INVARIANT NoDeferToReleased
INVARIANT ReleasedWhenEnded
+INVARIANT ExpiryOnlyWhileWaiting
INVARIANT RestoreLeavesNoOpenWrite
INVARIANT NoApplyAfterRevoke
INVARIANT CommitsInAdmissionOrder
diff --git a/spec/tla/refresh-barrier/RefreshBarrierEnv.cfg b/spec/tla/refresh-barrier/RefreshBarrierEnv.cfg
index 31127209c8..7de5fe94d5 100644
--- a/spec/tla/refresh-barrier/RefreshBarrierEnv.cfg
+++ b/spec/tla/refresh-barrier/RefreshBarrierEnv.cfg
@@ -8,7 +8,7 @@ CONSTANTS
MaxForeign = 0
MaxLocalEdits = 0
ImportHolds = FALSE
- HeldImportBlocks = FALSE
+ HoldBehindHeld = FALSE
ImportExpiryLocksSource = FALSE
RecoveryDefers = FALSE
AdoptLocalEdits = FALSE
diff --git a/spec/tla/refresh-barrier/RefreshBarrierImportExpiry.cfg b/spec/tla/refresh-barrier/RefreshBarrierImport.cfg
similarity index 55%
rename from spec/tla/refresh-barrier/RefreshBarrierImportExpiry.cfg
rename to spec/tla/refresh-barrier/RefreshBarrierImport.cfg
index edb7514ed0..5277ebc2c0 100644
--- a/spec/tla/refresh-barrier/RefreshBarrierImportExpiry.cfg
+++ b/spec/tla/refresh-barrier/RefreshBarrierImport.cfg
@@ -1,8 +1,6 @@
-\* #8705 as coded. Open finding: the import expiry re-checks last_commit without a lock on the source row, so a sync
-\* stamp that commits between the re-check and completeWrite leaves a write failed pending_import_expired after its import
-\* landed. Flip ImportExpiryLocksSource to TRUE once the re-check declares the protocol and locks the source row FOR SHARE.
-\* CommitsInAdmissionOrder is RefreshBarrierImportOrder.cfg's finding.
-\* OPEN_FINDING: ExpiryOnlyWhileWaiting
+\* #8705 with its fixes (03b89121): writes held for an import, a later write to the page held beside a held one, the
+\* import expiry re-checking under the source row lock, deferral behind recovery_required. Mutants hold-beside-held-off and
+\* import-expiry-skips-source-lock put back the two races this model found before the fixes.
SPECIFICATION Spec
CHECK_DEADLOCK FALSE
CONSTANTS
@@ -12,8 +10,8 @@ CONSTANTS
MaxForeign = 1
MaxLocalEdits = 0
ImportHolds = TRUE
- HeldImportBlocks = FALSE
- ImportExpiryLocksSource = FALSE
+ HoldBehindHeld = TRUE
+ ImportExpiryLocksSource = TRUE
RecoveryDefers = TRUE
AdoptLocalEdits = FALSE
MayRestore = FALSE
@@ -32,3 +30,4 @@ INVARIANT ReleasedWhenEnded
INVARIANT ExpiryOnlyWhileWaiting
INVARIANT RestoreLeavesNoOpenWrite
INVARIANT NoApplyAfterRevoke
+INVARIANT CommitsInAdmissionOrder
diff --git a/spec/tla/refresh-barrier/RefreshBarrierLiveness.cfg b/spec/tla/refresh-barrier/RefreshBarrierLiveness.cfg
index 768d2ca99b..178ced9844 100644
--- a/spec/tla/refresh-barrier/RefreshBarrierLiveness.cfg
+++ b/spec/tla/refresh-barrier/RefreshBarrierLiveness.cfg
@@ -1,14 +1,15 @@
-\* Liveness under fairness: every deferred write leaves deferral, and every admitted write ends.
+\* Liveness under fairness, one write: every deferred write leaves deferral and every admitted write ends.
+\* RefreshBarrierLivenessNightly.cfg checks two writes.
SPECIFICATION FairSpec
CHECK_DEADLOCK FALSE
CONSTANTS
- NW = 2
+ NW = 1
Blind = {1}
MaxRefreshes = 1
MaxForeign = 0
MaxLocalEdits = 0
ImportHolds = FALSE
- HeldImportBlocks = FALSE
+ HoldBehindHeld = FALSE
ImportExpiryLocksSource = FALSE
RecoveryDefers = FALSE
AdoptLocalEdits = FALSE
diff --git a/spec/tla/refresh-barrier/RefreshBarrierLivenessNightly.cfg b/spec/tla/refresh-barrier/RefreshBarrierLivenessNightly.cfg
new file mode 100644
index 0000000000..64acca86bc
--- /dev/null
+++ b/spec/tla/refresh-barrier/RefreshBarrierLivenessNightly.cfg
@@ -0,0 +1,18 @@
+\* Nightly only: RefreshBarrierLiveness.cfg with two writes (one blind).
+SPECIFICATION FairSpec
+CHECK_DEADLOCK FALSE
+CONSTANTS
+ NW = 2
+ Blind = {1}
+ MaxRefreshes = 1
+ MaxForeign = 0
+ MaxLocalEdits = 0
+ ImportHolds = FALSE
+ HoldBehindHeld = FALSE
+ ImportExpiryLocksSource = FALSE
+ RecoveryDefers = FALSE
+ AdoptLocalEdits = FALSE
+ MayRestore = FALSE
+ MayRevoke = FALSE
+PROPERTY DeferredLeaves
+PROPERTY EveryWriteEnds
diff --git a/spec/tla/refresh-barrier/mutants.json b/spec/tla/refresh-barrier/mutants.json
new file mode 100644
index 0000000000..7195af7336
--- /dev/null
+++ b/spec/tla/refresh-barrier/mutants.json
@@ -0,0 +1,373 @@
+{
+ "$comment": "Each mutant removes one guard of the refresh write barrier (kind guard) or a guard that is defense in depth at these bounds (kind survivor, expect survives). spec edits a copy of the model and TLC must report expect; code makes the same edit to the implementation, and each named test must fail under it and pass without it. Run: bun scripts/spec/mutants.ts refresh-barrier [--spec-only|--code-only] [--only <id>]. See spec/tla/README.md.",
+ "module": "RefreshBarrier",
+ "config": "RefreshBarrier.cfg",
+ "mutants": [
+ {
+ "id": "expiry-locks-refresh-first",
+ "kind": "guard",
+ "title": "the refresh-barrier expiry locks the refresh row before it declares the protocol (finding D reverted)",
+ "expect": "NoDeadlock",
+ "spec": {
+ "file": "RefreshBarrier.tla",
+ "before": "ExpiryLocks(i, w) == <<L(BRAIN, \"S\"), L(REF(i), \"NKU\"), L(CTR, \"U\"), L(REQ(w), \"U\")>>",
+ "after": "ExpiryLocks(i, w) == <<L(REF(i), \"NKU\"), L(BRAIN, \"S\"), L(CTR, \"U\"), L(REQ(w), \"U\")>>"
+ },
+ "code": {
+ "edits": [
+ {
+ "file": "src/core/persistence/refresh-barrier.ts",
+ "before": " await declarePersistenceProtocol(tx);\n const [held] = await tx.executeRaw('SELECT 1 FROM persistence_worktree_refreshes WHERE id=$1::uuid AND released_at IS NULL FOR NO KEY UPDATE', [row.deferred_refresh_id]);\n",
+ "after": " const [held] = await tx.executeRaw('SELECT 1 FROM persistence_worktree_refreshes WHERE id=$1::uuid AND released_at IS NULL FOR NO KEY UPDATE', [row.deferred_refresh_id]);\n await declarePersistenceProtocol(tx);\n"
+ }
+ ],
+ "tests": [
+ {
+ "test": "test/refresh-write-barrier.test.ts",
+ "filter": "an expiry that overlaps a fence waits on the brain row",
+ "postgres": true
+ }
+ ]
+ }
+ },
+ {
+ "id": "restore-keeps-deferred",
+ "kind": "guard",
+ "title": "a shared-skill restore leaves deferred writes open (finding C reverted)",
+ "expect": "RestoreLeavesNoOpenWrite",
+ "config": "RefreshBarrierEnv.cfg",
+ "spec": {
+ "file": "RefreshBarrier.tla",
+ "before": "RestoreCancels == {\"queued\", \"deferred\", \"running\"}",
+ "after": "RestoreCancels == {\"queued\", \"running\"}"
+ },
+ "code": {
+ "edits": [
+ {
+ "file": "src/core/shared-skills/restore.ts",
+ "before": "state IN ('queued','deferred','running') ORDER BY sequence FOR UPDATE",
+ "after": "state IN ('queued','running') ORDER BY sequence FOR UPDATE"
+ }
+ ],
+ "tests": [
+ {
+ "test": "test/refresh-write-barrier.test.ts",
+ "filter": "a shared-skill restore cancels a deferred write too"
+ }
+ ]
+ }
+ },
+ {
+ "id": "replay-ignores-refresh-change",
+ "kind": "guard",
+ "title": "a deferred write's replay ignores what its refresh changed",
+ "expect": "RefreshChangeNeverOverwritten",
+ "spec": {
+ "file": "RefreshBarrier.tla",
+ "before": "ReplayDiff(i) == rmoved[i]",
+ "after": "ReplayDiff(i) == FALSE"
+ },
+ "code": {
+ "edits": [
+ {
+ "file": "src/core/persistence/refresh-barrier.ts",
+ "before": "if (paths === null || paths.has(target)) touched.push(target);",
+ "after": "if (paths === null) touched.push(target);"
+ }
+ ],
+ "tests": [
+ {
+ "test": "test/refresh-write-barrier.test.ts",
+ "filter": "a deferred blind write to a page the refresh changed ends revision_conflict"
+ }
+ ]
+ }
+ },
+ {
+ "id": "release-only-on-completed",
+ "kind": "guard",
+ "title": "the release trigger stamps released_at only when a refresh completes, not when it aborts",
+ "expect": "ReleasedWhenEnded",
+ "spec": {
+ "file": "RefreshBarrier.tla",
+ "before": "ReleaseOn(s) == s \\in {\"completed\", \"aborted\"}",
+ "after": "ReleaseOn(s) == s = \"completed\""
+ },
+ "code": {
+ "edits": [
+ {
+ "file": "src/core/schema-migrations/v292-refresh-write-barrier.ts",
+ "before": " END $$;\n ALTER TABLE persistence_consumers ADD COLUMN IF NOT EXISTS capabilities text[] NOT NULL DEFAULT '{}';\n CREATE OR REPLACE FUNCTION gbrain_release_refresh_barrier() RETURNS trigger LANGUAGE plpgsql SET search_path = pg_catalog, public, pg_temp AS $$\n BEGIN\n NEW.released_at := COALESCE(NEW.released_at, now());\n RETURN NEW;\n END $$;\n DROP TRIGGER IF EXISTS gbrain_refresh_release ON persistence_worktree_refreshes;\n CREATE TRIGGER gbrain_refresh_release BEFORE UPDATE OF state ON persistence_worktree_refreshes FOR EACH ROW\n WHEN (NEW.state IN ('completed','aborted') AND OLD.state NOT IN ('completed','aborted'))",
+ "after": " END $$;\n ALTER TABLE persistence_consumers ADD COLUMN IF NOT EXISTS capabilities text[] NOT NULL DEFAULT '{}';\n CREATE OR REPLACE FUNCTION gbrain_release_refresh_barrier() RETURNS trigger LANGUAGE plpgsql SET search_path = pg_catalog, public, pg_temp AS $$\n BEGIN\n NEW.released_at := COALESCE(NEW.released_at, now());\n RETURN NEW;\n END $$;\n DROP TRIGGER IF EXISTS gbrain_refresh_release ON persistence_worktree_refreshes;\n CREATE TRIGGER gbrain_refresh_release BEFORE UPDATE OF state ON persistence_worktree_refreshes FOR EACH ROW\n WHEN (NEW.state='completed' AND OLD.state NOT IN ('completed','aborted'))"
+ },
+ {
+ "file": "src/core/schema-migrations/v292-refresh-write-barrier.ts",
+ "before": " ALTER TABLE persistence_requests ADD COLUMN IF NOT EXISTS deferred_refresh_id uuid REFERENCES persistence_worktree_refreshes(id);\n ALTER TABLE persistence_consumers ADD COLUMN IF NOT EXISTS capabilities text[] NOT NULL DEFAULT '{}';\n CREATE OR REPLACE FUNCTION gbrain_release_refresh_barrier() RETURNS trigger LANGUAGE plpgsql SET search_path = pg_catalog, public, pg_temp AS $$\n BEGIN\n NEW.released_at := COALESCE(NEW.released_at, now());\n RETURN NEW;\n END $$;\n DROP TRIGGER IF EXISTS gbrain_refresh_release ON persistence_worktree_refreshes;\n CREATE TRIGGER gbrain_refresh_release BEFORE UPDATE OF state ON persistence_worktree_refreshes FOR EACH ROW\n WHEN (NEW.state IN ('completed','aborted') AND OLD.state NOT IN ('completed','aborted'))",
+ "after": " ALTER TABLE persistence_requests ADD COLUMN IF NOT EXISTS deferred_refresh_id uuid REFERENCES persistence_worktree_refreshes(id);\n ALTER TABLE persistence_consumers ADD COLUMN IF NOT EXISTS capabilities text[] NOT NULL DEFAULT '{}';\n CREATE OR REPLACE FUNCTION gbrain_release_refresh_barrier() RETURNS trigger LANGUAGE plpgsql SET search_path = pg_catalog, public, pg_temp AS $$\n BEGIN\n NEW.released_at := COALESCE(NEW.released_at, now());\n RETURN NEW;\n END $$;\n DROP TRIGGER IF EXISTS gbrain_refresh_release ON persistence_worktree_refreshes;\n CREATE TRIGGER gbrain_refresh_release BEFORE UPDATE OF state ON persistence_worktree_refreshes FOR EACH ROW\n WHEN (NEW.state='completed' AND OLD.state NOT IN ('completed','aborted'))"
+ }
+ ],
+ "tests": [
+ {
+ "test": "test/refresh-write-barrier.test.ts",
+ "filter": "a write deferred behind a refresh that aborts commits once the refresh is released"
+ }
+ ]
+ }
+ },
+ {
+ "id": "expiry-skips-release-recheck",
+ "kind": "guard",
+ "title": "the refresh-barrier expiry does not re-read released_at under the refresh row lock",
+ "expect": "ExpiryOnlyWhileWaiting",
+ "spec": {
+ "file": "RefreshBarrier.tla",
+ "before": "ExpiryRecheck(i) == ~rel[i]",
+ "after": "ExpiryRecheck(i) == TRUE"
+ },
+ "code": {
+ "edits": [
+ {
+ "file": "src/core/persistence/refresh-barrier.ts",
+ "before": "'SELECT 1 FROM persistence_worktree_refreshes WHERE id=$1::uuid AND released_at IS NULL FOR NO KEY UPDATE'",
+ "after": "'SELECT 1 FROM persistence_worktree_refreshes WHERE id=$1::uuid FOR NO KEY UPDATE'"
+ }
+ ],
+ "tests": [
+ {
+ "test": "test/refresh-write-barrier.test.ts",
+ "filter": "racing the end",
+ "postgres": true
+ }
+ ]
+ }
+ },
+ {
+ "id": "admission-skips-refresh-lock",
+ "kind": "guard",
+ "title": "admission reads the active refresh without locking its row",
+ "expect": "NoDeferToReleased",
+ "spec": {
+ "file": "RefreshBarrier.tla",
+ "before": "AdmRefLock(i) == <<L(REF(i), \"NKU\")>>",
+ "after": "AdmRefLock(i) == <<>>"
+ },
+ "code": {
+ "edits": [
+ {
+ "file": "src/core/persistence/journal.ts",
+ "before": "LIMIT 1${barrier ? ' FOR NO KEY UPDATE' : ''}) f) AS refresh",
+ "after": "LIMIT 1) f) AS refresh"
+ }
+ ],
+ "tests": [
+ {
+ "test": "test/refresh-write-barrier.test.ts",
+ "filter": "while its refresh ends",
+ "postgres": true
+ }
+ ]
+ }
+ },
+ {
+ "id": "publication-skips-reauth",
+ "kind": "guard",
+ "title": "publication does not re-authorize the stored request",
+ "expect": "NoApplyAfterRevoke",
+ "config": "RefreshBarrierEnv.cfg",
+ "spec": {
+ "file": "RefreshBarrier.tla",
+ "before": "PublishAuth == TRUE",
+ "after": "PublishAuth == FALSE"
+ },
+ "code": {
+ "edits": [
+ {
+ "file": "src/core/persistence/coordinator.ts",
+ "before": "await authorizeStoredRequest(tx, row, true, { pageVisibility: skill });",
+ "after": "if (skill) await authorizeStoredRequest(tx, row, true, { pageVisibility: skill });"
+ },
+ {
+ "file": "src/core/persistence/group-publish.ts",
+ "before": "() => authorizeStoredRequest(tx, rows[i]!, true, { pageVisibility: false }),",
+ "after": "async () => undefined,"
+ }
+ ],
+ "tests": [
+ {
+ "test": "test/refresh-write-barrier.test.ts",
+ "filter": "revoked after it was claimed"
+ }
+ ]
+ }
+ },
+ {
+ "id": "preparation-ignores-drift",
+ "kind": "guard",
+ "title": "preparation writes over a file that differs from the page",
+ "expect": "ImportedRevisionNeverHidden",
+ "spec": {
+ "file": "RefreshBarrier.tla",
+ "before": "PrepRefusesDrift == TRUE",
+ "after": "PrepRefusesDrift == FALSE"
+ },
+ "code": {
+ "edits": [
+ {
+ "file": "src/core/persistence/page-prepare.ts",
+ "before": "if (!matches && !staleCache) {",
+ "after": "if (!matches && !staleCache && false) {"
+ }
+ ],
+ "tests": [
+ {
+ "test": "test/pending-import-async-git-7436.test.ts",
+ "filter": "keeps the event loop running"
+ }
+ ]
+ }
+ },
+ {
+ "id": "drain-ignores-running",
+ "kind": "guard",
+ "title": "the drain count leaves out claimed (running) writes",
+ "expect": "FenceNeverMidPrepare",
+ "spec": {
+ "file": "RefreshBarrier.tla",
+ "before": "DrainCounts(w) == st[w] \\in {\"queued\", \"running\"}",
+ "after": "DrainCounts(w) == st[w] = \"queued\""
+ },
+ "code": {
+ "edits": [
+ {
+ "file": "src/core/persistence/worktree-refresh.ts",
+ "before": "r.state IN ('queued','running','recovering')",
+ "after": "r.state IN ('queued','recovering')"
+ }
+ ],
+ "tests": [
+ {
+ "test": "test/persistence-worktree-refresh.test.ts",
+ "filter": "13e."
+ }
+ ]
+ }
+ },
+ {
+ "id": "trigger-allows-unreleased",
+ "kind": "survivor",
+ "title": "the v292 guard trigger lets any claim move a deferred row",
+ "expect": "survives",
+ "reason": "defense in depth in the model, whose only consumer claims through claimableWriteSql; the trigger stops an older binary's claim, which the model leaves out, and the named test pins it",
+ "spec": {
+ "file": "RefreshBarrier.tla",
+ "before": "TriggerAllows(w) == st[w] # \"deferred\" \\/ Released(w)",
+ "after": "TriggerAllows(w) == TRUE"
+ },
+ "code": {
+ "edits": [
+ {
+ "file": "src/core/schema-migrations/v292-refresh-write-barrier.ts",
+ "before": "WHEN (OLD.state='deferred' OR NEW.state='deferred')\n EXECUTE FUNCTION gbrain_guard_deferred_request();\n ALTER TABLE persistence_requests DROP CONSTRAINT IF EXISTS persistence_requests_state_check;\n ALTER TABLE persistence_requests ADD CONSTRAINT persistence_requests_state_check\n CHECK (state IN ('queued','deferred','running','recovering','committed','conflict','failed','cancelled')) NOT VALID;",
+ "after": "WHEN (false)\n EXECUTE FUNCTION gbrain_guard_deferred_request();\n ALTER TABLE persistence_requests DROP CONSTRAINT IF EXISTS persistence_requests_state_check;\n ALTER TABLE persistence_requests ADD CONSTRAINT persistence_requests_state_check\n CHECK (state IN ('queued','deferred','running','recovering','committed','conflict','failed','cancelled')) NOT VALID;"
+ },
+ {
+ "file": "src/core/schema-migrations/v292-refresh-write-barrier.ts",
+ "before": "WHEN (OLD.state='deferred' OR NEW.state='deferred')\n EXECUTE FUNCTION gbrain_guard_deferred_request();\n ALTER TABLE persistence_requests DROP CONSTRAINT IF EXISTS persistence_requests_state_check;\n ALTER TABLE persistence_requests ADD CONSTRAINT persistence_requests_state_check\n CHECK (state IN ('queued','deferred','running','recovering','committed','conflict','failed','cancelled'));",
+ "after": "WHEN (false)\n EXECUTE FUNCTION gbrain_guard_deferred_request();\n ALTER TABLE persistence_requests DROP CONSTRAINT IF EXISTS persistence_requests_state_check;\n ALTER TABLE persistence_requests ADD CONSTRAINT persistence_requests_state_check\n CHECK (state IN ('queued','deferred','running','recovering','committed','conflict','failed','cancelled'));"
+ }
+ ],
+ "tests": [
+ {
+ "test": "test/refresh-write-barrier.test.ts",
+ "filter": "a claim without the barrier declaration cannot move a deferred row"
+ }
+ ]
+ }
+ },
+ {
+ "id": "fence-skips-recount",
+ "kind": "survivor",
+ "title": "fenceAndMerge does not re-count the drain under its locks",
+ "expect": "survives",
+ "reason": "defense in depth at these bounds: every admission after the refresh row exists is deferred, so no queued work can appear between the drain poll and the fence",
+ "spec": {
+ "file": "RefreshBarrier.tla",
+ "before": "FenceRecount == TRUE",
+ "after": "FenceRecount == FALSE"
+ }
+ },
+ {
+ "id": "claim-ignores-fence",
+ "kind": "survivor",
+ "title": "a claim ignores the refresh fence",
+ "expect": "survives",
+ "reason": "defense in depth at these bounds: under the fence every queued row has drained and every new admission is deferred to the unreleased refresh, so nothing is claimable",
+ "spec": {
+ "file": "RefreshBarrier.tla",
+ "before": "FenceClear == ~RefIs({\"fenced\", \"merged\", \"recovery\"})",
+ "after": "FenceClear == TRUE"
+ }
+ },
+ {
+ "id": "claim-skips-protocol",
+ "kind": "survivor",
+ "title": "a claim does not declare the protocol (brain row FOR SHARE)",
+ "expect": "survives",
+ "reason": "defense in depth at these bounds: the fence's drain count and the claim's own request-row lock already order a claim against the fence",
+ "spec": {
+ "file": "RefreshBarrier.tla",
+ "before": "ClaimLocks == <<L(BRAIN, \"S\")>>",
+ "after": "ClaimLocks == <<>>"
+ }
+ },
+ {
+ "id": "replay-skips-reauth",
+ "kind": "survivor",
+ "title": "a deferred write's replay does not re-authorize it",
+ "expect": "survives",
+ "reason": "defense in depth: publication re-authorizes every write (publication-skips-reauth)",
+ "config": "RefreshBarrierEnv.cfg",
+ "spec": {
+ "file": "RefreshBarrier.tla",
+ "before": "ReplayAuth == TRUE",
+ "after": "ReplayAuth == FALSE"
+ }
+ },
+ {
+ "id": "hold-beside-held-off",
+ "kind": "guard",
+ "title": "a later write to a page with a write held for an import is not held beside it (#8705's first race, before 03b89121)",
+ "expect": "CommitsInAdmissionOrder",
+ "config": "RefreshBarrierImport.cfg",
+ "spec": {
+ "file": "RefreshBarrier.tla",
+ "before": " ELSE IF ImportHolds /\\ HoldBehindHeld /\\ HeldAhead(w) # {} THEN \"pending\"\n",
+ "after": " ELSE IF FALSE THEN \"pending\"\n"
+ }
+ },
+ {
+ "id": "import-expiry-locks-source-first",
+ "kind": "guard",
+ "title": "#8705's import expiry locks the source row before it declares the protocol",
+ "expect": "NoDeadlock",
+ "config": "RefreshBarrierImport.cfg",
+ "spec": {
+ "file": "RefreshBarrier.tla",
+ "before": "ImportExpiryLocks == <<L(BRAIN, \"S\"), L(SRC, \"S\")>>",
+ "after": "ImportExpiryLocks == <<L(SRC, \"S\"), L(BRAIN, \"S\")>>"
+ }
+ },
+ {
+ "id": "import-expiry-skips-source-lock",
+ "kind": "guard",
+ "title": "#8705's import expiry re-checks last_commit without locking the source row (#8705's second race, before 03b89121)",
+ "expect": "ExpiryOnlyWhileWaiting",
+ "config": "RefreshBarrierImport.cfg",
+ "spec": {
+ "file": "RefreshBarrier.tla",
+ "before": "Need(EXP, IF ImportExpiryLocksSource THEN ImportExpiryLocks ELSE <<>>)",
+ "after": "Need(EXP, <<>>)"
+ }
+ }
+ ]
+}
diff --git a/test/spec-refresh-barrier.test.ts b/test/spec-refresh-barrier.test.ts
new file mode 100644
index 0000000000..60e110a659
--- /dev/null
+++ b/test/spec-refresh-barrier.test.ts
@@ -0,0 +1,52 @@
+/**
+ * spec/tla/refresh-barrier models the refresh write barrier (spec/tla/README.md). This test keeps the model's configs
+ * honest about what they check: the default config is #8682 alone, every config checks a property, and the TLA+
+ * workflow runs when anything a mutant names changes.
+ */
+import { describe, expect, test } from 'bun:test';
+import { readdirSync, readFileSync } from 'node:fs';
+import { join } from 'node:path';
+
+const ROOT = join(import.meta.dir, '..');
+const SPEC = join(ROOT, 'spec/tla/refresh-barrier');
+const CONFIGS = readdirSync(SPEC).filter(f => f.endsWith('.cfg')).sort();
+const read = (cfg: string) => readFileSync(join(SPEC, cfg), 'utf8');
+const SAFETY = [...read('RefreshBarrier.cfg').matchAll(/^INVARIANT (\w+)$/gm)].map(m => m[1]!);
+
+function constants(cfg: string): Record<string, string> {
+ const out: Record<string, string> = {};
+ for (const line of read(cfg).split('\n')) {
+ const m = line.match(/^\s+(\w+)\s*=\s*(.+?)\s*$/);
+ if (m) out[m[1]!] = m[2]!;
+ }
+ return out;
+}
+
+describe('spec/tla/refresh-barrier configs', () => {
+ test('RefreshBarrier.cfg checks #8682 alone, as coded', () => {
+ const c = constants('RefreshBarrier.cfg');
+ expect({ ImportHolds: c.ImportHolds, RecoveryDefers: c.RecoveryDefers, AdoptLocalEdits: c.AdoptLocalEdits })
+ .toEqual({ ImportHolds: 'FALSE', RecoveryDefers: 'FALSE', AdoptLocalEdits: 'FALSE' });
+ });
+
+ for (const cfg of CONFIGS) {
+ test(cfg + ': checks every safety invariant, or the liveness properties', () => {
+ const text = read(cfg);
+ const checked = [...text.matchAll(/^(?:INVARIANT|PROPERTY) (\w+)$/gm)].map(m => m[1]!);
+ if (text.includes('SPECIFICATION FairSpec')) expect(checked).toEqual(['DeferredLeaves', 'EveryWriteEnds']);
+ else expect(checked).toEqual(SAFETY);
+ });
+ }
+});
+
+describe('the TLA+ workflow runs when anything a refresh-barrier mutant names changes', () => {
+ test('every code file and test in mutants.json is in tla-spec.yml\'s path filter', () => {
+ const workflow = readFileSync(join(ROOT, '.github/workflows/tla-spec.yml'), 'utf8');
+ const list = (from: string, to: string) => [...workflow.slice(workflow.indexOf(from), workflow.indexOf(to)).matchAll(/^\s+- '([^']+)'$/gm)].map(m => m[1]!);
+ const { mutants } = JSON.parse(read('../refresh-barrier/mutants.json')) as { mutants: Array<{ code?: { edits: Array<{ file: string }>; tests: Array<{ test: string }> } }> };
+ const named = new Set(mutants.flatMap(m => m.code ? [...m.code.edits.map(e => e.file), ...m.code.tests.map(t => t.test)] : []));
+ const paths = new Set(list(' pull_request:', ' push:'));
+ expect([...named].filter(file => !paths.has(file))).toEqual([]);
+ expect(workflow).toContain('spec: [drain-lease, refresh-barrier]');
+ });
+});