Garry's Opinionated OpenClaw/Hermes Agent Brain
TypeScript
Stable waits for a green upgrade test from every recent stable release (GST-2818) (#9462)
Garry Tanb7babb3e3860
diff --git a/.github/workflows/promote-stable.yml b/.github/workflows/promote-stable.ymlindex 97867329fb..c1411c59d0 100644--- a/.github/workflows/promote-stable.yml+++ b/.github/workflows/promote-stable.yml@@ -8,8 +8,9 @@ name: Promote stable# maintain role; v<version> is an existing release tag; soak_run_url is a# successful completed run of the workflow named by the STABLE_SOAK_WORKFLOW# repository variable; that run's head commit has the release's exact git-# tree; and the version is newer than the current stable (an older one only-# with rollback=true and a reason). It then creates the immutable annotated+# tree; matrix_run_url, when given, is a green upgrade matrix run of master's+# harness on this release's exact tag and tree; and the version is newer than+# the current stable (an older one only with rollback=true and a reason). It then creates the immutable annotated# record tag stable-v<version> (first promotion only) and moves the annotated# `stable` tag to the release commit, both carrying the same key: value body,# and commits the evidence record stable.json (version, commit, tree, soak run,@@ -35,6 +36,11 @@ on:description: GitHub Actions run URL of the passed soak on this release's treerequired: truetype: string+ matrix_run_url:+ description: GitHub Actions run URL of a green upgrade matrix run on this release's tag and tree (empty promotes on your decision, recorded as matrix_run null)+ required: false+ type: string+ default: ''scoreboard_url:description: Scoreboard URL (defaults to soak_run_url)required: false@@ -78,7 +84,7 @@ on:permissions:contents: write # create and push the stable-v<version> and stable tags and the stable-record branch- actions: read # read the soak run+ actions: read # read the soak run and the upgrade matrix run's receiptconcurrency:group: promote-stable@@ -103,6 +109,7 @@ jobs:STABLE_SOAK_WORKFLOW: ${{ vars.STABLE_SOAK_WORKFLOW }}PROMOTE_VERSION: ${{ inputs.version }}PROMOTE_SOAK_RUN_URL: ${{ inputs.soak_run_url }}+ PROMOTE_MATRIX_RUN_URL: ${{ inputs.matrix_run_url }}PROMOTE_SCOREBOARD_URL: ${{ inputs.scoreboard_url }}PROMOTE_ROLLBACK: ${{ inputs.rollback }}PROMOTE_REASON: ${{ inputs.reason }}diff --git a/.github/workflows/stable-daily.yml b/.github/workflows/stable-daily.ymlindex 6d9f352e07..1057b7a94f 100644--- a/.github/workflows/stable-daily.yml+++ b/.github/workflows/stable-daily.yml@@ -2,11 +2,14 @@ name: Stable daily# The daily stable check (docs/RELEASING.md#daily-stable-check,# scripts/stable-daily.ts). Twice a day it promotes the newest release whose-# exact tree passed the production soak with every integrity gate clean, but-# only when a reason exists: a merged PR labelled stable:important that is-# released but not yet in stable, or stable older than 7 days (the weekly-# floor). With nothing to promote yet it dispatches one soak on the newest-# release, and a later run promotes it. It never promotes without that soak,+# exact tree passed the production soak with every integrity gate clean and+# whose upgrade matrix run on that tag and tree is green, but only when a+# reason exists: a merged PR labelled stable:important that is released but+# not yet in stable, or stable older than 7 days (the weekly floor). While the+# upgrade matrix's own break issue is open it holds, except for a security or+# data-loss fix. With nothing to promote yet it dispatches one soak (and one+# upgrade matrix run) on the newest release, and a later run promotes it. It+# never promotes without that soak and matrix run,# never moves stable backward, pauses while stable is a rollback, and does# nothing while the STABLE_AUTO_PROMOTE repository variable is "off".# When the published stable.json is missing or names another commit than@@ -38,7 +41,8 @@ on:permissions:contents: write # push the stable-v<version> and stable tags and the stable-record branch (stable.json)- actions: write # read soak runs and their scoreboards; dispatch a soak+ actions: write # read soak and upgrade matrix runs and their artifacts; dispatch a soak or the matrix+ issues: read # find the upgrade matrix's open break issue, which holds stableconcurrency:group: promote-stablediff --git a/.github/workflows/upgrade-matrix.yml b/.github/workflows/upgrade-matrix.ymlindex 8a138ae8fa..815c11970e 100644--- a/.github/workflows/upgrade-matrix.yml+++ b/.github/workflows/upgrade-matrix.yml@@ -19,14 +19,17 @@ name: Upgrade matrix# newest release); optional explicit from-versions and a# proof-only fault injection.# schedule nightly, full matrix against the newest release; a red-# run opens or updates one upgrade-break issue (the label-# that holds stable promotion) and a green run closes it.+# run opens or updates one upgrade-break issue (its marker,+# not the label, holds stable promotion: stable-daily.ts)+# and a green run closes it.# pull_request reduced subset (N-1 and N-50, PGLite and Postgres,# keyless, repo-less) against the head, only when schema,# migration, self-upgrade or harness paths change.#-# This workflow does not gate releases to the latest lane; stable promotion-# reads its receipt (GBRA-224 W1).+# This workflow does not gate releases to the latest lane. Stable promotion+# reads its receipt: scripts/stable-daily.ts dispatches it on the candidate tag+# and promotes only on a green run of that exact tag and tree+# (scripts/stable-matrix.ts).on:workflow_dispatch:diff --git a/changelog.d/capy-stable-waits-for-a.md b/changelog.d/capy-stable-waits-for-a.mdnew file mode 100644index 0000000000..c63d936477--- /dev/null+++ b/changelog.d/capy-stable-waits-for-a.md@@ -0,0 +1,7 @@+**Stable now waits for a green upgrade test from every recent stable release.**++### Changed+- The daily stable check promotes a release only after the upgrade matrix ran green on that release's exact tag and tree. With no run yet it starts one; a red run holds stable with a warning and is not started again by the schedule.+- The upgrade matrix's full run also upgrades from the stable releases users may still be on, which are the last 8 stable releases plus the newest one of each older month, back 6 months. A test run from before this change no longer counts.+- While the upgrade matrix's own break issue is open, stable holds. Only a security or data-loss fix that fails on no from-version the issue doesn't name may pass it.+- `promote-stable.yml` takes an optional `matrix_run_url`, checks it the same way and records it as `matrix_run` in `stable.json` and the stable tag. A maintainer can still promote without one, recorded as `matrix_run: null`.diff --git a/docs/RELEASING.md b/docs/RELEASING.mdindex 18cee3d629..9d94b98366 100644--- a/docs/RELEASING.md+++ b/docs/RELEASING.md@@ -777,12 +777,15 @@ gh workflow run promote-stable.yml -f version=<VERSION> \version's first promotion and never moved or deleted. Both carry the samemessage: a title line (`gbrain`, then `stable channel: v<VERSION>`), then one`key: value` line each for `version`, `commit`, `tree`, `soak_run`,- `scoreboard`, `promoted_by`, `promoted_at` (UTC), `previous`+ `scoreboard`, `matrix_run` (only when the promotion named one; `stable.json`+ always has the key, `null` without one), `promoted_by`, `promoted_at` (UTC), `previous`(`none` on the first promotion), `rollback` and `reason` (`-` whenempty).- **Inputs.** `version` (an existing `v<VERSION>` release tag),`soak_run_url` (the soak's Actions run URL, optionally- `/attempts/<n>`), `scoreboard_url` (optional, defaults to the run URL),+ `/attempts/<n>`), `matrix_run_url` (optional for a maintainer: a green+ upgrade matrix run on this release, see below), `scoreboard_url` (optional,+ defaults to the run URL),`rollback` (default false), `reason` (required with rollback) and`withdraw` (versions added to the carried-forward `known_bad` list;clients on one are offered the current stable, even when older), and@@ -815,6 +818,16 @@ gh workflow run promote-stable.yml -f version=<VERSION> \variables → Actions → Variables; a bare file name like `soak.yml` or thefull `.github/workflows/soak.yml` path). With the variable unset, everypromotion is refused.+- **Upgrade matrix run.** When `matrix_run_url` is given, it must be a+ completed run of `.github/workflows/upgrade-matrix.yml` from master's harness+ (a schedule or dispatch run, not a pull request's), with the full from-version+ set and no fault injection, whose `upgrade-matrix-receipt` artifact names this+ release's tag, commit and tree and was written by that run's own commit, and+ it must be green; a red or foreign run is refused+ (`scripts/stable-matrix.ts` `judgeMatrixRun`). A maintainer may promote+ without one, for example past an upgrade break the matrix found: the record+ then says `matrix_run: null` and names the maintainer as `promoted_by`. A+ scheduled promotion always names one.- **Published release.** The tag's `VERSION` file must say the version, andits GitHub release must exist and not be a draft, so binary clients canfetch the stable release.@@ -893,6 +906,27 @@ and keeps stable between one day and one week behind what matters:(never while another soak runs, and never after a soak on any newer releasefailed to qualify: then a maintainer decides, and the check warns once stableis past the weekly floor), and a later run promotes it.+- **Upgrade matrix next.** The soaked candidate also needs a green+ [upgrade matrix](ci-upgrade-matrix.md) run on its exact tag and tree: a+ schedule or dispatch run of master's harness with the full from-version set+ (which includes every stable release in the window), checked as under+ "Upgrade matrix run" above and again by `promote()`, and recorded as+ `matrix_run`. With no run yet it dispatches one on the candidate's tag+ (`upgrade-matrix.yml` on master, `candidate=v<version>`), unless a matrix run+ is active, and a later run promotes; when it dispatches a soak on the+ newest release it dispatches the matrix on that release too. A red or+ foreign run holds stable with a warning and is never re-dispatched by the+ schedule: a maintainer fixes the break or promotes by hand.+- **Upgrade breaks hold.** While the matrix's own break issue is open (the+ one the nightly run opens with the `<!-- gbrain-upgrade-matrix:nightly -->`+ marker; another issue with the `upgrade-break` label holds nothing), the+ check holds stable, the weekly floor included, and dispatches nothing. The+ one way past it on the schedule is a released `stable:important` fix whose+ `changelog.d` fragment says `important: security` or `important: data-loss`,+ on a candidate whose matrix run fails on no from-version the issue names+ (its `red-from` line) and on nothing else; the record's `reason` names the+ excused from-versions. Everything else waits for the nightly run to close+ the issue, or for a maintainer's promotion.- **Who.** A schedule event runs only the workflow file as merged on thedefault branch, so a scheduled run promotes by rule without a maintainer. Amanual dispatch defaults to `dry_run` (report only); `dry_run=false` needs andiff --git a/docs/architecture/key-files/ci-health-and-test-guards.md b/docs/architecture/key-files/ci-health-and-test-guards.mdindex 0b6193fe83..7f998907a0 100644--- a/docs/architecture/key-files/ci-health-and-test-guards.md+++ b/docs/architecture/key-files/ci-health-and-test-guards.md@@ -4,15 +4,15 @@Current behavior and load-bearing invariants; history belongs in Git and CHANGELOG.- `.github/workflows/nightly-watch.yml` + `scripts/nightly-issue.ts` — nightly-red and master-red incidents. `workflow_run` fires only for master-branch completions; the `resolve` job admits scheduled runs and push-to-master runs of Test and E2E Tests from this repository (head repository checked) plus dispatch, and `nightly-issue.ts --resolve` turns every entry path into the target workflow id and track (`schedule`, `push`; a replay writes `push`); `watch` queues on one group per repository + workflow + track. `nightly-issue.ts --run-id <id> [--replay-issue <n>] [--dry-run]` routes the run (`routeRun`: pull-request, fork, other-branch and non-allowlisted push runs are refused with a next step). Scheduled runs keep one `Nightly red: <workflow>` issue (label `nightly-red`): open on red with job/step pairs, every failing test file with its reproduce line, sanitized error excerpts, the first red run, the last green SHA and compare link, a "Next step for the agent" and a JSON block; update while red; reopen on a flap; close only on a later green scheduled run with complete evidence where every previously failing job executed. Rows in `.github/nightly-known-red.tsv` (at most 3; race-hunt jobs refused) keep a matching incident open under `known-red`. A scheduled Test or E2E Tests run also feeds that workflow's open master-red issue. Push runs go to `scripts/lib/master-red.ts`. A ratchet test fails when a scheduled workflow is not watched. Tests: `test/scripts/nightly-issue.test.ts`. Process: `docs/RELEASING.md#nightly-red-issues`, `docs/ci-red-runbook.md`.-- `.github/workflows/upgrade-matrix.yml` + `scripts/upgrade-matrix.ts` (modules in `scripts/upgrade-matrix/`) — upgrade-from-N matrix (GBRA-224 W2). `plan` resolves the candidate and from-versions (N-1, N-10, N-50, 30 days, stable) from release tags; `cell` installs a from-version with `bun install -g github:garrytan/gbrain` in an isolated HOME, seeds a brain, runs that release's `gbrain upgrade` against `github-fixture.ts` (per-cell bare mirror via git `insteadOf`, loopback tarball API, TLS fixture behind `HTTPS_PROXY`; every other host refused), then checks install identity, one build across a running serve, apply-migrations, schema head, doctor, canonical preservation (`snapshot.ts`), writes, keyed vector query on the mock provider and the old binary on the upgraded brain; `summarize` judges red/hold/green (`verdict.ts`, `known-exceptions.json`, the `W4 SWITCH POINT` for `doctor --alert`); `issue` keeps one `upgrade-break` issue. Tests: `test/scripts/upgrade-matrix.test.ts`. Docs: `docs/ci-upgrade-matrix.md`.+- `.github/workflows/upgrade-matrix.yml` + `scripts/upgrade-matrix.ts` (modules in `scripts/upgrade-matrix/`) — upgrade-from-N matrix (GBRA-224 W2). `plan` resolves the candidate and from-versions (N-1, N-10, N-50, 30 days, stable, and `stable-window`: `versions.ts` `stableWindow` over the `stable-v*` record tags, the newest 8 plus one per older month back 183 days) from release tags; `cell` installs a from-version with `bun install -g github:garrytan/gbrain` in an isolated HOME, seeds a brain, runs that release's `gbrain upgrade` against `github-fixture.ts` (per-cell bare mirror via git `insteadOf`, loopback tarball API, TLS fixture behind `HTTPS_PROXY`; every other host refused), then checks install identity, one build across a running serve, apply-migrations, schema head, doctor, canonical preservation (`snapshot.ts`), writes, keyed vector query on the mock provider and the old binary on the upgraded brain; `summarize` judges red/hold/green (`verdict.ts`, `known-exceptions.json`, the `W4 SWITCH POINT` for `doctor --alert`); `issue` keeps one `upgrade-break` issue with `marker.ts`'s `ISSUE_MARKER` and a `red-from` line naming the failing from-versions. Tests: `test/scripts/upgrade-matrix.test.ts`. Docs: `docs/ci-upgrade-matrix.md`.- `scripts/lib/gh-issue.ts` — the GitHub seam shared by the CI watchers. `inert()` makes run-derived text (job, step and test names, error text) safe to show an agent: control characters stripped, length capped, backticks and pipes replaced, mentions and links neutralized; `redactCredentials()` strips user:password from database URLs. `GitHubClient` is the one interface the watchers call (`restClient()` uses `GH_TOKEN`/`GITHUB_TOKEN` in Actions or the authenticated `gh api` locally; `download()` reads artifact archives without forwarding the token to blob storage). `findIssueByTitle` finds an issue by label and exact title, open first, else the most recently updated closed one; `listLabeled`, `getIssue` and `linkedPulls` (PRs cross-referencing an issue) back master-red and CI health; `fetchArtifactJson` + `readZipEntry` read one JSON file from a run artifact and return a reason instead of throwing. `createIssue` adds `AGENT_FILED_LABEL` (`agent-filed`) to every issue it opens, so CONTRIBUTING.md's community-issues search can leave automation's issues out; `test/scripts/gh-issue-agent-filed.test.ts` fails if a script or workflow opens an issue any other way.- `.github/workflows/ci-health.yml` + `scripts/ci-health-report.ts` — CI health report. Reads one window (`--days`, default 7, or `--since/--until`, also dispatch inputs) of same-repository pull-request and push-to-master runs of test.yml and e2e.yml (runs, jobs, check-run annotations, issues; never logs, fork runs or fork artifacts), paging the whole window (`listRuns` splits a query over the 1000-result search cap). It reports first-attempt pass rates (`firstAttempt()`), the exit criteria (master >= 95%, PR >= 90%; met, missed or indeterminate when the window cannot be read in full), stress-gate failures, failure classes, cancellation and completion coverage, master-red time to first response, the nightly-watch residual, the test files with the most failure annotations, shard spread and the count of fork runs it did not read; an explicit window with a missed target opens a follow-up issue. Tests: `test/scripts/ci-health-report.test.ts`. Docs: `docs/ci-red-runbook.md#ci-health-report`.- `scripts/lib/master-red.ts` — master-red reconciliation for push-to-master runs of Test and E2E Tests (`PUSH_ALLOWLIST`). Each watch evaluates every completed push run above the high-water mark (`evaluated`: run number, then attempt) stored in the newest `Master red: <workflow>` issue's `master-red:json` block (`applyRun` folds runs; cancelled runs change nothing); a missing or malformed block is rebuilt from the actions API (`pendingRuns`). It opens on red with the suspect range (`suspectRange`: compare plus commit-to-PR lookups), updates while red, closes when every previously failing job (E2E Tests: every failing file, from the manifest) ran and passed with complete evidence, records an incident that opened and closed between watches, and keeps a not-run-since failure open with the manual-close step. A newest issue closed by hand while its block is red ends that incident (`closed_by_hand`): later runs are recorded on it as green, and only a new red run opens a new issue. `externalPass` lets a scheduled run or an issue-bound E2E replay clear failures. Closing without a merged linked repair PR opens `flake` issues (`flakeFor`, `flake:json`); `closeFlakes` closes one after its repair PR merged and a later complete run passed its file and arm. Writes re-read the issue first and re-plan when it moved; a create race closes the newer duplicate. Tests: `test/scripts/nightly-issue.test.ts`. Docs: `docs/ci-red-runbook.md`.- `scripts/lib/ci-run.ts` — one run's failure evidence (`collectEvidence`): jobs, failing jobs' annotations and, for test.yml and e2e.yml, the `ci-manifest` artifact, returned as structured records (job, step, every failing test file and identity) apart from sanitized excerpts; a cancelled job is not run (listed in `cancelled`) unless its annotation says it timed out, and a red run whose only failures are aggregators of cancelled jobs is `ignored`; with `complete`/`problems` (an unreadable page, annotation or manifest is incomplete evidence, never zero failures). A push run with a succeeded `reused-full-suite` job is judged by the run its `reuse.json` names (`checkRecord` from `scripts/ci-reuse.ts`, recorded attempt, identical commit tree): that run's succeeded jobs and complete `ci-manifest` stand in for the push, and any gap is incomplete. `reproduceCommand` builds `bun run test:stress <file> --iterations 10 [--postgres]` from a validated test path only. `exemptionRows` turns failing identities with an exact test name and signature into the stress gate's `<!-- gbrain-stress-exemption -->` records (backend per `stressBackend`, owner `agent on release duty`), written into master-red, flake and nightly-red bodies; `test/fixtures/nightly-watch/stress-exemption-contract.ts` vendors the gate's parser so the tests prove it accepts them.- `scripts/ci-manifest.ts` — the versioned failure and execution manifest (`gbrain-ci-manifest/v1`). The `executed-receipts` job of test.yml and e2e.yml builds it from every lane's receipts (via `scripts/ci-executed-counts.ts`, declared skips from the expected-deltas TSV), binds it to the workflow, event, SHA, run id and attempt, and uploads it as `ci-manifest`; each failure carries the JUnit message (credentials redacted) and the stress gate's `failureSignature` (omitted when the message holds a credential); `checkManifest` refuses a manifest for another run or attempt; `passedFiles` lists files that executed tests and failed none. Docs: `docs/ci-red-runbook.md#ci-failure-manifest`.- `scripts/release-gate.ts` — release.yml's `ci-gate` job. Polls the push-to-master runs of Test and E2E Tests on the release commit and writes `publish`/`sha`: both green publishes; a failed run publishes nothing and fails with the next step; a cancelled or skipped run publishes nothing and fails with `gh run rerun <id>` (master push runs are keyed by sha, and between releases many commits share one VERSION, so it never follows to a later commit); non-master refs are refused; a `workflow_dispatch` (`--event`) gates only the release commit (the first-parent commit that last changed VERSION) and refuses a later master HEAD with the re-run of that commit's push Release run; it fails with the backfill command after 110 minutes. Build, release, `latest-stable` and the publish jobs use the gated commit. Tests: `test/scripts/release-gate.test.ts`. Docs: `docs/RELEASING.md#release-ci-gate`.-- `.github/workflows/promote-stable.yml` + `scripts/promote-stable.ts` — stable channel promotion (`workflow_dispatch`: `version`, `soak_run_url`, `scoreboard_url`, `rollback`, `reason`, `withdraw`, `not_promotable`, `not_promotable_reason`, `not_promotable_fixed_in`, `not_promotable_remove`; concurrency group `promote-stable`). `promote()` refuses unless the actor's `role_name` is admin or maintain (`authorize()`; the one exception is `PromoteDeps.scheduled`, set by `scheduleAuthorized()` only for a schedule event of `stable-daily.yml` on the default branch, which also requires the release commit on master and records `promoted_by: schedule (.github/workflows/stable-daily.yml)`), `v<VERSION>` exists with a `VERSION` file naming it and a published (non-draft) GitHub release, the soak run URL names a completed, successful run of this repository's `STABLE_SOAK_WORKFLOW` workflow (repository variable; bare file name or path), the run's head commit has the release tag's exact tree (fetched when missing), and the version is newer than the current stable (the highest `v*` tag on `refs/tags/stable^{commit}`), or older with `rollback` and a reason. Re-promoting the current stable (no rollback) moves no tag: `recordCurrentStable()` writes `stable.json` from the stable tag's message (`parseTagMessage()`, which must name the same commit, tree and soak run) when the published record does not name that release, leasing `stable` unchanged and the record branch, and refuses as a no-op otherwise (GBRA-346 backfill for releases promoted before the record existed). It creates the immutable annotated `stable-v<VERSION>` record (refused when it exists, except on a rollback re-promotion of the same commit) and moves the annotated `stable` tag, both with `renderTagMessage()`'s `key: value` body, commits the evidence record (`renderRecord()`, `stable.json`, `known_bad` carried forward and merged with `withdraw` by `mergeKnownBad()`, `not_promotable` carried forward, added to and removed from by `mergeNotPromotable()`, which caps a reason at 500 characters; a record over `STABLE_RECORD_MAX_BYTES`, the client's `RECORD_MAX_BYTES`, is refused before the push) on the `stable-record` branch, then pushes all three in one `--atomic` push (`stable` and `stable-record` leased on the values it read, so a failed push changes nothing) and writes the step summary. Side effects come in through `PromoteDeps`. Tests: `test/scripts/promote-stable.test.ts`. Docs: `docs/RELEASING.md#stable-channel-promotion`.-- `.github/workflows/stable-daily.yml` + `scripts/stable-daily.ts` — the daily stable check (05:17 and 13:17 UTC; dispatch defaults to `dry_run`, and `dry_run=false` needs admin or maintain; concurrency group `promote-stable`). Off when `STABLE_AUTO_PROMOTE` is `off`, paused while the stable record says `rollback: true`. Acts only on a reason: a merged `stable:important` PR released but not in stable, or stable older than `WEEKLY_FLOOR_DAYS` (7) with a newer release. Promotes, through `promote()`, the newest release on master newer than stable whose exact tree passed a full `SOAK_SHAPE` `STABLE_SOAK_WORKFLOW` run with every `INTEGRITY_GATES` gate passing in every cell (on the important-only path it must contain the fix), skipping a release with a `stable-v<version>` tag (stable before, then rolled back) or on stable.json's `known_bad` (#8792) or `not_promotable` list; otherwise dispatches one soak on the newest release's tag, never while a soak runs or after a soak on a newer release failed to qualify. Tests: `test/scripts/stable-daily.test.ts`. Docs: `docs/RELEASING.md#daily-stable-check`.+- `.github/workflows/promote-stable.yml` + `scripts/promote-stable.ts` — stable channel promotion (`workflow_dispatch`: `version`, `soak_run_url`, `matrix_run_url`, `scoreboard_url`, `rollback`, `reason`, `withdraw`, `not_promotable`, `not_promotable_reason`, `not_promotable_fixed_in`, `not_promotable_remove`; concurrency group `promote-stable`). `promote()` refuses unless the actor's `role_name` is admin or maintain (`authorize()`; the one exception is `PromoteDeps.scheduled`, set by `scheduleAuthorized()` only for a schedule event of `stable-daily.yml` on the default branch, which also requires the release commit on master and records `promoted_by: schedule (.github/workflows/stable-daily.yml)`), `v<VERSION>` exists with a `VERSION` file naming it and a published (non-draft) GitHub release, the soak run URL names a completed, successful run of this repository's `STABLE_SOAK_WORKFLOW` workflow (repository variable; bare file name or path), the run's head commit has the release tag's exact tree (fetched when missing), `checkMatrix()` accepts the named upgrade matrix run only through `scripts/stable-matrix.ts` `judgeMatrixRun()` (a completed schedule or dispatch run of master's harness, full profile, no fault, whose `upgrade-matrix-receipt` names the release's tag, commit and tree and the run's own harness, and green; a scheduled promotion must name one, a maintainer may not and the record says `matrix_run: null`), and the version is newer than the current stable (the highest `v*` tag on `refs/tags/stable^{commit}`), or older with `rollback` and a reason. Re-promoting the current stable (no rollback) moves no tag: `recordCurrentStable()` writes `stable.json` from the stable tag's message (`parseTagMessage()`, which must name the same commit, tree and soak run) when the published record does not name that release, leasing `stable` unchanged and the record branch, and refuses as a no-op otherwise (GBRA-346 backfill for releases promoted before the record existed). It creates the immutable annotated `stable-v<VERSION>` record (refused when it exists, except on a rollback re-promotion of the same commit) and moves the annotated `stable` tag, both with `renderTagMessage()`'s `key: value` body, commits the evidence record (`renderRecord()`, `stable.json`, `known_bad` carried forward and merged with `withdraw` by `mergeKnownBad()`, `not_promotable` carried forward, added to and removed from by `mergeNotPromotable()`, which caps a reason at 500 characters; a record over `STABLE_RECORD_MAX_BYTES`, the client's `RECORD_MAX_BYTES`, is refused before the push) on the `stable-record` branch, then pushes all three in one `--atomic` push (`stable` and `stable-record` leased on the values it read, so a failed push changes nothing) and writes the step summary. Side effects come in through `PromoteDeps`. Tests: `test/scripts/promote-stable.test.ts`. Docs: `docs/RELEASING.md#stable-channel-promotion`.+- `.github/workflows/stable-daily.yml` + `scripts/stable-daily.ts` — the daily stable check (05:17 and 13:17 UTC; dispatch defaults to `dry_run`, and `dry_run=false` needs admin or maintain; concurrency group `promote-stable`). Off when `STABLE_AUTO_PROMOTE` is `off`, paused while the stable record says `rollback: true`. Acts only on a reason: a merged `stable:important` PR released but not in stable, or stable older than `WEEKLY_FLOOR_DAYS` (7) with a newer release. Promotes, through `promote()`, the newest release on master newer than stable whose exact tree passed a full `SOAK_SHAPE` `STABLE_SOAK_WORKFLOW` run with every `INTEGRITY_GATES` gate passing in every cell (on the important-only path it must contain the fix), skipping a release with a `stable-v<version>` tag (stable before, then rolled back) or on stable.json's `known_bad` (#8792) or `not_promotable` list; the soaked candidate also needs a green upgrade matrix gate run on its exact tag and tree (`judgeMatrixRun`), and with none it dispatches `upgrade-matrix.yml` on master with `candidate=v<version>` (not while a matrix run is active); a red or foreign run returns `held`. While the matrix's own break issue is open (`findBreakIssue`: `ISSUE_MARKER`, not the label alone) it returns `held` unless a released fix's `changelog.d` fragment says `important: security` or `data-loss` (`BREAK_PASS_CLASSES`) and the candidate's run fails only on from-versions the issue's `red-from` line names. Otherwise dispatches one soak (and the matrix, when it has no run) on the newest release's tag, never while a soak runs or after a soak on a newer release failed to qualify. Tests: `test/scripts/stable-daily.test.ts`. Docs: `docs/RELEASING.md#daily-stable-check`.- `scripts/check-install-ref.ts` (`bun run check:install-ref`, in verify) — every install pin, bootstrap runbook fetch and setup-helper fetch in tracked files (not CHANGELOG.md or test/) names `INSTALL_REF` from `src/core/install-ref.ts`; exact `v<version>` and commit pins are left alone; TypeScript must import the constant; `--write` rewrites the text surfaces. `scripts/check-bootstrap-tag.sh` reads its sanctioned ref from the same constant. Tests: `test/install-ref.test.ts`.- `scripts/regen-all.ts` — one entry point for every generated artifact (`ARTIFACTS`: regen and check command per artifact). `bun run regen:all` regenerates offline and keyless (provider keys and database URLs removed) and lists what changed; a second run changes nothing. `--check` is read-only and runs in `verify` as `check:regen-all`; `--goldens` also regenerates the offline contract goldens. Postgres goldens are named with their own commands, and `llms.txt` regenerates last because it inlines other generated docs. Every `bun test` it spawns (two checks and the goldens) carries `--timeout=60000` through one `BUN_TEST` prefix, because Bun does not apply bunfig.toml's `[test] timeout`. Tests: `test/scripts/regen-all.test.ts`.- `scripts/build-behavior-changes.ts` — writes (or `--check`s) `src/core/behavior-changes.generated.ts` from `behavior-changes.d/<id>.md` and the `<!-- behavior-change: <id> -->` markers in released CHANGELOG sections (and in `changelog.d` fragments while VERSION is newer than the top heading, i.e. inside a release PR): `RELEASED_BEHAVIOR_CHANGES` rows dated by their section, `BEHAVIOR_CHANGE_RELEASES` for `releasedIn(id)` rows. Refuses unmarked files, dangling markers, duplicate ids, bad names, texts over 2,000 characters and a `changelog.d` fragment whose `bump:` is not `patch`. The release PR's `release_tool`; in `regen:all`. Tests: `test/scripts/build-behavior-changes.test.ts`.diff --git a/docs/ci-upgrade-matrix.md b/docs/ci-upgrade-matrix.mdindex a91dcf37e2..4461d00e7b 100644--- a/docs/ci-upgrade-matrix.md+++ b/docs/ci-upgrade-matrix.md@@ -7,8 +7,11 @@ checks the candidate on the upgraded brain. The driver is[scripts/upgrade-matrix.ts](../scripts/upgrade-matrix.ts) with its modules in[scripts/upgrade-matrix/](../scripts/upgrade-matrix/).-It does not gate releases to the latest lane. Stable promotion reads its run-receipt (GBRA-224 W1).+It does not gate releases to the latest lane. It gates stable: the daily+stable check dispatches it on the stable candidate's tag and promotes only on+a green run of that exact tag and tree, and holds while the nightly run's+break issue is open ([daily stable check](RELEASING.md#daily-stable-check),+`scripts/stable-matrix.ts`).## What a run tests@@ -16,9 +19,12 @@ receipt (GBRA-224 W1).releases API, authenticated with the workflow token) and recorded in the plan:N-1, N-10 and N-50 (the 1st, 10th and 50th newest release older than thecandidate), the release published closest to 30 days before the candidate, and-the current `stable` tag when it exists and is older. A role that cannot be-filled is recorded as skipped with its reason. Pull requests run N-1 and N-50-only.+the current `stable` tag when it exists and is older, and the stable+releases in the window (`stable-window`, `versions.ts` `stableWindow`): every+`stable-v<version>` record older than the candidate that was stable during the+183 days before it, capped at the newest 8 by promotion plus the newest one of+each older calendar month. A role that cannot be filled is recorded as+skipped with its reason. Pull requests run N-1 and N-50 only.**Cells.** Every from-version runs on PGLite and Postgres, keyless and keyed,repo-less and repo brain (32 to 40 cells; one job per from-version, engine and@@ -143,8 +149,13 @@ branch.- **Nightly** (06:41 UTC): full matrix against the newest release. A red runopens or updates one issue carrying the `upgrade-break` label (found by themarker `<!-- gbrain-upgrade-matrix:nightly -->`) and a later green run- comments and closes it. Nightly watch also tracks the workflow like every- scheduled run.+ comments and closes it. The issue's+ `<!-- gbrain-upgrade-matrix:red-from <tags> -->` line names the from-versions+ that failed or held. While it is open the daily stable check holds stable+ (`upgrade-matrix/marker.ts`). Nightly watch also tracks the workflow like+ every scheduled run.+- **Stable candidate**: the daily stable check dispatches the full matrix on+ its candidate's tag when that release has no run yet (`scripts/stable-daily.ts`).- **Dispatch**: `gh workflow run upgrade-matrix.yml --ref master -f candidate=v0.60.475.0`(empty candidate: the newest release; `-f from=v0.60.390.0,v0.60.425.0`replaces the resolved from-versions; `-f fault=corrupt-after-upgrade` is the@@ -195,7 +206,11 @@ older Bun versions.A cell takes 40 to 70 seconds on a 4-vCPU runner (install from the Bun cache,seed, upgrade, checks, mixed-version probe); a job adds about two minutes of-checkout, `bun install` and cache restore. A full run is 20 cell jobs of about-four minutes on `ubicloud-standard-4` plus the plan, summary and issue jobs on-`ubicloud-standard-2`: about 330 vCPU-minutes. The pull-request subset is-4 jobs, about 50 vCPU-minutes.+checkout, `bun install` and cache restore. A full run is 4 cell jobs per+from-version, each about four minutes on `ubicloud-standard-4`, plus the+plan, summary and issue jobs on `ubicloud-standard-2`. With the stable window+(on 2026-11-16, candidate v0.60.729.0: 14 from-versions) that is 56 cell jobs+and 112 cells, about 900 vCPU-minutes, against 20 cell jobs (5 from-versions,+about 330 vCPU-minutes) before it. The window adds at most one from-version+per stable promotion until it holds 8 recent ones plus one per older month.+The pull-request subset is 4 jobs, about 50 vCPU-minutes.diff --git a/scripts/promote-stable.ts b/scripts/promote-stable.tsindex cb18affc6e..1fce2aa5e5 100644--- a/scripts/promote-stable.ts+++ b/scripts/promote-stable.ts@@ -20,6 +20,11 @@* the STABLE_SOAK_WORKFLOW repository variable, completed with success;* 4. the run's head commit has the release tree (the same bytes, not merely* the same version);+ * 4b. a forward promotion by the schedule names a green upgrade matrix run on+ * the release's exact tag and tree (matrix_run_url; scripts/stable-matrix.ts+ * judgeMatrixRun), and any matrix run named is checked the same way, so a+ * red or foreign run is refused; a maintainer may promote without one, and+ * the record then says matrix_run: null with the maintainer as promoted_by;* 5. the version is strictly newer than the current stable, unless this is a* rollback with a reason; promoting the current stable again moves no* tag and only writes its evidence record when the published one does@@ -46,16 +51,19 @@* Clients read the record at* https://raw.githubusercontent.com/garrytan/gbrain/stable-record/stable.json* (src/core/stable-channel.ts: no API call, no git) and an explicit upgrade- * checks it against `refs/tags/stable^{}` and `v<VERSION>`. Generation 1- * records carry soak evidence only; the upgrade matrix (W2) adds matrix_run- * and sets MATRIX_REQUIRED_FROM in src/core/self-upgrade.ts.+ * checks it against `refs/tags/stable^{}` and `v<VERSION>`. Records carry the+ * soak run and, from the matrix gate on, matrix_run; clients require matrix_run+ * only once MATRIX_REQUIRED_FROM in src/core/self-upgrade.ts is set.* Side effects (gh, git, clock, log, step summary) come in through* PromoteDeps, so the decisions are tested without network or a real remote* (test/scripts/promote-stable.test.ts). Every refusal exits 1 with one* sentence saying what to do. Docs: docs/RELEASING.md#stable-channel-promotion*/import { spawnSync } from 'node:child_process';-import { appendFileSync } from 'node:fs';+import { appendFileSync, mkdtempSync, readFileSync, rmSync } from 'node:fs';+import { tmpdir } from 'node:os';+import { join } from 'node:path';+import { judgeMatrixRun, MATRIX_RECEIPT_ARTIFACT, type BreakIssue, type MatrixReceipt, type MatrixRunRow } from './stable-matrix.ts';export const DOCS = 'docs/RELEASING.md#stable-channel-promotion';export const STABLE_REF = 'refs/tags/stable';@@ -81,6 +89,10 @@ export interface PromoteInputs {notPromotable?: NotPromotable[];/** Releases to take off the record's not_promotable list (one listed by mistake, say). */notPromotableRemove?: string[];+ /** The upgrade matrix run on this release (required for a scheduled forward promotion). */+ matrixRunUrl?: string;+ /** The open break issue a security or data-loss promotion passes (stable-daily only); the run may fail only on its named from-versions. */+ matrixExcuse?: BreakIssue | null;}/** A release outside evidence shows is bad although it may soak clean, with the release that fixes it. */@@ -102,6 +114,8 @@ export interface PromoteDeps {/** A write to the GitHub API (only the release edit that marks stable Latest). */ghApiWrite(method: 'PATCH', path: string, body: Record<string, unknown>): Promise<unknown>;git(args: string[], input?: string): Promise<GitResult>;+ /** The upgrade-matrix-receipt artifact of a matrix run, or null when it cannot be read. */+ matrixReceipt?(runId: number): Promise<MatrixReceipt | null>;now(): Date;log(line: string): void;summary(markdown: string): void;@@ -120,6 +134,8 @@ export interface TagFields {tree: string;soakRun: string;scoreboard: string;+ /** The upgrade matrix run; absent or empty when a maintainer promoted without one. */+ matrixRun?: string;promotedBy: string;promotedAt: string;previous: string | null;@@ -139,6 +155,7 @@ export function renderRecord(f: TagFields, knownBad: readonly string[], notPromotree: f.tree,soak_run: f.soakRun,scoreboard: f.scoreboard,+ matrix_run: f.matrixRun || null,promoted_by: f.promotedBy,promoted_at: f.promotedAt,previous: f.previous,@@ -221,10 +238,11 @@ export function compareVersions(a: string, b: string): number {return 0;}-export function parseRunUrl(url: string, repo: string): { runId: string; attempt: string | null } {+export function parseRunUrl(url: string, repo: string, input: 'soak_run_url' | 'matrix_run_url' = 'soak_run_url'): { runId: string; attempt: string | null } {+ const what = input === 'soak_run_url' ? 'soak' : 'upgrade matrix';const m = /^https:\/\/github\.com\/([\w.-]+\/[\w.-]+)\/actions\/runs\/(\d+)(?:\/attempts\/(\d+))?\/?$/.exec(url.trim());- if (!m) throw new Refusal(`soak_run_url "${url}" is not a GitHub Actions run URL; pass https://github.com/${repo}/actions/runs/<run id> (optionally /attempts/<n>) of the soak run.`);- if (m[1]!.toLowerCase() !== repo.toLowerCase()) throw new Refusal(`soak_run_url names a run of ${m[1]}, not ${repo}; pass the soak run from this repository's Actions tab.`);+ if (!m) throw new Refusal(`${input} "${url}" is not a GitHub Actions run URL; pass https://github.com/${repo}/actions/runs/<run id> (optionally /attempts/<n>) of the ${what} run.`);+ if (m[1]!.toLowerCase() !== repo.toLowerCase()) throw new Refusal(`${input} names a run of ${m[1]}, not ${repo}; pass the ${what} run from this repository's Actions tab.`);return { runId: m[2]!, attempt: m[3] ?? null };}@@ -263,6 +281,7 @@ export function renderTagMessage(f: TagFields): string {`tree: ${f.tree}`,`soak_run: ${f.soakRun}`,`scoreboard: ${f.scoreboard}`,+ ...(f.matrixRun ? [`matrix_run: ${f.matrixRun}`] : []),`promoted_by: ${f.promotedBy}`,`promoted_at: ${f.promotedAt}`,`previous: ${f.previous ?? 'none'}`,@@ -289,8 +308,9 @@ export function parseTagMessage(message: string): TagFields | null {if (!version || !VERSION_RE.test(version) || !commit || !SHA_RE.test(commit) || !tree || !SHA_RE.test(tree)) return null;if (!soakRun || !scoreboard || !promotedBy || !promotedAt || !reason || (rollback !== 'true' && rollback !== 'false')) return null;if (!previous || (previous !== 'none' && !VERSION_RE.test(previous))) return null;+ const matrixRun = kv.get('matrix_run');return {- version, commit, tree, soakRun, scoreboard, promotedBy, promotedAt,+ version, commit, tree, soakRun, scoreboard, ...(matrixRun ? { matrixRun } : {}), promotedBy, promotedAt,previous: previous === 'none' ? null : previous,rollback: rollback === 'true',reason: reason === '-' ? '' : reason,@@ -304,6 +324,7 @@ export function renderSummary(p: Promotion, notPromotable: readonly NotPromotabl['Tree', `\`${p.tree}\``],['Soak run', p.soakRun],['Scoreboard', p.scoreboard],+ ['Upgrade matrix', p.matrixRun || 'none (promoted without one)'],['Previous stable', p.previous ? `v${p.previous}` : 'none'],['Rollback', p.rollback ? `yes: ${p.reason.trim()}` : 'no'],['Known bad', p.knownBad.length ? p.knownBad.map(v => `v${v}`).join(', ') : 'none'],@@ -455,6 +476,7 @@ export async function promote(deps: PromoteDeps, inputs: PromoteInputs): Promiseconst current = await currentStable(deps);if (current && !rollback && compareVersions(version, current.version) === 0) return recordCurrentStable(deps, inputs, { version, commit, tree }, current);checkOrder(version, current?.version ?? null, rollback, reason);+ const matrixRun = await checkMatrix(deps, inputs, { version, commit, tree });deps.log(`[promote-stable] current stable is ${current ? `v${current.version}` : 'none'}; ${rollback ? `rollback to v${version}: ${reason.trim()}` : `v${version} is newer`}.`);const recordRef = `refs/tags/stable-v${version}`;@@ -470,6 +492,7 @@ export async function promote(deps: PromoteDeps, inputs: PromoteInputs): Promiseversion, commit, tree,soakRun: inputs.soakRunUrl.trim(),scoreboard: inputs.scoreboardUrl.trim() || inputs.soakRunUrl.trim(),+ ...(matrixRun ? { matrixRun } : {}),promotedBy: deps.scheduled ? `schedule (${DAILY_WORKFLOW})` : deps.actor,promotedAt: deps.now().toISOString().replace(/\.\d{3}Z$/, 'Z'),previous: current?.version ?? null,@@ -506,6 +529,30 @@ export async function promote(deps: PromoteDeps, inputs: PromoteInputs): Promisereturn promotion;}+/**+ * The upgrade matrix run a promotion names, judged by judgeMatrixRun (scripts/stable-matrix.ts); refuses a red,+ * foreign or unreadable run. A scheduled forward promotion must name one; a maintainer may promote without one.+ */+async function checkMatrix(deps: PromoteDeps, inputs: PromoteInputs, release: { version: string; commit: string; tree: string }): Promise<string> {+ const url = (inputs.matrixRunUrl ?? '').trim();+ if (!url) {+ if (inputs.rollback) return '';+ if (deps.scheduled) throw new Refusal(`a scheduled promotion needs a green upgrade matrix run on v${release.version}'s exact tree; the daily check dispatches one (${MATRIX_DOCS}).`);+ deps.log(`[promote-stable] no upgrade matrix run given: v${release.version} is promoted on ${deps.actor}'s decision, and the record's matrix_run is null.`);+ return '';+ }+ const { runId, attempt } = parseRunUrl(url, deps.repo, 'matrix_run_url');+ if (attempt) throw new Refusal(`matrix_run_url names attempt ${attempt}; pass the run URL without /attempts/, whose receipt is the run's latest.`);+ if (!deps.matrixReceipt) throw new Refusal('This caller cannot read upgrade matrix receipts; run the promotion from promote-stable.yml or stable-daily.yml.');+ const run = await deps.ghApi(`repos/${deps.repo}/actions/runs/${runId}`) as MatrixRunRow;+ const judged = judgeMatrixRun({ ...run, html_url: url }, await deps.matrixReceipt(Number(runId)), release, inputs.matrixExcuse);+ if (!judged.ok) throw new Refusal(`${judged.why}; promote only a release whose upgrade matrix run passed on its exact tree (${MATRIX_DOCS}).`);+ deps.log(`[promote-stable] ${judged.why}.`);+ return url;+}++const MATRIX_DOCS = 'docs/RELEASING.md#daily-stable-check';+/** GitHub's limit on a release body is 125,000 characters; stay under it with room for the trailer. */export const RELEASE_NOTES_MAX = 120_000;@@ -609,6 +656,18 @@ export function processDeps(env: NodeJS.ProcessEnv = process.env, cwd = process.const r = spawnSync('git', args, { cwd, env, encoding: 'utf8', maxBuffer: 64 * 1024 * 1024, ...(input !== undefined ? { input } : {}) });return { code: r.status ?? 1, stdout: r.stdout ?? '', stderr: r.stderr ?? '' };},+ async matrixReceipt(runId) {+ const dir = mkdtempSync(join(tmpdir(), 'matrix-'));+ try {+ const r = spawnSync('gh', ['run', 'download', String(runId), '-R', env.GITHUB_REPOSITORY ?? '', '-n', MATRIX_RECEIPT_ARTIFACT, '-D', dir], { cwd, env, encoding: 'utf8' });+ if (r.status !== 0) return null;+ return JSON.parse(readFileSync(join(dir, `${MATRIX_RECEIPT_ARTIFACT}.json`), 'utf8')) as MatrixReceipt;+ } catch {+ return null;+ } finally {+ rmSync(dir, { recursive: true, force: true });+ }+ },now: () => new Date(),log: line => console.log(line),summary: markdown => { if (env.GITHUB_STEP_SUMMARY) appendFileSync(env.GITHUB_STEP_SUMMARY, markdown); },@@ -627,6 +686,7 @@ if (import.meta.main) {notPromotable: (env.PROMOTE_NOT_PROMOTABLE ?? '').split(',').map(v => v.trim().replace(/^v/, '')).filter(Boolean).map(version => ({ version, reason: env.PROMOTE_NOT_PROMOTABLE_REASON ?? '', fixed_in: (env.PROMOTE_NOT_PROMOTABLE_FIXED_IN ?? '').trim().replace(/^v/, '') })),notPromotableRemove: (env.PROMOTE_NOT_PROMOTABLE_REMOVE ?? '').split(',').map(v => v.trim().replace(/^v/, '')).filter(Boolean),+ matrixRunUrl: env.PROMOTE_MATRIX_RUN_URL ?? '',};const deps = processDeps();try {diff --git a/scripts/stable-daily.ts b/scripts/stable-daily.tsindex 1ffd378bdf..50c22dbae7 100644--- a/scripts/stable-daily.ts+++ b/scripts/stable-daily.ts@@ -15,10 +15,15 @@* promote()'s re-promotion path (no tag moves; the release, exact tree,* soak run and leases are checked as for any promotion), and stops there* (GBRA-346: stable was promoted before the record existed).- * 2. A reason must exist: a merged PR labelled `stable:important` (security,- * regression, data-loss or blocking fix, set by the merge lane) that is in a- * published release but not in stable, or stable older than 7 days with a- * newer release (the weekly floor). No reason: nothing happens.+ * 2. A reason must exist: a merged PR labelled `stable:important` (set by the+ * merge lane; the classes are in docs/RELEASING.md) that is in a published+ * release but not in stable, or stable older than 7 days with a newer+ * release (the weekly floor). No reason: nothing happens.+ * 2b. While the upgrade matrix's own break issue is open (scripts/stable-matrix.ts+ * findBreakIssue: the ISSUE_MARKER issue, not any issue with the label), stable+ * holds: only an important fix whose fragment says `important: security` or+ * `important: data-loss` may pass it, and only on a candidate whose matrix run+ * fails on no from-version the issue does not name. The weekly floor waits.* 3. The candidate is the newest release on master newer than stable whose exact* git tree passed a full (SOAK_SHAPE) successful run of the STABLE_SOAK_WORKFLOW* soak with every integrity gate (INTEGRITY_GATES) passing in every cell; on the@@ -29,8 +34,14 @@* then rolled back) or that stable.json lists as known_bad or not_promotable* (bad on outside evidence the soak cannot see) is never a candidate or soaked* (#8792).+ * 3b. The candidate also needs an upgrade matrix run on its exact tag and tree+ * (stable-matrix.ts judgeMatrixRun): green, or past an open break as in 2b. No+ * run yet: one matrix dispatch on the candidate tag, unless a matrix run is+ * active. A red or foreign run holds stable (a maintainer decides) and is never+ * re-dispatched by the schedule. promote() checks the run again.* 4. Nothing promotable yet: the newest release is soaked (one dispatch of the- * soak on its tag) unless a soak is already running, or a soak on any newer+ * soak on its tag, with an upgrade matrix dispatch on the same tag when it has+ * no run) unless a soak is already running, or a soak on any newer* release did not qualify (a maintainer decides; past the weekly floor it* warns); a failed soak is never retried by the schedule.*@@ -44,6 +55,10 @@ import {authorize, compareVersions, currentStable, DOCS, onReleaseBranch, processDeps, promote, readPriorRecord, refExists, Refusal, requireSoakWorkflow, scheduleAuthorized, STABLE_REF,type PromoteDeps, type Promotion,} from './promote-stable.ts';+import {+ BREAK_PASS_CLASSES, breakIssueQuery, findBreakIssue, fragmentImportance, isGateRun, judgeMatrixRun, MATRIX_WORKFLOW, receiptNames,+ type MatrixReceipt, type MatrixRunRow,+} from './stable-matrix.ts';export const IMPORTANT_LABEL = 'stable:important';export const WEEKLY_FLOOR_DAYS = 7;@@ -64,14 +79,15 @@ export interface Scoreboard {export interface DailyDeps extends PromoteDeps {ghPost(path: string, body: unknown): Promise<void>;scoreboards(runId: number): Promise<Scoreboard[]>;+ matrixReceipt(runId: number): Promise<MatrixReceipt | null>;autoPromote: string;dryRun: boolean;}export type DailyResult =- | { action: 'off' | 'paused' | 'none'; why: string }+ | { action: 'off' | 'paused' | 'none' | 'held'; why: string }| { action: 'promoted' | 'backfilled'; why: string; promotion: Promotion }- | { action: 'soak_dispatched' | 'waiting'; why: string; version: string }+ | { action: 'soak_dispatched' | 'matrix_dispatched' | 'waiting'; why: string; version: string }| { action: 'dry_run'; why: string; would: string };export function parseRecord(message: string): Record<string, string> {@@ -116,6 +132,18 @@ async function ok(deps: PromoteDeps, args: string[]): Promise<string> {return r.stdout.trim();}+/** The `important:` class of the changelog.d fragment a merge commit added, or null. */+async function importanceOf(deps: PromoteDeps, commit: string): Promise<string | null> {+ const added = await deps.git(['diff-tree', '--no-commit-id', '--name-only', '-r', '--diff-filter=A', commit, '--', 'changelog.d']);+ if (added.code !== 0) return null;+ for (const path of added.stdout.split('\n').map(p => p.trim()).filter(p => /^changelog\.d\/[^/]+\.md$/.test(p) && !p.endsWith('/README.md'))) {+ const shown = await deps.git(['show', `${commit}:${path}`]);+ const cls = shown.code === 0 ? fragmentImportance(shown.stdout) : null;+ if (cls) return cls;+ }+ return null;+}+const isAncestor = async (deps: PromoteDeps, a: string, b: string) => (await deps.git(['merge-base', '--is-ancestor', a, b])).code === 0;async function treeOf(deps: PromoteDeps, commit: string): Promise<string | null> {@@ -166,34 +194,58 @@ export async function daily(deps: DailyDeps): Promise<DailyResult> {const since = new Date((Number.isFinite(promotedAt) ? promotedAt : deps.now().getTime()) - 14 * DAY_MS).toISOString().slice(0, 10);const q = `repo:${deps.repo} is:pr is:merged label:"${IMPORTANT_LABEL}" merged:>=${since}`;const found = await deps.ghApi(`search/issues?q=${encodeURIComponent(q)}&per_page=100`) as { items?: Array<{ number: number }> };- const important: Array<{ pr: number; version: string }> = [];+ const important: Array<{ pr: number; version: string; passesBreak: boolean }> = [];for (const { number } of found.items ?? []) {const pr = await deps.ghApi(`repos/${deps.repo}/pulls/${number}`) as { merge_commit_sha?: string | null };const sha = pr.merge_commit_sha;if (!sha || (await treeOf(deps, sha)) === null || await isAncestor(deps, sha, stableCommit)) continue;for (const r of [...releases].reverse()) {if (!(await isAncestor(deps, sha, r.commit))) continue;- important.push({ pr: number, version: r.version });+ const cls = await importanceOf(deps, sha);+ important.push({ pr: number, version: r.version, passesBreak: (BREAK_PASS_CLASSES as readonly string[]).includes(cls ?? '') });break;}}- const weekly = ageDays > WEEKLY_FLOOR_DAYS;- if (important.length === 0 && !weekly) {+ const weeklyDue = ageDays > WEEKLY_FLOOR_DAYS;+ if (important.length === 0 && !weeklyDue) {return { action: 'none', why: `no released ${IMPORTANT_LABEL} fix is missing from stable v${current.version}, and stable is ${ageDays.toFixed(1)} days old (floor ${WEEKLY_FLOOR_DAYS}).` };}+ const breakIssue = findBreakIssue(await deps.ghApi(breakIssueQuery(deps.repo)));+ const passing = breakIssue ? important.filter(i => i.passesBreak) : important;+ const weekly = weeklyDue && !breakIssue;+ if (breakIssue && passing.length === 0) {+ const why = `the upgrade matrix's break issue #${breakIssue.number} (${breakIssue.url}) is open, so stable v${current.version} holds; only a released important: security or data-loss fix may pass it${important.length ? ` (waiting: ${important.map(i => `#${i.pr}`).join(', ')})` : ''}.`;+ deps.log(`::warning::Daily stable check: ${why} Docs: docs/RELEASING.md#daily-stable-check`);+ return { action: 'held', why };+ }const reason = [- ...(important.length ? [`important: ${important.map(i => `#${i.pr} (v${i.version})`).join(', ')}`] : []),+ ...(passing.length ? [`important: ${passing.map(i => `#${i.pr} (v${i.version})`).join(', ')}`] : []),...(weekly ? [`weekly floor: stable v${current.version} is ${ageDays.toFixed(1)} days old`] : []),+ ...(breakIssue ? [`passing open upgrade break #${breakIssue.number} as a security or data-loss fix`] : []),].join('; ');deps.log(`[stable-daily] reason: ${reason}.`);+ const matrixFile = MATRIX_WORKFLOW.split('/').at(-1);+ const matrixRuns = (((await deps.ghApi(`repos/${deps.repo}/actions/workflows/${matrixFile}/runs?branch=master&per_page=100`)) as { workflow_runs?: MatrixRunRow[] }).workflow_runs ?? []).filter(isGateRun);+ const receipts = new Map<number, MatrixReceipt | null>();+ /** The newest completed gate run whose receipt tests this release, or null. */+ const matrixRunFor = async (version: string): Promise<{ run: MatrixRunRow; receipt: MatrixReceipt | null } | null> => {+ for (const run of matrixRuns.filter(r => r.status === 'completed')) {+ if (!receipts.has(run.id)) receipts.set(run.id, await deps.matrixReceipt(run.id));+ if (receiptNames(receipts.get(run.id)!, version)) return { run, receipt: receipts.get(run.id)! };+ }+ return null;+ };+ const activeMatrix = matrixRuns.find(r => r.status !== 'completed');+ const dispatchMatrix = (version: string) => deps.ghPost(`repos/${deps.repo}/actions/workflows/${matrixFile}/dispatches`, { ref: 'master', inputs: { candidate: `v${version}` } });+const runs = ((await deps.ghApi(`repos/${deps.repo}/actions/workflows/${soakPath.split('/').at(-1)}/runs?per_page=100`)) as { workflow_runs?: SoakRunRow[] }).workflow_runs ?? [];const runsByTree = new Map<string, SoakRunRow[]>();for (const run of runs) {const tree = await treeOf(deps, run.head_sha);if (tree) runsByTree.set(tree, [...(runsByTree.get(tree) ?? []), run]);}- const oldestImportant = important.reduce<string | null>((v, i) => (!v || compareVersions(i.version, v) < 0 ? i.version : v), null);+ const oldestImportant = passing.reduce<string | null>((v, i) => (!v || compareVersions(i.version, v) < 0 ? i.version : v), null);let failedSoak: { version: string; run: SoakRunRow } | null = null;for (const rel of releases) {if (!weekly && oldestImportant && compareVersions(rel.version, oldestImportant) < 0) break;@@ -208,8 +260,25 @@ export async function daily(deps: DailyDeps): Promise<DailyResult> {deps.log(`[stable-daily] not promotable: v${rel.version} soak ${run.html_url}: ${failures.join(', ')}.`);}if (!chosen) { failedSoak ??= completed[0] ? { version: rel.version, run: completed[0] } : null; continue; }- if (deps.dryRun) return { action: 'dry_run', why: reason, would: `promote v${rel.version} on soak ${chosen.html_url}` };- const promotion = await promote(deps, { version: rel.version, soakRunUrl: chosen.html_url, scoreboardUrl: '', rollback: false, reason: `scheduled daily check; ${reason}` });+ const matrix = await matrixRunFor(rel.version);+ if (!matrix) {+ if (activeMatrix) return { action: 'waiting', why: `${reason}; v${rel.version} passed its soak and upgrade matrix run ${activeMatrix.html_url} is still running.`, version: rel.version };+ if (deps.dryRun) return { action: 'dry_run', why: reason, would: `dispatch the upgrade matrix on v${rel.version}` };+ await dispatchMatrix(rel.version);+ return { action: 'matrix_dispatched', why: `${reason}; v${rel.version} passed its soak and has no upgrade matrix run yet.`, version: rel.version };+ }+ const judged = judgeMatrixRun(matrix.run, matrix.receipt, { version: rel.version, commit: rel.commit, tree: tree! }, breakIssue);+ if (!judged.ok) {+ const why = `${reason}; ${judged.why}, so stable holds and a maintainer decides.`;+ deps.log(`::warning::Daily stable check: ${why} Docs: docs/RELEASING.md#daily-stable-check`);+ return { action: 'held', why };+ }+ if (deps.dryRun) return { action: 'dry_run', why: reason, would: `promote v${rel.version} on soak ${chosen.html_url} and upgrade matrix ${matrix.run.html_url}` };+ const excused = judged.excused.length ? `; upgrade break #${breakIssue!.number} excused for ${judged.excused.join(', ')}` : '';+ const promotion = await promote(deps, {+ version: rel.version, soakRunUrl: chosen.html_url, scoreboardUrl: '', rollback: false, reason: `scheduled daily check; ${reason}${excused}`,+ matrixRunUrl: matrix.run.html_url, matrixExcuse: breakIssue,+ });return { action: 'promoted', why: reason, promotion };}@@ -220,8 +289,10 @@ export async function daily(deps: DailyDeps): Promise<DailyResult> {if (weekly) deps.log(`::warning::Daily stable check: stable v${current.version} is past the ${WEEKLY_FLOOR_DAYS}-day floor and no soak qualifies; a maintainer decides. Docs: docs/RELEASING.md#daily-stable-check`);return { action: 'waiting', why: `${reason}; v${failedSoak.version} already had a soak (${failedSoak.run.html_url}, ${failedSoak.run.conclusion}) that did not qualify, so no new soak is dispatched and a maintainer decides.`, version: newest.version };}- if (deps.dryRun) return { action: 'dry_run', why: reason, would: `dispatch the soak on v${newest.version}` };+ const matrixToo = !activeMatrix && !(await matrixRunFor(newest.version));+ if (deps.dryRun) return { action: 'dry_run', why: reason, would: `dispatch the soak${matrixToo ? ' and the upgrade matrix' : ''} on v${newest.version}` };await deps.ghPost(`repos/${deps.repo}/actions/workflows/${soakPath.split('/').at(-1)}/dispatches`, { ref: `v${newest.version}` });+ if (matrixToo) await dispatchMatrix(newest.version);return { action: 'soak_dispatched', why: reason, version: newest.version };}@@ -230,6 +301,7 @@ export function dailyDeps(env: NodeJS.ProcessEnv = process.env, cwd = process.cwreturn {...base,scheduled: scheduleAuthorized(env, base.repo),+ matrixReceipt: base.matrixReceipt!,autoPromote: env.STABLE_AUTO_PROMOTE ?? '',dryRun: env.DAILY_DRY_RUN === 'true',async ghPost(path, body) {diff --git a/scripts/stable-matrix.ts b/scripts/stable-matrix.tsnew file mode 100644index 0000000000..9518ea2cfb--- /dev/null+++ b/scripts/stable-matrix.ts@@ -0,0 +1,112 @@+/**+ * The upgrade matrix as a condition of stable promotion (stability plan item 3a, GST-2818).+ *+ * A forward promotion names one run of .github/workflows/upgrade-matrix.yml (`matrix_run` in the record and the+ * promote-stable.yml input). judgeMatrixRun() accepts it only when the run is a full-profile gate run (schedule+ * or dispatch, from master's harness, no fault injection) whose receipt (the upgrade-matrix-receipt artifact the+ * summary job writes) names exactly the release being promoted (tag, commit and tree) and was written by that+ * run's own harness, and when the run is green. The one way past a red run is the break path below.+ *+ * The daily check (scripts/stable-daily.ts) also holds stable while the matrix's own break issue is open+ * (ISSUE_MARKER, upgrade-matrix/marker.ts); any other issue with the label holds nothing. A security or data-loss+ * fix may pass an open break when its candidate's run fails only on from-versions that issue names: the break is+ * already known and the fix makes nothing worse elsewhere. Otherwise a maintainer promotes by hand+ * (promote-stable.yml), which the record shows as promoted_by.+ *+ * No dependencies beyond node: stable-daily.yml and promote-stable.yml install none.+ */+import { failingFrom, ISSUE_LABEL, ISSUE_MARKER, parseRedFrom } from './upgrade-matrix/marker.ts';++export const MATRIX_WORKFLOW = '.github/workflows/upgrade-matrix.yml';+export const MATRIX_RECEIPT_ARTIFACT = 'upgrade-matrix-receipt';+export const MATRIX_RECEIPT_SCHEMA = 'gbrain-upgrade-matrix-run/1';+/** The `important:` classes that may pass an open upgrade break (with a run that fails only on the from-versions it names). */+export const BREAK_PASS_CLASSES = ['security', 'data-loss'] as const;++export interface MatrixRunRow {+ id: number;+ status: string | null;+ conclusion: string | null;+ head_sha: string;+ head_branch?: string | null;+ event?: string | null;+ path?: string | null;+ html_url: string;+}++/** The run receipt (scripts/upgrade-matrix.ts buildSummary), as far as promotion reads it. */+export interface MatrixReceipt {+ schema?: string;+ harness?: { sha?: string };+ candidate?: { tag?: string | null; sha?: string; tree?: string };+ profile?: string;+ from?: Array<{ tag: string; roles?: string[] }>;+ skipped?: Array<{ role: string; reason?: string }>;+ status?: string;+ cells?: Array<{ id: string; from: string; verdict: string; fault?: string | null }>;+ red?: string[];+ holds?: string[];+ foreign_receipts?: string[];+}++export interface BreakIssue { number: number; url: string; from: string[] | null }++export type MatrixJudgement = { ok: true; why: string; excused: string[] } | { ok: false; why: string };++const normalise = (p: string) => p.trim().replace(/@.*$/, '').replace(/^\.\//, '');++/** Runs whose receipt can be gate evidence: master's harness, by schedule or dispatch (pull-request runs test their own harness). */+export const isGateRun = (run: MatrixRunRow) => (run.event === 'schedule' || run.event === 'workflow_dispatch') && run.head_branch === 'master';++/** A receipt from a harness that resolves the stable-window role (scripts/upgrade-matrix/versions.ts): it tested or skipped it. Older receipts never tested the stable releases users may still be on. */+const coversStableWindow = (r: MatrixReceipt) =>+ (r.from ?? []).some(f => (f.roles ?? []).includes('stable-window')) || (r.skipped ?? []).some(s => s.role === 'stable-window');++/** Does this receipt test this release as a gate run (full profile with the stable window, no fault injection)? A candidate tag alone picks the run; judgeMatrixRun checks the rest. */+export const receiptNames = (r: MatrixReceipt | null, version: string) =>+ !!r && r.candidate?.tag === `v${version}` && r.profile === 'full' && coversStableWindow(r) && !(r.cells ?? []).some(c => c.fault);++/**+ * Whether a matrix run qualifies the release for stable. `excuse` is the break path: the open break issue's named+ * from-versions, given only for a security or data-loss candidate while that issue is open.+ */+export function judgeMatrixRun(run: MatrixRunRow, receipt: MatrixReceipt | null, release: { version: string; commit: string; tree: string }, excuse?: BreakIssue | null): MatrixJudgement {+ const url = run.html_url;+ if (run.path && normalise(run.path) !== MATRIX_WORKFLOW) return { ok: false, why: `run ${url} ran ${normalise(run.path)}, not the upgrade matrix ${MATRIX_WORKFLOW}` };+ if (!isGateRun(run)) return { ok: false, why: `upgrade matrix run ${url} is a ${run.event ?? 'unknown'} run on ${run.head_branch ?? 'an unknown branch'}; only a schedule or dispatch run of master's harness is gate evidence` };+ if (run.status !== 'completed') return { ok: false, why: `upgrade matrix run ${url} is ${run.status ?? 'in an unknown state'}, not completed` };+ if (!receipt) return { ok: false, why: `upgrade matrix run ${url} has no ${MATRIX_RECEIPT_ARTIFACT} artifact to read` };+ if (receipt.schema !== MATRIX_RECEIPT_SCHEMA) return { ok: false, why: `upgrade matrix run ${url}'s receipt is not a ${MATRIX_RECEIPT_SCHEMA} receipt` };+ if (receipt.harness?.sha !== run.head_sha) return { ok: false, why: `upgrade matrix run ${url}'s receipt names harness ${receipt.harness?.sha ?? 'none'}, not the run's commit ${run.head_sha}` };+ const c = receipt.candidate ?? {};+ if (c.tag !== `v${release.version}` || c.sha !== release.commit || c.tree !== release.tree) {+ return { ok: false, why: `upgrade matrix run ${url} tested ${c.tag ?? 'an untagged commit'} (commit ${c.sha ?? '?'}, tree ${c.tree ?? '?'}), not v${release.version} (commit ${release.commit}, tree ${release.tree})` };+ }+ if (receipt.profile !== 'full') return { ok: false, why: `upgrade matrix run ${url} ran the ${receipt.profile ?? 'unknown'} profile, not the full from-version set` };+ if (!coversStableWindow(receipt)) return { ok: false, why: `upgrade matrix run ${url} predates the stable-window from-versions, so it never tested upgrades from the stable releases users may still be on` };+ if ((receipt.cells ?? []).some(cell => cell.fault)) return { ok: false, why: `upgrade matrix run ${url} injected a fault (a proof run), so it qualifies nothing` };+ if (receipt.status === 'green' && run.conclusion === 'success') return { ok: true, why: `upgrade matrix run ${url} is green on v${release.version}`, excused: [] };+ const failing = failingFrom(receipt);+ const what = `upgrade matrix run ${url} is ${receipt.status ?? run.conclusion ?? 'not green'} on v${release.version}${failing.from.length ? ` (failing from ${failing.from.join(', ')})` : ''}${failing.door ? ' with a failing door' : ''}`;+ if (!excuse) return { ok: false, why: what };+ if (!excuse.from) return { ok: false, why: `${what}, and break issue #${excuse.number} names no from-versions, so nothing is excused` };+ if ((receipt.foreign_receipts ?? []).length || failing.door || failing.from.length === 0) return { ok: false, why: `${what}, which the break issue cannot excuse` };+ const outside = failing.from.filter(f => !excuse.from!.includes(f));+ if (outside.length) return { ok: false, why: `${what}; break issue #${excuse.number} names only ${excuse.from.join(', ')}, so ${outside.join(', ')} is a new break` };+ return { ok: true, why: `${what}, all named by break issue #${excuse.number}`, excused: failing.from };+}++/** The matrix's own open break issue (labelled and carrying ISSUE_MARKER), or null. A labelled issue without the marker holds nothing. */+export function findBreakIssue(issues: unknown): BreakIssue | null {+ const rows = Array.isArray(issues) ? issues as Array<{ number?: number; html_url?: string; body?: string | null; state?: string; pull_request?: unknown }> : [];+ const hit = rows.find(i => !i.pull_request && i.state !== 'closed' && typeof i.number === 'number' && String(i.body ?? '').includes(ISSUE_MARKER));+ return hit ? { number: hit.number!, url: hit.html_url ?? `#${hit.number}`, from: parseRedFrom(String(hit.body ?? '')) } : null;+}++export const breakIssueQuery = (repo: string) => `repos/${repo}/issues?state=open&labels=${ISSUE_LABEL}&per_page=100`;++/** The `important:` class in a changelog.d fragment's front matter, or null. */+export function fragmentImportance(text: string): string | null {+ const m = /^---\r?\n([\s\S]*?)\r?\n---/.exec(text);+ return m ? /^important:\s*([a-z-]+)\s*$/m.exec(m[1]!)?.[1] ?? null : null;+}diff --git a/scripts/upgrade-matrix.ts b/scripts/upgrade-matrix.tsindex a03da238fd..6329dfbf27 100644--- a/scripts/upgrade-matrix.ts+++ b/scripts/upgrade-matrix.ts@@ -18,7 +18,8 @@import { appendFileSync, existsSync, mkdirSync, readFileSync, readdirSync, writeFileSync } from 'node:fs';import { execFileSync, spawnSync } from 'node:child_process';import { join, resolve } from 'node:path';-import { fetchReleases, hasEnoughHistory, RELEASE_TAG, resolveFromVersions, type Candidate, type FromVersion, type Profile, type Release } from './upgrade-matrix/versions.ts';+import { fetchReleases, hasEnoughHistory, RELEASE_TAG, resolveFromVersions, stableWindow, type Candidate, type FromVersion, type Profile, type Release, type StableRecordTag } from './upgrade-matrix/versions.ts';+import { failingFrom, ISSUE_LABEL, ISSUE_MARKER, renderRedFrom } from './upgrade-matrix/marker.ts';import { cellId, runCell, type BrainShape, type Engine, type Fault, type Keys } from './upgrade-matrix/cell.ts';import { CHECKS, loadKnownExceptions, summarize, type CellSummary } from './upgrade-matrix/verdict.ts';import { AGENT_FILED_LABEL } from './lib/gh-issue.ts';@@ -26,8 +27,7 @@ import { AGENT_FILED_LABEL } from './lib/gh-issue.ts';const REPO = 'garrytan/gbrain';const ROOT = resolve(import.meta.dir, '..');const BT = '\x60';-export const ISSUE_MARKER = '<!-- gbrain-upgrade-matrix:nightly -->';-export const ISSUE_LABEL = 'upgrade-break';+export { ISSUE_LABEL, ISSUE_MARKER };const argv = process.argv.slice(2);const flag = (name: string, fallback?: string) => { const i = argv.indexOf(name); return i >= 0 && i + 1 < argv.length ? argv[i + 1]! : fallback; };@@ -61,9 +61,12 @@ async function plan(): Promise<void> {const profile = flag('--profile', 'full') as Profile;if (profile !== 'full' && profile !== 'pr') throw new Error('--profile takes full or pr');const input = (flag('--candidate', '') ?? '').trim();+ const stableRecords = profile === 'full' ? stableRecordTags() : [];const releases = await fetchReleases(REPO, rs => {const newest = rs[0];- return !!newest && hasEnoughHistory(rs, candidateHint(rs, input) ?? newest);+ if (!newest) return false;+ const hint = candidateHint(rs, input) ?? newest;+ return hasEnoughHistory(rs, hint) && stableWindow(stableRecords, hint).every(v => rs.some(r => r.version === v));});if (!releases.length) throw new Error('no published releases found');let candidate: Candidate;@@ -86,7 +89,7 @@ async function plan(): Promise<void> {return { tag, version: m[1]!, published_at: releases.find(r => r.tag === tag)?.published_at ?? '', roles: ['explicit' as never] };});} else {- const resolved = resolveFromVersions(releases, candidate, { profile, stableVersion: stableVersion() });+ const resolved = resolveFromVersions(releases, candidate, { profile, stableVersion: stableVersion(), stableRecords });from = resolved.from;skipped = resolved.skipped;}@@ -118,6 +121,17 @@ function stableVersion(): string | null {try { return commitVersion(ensureCommit('stable')); } catch { return null; }}+/** Every stable-v<version> record tag on origin with its creation time (the promotion), fetched first. */+function stableRecordTags(): StableRecordTag[] {+ execFileSync('git', ['-C', ROOT, 'fetch', '--quiet', '--no-tags', '--force', 'origin', '+refs/tags/stable-v*:refs/tags/stable-v*'], { stdio: ['ignore', 'pipe', 'pipe'] });+ const out = git('for-each-ref', '--format=%(refname:strip=2)%09%(creatordate:iso-strict)', 'refs/tags/stable-v*');+ return out.split('\n').filter(Boolean).flatMap(line => {+ const [name, date] = line.split('\t');+ const m = /^stable-v(\d+\.\d+\.\d+(?:\.\d+)?)$/.exec(name ?? '');+ return m && date ? [{ version: m[1]!, promoted_at: new Date(date).toISOString() }] : [];+ });+}+function candidateHint(releases: Release[], input: string): { version: string; published_at: string } | null {if (!input) return null;const rel = releases.find(r => r.tag === input);@@ -228,7 +242,8 @@ async function issue(): Promise<void> {return res.status === 204 ? null : res.json();};const existing = (await api('GET', '/issues?state=all&labels=' + ISSUE_LABEL + '&per_page=100') as any[]).find(i => !i.pull_request && String(i.body ?? '').includes(ISSUE_MARKER));- const body = ISSUE_MARKER + '\n' + renderSummary(s) + '\n\nRun: ' + (s.harness.run_url ?? '(local)') + '\n\nThis issue carries the ' + BT + ISSUE_LABEL + BT + ' label, which holds stable promotion while it is open. The nightly matrix closes it when a later run is green.';+ const body = ISSUE_MARKER + '\n' + renderRedFrom(failingFrom(s).from) + '\n' + renderSummary(s) + '\n\nRun: ' + (s.harness.run_url ?? '(local)')+ + '\n\nWhile this issue is open the daily stable check holds stable (docs/RELEASING.md#daily-stable-check); only a security or data-loss fix that is green on every from-version this issue does not name may pass it. The nightly matrix closes it when a later run is green.';if (s.status === 'red') {const title = 'Upgrade matrix red: upgrading to ' + (s.candidate.tag ?? s.candidate.sha.slice(0, 12)) + ' fails ' + s.red.length + ' cell(s)';if (dry) { console.log((existing ? 'update #' + existing.number : 'create') + ': ' + title + '\n' + body); return; }diff --git a/scripts/upgrade-matrix/marker.ts b/scripts/upgrade-matrix/marker.tsnew file mode 100644index 0000000000..0c85a1f20c--- /dev/null+++ b/scripts/upgrade-matrix/marker.ts@@ -0,0 +1,47 @@+/**+ * The upgrade matrix's own break issue (GBRA-224 W2, stability plan item 3a): the nightly run opens or+ * updates one issue carrying ISSUE_MARKER and the ISSUE_LABEL label when it is red, and closes it when a later+ * run is green. scripts/stable-daily.ts holds stable while that issue is open; any other issue with the label+ * holds nothing. The issue names the from-versions that failed on a RED_FROM_MARKER line, which is what lets a+ * security or data-loss fix pass an open break when it is green on every other from-version.+ * No imports: stable-daily.yml installs no dependencies.+ */+export const ISSUE_MARKER = '<!-- gbrain-upgrade-matrix:nightly -->';+export const ISSUE_LABEL = 'upgrade-break';+const RED_FROM_PREFIX = '<!-- gbrain-upgrade-matrix:red-from ';++/** A run summary's shape as far as the failing from-versions go (scripts/upgrade-matrix.ts buildSummary). */+export interface RunSummaryCells {+ cells?: Array<{ id: string; from: string; verdict: string; fault?: string | null }>;+ red?: string[];+ holds?: string[];+}++/** Cell ids are `<from tag>__<engine>__<keys>__<brain>` (cell.ts cellId); a door id (`door:<name>`) has no from-version. */+const fromOfId = (id: string): string | null => (id.startsWith('door:') ? null : id.split('__')[0] || null);++/**+ * The from-versions with a red or held cell, sorted, and whether a door failed or held (a door has no+ * from-version, so nothing can excuse it).+ */+export function failingFrom(s: RunSummaryCells): { from: string[]; door: boolean } {+ const out = new Set<string>();+ let door = false;+ for (const id of [...(s.red ?? []), ...(s.holds ?? [])]) {+ const cell = s.cells?.find(c => c.id === id);+ const from = cell ? cell.from : fromOfId(id);+ if (from === null) door = true;+ else out.add(from);+ }+ return { from: [...out].sort(), door };+}++export const renderRedFrom = (from: readonly string[]) => RED_FROM_PREFIX + from.join(',') + ' -->';++/** The from-versions an issue body names, or null when it names none (an issue written before this line existed). */+export function parseRedFrom(body: string): string[] | null {+ const line = body.split('\n').find(l => l.startsWith(RED_FROM_PREFIX) && l.trimEnd().endsWith('-->'));+ if (!line) return null;+ const from = line.slice(RED_FROM_PREFIX.length).replace(/-->\s*$/, '').split(',').map(t => t.trim()).filter(t => /^v\d+\.\d+\.\d+(\.\d+)?$/.test(t));+ return from.length ? from : null;+}diff --git a/scripts/upgrade-matrix/versions.ts b/scripts/upgrade-matrix/versions.tsindex 5827dfa668..3ead67b2f1 100644--- a/scripts/upgrade-matrix/versions.ts+++ b/scripts/upgrade-matrix/versions.ts@@ -4,16 +4,23 @@* Resolved once per run from published release tags and recorded in the run* receipt: N-1, N-10 and N-50 (the 1st, 10th and 50th newest release older* than the candidate), the release published closest to 30 days before the- * candidate, and the current stable tag when one exists and is older. The- * pull-request profile keeps N-1 and N-50. A role that cannot be filled (too- * few older releases, stable missing or not older) is recorded as skipped- * with the reason, never silently dropped.+ * candidate, the current stable tag when one exists and is older, and the+ * stable releases in the promise window (stableWindow: the last 8 plus one per+ * older month, back 6 months). The pull-request profile keeps N-1 and N-50. A+ * role that cannot be filled (too few older releases, stable missing or not+ * older) is recorded as skipped with the reason, never silently dropped.*/import { isNewerVersion, isValidVersionString } from '../../src/core/semver.ts';-export type FromRole = 'N-1' | 'N-10' | 'N-50' | '30d' | 'stable';+export type FromRole = 'N-1' | 'N-10' | 'N-50' | '30d' | 'stable' | 'stable-window';export type Profile = 'full' | 'pr';-export const PROFILE_ROLES: Record<Profile, FromRole[]> = { full: ['N-1', 'N-10', 'N-50', '30d', 'stable'], pr: ['N-1', 'N-50'] };+export const PROFILE_ROLES: Record<Profile, FromRole[]> = { full: ['N-1', 'N-10', 'N-50', '30d', 'stable', 'stable-window'], pr: ['N-1', 'N-50'] };++/** A stable record tag (stable-v<version>) and when it was created, which is when that release was promoted. */+export interface StableRecordTag { version: string; promoted_at: string }+/** How far back a stable user is still promised an upgrade (docs/STABILITY.md), and how many recent stable releases are all kept. */+export const STABLE_WINDOW_DAYS = 183;+export const STABLE_WINDOW_RECENT = 8;export interface Release { tag: string; version: string; published_at: string }export interface Candidate { tag: string | null; sha: string; tree: string; version: string; published_at: string }@@ -33,7 +40,7 @@ export function olderReleases(releases: readonly Release[], candidateVersion: stexport function resolveFromVersions(releases: readonly Release[],candidate: Pick<Candidate, 'version' | 'published_at'>,- opts: { profile: Profile; stableVersion?: string | null },+ opts: { profile: Profile; stableVersion?: string | null; stableRecords?: readonly StableRecordTag[] },): { from: FromVersion[]; skipped: SkippedRole[] } {const older = olderReleases(releases, candidate.version);const picked = new Map<string, FromVersion>();@@ -56,6 +63,10 @@ export function resolveFromVersions(if (!best || d < Math.abs(Date.parse(best.published_at) - target)) best = r;}take(role, best, 'no published release is older than the candidate');+ } else if (role === 'stable-window') {+ const window = stableWindow(opts.stableRecords ?? [], candidate);+ if (window.length === 0) skipped.push({ role, reason: 'no stable release older than the candidate is in the window' });+ for (const v of window) take(role, older.find(r => r.version === v), 'stable ' + v + ' has no published release');} else {const stable = opts.stableVersion ?? null;if (!stable) { skipped.push({ role, reason: 'no stable tag exists' }); continue; }@@ -66,6 +77,29 @@ export function resolveFromVersions(return { from: [...picked.values()], skipped };}+/**+ * The stable releases a stable user may still be upgrading from: every stable release older than the candidate+ * that was stable at some point in the last STABLE_WINDOW_DAYS before the candidate (promoted inside the window, or+ * the one that was stable when the window opened), capped at the newest STABLE_WINDOW_RECENT by promotion plus the+ * newest one of each older calendar month (UTC). Newest first.+ */+export function stableWindow(records: readonly StableRecordTag[], candidate: Pick<Candidate, 'version' | 'published_at'>): string[] {+ const horizon = Date.parse(candidate.published_at) - STABLE_WINDOW_DAYS * DAY_MS;+ const older = records+ .filter(r => isValidVersionString(r.version) && isNewerVersion(r.version, candidate.version) && Number.isFinite(Date.parse(r.promoted_at)))+ .sort((a, b) => Date.parse(b.promoted_at) - Date.parse(a.promoted_at));+ const inWindow = older.filter((_, i) => i === 0 || Date.parse(older[i - 1]!.promoted_at) >= horizon);+ const months = new Set<string>();+ const out: string[] = [];+ inWindow.forEach((r, i) => {+ const month = r.promoted_at.slice(0, 7);+ if (i < STABLE_WINDOW_RECENT) out.push(r.version);+ else if (!months.has(month)) out.push(r.version);+ if (i >= STABLE_WINDOW_RECENT) months.add(month);+ });+ return [...new Set(out)];+}+function token(): string | undefined {return process.env.GITHUB_TOKEN || process.env.GH_TOKEN || undefined;}diff --git a/test/scripts/promote-stable.test.ts b/test/scripts/promote-stable.test.tsindex 5930ec0285..14282ffb7e 100644--- a/test/scripts/promote-stable.test.ts+++ b/test/scripts/promote-stable.test.ts@@ -22,6 +22,7 @@ import {checkOrder, checkSoakRun, compareVersions, RELEASE_NOTES_MAX, rolledUpNotes, mergeKnownBad, mergeNotPromotable, normaliseWorkflowPath, parseRunUrl, parseTagMessage, processDeps, promote, Refusal, renderRecord, renderSummary, renderTagMessage, STABLE_RECORD_MAX_BYTES,type GitResult, type PromoteDeps, type PromoteInputs, type SoakRun, type TagFields,} from '../../scripts/promote-stable.ts';+import type { MatrixReceipt } from '../../scripts/stable-matrix.ts';import { offeredVersion } from '../../src/core/self-upgrade.ts';import { RECORD_MAX_BYTES } from '../../src/core/stable-channel.ts';import { parseStableRecord } from '../../src/core/stable-channel.ts';@@ -732,6 +733,64 @@ describe('stable.json bridge for a stable promoted before the evidence record (G});});+describe('promote: the upgrade matrix run (stability plan 3a)', () => {+ const MATRIX_URL = `https://github.com/${REPO}/actions/runs/777`;+ const HARNESS = sha('f');+ const greenRun = { id: 777, status: 'completed', conclusion: 'success', head_sha: HARNESS, head_branch: 'master', event: 'schedule', path: '.github/workflows/upgrade-matrix.yml' };+ const greenReceipt = (): MatrixReceipt => ({+ schema: 'gbrain-upgrade-matrix-run/1', harness: { sha: HARNESS }, candidate: { tag: 'v0.60.361.0', sha: sha('b'), tree: sha('2') }, profile: 'full',+ status: 'green', cells: [], red: [], holds: [], foreign_receipts: [], from: [{ tag: 'v0.60.360.0', roles: ['N-1', 'stable-window'] }], skipped: [],+ });+ function withMatrix(run: Record<string, unknown>, receipt: MatrixReceipt | null, setup: Parameters<typeof harness>[0] = {}) {+ const h = harness(setup);+ const ghApi = h.deps.ghApi;+ h.deps.ghApi = async path => (path.endsWith('/actions/runs/777') ? (h.api.push(path), run) : ghApi(path));+ h.deps.matrixReceipt = async id => (id === 777 ? structuredClone(receipt) : null);+ return h;+ }++ test('a green gate run on the exact tag and tree lands in stable.json and the tag message, and round-trips', async () => {+ const h = withMatrix(greenRun, greenReceipt());+ const p = await promote(h.deps, inputs({ matrixRunUrl: MATRIX_URL }));+ expect(p.matrixRun).toBe(MATRIX_URL);+ expect(JSON.parse(h.repo.records![h.repo.recordBranch!]!)).toMatchObject({ matrix_run: MATRIX_URL });+ expect(parseStableRecord(h.repo.records![h.repo.recordBranch!]!, '0.60.361.0')).toMatchObject({ ok: true, record: { matrix_run: MATRIX_URL } });+ expect(h.repo.tags.stable!.message).toContain(`soak_run: ${RUN_URL}\nscoreboard: ${RUN_URL}\nmatrix_run: ${MATRIX_URL}\npromoted_by: `);+ expect(parseTagMessage(h.repo.tags.stable!.message!)).toMatchObject({ matrixRun: MATRIX_URL });+ expect(renderSummary(p)).toContain(MATRIX_URL);+ });++ test('a maintainer may promote without a matrix run, recorded as matrix_run null with the maintainer as promoted_by', async () => {+ const h = harness();+ await promote(h.deps, inputs());+ expect(JSON.parse(h.repo.records![h.repo.recordBranch!]!)).toMatchObject({ matrix_run: null, promoted_by: 'alice-example' });+ expect(h.logs.some(l => l.includes("promoted on alice-example's decision"))).toBe(true);+ });++ test('a red, foreign, unfinished, pull-request, partial or unreadable run is refused and nothing moves', async () => {+ const red = { ...greenReceipt(), status: 'red', red: ['v0.60.241.0__postgres__keyed__repo'], cells: [{ id: 'v0.60.241.0__postgres__keyed__repo', from: 'v0.60.241.0', verdict: 'fail' }] };+ const cases: Array<[Record<string, unknown>, MatrixReceipt | null, RegExp, string?]> = [+ [{ ...greenRun, conclusion: 'failure' }, red, /is red on v0\.60\.361\.0 \(failing from v0\.60\.241\.0\)/],+ [greenRun, { ...greenReceipt(), candidate: { tag: 'v0.60.360.0', sha: sha('a'), tree: sha('1') } }, /tested v0\.60\.360\.0 .*not v0\.60\.361\.0/],+ [greenRun, { ...greenReceipt(), candidate: { tag: 'v0.60.361.0', sha: sha('b'), tree: sha('9') } }, /tree 9+\), not v0\.60\.361\.0/],+ [greenRun, { ...greenReceipt(), harness: { sha: sha('e') } }, /receipt names harness e+, not the run's commit f+/],+ [{ ...greenRun, event: 'pull_request', head_branch: 'fix/x' }, greenReceipt(), /is a pull_request run on fix\/x/],+ [{ ...greenRun, status: 'in_progress', conclusion: null }, greenReceipt(), /is in_progress, not completed/],+ [{ ...greenRun, path: '.github/workflows/soak.yml' }, greenReceipt(), /ran \.github\/workflows\/soak\.yml, not the upgrade matrix/],+ [greenRun, { ...greenReceipt(), profile: 'explicit' }, /ran the explicit profile/],+ [greenRun, { ...greenReceipt(), from: [{ tag: 'v0.60.360.0', roles: ['N-1'] }], skipped: [] }, /predates the stable-window from-versions/],+ [greenRun, null, /has no upgrade-matrix-receipt artifact/],+ [greenRun, greenReceipt(), /matrix_run_url names a run of acme-example\/gbrain/, 'https://github.com/acme-example/gbrain/actions/runs/777'],+ [greenRun, greenReceipt(), /matrix_run_url names attempt 2/, `${MATRIX_URL}/attempts/2`],+ ];+ for (const [run, receipt, pattern, url] of cases) {+ const h = withMatrix(run, receipt);+ await refusedAsync(promote(h.deps, inputs({ matrixRunUrl: url ?? MATRIX_URL })), pattern);+ expect(h.calls.filter(c => c.startsWith('push') || c.startsWith('tag -a'))).toEqual([]);+ }+ });+});+describe('promote-stable.yml', () => {const workflowYml = readFileSync(join(ROOT, '.github/workflows/promote-stable.yml'), 'utf8');const wf = load(workflowYml) as {@@ -744,8 +803,8 @@ describe('promote-stable.yml', () => {test('dispatch only, with the decided inputs, one serial group and scoped permissions', () => {expect(Object.keys(wf.on)).toEqual(['workflow_dispatch']);const i = wf.on.workflow_dispatch!.inputs;- expect(Object.keys(i)).toEqual(['version', 'soak_run_url', 'scoreboard_url', 'rollback', 'reason', 'withdraw', 'not_promotable', 'not_promotable_reason', 'not_promotable_fixed_in', 'not_promotable_remove']);- for (const k of ['withdraw', 'not_promotable', 'not_promotable_reason', 'not_promotable_fixed_in', 'not_promotable_remove']) expect(i[k]).toMatchObject({ required: false, type: 'string', default: '' });+ expect(Object.keys(i)).toEqual(['version', 'soak_run_url', 'matrix_run_url', 'scoreboard_url', 'rollback', 'reason', 'withdraw', 'not_promotable', 'not_promotable_reason', 'not_promotable_fixed_in', 'not_promotable_remove']);+ for (const k of ['matrix_run_url', 'withdraw', 'not_promotable', 'not_promotable_reason', 'not_promotable_fixed_in', 'not_promotable_remove']) expect(i[k]).toMatchObject({ required: false, type: 'string', default: '' });expect([i.version!.required, i.soak_run_url!.required, i.scoreboard_url!.required]).toEqual([true, true, false]);expect(i.rollback).toMatchObject({ type: 'boolean', default: false });expect(wf.concurrency).toEqual({ group: 'promote-stable', 'cancel-in-progress': false });@@ -760,7 +819,7 @@ describe('promote-stable.yml', () => {PROMOTE_VERSION: '${{ inputs.version }}', PROMOTE_SOAK_RUN_URL: '${{ inputs.soak_run_url }}', PROMOTE_SCOREBOARD_URL: '${{ inputs.scoreboard_url }}',PROMOTE_ROLLBACK: '${{ inputs.rollback }}', PROMOTE_REASON: '${{ inputs.reason }}', PROMOTE_WITHDRAW: '${{ inputs.withdraw }}',PROMOTE_NOT_PROMOTABLE: '${{ inputs.not_promotable }}', PROMOTE_NOT_PROMOTABLE_REASON: '${{ inputs.not_promotable_reason }}', PROMOTE_NOT_PROMOTABLE_FIXED_IN: '${{ inputs.not_promotable_fixed_in }}',- PROMOTE_NOT_PROMOTABLE_REMOVE: '${{ inputs.not_promotable_remove }}',+ PROMOTE_NOT_PROMOTABLE_REMOVE: '${{ inputs.not_promotable_remove }}', PROMOTE_MATRIX_RUN_URL: '${{ inputs.matrix_run_url }}',});for (const s of steps) expect(s.run ?? '').not.toContain('${{');const releaseYml = readFileSync(join(ROOT, '.github/workflows/release.yml'), 'utf8');diff --git a/test/scripts/stable-daily.test.ts b/test/scripts/stable-daily.test.tsindex 4ef785d989..1882aaa197 100644--- a/test/scripts/stable-daily.test.ts+++ b/test/scripts/stable-daily.test.ts@@ -18,6 +18,7 @@ import { join } from 'node:path';import { load } from 'js-yaml';import { promote, renderTagMessage, scheduleAuthorized, Refusal, type GitResult } from '../../scripts/promote-stable.ts';import { daily, integrityFailures, parseRecord, shapeFailures, type DailyDeps, type Scoreboard, type SoakRunRow } from '../../scripts/stable-daily.ts';+import type { MatrixReceipt, MatrixRunRow } from '../../scripts/stable-matrix.ts';import { parseStableRecord } from '../../src/core/stable-channel.ts';const ROOT = join(import.meta.dir, '..', '..');@@ -40,8 +41,13 @@ interface World {master?: number;releases: Record<string, number>;stable: { version: string; commit: number; message: string };- importantPrs?: Array<{ number: number; merge: number }>;+ /** `important`: the class its changelog.d fragment's front matter names. */+ importantPrs?: Array<{ number: number; merge: number; important?: string }>;runs?: Array<SoakRunRow & { boards?: Scoreboard[] }>;+ /** Upgrade matrix runs, newest first; default: one green gate run on every release (MATRIX_ID + its commit). */+ matrix?: Array<MatrixRunRow & { receipt?: MatrixReceipt | null }>;+ /** Open issues with the upgrade-break label (the issues query's answer). */+ breakIssues?: unknown[];autoPromote?: string;dryRun?: boolean;scheduled?: boolean;@@ -102,6 +108,14 @@ function world(w: World) {tags[name] = { commit: commitOf(commit)!, object: `tagobj-${name}-new`, message: rest[rest.indexOf('-m') + 1] };return ok();}+ if (cmd === 'diff-tree') {+ const pr = (w.importantPrs ?? []).find(p => p.important && sha(p.merge) === rest.at(-3));+ return pr ? ok(`changelog.d/fix-${pr.number}.md`) : ok();+ }+ if (cmd === 'show' && rest[0]!.includes(':changelog.d/')) {+ const pr = (w.importantPrs ?? []).find(p => rest[0] === `${sha(p.merge)}:changelog.d/fix-${p.number}.md`);+ return pr ? ok(`---\nimportant: ${pr.important}\n---\n**A fix.**\n`) : fail;+ }if (cmd === 'show') return ok(rest[0]!.replace(/^refs\/tags\/v/, '').replace(/:VERSION$/, ''));if (cmd === 'merge-base') {const a = commitOf(rest[1]!); const b = commitOf(rest[2]!);@@ -117,6 +131,7 @@ function world(w: World) {return fail;};const runs = w.runs ?? [];+ const matrix = w.matrix ?? Object.values(w.releases).map(c => matrixRun(MATRIX_ID + c, c, w.releases));const deps: DailyDeps = {repo: REPO,actor: 'github-actions[bot]',@@ -127,6 +142,14 @@ function world(w: World) {async ghApi(path) {api.push(path);if (path.startsWith('search/issues')) return { items: (w.importantPrs ?? []).map(p => ({ number: p.number })) };+ if (path.includes('/issues?')) return w.breakIssues ?? [];+ if (path.includes('/actions/workflows/upgrade-matrix.yml/runs')) {+ const q = new URLSearchParams(path.split('?')[1] ?? '');+ const rows = matrix.filter(r => !q.get('branch') || r.head_branch === q.get('branch'));+ return { workflow_runs: rows.slice(0, Number(q.get('per_page') ?? 30)).map(({ receipt: _, ...run }) => run) };+ }+ const matrixHit = matrix.find(r => path.endsWith(`/actions/runs/${r.id}`));+ if (matrixHit) { const { receipt: _, ...run } = matrixHit; return { ...run, path: '.github/workflows/upgrade-matrix.yml' }; }const pull = /\/pulls\/(\d+)$/.exec(path);if (pull) return { merge_commit_sha: sha(w.importantPrs!.find(p => p.number === Number(pull[1]))!.merge) };if (path.includes('/actions/workflows/')) return { workflow_runs: runs };@@ -146,10 +169,27 @@ function world(w: World) {async ghPost(path, body) { posts.push({ path, body }); },async ghApiWrite() { return {}; },async scoreboards(id) { return runs.find(r => r.id === id)?.boards ?? CLEAN; },+ async matrixReceipt(id) { const hit = matrix.find(r => r.id === id); return hit ? hit.receipt ?? null : null; },};return { deps, posts, pushes, api, logs, tags, record: () => (record.head ? objects[record.head] : undefined) };}+const MATRIX_ID = 700;+const HARNESS = 'f'.repeat(40);+/** A finished gate run of the upgrade matrix on the release at `commit`; green unless cells say otherwise. */+function matrixRun(id: number, commit: number, releases: Record<string, number>, over: Partial<MatrixRunRow> = {}, receipt: Partial<MatrixReceipt> = {}): MatrixRunRow & { receipt: MatrixReceipt } {+ const version = Object.entries(releases).find(([, c]) => c === commit)![0];+ const red = receipt.red ?? [];+ return {+ id, status: 'completed', conclusion: red.length ? 'failure' : 'success', head_sha: HARNESS, head_branch: 'master', event: 'schedule',+ html_url: `https://github.com/${REPO}/actions/runs/${id}`, ...over,+ receipt: {+ schema: 'gbrain-upgrade-matrix-run/1', harness: { sha: HARNESS }, candidate: { tag: `v${version}`, sha: sha(commit), tree: tree(commit) }, profile: 'full',+ status: red.length ? 'red' : 'green', cells: [], red, holds: [], foreign_receipts: [], from: [{ tag: 'v1.0.1', roles: ['N-1', 'stable', 'stable-window'] }], skipped: [], ...receipt,+ },+ };+}+const soak = (id: number, commit: number, over: Partial<SoakRunRow> & { boards?: Scoreboard[] } = {}) =>({ id, status: 'completed', conclusion: 'success', head_sha: sha(commit), html_url: `https://github.com/${REPO}/actions/runs/${id}`, ...over });@@ -173,7 +213,7 @@ describe('scheduleAuthorized', () => {test('a scheduled promotion skips the role read and records the schedule as the promoter; a dispatch keeps the maintainer check', async () => {const w = world(base({ runs: [soak(9, 2)] }));- const p = await promote(w.deps, { version: '1.0.2', soakRunUrl: `https://github.com/${REPO}/actions/runs/9`, scoreboardUrl: '', rollback: false, reason: '' });+ const p = await promote(w.deps, { version: '1.0.2', soakRunUrl: `https://github.com/${REPO}/actions/runs/9`, scoreboardUrl: '', rollback: false, reason: '', matrixRunUrl: `https://github.com/${REPO}/actions/runs/${MATRIX_ID + 2}` });expect(w.api.some(a => a.endsWith('/permission'))).toBe(false);expect(p.promotedBy).toBe('schedule (.github/workflows/stable-daily.yml)');const d = world(base({ runs: [soak(9, 2)], scheduled: false }));@@ -183,6 +223,16 @@ describe('scheduleAuthorized', () => {});});+describe('a scheduled promotion names a matrix run', () => {+ test('promote() refuses a scheduled forward promotion without one, before any tag moves', async () => {+ const w = world(base({ runs: [soak(9, 2)] }));+ const error = await promote(w.deps, { version: '1.0.2', soakRunUrl: `https://github.com/${REPO}/actions/runs/9`, scoreboardUrl: '', rollback: false, reason: '' }).then(() => null, e => e);+ expect(error).toBeInstanceOf(Refusal);+ expect((error as Error).message).toMatch(/a scheduled promotion needs a green upgrade matrix run on v1\.0\.2's exact tree/);+ expect(w.pushes).toEqual([]);+ });+});+describe('integrity gates', () => {test('every integrity gate must pass in every cell; latency gates stay record-only; no data or no cells fails closed', () => {expect(integrityFailures(CLEAN)).toEqual([]);@@ -393,6 +443,145 @@ describe('daily: weekly floor', () => {});});+describe('daily: the upgrade matrix gates stable (stability plan 3a)', () => {+ const old = { version: '1.0.1', commit: 1, message: stableMessage('1.0.1', '2026-11-02T05:17:00Z') };+ const RELEASES = { '1.0.1': 1, '1.0.2': 2, '1.0.3': 3, '1.0.4': 4 };+ const OLD_UPDATER = 'v0.60.241.0__postgres__keyed__repo';+ const redFrom = (id: number, commit: number, cellIds: string[], over: Partial<MatrixRunRow> = {}) =>+ matrixRun(id, commit, RELEASES, { event: 'workflow_dispatch', ...over }, { red: cellIds, cells: cellIds.map(c => ({ id: c, from: c.split('__')[0]!, verdict: 'fail' })) });+ const breakIssue = (number: number, from?: string[]) => ({+ number, state: 'open', html_url: `https://github.com/${REPO}/issues/${number}`,+ body: `<!-- gbrain-upgrade-matrix:nightly -->\n${from ? `<!-- gbrain-upgrade-matrix:red-from ${from.join(',')} -->\n` : ''}## Upgrade matrix: RED`,+ });++ test('a dry run whose candidate is red upgrading from v0.60.241.0 (the updater that exited 1) holds and writes nothing; so does a real run', async () => {+ for (const dryRun of [true, false]) {+ const w = world(base({ stable: old, runs: [soak(9, 4)], matrix: [redFrom(704, 4, [OLD_UPDATER])], dryRun }));+ const r = await daily(w.deps);+ expect(r.action).toBe('held');+ expect(r.why).toContain('failing from v0.60.241.0');+ expect(r.why).toContain('a maintainer decides');+ expect(w.pushes).toEqual([]);+ expect(w.posts).toEqual([]);+ expect(w.logs.some(l => l.startsWith('::warning::'))).toBe(true);+ }+ });++ test("an open break issue from the matrix holds stable, on the weekly floor and for a regression fix alike, and dispatches nothing; a labelled issue without the matrix's marker holds nothing", async () => {+ const held = world(base({ stable: old, runs: [soak(9, 4)], importantPrs: [{ number: 30, merge: 3, important: 'regression' }], breakIssues: [breakIssue(9329)] }));+ const r = await daily(held.deps);+ expect(r.action).toBe('held');+ expect(r.why).toContain('#9329');+ expect(r.why).toContain('waiting: #30');+ expect([...held.pushes, ...held.posts]).toEqual([]);+ const labelled = world(base({ stable: old, runs: [soak(9, 4)], breakIssues: [{ number: 41, state: 'open', html_url: 'x', body: 'Upgrading from 0.59 fails for me' }] }));+ expect((await daily(labelled.deps)).action).toBe('promoted');+ });++ test('a soaked candidate with no matrix run gets one dispatch on its tag from master; an active matrix run means wait; a dry run says what it would dispatch', async () => {+ const none = world(base({ stable: old, runs: [soak(9, 4)], matrix: [] }));+ expect(await daily(none.deps)).toMatchObject({ action: 'matrix_dispatched', version: '1.0.4' });+ expect(none.posts).toEqual([{ path: `repos/${REPO}/actions/workflows/upgrade-matrix.yml/dispatches`, body: { ref: 'master', inputs: { candidate: 'v1.0.4' } } }]);+ expect(none.pushes).toEqual([]);+ const active = world(base({ stable: old, runs: [soak(9, 4)], matrix: [{ ...matrixRun(800, 4, RELEASES), status: 'in_progress', conclusion: null, receipt: null }] }));+ expect((await daily(active.deps)).action).toBe('waiting');+ expect(active.posts).toEqual([]);+ const dry = world(base({ stable: old, runs: [soak(9, 4)], matrix: [], dryRun: true }));+ expect(await daily(dry.deps)).toMatchObject({ action: 'dry_run', would: 'dispatch the upgrade matrix on v1.0.4' });+ expect(dry.posts).toEqual([]);+ });++ test('the soak dispatch on the newest release also dispatches the matrix on it when it has no run', async () => {+ const w = world(base({ stable: old, matrix: [] }));+ expect(await daily(w.deps)).toMatchObject({ action: 'soak_dispatched', version: '1.0.4' });+ expect(w.posts.map(p => p.path)).toEqual([`repos/${REPO}/actions/workflows/production-soak.yml/dispatches`, `repos/${REPO}/actions/workflows/upgrade-matrix.yml/dispatches`]);+ const dry = world(base({ stable: old, matrix: [], dryRun: true }));+ expect(await daily(dry.deps)).toMatchObject({ action: 'dry_run', would: 'dispatch the soak and the upgrade matrix on v1.0.4' });+ });++ test('a run on another release, a pull-request run, a run with explicit from-versions and a fault-injection proof run do not qualify the candidate', async () => {+ const cases: Array<MatrixRunRow & { receipt?: MatrixReceipt | null }> = [+ matrixRun(801, 3, RELEASES),+ matrixRun(802, 4, RELEASES, { event: 'pull_request', head_branch: 'fix/x' }),+ matrixRun(803, 4, RELEASES, {}, { profile: 'explicit' }),+ matrixRun(804, 4, RELEASES, {}, { cells: [{ id: 'v1.0.1__pglite__keyless__repo', from: 'v1.0.1', verdict: 'pass', fault: 'corrupt-after-upgrade' }] }),+ ];+ for (const run of cases) {+ const w = world(base({ stable: old, runs: [soak(9, 4)], matrix: [run] }));+ expect(await daily(w.deps)).toMatchObject({ action: 'matrix_dispatched', version: '1.0.4' });+ expect(w.pushes).toEqual([]);+ }+ });++ test('a red run on the candidate behind 30 newer pull-request runs is still found: stable holds and nothing is re-dispatched', async () => {+ const prRuns = Array.from({ length: 30 }, (_, i) => ({ ...matrixRun(900 + i, 4, RELEASES, { event: 'pull_request', head_branch: `fix/pr-${i}` }), receipt: null }));+ const w = world(base({ stable: old, runs: [soak(9, 4)], matrix: [...prRuns, redFrom(704, 4, [OLD_UPDATER])] }));+ const r = await daily(w.deps);+ expect(r.action).toBe('held');+ expect(r.why).toContain('failing from v0.60.241.0');+ expect(w.posts).toEqual([]);+ });++ test('a green run whose receipt predates the stable-window from-versions does not qualify: the check dispatches a fresh run', async () => {+ const preWindow = matrixRun(807, 4, RELEASES, {}, { from: [{ tag: 'v1.0.1', roles: ['N-1', 'stable'] }], skipped: [] });+ const w = world(base({ stable: old, runs: [soak(9, 4)], matrix: [preWindow] }));+ expect(await daily(w.deps)).toMatchObject({ action: 'matrix_dispatched', version: '1.0.4' });+ expect(w.pushes).toEqual([]);+ const skippedWindow = world(base({ stable: old, runs: [soak(9, 4)], matrix: [matrixRun(808, 4, RELEASES, {}, { from: [{ tag: 'v1.0.1', roles: ['N-1'] }], skipped: [{ role: 'stable-window', reason: 'no stable release older than the candidate is in the window' }] })] }));+ expect((await daily(skippedWindow.deps)).action).toBe('promoted');+ });++ test('a green run on the exact tag and tree promotes, and stable.json and the tag carry matrix_run', async () => {+ const w = world(base({ stable: old, runs: [soak(9, 4)] }));+ const r = await daily(w.deps);+ expect(r.action === 'promoted' && r.promotion.version).toBe('1.0.4');+ const url = `https://github.com/${REPO}/actions/runs/${MATRIX_ID + 4}`;+ expect(JSON.parse(w.record()!)).toMatchObject({ version: '1.0.4', matrix_run: url });+ expect(parseStableRecord(w.record()!, '1.0.4')).toMatchObject({ ok: true, record: { matrix_run: url } });+ expect(w.tags.stable!.message).toContain(`matrix_run: ${url}\n`);+ });++ test('a run whose receipt names the tag but another tree, or was written by another harness, is foreign and holds', async () => {+ const foreignTree = matrixRun(805, 4, RELEASES, {}, { candidate: { tag: 'v1.0.4', sha: sha(4), tree: tree(9) } });+ const foreignHarness = matrixRun(806, 4, RELEASES, { head_sha: 'a'.repeat(40) });+ for (const run of [foreignTree, foreignHarness]) {+ const w = world(base({ stable: old, runs: [soak(9, 4)], matrix: [run] }));+ expect((await daily(w.deps)).action).toBe('held');+ expect(w.pushes).toEqual([]);+ }+ });++ describe('a security or data-loss fix past an open break', () => {+ const important = (cls: string) => [{ number: 50, merge: 4, important: cls }];++ test('green on every from-version the break issue does not name: promotes, and the record says what was excused', async () => {+ for (const cls of ['security', 'data-loss']) {+ const w = world(base({ importantPrs: important(cls), runs: [soak(9, 4)], matrix: [redFrom(704, 4, [OLD_UPDATER])], breakIssues: [breakIssue(9329, ['v0.60.241.0'])] }));+ const r = await daily(w.deps);+ expect(r.action === 'promoted' && r.promotion.version).toBe('1.0.4');+ expect(JSON.parse(w.record()!)).toMatchObject({ matrix_run: `https://github.com/${REPO}/actions/runs/704`, reason: expect.stringContaining('upgrade break #9329 excused for v0.60.241.0') });+ }+ });++ test('also green: promotes without excusing anything', async () => {+ const w = world(base({ importantPrs: important('security'), runs: [soak(9, 4)], breakIssues: [breakIssue(9329, ['v0.60.241.0'])] }));+ expect((await daily(w.deps)).action).toBe('promoted');+ });++ test('a failure on a from-version the issue does not name, an issue that names none, or a fix of another class still holds', async () => {+ const newBreak = world(base({ importantPrs: important('security'), runs: [soak(9, 4)], matrix: [redFrom(704, 4, [OLD_UPDATER, 'v1.0.2__pglite__keyless__repo-less'])], breakIssues: [breakIssue(9329, ['v0.60.241.0'])] }));+ const r = await daily(newBreak.deps);+ expect(r.action).toBe('held');+ expect(r.why).toContain('v1.0.2 is a new break');+ const unnamed = world(base({ importantPrs: important('data-loss'), runs: [soak(9, 4)], matrix: [redFrom(704, 4, [OLD_UPDATER])], breakIssues: [breakIssue(9329)] }));+ expect((await daily(unnamed.deps)).why).toContain('names no from-versions');+ const regression = world(base({ importantPrs: important('regression'), runs: [soak(9, 4)], breakIssues: [breakIssue(9329, ['v0.60.241.0'])] }));+ expect((await daily(regression.deps)).action).toBe('held');+ expect([...newBreak.pushes, ...unnamed.pushes, ...regression.pushes]).toEqual([]);+ });+ });+});+describe('daily: stable.json backfill (GBRA-346)', () => {const SOAK_7 = `https://github.com/${REPO}/actions/runs/7`;const tagged = (version: string, commit: number, rollback = false) => renderTagMessage({@@ -470,7 +659,7 @@ describe('stable-daily.yml', () => {expect(Object.keys(wf.on).sort()).toEqual(['schedule', 'workflow_dispatch']);expect(wf.on.schedule.map(s => s.cron)).toEqual(['17 5 * * *', '17 13 * * *']);expect(wf.on.workflow_dispatch.inputs.dry_run).toMatchObject({ type: 'boolean', default: true });- expect(wf.permissions).toEqual({ contents: 'write', actions: 'write' });+ expect(wf.permissions).toEqual({ contents: 'write', actions: 'write', issues: 'read' });expect(wf.concurrency).toEqual({ group: 'promote-stable', 'cancel-in-progress': false });});diff --git a/test/scripts/upgrade-matrix.test.ts b/test/scripts/upgrade-matrix.test.tsindex be9ce06365..2dd3f0dec7 100644--- a/test/scripts/upgrade-matrix.test.ts+++ b/test/scripts/upgrade-matrix.test.ts@@ -9,7 +9,8 @@ import { execFileSync } from 'node:child_process';import { mkdirSync, mkdtempSync, rmSync, unlinkSync, writeFileSync } from 'node:fs';import { tmpdir } from 'node:os';import { join } from 'node:path';-import { resolveFromVersions, type Release } from '../../scripts/upgrade-matrix/versions.ts';+import { resolveFromVersions, stableWindow, type Release, type StableRecordTag } from '../../scripts/upgrade-matrix/versions.ts';+import { failingFrom, parseRedFrom, renderRedFrom } from '../../scripts/upgrade-matrix/marker.ts';import { createCertificates, parseFixturePath, startGithubFixture } from '../../scripts/upgrade-matrix/github-fixture.ts';import { CELL_RELEASE_CHANNEL_ENV, compareInstalledTree, upgradeRefs } from '../../scripts/upgrade-matrix/cell.ts';import { diffCatalog, diffContent, takeSnapshot, type Query } from '../../scripts/upgrade-matrix/snapshot.ts';@@ -28,13 +29,13 @@ describe('from-version resolution', () => {const r = resolveFromVersions(releases, candidate, { profile: 'full', stableVersion: null });const byRole = Object.fromEntries(r.from.flatMap(f => f.roles.map(role => [role, f.tag])));expect(byRole).toEqual({ 'N-1': 'v0.60.79.0', 'N-10': 'v0.60.70.0', 'N-50': 'v0.60.30.0', '30d': 'v0.60.50.0' });- expect(r.skipped).toEqual([{ role: 'stable', reason: expect.stringContaining('stable') }]);+ expect(r.skipped).toEqual([{ role: 'stable', reason: expect.stringContaining('stable') }, { role: 'stable-window', reason: expect.stringContaining('window') }]);});test('a role shared by two picks is one from-version with both roles; too few releases skips N-50 instead of dropping it', () => {const few = releases.slice(0, 12);const r = resolveFromVersions(few, few[11]!, { profile: 'full', stableVersion: '0.60.2.0' });- expect(r.skipped.map(s => s.role)).toEqual(['N-50']);+ expect(r.skipped.map(s => s.role)).toEqual(['N-50', 'stable-window']);expect(r.skipped[0]!.reason).toContain('only 11 published releases');const stable = r.from.find(f => f.roles.includes('stable'))!;expect(stable.tag).toBe('v0.60.2.0');@@ -49,6 +50,52 @@ describe('from-version resolution', () => {});});+describe('the stable window (stability plan 3a)', () => {+ const at = (iso: string) => new Date(iso).toISOString();+ const candidate = { version: '0.60.730.0', published_at: at('2026-11-16T03:00:00Z') };+ const records: StableRecordTag[] = [+ { version: '0.50.1.0', promoted_at: at('2026-03-01T00:00:00Z') },+ { version: '0.50.2.0', promoted_at: at('2026-04-01T00:00:00Z') },+ { version: '0.55.1.0', promoted_at: at('2026-08-03T00:00:00Z') },+ { version: '0.55.2.0', promoted_at: at('2026-08-20T00:00:00Z') },+ { version: '0.60.474.0', promoted_at: at('2026-10-28T04:52:51Z') },+ { version: '0.60.593.0', promoted_at: at('2026-11-06T20:40:18Z') },+ ...['613', '620', '638', '654', '672', '707', '717', '722'].map((n, i) => ({ version: '0.60.' + n + '.0', promoted_at: at('2026-11-' + String(8 + i).padStart(2, '0') + 'T13:22:00Z') })),+ { version: '0.60.731.0', promoted_at: at('2026-11-16T13:22:00Z') },+ ];++ test('keeps the last 8 stable releases, the newest of each older month, and the release stable when the window opened; nothing newer than the candidate', () => {+ expect(stableWindow(records, candidate)).toEqual([+ '0.60.722.0', '0.60.717.0', '0.60.707.0', '0.60.672.0', '0.60.654.0', '0.60.638.0', '0.60.620.0', '0.60.613.0',+ '0.60.593.0', '0.60.474.0', '0.55.2.0', '0.50.2.0',+ ]);+ expect(stableWindow([], candidate)).toEqual([]);+ });++ test('the full profile upgrades from every window release that has a published release; one without is skipped with its reason; the pr profile has no window', () => {+ const rel = (v: string, d: string): Release => ({ tag: 'v' + v, version: v, published_at: at(d) });+ const published = [rel('0.60.722.0', '2026-11-15T00:00:00Z'), rel('0.60.717.0', '2026-11-14T00:00:00Z'), rel('0.60.474.0', '2026-10-27T00:00:00Z'), rel('0.60.729.0', '2026-11-16T00:00:00Z')];+ const few = [{ version: '0.60.722.0', promoted_at: at('2026-11-15T13:22:00Z') }, { version: '0.60.717.0', promoted_at: at('2026-11-14T13:22:00Z') }, { version: '0.60.593.0', promoted_at: at('2026-11-06T20:40:18Z') }];+ const r = resolveFromVersions(published, candidate, { profile: 'full', stableVersion: '0.60.722.0', stableRecords: few });+ expect(r.from.find(f => f.tag === 'v0.60.722.0')!.roles).toEqual(['stable', 'stable-window']);+ expect(r.from.find(f => f.tag === 'v0.60.717.0')!.roles).toEqual(['stable-window']);+ expect(r.skipped).toContainEqual({ role: 'stable-window', reason: 'stable 0.60.593.0 has no published release' });+ expect(resolveFromVersions(published, candidate, { profile: 'pr', stableRecords: few }).from.some(f => f.roles.includes('stable-window'))).toBe(false);+ });++ test("the break issue names the from-versions that failed or held, and only the matrix's marker line is read back", () => {+ const summary = {+ cells: [{ id: 'v0.60.241.0__postgres__keyed__repo', from: 'v0.60.241.0', verdict: 'fail' }, { id: 'v0.60.722.0__pglite__keyless__repo', from: 'v0.60.722.0', verdict: 'pass' }],+ red: ['v0.60.241.0__postgres__keyed__repo', 'v0.60.150.0__pglite__keyed__repo-less'], holds: ['door:hermes'],+ };+ expect(failingFrom(summary)).toEqual({ from: ['v0.60.150.0', 'v0.60.241.0'], door: true });+ const line = renderRedFrom(failingFrom(summary).from);+ expect(line).toBe('<!-- gbrain-upgrade-matrix:red-from v0.60.150.0,v0.60.241.0 -->');+ expect(parseRedFrom('<!-- gbrain-upgrade-matrix:nightly -->\n' + line + '\n## Upgrade matrix: RED')).toEqual(['v0.60.150.0', 'v0.60.241.0']);+ expect(parseRedFrom('<!-- gbrain-upgrade-matrix:nightly -->\nUpgrading from v0.60.241.0 fails')).toBeNull();+ });+});+function gitRepo(): { dir: string; first: string; second: string } {const dir = mkdtempSync(join(tmpdir(), 'um-src-'));const git = (...args: string[]) => execFileSync('git', ['-C', dir, '-c', 'user.name=t', '-c', '[email protected]', ...args], { encoding: 'utf8' }).trim();