Skip to content

Garry's Opinionated OpenClaw/Hermes Agent Brain

TypeScript

Stable waits for a green upgrade test from every recent stable release (GST-2818) (#9462)

Garry Tanb7babb3e3860
diff --git a/.github/workflows/promote-stable.yml b/.github/workflows/promote-stable.yml
index 97867329fb..c1411c59d0 100644
--- a/.github/workflows/promote-stable.yml
+++ b/.github/workflows/promote-stable.yml
@@ -8,8 +8,9 @@ name: Promote stable
# maintain role; v<version> is an existing release tag; soak_run_url is a
# successful completed run of the workflow named by the STABLE_SOAK_WORKFLOW
# repository variable; that run's head commit has the release's exact git
-# tree; and the version is newer than the current stable (an older one only
-# with rollback=true and a reason). It then creates the immutable annotated
+# tree; matrix_run_url, when given, is a green upgrade matrix run of master's
+# harness on this release's exact tag and tree; and the version is newer than
+# the current stable (an older one only with rollback=true and a reason). It then creates the immutable annotated
# record tag stable-v<version> (first promotion only) and moves the annotated
# `stable` tag to the release commit, both carrying the same key: value body,
# and commits the evidence record stable.json (version, commit, tree, soak run,
@@ -35,6 +36,11 @@ on:
description: GitHub Actions run URL of the passed soak on this release's tree
required: true
type: string
+ matrix_run_url:
+ description: GitHub Actions run URL of a green upgrade matrix run on this release's tag and tree (empty promotes on your decision, recorded as matrix_run null)
+ required: false
+ type: string
+ default: ''
scoreboard_url:
description: Scoreboard URL (defaults to soak_run_url)
required: false
@@ -78,7 +84,7 @@ on:
permissions:
contents: write # create and push the stable-v<version> and stable tags and the stable-record branch
- actions: read # read the soak run
+ actions: read # read the soak run and the upgrade matrix run's receipt
concurrency:
group: promote-stable
@@ -103,6 +109,7 @@ jobs:
STABLE_SOAK_WORKFLOW: ${{ vars.STABLE_SOAK_WORKFLOW }}
PROMOTE_VERSION: ${{ inputs.version }}
PROMOTE_SOAK_RUN_URL: ${{ inputs.soak_run_url }}
+ PROMOTE_MATRIX_RUN_URL: ${{ inputs.matrix_run_url }}
PROMOTE_SCOREBOARD_URL: ${{ inputs.scoreboard_url }}
PROMOTE_ROLLBACK: ${{ inputs.rollback }}
PROMOTE_REASON: ${{ inputs.reason }}
diff --git a/.github/workflows/stable-daily.yml b/.github/workflows/stable-daily.yml
index 6d9f352e07..1057b7a94f 100644
--- a/.github/workflows/stable-daily.yml
+++ b/.github/workflows/stable-daily.yml
@@ -2,11 +2,14 @@ name: Stable daily
# The daily stable check (docs/RELEASING.md#daily-stable-check,
# scripts/stable-daily.ts). Twice a day it promotes the newest release whose
-# exact tree passed the production soak with every integrity gate clean, but
-# only when a reason exists: a merged PR labelled stable:important that is
-# released but not yet in stable, or stable older than 7 days (the weekly
-# floor). With nothing to promote yet it dispatches one soak on the newest
-# release, and a later run promotes it. It never promotes without that soak,
+# exact tree passed the production soak with every integrity gate clean and
+# whose upgrade matrix run on that tag and tree is green, but only when a
+# reason exists: a merged PR labelled stable:important that is released but
+# not yet in stable, or stable older than 7 days (the weekly floor). While the
+# upgrade matrix's own break issue is open it holds, except for a security or
+# data-loss fix. With nothing to promote yet it dispatches one soak (and one
+# upgrade matrix run) on the newest release, and a later run promotes it. It
+# never promotes without that soak and matrix run,
# never moves stable backward, pauses while stable is a rollback, and does
# nothing while the STABLE_AUTO_PROMOTE repository variable is "off".
# When the published stable.json is missing or names another commit than
@@ -38,7 +41,8 @@ on:
permissions:
contents: write # push the stable-v<version> and stable tags and the stable-record branch (stable.json)
- actions: write # read soak runs and their scoreboards; dispatch a soak
+ actions: write # read soak and upgrade matrix runs and their artifacts; dispatch a soak or the matrix
+ issues: read # find the upgrade matrix's open break issue, which holds stable
concurrency:
group: promote-stable
diff --git a/.github/workflows/upgrade-matrix.yml b/.github/workflows/upgrade-matrix.yml
index 8a138ae8fa..815c11970e 100644
--- a/.github/workflows/upgrade-matrix.yml
+++ b/.github/workflows/upgrade-matrix.yml
@@ -19,14 +19,17 @@ name: Upgrade matrix
# newest release); optional explicit from-versions and a
# proof-only fault injection.
# schedule nightly, full matrix against the newest release; a red
-# run opens or updates one upgrade-break issue (the label
-# that holds stable promotion) and a green run closes it.
+# run opens or updates one upgrade-break issue (its marker,
+# not the label, holds stable promotion: stable-daily.ts)
+# and a green run closes it.
# pull_request reduced subset (N-1 and N-50, PGLite and Postgres,
# keyless, repo-less) against the head, only when schema,
# migration, self-upgrade or harness paths change.
#
-# This workflow does not gate releases to the latest lane; stable promotion
-# reads its receipt (GBRA-224 W1).
+# This workflow does not gate releases to the latest lane. Stable promotion
+# reads its receipt: scripts/stable-daily.ts dispatches it on the candidate tag
+# and promotes only on a green run of that exact tag and tree
+# (scripts/stable-matrix.ts).
on:
workflow_dispatch:
diff --git a/changelog.d/capy-stable-waits-for-a.md b/changelog.d/capy-stable-waits-for-a.md
new file mode 100644
index 0000000000..c63d936477
--- /dev/null
+++ b/changelog.d/capy-stable-waits-for-a.md
@@ -0,0 +1,7 @@
+**Stable now waits for a green upgrade test from every recent stable release.**
+
+### Changed
+- The daily stable check promotes a release only after the upgrade matrix ran green on that release's exact tag and tree. With no run yet it starts one; a red run holds stable with a warning and is not started again by the schedule.
+- The upgrade matrix's full run also upgrades from the stable releases users may still be on, which are the last 8 stable releases plus the newest one of each older month, back 6 months. A test run from before this change no longer counts.
+- While the upgrade matrix's own break issue is open, stable holds. Only a security or data-loss fix that fails on no from-version the issue doesn't name may pass it.
+- `promote-stable.yml` takes an optional `matrix_run_url`, checks it the same way and records it as `matrix_run` in `stable.json` and the stable tag. A maintainer can still promote without one, recorded as `matrix_run: null`.
diff --git a/docs/RELEASING.md b/docs/RELEASING.md
index 18cee3d629..9d94b98366 100644
--- a/docs/RELEASING.md
+++ b/docs/RELEASING.md
@@ -777,12 +777,15 @@ gh workflow run promote-stable.yml -f version=<VERSION> \
version's first promotion and never moved or deleted. Both carry the same
message: a title line (`gbrain`, then `stable channel: v<VERSION>`), then one
`key: value` line each for `version`, `commit`, `tree`, `soak_run`,
- `scoreboard`, `promoted_by`, `promoted_at` (UTC), `previous`
+ `scoreboard`, `matrix_run` (only when the promotion named one; `stable.json`
+ always has the key, `null` without one), `promoted_by`, `promoted_at` (UTC), `previous`
(`none` on the first promotion), `rollback` and `reason` (`-` when
empty).
- **Inputs.** `version` (an existing `v<VERSION>` release tag),
`soak_run_url` (the soak's Actions run URL, optionally
- `/attempts/<n>`), `scoreboard_url` (optional, defaults to the run URL),
+ `/attempts/<n>`), `matrix_run_url` (optional for a maintainer: a green
+ upgrade matrix run on this release, see below), `scoreboard_url` (optional,
+ defaults to the run URL),
`rollback` (default false), `reason` (required with rollback) and
`withdraw` (versions added to the carried-forward `known_bad` list;
clients on one are offered the current stable, even when older), and
@@ -815,6 +818,16 @@ gh workflow run promote-stable.yml -f version=<VERSION> \
variables → Actions → Variables; a bare file name like `soak.yml` or the
full `.github/workflows/soak.yml` path). With the variable unset, every
promotion is refused.
+- **Upgrade matrix run.** When `matrix_run_url` is given, it must be a
+ completed run of `.github/workflows/upgrade-matrix.yml` from master's harness
+ (a schedule or dispatch run, not a pull request's), with the full from-version
+ set and no fault injection, whose `upgrade-matrix-receipt` artifact names this
+ release's tag, commit and tree and was written by that run's own commit, and
+ it must be green; a red or foreign run is refused
+ (`scripts/stable-matrix.ts` `judgeMatrixRun`). A maintainer may promote
+ without one, for example past an upgrade break the matrix found: the record
+ then says `matrix_run: null` and names the maintainer as `promoted_by`. A
+ scheduled promotion always names one.
- **Published release.** The tag's `VERSION` file must say the version, and
its GitHub release must exist and not be a draft, so binary clients can
fetch the stable release.
@@ -893,6 +906,27 @@ and keeps stable between one day and one week behind what matters:
(never while another soak runs, and never after a soak on any newer release
failed to qualify: then a maintainer decides, and the check warns once stable
is past the weekly floor), and a later run promotes it.
+- **Upgrade matrix next.** The soaked candidate also needs a green
+ [upgrade matrix](ci-upgrade-matrix.md) run on its exact tag and tree: a
+ schedule or dispatch run of master's harness with the full from-version set
+ (which includes every stable release in the window), checked as under
+ "Upgrade matrix run" above and again by `promote()`, and recorded as
+ `matrix_run`. With no run yet it dispatches one on the candidate's tag
+ (`upgrade-matrix.yml` on master, `candidate=v<version>`), unless a matrix run
+ is active, and a later run promotes; when it dispatches a soak on the
+ newest release it dispatches the matrix on that release too. A red or
+ foreign run holds stable with a warning and is never re-dispatched by the
+ schedule: a maintainer fixes the break or promotes by hand.
+- **Upgrade breaks hold.** While the matrix's own break issue is open (the
+ one the nightly run opens with the `<!-- gbrain-upgrade-matrix:nightly -->`
+ marker; another issue with the `upgrade-break` label holds nothing), the
+ check holds stable, the weekly floor included, and dispatches nothing. The
+ one way past it on the schedule is a released `stable:important` fix whose
+ `changelog.d` fragment says `important: security` or `important: data-loss`,
+ on a candidate whose matrix run fails on no from-version the issue names
+ (its `red-from` line) and on nothing else; the record's `reason` names the
+ excused from-versions. Everything else waits for the nightly run to close
+ the issue, or for a maintainer's promotion.
- **Who.** A schedule event runs only the workflow file as merged on the
default branch, so a scheduled run promotes by rule without a maintainer. A
manual dispatch defaults to `dry_run` (report only); `dry_run=false` needs an
diff --git a/docs/architecture/key-files/ci-health-and-test-guards.md b/docs/architecture/key-files/ci-health-and-test-guards.md
index 0b6193fe83..7f998907a0 100644
--- a/docs/architecture/key-files/ci-health-and-test-guards.md
+++ b/docs/architecture/key-files/ci-health-and-test-guards.md
@@ -4,15 +4,15 @@
Current behavior and load-bearing invariants; history belongs in Git and CHANGELOG.
- `.github/workflows/nightly-watch.yml` + `scripts/nightly-issue.ts` — nightly-red and master-red incidents. `workflow_run` fires only for master-branch completions; the `resolve` job admits scheduled runs and push-to-master runs of Test and E2E Tests from this repository (head repository checked) plus dispatch, and `nightly-issue.ts --resolve` turns every entry path into the target workflow id and track (`schedule`, `push`; a replay writes `push`); `watch` queues on one group per repository + workflow + track. `nightly-issue.ts --run-id <id> [--replay-issue <n>] [--dry-run]` routes the run (`routeRun`: pull-request, fork, other-branch and non-allowlisted push runs are refused with a next step). Scheduled runs keep one `Nightly red: <workflow>` issue (label `nightly-red`): open on red with job/step pairs, every failing test file with its reproduce line, sanitized error excerpts, the first red run, the last green SHA and compare link, a "Next step for the agent" and a JSON block; update while red; reopen on a flap; close only on a later green scheduled run with complete evidence where every previously failing job executed. Rows in `.github/nightly-known-red.tsv` (at most 3; race-hunt jobs refused) keep a matching incident open under `known-red`. A scheduled Test or E2E Tests run also feeds that workflow's open master-red issue. Push runs go to `scripts/lib/master-red.ts`. A ratchet test fails when a scheduled workflow is not watched. Tests: `test/scripts/nightly-issue.test.ts`. Process: `docs/RELEASING.md#nightly-red-issues`, `docs/ci-red-runbook.md`.
-- `.github/workflows/upgrade-matrix.yml` + `scripts/upgrade-matrix.ts` (modules in `scripts/upgrade-matrix/`) — upgrade-from-N matrix (GBRA-224 W2). `plan` resolves the candidate and from-versions (N-1, N-10, N-50, 30 days, stable) from release tags; `cell` installs a from-version with `bun install -g github:garrytan/gbrain` in an isolated HOME, seeds a brain, runs that release's `gbrain upgrade` against `github-fixture.ts` (per-cell bare mirror via git `insteadOf`, loopback tarball API, TLS fixture behind `HTTPS_PROXY`; every other host refused), then checks install identity, one build across a running serve, apply-migrations, schema head, doctor, canonical preservation (`snapshot.ts`), writes, keyed vector query on the mock provider and the old binary on the upgraded brain; `summarize` judges red/hold/green (`verdict.ts`, `known-exceptions.json`, the `W4 SWITCH POINT` for `doctor --alert`); `issue` keeps one `upgrade-break` issue. Tests: `test/scripts/upgrade-matrix.test.ts`. Docs: `docs/ci-upgrade-matrix.md`.
+- `.github/workflows/upgrade-matrix.yml` + `scripts/upgrade-matrix.ts` (modules in `scripts/upgrade-matrix/`) — upgrade-from-N matrix (GBRA-224 W2). `plan` resolves the candidate and from-versions (N-1, N-10, N-50, 30 days, stable, and `stable-window`: `versions.ts` `stableWindow` over the `stable-v*` record tags, the newest 8 plus one per older month back 183 days) from release tags; `cell` installs a from-version with `bun install -g github:garrytan/gbrain` in an isolated HOME, seeds a brain, runs that release's `gbrain upgrade` against `github-fixture.ts` (per-cell bare mirror via git `insteadOf`, loopback tarball API, TLS fixture behind `HTTPS_PROXY`; every other host refused), then checks install identity, one build across a running serve, apply-migrations, schema head, doctor, canonical preservation (`snapshot.ts`), writes, keyed vector query on the mock provider and the old binary on the upgraded brain; `summarize` judges red/hold/green (`verdict.ts`, `known-exceptions.json`, the `W4 SWITCH POINT` for `doctor --alert`); `issue` keeps one `upgrade-break` issue with `marker.ts`'s `ISSUE_MARKER` and a `red-from` line naming the failing from-versions. Tests: `test/scripts/upgrade-matrix.test.ts`. Docs: `docs/ci-upgrade-matrix.md`.
- `scripts/lib/gh-issue.ts` — the GitHub seam shared by the CI watchers. `inert()` makes run-derived text (job, step and test names, error text) safe to show an agent: control characters stripped, length capped, backticks and pipes replaced, mentions and links neutralized; `redactCredentials()` strips user:password from database URLs. `GitHubClient` is the one interface the watchers call (`restClient()` uses `GH_TOKEN`/`GITHUB_TOKEN` in Actions or the authenticated `gh api` locally; `download()` reads artifact archives without forwarding the token to blob storage). `findIssueByTitle` finds an issue by label and exact title, open first, else the most recently updated closed one; `listLabeled`, `getIssue` and `linkedPulls` (PRs cross-referencing an issue) back master-red and CI health; `fetchArtifactJson` + `readZipEntry` read one JSON file from a run artifact and return a reason instead of throwing. `createIssue` adds `AGENT_FILED_LABEL` (`agent-filed`) to every issue it opens, so CONTRIBUTING.md's community-issues search can leave automation's issues out; `test/scripts/gh-issue-agent-filed.test.ts` fails if a script or workflow opens an issue any other way.
- `.github/workflows/ci-health.yml` + `scripts/ci-health-report.ts` — CI health report. Reads one window (`--days`, default 7, or `--since/--until`, also dispatch inputs) of same-repository pull-request and push-to-master runs of test.yml and e2e.yml (runs, jobs, check-run annotations, issues; never logs, fork runs or fork artifacts), paging the whole window (`listRuns` splits a query over the 1000-result search cap). It reports first-attempt pass rates (`firstAttempt()`), the exit criteria (master >= 95%, PR >= 90%; met, missed or indeterminate when the window cannot be read in full), stress-gate failures, failure classes, cancellation and completion coverage, master-red time to first response, the nightly-watch residual, the test files with the most failure annotations, shard spread and the count of fork runs it did not read; an explicit window with a missed target opens a follow-up issue. Tests: `test/scripts/ci-health-report.test.ts`. Docs: `docs/ci-red-runbook.md#ci-health-report`.
- `scripts/lib/master-red.ts` — master-red reconciliation for push-to-master runs of Test and E2E Tests (`PUSH_ALLOWLIST`). Each watch evaluates every completed push run above the high-water mark (`evaluated`: run number, then attempt) stored in the newest `Master red: <workflow>` issue's `master-red:json` block (`applyRun` folds runs; cancelled runs change nothing); a missing or malformed block is rebuilt from the actions API (`pendingRuns`). It opens on red with the suspect range (`suspectRange`: compare plus commit-to-PR lookups), updates while red, closes when every previously failing job (E2E Tests: every failing file, from the manifest) ran and passed with complete evidence, records an incident that opened and closed between watches, and keeps a not-run-since failure open with the manual-close step. A newest issue closed by hand while its block is red ends that incident (`closed_by_hand`): later runs are recorded on it as green, and only a new red run opens a new issue. `externalPass` lets a scheduled run or an issue-bound E2E replay clear failures. Closing without a merged linked repair PR opens `flake` issues (`flakeFor`, `flake:json`); `closeFlakes` closes one after its repair PR merged and a later complete run passed its file and arm. Writes re-read the issue first and re-plan when it moved; a create race closes the newer duplicate. Tests: `test/scripts/nightly-issue.test.ts`. Docs: `docs/ci-red-runbook.md`.
- `scripts/lib/ci-run.ts` — one run's failure evidence (`collectEvidence`): jobs, failing jobs' annotations and, for test.yml and e2e.yml, the `ci-manifest` artifact, returned as structured records (job, step, every failing test file and identity) apart from sanitized excerpts; a cancelled job is not run (listed in `cancelled`) unless its annotation says it timed out, and a red run whose only failures are aggregators of cancelled jobs is `ignored`; with `complete`/`problems` (an unreadable page, annotation or manifest is incomplete evidence, never zero failures). A push run with a succeeded `reused-full-suite` job is judged by the run its `reuse.json` names (`checkRecord` from `scripts/ci-reuse.ts`, recorded attempt, identical commit tree): that run's succeeded jobs and complete `ci-manifest` stand in for the push, and any gap is incomplete. `reproduceCommand` builds `bun run test:stress <file> --iterations 10 [--postgres]` from a validated test path only. `exemptionRows` turns failing identities with an exact test name and signature into the stress gate's `<!-- gbrain-stress-exemption -->` records (backend per `stressBackend`, owner `agent on release duty`), written into master-red, flake and nightly-red bodies; `test/fixtures/nightly-watch/stress-exemption-contract.ts` vendors the gate's parser so the tests prove it accepts them.
- `scripts/ci-manifest.ts` — the versioned failure and execution manifest (`gbrain-ci-manifest/v1`). The `executed-receipts` job of test.yml and e2e.yml builds it from every lane's receipts (via `scripts/ci-executed-counts.ts`, declared skips from the expected-deltas TSV), binds it to the workflow, event, SHA, run id and attempt, and uploads it as `ci-manifest`; each failure carries the JUnit message (credentials redacted) and the stress gate's `failureSignature` (omitted when the message holds a credential); `checkManifest` refuses a manifest for another run or attempt; `passedFiles` lists files that executed tests and failed none. Docs: `docs/ci-red-runbook.md#ci-failure-manifest`.
- `scripts/release-gate.ts` — release.yml's `ci-gate` job. Polls the push-to-master runs of Test and E2E Tests on the release commit and writes `publish`/`sha`: both green publishes; a failed run publishes nothing and fails with the next step; a cancelled or skipped run publishes nothing and fails with `gh run rerun <id>` (master push runs are keyed by sha, and between releases many commits share one VERSION, so it never follows to a later commit); non-master refs are refused; a `workflow_dispatch` (`--event`) gates only the release commit (the first-parent commit that last changed VERSION) and refuses a later master HEAD with the re-run of that commit's push Release run; it fails with the backfill command after 110 minutes. Build, release, `latest-stable` and the publish jobs use the gated commit. Tests: `test/scripts/release-gate.test.ts`. Docs: `docs/RELEASING.md#release-ci-gate`.
-- `.github/workflows/promote-stable.yml` + `scripts/promote-stable.ts` — stable channel promotion (`workflow_dispatch`: `version`, `soak_run_url`, `scoreboard_url`, `rollback`, `reason`, `withdraw`, `not_promotable`, `not_promotable_reason`, `not_promotable_fixed_in`, `not_promotable_remove`; concurrency group `promote-stable`). `promote()` refuses unless the actor's `role_name` is admin or maintain (`authorize()`; the one exception is `PromoteDeps.scheduled`, set by `scheduleAuthorized()` only for a schedule event of `stable-daily.yml` on the default branch, which also requires the release commit on master and records `promoted_by: schedule (.github/workflows/stable-daily.yml)`), `v<VERSION>` exists with a `VERSION` file naming it and a published (non-draft) GitHub release, the soak run URL names a completed, successful run of this repository's `STABLE_SOAK_WORKFLOW` workflow (repository variable; bare file name or path), the run's head commit has the release tag's exact tree (fetched when missing), and the version is newer than the current stable (the highest `v*` tag on `refs/tags/stable^{commit}`), or older with `rollback` and a reason. Re-promoting the current stable (no rollback) moves no tag: `recordCurrentStable()` writes `stable.json` from the stable tag's message (`parseTagMessage()`, which must name the same commit, tree and soak run) when the published record does not name that release, leasing `stable` unchanged and the record branch, and refuses as a no-op otherwise (GBRA-346 backfill for releases promoted before the record existed). It creates the immutable annotated `stable-v<VERSION>` record (refused when it exists, except on a rollback re-promotion of the same commit) and moves the annotated `stable` tag, both with `renderTagMessage()`'s `key: value` body, commits the evidence record (`renderRecord()`, `stable.json`, `known_bad` carried forward and merged with `withdraw` by `mergeKnownBad()`, `not_promotable` carried forward, added to and removed from by `mergeNotPromotable()`, which caps a reason at 500 characters; a record over `STABLE_RECORD_MAX_BYTES`, the client's `RECORD_MAX_BYTES`, is refused before the push) on the `stable-record` branch, then pushes all three in one `--atomic` push (`stable` and `stable-record` leased on the values it read, so a failed push changes nothing) and writes the step summary. Side effects come in through `PromoteDeps`. Tests: `test/scripts/promote-stable.test.ts`. Docs: `docs/RELEASING.md#stable-channel-promotion`.
-- `.github/workflows/stable-daily.yml` + `scripts/stable-daily.ts` — the daily stable check (05:17 and 13:17 UTC; dispatch defaults to `dry_run`, and `dry_run=false` needs admin or maintain; concurrency group `promote-stable`). Off when `STABLE_AUTO_PROMOTE` is `off`, paused while the stable record says `rollback: true`. Acts only on a reason: a merged `stable:important` PR released but not in stable, or stable older than `WEEKLY_FLOOR_DAYS` (7) with a newer release. Promotes, through `promote()`, the newest release on master newer than stable whose exact tree passed a full `SOAK_SHAPE` `STABLE_SOAK_WORKFLOW` run with every `INTEGRITY_GATES` gate passing in every cell (on the important-only path it must contain the fix), skipping a release with a `stable-v<version>` tag (stable before, then rolled back) or on stable.json's `known_bad` (#8792) or `not_promotable` list; otherwise dispatches one soak on the newest release's tag, never while a soak runs or after a soak on a newer release failed to qualify. Tests: `test/scripts/stable-daily.test.ts`. Docs: `docs/RELEASING.md#daily-stable-check`.
+- `.github/workflows/promote-stable.yml` + `scripts/promote-stable.ts` — stable channel promotion (`workflow_dispatch`: `version`, `soak_run_url`, `matrix_run_url`, `scoreboard_url`, `rollback`, `reason`, `withdraw`, `not_promotable`, `not_promotable_reason`, `not_promotable_fixed_in`, `not_promotable_remove`; concurrency group `promote-stable`). `promote()` refuses unless the actor's `role_name` is admin or maintain (`authorize()`; the one exception is `PromoteDeps.scheduled`, set by `scheduleAuthorized()` only for a schedule event of `stable-daily.yml` on the default branch, which also requires the release commit on master and records `promoted_by: schedule (.github/workflows/stable-daily.yml)`), `v<VERSION>` exists with a `VERSION` file naming it and a published (non-draft) GitHub release, the soak run URL names a completed, successful run of this repository's `STABLE_SOAK_WORKFLOW` workflow (repository variable; bare file name or path), the run's head commit has the release tag's exact tree (fetched when missing), `checkMatrix()` accepts the named upgrade matrix run only through `scripts/stable-matrix.ts` `judgeMatrixRun()` (a completed schedule or dispatch run of master's harness, full profile, no fault, whose `upgrade-matrix-receipt` names the release's tag, commit and tree and the run's own harness, and green; a scheduled promotion must name one, a maintainer may not and the record says `matrix_run: null`), and the version is newer than the current stable (the highest `v*` tag on `refs/tags/stable^{commit}`), or older with `rollback` and a reason. Re-promoting the current stable (no rollback) moves no tag: `recordCurrentStable()` writes `stable.json` from the stable tag's message (`parseTagMessage()`, which must name the same commit, tree and soak run) when the published record does not name that release, leasing `stable` unchanged and the record branch, and refuses as a no-op otherwise (GBRA-346 backfill for releases promoted before the record existed). It creates the immutable annotated `stable-v<VERSION>` record (refused when it exists, except on a rollback re-promotion of the same commit) and moves the annotated `stable` tag, both with `renderTagMessage()`'s `key: value` body, commits the evidence record (`renderRecord()`, `stable.json`, `known_bad` carried forward and merged with `withdraw` by `mergeKnownBad()`, `not_promotable` carried forward, added to and removed from by `mergeNotPromotable()`, which caps a reason at 500 characters; a record over `STABLE_RECORD_MAX_BYTES`, the client's `RECORD_MAX_BYTES`, is refused before the push) on the `stable-record` branch, then pushes all three in one `--atomic` push (`stable` and `stable-record` leased on the values it read, so a failed push changes nothing) and writes the step summary. Side effects come in through `PromoteDeps`. Tests: `test/scripts/promote-stable.test.ts`. Docs: `docs/RELEASING.md#stable-channel-promotion`.
+- `.github/workflows/stable-daily.yml` + `scripts/stable-daily.ts` — the daily stable check (05:17 and 13:17 UTC; dispatch defaults to `dry_run`, and `dry_run=false` needs admin or maintain; concurrency group `promote-stable`). Off when `STABLE_AUTO_PROMOTE` is `off`, paused while the stable record says `rollback: true`. Acts only on a reason: a merged `stable:important` PR released but not in stable, or stable older than `WEEKLY_FLOOR_DAYS` (7) with a newer release. Promotes, through `promote()`, the newest release on master newer than stable whose exact tree passed a full `SOAK_SHAPE` `STABLE_SOAK_WORKFLOW` run with every `INTEGRITY_GATES` gate passing in every cell (on the important-only path it must contain the fix), skipping a release with a `stable-v<version>` tag (stable before, then rolled back) or on stable.json's `known_bad` (#8792) or `not_promotable` list; the soaked candidate also needs a green upgrade matrix gate run on its exact tag and tree (`judgeMatrixRun`), and with none it dispatches `upgrade-matrix.yml` on master with `candidate=v<version>` (not while a matrix run is active); a red or foreign run returns `held`. While the matrix's own break issue is open (`findBreakIssue`: `ISSUE_MARKER`, not the label alone) it returns `held` unless a released fix's `changelog.d` fragment says `important: security` or `data-loss` (`BREAK_PASS_CLASSES`) and the candidate's run fails only on from-versions the issue's `red-from` line names. Otherwise dispatches one soak (and the matrix, when it has no run) on the newest release's tag, never while a soak runs or after a soak on a newer release failed to qualify. Tests: `test/scripts/stable-daily.test.ts`. Docs: `docs/RELEASING.md#daily-stable-check`.
- `scripts/check-install-ref.ts` (`bun run check:install-ref`, in verify) — every install pin, bootstrap runbook fetch and setup-helper fetch in tracked files (not CHANGELOG.md or test/) names `INSTALL_REF` from `src/core/install-ref.ts`; exact `v<version>` and commit pins are left alone; TypeScript must import the constant; `--write` rewrites the text surfaces. `scripts/check-bootstrap-tag.sh` reads its sanctioned ref from the same constant. Tests: `test/install-ref.test.ts`.
- `scripts/regen-all.ts` — one entry point for every generated artifact (`ARTIFACTS`: regen and check command per artifact). `bun run regen:all` regenerates offline and keyless (provider keys and database URLs removed) and lists what changed; a second run changes nothing. `--check` is read-only and runs in `verify` as `check:regen-all`; `--goldens` also regenerates the offline contract goldens. Postgres goldens are named with their own commands, and `llms.txt` regenerates last because it inlines other generated docs. Every `bun test` it spawns (two checks and the goldens) carries `--timeout=60000` through one `BUN_TEST` prefix, because Bun does not apply bunfig.toml's `[test] timeout`. Tests: `test/scripts/regen-all.test.ts`.
- `scripts/build-behavior-changes.ts` — writes (or `--check`s) `src/core/behavior-changes.generated.ts` from `behavior-changes.d/<id>.md` and the `<!-- behavior-change: <id> -->` markers in released CHANGELOG sections (and in `changelog.d` fragments while VERSION is newer than the top heading, i.e. inside a release PR): `RELEASED_BEHAVIOR_CHANGES` rows dated by their section, `BEHAVIOR_CHANGE_RELEASES` for `releasedIn(id)` rows. Refuses unmarked files, dangling markers, duplicate ids, bad names, texts over 2,000 characters and a `changelog.d` fragment whose `bump:` is not `patch`. The release PR's `release_tool`; in `regen:all`. Tests: `test/scripts/build-behavior-changes.test.ts`.
diff --git a/docs/ci-upgrade-matrix.md b/docs/ci-upgrade-matrix.md
index a91dcf37e2..4461d00e7b 100644
--- a/docs/ci-upgrade-matrix.md
+++ b/docs/ci-upgrade-matrix.md
@@ -7,8 +7,11 @@ checks the candidate on the upgraded brain. The driver is
[scripts/upgrade-matrix.ts](../scripts/upgrade-matrix.ts) with its modules in
[scripts/upgrade-matrix/](../scripts/upgrade-matrix/).
-It does not gate releases to the latest lane. Stable promotion reads its run
-receipt (GBRA-224 W1).
+It does not gate releases to the latest lane. It gates stable: the daily
+stable check dispatches it on the stable candidate's tag and promotes only on
+a green run of that exact tag and tree, and holds while the nightly run's
+break issue is open ([daily stable check](RELEASING.md#daily-stable-check),
+`scripts/stable-matrix.ts`).
## What a run tests
@@ -16,9 +19,12 @@ receipt (GBRA-224 W1).
releases API, authenticated with the workflow token) and recorded in the plan:
N-1, N-10 and N-50 (the 1st, 10th and 50th newest release older than the
candidate), the release published closest to 30 days before the candidate, and
-the current `stable` tag when it exists and is older. A role that cannot be
-filled is recorded as skipped with its reason. Pull requests run N-1 and N-50
-only.
+the current `stable` tag when it exists and is older, and the stable
+releases in the window (`stable-window`, `versions.ts` `stableWindow`): every
+`stable-v<version>` record older than the candidate that was stable during the
+183 days before it, capped at the newest 8 by promotion plus the newest one of
+each older calendar month. A role that cannot be filled is recorded as
+skipped with its reason. Pull requests run N-1 and N-50 only.
**Cells.** Every from-version runs on PGLite and Postgres, keyless and keyed,
repo-less and repo brain (32 to 40 cells; one job per from-version, engine and
@@ -143,8 +149,13 @@ branch.
- **Nightly** (06:41 UTC): full matrix against the newest release. A red run
opens or updates one issue carrying the `upgrade-break` label (found by the
marker `<!-- gbrain-upgrade-matrix:nightly -->`) and a later green run
- comments and closes it. Nightly watch also tracks the workflow like every
- scheduled run.
+ comments and closes it. The issue's
+ `<!-- gbrain-upgrade-matrix:red-from <tags> -->` line names the from-versions
+ that failed or held. While it is open the daily stable check holds stable
+ (`upgrade-matrix/marker.ts`). Nightly watch also tracks the workflow like
+ every scheduled run.
+- **Stable candidate**: the daily stable check dispatches the full matrix on
+ its candidate's tag when that release has no run yet (`scripts/stable-daily.ts`).
- **Dispatch**: `gh workflow run upgrade-matrix.yml --ref master -f candidate=v0.60.475.0`
(empty candidate: the newest release; `-f from=v0.60.390.0,v0.60.425.0`
replaces the resolved from-versions; `-f fault=corrupt-after-upgrade` is the
@@ -195,7 +206,11 @@ older Bun versions.
A cell takes 40 to 70 seconds on a 4-vCPU runner (install from the Bun cache,
seed, upgrade, checks, mixed-version probe); a job adds about two minutes of
-checkout, `bun install` and cache restore. A full run is 20 cell jobs of about
-four minutes on `ubicloud-standard-4` plus the plan, summary and issue jobs on
-`ubicloud-standard-2`: about 330 vCPU-minutes. The pull-request subset is
-4 jobs, about 50 vCPU-minutes.
+checkout, `bun install` and cache restore. A full run is 4 cell jobs per
+from-version, each about four minutes on `ubicloud-standard-4`, plus the
+plan, summary and issue jobs on `ubicloud-standard-2`. With the stable window
+(on 2026-11-16, candidate v0.60.729.0: 14 from-versions) that is 56 cell jobs
+and 112 cells, about 900 vCPU-minutes, against 20 cell jobs (5 from-versions,
+about 330 vCPU-minutes) before it. The window adds at most one from-version
+per stable promotion until it holds 8 recent ones plus one per older month.
+The pull-request subset is 4 jobs, about 50 vCPU-minutes.
diff --git a/scripts/promote-stable.ts b/scripts/promote-stable.ts
index cb18affc6e..1fce2aa5e5 100644
--- a/scripts/promote-stable.ts
+++ b/scripts/promote-stable.ts
@@ -20,6 +20,11 @@
* the STABLE_SOAK_WORKFLOW repository variable, completed with success;
* 4. the run's head commit has the release tree (the same bytes, not merely
* the same version);
+ * 4b. a forward promotion by the schedule names a green upgrade matrix run on
+ * the release's exact tag and tree (matrix_run_url; scripts/stable-matrix.ts
+ * judgeMatrixRun), and any matrix run named is checked the same way, so a
+ * red or foreign run is refused; a maintainer may promote without one, and
+ * the record then says matrix_run: null with the maintainer as promoted_by;
* 5. the version is strictly newer than the current stable, unless this is a
* rollback with a reason; promoting the current stable again moves no
* tag and only writes its evidence record when the published one does
@@ -46,16 +51,19 @@
* Clients read the record at
* https://raw.githubusercontent.com/garrytan/gbrain/stable-record/stable.json
* (src/core/stable-channel.ts: no API call, no git) and an explicit upgrade
- * checks it against `refs/tags/stable^{}` and `v<VERSION>`. Generation 1
- * records carry soak evidence only; the upgrade matrix (W2) adds matrix_run
- * and sets MATRIX_REQUIRED_FROM in src/core/self-upgrade.ts.
+ * checks it against `refs/tags/stable^{}` and `v<VERSION>`. Records carry the
+ * soak run and, from the matrix gate on, matrix_run; clients require matrix_run
+ * only once MATRIX_REQUIRED_FROM in src/core/self-upgrade.ts is set.
* Side effects (gh, git, clock, log, step summary) come in through
* PromoteDeps, so the decisions are tested without network or a real remote
* (test/scripts/promote-stable.test.ts). Every refusal exits 1 with one
* sentence saying what to do. Docs: docs/RELEASING.md#stable-channel-promotion
*/
import { spawnSync } from 'node:child_process';
-import { appendFileSync } from 'node:fs';
+import { appendFileSync, mkdtempSync, readFileSync, rmSync } from 'node:fs';
+import { tmpdir } from 'node:os';
+import { join } from 'node:path';
+import { judgeMatrixRun, MATRIX_RECEIPT_ARTIFACT, type BreakIssue, type MatrixReceipt, type MatrixRunRow } from './stable-matrix.ts';
export const DOCS = 'docs/RELEASING.md#stable-channel-promotion';
export const STABLE_REF = 'refs/tags/stable';
@@ -81,6 +89,10 @@ export interface PromoteInputs {
notPromotable?: NotPromotable[];
/** Releases to take off the record's not_promotable list (one listed by mistake, say). */
notPromotableRemove?: string[];
+ /** The upgrade matrix run on this release (required for a scheduled forward promotion). */
+ matrixRunUrl?: string;
+ /** The open break issue a security or data-loss promotion passes (stable-daily only); the run may fail only on its named from-versions. */
+ matrixExcuse?: BreakIssue | null;
}
/** A release outside evidence shows is bad although it may soak clean, with the release that fixes it. */
@@ -102,6 +114,8 @@ export interface PromoteDeps {
/** A write to the GitHub API (only the release edit that marks stable Latest). */
ghApiWrite(method: 'PATCH', path: string, body: Record<string, unknown>): Promise<unknown>;
git(args: string[], input?: string): Promise<GitResult>;
+ /** The upgrade-matrix-receipt artifact of a matrix run, or null when it cannot be read. */
+ matrixReceipt?(runId: number): Promise<MatrixReceipt | null>;
now(): Date;
log(line: string): void;
summary(markdown: string): void;
@@ -120,6 +134,8 @@ export interface TagFields {
tree: string;
soakRun: string;
scoreboard: string;
+ /** The upgrade matrix run; absent or empty when a maintainer promoted without one. */
+ matrixRun?: string;
promotedBy: string;
promotedAt: string;
previous: string | null;
@@ -139,6 +155,7 @@ export function renderRecord(f: TagFields, knownBad: readonly string[], notPromo
tree: f.tree,
soak_run: f.soakRun,
scoreboard: f.scoreboard,
+ matrix_run: f.matrixRun || null,
promoted_by: f.promotedBy,
promoted_at: f.promotedAt,
previous: f.previous,
@@ -221,10 +238,11 @@ export function compareVersions(a: string, b: string): number {
return 0;
}
-export function parseRunUrl(url: string, repo: string): { runId: string; attempt: string | null } {
+export function parseRunUrl(url: string, repo: string, input: 'soak_run_url' | 'matrix_run_url' = 'soak_run_url'): { runId: string; attempt: string | null } {
+ const what = input === 'soak_run_url' ? 'soak' : 'upgrade matrix';
const m = /^https:\/\/github\.com\/([\w.-]+\/[\w.-]+)\/actions\/runs\/(\d+)(?:\/attempts\/(\d+))?\/?$/.exec(url.trim());
- if (!m) throw new Refusal(`soak_run_url "${url}" is not a GitHub Actions run URL; pass https://github.com/${repo}/actions/runs/<run id> (optionally /attempts/<n>) of the soak run.`);
- if (m[1]!.toLowerCase() !== repo.toLowerCase()) throw new Refusal(`soak_run_url names a run of ${m[1]}, not ${repo}; pass the soak run from this repository's Actions tab.`);
+ if (!m) throw new Refusal(`${input} "${url}" is not a GitHub Actions run URL; pass https://github.com/${repo}/actions/runs/<run id> (optionally /attempts/<n>) of the ${what} run.`);
+ if (m[1]!.toLowerCase() !== repo.toLowerCase()) throw new Refusal(`${input} names a run of ${m[1]}, not ${repo}; pass the ${what} run from this repository's Actions tab.`);
return { runId: m[2]!, attempt: m[3] ?? null };
}
@@ -263,6 +281,7 @@ export function renderTagMessage(f: TagFields): string {
`tree: ${f.tree}`,
`soak_run: ${f.soakRun}`,
`scoreboard: ${f.scoreboard}`,
+ ...(f.matrixRun ? [`matrix_run: ${f.matrixRun}`] : []),
`promoted_by: ${f.promotedBy}`,
`promoted_at: ${f.promotedAt}`,
`previous: ${f.previous ?? 'none'}`,
@@ -289,8 +308,9 @@ export function parseTagMessage(message: string): TagFields | null {
if (!version || !VERSION_RE.test(version) || !commit || !SHA_RE.test(commit) || !tree || !SHA_RE.test(tree)) return null;
if (!soakRun || !scoreboard || !promotedBy || !promotedAt || !reason || (rollback !== 'true' && rollback !== 'false')) return null;
if (!previous || (previous !== 'none' && !VERSION_RE.test(previous))) return null;
+ const matrixRun = kv.get('matrix_run');
return {
- version, commit, tree, soakRun, scoreboard, promotedBy, promotedAt,
+ version, commit, tree, soakRun, scoreboard, ...(matrixRun ? { matrixRun } : {}), promotedBy, promotedAt,
previous: previous === 'none' ? null : previous,
rollback: rollback === 'true',
reason: reason === '-' ? '' : reason,
@@ -304,6 +324,7 @@ export function renderSummary(p: Promotion, notPromotable: readonly NotPromotabl
['Tree', `\`${p.tree}\``],
['Soak run', p.soakRun],
['Scoreboard', p.scoreboard],
+ ['Upgrade matrix', p.matrixRun || 'none (promoted without one)'],
['Previous stable', p.previous ? `v${p.previous}` : 'none'],
['Rollback', p.rollback ? `yes: ${p.reason.trim()}` : 'no'],
['Known bad', p.knownBad.length ? p.knownBad.map(v => `v${v}`).join(', ') : 'none'],
@@ -455,6 +476,7 @@ export async function promote(deps: PromoteDeps, inputs: PromoteInputs): Promise
const current = await currentStable(deps);
if (current && !rollback && compareVersions(version, current.version) === 0) return recordCurrentStable(deps, inputs, { version, commit, tree }, current);
checkOrder(version, current?.version ?? null, rollback, reason);
+ const matrixRun = await checkMatrix(deps, inputs, { version, commit, tree });
deps.log(`[promote-stable] current stable is ${current ? `v${current.version}` : 'none'}; ${rollback ? `rollback to v${version}: ${reason.trim()}` : `v${version} is newer`}.`);
const recordRef = `refs/tags/stable-v${version}`;
@@ -470,6 +492,7 @@ export async function promote(deps: PromoteDeps, inputs: PromoteInputs): Promise
version, commit, tree,
soakRun: inputs.soakRunUrl.trim(),
scoreboard: inputs.scoreboardUrl.trim() || inputs.soakRunUrl.trim(),
+ ...(matrixRun ? { matrixRun } : {}),
promotedBy: deps.scheduled ? `schedule (${DAILY_WORKFLOW})` : deps.actor,
promotedAt: deps.now().toISOString().replace(/\.\d{3}Z$/, 'Z'),
previous: current?.version ?? null,
@@ -506,6 +529,30 @@ export async function promote(deps: PromoteDeps, inputs: PromoteInputs): Promise
return promotion;
}
+/**
+ * The upgrade matrix run a promotion names, judged by judgeMatrixRun (scripts/stable-matrix.ts); refuses a red,
+ * foreign or unreadable run. A scheduled forward promotion must name one; a maintainer may promote without one.
+ */
+async function checkMatrix(deps: PromoteDeps, inputs: PromoteInputs, release: { version: string; commit: string; tree: string }): Promise<string> {
+ const url = (inputs.matrixRunUrl ?? '').trim();
+ if (!url) {
+ if (inputs.rollback) return '';
+ if (deps.scheduled) throw new Refusal(`a scheduled promotion needs a green upgrade matrix run on v${release.version}'s exact tree; the daily check dispatches one (${MATRIX_DOCS}).`);
+ deps.log(`[promote-stable] no upgrade matrix run given: v${release.version} is promoted on ${deps.actor}'s decision, and the record's matrix_run is null.`);
+ return '';
+ }
+ const { runId, attempt } = parseRunUrl(url, deps.repo, 'matrix_run_url');
+ if (attempt) throw new Refusal(`matrix_run_url names attempt ${attempt}; pass the run URL without /attempts/, whose receipt is the run's latest.`);
+ if (!deps.matrixReceipt) throw new Refusal('This caller cannot read upgrade matrix receipts; run the promotion from promote-stable.yml or stable-daily.yml.');
+ const run = await deps.ghApi(`repos/${deps.repo}/actions/runs/${runId}`) as MatrixRunRow;
+ const judged = judgeMatrixRun({ ...run, html_url: url }, await deps.matrixReceipt(Number(runId)), release, inputs.matrixExcuse);
+ if (!judged.ok) throw new Refusal(`${judged.why}; promote only a release whose upgrade matrix run passed on its exact tree (${MATRIX_DOCS}).`);
+ deps.log(`[promote-stable] ${judged.why}.`);
+ return url;
+}
+
+const MATRIX_DOCS = 'docs/RELEASING.md#daily-stable-check';
+
/** GitHub's limit on a release body is 125,000 characters; stay under it with room for the trailer. */
export const RELEASE_NOTES_MAX = 120_000;
@@ -609,6 +656,18 @@ export function processDeps(env: NodeJS.ProcessEnv = process.env, cwd = process.
const r = spawnSync('git', args, { cwd, env, encoding: 'utf8', maxBuffer: 64 * 1024 * 1024, ...(input !== undefined ? { input } : {}) });
return { code: r.status ?? 1, stdout: r.stdout ?? '', stderr: r.stderr ?? '' };
},
+ async matrixReceipt(runId) {
+ const dir = mkdtempSync(join(tmpdir(), 'matrix-'));
+ try {
+ const r = spawnSync('gh', ['run', 'download', String(runId), '-R', env.GITHUB_REPOSITORY ?? '', '-n', MATRIX_RECEIPT_ARTIFACT, '-D', dir], { cwd, env, encoding: 'utf8' });
+ if (r.status !== 0) return null;
+ return JSON.parse(readFileSync(join(dir, `${MATRIX_RECEIPT_ARTIFACT}.json`), 'utf8')) as MatrixReceipt;
+ } catch {
+ return null;
+ } finally {
+ rmSync(dir, { recursive: true, force: true });
+ }
+ },
now: () => new Date(),
log: line => console.log(line),
summary: markdown => { if (env.GITHUB_STEP_SUMMARY) appendFileSync(env.GITHUB_STEP_SUMMARY, markdown); },
@@ -627,6 +686,7 @@ if (import.meta.main) {
notPromotable: (env.PROMOTE_NOT_PROMOTABLE ?? '').split(',').map(v => v.trim().replace(/^v/, '')).filter(Boolean)
.map(version => ({ version, reason: env.PROMOTE_NOT_PROMOTABLE_REASON ?? '', fixed_in: (env.PROMOTE_NOT_PROMOTABLE_FIXED_IN ?? '').trim().replace(/^v/, '') })),
notPromotableRemove: (env.PROMOTE_NOT_PROMOTABLE_REMOVE ?? '').split(',').map(v => v.trim().replace(/^v/, '')).filter(Boolean),
+ matrixRunUrl: env.PROMOTE_MATRIX_RUN_URL ?? '',
};
const deps = processDeps();
try {
diff --git a/scripts/stable-daily.ts b/scripts/stable-daily.ts
index 1ffd378bdf..50c22dbae7 100644
--- a/scripts/stable-daily.ts
+++ b/scripts/stable-daily.ts
@@ -15,10 +15,15 @@
* promote()'s re-promotion path (no tag moves; the release, exact tree,
* soak run and leases are checked as for any promotion), and stops there
* (GBRA-346: stable was promoted before the record existed).
- * 2. A reason must exist: a merged PR labelled `stable:important` (security,
- * regression, data-loss or blocking fix, set by the merge lane) that is in a
- * published release but not in stable, or stable older than 7 days with a
- * newer release (the weekly floor). No reason: nothing happens.
+ * 2. A reason must exist: a merged PR labelled `stable:important` (set by the
+ * merge lane; the classes are in docs/RELEASING.md) that is in a published
+ * release but not in stable, or stable older than 7 days with a newer
+ * release (the weekly floor). No reason: nothing happens.
+ * 2b. While the upgrade matrix's own break issue is open (scripts/stable-matrix.ts
+ * findBreakIssue: the ISSUE_MARKER issue, not any issue with the label), stable
+ * holds: only an important fix whose fragment says `important: security` or
+ * `important: data-loss` may pass it, and only on a candidate whose matrix run
+ * fails on no from-version the issue does not name. The weekly floor waits.
* 3. The candidate is the newest release on master newer than stable whose exact
* git tree passed a full (SOAK_SHAPE) successful run of the STABLE_SOAK_WORKFLOW
* soak with every integrity gate (INTEGRITY_GATES) passing in every cell; on the
@@ -29,8 +34,14 @@
* then rolled back) or that stable.json lists as known_bad or not_promotable
* (bad on outside evidence the soak cannot see) is never a candidate or soaked
* (#8792).
+ * 3b. The candidate also needs an upgrade matrix run on its exact tag and tree
+ * (stable-matrix.ts judgeMatrixRun): green, or past an open break as in 2b. No
+ * run yet: one matrix dispatch on the candidate tag, unless a matrix run is
+ * active. A red or foreign run holds stable (a maintainer decides) and is never
+ * re-dispatched by the schedule. promote() checks the run again.
* 4. Nothing promotable yet: the newest release is soaked (one dispatch of the
- * soak on its tag) unless a soak is already running, or a soak on any newer
+ * soak on its tag, with an upgrade matrix dispatch on the same tag when it has
+ * no run) unless a soak is already running, or a soak on any newer
* release did not qualify (a maintainer decides; past the weekly floor it
* warns); a failed soak is never retried by the schedule.
*
@@ -44,6 +55,10 @@ import {
authorize, compareVersions, currentStable, DOCS, onReleaseBranch, processDeps, promote, readPriorRecord, refExists, Refusal, requireSoakWorkflow, scheduleAuthorized, STABLE_REF,
type PromoteDeps, type Promotion,
} from './promote-stable.ts';
+import {
+ BREAK_PASS_CLASSES, breakIssueQuery, findBreakIssue, fragmentImportance, isGateRun, judgeMatrixRun, MATRIX_WORKFLOW, receiptNames,
+ type MatrixReceipt, type MatrixRunRow,
+} from './stable-matrix.ts';
export const IMPORTANT_LABEL = 'stable:important';
export const WEEKLY_FLOOR_DAYS = 7;
@@ -64,14 +79,15 @@ export interface Scoreboard {
export interface DailyDeps extends PromoteDeps {
ghPost(path: string, body: unknown): Promise<void>;
scoreboards(runId: number): Promise<Scoreboard[]>;
+ matrixReceipt(runId: number): Promise<MatrixReceipt | null>;
autoPromote: string;
dryRun: boolean;
}
export type DailyResult =
- | { action: 'off' | 'paused' | 'none'; why: string }
+ | { action: 'off' | 'paused' | 'none' | 'held'; why: string }
| { action: 'promoted' | 'backfilled'; why: string; promotion: Promotion }
- | { action: 'soak_dispatched' | 'waiting'; why: string; version: string }
+ | { action: 'soak_dispatched' | 'matrix_dispatched' | 'waiting'; why: string; version: string }
| { action: 'dry_run'; why: string; would: string };
export function parseRecord(message: string): Record<string, string> {
@@ -116,6 +132,18 @@ async function ok(deps: PromoteDeps, args: string[]): Promise<string> {
return r.stdout.trim();
}
+/** The `important:` class of the changelog.d fragment a merge commit added, or null. */
+async function importanceOf(deps: PromoteDeps, commit: string): Promise<string | null> {
+ const added = await deps.git(['diff-tree', '--no-commit-id', '--name-only', '-r', '--diff-filter=A', commit, '--', 'changelog.d']);
+ if (added.code !== 0) return null;
+ for (const path of added.stdout.split('\n').map(p => p.trim()).filter(p => /^changelog\.d\/[^/]+\.md$/.test(p) && !p.endsWith('/README.md'))) {
+ const shown = await deps.git(['show', `${commit}:${path}`]);
+ const cls = shown.code === 0 ? fragmentImportance(shown.stdout) : null;
+ if (cls) return cls;
+ }
+ return null;
+}
+
const isAncestor = async (deps: PromoteDeps, a: string, b: string) => (await deps.git(['merge-base', '--is-ancestor', a, b])).code === 0;
async function treeOf(deps: PromoteDeps, commit: string): Promise<string | null> {
@@ -166,34 +194,58 @@ export async function daily(deps: DailyDeps): Promise<DailyResult> {
const since = new Date((Number.isFinite(promotedAt) ? promotedAt : deps.now().getTime()) - 14 * DAY_MS).toISOString().slice(0, 10);
const q = `repo:${deps.repo} is:pr is:merged label:"${IMPORTANT_LABEL}" merged:>=${since}`;
const found = await deps.ghApi(`search/issues?q=${encodeURIComponent(q)}&per_page=100`) as { items?: Array<{ number: number }> };
- const important: Array<{ pr: number; version: string }> = [];
+ const important: Array<{ pr: number; version: string; passesBreak: boolean }> = [];
for (const { number } of found.items ?? []) {
const pr = await deps.ghApi(`repos/${deps.repo}/pulls/${number}`) as { merge_commit_sha?: string | null };
const sha = pr.merge_commit_sha;
if (!sha || (await treeOf(deps, sha)) === null || await isAncestor(deps, sha, stableCommit)) continue;
for (const r of [...releases].reverse()) {
if (!(await isAncestor(deps, sha, r.commit))) continue;
- important.push({ pr: number, version: r.version });
+ const cls = await importanceOf(deps, sha);
+ important.push({ pr: number, version: r.version, passesBreak: (BREAK_PASS_CLASSES as readonly string[]).includes(cls ?? '') });
break;
}
}
- const weekly = ageDays > WEEKLY_FLOOR_DAYS;
- if (important.length === 0 && !weekly) {
+ const weeklyDue = ageDays > WEEKLY_FLOOR_DAYS;
+ if (important.length === 0 && !weeklyDue) {
return { action: 'none', why: `no released ${IMPORTANT_LABEL} fix is missing from stable v${current.version}, and stable is ${ageDays.toFixed(1)} days old (floor ${WEEKLY_FLOOR_DAYS}).` };
}
+ const breakIssue = findBreakIssue(await deps.ghApi(breakIssueQuery(deps.repo)));
+ const passing = breakIssue ? important.filter(i => i.passesBreak) : important;
+ const weekly = weeklyDue && !breakIssue;
+ if (breakIssue && passing.length === 0) {
+ const why = `the upgrade matrix's break issue #${breakIssue.number} (${breakIssue.url}) is open, so stable v${current.version} holds; only a released important: security or data-loss fix may pass it${important.length ? ` (waiting: ${important.map(i => `#${i.pr}`).join(', ')})` : ''}.`;
+ deps.log(`::warning::Daily stable check: ${why} Docs: docs/RELEASING.md#daily-stable-check`);
+ return { action: 'held', why };
+ }
const reason = [
- ...(important.length ? [`important: ${important.map(i => `#${i.pr} (v${i.version})`).join(', ')}`] : []),
+ ...(passing.length ? [`important: ${passing.map(i => `#${i.pr} (v${i.version})`).join(', ')}`] : []),
...(weekly ? [`weekly floor: stable v${current.version} is ${ageDays.toFixed(1)} days old`] : []),
+ ...(breakIssue ? [`passing open upgrade break #${breakIssue.number} as a security or data-loss fix`] : []),
].join('; ');
deps.log(`[stable-daily] reason: ${reason}.`);
+ const matrixFile = MATRIX_WORKFLOW.split('/').at(-1);
+ const matrixRuns = (((await deps.ghApi(`repos/${deps.repo}/actions/workflows/${matrixFile}/runs?branch=master&per_page=100`)) as { workflow_runs?: MatrixRunRow[] }).workflow_runs ?? []).filter(isGateRun);
+ const receipts = new Map<number, MatrixReceipt | null>();
+ /** The newest completed gate run whose receipt tests this release, or null. */
+ const matrixRunFor = async (version: string): Promise<{ run: MatrixRunRow; receipt: MatrixReceipt | null } | null> => {
+ for (const run of matrixRuns.filter(r => r.status === 'completed')) {
+ if (!receipts.has(run.id)) receipts.set(run.id, await deps.matrixReceipt(run.id));
+ if (receiptNames(receipts.get(run.id)!, version)) return { run, receipt: receipts.get(run.id)! };
+ }
+ return null;
+ };
+ const activeMatrix = matrixRuns.find(r => r.status !== 'completed');
+ const dispatchMatrix = (version: string) => deps.ghPost(`repos/${deps.repo}/actions/workflows/${matrixFile}/dispatches`, { ref: 'master', inputs: { candidate: `v${version}` } });
+
const runs = ((await deps.ghApi(`repos/${deps.repo}/actions/workflows/${soakPath.split('/').at(-1)}/runs?per_page=100`)) as { workflow_runs?: SoakRunRow[] }).workflow_runs ?? [];
const runsByTree = new Map<string, SoakRunRow[]>();
for (const run of runs) {
const tree = await treeOf(deps, run.head_sha);
if (tree) runsByTree.set(tree, [...(runsByTree.get(tree) ?? []), run]);
}
- const oldestImportant = important.reduce<string | null>((v, i) => (!v || compareVersions(i.version, v) < 0 ? i.version : v), null);
+ const oldestImportant = passing.reduce<string | null>((v, i) => (!v || compareVersions(i.version, v) < 0 ? i.version : v), null);
let failedSoak: { version: string; run: SoakRunRow } | null = null;
for (const rel of releases) {
if (!weekly && oldestImportant && compareVersions(rel.version, oldestImportant) < 0) break;
@@ -208,8 +260,25 @@ export async function daily(deps: DailyDeps): Promise<DailyResult> {
deps.log(`[stable-daily] not promotable: v${rel.version} soak ${run.html_url}: ${failures.join(', ')}.`);
}
if (!chosen) { failedSoak ??= completed[0] ? { version: rel.version, run: completed[0] } : null; continue; }
- if (deps.dryRun) return { action: 'dry_run', why: reason, would: `promote v${rel.version} on soak ${chosen.html_url}` };
- const promotion = await promote(deps, { version: rel.version, soakRunUrl: chosen.html_url, scoreboardUrl: '', rollback: false, reason: `scheduled daily check; ${reason}` });
+ const matrix = await matrixRunFor(rel.version);
+ if (!matrix) {
+ if (activeMatrix) return { action: 'waiting', why: `${reason}; v${rel.version} passed its soak and upgrade matrix run ${activeMatrix.html_url} is still running.`, version: rel.version };
+ if (deps.dryRun) return { action: 'dry_run', why: reason, would: `dispatch the upgrade matrix on v${rel.version}` };
+ await dispatchMatrix(rel.version);
+ return { action: 'matrix_dispatched', why: `${reason}; v${rel.version} passed its soak and has no upgrade matrix run yet.`, version: rel.version };
+ }
+ const judged = judgeMatrixRun(matrix.run, matrix.receipt, { version: rel.version, commit: rel.commit, tree: tree! }, breakIssue);
+ if (!judged.ok) {
+ const why = `${reason}; ${judged.why}, so stable holds and a maintainer decides.`;
+ deps.log(`::warning::Daily stable check: ${why} Docs: docs/RELEASING.md#daily-stable-check`);
+ return { action: 'held', why };
+ }
+ if (deps.dryRun) return { action: 'dry_run', why: reason, would: `promote v${rel.version} on soak ${chosen.html_url} and upgrade matrix ${matrix.run.html_url}` };
+ const excused = judged.excused.length ? `; upgrade break #${breakIssue!.number} excused for ${judged.excused.join(', ')}` : '';
+ const promotion = await promote(deps, {
+ version: rel.version, soakRunUrl: chosen.html_url, scoreboardUrl: '', rollback: false, reason: `scheduled daily check; ${reason}${excused}`,
+ matrixRunUrl: matrix.run.html_url, matrixExcuse: breakIssue,
+ });
return { action: 'promoted', why: reason, promotion };
}
@@ -220,8 +289,10 @@ export async function daily(deps: DailyDeps): Promise<DailyResult> {
if (weekly) deps.log(`::warning::Daily stable check: stable v${current.version} is past the ${WEEKLY_FLOOR_DAYS}-day floor and no soak qualifies; a maintainer decides. Docs: docs/RELEASING.md#daily-stable-check`);
return { action: 'waiting', why: `${reason}; v${failedSoak.version} already had a soak (${failedSoak.run.html_url}, ${failedSoak.run.conclusion}) that did not qualify, so no new soak is dispatched and a maintainer decides.`, version: newest.version };
}
- if (deps.dryRun) return { action: 'dry_run', why: reason, would: `dispatch the soak on v${newest.version}` };
+ const matrixToo = !activeMatrix && !(await matrixRunFor(newest.version));
+ if (deps.dryRun) return { action: 'dry_run', why: reason, would: `dispatch the soak${matrixToo ? ' and the upgrade matrix' : ''} on v${newest.version}` };
await deps.ghPost(`repos/${deps.repo}/actions/workflows/${soakPath.split('/').at(-1)}/dispatches`, { ref: `v${newest.version}` });
+ if (matrixToo) await dispatchMatrix(newest.version);
return { action: 'soak_dispatched', why: reason, version: newest.version };
}
@@ -230,6 +301,7 @@ export function dailyDeps(env: NodeJS.ProcessEnv = process.env, cwd = process.cw
return {
...base,
scheduled: scheduleAuthorized(env, base.repo),
+ matrixReceipt: base.matrixReceipt!,
autoPromote: env.STABLE_AUTO_PROMOTE ?? '',
dryRun: env.DAILY_DRY_RUN === 'true',
async ghPost(path, body) {
diff --git a/scripts/stable-matrix.ts b/scripts/stable-matrix.ts
new file mode 100644
index 0000000000..9518ea2cfb
--- /dev/null
+++ b/scripts/stable-matrix.ts
@@ -0,0 +1,112 @@
+/**
+ * The upgrade matrix as a condition of stable promotion (stability plan item 3a, GST-2818).
+ *
+ * A forward promotion names one run of .github/workflows/upgrade-matrix.yml (`matrix_run` in the record and the
+ * promote-stable.yml input). judgeMatrixRun() accepts it only when the run is a full-profile gate run (schedule
+ * or dispatch, from master's harness, no fault injection) whose receipt (the upgrade-matrix-receipt artifact the
+ * summary job writes) names exactly the release being promoted (tag, commit and tree) and was written by that
+ * run's own harness, and when the run is green. The one way past a red run is the break path below.
+ *
+ * The daily check (scripts/stable-daily.ts) also holds stable while the matrix's own break issue is open
+ * (ISSUE_MARKER, upgrade-matrix/marker.ts); any other issue with the label holds nothing. A security or data-loss
+ * fix may pass an open break when its candidate's run fails only on from-versions that issue names: the break is
+ * already known and the fix makes nothing worse elsewhere. Otherwise a maintainer promotes by hand
+ * (promote-stable.yml), which the record shows as promoted_by.
+ *
+ * No dependencies beyond node: stable-daily.yml and promote-stable.yml install none.
+ */
+import { failingFrom, ISSUE_LABEL, ISSUE_MARKER, parseRedFrom } from './upgrade-matrix/marker.ts';
+
+export const MATRIX_WORKFLOW = '.github/workflows/upgrade-matrix.yml';
+export const MATRIX_RECEIPT_ARTIFACT = 'upgrade-matrix-receipt';
+export const MATRIX_RECEIPT_SCHEMA = 'gbrain-upgrade-matrix-run/1';
+/** The `important:` classes that may pass an open upgrade break (with a run that fails only on the from-versions it names). */
+export const BREAK_PASS_CLASSES = ['security', 'data-loss'] as const;
+
+export interface MatrixRunRow {
+ id: number;
+ status: string | null;
+ conclusion: string | null;
+ head_sha: string;
+ head_branch?: string | null;
+ event?: string | null;
+ path?: string | null;
+ html_url: string;
+}
+
+/** The run receipt (scripts/upgrade-matrix.ts buildSummary), as far as promotion reads it. */
+export interface MatrixReceipt {
+ schema?: string;
+ harness?: { sha?: string };
+ candidate?: { tag?: string | null; sha?: string; tree?: string };
+ profile?: string;
+ from?: Array<{ tag: string; roles?: string[] }>;
+ skipped?: Array<{ role: string; reason?: string }>;
+ status?: string;
+ cells?: Array<{ id: string; from: string; verdict: string; fault?: string | null }>;
+ red?: string[];
+ holds?: string[];
+ foreign_receipts?: string[];
+}
+
+export interface BreakIssue { number: number; url: string; from: string[] | null }
+
+export type MatrixJudgement = { ok: true; why: string; excused: string[] } | { ok: false; why: string };
+
+const normalise = (p: string) => p.trim().replace(/@.*$/, '').replace(/^\.\//, '');
+
+/** Runs whose receipt can be gate evidence: master's harness, by schedule or dispatch (pull-request runs test their own harness). */
+export const isGateRun = (run: MatrixRunRow) => (run.event === 'schedule' || run.event === 'workflow_dispatch') && run.head_branch === 'master';
+
+/** A receipt from a harness that resolves the stable-window role (scripts/upgrade-matrix/versions.ts): it tested or skipped it. Older receipts never tested the stable releases users may still be on. */
+const coversStableWindow = (r: MatrixReceipt) =>
+ (r.from ?? []).some(f => (f.roles ?? []).includes('stable-window')) || (r.skipped ?? []).some(s => s.role === 'stable-window');
+
+/** Does this receipt test this release as a gate run (full profile with the stable window, no fault injection)? A candidate tag alone picks the run; judgeMatrixRun checks the rest. */
+export const receiptNames = (r: MatrixReceipt | null, version: string) =>
+ !!r && r.candidate?.tag === `v${version}` && r.profile === 'full' && coversStableWindow(r) && !(r.cells ?? []).some(c => c.fault);
+
+/**
+ * Whether a matrix run qualifies the release for stable. `excuse` is the break path: the open break issue's named
+ * from-versions, given only for a security or data-loss candidate while that issue is open.
+ */
+export function judgeMatrixRun(run: MatrixRunRow, receipt: MatrixReceipt | null, release: { version: string; commit: string; tree: string }, excuse?: BreakIssue | null): MatrixJudgement {
+ const url = run.html_url;
+ if (run.path && normalise(run.path) !== MATRIX_WORKFLOW) return { ok: false, why: `run ${url} ran ${normalise(run.path)}, not the upgrade matrix ${MATRIX_WORKFLOW}` };
+ if (!isGateRun(run)) return { ok: false, why: `upgrade matrix run ${url} is a ${run.event ?? 'unknown'} run on ${run.head_branch ?? 'an unknown branch'}; only a schedule or dispatch run of master's harness is gate evidence` };
+ if (run.status !== 'completed') return { ok: false, why: `upgrade matrix run ${url} is ${run.status ?? 'in an unknown state'}, not completed` };
+ if (!receipt) return { ok: false, why: `upgrade matrix run ${url} has no ${MATRIX_RECEIPT_ARTIFACT} artifact to read` };
+ if (receipt.schema !== MATRIX_RECEIPT_SCHEMA) return { ok: false, why: `upgrade matrix run ${url}'s receipt is not a ${MATRIX_RECEIPT_SCHEMA} receipt` };
+ if (receipt.harness?.sha !== run.head_sha) return { ok: false, why: `upgrade matrix run ${url}'s receipt names harness ${receipt.harness?.sha ?? 'none'}, not the run's commit ${run.head_sha}` };
+ const c = receipt.candidate ?? {};
+ if (c.tag !== `v${release.version}` || c.sha !== release.commit || c.tree !== release.tree) {
+ return { ok: false, why: `upgrade matrix run ${url} tested ${c.tag ?? 'an untagged commit'} (commit ${c.sha ?? '?'}, tree ${c.tree ?? '?'}), not v${release.version} (commit ${release.commit}, tree ${release.tree})` };
+ }
+ if (receipt.profile !== 'full') return { ok: false, why: `upgrade matrix run ${url} ran the ${receipt.profile ?? 'unknown'} profile, not the full from-version set` };
+ if (!coversStableWindow(receipt)) return { ok: false, why: `upgrade matrix run ${url} predates the stable-window from-versions, so it never tested upgrades from the stable releases users may still be on` };
+ if ((receipt.cells ?? []).some(cell => cell.fault)) return { ok: false, why: `upgrade matrix run ${url} injected a fault (a proof run), so it qualifies nothing` };
+ if (receipt.status === 'green' && run.conclusion === 'success') return { ok: true, why: `upgrade matrix run ${url} is green on v${release.version}`, excused: [] };
+ const failing = failingFrom(receipt);
+ const what = `upgrade matrix run ${url} is ${receipt.status ?? run.conclusion ?? 'not green'} on v${release.version}${failing.from.length ? ` (failing from ${failing.from.join(', ')})` : ''}${failing.door ? ' with a failing door' : ''}`;
+ if (!excuse) return { ok: false, why: what };
+ if (!excuse.from) return { ok: false, why: `${what}, and break issue #${excuse.number} names no from-versions, so nothing is excused` };
+ if ((receipt.foreign_receipts ?? []).length || failing.door || failing.from.length === 0) return { ok: false, why: `${what}, which the break issue cannot excuse` };
+ const outside = failing.from.filter(f => !excuse.from!.includes(f));
+ if (outside.length) return { ok: false, why: `${what}; break issue #${excuse.number} names only ${excuse.from.join(', ')}, so ${outside.join(', ')} is a new break` };
+ return { ok: true, why: `${what}, all named by break issue #${excuse.number}`, excused: failing.from };
+}
+
+/** The matrix's own open break issue (labelled and carrying ISSUE_MARKER), or null. A labelled issue without the marker holds nothing. */
+export function findBreakIssue(issues: unknown): BreakIssue | null {
+ const rows = Array.isArray(issues) ? issues as Array<{ number?: number; html_url?: string; body?: string | null; state?: string; pull_request?: unknown }> : [];
+ const hit = rows.find(i => !i.pull_request && i.state !== 'closed' && typeof i.number === 'number' && String(i.body ?? '').includes(ISSUE_MARKER));
+ return hit ? { number: hit.number!, url: hit.html_url ?? `#${hit.number}`, from: parseRedFrom(String(hit.body ?? '')) } : null;
+}
+
+export const breakIssueQuery = (repo: string) => `repos/${repo}/issues?state=open&labels=${ISSUE_LABEL}&per_page=100`;
+
+/** The `important:` class in a changelog.d fragment's front matter, or null. */
+export function fragmentImportance(text: string): string | null {
+ const m = /^---\r?\n([\s\S]*?)\r?\n---/.exec(text);
+ return m ? /^important:\s*([a-z-]+)\s*$/m.exec(m[1]!)?.[1] ?? null : null;
+}
diff --git a/scripts/upgrade-matrix.ts b/scripts/upgrade-matrix.ts
index a03da238fd..6329dfbf27 100644
--- a/scripts/upgrade-matrix.ts
+++ b/scripts/upgrade-matrix.ts
@@ -18,7 +18,8 @@
import { appendFileSync, existsSync, mkdirSync, readFileSync, readdirSync, writeFileSync } from 'node:fs';
import { execFileSync, spawnSync } from 'node:child_process';
import { join, resolve } from 'node:path';
-import { fetchReleases, hasEnoughHistory, RELEASE_TAG, resolveFromVersions, type Candidate, type FromVersion, type Profile, type Release } from './upgrade-matrix/versions.ts';
+import { fetchReleases, hasEnoughHistory, RELEASE_TAG, resolveFromVersions, stableWindow, type Candidate, type FromVersion, type Profile, type Release, type StableRecordTag } from './upgrade-matrix/versions.ts';
+import { failingFrom, ISSUE_LABEL, ISSUE_MARKER, renderRedFrom } from './upgrade-matrix/marker.ts';
import { cellId, runCell, type BrainShape, type Engine, type Fault, type Keys } from './upgrade-matrix/cell.ts';
import { CHECKS, loadKnownExceptions, summarize, type CellSummary } from './upgrade-matrix/verdict.ts';
import { AGENT_FILED_LABEL } from './lib/gh-issue.ts';
@@ -26,8 +27,7 @@ import { AGENT_FILED_LABEL } from './lib/gh-issue.ts';
const REPO = 'garrytan/gbrain';
const ROOT = resolve(import.meta.dir, '..');
const BT = '\x60';
-export const ISSUE_MARKER = '<!-- gbrain-upgrade-matrix:nightly -->';
-export const ISSUE_LABEL = 'upgrade-break';
+export { ISSUE_LABEL, ISSUE_MARKER };
const argv = process.argv.slice(2);
const flag = (name: string, fallback?: string) => { const i = argv.indexOf(name); return i >= 0 && i + 1 < argv.length ? argv[i + 1]! : fallback; };
@@ -61,9 +61,12 @@ async function plan(): Promise<void> {
const profile = flag('--profile', 'full') as Profile;
if (profile !== 'full' && profile !== 'pr') throw new Error('--profile takes full or pr');
const input = (flag('--candidate', '') ?? '').trim();
+ const stableRecords = profile === 'full' ? stableRecordTags() : [];
const releases = await fetchReleases(REPO, rs => {
const newest = rs[0];
- return !!newest && hasEnoughHistory(rs, candidateHint(rs, input) ?? newest);
+ if (!newest) return false;
+ const hint = candidateHint(rs, input) ?? newest;
+ return hasEnoughHistory(rs, hint) && stableWindow(stableRecords, hint).every(v => rs.some(r => r.version === v));
});
if (!releases.length) throw new Error('no published releases found');
let candidate: Candidate;
@@ -86,7 +89,7 @@ async function plan(): Promise<void> {
return { tag, version: m[1]!, published_at: releases.find(r => r.tag === tag)?.published_at ?? '', roles: ['explicit' as never] };
});
} else {
- const resolved = resolveFromVersions(releases, candidate, { profile, stableVersion: stableVersion() });
+ const resolved = resolveFromVersions(releases, candidate, { profile, stableVersion: stableVersion(), stableRecords });
from = resolved.from;
skipped = resolved.skipped;
}
@@ -118,6 +121,17 @@ function stableVersion(): string | null {
try { return commitVersion(ensureCommit('stable')); } catch { return null; }
}
+/** Every stable-v<version> record tag on origin with its creation time (the promotion), fetched first. */
+function stableRecordTags(): StableRecordTag[] {
+ execFileSync('git', ['-C', ROOT, 'fetch', '--quiet', '--no-tags', '--force', 'origin', '+refs/tags/stable-v*:refs/tags/stable-v*'], { stdio: ['ignore', 'pipe', 'pipe'] });
+ const out = git('for-each-ref', '--format=%(refname:strip=2)%09%(creatordate:iso-strict)', 'refs/tags/stable-v*');
+ return out.split('\n').filter(Boolean).flatMap(line => {
+ const [name, date] = line.split('\t');
+ const m = /^stable-v(\d+\.\d+\.\d+(?:\.\d+)?)$/.exec(name ?? '');
+ return m && date ? [{ version: m[1]!, promoted_at: new Date(date).toISOString() }] : [];
+ });
+}
+
function candidateHint(releases: Release[], input: string): { version: string; published_at: string } | null {
if (!input) return null;
const rel = releases.find(r => r.tag === input);
@@ -228,7 +242,8 @@ async function issue(): Promise<void> {
return res.status === 204 ? null : res.json();
};
const existing = (await api('GET', '/issues?state=all&labels=' + ISSUE_LABEL + '&per_page=100') as any[]).find(i => !i.pull_request && String(i.body ?? '').includes(ISSUE_MARKER));
- const body = ISSUE_MARKER + '\n' + renderSummary(s) + '\n\nRun: ' + (s.harness.run_url ?? '(local)') + '\n\nThis issue carries the ' + BT + ISSUE_LABEL + BT + ' label, which holds stable promotion while it is open. The nightly matrix closes it when a later run is green.';
+ const body = ISSUE_MARKER + '\n' + renderRedFrom(failingFrom(s).from) + '\n' + renderSummary(s) + '\n\nRun: ' + (s.harness.run_url ?? '(local)')
+ + '\n\nWhile this issue is open the daily stable check holds stable (docs/RELEASING.md#daily-stable-check); only a security or data-loss fix that is green on every from-version this issue does not name may pass it. The nightly matrix closes it when a later run is green.';
if (s.status === 'red') {
const title = 'Upgrade matrix red: upgrading to ' + (s.candidate.tag ?? s.candidate.sha.slice(0, 12)) + ' fails ' + s.red.length + ' cell(s)';
if (dry) { console.log((existing ? 'update #' + existing.number : 'create') + ': ' + title + '\n' + body); return; }
diff --git a/scripts/upgrade-matrix/marker.ts b/scripts/upgrade-matrix/marker.ts
new file mode 100644
index 0000000000..0c85a1f20c
--- /dev/null
+++ b/scripts/upgrade-matrix/marker.ts
@@ -0,0 +1,47 @@
+/**
+ * The upgrade matrix's own break issue (GBRA-224 W2, stability plan item 3a): the nightly run opens or
+ * updates one issue carrying ISSUE_MARKER and the ISSUE_LABEL label when it is red, and closes it when a later
+ * run is green. scripts/stable-daily.ts holds stable while that issue is open; any other issue with the label
+ * holds nothing. The issue names the from-versions that failed on a RED_FROM_MARKER line, which is what lets a
+ * security or data-loss fix pass an open break when it is green on every other from-version.
+ * No imports: stable-daily.yml installs no dependencies.
+ */
+export const ISSUE_MARKER = '<!-- gbrain-upgrade-matrix:nightly -->';
+export const ISSUE_LABEL = 'upgrade-break';
+const RED_FROM_PREFIX = '<!-- gbrain-upgrade-matrix:red-from ';
+
+/** A run summary's shape as far as the failing from-versions go (scripts/upgrade-matrix.ts buildSummary). */
+export interface RunSummaryCells {
+ cells?: Array<{ id: string; from: string; verdict: string; fault?: string | null }>;
+ red?: string[];
+ holds?: string[];
+}
+
+/** Cell ids are `<from tag>__<engine>__<keys>__<brain>` (cell.ts cellId); a door id (`door:<name>`) has no from-version. */
+const fromOfId = (id: string): string | null => (id.startsWith('door:') ? null : id.split('__')[0] || null);
+
+/**
+ * The from-versions with a red or held cell, sorted, and whether a door failed or held (a door has no
+ * from-version, so nothing can excuse it).
+ */
+export function failingFrom(s: RunSummaryCells): { from: string[]; door: boolean } {
+ const out = new Set<string>();
+ let door = false;
+ for (const id of [...(s.red ?? []), ...(s.holds ?? [])]) {
+ const cell = s.cells?.find(c => c.id === id);
+ const from = cell ? cell.from : fromOfId(id);
+ if (from === null) door = true;
+ else out.add(from);
+ }
+ return { from: [...out].sort(), door };
+}
+
+export const renderRedFrom = (from: readonly string[]) => RED_FROM_PREFIX + from.join(',') + ' -->';
+
+/** The from-versions an issue body names, or null when it names none (an issue written before this line existed). */
+export function parseRedFrom(body: string): string[] | null {
+ const line = body.split('\n').find(l => l.startsWith(RED_FROM_PREFIX) && l.trimEnd().endsWith('-->'));
+ if (!line) return null;
+ const from = line.slice(RED_FROM_PREFIX.length).replace(/-->\s*$/, '').split(',').map(t => t.trim()).filter(t => /^v\d+\.\d+\.\d+(\.\d+)?$/.test(t));
+ return from.length ? from : null;
+}
diff --git a/scripts/upgrade-matrix/versions.ts b/scripts/upgrade-matrix/versions.ts
index 5827dfa668..3ead67b2f1 100644
--- a/scripts/upgrade-matrix/versions.ts
+++ b/scripts/upgrade-matrix/versions.ts
@@ -4,16 +4,23 @@
* Resolved once per run from published release tags and recorded in the run
* receipt: N-1, N-10 and N-50 (the 1st, 10th and 50th newest release older
* than the candidate), the release published closest to 30 days before the
- * candidate, and the current stable tag when one exists and is older. The
- * pull-request profile keeps N-1 and N-50. A role that cannot be filled (too
- * few older releases, stable missing or not older) is recorded as skipped
- * with the reason, never silently dropped.
+ * candidate, the current stable tag when one exists and is older, and the
+ * stable releases in the promise window (stableWindow: the last 8 plus one per
+ * older month, back 6 months). The pull-request profile keeps N-1 and N-50. A
+ * role that cannot be filled (too few older releases, stable missing or not
+ * older) is recorded as skipped with the reason, never silently dropped.
*/
import { isNewerVersion, isValidVersionString } from '../../src/core/semver.ts';
-export type FromRole = 'N-1' | 'N-10' | 'N-50' | '30d' | 'stable';
+export type FromRole = 'N-1' | 'N-10' | 'N-50' | '30d' | 'stable' | 'stable-window';
export type Profile = 'full' | 'pr';
-export const PROFILE_ROLES: Record<Profile, FromRole[]> = { full: ['N-1', 'N-10', 'N-50', '30d', 'stable'], pr: ['N-1', 'N-50'] };
+export const PROFILE_ROLES: Record<Profile, FromRole[]> = { full: ['N-1', 'N-10', 'N-50', '30d', 'stable', 'stable-window'], pr: ['N-1', 'N-50'] };
+
+/** A stable record tag (stable-v<version>) and when it was created, which is when that release was promoted. */
+export interface StableRecordTag { version: string; promoted_at: string }
+/** How far back a stable user is still promised an upgrade (docs/STABILITY.md), and how many recent stable releases are all kept. */
+export const STABLE_WINDOW_DAYS = 183;
+export const STABLE_WINDOW_RECENT = 8;
export interface Release { tag: string; version: string; published_at: string }
export interface Candidate { tag: string | null; sha: string; tree: string; version: string; published_at: string }
@@ -33,7 +40,7 @@ export function olderReleases(releases: readonly Release[], candidateVersion: st
export function resolveFromVersions(
releases: readonly Release[],
candidate: Pick<Candidate, 'version' | 'published_at'>,
- opts: { profile: Profile; stableVersion?: string | null },
+ opts: { profile: Profile; stableVersion?: string | null; stableRecords?: readonly StableRecordTag[] },
): { from: FromVersion[]; skipped: SkippedRole[] } {
const older = olderReleases(releases, candidate.version);
const picked = new Map<string, FromVersion>();
@@ -56,6 +63,10 @@ export function resolveFromVersions(
if (!best || d < Math.abs(Date.parse(best.published_at) - target)) best = r;
}
take(role, best, 'no published release is older than the candidate');
+ } else if (role === 'stable-window') {
+ const window = stableWindow(opts.stableRecords ?? [], candidate);
+ if (window.length === 0) skipped.push({ role, reason: 'no stable release older than the candidate is in the window' });
+ for (const v of window) take(role, older.find(r => r.version === v), 'stable ' + v + ' has no published release');
} else {
const stable = opts.stableVersion ?? null;
if (!stable) { skipped.push({ role, reason: 'no stable tag exists' }); continue; }
@@ -66,6 +77,29 @@ export function resolveFromVersions(
return { from: [...picked.values()], skipped };
}
+/**
+ * The stable releases a stable user may still be upgrading from: every stable release older than the candidate
+ * that was stable at some point in the last STABLE_WINDOW_DAYS before the candidate (promoted inside the window, or
+ * the one that was stable when the window opened), capped at the newest STABLE_WINDOW_RECENT by promotion plus the
+ * newest one of each older calendar month (UTC). Newest first.
+ */
+export function stableWindow(records: readonly StableRecordTag[], candidate: Pick<Candidate, 'version' | 'published_at'>): string[] {
+ const horizon = Date.parse(candidate.published_at) - STABLE_WINDOW_DAYS * DAY_MS;
+ const older = records
+ .filter(r => isValidVersionString(r.version) && isNewerVersion(r.version, candidate.version) && Number.isFinite(Date.parse(r.promoted_at)))
+ .sort((a, b) => Date.parse(b.promoted_at) - Date.parse(a.promoted_at));
+ const inWindow = older.filter((_, i) => i === 0 || Date.parse(older[i - 1]!.promoted_at) >= horizon);
+ const months = new Set<string>();
+ const out: string[] = [];
+ inWindow.forEach((r, i) => {
+ const month = r.promoted_at.slice(0, 7);
+ if (i < STABLE_WINDOW_RECENT) out.push(r.version);
+ else if (!months.has(month)) out.push(r.version);
+ if (i >= STABLE_WINDOW_RECENT) months.add(month);
+ });
+ return [...new Set(out)];
+}
+
function token(): string | undefined {
return process.env.GITHUB_TOKEN || process.env.GH_TOKEN || undefined;
}
diff --git a/test/scripts/promote-stable.test.ts b/test/scripts/promote-stable.test.ts
index 5930ec0285..14282ffb7e 100644
--- a/test/scripts/promote-stable.test.ts
+++ b/test/scripts/promote-stable.test.ts
@@ -22,6 +22,7 @@ import {
checkOrder, checkSoakRun, compareVersions, RELEASE_NOTES_MAX, rolledUpNotes, mergeKnownBad, mergeNotPromotable, normaliseWorkflowPath, parseRunUrl, parseTagMessage, processDeps, promote, Refusal, renderRecord, renderSummary, renderTagMessage, STABLE_RECORD_MAX_BYTES,
type GitResult, type PromoteDeps, type PromoteInputs, type SoakRun, type TagFields,
} from '../../scripts/promote-stable.ts';
+import type { MatrixReceipt } from '../../scripts/stable-matrix.ts';
import { offeredVersion } from '../../src/core/self-upgrade.ts';
import { RECORD_MAX_BYTES } from '../../src/core/stable-channel.ts';
import { parseStableRecord } from '../../src/core/stable-channel.ts';
@@ -732,6 +733,64 @@ describe('stable.json bridge for a stable promoted before the evidence record (G
});
});
+describe('promote: the upgrade matrix run (stability plan 3a)', () => {
+ const MATRIX_URL = `https://github.com/${REPO}/actions/runs/777`;
+ const HARNESS = sha('f');
+ const greenRun = { id: 777, status: 'completed', conclusion: 'success', head_sha: HARNESS, head_branch: 'master', event: 'schedule', path: '.github/workflows/upgrade-matrix.yml' };
+ const greenReceipt = (): MatrixReceipt => ({
+ schema: 'gbrain-upgrade-matrix-run/1', harness: { sha: HARNESS }, candidate: { tag: 'v0.60.361.0', sha: sha('b'), tree: sha('2') }, profile: 'full',
+ status: 'green', cells: [], red: [], holds: [], foreign_receipts: [], from: [{ tag: 'v0.60.360.0', roles: ['N-1', 'stable-window'] }], skipped: [],
+ });
+ function withMatrix(run: Record<string, unknown>, receipt: MatrixReceipt | null, setup: Parameters<typeof harness>[0] = {}) {
+ const h = harness(setup);
+ const ghApi = h.deps.ghApi;
+ h.deps.ghApi = async path => (path.endsWith('/actions/runs/777') ? (h.api.push(path), run) : ghApi(path));
+ h.deps.matrixReceipt = async id => (id === 777 ? structuredClone(receipt) : null);
+ return h;
+ }
+
+ test('a green gate run on the exact tag and tree lands in stable.json and the tag message, and round-trips', async () => {
+ const h = withMatrix(greenRun, greenReceipt());
+ const p = await promote(h.deps, inputs({ matrixRunUrl: MATRIX_URL }));
+ expect(p.matrixRun).toBe(MATRIX_URL);
+ expect(JSON.parse(h.repo.records![h.repo.recordBranch!]!)).toMatchObject({ matrix_run: MATRIX_URL });
+ expect(parseStableRecord(h.repo.records![h.repo.recordBranch!]!, '0.60.361.0')).toMatchObject({ ok: true, record: { matrix_run: MATRIX_URL } });
+ expect(h.repo.tags.stable!.message).toContain(`soak_run: ${RUN_URL}\nscoreboard: ${RUN_URL}\nmatrix_run: ${MATRIX_URL}\npromoted_by: `);
+ expect(parseTagMessage(h.repo.tags.stable!.message!)).toMatchObject({ matrixRun: MATRIX_URL });
+ expect(renderSummary(p)).toContain(MATRIX_URL);
+ });
+
+ test('a maintainer may promote without a matrix run, recorded as matrix_run null with the maintainer as promoted_by', async () => {
+ const h = harness();
+ await promote(h.deps, inputs());
+ expect(JSON.parse(h.repo.records![h.repo.recordBranch!]!)).toMatchObject({ matrix_run: null, promoted_by: 'alice-example' });
+ expect(h.logs.some(l => l.includes("promoted on alice-example's decision"))).toBe(true);
+ });
+
+ test('a red, foreign, unfinished, pull-request, partial or unreadable run is refused and nothing moves', async () => {
+ const red = { ...greenReceipt(), status: 'red', red: ['v0.60.241.0__postgres__keyed__repo'], cells: [{ id: 'v0.60.241.0__postgres__keyed__repo', from: 'v0.60.241.0', verdict: 'fail' }] };
+ const cases: Array<[Record<string, unknown>, MatrixReceipt | null, RegExp, string?]> = [
+ [{ ...greenRun, conclusion: 'failure' }, red, /is red on v0\.60\.361\.0 \(failing from v0\.60\.241\.0\)/],
+ [greenRun, { ...greenReceipt(), candidate: { tag: 'v0.60.360.0', sha: sha('a'), tree: sha('1') } }, /tested v0\.60\.360\.0 .*not v0\.60\.361\.0/],
+ [greenRun, { ...greenReceipt(), candidate: { tag: 'v0.60.361.0', sha: sha('b'), tree: sha('9') } }, /tree 9+\), not v0\.60\.361\.0/],
+ [greenRun, { ...greenReceipt(), harness: { sha: sha('e') } }, /receipt names harness e+, not the run's commit f+/],
+ [{ ...greenRun, event: 'pull_request', head_branch: 'fix/x' }, greenReceipt(), /is a pull_request run on fix\/x/],
+ [{ ...greenRun, status: 'in_progress', conclusion: null }, greenReceipt(), /is in_progress, not completed/],
+ [{ ...greenRun, path: '.github/workflows/soak.yml' }, greenReceipt(), /ran \.github\/workflows\/soak\.yml, not the upgrade matrix/],
+ [greenRun, { ...greenReceipt(), profile: 'explicit' }, /ran the explicit profile/],
+ [greenRun, { ...greenReceipt(), from: [{ tag: 'v0.60.360.0', roles: ['N-1'] }], skipped: [] }, /predates the stable-window from-versions/],
+ [greenRun, null, /has no upgrade-matrix-receipt artifact/],
+ [greenRun, greenReceipt(), /matrix_run_url names a run of acme-example\/gbrain/, 'https://github.com/acme-example/gbrain/actions/runs/777'],
+ [greenRun, greenReceipt(), /matrix_run_url names attempt 2/, `${MATRIX_URL}/attempts/2`],
+ ];
+ for (const [run, receipt, pattern, url] of cases) {
+ const h = withMatrix(run, receipt);
+ await refusedAsync(promote(h.deps, inputs({ matrixRunUrl: url ?? MATRIX_URL })), pattern);
+ expect(h.calls.filter(c => c.startsWith('push') || c.startsWith('tag -a'))).toEqual([]);
+ }
+ });
+});
+
describe('promote-stable.yml', () => {
const workflowYml = readFileSync(join(ROOT, '.github/workflows/promote-stable.yml'), 'utf8');
const wf = load(workflowYml) as {
@@ -744,8 +803,8 @@ describe('promote-stable.yml', () => {
test('dispatch only, with the decided inputs, one serial group and scoped permissions', () => {
expect(Object.keys(wf.on)).toEqual(['workflow_dispatch']);
const i = wf.on.workflow_dispatch!.inputs;
- expect(Object.keys(i)).toEqual(['version', 'soak_run_url', 'scoreboard_url', 'rollback', 'reason', 'withdraw', 'not_promotable', 'not_promotable_reason', 'not_promotable_fixed_in', 'not_promotable_remove']);
- for (const k of ['withdraw', 'not_promotable', 'not_promotable_reason', 'not_promotable_fixed_in', 'not_promotable_remove']) expect(i[k]).toMatchObject({ required: false, type: 'string', default: '' });
+ expect(Object.keys(i)).toEqual(['version', 'soak_run_url', 'matrix_run_url', 'scoreboard_url', 'rollback', 'reason', 'withdraw', 'not_promotable', 'not_promotable_reason', 'not_promotable_fixed_in', 'not_promotable_remove']);
+ for (const k of ['matrix_run_url', 'withdraw', 'not_promotable', 'not_promotable_reason', 'not_promotable_fixed_in', 'not_promotable_remove']) expect(i[k]).toMatchObject({ required: false, type: 'string', default: '' });
expect([i.version!.required, i.soak_run_url!.required, i.scoreboard_url!.required]).toEqual([true, true, false]);
expect(i.rollback).toMatchObject({ type: 'boolean', default: false });
expect(wf.concurrency).toEqual({ group: 'promote-stable', 'cancel-in-progress': false });
@@ -760,7 +819,7 @@ describe('promote-stable.yml', () => {
PROMOTE_VERSION: '${{ inputs.version }}', PROMOTE_SOAK_RUN_URL: '${{ inputs.soak_run_url }}', PROMOTE_SCOREBOARD_URL: '${{ inputs.scoreboard_url }}',
PROMOTE_ROLLBACK: '${{ inputs.rollback }}', PROMOTE_REASON: '${{ inputs.reason }}', PROMOTE_WITHDRAW: '${{ inputs.withdraw }}',
PROMOTE_NOT_PROMOTABLE: '${{ inputs.not_promotable }}', PROMOTE_NOT_PROMOTABLE_REASON: '${{ inputs.not_promotable_reason }}', PROMOTE_NOT_PROMOTABLE_FIXED_IN: '${{ inputs.not_promotable_fixed_in }}',
- PROMOTE_NOT_PROMOTABLE_REMOVE: '${{ inputs.not_promotable_remove }}',
+ PROMOTE_NOT_PROMOTABLE_REMOVE: '${{ inputs.not_promotable_remove }}', PROMOTE_MATRIX_RUN_URL: '${{ inputs.matrix_run_url }}',
});
for (const s of steps) expect(s.run ?? '').not.toContain('${{');
const releaseYml = readFileSync(join(ROOT, '.github/workflows/release.yml'), 'utf8');
diff --git a/test/scripts/stable-daily.test.ts b/test/scripts/stable-daily.test.ts
index 4ef785d989..1882aaa197 100644
--- a/test/scripts/stable-daily.test.ts
+++ b/test/scripts/stable-daily.test.ts
@@ -18,6 +18,7 @@ import { join } from 'node:path';
import { load } from 'js-yaml';
import { promote, renderTagMessage, scheduleAuthorized, Refusal, type GitResult } from '../../scripts/promote-stable.ts';
import { daily, integrityFailures, parseRecord, shapeFailures, type DailyDeps, type Scoreboard, type SoakRunRow } from '../../scripts/stable-daily.ts';
+import type { MatrixReceipt, MatrixRunRow } from '../../scripts/stable-matrix.ts';
import { parseStableRecord } from '../../src/core/stable-channel.ts';
const ROOT = join(import.meta.dir, '..', '..');
@@ -40,8 +41,13 @@ interface World {
master?: number;
releases: Record<string, number>;
stable: { version: string; commit: number; message: string };
- importantPrs?: Array<{ number: number; merge: number }>;
+ /** `important`: the class its changelog.d fragment's front matter names. */
+ importantPrs?: Array<{ number: number; merge: number; important?: string }>;
runs?: Array<SoakRunRow & { boards?: Scoreboard[] }>;
+ /** Upgrade matrix runs, newest first; default: one green gate run on every release (MATRIX_ID + its commit). */
+ matrix?: Array<MatrixRunRow & { receipt?: MatrixReceipt | null }>;
+ /** Open issues with the upgrade-break label (the issues query's answer). */
+ breakIssues?: unknown[];
autoPromote?: string;
dryRun?: boolean;
scheduled?: boolean;
@@ -102,6 +108,14 @@ function world(w: World) {
tags[name] = { commit: commitOf(commit)!, object: `tagobj-${name}-new`, message: rest[rest.indexOf('-m') + 1] };
return ok();
}
+ if (cmd === 'diff-tree') {
+ const pr = (w.importantPrs ?? []).find(p => p.important && sha(p.merge) === rest.at(-3));
+ return pr ? ok(`changelog.d/fix-${pr.number}.md`) : ok();
+ }
+ if (cmd === 'show' && rest[0]!.includes(':changelog.d/')) {
+ const pr = (w.importantPrs ?? []).find(p => rest[0] === `${sha(p.merge)}:changelog.d/fix-${p.number}.md`);
+ return pr ? ok(`---\nimportant: ${pr.important}\n---\n**A fix.**\n`) : fail;
+ }
if (cmd === 'show') return ok(rest[0]!.replace(/^refs\/tags\/v/, '').replace(/:VERSION$/, ''));
if (cmd === 'merge-base') {
const a = commitOf(rest[1]!); const b = commitOf(rest[2]!);
@@ -117,6 +131,7 @@ function world(w: World) {
return fail;
};
const runs = w.runs ?? [];
+ const matrix = w.matrix ?? Object.values(w.releases).map(c => matrixRun(MATRIX_ID + c, c, w.releases));
const deps: DailyDeps = {
repo: REPO,
actor: 'github-actions[bot]',
@@ -127,6 +142,14 @@ function world(w: World) {
async ghApi(path) {
api.push(path);
if (path.startsWith('search/issues')) return { items: (w.importantPrs ?? []).map(p => ({ number: p.number })) };
+ if (path.includes('/issues?')) return w.breakIssues ?? [];
+ if (path.includes('/actions/workflows/upgrade-matrix.yml/runs')) {
+ const q = new URLSearchParams(path.split('?')[1] ?? '');
+ const rows = matrix.filter(r => !q.get('branch') || r.head_branch === q.get('branch'));
+ return { workflow_runs: rows.slice(0, Number(q.get('per_page') ?? 30)).map(({ receipt: _, ...run }) => run) };
+ }
+ const matrixHit = matrix.find(r => path.endsWith(`/actions/runs/${r.id}`));
+ if (matrixHit) { const { receipt: _, ...run } = matrixHit; return { ...run, path: '.github/workflows/upgrade-matrix.yml' }; }
const pull = /\/pulls\/(\d+)$/.exec(path);
if (pull) return { merge_commit_sha: sha(w.importantPrs!.find(p => p.number === Number(pull[1]))!.merge) };
if (path.includes('/actions/workflows/')) return { workflow_runs: runs };
@@ -146,10 +169,27 @@ function world(w: World) {
async ghPost(path, body) { posts.push({ path, body }); },
async ghApiWrite() { return {}; },
async scoreboards(id) { return runs.find(r => r.id === id)?.boards ?? CLEAN; },
+ async matrixReceipt(id) { const hit = matrix.find(r => r.id === id); return hit ? hit.receipt ?? null : null; },
};
return { deps, posts, pushes, api, logs, tags, record: () => (record.head ? objects[record.head] : undefined) };
}
+const MATRIX_ID = 700;
+const HARNESS = 'f'.repeat(40);
+/** A finished gate run of the upgrade matrix on the release at `commit`; green unless cells say otherwise. */
+function matrixRun(id: number, commit: number, releases: Record<string, number>, over: Partial<MatrixRunRow> = {}, receipt: Partial<MatrixReceipt> = {}): MatrixRunRow & { receipt: MatrixReceipt } {
+ const version = Object.entries(releases).find(([, c]) => c === commit)![0];
+ const red = receipt.red ?? [];
+ return {
+ id, status: 'completed', conclusion: red.length ? 'failure' : 'success', head_sha: HARNESS, head_branch: 'master', event: 'schedule',
+ html_url: `https://github.com/${REPO}/actions/runs/${id}`, ...over,
+ receipt: {
+ schema: 'gbrain-upgrade-matrix-run/1', harness: { sha: HARNESS }, candidate: { tag: `v${version}`, sha: sha(commit), tree: tree(commit) }, profile: 'full',
+ status: red.length ? 'red' : 'green', cells: [], red, holds: [], foreign_receipts: [], from: [{ tag: 'v1.0.1', roles: ['N-1', 'stable', 'stable-window'] }], skipped: [], ...receipt,
+ },
+ };
+}
+
const soak = (id: number, commit: number, over: Partial<SoakRunRow> & { boards?: Scoreboard[] } = {}) =>
({ id, status: 'completed', conclusion: 'success', head_sha: sha(commit), html_url: `https://github.com/${REPO}/actions/runs/${id}`, ...over });
@@ -173,7 +213,7 @@ describe('scheduleAuthorized', () => {
test('a scheduled promotion skips the role read and records the schedule as the promoter; a dispatch keeps the maintainer check', async () => {
const w = world(base({ runs: [soak(9, 2)] }));
- const p = await promote(w.deps, { version: '1.0.2', soakRunUrl: `https://github.com/${REPO}/actions/runs/9`, scoreboardUrl: '', rollback: false, reason: '' });
+ const p = await promote(w.deps, { version: '1.0.2', soakRunUrl: `https://github.com/${REPO}/actions/runs/9`, scoreboardUrl: '', rollback: false, reason: '', matrixRunUrl: `https://github.com/${REPO}/actions/runs/${MATRIX_ID + 2}` });
expect(w.api.some(a => a.endsWith('/permission'))).toBe(false);
expect(p.promotedBy).toBe('schedule (.github/workflows/stable-daily.yml)');
const d = world(base({ runs: [soak(9, 2)], scheduled: false }));
@@ -183,6 +223,16 @@ describe('scheduleAuthorized', () => {
});
});
+describe('a scheduled promotion names a matrix run', () => {
+ test('promote() refuses a scheduled forward promotion without one, before any tag moves', async () => {
+ const w = world(base({ runs: [soak(9, 2)] }));
+ const error = await promote(w.deps, { version: '1.0.2', soakRunUrl: `https://github.com/${REPO}/actions/runs/9`, scoreboardUrl: '', rollback: false, reason: '' }).then(() => null, e => e);
+ expect(error).toBeInstanceOf(Refusal);
+ expect((error as Error).message).toMatch(/a scheduled promotion needs a green upgrade matrix run on v1\.0\.2's exact tree/);
+ expect(w.pushes).toEqual([]);
+ });
+});
+
describe('integrity gates', () => {
test('every integrity gate must pass in every cell; latency gates stay record-only; no data or no cells fails closed', () => {
expect(integrityFailures(CLEAN)).toEqual([]);
@@ -393,6 +443,145 @@ describe('daily: weekly floor', () => {
});
});
+describe('daily: the upgrade matrix gates stable (stability plan 3a)', () => {
+ const old = { version: '1.0.1', commit: 1, message: stableMessage('1.0.1', '2026-11-02T05:17:00Z') };
+ const RELEASES = { '1.0.1': 1, '1.0.2': 2, '1.0.3': 3, '1.0.4': 4 };
+ const OLD_UPDATER = 'v0.60.241.0__postgres__keyed__repo';
+ const redFrom = (id: number, commit: number, cellIds: string[], over: Partial<MatrixRunRow> = {}) =>
+ matrixRun(id, commit, RELEASES, { event: 'workflow_dispatch', ...over }, { red: cellIds, cells: cellIds.map(c => ({ id: c, from: c.split('__')[0]!, verdict: 'fail' })) });
+ const breakIssue = (number: number, from?: string[]) => ({
+ number, state: 'open', html_url: `https://github.com/${REPO}/issues/${number}`,
+ body: `<!-- gbrain-upgrade-matrix:nightly -->\n${from ? `<!-- gbrain-upgrade-matrix:red-from ${from.join(',')} -->\n` : ''}## Upgrade matrix: RED`,
+ });
+
+ test('a dry run whose candidate is red upgrading from v0.60.241.0 (the updater that exited 1) holds and writes nothing; so does a real run', async () => {
+ for (const dryRun of [true, false]) {
+ const w = world(base({ stable: old, runs: [soak(9, 4)], matrix: [redFrom(704, 4, [OLD_UPDATER])], dryRun }));
+ const r = await daily(w.deps);
+ expect(r.action).toBe('held');
+ expect(r.why).toContain('failing from v0.60.241.0');
+ expect(r.why).toContain('a maintainer decides');
+ expect(w.pushes).toEqual([]);
+ expect(w.posts).toEqual([]);
+ expect(w.logs.some(l => l.startsWith('::warning::'))).toBe(true);
+ }
+ });
+
+ test("an open break issue from the matrix holds stable, on the weekly floor and for a regression fix alike, and dispatches nothing; a labelled issue without the matrix's marker holds nothing", async () => {
+ const held = world(base({ stable: old, runs: [soak(9, 4)], importantPrs: [{ number: 30, merge: 3, important: 'regression' }], breakIssues: [breakIssue(9329)] }));
+ const r = await daily(held.deps);
+ expect(r.action).toBe('held');
+ expect(r.why).toContain('#9329');
+ expect(r.why).toContain('waiting: #30');
+ expect([...held.pushes, ...held.posts]).toEqual([]);
+ const labelled = world(base({ stable: old, runs: [soak(9, 4)], breakIssues: [{ number: 41, state: 'open', html_url: 'x', body: 'Upgrading from 0.59 fails for me' }] }));
+ expect((await daily(labelled.deps)).action).toBe('promoted');
+ });
+
+ test('a soaked candidate with no matrix run gets one dispatch on its tag from master; an active matrix run means wait; a dry run says what it would dispatch', async () => {
+ const none = world(base({ stable: old, runs: [soak(9, 4)], matrix: [] }));
+ expect(await daily(none.deps)).toMatchObject({ action: 'matrix_dispatched', version: '1.0.4' });
+ expect(none.posts).toEqual([{ path: `repos/${REPO}/actions/workflows/upgrade-matrix.yml/dispatches`, body: { ref: 'master', inputs: { candidate: 'v1.0.4' } } }]);
+ expect(none.pushes).toEqual([]);
+ const active = world(base({ stable: old, runs: [soak(9, 4)], matrix: [{ ...matrixRun(800, 4, RELEASES), status: 'in_progress', conclusion: null, receipt: null }] }));
+ expect((await daily(active.deps)).action).toBe('waiting');
+ expect(active.posts).toEqual([]);
+ const dry = world(base({ stable: old, runs: [soak(9, 4)], matrix: [], dryRun: true }));
+ expect(await daily(dry.deps)).toMatchObject({ action: 'dry_run', would: 'dispatch the upgrade matrix on v1.0.4' });
+ expect(dry.posts).toEqual([]);
+ });
+
+ test('the soak dispatch on the newest release also dispatches the matrix on it when it has no run', async () => {
+ const w = world(base({ stable: old, matrix: [] }));
+ expect(await daily(w.deps)).toMatchObject({ action: 'soak_dispatched', version: '1.0.4' });
+ expect(w.posts.map(p => p.path)).toEqual([`repos/${REPO}/actions/workflows/production-soak.yml/dispatches`, `repos/${REPO}/actions/workflows/upgrade-matrix.yml/dispatches`]);
+ const dry = world(base({ stable: old, matrix: [], dryRun: true }));
+ expect(await daily(dry.deps)).toMatchObject({ action: 'dry_run', would: 'dispatch the soak and the upgrade matrix on v1.0.4' });
+ });
+
+ test('a run on another release, a pull-request run, a run with explicit from-versions and a fault-injection proof run do not qualify the candidate', async () => {
+ const cases: Array<MatrixRunRow & { receipt?: MatrixReceipt | null }> = [
+ matrixRun(801, 3, RELEASES),
+ matrixRun(802, 4, RELEASES, { event: 'pull_request', head_branch: 'fix/x' }),
+ matrixRun(803, 4, RELEASES, {}, { profile: 'explicit' }),
+ matrixRun(804, 4, RELEASES, {}, { cells: [{ id: 'v1.0.1__pglite__keyless__repo', from: 'v1.0.1', verdict: 'pass', fault: 'corrupt-after-upgrade' }] }),
+ ];
+ for (const run of cases) {
+ const w = world(base({ stable: old, runs: [soak(9, 4)], matrix: [run] }));
+ expect(await daily(w.deps)).toMatchObject({ action: 'matrix_dispatched', version: '1.0.4' });
+ expect(w.pushes).toEqual([]);
+ }
+ });
+
+ test('a red run on the candidate behind 30 newer pull-request runs is still found: stable holds and nothing is re-dispatched', async () => {
+ const prRuns = Array.from({ length: 30 }, (_, i) => ({ ...matrixRun(900 + i, 4, RELEASES, { event: 'pull_request', head_branch: `fix/pr-${i}` }), receipt: null }));
+ const w = world(base({ stable: old, runs: [soak(9, 4)], matrix: [...prRuns, redFrom(704, 4, [OLD_UPDATER])] }));
+ const r = await daily(w.deps);
+ expect(r.action).toBe('held');
+ expect(r.why).toContain('failing from v0.60.241.0');
+ expect(w.posts).toEqual([]);
+ });
+
+ test('a green run whose receipt predates the stable-window from-versions does not qualify: the check dispatches a fresh run', async () => {
+ const preWindow = matrixRun(807, 4, RELEASES, {}, { from: [{ tag: 'v1.0.1', roles: ['N-1', 'stable'] }], skipped: [] });
+ const w = world(base({ stable: old, runs: [soak(9, 4)], matrix: [preWindow] }));
+ expect(await daily(w.deps)).toMatchObject({ action: 'matrix_dispatched', version: '1.0.4' });
+ expect(w.pushes).toEqual([]);
+ const skippedWindow = world(base({ stable: old, runs: [soak(9, 4)], matrix: [matrixRun(808, 4, RELEASES, {}, { from: [{ tag: 'v1.0.1', roles: ['N-1'] }], skipped: [{ role: 'stable-window', reason: 'no stable release older than the candidate is in the window' }] })] }));
+ expect((await daily(skippedWindow.deps)).action).toBe('promoted');
+ });
+
+ test('a green run on the exact tag and tree promotes, and stable.json and the tag carry matrix_run', async () => {
+ const w = world(base({ stable: old, runs: [soak(9, 4)] }));
+ const r = await daily(w.deps);
+ expect(r.action === 'promoted' && r.promotion.version).toBe('1.0.4');
+ const url = `https://github.com/${REPO}/actions/runs/${MATRIX_ID + 4}`;
+ expect(JSON.parse(w.record()!)).toMatchObject({ version: '1.0.4', matrix_run: url });
+ expect(parseStableRecord(w.record()!, '1.0.4')).toMatchObject({ ok: true, record: { matrix_run: url } });
+ expect(w.tags.stable!.message).toContain(`matrix_run: ${url}\n`);
+ });
+
+ test('a run whose receipt names the tag but another tree, or was written by another harness, is foreign and holds', async () => {
+ const foreignTree = matrixRun(805, 4, RELEASES, {}, { candidate: { tag: 'v1.0.4', sha: sha(4), tree: tree(9) } });
+ const foreignHarness = matrixRun(806, 4, RELEASES, { head_sha: 'a'.repeat(40) });
+ for (const run of [foreignTree, foreignHarness]) {
+ const w = world(base({ stable: old, runs: [soak(9, 4)], matrix: [run] }));
+ expect((await daily(w.deps)).action).toBe('held');
+ expect(w.pushes).toEqual([]);
+ }
+ });
+
+ describe('a security or data-loss fix past an open break', () => {
+ const important = (cls: string) => [{ number: 50, merge: 4, important: cls }];
+
+ test('green on every from-version the break issue does not name: promotes, and the record says what was excused', async () => {
+ for (const cls of ['security', 'data-loss']) {
+ const w = world(base({ importantPrs: important(cls), runs: [soak(9, 4)], matrix: [redFrom(704, 4, [OLD_UPDATER])], breakIssues: [breakIssue(9329, ['v0.60.241.0'])] }));
+ const r = await daily(w.deps);
+ expect(r.action === 'promoted' && r.promotion.version).toBe('1.0.4');
+ expect(JSON.parse(w.record()!)).toMatchObject({ matrix_run: `https://github.com/${REPO}/actions/runs/704`, reason: expect.stringContaining('upgrade break #9329 excused for v0.60.241.0') });
+ }
+ });
+
+ test('also green: promotes without excusing anything', async () => {
+ const w = world(base({ importantPrs: important('security'), runs: [soak(9, 4)], breakIssues: [breakIssue(9329, ['v0.60.241.0'])] }));
+ expect((await daily(w.deps)).action).toBe('promoted');
+ });
+
+ test('a failure on a from-version the issue does not name, an issue that names none, or a fix of another class still holds', async () => {
+ const newBreak = world(base({ importantPrs: important('security'), runs: [soak(9, 4)], matrix: [redFrom(704, 4, [OLD_UPDATER, 'v1.0.2__pglite__keyless__repo-less'])], breakIssues: [breakIssue(9329, ['v0.60.241.0'])] }));
+ const r = await daily(newBreak.deps);
+ expect(r.action).toBe('held');
+ expect(r.why).toContain('v1.0.2 is a new break');
+ const unnamed = world(base({ importantPrs: important('data-loss'), runs: [soak(9, 4)], matrix: [redFrom(704, 4, [OLD_UPDATER])], breakIssues: [breakIssue(9329)] }));
+ expect((await daily(unnamed.deps)).why).toContain('names no from-versions');
+ const regression = world(base({ importantPrs: important('regression'), runs: [soak(9, 4)], breakIssues: [breakIssue(9329, ['v0.60.241.0'])] }));
+ expect((await daily(regression.deps)).action).toBe('held');
+ expect([...newBreak.pushes, ...unnamed.pushes, ...regression.pushes]).toEqual([]);
+ });
+ });
+});
+
describe('daily: stable.json backfill (GBRA-346)', () => {
const SOAK_7 = `https://github.com/${REPO}/actions/runs/7`;
const tagged = (version: string, commit: number, rollback = false) => renderTagMessage({
@@ -470,7 +659,7 @@ describe('stable-daily.yml', () => {
expect(Object.keys(wf.on).sort()).toEqual(['schedule', 'workflow_dispatch']);
expect(wf.on.schedule.map(s => s.cron)).toEqual(['17 5 * * *', '17 13 * * *']);
expect(wf.on.workflow_dispatch.inputs.dry_run).toMatchObject({ type: 'boolean', default: true });
- expect(wf.permissions).toEqual({ contents: 'write', actions: 'write' });
+ expect(wf.permissions).toEqual({ contents: 'write', actions: 'write', issues: 'read' });
expect(wf.concurrency).toEqual({ group: 'promote-stable', 'cancel-in-progress': false });
});
diff --git a/test/scripts/upgrade-matrix.test.ts b/test/scripts/upgrade-matrix.test.ts
index be9ce06365..2dd3f0dec7 100644
--- a/test/scripts/upgrade-matrix.test.ts
+++ b/test/scripts/upgrade-matrix.test.ts
@@ -9,7 +9,8 @@ import { execFileSync } from 'node:child_process';
import { mkdirSync, mkdtempSync, rmSync, unlinkSync, writeFileSync } from 'node:fs';
import { tmpdir } from 'node:os';
import { join } from 'node:path';
-import { resolveFromVersions, type Release } from '../../scripts/upgrade-matrix/versions.ts';
+import { resolveFromVersions, stableWindow, type Release, type StableRecordTag } from '../../scripts/upgrade-matrix/versions.ts';
+import { failingFrom, parseRedFrom, renderRedFrom } from '../../scripts/upgrade-matrix/marker.ts';
import { createCertificates, parseFixturePath, startGithubFixture } from '../../scripts/upgrade-matrix/github-fixture.ts';
import { CELL_RELEASE_CHANNEL_ENV, compareInstalledTree, upgradeRefs } from '../../scripts/upgrade-matrix/cell.ts';
import { diffCatalog, diffContent, takeSnapshot, type Query } from '../../scripts/upgrade-matrix/snapshot.ts';
@@ -28,13 +29,13 @@ describe('from-version resolution', () => {
const r = resolveFromVersions(releases, candidate, { profile: 'full', stableVersion: null });
const byRole = Object.fromEntries(r.from.flatMap(f => f.roles.map(role => [role, f.tag])));
expect(byRole).toEqual({ 'N-1': 'v0.60.79.0', 'N-10': 'v0.60.70.0', 'N-50': 'v0.60.30.0', '30d': 'v0.60.50.0' });
- expect(r.skipped).toEqual([{ role: 'stable', reason: expect.stringContaining('stable') }]);
+ expect(r.skipped).toEqual([{ role: 'stable', reason: expect.stringContaining('stable') }, { role: 'stable-window', reason: expect.stringContaining('window') }]);
});
test('a role shared by two picks is one from-version with both roles; too few releases skips N-50 instead of dropping it', () => {
const few = releases.slice(0, 12);
const r = resolveFromVersions(few, few[11]!, { profile: 'full', stableVersion: '0.60.2.0' });
- expect(r.skipped.map(s => s.role)).toEqual(['N-50']);
+ expect(r.skipped.map(s => s.role)).toEqual(['N-50', 'stable-window']);
expect(r.skipped[0]!.reason).toContain('only 11 published releases');
const stable = r.from.find(f => f.roles.includes('stable'))!;
expect(stable.tag).toBe('v0.60.2.0');
@@ -49,6 +50,52 @@ describe('from-version resolution', () => {
});
});
+describe('the stable window (stability plan 3a)', () => {
+ const at = (iso: string) => new Date(iso).toISOString();
+ const candidate = { version: '0.60.730.0', published_at: at('2026-11-16T03:00:00Z') };
+ const records: StableRecordTag[] = [
+ { version: '0.50.1.0', promoted_at: at('2026-03-01T00:00:00Z') },
+ { version: '0.50.2.0', promoted_at: at('2026-04-01T00:00:00Z') },
+ { version: '0.55.1.0', promoted_at: at('2026-08-03T00:00:00Z') },
+ { version: '0.55.2.0', promoted_at: at('2026-08-20T00:00:00Z') },
+ { version: '0.60.474.0', promoted_at: at('2026-10-28T04:52:51Z') },
+ { version: '0.60.593.0', promoted_at: at('2026-11-06T20:40:18Z') },
+ ...['613', '620', '638', '654', '672', '707', '717', '722'].map((n, i) => ({ version: '0.60.' + n + '.0', promoted_at: at('2026-11-' + String(8 + i).padStart(2, '0') + 'T13:22:00Z') })),
+ { version: '0.60.731.0', promoted_at: at('2026-11-16T13:22:00Z') },
+ ];
+
+ test('keeps the last 8 stable releases, the newest of each older month, and the release stable when the window opened; nothing newer than the candidate', () => {
+ expect(stableWindow(records, candidate)).toEqual([
+ '0.60.722.0', '0.60.717.0', '0.60.707.0', '0.60.672.0', '0.60.654.0', '0.60.638.0', '0.60.620.0', '0.60.613.0',
+ '0.60.593.0', '0.60.474.0', '0.55.2.0', '0.50.2.0',
+ ]);
+ expect(stableWindow([], candidate)).toEqual([]);
+ });
+
+ test('the full profile upgrades from every window release that has a published release; one without is skipped with its reason; the pr profile has no window', () => {
+ const rel = (v: string, d: string): Release => ({ tag: 'v' + v, version: v, published_at: at(d) });
+ const published = [rel('0.60.722.0', '2026-11-15T00:00:00Z'), rel('0.60.717.0', '2026-11-14T00:00:00Z'), rel('0.60.474.0', '2026-10-27T00:00:00Z'), rel('0.60.729.0', '2026-11-16T00:00:00Z')];
+ const few = [{ version: '0.60.722.0', promoted_at: at('2026-11-15T13:22:00Z') }, { version: '0.60.717.0', promoted_at: at('2026-11-14T13:22:00Z') }, { version: '0.60.593.0', promoted_at: at('2026-11-06T20:40:18Z') }];
+ const r = resolveFromVersions(published, candidate, { profile: 'full', stableVersion: '0.60.722.0', stableRecords: few });
+ expect(r.from.find(f => f.tag === 'v0.60.722.0')!.roles).toEqual(['stable', 'stable-window']);
+ expect(r.from.find(f => f.tag === 'v0.60.717.0')!.roles).toEqual(['stable-window']);
+ expect(r.skipped).toContainEqual({ role: 'stable-window', reason: 'stable 0.60.593.0 has no published release' });
+ expect(resolveFromVersions(published, candidate, { profile: 'pr', stableRecords: few }).from.some(f => f.roles.includes('stable-window'))).toBe(false);
+ });
+
+ test("the break issue names the from-versions that failed or held, and only the matrix's marker line is read back", () => {
+ const summary = {
+ cells: [{ id: 'v0.60.241.0__postgres__keyed__repo', from: 'v0.60.241.0', verdict: 'fail' }, { id: 'v0.60.722.0__pglite__keyless__repo', from: 'v0.60.722.0', verdict: 'pass' }],
+ red: ['v0.60.241.0__postgres__keyed__repo', 'v0.60.150.0__pglite__keyed__repo-less'], holds: ['door:hermes'],
+ };
+ expect(failingFrom(summary)).toEqual({ from: ['v0.60.150.0', 'v0.60.241.0'], door: true });
+ const line = renderRedFrom(failingFrom(summary).from);
+ expect(line).toBe('<!-- gbrain-upgrade-matrix:red-from v0.60.150.0,v0.60.241.0 -->');
+ expect(parseRedFrom('<!-- gbrain-upgrade-matrix:nightly -->\n' + line + '\n## Upgrade matrix: RED')).toEqual(['v0.60.150.0', 'v0.60.241.0']);
+ expect(parseRedFrom('<!-- gbrain-upgrade-matrix:nightly -->\nUpgrading from v0.60.241.0 fails')).toBeNull();
+ });
+});
+
function gitRepo(): { dir: string; first: string; second: string } {
const dir = mkdtempSync(join(tmpdir(), 'um-src-'));
const git = (...args: string[]) => execFileSync('git', ['-C', dir, '-c', 'user.name=t', '-c', '[email protected]', ...args], { encoding: 'utf8' }).trim();