Skip to content

Self-hosted, Self-Improving, Federated Everything App — dev management, agent orchestration, persistent mind, digital twin, second brain, and creative pipeline from a single dashboard over Tailscale to port the user into the OS

CI: let failed jobs finish, split browser suites from DB tests, drop the local fidelity gate

Claudee811a035d3cc
diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
index 59b083c2eb..0fdc6eea64 100644
--- a/.github/workflows/ci.yml
+++ b/.github/workflows/ci.yml
@@ -62,6 +62,7 @@ jobs:
client_shards: ${{ steps.plan.outputs.client_shards }}
windows_shards: ${{ steps.plan.outputs.windows_shards }}
suite_reasons: ${{ steps.plan.outputs.suite_reasons }}
+ node_minimum: ${{ steps.node-minimum.outputs.run }}
steps:
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
with:
@@ -98,6 +99,22 @@ jobs:
CI_BASE_REF: ${{ github.base_ref }}
run: node scripts/ci-test-plan.js
+ # The Node-minimum job proves the lockfiles install and the runtime gate
+ # loads on the oldest supported Node. Only these files can change that
+ # answer, so a PR that touches none of them skips it; non-PR runs always
+ # run it.
+ - name: Decide whether the Node minimum check applies
+ id: node-minimum
+ env:
+ CI_EVENT: ${{ github.event_name }}
+ run: |
+ pattern='^((server/|client/|autofixer/)?package(-lock)?\.json|\.nvmrc|scripts/checkNodeVersion(\.test)?\.js|scripts/node-version-drift\.test\.js|\.github/workflows/ci\.yml)$'
+ if [ "$CI_EVENT" != pull_request ] || git diff --name-only "$CI_BASE_SHA"...HEAD | grep -qE "$pattern"; then
+ echo run=true >> "$GITHUB_OUTPUT"
+ else
+ echo run=false >> "$GITHUB_OUTPUT"
+ fi
+
- name: Publish plan summary
env:
PLAN_REASON: ${{ steps.plan.outputs.reason }}
@@ -142,11 +159,13 @@ jobs:
runs-on: ubuntu-latest
timeout-minutes: 30
strategy:
+ # A failing shard must not cancel its siblings: "Re-run failed jobs"
+ # then repeats only the red shard instead of the whole matrix.
+ fail-fast: false
matrix:
shard: ${{ fromJSON(needs.impact.outputs.server_shards) }}
permissions:
contents: read
- actions: write
steps:
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
with:
@@ -322,16 +341,6 @@ jobs:
if: needs.impact.outputs.smoke == 'true' && matrix.shard == 1
run: npm run smoke
- - name: Report the failure, then cancel sibling CI jobs
- if: failure() && github.event_name == 'pull_request'
- env:
- # Actions exposes no variable for the matrix leg, and the annotation
- # has to name the shard even when the API lookup behind it fails.
- # Empty on the unsharded jobs, which the script treats as absent.
- CI_FAILED_SHARD: ${{ matrix.shard }}
- GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
- run: node scripts/cancel-current-ci-run.js
-
client:
# Sharded like the server job; the build and the bundle budget run on shard 1.
# Lint is deliberately NOT here — it has its own job.
@@ -341,11 +350,13 @@ jobs:
runs-on: ubuntu-latest
timeout-minutes: 30
strategy:
+ # A failing shard must not cancel its siblings: "Re-run failed jobs"
+ # then repeats only the red shard instead of the whole matrix.
+ fail-fast: false
matrix:
shard: ${{ fromJSON(needs.impact.outputs.client_shards) }}
permissions:
contents: read
- actions: write
steps:
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
with:
@@ -428,16 +439,6 @@ jobs:
if: needs.impact.outputs.build == 'true' && matrix.shard == 1
run: npm run test --prefix client -- ApiExplorer.bundle
- - name: Report the failure, then cancel sibling CI jobs
- if: failure() && github.event_name == 'pull_request'
- env:
- # Actions exposes no variable for the matrix leg, and the annotation
- # has to name the shard even when the API lookup behind it fails.
- # Empty on the unsharded jobs, which the script treats as absent.
- CI_FAILED_SHARD: ${{ matrix.shard }}
- GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
- run: node scripts/cancel-current-ci-run.js
-
lint:
# Its own job rather than a step on client shard 1, for cost and latency —
# NOT as a flake fix. Steps within one job run sequentially, so the ~31s of
@@ -461,7 +462,6 @@ jobs:
timeout-minutes: 15
permissions:
contents: read
- actions: write
steps:
# No `fetch-depth` and no `Resolve the pull-request diff base`, unlike the
# test jobs: the linter's file list arrives from the impact plan, so this
@@ -509,28 +509,17 @@ jobs:
CI_LINT_FILES: ${{ needs.impact.outputs.lint_files }}
run: node scripts/run-ci-lint.js
- - name: Report the failure, then cancel sibling CI jobs
- if: failure() && github.event_name == 'pull_request'
- env:
- # Actions exposes no variable for the matrix leg, and the annotation
- # has to name the shard even when the API lookup behind it fails.
- # Empty on the unsharded jobs, which the script treats as absent.
- CI_FAILED_SHARD: ${{ matrix.shard }}
- GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
- run: node scripts/cancel-current-ci-run.js
-
database:
- # Owns the registered browser suites; the Linux unit job excludes that
- # exact registry. PRs run smoke here and require exact-head fidelity evidence
- # at the gate. Scheduled/release runs retain the complete browser profile.
+ # The serial Postgres suite. The browser suites used to run after it on
+ # this job; they are now the `browser` job beside it, because together they
+ # set the run's length more often than any other job.
name: DB tests
needs: impact
- if: needs.impact.outputs.db == 'true' || needs.impact.outputs.browser_files != '[]'
+ if: needs.impact.outputs.db == 'true'
runs-on: ubuntu-latest
timeout-minutes: 20
permissions:
contents: read
- actions: write
services:
postgres:
image: pgvector/pgvector:pg17@sha256:ac08538c6f8b9904c33c8224c5e5706dbe760aca29db1d096972b4052c22a75d
@@ -645,7 +634,6 @@ jobs:
vitest-server-${{ runner.os }}-
- name: Provision test database (portos_test)
- if: needs.impact.outputs.db == 'true'
run: npm run setup:db:test --prefix server
# Backup restore coverage takes a real pg_dump. The runner's default
@@ -664,19 +652,7 @@ jobs:
PG_MAJOR: '17'
run: bash scripts/ci-install-db-tools.sh
- # Read the runner's Chrome tree into the page cache before any suite
- # launches it (#9641). The recurring startup hang was captured with the
- # browser's lead thread waiting on a page-cache read and I/O pressure
- # above 50 right after the installs above, suggesting a cold read. The
- # launcher's 20s startup deadline is unchanged; the printed duration shows
- # whether the cold read is as slow as the failures suggest. Completion
- # requires every read to succeed within the warm-up budget; a passing
- # browser run remains a non-reproduction, not proof of a causal repair.
- - name: Warm the Chrome install
- run: bash scripts/ci-warm-test-chrome.sh
-
- name: Run DB-backed tests (portos_test)
- if: needs.impact.outputs.db == 'true'
run: |
set +e
start=$(date +%s)
@@ -688,25 +664,155 @@ jobs:
env:
PGDATABASE: portos_test
+ browser:
+ # Owns the registered browser suites; the Linux unit job excludes that
+ # exact registry. PRs run the smoke profile; scheduled, release and
+ # dispatched runs run the complete profile. No Postgres: the suites run on
+ # the file backend under NODE_ENV=test.
+ name: Browser suites
+ needs: impact
+ if: needs.impact.outputs.browser_files != '[]'
+ runs-on: ubuntu-latest
+ timeout-minutes: 20
+ permissions:
+ contents: read
+ steps:
+ - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
+ with:
+ persist-credentials: false
+
+ - name: Use Node.js 24.x
+ uses: actions/setup-node@949feb2413d6458794dcd2491c4babbbce0c15c1 # v7.1.0
+ with:
+ node-version: 24.x
+ cache: npm
+ cache-dependency-path: server/package-lock.json
+
+ # Same entry and same contract as the server job — see it for the key, and
+ # for why the install and rebuild below are gated on the mark rather than
+ # on the cache hit.
+ - name: Cache server node_modules
+ id: server-modules
+ uses: actions/cache@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0
+ with:
+ path: server/node_modules
+ key: server-modules-${{ runner.os }}-${{ runner.arch }}-node24-${{ hashFiles('server/package-lock.json', 'server/package.json', 'server/.npmrc', 'scripts/trusted-rebuilds.js') }}
+
+ # A restored tree is only reusable if it was rebuilt before it was saved,
+ # and that is not answerable by looking at the tree — with today's
+ # versions the rebuild is nearly a no-op, so every allowlisted package
+ # imports fine on a never-rebuilt one. The rebuild step below leaves an
+ # explicit mark; this reads it back.
+ #
+ # `continue-on-error` on purpose: a bad entry must not wedge every PR
+ # until a human clears the cache by hand. It fails, the two steps below
+ # see a non-success outcome, and the job builds the tree it needs. It does
+ # not *replace* the bad entry — cache keys are immutable and actions/cache
+ # skips the save on an exact hit — so that entry keeps costing each run a
+ # reinstall until the key moves or it ages out. That is precisely the
+ # pre-cache cost: the worst case degrades to the old behaviour, never to a
+ # red repo.
+ #
+ # `outcome != 'success'` covers all three cases in one expression: on a
+ # cache miss this step is skipped (outcome `skipped`), on a hit it is
+ # `success` or `failure`.
+ - name: Check the restored node_modules was rebuilt
+ id: server-modules-usable
+ if: steps.server-modules.outputs.cache-hit == 'true'
+ continue-on-error: true
+ run: node scripts/trusted-rebuild-stamp.js check server
+
+ - name: Install server dependencies
+ if: steps.server-modules-usable.outcome != 'success'
+ run: npm ci --prefix server
+
+ # server/.npmrc pins ignore-scripts=true, so the allowlisted packages need
+ # this explicit rebuild. Same condition as the install: whenever this job
+ # builds the tree it will cache, it must build a rebuilt one — and the
+ # mark is written in the same block, so a failed rebuild cannot leave one
+ # behind for a later job to trust.
+ #
+ # `shell: bash` is load-bearing, not style. windows-server would default
+ # to pwsh, where a *native* command's non-zero exit neither throws nor
+ # stops the block (`$PSNativeCommandUseErrorActionPreference` is false)
+ # and only the last command's code becomes the step result — so a failed
+ # rebuild would run the mark anyway, reset the exit code to 0, and publish
+ # a green, marked, un-rebuilt tree. Under `bash -e` the block stops at
+ # the rebuild, the job fails, and actions/cache (`post-if: success()`)
+ # never saves it.
+ - name: Rebuild trusted native dependencies
+ if: steps.server-modules-usable.outcome != 'success'
+ shell: bash
+ env:
+ # Skip onnxruntime-node's CUDA execution provider. Its postinstall
+ # downloads it on linux-x64 whenever the .so is absent — several
+ # hundred MB, on two jobs, every run — and no hosted runner has an
+ # NVIDIA GPU to use it with. The CPU binaries ship in the tarball, so
+ # inference still works; this is the single largest cost in the job.
+ ONNXRUNTIME_NODE_INSTALL_CUDA: skip
+ run: |
+ node scripts/trusted-rebuilds.js server
+ node scripts/trusted-rebuild-stamp.js write server
+
+ # Reuse Vite/Vitest artifacts from checkout-owned .cache paths, outside
+ # node_modules (which may be shared by worktrees). Unlike the installed
+ # dependency tree above, these artifacts can safely use a near-miss
+ # restore and stay warm across a lockfile bump.
+ - name: Cache Vitest transform artifacts
+ uses: actions/cache@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0
+ with:
+ path: |
+ server/.cache/vitest
+ server/.cache/vitest-db
+ key: vitest-server-${{ runner.os }}-${{ hashFiles('server/package-lock.json', 'server/vitest.config.js', 'server/vitest.config.db.js', 'scripts/vitestCiPool.js') }}-browser
+ restore-keys: |
+ vitest-server-${{ runner.os }}-${{ hashFiles('server/package-lock.json') }}-
+ vitest-server-${{ runner.os }}-
+
+ # Backup restore coverage takes a real pg_dump. The runner's default
+ # PostgreSQL 16 client cannot dump the PostgreSQL 17 service above.
+ #
+ # The pull is ~95 ffmpeg dependency packages from the runner's mirror, and
+ # a slow mirror has eaten this job's budget before any test ran (#10369,
+ # #10562): apt's retries and timeouts cannot bound a connection that
+ # trickles bytes. The script bounds the download, falls back from the
+ # runner mirror to the official Ubuntu archive with signing intact, and
+ # fails with an installation-failure annotation once its budget is spent.
+ # The step timeout stays as a backstop above the script's own budget.
+ - name: Install PostgreSQL client and audio verification tools
+ timeout-minutes: 9
+ env:
+ PG_MAJOR: '17'
+ run: bash scripts/ci-install-db-tools.sh
+
+ # Read the runner's Chrome tree into the page cache before any suite
+ # launches it (#9641). The recurring startup hang was captured with the
+ # browser's lead thread waiting on a page-cache read and I/O pressure
+ # above 50 right after the installs above, suggesting a cold read. The
+ # launcher's 20s startup deadline is unchanged; the printed duration shows
+ # whether the cold read is as slow as the failures suggest. Completion
+ # requires every read to succeed within the warm-up budget; a passing
+ # browser run remains a non-reproduction, not proof of a causal repair.
+ - name: Warm the Chrome install
+ run: bash scripts/ci-warm-test-chrome.sh
+
# Same client cache entry as the `client` job (see it for the key and why
# an exact hit needs no rebuild).
- name: Cache client node_modules
id: client-modules
- if: needs.impact.outputs.browser_files != '[]'
uses: actions/cache@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0
with:
path: client/node_modules
key: client-modules-${{ runner.os }}-${{ runner.arch }}-node24-${{ hashFiles('client/package-lock.json', 'client/package.json', 'client/.npmrc', 'scripts/trusted-rebuilds.js') }}
- name: Install client dependencies
- if: needs.impact.outputs.browser_files != '[]' && steps.client-modules.outputs.cache-hit != 'true'
+ if: steps.client-modules.outputs.cache-hit != 'true'
run: npm ci --prefix client
# Fail-closed: PORTOS_REQUIRE_BROWSER_SUITES makes the suites' canRun guard
# throw (see server/lib/browserSuiteGate.js) when Chrome, ffmpeg or the
# client bundler is missing, so a skip can never pass this step.
- name: Run cross-workspace browser suites
- if: needs.impact.outputs.browser_files != '[]'
# The independent supervisor emits progress and owns a 12m deadline;
# this 15m step limit is only a fallback before the 20m job boundary.
timeout-minutes: 15
@@ -720,23 +826,13 @@ jobs:
run: node scripts/watch-ci-tests.js server
- name: Preserve browser test failure diagnostics
- if: failure() && needs.impact.outputs.browser_files != '[]'
+ if: failure()
continue-on-error: true
timeout-minutes: 1
env:
PORTOS_CI_DIAGNOSTICS_DIR: ${{ runner.temp }}/portos-ci-diagnostics
run: node scripts/ci-test-diagnostics.js
- - name: Report the failure, then cancel sibling CI jobs
- if: failure() && github.event_name == 'pull_request'
- env:
- # Actions exposes no variable for the matrix leg, and the annotation
- # has to name the shard even when the API lookup behind it fails.
- # Empty on the unsharded jobs, which the script treats as absent.
- CI_FAILED_SHARD: ${{ matrix.shard }}
- GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
- run: node scripts/cancel-current-ci-run.js
-
windows-server:
# Sharded like the server job.
name: Windows server unit tests (${{ matrix.shard }}/${{ strategy.job-total }})
@@ -745,11 +841,13 @@ jobs:
runs-on: windows-latest
timeout-minutes: 40
strategy:
+ # A failing shard must not cancel its siblings: "Re-run failed jobs"
+ # then repeats only the red shard instead of the whole matrix.
+ fail-fast: false
matrix:
shard: ${{ fromJSON(needs.impact.outputs.windows_shards) }}
permissions:
contents: read
- actions: write
steps:
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
with:
@@ -879,18 +977,10 @@ jobs:
PORTOS_CI_DIAGNOSTICS_DIR: ${{ runner.temp }}/portos-ci-diagnostics
run: node scripts/ci-test-diagnostics.js
- - name: Report the failure, then cancel sibling CI jobs
- if: failure() && github.event_name == 'pull_request'
- env:
- # Actions exposes no variable for the matrix leg, and the annotation
- # has to name the shard even when the API lookup behind it fails.
- # Empty on the unsharded jobs, which the script treats as absent.
- CI_FAILED_SHARD: ${{ matrix.shard }}
- GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
- run: node scripts/cancel-current-ci-run.js
-
node-minimum:
name: Node minimum compatibility
+ needs: impact
+ if: needs.impact.outputs.node_minimum == 'true'
runs-on: ubuntu-latest
timeout-minutes: 15
permissions:
@@ -932,7 +1022,6 @@ jobs:
timeout-minutes: 10
permissions:
contents: read
- actions: write
steps:
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
with:
@@ -953,26 +1042,17 @@ jobs:
PORTOS_REQUIRE_LINUX_CONTAINMENT: '1'
run: npm test --prefix server -- --run lib/codeAnimationContainment.test.js routes/codeAnimationExecution.test.js services/codeAnimation/containedWorker.test.js
- - name: Report the failure, then cancel sibling CI jobs
- if: failure() && github.event_name == 'pull_request'
- env:
- CI_FAILED_SHARD: ${{ matrix.shard }}
- GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
- run: node scripts/cancel-current-ci-run.js
-
gate:
name: CI Gate
if: always()
- needs: [impact, server, client, lint, database, windows-server, node-minimum, linux-containment]
+ needs: [impact, server, client, lint, database, browser, windows-server, node-minimum, linux-containment]
runs-on: ubuntu-latest
timeout-minutes: 10
- # READ, not write: the reporter only lists this run's jobs, to name the one
- # whose STEP failed when the fail-fast cancel beat GitHub to writing the job
- # conclusion. Cancelling the run stays a leaf-job privilege.
+ # READ only: the reporter lists this run's jobs to name the one whose STEP
+ # failed when a mid-step cancel hid the job's `failure` conclusion.
permissions:
contents: read
actions: read
- statuses: read
steps:
# The gate needs the repo only for scripts/ci-gate-report.js, which
# imports Node builtins only — no dependency install, no setup-node.
@@ -1001,20 +1081,12 @@ jobs:
CI_GATE_RESULT_CLIENT: ${{ needs.client.result }}
CI_GATE_RESULT_LINT: ${{ needs.lint.result }}
CI_GATE_RESULT_DATABASE: ${{ needs.database.result }}
+ CI_GATE_RESULT_BROWSER: ${{ needs.browser.result }}
CI_GATE_RESULT_WINDOWS_SERVER: ${{ needs.windows-server.result }}
CI_GATE_RESULT_NODE_MINIMUM: ${{ needs.node-minimum.result }}
CI_GATE_RESULT_LINUX_CONTAINMENT: ${{ needs.linux-containment.result }}
run: node scripts/ci-gate-report.js
- - name: Require current browser fidelity evidence
- if: github.event_name == 'pull_request' && github.base_ref != 'release' && inputs.full != true
- env:
- GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
- CI_BROWSER_FILES: ${{ needs.impact.outputs.browser_files }}
- CI_PULL_REQUEST: ${{ github.event.pull_request.number }}
- CI_EXPECTED_HEAD: ${{ github.event.pull_request.head.sha }}
- run: node scripts/verify-browser-fidelity.js
-
full-gate:
# A second check run published ONLY when the impact plan chose the complete
# suite. release.yml keys on this name to skip re-running CI on the push to
diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml
index d3ad0c2be1..d1c2fd276b 100644
--- a/.github/workflows/release.yml
+++ b/.github/workflows/release.yml
@@ -51,17 +51,13 @@ jobs:
full-ci:
needs: verify-ci
if: needs.verify-ci.outputs.verified != 'true'
- # `actions: write` looks unused from here — the fail-fast cancellation step
- # in ci.yml's leaf jobs is gated on `github.event_name == 'pull_request'`,
- # and a called workflow inherits the caller's event, which is this push.
- # It still has to be granted: a called workflow's jobs cannot hold a
- # permission the calling job does not, so narrowing this to `contents: read`
- # strips `actions: write` from leaf jobs that declare it. This block also
- # scopes full-ci DOWN from the workflow-level `contents: write` the release
- # job needs; ci.yml only ever checks out. ci-fail-fast.test.js pins it.
+ # A called workflow's jobs cannot hold a permission the calling job lacks,
+ # so this grants what ci.yml's gate jobs read (`actions: read`, to name a
+ # failed step) and scopes full-ci DOWN from the workflow-level
+ # `contents: write` the release job needs; ci.yml only ever checks out.
permissions:
contents: read
- actions: write
+ actions: read
uses: ./.github/workflows/ci.yml
with:
full: true
diff --git a/docs/CONTRIBUTING.md b/docs/CONTRIBUTING.md
index 14c323a90e..46c14081c8 100644
--- a/docs/CONTRIBUTING.md
+++ b/docs/CONTRIBUTING.md
@@ -89,11 +89,10 @@ See [VERSIONING.md](./VERSIONING.md) for full details.
Pregate proves only the stages it runs; it does not replace required CI or resource-dependent checks. A full-suite plan runs only the always-run guards unless `--full` is supplied; DB suites, Windows, client build, and boot smoke are reported but not run by pregate.
-Rendering PRs that select fidelity cases also run
-`node scripts/browser-fidelity.js --pr NUMBER` from the clean published head.
-The command runs the cases and publishes exact-head evidence using existing GitHub
-authentication; missing or stale evidence blocks CI Gate. See
-[Browser fidelity evidence](GITHUB_ACTIONS.md#browser-fidelity-evidence).
+PRs run the browser suites' smoke profile; the `[fidelity]` cases run nightly
+and on the release PR. To check them on a rendering PR before merging, run
+`node scripts/browser-fidelity.js --pr NUMBER` from the clean published head. See
+[Browser fidelity cases](GITHUB_ACTIONS.md#browser-fidelity-cases).
### Checkout-owned development caches
diff --git a/docs/GITHUB_ACTIONS.md b/docs/GITHUB_ACTIONS.md
index 00a40ad579..4a83e77211 100644
--- a/docs/GITHUB_ACTIONS.md
+++ b/docs/GITHUB_ACTIONS.md
@@ -286,8 +286,8 @@ nothing for this workspace, so a restored tree is usable as-is.
### Reusing the server install
-Three jobs install the server workspace — `server`, `database`, and
-`windows-server`. `setup-node`'s `cache: npm` only preserves `~/.npm`, the
+Four jobs install the server workspace — `server`, `database`, `browser`,
+and `windows-server`. `setup-node`'s `cache: npm` only preserves `~/.npm`, the
tarball cache, so `npm ci` still wipes and repopulates a ~570 MB
`node_modules` on each of them. Two changes cut that.
@@ -295,7 +295,7 @@ tarball cache, so `npm ci` still wipes and repopulates a ~570 MB
in the npm tarball, but its postinstall *downloads* the CUDA EP — several
hundred MB — on linux-x64 whenever `libonnxruntime_providers_cuda.so` is
absent. `server/.npmrc` pins `ignore-scripts=true`, so that runs inside
-`scripts/trusted-rebuilds.js`, on the `server` and `database` jobs, every run.
+`scripts/trusted-rebuilds.js`, on the Linux server-installing jobs, every run.
No hosted runner has an NVIDIA GPU to use it with. Setting
`ONNXRUNTIME_NODE_INSTALL_CUDA=skip` on the rebuild step removes the download
outright, and inference is unaffected. This is the largest single saving here,
@@ -379,7 +379,7 @@ a cache written by a pull request is visible only to that branch — so the 09:1
UTC schedule is what seeds the entries on the default branch, the one scope every
PR branch can read. A PR that changes the lockfile misses by design.
-Two entries exist per key state (Linux, shared by `server` and `database`, plus
+Two entries exist per key state (Linux, shared by `server`, `database` and `browser`, plus
Windows) at roughly 570 MB each, against GitHub's 10 GB per-repo cache budget.
Eviction is LRU, so the constantly-read `main` entries outlive the PR-scoped ones
— but a burst of lockfile-churning PRs can still push out the transform caches
@@ -448,7 +448,7 @@ The selected work is split across parallel jobs:
`Cache client node_modules` with the client job, so on a hit it pays no install.
- **DB tests** — provisions only the isolated `portos_test` database and runs
the serial DB suite when database-sensitive files changed.
-- **Cross-workspace browser suites** (same `database` job) — the exact
+- **Browser suites** (their own `browser` job, beside DB tests) — the exact
`BROWSER_SUITES` registry in `scripts/lib/browserSuites.js` owns these tests.
Linux server shards exclude that registry with `PORTOS_CI_BROWSER_OWNER=unit`;
other browser files, including `overlayTextService`, keep their existing owner.
@@ -461,8 +461,9 @@ The selected work is split across parallel jobs:
hosted CI selects `full`. Hosted profiles retain generated Three render/export
and owned cleanup coverage. Profiles do not change pixels,
render settings, tolerances, or cleanup assertions.
- A PR selecting a fidelity-owning file also requires the exact-head receipt
- described below; a smoke pass alone cannot admit it.
+ They ran after the DB suite on one job until that job set the run's length
+ more often than any other; split, each finishes in about the time of its own
+ steps.
- **Windows server tests** — the same server selection, but only on full CI
(the `main` → `release` PR, nightly, release, workflow dispatch) or when a
Windows-sensitive surface changed (`.ps1` / `.cmd` spawn, PowerShell BOM,
@@ -470,6 +471,11 @@ The selected work is split across parallel jobs:
Linux-faithful PRs skip this job. `pinPlatform('win32')` tests still run on
Linux. A full plan does not automatically mean a full *Windows* run — see
"What a full plan costs the Windows job" below.
+- **Node minimum compatibility** — installs every workspace and runs the
+ runtime gate on the oldest supported Node. On a pull request it runs only
+ when a `package.json`, a lockfile, `.nvmrc`, the gate script, its drift test
+ or `ci.yml` changed, since nothing else can change its answer; non-PR runs
+ always run it.
- **CI Gate** — always reports one stable required-check result and fails if any
selected job failed or was cancelled. Both block, but the message distinguishes
them: `scripts/ci-gate-report.js` says *cancelled, not failed* and names the
@@ -478,40 +484,25 @@ The selected work is split across parallel jobs:
automatic retry" below.
- **Full CI Gate** — published only when the plan chose the complete suite and
full Windows coverage, with the full browser profile executed in hosted CI.
- Ordinary PR smoke runs cannot publish it, even with a local fidelity receipt.
+ Ordinary PR smoke runs cannot publish it.
It mirrors `CI Gate`'s result. This is the check the release workflow looks for;
see "Reusing the release PR's CI run" below.
-### Browser fidelity evidence
-
-After publishing a relevant PR, run `node scripts/browser-fidelity.js --pr NUMBER`
-from its clean checkout using existing `gh` authentication. This command runs the
-four registered fidelity cases automatically, retains a local receipt, and publishes
-an exact-commit status only after complete passing assertions and unchanged-source
-verification. It binds the PR head and that commit's source tree. It does not
-provision a runner, install credentials, or start PortOS.
-
-The required CI gate checks the current head and its source tree, profile digest,
-trusted repository-owner publisher and latest status. Missing, stale, skipped, failed or incomplete
-results block admission. Local receipt JSON is an audit record; the trusted
-publisher's status attests execution, not a cryptographic proof of test behavior.
-Keep credentials outside the test subprocess.
-
-If evidence finishes after CI Gate failed only for missing fidelity evidence,
-rerun that gate job on the unchanged run once its other required jobs have passed.
-A receipt cannot turn a cancelled or failed browser, Windows, or containment job
-green. A changed PR head requires new evidence; base-only movement does not. Hosted smoke tests the merge checkout; local fidelity proves the PR head only,
-not native integration of subsequent base changes. Scheduled/release full coverage
-remains the integration check for the ordinary fidelity cases; hardware-sensitive
-`[local-performance]` acceptance stays local. Default
-pregate is not a substitute for this command or required CI.
-
-The local playback case preserves full-resolution transport, frame freshness,
-preview/export parity and lens-coordinate regressions. Creative quality receives
-human review; hosted acceptance does not impose this playback performance budget.
-Software WebGL results do not certify physical-GPU performance, and
-Mac-local results cannot replace Windows compatibility or Linux containment.
-The profile split does not explain or repair previously unidentified hosted stalls.
+### Browser fidelity cases
+
+Pull requests run the browser suites' smoke profile, which skips the
+`[fidelity]` grading cases. Those cases run in hosted CI on the nightly
+schedule, on the `main` -> `release` pull request and on dispatched full runs,
+so a fidelity regression shows up there rather than blocking the PR that
+caused it. The `[local-performance]` playback case runs only locally.
+
+To check the fidelity cases on a PR before merging, run
+`node scripts/browser-fidelity.js --pr NUMBER` from its clean checkout with
+existing `gh` authentication. It runs the registered fidelity cases, keeps a
+local receipt, and publishes an exact-commit status for the PR head. That
+status is informational: no required check reads it. Software WebGL results do
+not certify physical-GPU performance, and Mac-local results cannot replace
+Windows compatibility or Linux containment.
### Full-suite sharding
@@ -689,69 +680,20 @@ green PRs, and the `main` → `release` PR catches it before a release ships.
Changes to CI/test configuration also force the full suite on their own PR.
`[skip ci]` remains honored for push events only; PR CI always runs.
-### Fail-fast sibling cancellation
-
-Each selected leaf job (`server`, `client`, `lint`, `database`, and
-`windows-server`) ends with an `if: failure() && github.event_name ==
-'pull_request'` step that asks GitHub to cancel the current pull-request
-workflow run. The event guard is important because the same workflow is reused
-by the release workflow: a failing reusable `full-ci` job must not cancel its
-parent release run before that workflow can report the release failure. The
-request uses the repository-owned
-`scripts/cancel-current-ci-run.js` helper and the standard workflow-run cancel
-endpoint. The helper accepts no repository or run arguments: it validates and
-uses only `GITHUB_REPOSITORY` and `GITHUB_RUN_ID` supplied by Actions, with the
-step-scoped `GITHUB_TOKEN`.
-
-The leaf jobs request only `contents: read` and `actions: write`, and check
-out with `persist-credentials: false` so a token that can now cancel runs is
-not left in `.git/config` for the test suite's own subprocesses to read. The
-token is present in the environment only for the cancellation step, and no
-third-party action or long-lived secret is involved. The failing test/build
-step runs before cancellation, so its annotations and logs remain the evidence
-for the failure. A successful cancellation returns `202`; a `409` means the run
-is already terminal and is treated as a no-op.
-
-`release.yml`'s `full-ci` job must grant `actions: write` to the workflow it
-calls even though the cancellation step never fires there — a called workflow's
-jobs cannot hold a permission the calling job lacks. See the comment on that
-job; `scripts/ci-fail-fast.test.js` pins it.
-
-Cancellation is deliberately best-effort. Fork pull requests and other
-read-only-token runs may receive a permission failure, and transient API or
-network failures are also possible. The helper logs the unavailable
-cancellation and exits normally, preserving the original failed step and its
-failed job result when the API is unavailable. It imports only Node builtins,
-so it still runs from a job that failed before `npm ci`
-(`scripts/pre-install-entrypoints.test.js` enforces that).
-
-**Canceled is not mergeable.** A run-wide cancellation lands on the requesting
-job and on `CI Gate`, which is usually still waiting on its `needs` and so ends
-`cancelled` rather than running its comparison. Every consumer treats that as a
-non-pass, by allowlisting the green results rather than denylisting the red
-ones:
-
-- Branch protection requires the `CI Gate` context to conclude `success`;
- `cancelled` does not satisfy it, and a gate that never publishes leaves the
- required context unreported, which also blocks.
-- If the gate job does run, it accepts only `success` or `skipped` per job, so
- the failed leaf (or its own `cancelled` result) fails the gate. It reports a
- cancel and a failure differently — see the next section — but neither passes.
-- `scripts/verify-ci-status.js` accepts a `Full CI Gate` only at
- `conclusion === 'success'`, so a canceled run can never let a release skip
- the full suite.
-- PortOS's own auto-merge watcher (`server/services/prWatcher.js`) counts only
- `SUCCESS`/`NEUTRAL`/`SKIPPED` as green.
-
-The consequence to expect in the UI: on a fail-fast run the required check
-reads *canceled*, not *failed*. The failing step's log and annotations are
-still the diagnosis. The target is for siblings to become canceled within 30
-seconds of the first failing job completing, while the existing workflow-level
-`concurrency.cancel-in-progress` continues to handle superseded runs
-independently — the two are orthogonal, one canceling this run by id and the
-other canceling an older run when a newer commit arrives. Scheduled, manually
-dispatched, and release-called runs skip this sibling cancellation so their
-aggregate diagnostics and cache post-steps can complete normally.
+### Failures do not cancel other jobs
+
+A failing job does not cancel its siblings, and every matrix sets
+`fail-fast: false`. The run finishes with the failing job concluded `failure`
+and the rest concluded on their own results, so **Re-run failed jobs** repeats
+only the red shard. Runners on this public repository are not billed, so the
+siblings that keep running cost concurrency, not money.
+
+**Cancelled is not mergeable.** Every consumer allowlists green results:
+branch protection requires `CI Gate` to conclude `success`; the gate accepts
+only `success` or `skipped` per job; `scripts/verify-ci-status.js` accepts a
+`Full CI Gate` only at `success`; and PortOS's auto-merge watcher
+(`server/services/prWatcher.js`) counts only `SUCCESS`/`NEUTRAL`/`SKIPPED` as
+green.
### Windows native worker crash evidence
@@ -774,8 +716,7 @@ handling (#8152). Microsoft documents these fields in its
### External cancellation and one automatic retry
Not every cancel comes from this repository. When several PRs build at once,
-GitHub itself cancels in-flight runs — no job fails, the fail-fast step above
-stays `skipped`, and **no successor run exists for the branch** (issue 7437).
+GitHub itself cancels in-flight runs — no job fails, and **no successor run exists for the branch** (issue 7437).
That last property is the whole diagnosis: a `cancel-in-progress` supersession
always leaves a newer run for the same PR, and an external cancel leaves none.
@@ -788,7 +729,7 @@ retries only when every one of these holds:
| conclusion is `cancelled` | A failure is a failure. |
| event is `pull_request` | Nightly, dispatch, and release-called runs are not ours to re-drive. |
| `run_attempt == 1` | The retry budget. `POST /rerun` makes attempt 2, whose cancel sees attempt 2 and stops — one retry per run, and a PR run is one run per head SHA. |
-| no job concluded `failure`/`timed_out` | Fail-fast cancellation makes a genuinely red run *look* cancelled. Retrying it would re-run the suite on a broken tree. |
+| no job concluded `failure`/`timed_out` | A run cancelled while one job was already red *looks* cancelled. Retrying it would re-run the suite on a broken tree. |
| no job hit its own `timeout-minutes` | GitHub reports a timed-out job as `cancelled`; only its check-run annotation ("exceeded the maximum execution time") tells it from an external cancel (#11910). A re-run would time out again, so it skips as `job-timed-out`. |
| no newer run for the branch | That is a supersession; the newer run already covers this code. |
@@ -811,8 +752,7 @@ raising the delay fails the build rather than quietly eating the margin.
Every API-backed guard in the table is evaluated **twice**: once before the wait
and once after it. A push or a human re-run that lands during the delay
therefore still wins, and the post-wait job listing is the more reliable one —
-a failing job's step records can still be settling when the run's own fail-fast
-cancel lands.
+a failing job's step records can still be settling when a cancel lands.
The two passes are **not** equal in authority. The post-wait pass is the reading
the re-dispatch is made on, and it fails CLOSED as before. The pre-wait pass is
@@ -833,14 +773,9 @@ retrying only once the repository is quiet. On a busy repo that can mean never
retrying, which is worse than retrying into a cancel.
**Why the failing-job guard reads the jobs' own conclusions** rather than a
-marker written by the fail-fast step: a marker fails OPEN. It would be written
-by a job that is already failing, on a run about to be cancelled out from under
-it, so a lost write makes a red run look externally cancelled — and get
-retried. Reading conclusions fails CLOSED: an unreadable listing is
+marker a failing job writes: a marker fails OPEN. A lost write makes a red run
+look externally cancelled, and it gets retried. Reading conclusions fails CLOSED: an unreadable listing is
`jobs-unavailable` and skips the retry. Do not "simplify" this into a marker.
-The ambiguity itself is the price of cancelling the run on first failure rather
-than letting the remaining jobs fail naturally; that trade buys the 30-second
-fail-fast above and is not on the table.
**This is also the only layer that can explain a run-wide cancel.** `if:
always()` on the gate defeats an upstream failure, not a cancellation of the
diff --git a/docs/TROUBLESHOOTING.md b/docs/TROUBLESHOOTING.md
index 2779b5db05..9d4566d56e 100644
--- a/docs/TROUBLESHOOTING.md
+++ b/docs/TROUBLESHOOTING.md
@@ -590,174 +590,49 @@ npm run test:watch
### CI red with every job "cancelled"
-**Symptom**: A pull request's `CI Gate` / `Full CI Gate` goes red. Opening the
-run shows no failing assertion — jobs report `##[error]The operation was
-canceled.` mid-step, the run's top-level conclusion is `cancelled` rather than
-`failure`, and `gh pr checks` lists the long-running shards as `fail` with
-nothing saying why.
-
-**Two very different causes produce that identical surface**, and the run
-and job conclusions do not tell them apart:
-
-1. **A real test failure.** `scripts/cancel-current-ci-run.js` cancels the
- sibling jobs the moment one job fails, so the failing job is cancelled out
- from under its own reporting step and its conclusion reads `cancelled` too.
- Nothing in the run reads `failure`.
-2. **An external cancel** — the Actions spending limit, or a human pressing
- cancel.
-
-**Always rule out (1) first.** It is the common case, it is the one you can fix,
-and assuming billing sends you to a dashboard instead of the one-line assertion
-that is actually red. Two PortOS incidents were misdiagnosed as billing this way
-(#7482 on 2026-09-16, #7571 on 2026-09-17) and burned full-matrix re-runs before
-anyone read a job log.
-
-#### Step 0 — read the annotation, then the gate (usually this is the whole answer)
-
-Both now name the job and the failing step, and neither depends on a job
-conclusion, so both survive the cancel:
-
-- **The Checks tab's `::error` annotation.** `scripts/cancel-current-ci-run.js`
- prints it *before* it POSTs the cancel, and Actions records an annotation the
- instant the line is printed. It renders above the fold on the pull request.
-- **The `CI Gate` summary.** `scripts/ci-gate-report.js` queries this run's job
- steps whenever its verdict would otherwise be `cancelled`, and reports a
- failure naming the culprit instead.
-
-Both are best-effort. If the annotation shows only a job-and-shard label, or
-the gate still says `CANCELLED, not failed`, the lookup behind them did not
-land — fall through to step 1 and run it by hand. A run from before #7574 has
-neither.
-
-#### Step 1 — ask whether a STEP failed (this is the primary diagnostic)
-
-This is what step 0's two surfaces do for you; run it by hand when they are
-missing or came back empty. A cancelled job keeps its steps' conclusions, so the
-failing step survives even though the job that owns it reads `cancelled`.
-**`gh run view --json jobs` does not expose steps** — that omission is precisely
-why both incidents above were misread. Go to the REST API instead:
+A failing job no longer cancels its siblings, so a real test failure shows as a
+job that concluded `failure`. Open it and read the assertion, or use
+`gh run view --log-failed`. To rerun only that shard, use **Re-run failed jobs**.
-```bash
-gh api "/repos/atomantic/PortOS/actions/runs/<run-id>/jobs?per_page=100" \
- --jq '.jobs[] | select(any(.steps[]?; .conclusion=="failure"))
- | "\(.name) job=\(.conclusion) failedStep=\([.steps[] | select(.conclusion=="failure") | .name] | join(", "))"'
-```
-
-On a run where every job read `cancelled`, that printed:
-
-```
-Windows server unit tests (3/3) job=cancelled failedStep=Run server tests on Windows
-CI Gate job=failure failedStep=Require every selected job to pass
-Full CI Gate job=failure failedStep=Require the aggregate gate to have passed
-```
-
-Ignore the two gate rows — they always fail downstream of a red run. **Any other
-row is your culprit.** This is the same signal
-`scripts/ci-retry-cancelled-run.js` uses (`jobFailed` reads step conclusions,
-not just the job's), which is why the recovery workflow correctly refuses to
-retry a fail-fast run.
-
-Then read the assertion out of that job's log. **`--log-failed` prints nothing
-here** — GitHub populates it only for steps GitHub itself marked failed on a
-job that concluded `failure`, so a self-cancelled job yields an empty result
-and the failure looks absent. Use the full log:
-
-```bash
-gh run view --job <job-id> --log 2>&1 \
- | grep -nE '🛑 Requested cancellation|FAIL |AssertionError|✕ |Test Files .*failed'
-```
-
-The `🛑 Requested cancellation` line is written by
-`scripts/cancel-current-ci-run.js` and exists **only** when a job really failed
-— it is the confirming tell if the step query is ever inconclusive.
+A run where every job reads `cancelled` has one of three causes:
-If no job has a failed step and no log shows a failure, go to step 2.
+1. **A newer push superseded it.** `cancel-in-progress` is per PR, and a
+ supersession always leaves a newer run for the same branch:
-A Windows-only failure is a recurring shape: a test that hardcodes a POSIX
-answer for platform-dependent behaviour (shell quoting, path separators,
-process-group flags) is green locally and on Linux, and red only on the Windows
-shard. See "Windows server unit tests fail but macOS/Linux pass" below.
-
-#### Step 2 — only now consider an external cancel
-
-Confirm there is no successor run. `cancel-in-progress` is per-PR
-(`.github/workflows/ci.yml`), and a legitimate supersession always leaves a
-**newer** run for the same branch; an external cancel leaves none.
-
-```bash
-gh run list --branch "<head-branch>" --workflow CI \
- --json databaseId,headSha,status,conclusion,createdAt,attempt
-```
-
-Supporting signals for the spending-limit case, none of them sufficient alone:
-
-- Every runner is cancelled within a short window of one another, rather than
- one job finishing ~30–60 s ahead of the rest.
-- The logs show tests still passing right up to the cancel.
-- `scripts/ci-gate-report.js` distinguishes the two now, so its wording IS a
- signal — but read which sentence it printed. "Checked every job's STEP
- conclusions too, and none failed" means the lookup ran and cleared a
- fail-fast cancel. "could NOT be read" means it did not, and rules nothing
- out. A run from before #7574 says neither.
+ ```bash
+ gh run list --branch "<head-branch>" --workflow CI \
+ --json databaseId,headSha,status,conclusion,createdAt,attempt
+ ```
-Exceeding the concurrent-**job cap** makes GitHub queue runs; cancelling
-in-flight runs is the **spending-limit** behaviour. Full runs are expensive
-because the three Windows shards bill at a 2× minute multiplier.
+2. **A job hit its own `timeout-minutes`.** GitHub records that as `cancelled`,
+ not `timed_out`. The job carries an annotation reading "exceeded the maximum
+ execution time", and the step that was running is the slow one (a stalled
+ package download, a hang). A cancelled job keeps its steps' conclusions, so
+ the `CI Gate` summary (`scripts/ci-gate-report.js`) names any step that
+ failed before the cancel. By hand:
-**Account-level confirmation** (needs a scope an unattended agent cannot grant
-itself — run it yourself):
+ ```bash
+ gh api "/repos/atomantic/PortOS/actions/runs/<run-id>/jobs?per_page=100" \
+ --jq '.jobs[] | select(any(.steps[]?; .conclusion=="failure")) | .name'
+ ```
-```bash
-gh auth refresh -s user
-gh api /users/<your-login>/settings/billing/actions
-```
+3. **An external cancel**, such as GitHub cancelling in-flight runs under load,
+ the Actions spending limit, or someone pressing cancel. There is no newer
+ run and no failed step. Only the account owner can check or raise the
+ spending limit (`gh auth refresh -s user`, then
+ `gh api /users/<your-login>/settings/billing/actions`).
-Only the account owner can raise or clear the limit. If that is the cause,
-re-running spends more of the budget the limit is protecting — stop after one
-manual retry, arm auto-merge (`gh pr merge <n> --merge --auto --delete-branch`,
-which branch protection still gates), and leave the PR for the owner.
-
-#### What PortOS already does about it
-
-- One automatic retry. `.github/workflows/ci-cancel-recovery.yml` watches for a
- completed CI run and re-dispatches it exactly once when it was cancelled on a
- pull request, **no job failed**, and **no newer run exists for the branch**.
- The budget is the run attempt: `POST /rerun` produces attempt 2, and attempt
- 2 is never retried. A supersession and a self-cancel after a real failure are
- both skipped, and the recovery run's own summary page states which of those
- applied. Its "no job failed" test reads **step** conclusions as well as the
- job's (`jobFailed` in `scripts/ci-retry-cancelled-run.js`), which is what
- keeps it from burning an attempt on a fail-fast run — the same signal step 1
- above uses by hand.
-- The gate reports the difference **when the gate itself runs**.
- `scripts/ci-gate-report.js` prints `this run was CANCELLED, not failed`,
- names the cancelled jobs, and points back here — instead of the old
- undifferentiated "did not pass". Since #7574 it reaches that wording only
- after querying the run's job **steps** and finding none failed, so a
- fail-fast cancel is reported as the red run it is rather than sharing the
- external cancel's line. Two limits remain: `if: always()` defeats an upstream
- failure, not a run-wide cancellation, so in the full external-cancel case the
- gate job is cancelled too and prints nothing; and when the step lookup itself
- fails the gate says so and falls back to the conclusion-only wording, which
- proves nothing about causes. It covers a partial cancel; the recovery run
- covers the rest.
-
-**A caveat worth knowing**: the retry re-runs the whole suite, Windows shards
-included, so if the cause really is the spending limit then recovery spends
-more of it. That is the same cost as the manual re-run it replaces, but it
-means recovery is not a substitute for reducing billable minutes — tracked in
-issue #7440.
-
-One deliberate rough edge: a run **you** cancel by hand looks identical to an
-external cancel from the API, so it gets the same single retry. Push a new
-commit (or close the PR) rather than cancelling if you want the run to stay
-stopped.
+**What PortOS does about it.** `.github/workflows/ci-cancel-recovery.yml`
+re-dispatches a cancelled pull-request run exactly once when no job failed, no
+job hit its timeout, and no newer run exists for the branch. Its summary page
+says which of those applied. A run you cancel by hand looks like an external
+cancel and gets the same single retry; push a new commit or close the PR if you
+want it to stay stopped.
### Windows server unit tests fail but macOS/Linux pass
**Symptom**: `Windows server unit tests (n/3)` is the only shard with a real
-assertion failure. Because that shard fail-fasts, the run then reads as a
-run-wide cancel — see the section above for how to find the assertion at all.
+assertion failure.
The cause is almost always a **test** that hardcodes the POSIX answer to a
platform-dependent question, not a production bug. Three recurring shapes:
diff --git a/scripts/cancel-current-ci-run.js b/scripts/cancel-current-ci-run.js
deleted file mode 100644
index 64d747d37b..0000000000
--- a/scripts/cancel-current-ci-run.js
+++ /dev/null
@@ -1,164 +0,0 @@
-#!/usr/bin/env node
-
-/**
- * Report the failure, then cancel the workflow run executing this step.
- *
- * The workflow deliberately supplies no target arguments. The repository and
- * run id come only from GitHub Actions' environment, so a PR cannot redirect
- * this request to another run or repository through script arguments.
- *
- * REPORTING COMES FIRST, and it is the reason this script is not just a POST.
- * Cancelling the run stops the failing job before GitHub writes its `failure`
- * conclusion, so afterwards `gh pr checks`, `gh run view --json jobs` and
- * `gh run view --log-failed` all show a wall of `cancelled` with no cause —
- * three debugging sessions went looking for a billing problem instead of the
- * one-line assertion sitting in a job log (issue 7574). A workflow annotation
- * is written the instant it is printed and survives the cancellation, so the
- * run says what failed before it stops saying anything at all.
- *
- * This script runs after a real CI failure. NOTHING here may replace that
- * failure: every diagnostic and cancellation path returns normally after
- * emitting a single line, and the cancel is attempted even when the reporting
- * half throws.
- */
-
-import { isDirectlyInvoked } from './lib/directInvocation.js';
-import {
- currentRunTargetFromEnv, fetchJobsWithFailedSteps, formatFailedJob, githubRequest, isSuccess, trimmed,
-} from './lib/githubActionsApi.js';
-import { formatErrorAnnotation, safeWorkflowText, writeStepSummary } from './lib/githubOutput.js';
-
-const CANCELLATION_TIMEOUT_MS = 10_000;
-/**
- * Short, because this lookup sits between the failure and the cancel and every
- * second of it is a second ~13 doomed sibling legs keep billing (the Windows
- * shards at a 2x minute multiplier). Three seconds is ~40 runner-seconds in the
- * worst case and a few in the normal one.
- *
- * Running the lookup CONCURRENTLY with the cancel would cost nothing at all,
- * and is deliberately not done: once the cancel lands, Actions may kill this
- * job mid-step, and the enriched annotation — the entire point of issue 7574 —
- * would be lost to that race on exactly the runs it exists to explain.
- */
-const DIAGNOSTIC_TIMEOUT_MS = 3_000;
-
-/**
- * This job, as a human reads it in the Checks tab.
- *
- * `GITHUB_JOB` is the job ID (`windows-server`), not the rendered name, and
- * Actions exposes no variable for the matrix leg — so the workflow passes the
- * shard in. Together they identify the row without an API call, which is what
- * keeps the annotation useful when the lookup below is the thing that failed.
- */
-function localJobLabel(env) {
- const job = safeWorkflowText(env.GITHUB_JOB, 'this job');
- const shard = trimmed(env.CI_FAILED_SHARD);
- return /^\d+$/.test(shard) ? `${job} (shard ${shard})` : job;
-}
-
-/**
- * Write the annotation and the run-summary block naming what failed.
- *
- * Never throws: it runs after a real CI failure, and a diagnostic that took the
- * cancel down with it would cost money on every red build.
- */
-async function reportFailureBeforeCancel({ env, fetchImpl, logger, writeSummary, target }) {
- const label = localJobLabel(env);
- const found = await fetchJobsWithFailedSteps(fetchImpl, target && {
- ...target,
- timeoutMs: DIAGNOSTIC_TIMEOUT_MS,
- });
- // `scripts/run-ci-tests.js` sets this via $GITHUB_ENV when a Vitest worker
- // crashed natively (a Windows fail-fast abort, not an assertion) and the
- // crash reproduced on a same-file retry — the API lookup above can't see
- // this because it names FAILED STEPS, and the crash killed the step before
- // any per-test output named the file (issue 8152).
- const crashedFile = safeWorkflowText(env.CI_CRASHED_TEST_FILE);
- const lines = found?.length
- ? found.map(formatFailedJob)
- : [crashedFile
- ? `${label} — the failing step could not be read from the Actions API; the Vitest worker crashed natively on ${crashedFile} (see this job's log)`
- : `${label} — the failing step could not be read from the Actions API; open this job's log`];
-
- try {
- for (const line of lines) logger.log?.(formatErrorAnnotation(`CI failed: ${label}`, line));
- writeSummary(
- [
- `### ❌ CI failed in ${label}`,
- '',
- ...lines.map((line) => `- ${line}`),
- '',
- 'Every other job was cancelled on purpose by the fail-fast step, so they are'
- + ' collateral and not the cause. `gh run view --log-failed` is empty for this'
- + ' run because the cancel lands before a `failure` conclusion is written —'
- + " open the job named above and read its log.",
- ].join('\n'),
- env,
- );
- } catch (error) {
- logger.error?.(`⚠️ Could not write the CI failure annotation: ${error?.message || 'unknown error'}`);
- }
-}
-
-/**
- * Report the failure, then ask GitHub to cancel the workflow run containing
- * this step.
- *
- * @param {object} [options]
- * @param {NodeJS.ProcessEnv} [options.env] - Actions environment to read.
- * @param {typeof fetch} [options.fetchImpl] - Injectable fetch for tests.
- * @param {{log?: Function, error?: Function}} [options.logger] - Injectable logger.
- * @param {Function} [options.writeSummary] - Injectable step-summary writer.
- * @returns {Promise<{outcome: string, status?: number, reason?: string}>}
- */
-export async function cancelCurrentCiRun({
- env = process.env,
- fetchImpl = globalThis.fetch,
- logger = console,
- writeSummary = writeStepSummary,
-} = {}) {
- const target = currentRunTargetFromEnv(env);
-
- // Before the POST, and ahead of the early returns below: a job whose token or
- // repository is unusable still failed, and the annotation needs neither.
- await reportFailureBeforeCancel({ env, fetchImpl, logger, writeSummary, target });
-
- if (!target) {
- logger.error?.('⚠️ CI run cancellation skipped: required GitHub Actions environment is unavailable');
- return { outcome: 'skipped', reason: 'invalid-environment' };
- }
-
- if (typeof fetchImpl !== 'function') {
- logger.error?.('⚠️ CI run cancellation unavailable: fetch is not available');
- return { outcome: 'unavailable', reason: 'fetch-unavailable' };
- }
-
- try {
- const url = `${target.repoPath}/actions/runs/${target.runId}/cancel`;
- const response = await githubRequest(fetchImpl, url, target.token, {
- method: 'POST',
- timeoutMs: CANCELLATION_TIMEOUT_MS,
- });
- const status = Number(response?.status) || 0;
-
- if (isSuccess(response)) {
- logger.log?.('🛑 Requested cancellation of the current CI run');
- return { outcome: 'requested', status };
- }
-
- if (status === 409) {
- logger.log?.('ℹ️ Current CI run is already terminal; sibling cancellation was unnecessary');
- return { outcome: 'already-terminal', status };
- }
-
- logger.error?.(`⚠️ Could not cancel the current CI run: GitHub API returned ${status || 'an unknown status'}`);
- return { outcome: 'unavailable', status };
- } catch (error) {
- logger.error?.(`⚠️ Could not cancel the current CI run: ${error?.message || 'network request failed'}`);
- return { outcome: 'unavailable', reason: 'request-failed' };
- }
-}
-
-if (isDirectlyInvoked(import.meta.url)) {
- await cancelCurrentCiRun();
-}
diff --git a/scripts/cancel-current-ci-run.test.js b/scripts/cancel-current-ci-run.test.js
deleted file mode 100644
index 76267e2757..0000000000
--- a/scripts/cancel-current-ci-run.test.js
+++ /dev/null
@@ -1,266 +0,0 @@
-import { describe, expect, it, vi } from 'vitest';
-
-import { cancelCurrentCiRun } from './cancel-current-ci-run.js';
-
-const ENV = {
- GITHUB_REPOSITORY: 'example/portos',
- GITHUB_RUN_ID: '123456789',
- GITHUB_TOKEN: 'ephemeral-test-token',
-};
-
-const response = (status) => ({ ok: status >= 200 && status < 300, status });
-
-function setup() {
- return {
- fetchImpl: vi.fn(),
- logger: { log: vi.fn(), error: vi.fn() },
- };
-}
-
-describe('cancelCurrentCiRun', () => {
- it('cancels the current repository run with the step-scoped token', async () => {
- const { fetchImpl, logger } = setup();
- fetchImpl.mockResolvedValue(response(202));
-
- await expect(cancelCurrentCiRun({ env: ENV, fetchImpl, logger })).resolves.toEqual({
- outcome: 'requested',
- status: 202,
- });
-
- expect(fetchImpl).toHaveBeenCalledWith(
- 'https://api.github.com/repos/example/portos/actions/runs/123456789/cancel',
- expect.objectContaining({
- method: 'POST',
- headers: {
- Accept: 'application/vnd.github+json',
- Authorization: 'Bearer ephemeral-test-token',
- 'X-GitHub-Api-Version': '2022-11-28',
- },
- signal: expect.any(AbortSignal),
- }),
- );
- expect(logger.error).not.toHaveBeenCalled();
- });
-
- it('treats an already-terminal run as a successful no-op', async () => {
- const { fetchImpl, logger } = setup();
- fetchImpl.mockResolvedValue(response(409));
-
- await expect(cancelCurrentCiRun({ env: ENV, fetchImpl, logger })).resolves.toEqual({
- outcome: 'already-terminal',
- status: 409,
- });
-
- expect(logger.error).not.toHaveBeenCalled();
- });
-
- it('uses the Actions API base and falls back to the public API when it is absent', async () => {
- const { fetchImpl, logger } = setup();
- fetchImpl.mockResolvedValue(response(202));
-
- await cancelCurrentCiRun({
- env: { ...ENV, GITHUB_API_URL: 'https://github.example.test/api/v3/' },
- fetchImpl,
- logger,
- });
- expect(fetchImpl).toHaveBeenLastCalledWith(
- 'https://github.example.test/api/v3/repos/example/portos/actions/runs/123456789/cancel',
- expect.any(Object),
- );
-
- fetchImpl.mockClear();
- await cancelCurrentCiRun({ env: ENV, fetchImpl, logger });
- expect(fetchImpl).toHaveBeenCalledWith(
- 'https://api.github.com/repos/example/portos/actions/runs/123456789/cancel',
- expect.any(Object),
- );
- });
-
- it('rejects an unsafe API base without making a request', async () => {
- const { fetchImpl, logger } = setup();
-
- await expect(cancelCurrentCiRun({
- env: { ...ENV, GITHUB_API_URL: 'http://example.invalid/api/v3' },
- fetchImpl,
- logger,
- })).resolves.toEqual({ outcome: 'skipped', reason: 'invalid-environment' });
-
- expect(fetchImpl).not.toHaveBeenCalled();
-
- await expect(cancelCurrentCiRun({
- env: { ...ENV, GITHUB_API_URL: 'not-a-url' },
- fetchImpl,
- logger,
- })).resolves.toEqual({ outcome: 'skipped', reason: 'invalid-environment' });
-
- expect(fetchImpl).not.toHaveBeenCalled();
- });
-
- it('keeps a permission failure best-effort and preserves the original job failure', async () => {
- const { fetchImpl, logger } = setup();
- fetchImpl.mockResolvedValue(response(403));
-
- await expect(cancelCurrentCiRun({ env: ENV, fetchImpl, logger })).resolves.toEqual({
- outcome: 'unavailable',
- status: 403,
- });
-
- expect(logger.error).toHaveBeenCalledWith(expect.stringContaining('403'));
- });
-
- it('keeps a network failure best-effort', async () => {
- const { fetchImpl, logger } = setup();
- fetchImpl.mockRejectedValue(new Error('network unavailable'));
-
- await expect(cancelCurrentCiRun({ env: ENV, fetchImpl, logger })).resolves.toEqual({
- outcome: 'unavailable',
- reason: 'request-failed',
- });
-
- expect(logger.error).toHaveBeenCalledWith(expect.stringContaining('network unavailable'));
- });
-
- it('does not make a request when the target cannot come from Actions environment', async () => {
- const { fetchImpl, logger } = setup();
-
- await expect(cancelCurrentCiRun({
- env: { ...ENV, GITHUB_REPOSITORY: 'https://example.invalid/other', GITHUB_RUN_ID: 'run-from-args' },
- fetchImpl,
- logger,
- })).resolves.toEqual({ outcome: 'skipped', reason: 'invalid-environment' });
-
- expect(fetchImpl).not.toHaveBeenCalled();
- expect(logger.error).toHaveBeenCalledWith(expect.stringContaining('environment'));
- });
-});
-
-/**
- * Issue 7574: cancelling the run stops the failing job before GitHub records
- * its `failure` conclusion, so the annotation printed here is the ONLY surface
- * that survives to name the culprit. These cover the ordering that makes that
- * true, and the contract that a broken diagnostic never costs the cancel.
- */
-describe('reportFailureBeforeCancel', () => {
- const ANNOTATION_ENV = { ...ENV, GITHUB_JOB: 'windows-server', CI_FAILED_SHARD: '3' };
- const JOBS_URL = 'https://api.github.com/repos/example/portos/actions/runs/123456789/jobs?per_page=100';
- const CANCEL_URL = 'https://api.github.com/repos/example/portos/actions/runs/123456789/cancel';
- const jobsResponse = (jobs) => ({ ok: true, status: 200, json: async () => ({ jobs }) });
- const FAILED_RUN = [
- { name: 'Server tests (1/2)', conclusion: 'cancelled', steps: [{ name: 'Run server tests', conclusion: 'cancelled' }] },
- {
- name: 'Windows server unit tests (3/3)',
- conclusion: 'cancelled',
- steps: [
- { name: 'Checkout', conclusion: 'success' },
- { name: 'Run server tests on Windows', conclusion: 'failure' },
- ],
- },
- ];
-
- it('annotates the failing job and step BEFORE it asks for the cancel', async () => {
- const { fetchImpl, logger } = setup();
- const summaries = [];
- fetchImpl.mockImplementation((url) => (
- url === JOBS_URL ? Promise.resolve(jobsResponse(FAILED_RUN)) : Promise.resolve(response(202))
- ));
-
- await cancelCurrentCiRun({
- env: ANNOTATION_ENV, fetchImpl, logger, writeSummary: (markdown) => summaries.push(markdown),
- });
-
- const annotation = logger.log.mock.calls.map(([line]) => line).find((line) => line.startsWith('::error'));
- expect(annotation).toContain('Windows server unit tests (3/3) — failed step: Run server tests on Windows');
- // Ordering is the whole point: a cancel that lands first can kill this job
- // mid-step, and an unwritten annotation explains nothing.
- expect(fetchImpl.mock.calls.map(([url]) => url)).toEqual([JOBS_URL, CANCEL_URL]);
- expect(summaries[0]).toContain('Run server tests on Windows');
- expect(logger.error).not.toHaveBeenCalled();
- });
-
- it('still names this job and shard when the step lookup is unavailable', async () => {
- // The fallback is why the workflow passes the shard in: `GITHUB_JOB` alone
- // cannot tell three identical Windows rows apart.
- const { fetchImpl, logger } = setup();
- fetchImpl.mockImplementation((url) => (
- url === JOBS_URL ? Promise.reject(new Error('network unavailable')) : Promise.resolve(response(202))
- ));
-
- await expect(cancelCurrentCiRun({ env: ANNOTATION_ENV, fetchImpl, logger }))
- .resolves.toEqual({ outcome: 'requested', status: 202 });
-
- const annotation = logger.log.mock.calls.map(([line]) => line).find((line) => line.startsWith('::error'));
- expect(annotation).toContain('windows-server (shard 3)');
- });
-
- it('names the crashed test file in the fallback when a native Vitest worker crash killed the step (issue 8152)', async () => {
- // `run-ci-tests.js` writes CI_CRASHED_TEST_FILE via $GITHUB_ENV when a
- // worker crash reproduces on retry — the failing-step lookup can't see
- // this itself because the crash killed the step before Actions recorded
- // a per-step failure the jobs API can read back.
- const { fetchImpl, logger } = setup();
- fetchImpl.mockImplementation((url) => (
- url === JOBS_URL ? Promise.reject(new Error('network unavailable')) : Promise.resolve(response(202))
- ));
-
- await cancelCurrentCiRun({
- env: { ...ANNOTATION_ENV, CI_CRASHED_TEST_FILE: 'server/services/sprites/importer.test.js' },
- fetchImpl,
- logger,
- });
-
- const annotation = logger.log.mock.calls.map(([line]) => line).find((line) => line.startsWith('::error'));
- expect(annotation).toContain('server/services/sprites/importer.test.js');
- expect(annotation).not.toContain('open this job\'s log');
- });
-
- it('annotates even when the environment cannot authorize a cancel', async () => {
- const { fetchImpl, logger } = setup();
-
- await expect(cancelCurrentCiRun({ env: { GITHUB_JOB: 'lint' }, fetchImpl, logger }))
- .resolves.toEqual({ outcome: 'skipped', reason: 'invalid-environment' });
-
- expect(logger.log).toHaveBeenCalledWith(expect.stringContaining('::error'));
- expect(logger.log).toHaveBeenCalledWith(expect.stringContaining('lint'));
- expect(fetchImpl).not.toHaveBeenCalled();
- });
-
- it('still attempts the cancel when the annotation path throws', async () => {
- // The script runs after a real failure. A diagnostic that takes the cancel
- // down with it would cost money on every red build.
- const { fetchImpl } = setup();
- const logger = {
- log: vi.fn((line) => {
- if (String(line).startsWith('::error')) throw new Error('stdout closed');
- }),
- error: vi.fn(),
- };
- fetchImpl.mockResolvedValue(response(202));
-
- await expect(cancelCurrentCiRun({ env: ANNOTATION_ENV, fetchImpl, logger }))
- .resolves.toMatchObject({ outcome: 'requested' });
-
- expect(logger.error).toHaveBeenCalledWith(expect.stringContaining('annotation'));
- expect(fetchImpl).toHaveBeenCalledWith(CANCEL_URL, expect.objectContaining({ method: 'POST' }));
- });
-
- it('keeps a workflow-sourced name from forging its own workflow command', async () => {
- // Job and step names come from a workflow file, and Actions reads a leading
- // `::` as a command — so a name must never be able to open one.
- const { fetchImpl, logger } = setup();
- fetchImpl.mockImplementation((url) => (
- url === JOBS_URL
- ? Promise.resolve(jobsResponse([{
- name: 'evil\n::error::forged',
- steps: [{ name: 'step', conclusion: 'failure' }],
- }]))
- : Promise.resolve(response(202))
- ));
-
- await cancelCurrentCiRun({ env: ANNOTATION_ENV, fetchImpl, logger });
-
- const printed = logger.log.mock.calls.map(([line]) => line).filter((line) => line.startsWith('::error'));
- expect(printed).toHaveLength(1);
- expect(printed[0]).not.toContain('::error::forged');
- expect(printed[0]).toContain('evil');
- });
-});
diff --git a/scripts/ci-base-sha.test.js b/scripts/ci-base-sha.test.js
index 70e461a47c..a3caf15ea0 100644
--- a/scripts/ci-base-sha.test.js
+++ b/scripts/ci-base-sha.test.js
@@ -122,7 +122,7 @@ describe('ci.yml server node_modules cache', () => {
it('caches the installed tree on every job that installs it', () => {
expect(installers.map(([id]) => id).sort())
- .toEqual(['database', 'server', 'windows-server']);
+ .toEqual(['browser', 'database', 'server', 'windows-server']);
for (const [, body] of installers) expect(cacheEntry(body)).toMatch(/id: server-modules\b/);
});
diff --git a/scripts/ci-fail-fast.test.js b/scripts/ci-fail-fast.test.js
deleted file mode 100644
index 96393a99cc..0000000000
--- a/scripts/ci-fail-fast.test.js
+++ /dev/null
@@ -1,112 +0,0 @@
-import { readFileSync } from 'fs';
-import { dirname, join } from 'path';
-import { fileURLToPath } from 'url';
-
-import { describe, expect, it } from 'vitest';
-
-import { workflowJobs } from './lib/workflowJobs.js';
-
-const REPO_ROOT = join(dirname(fileURLToPath(import.meta.url)), '..');
-const WORKFLOW = readFileSync(join(REPO_ROOT, '.github/workflows/ci.yml'), 'utf8');
-const NON_LEAF_JOBS = new Set(['impact', 'gate', 'full-gate']);
-// The expensive fan-out jobs that exist today. A FLOOR, not a snapshot: a new
-// leaf job has to be discovered on its own and held to the contract below, so
-// nothing here has to be edited to add one. What this list catches is the
-// opposite — discovery silently ceasing to see one of the four while the other
-// three keep every assertion green.
-const EXPECTED_LEAF_JOBS = ['client', 'database', 'server', 'windows-server'];
-const CANCEL_STEP_HEADING = ' - name: Report the failure, then cancel sibling CI jobs';
-// Both spellings of the dependency. Under a scalar-only pattern, a leaf job
-// rewritten to the list form (`needs: [impact]`) dropped out of every assertion
-// below without failing anything — the exact silent coverage loss this contract
-// exists to prevent.
-const NEEDS_IMPACT = /^\s*needs:\s*(?:impact\s*$|\[[^\]]*\bimpact\b)/m;
-const FAIL_FAST_STEP = [
- CANCEL_STEP_HEADING,
- " if: failure() && github.event_name == 'pull_request'",
- ' env:',
- // The annotation names the matrix leg, and Actions exposes no variable for
- // it — so the shard is passed in. Empty on the unsharded jobs, which the
- // script treats as absent.
- ' CI_FAILED_SHARD: ${{ matrix.shard }}',
- ' GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}',
- ' run: node scripts/cancel-current-ci-run.js',
-].join('\n');
-/**
- * Comments are explanation, not contract. Stripping them before the exact match
- * below keeps the guard on the step's BEHAVIOUR — `if:`, env, `run:` — so
- * rewording a comment in ci.yml cannot fail CI in five jobs at once.
- */
-const withoutComments = (yaml) => yaml
- .split('\n')
- .filter((line) => !/^\s*#/.test(line))
- .join('\n');
-
-describe('ci.yml fail-fast cancellation contract', () => {
- const jobs = workflowJobs(WORKFLOW);
- const leafJobs = Object.keys(jobs).filter((id) => (
- !NON_LEAF_JOBS.has(id) && NEEDS_IMPACT.test(jobs[id])
- ));
-
- it('discovers every direct impact leaf job for contract checks', () => {
- expect(leafJobs).toEqual(expect.arrayContaining(EXPECTED_LEAF_JOBS));
- });
-
- it('covers every cancellation step the workflow actually installs', () => {
- // The discovery floor from the other side. A cancellation step added to a
- // job this file does not classify as a leaf would never be checked for the
- // `actions: write` grant it needs, and a missing grant fails only at
- // runtime — inside a job that is otherwise passing.
- const installed = WORKFLOW.split(CANCEL_STEP_HEADING).length - 1;
- expect(installed).toBe(leafJobs.length);
- });
-
- it('adds the cancellation step as the final step of every expensive leaf job', () => {
- for (const id of leafJobs) {
- const body = jobs[id];
- expect(body, id).toBeTruthy();
- const step = withoutComments(body.slice(body.lastIndexOf(CANCEL_STEP_HEADING))).trimEnd();
- expect(step, id).toBe(FAIL_FAST_STEP);
- expect(body.match(/GITHUB_TOKEN:/g), id).toHaveLength(1);
- }
- });
-
- it('grants only the leaf jobs the minimum cancellation permissions', () => {
- for (const id of leafJobs) {
- const body = jobs[id];
- expect(body, id).toMatch(/\n permissions:\n contents: read\n actions: write\n/);
- }
- for (const id of NON_LEAF_JOBS) {
- expect(jobs[id], id).not.toContain('actions: write');
- }
- });
-
- it('does not pass arbitrary repository or run targets to the helper', () => {
- for (const id of leafJobs) {
- const body = jobs[id];
- expect(body, id).not.toMatch(/GITHUB_(?:REPOSITORY|RUN_ID):/);
- const step = body.slice(body.lastIndexOf(CANCEL_STEP_HEADING));
- expect(step, id).not.toContain('\n with:');
- expect(body, id).toContain('if: failure()');
- }
- });
-
- it('does not persist the elevated checkout token into leaf job steps', () => {
- for (const id of leafJobs) {
- const body = jobs[id];
- const checkoutStart = body.indexOf(' - uses: actions/checkout@');
- const nextStep = body.indexOf('\n - ', checkoutStart + 1);
- const checkout = body.slice(checkoutStart, nextStep === -1 ? undefined : nextStep);
- expect(checkout, id).toContain('persist-credentials: false');
- }
- });
-
- it('grants the reusable release caller the permission leaf jobs require', () => {
- // Bounded to the full-ci job: slicing to end of file would let a
- // permissions block on the later `release` job satisfy this instead.
- const releaseWorkflow = readFileSync(join(REPO_ROOT, '.github/workflows/release.yml'), 'utf8');
- const fullCi = workflowJobs(releaseWorkflow)['full-ci'];
- expect(fullCi).toBeTruthy();
- expect(fullCi).toMatch(/\n permissions:\n contents: read\n actions: write\n/);
- });
-});
diff --git a/scripts/ci-gate-report.js b/scripts/ci-gate-report.js
index d5db6382b5..9c1160cd63 100644
--- a/scripts/ci-gate-report.js
+++ b/scripts/ci-gate-report.js
@@ -14,26 +14,23 @@
* - cancelled — at least one job was `cancelled` and NOTHING failed. There
* is no test failure to chase. Still non-zero: a cancelled
* run proves nothing and must not be merged.
- * - failure — at least one job genuinely failed. Cancelled siblings are
- * reported as collateral, because the fail-fast step in
- * ci.yml cancels them on purpose.
+ * - failure — at least one job genuinely failed. Any cancelled jobs are
+ * listed beside it.
*
* The catch, and why this talks to the API at all: `needs.<job>.result` is the
- * job's CONCLUSION, and the fail-fast step cancels the run before GitHub writes
- * the failing job's `failure` conclusion. The culprit therefore arrives here as
- * `cancelled` like everything else, and the gate reported the cancelled verdict
- * on a run that really was red (issue 7574). A cancelled job keeps its STEPS'
+ * job's CONCLUSION, and a job that is cancelled mid-step (a run-wide cancel, or
+ * its own `timeout-minutes`) reads `cancelled` even when a step had already
+ * failed (issue 7574). A cancelled job keeps its STEPS'
* conclusions, so when — and only when — the cancelled verdict is about to be
* printed, the run's jobs are queried for a failed step. Best-effort by design:
* an API failure falls back to today's wording rather than turning a cancelled
* run red or green on a network blip.
*
* Scope, so the next reader is not misled: this runs only when the GATE job
- * itself runs. The two cancels differ here, and it matters. Our OWN fail-fast
- * cancel leaves the gate running — `if: always()` survives it, and the gate is
- * where a red run gets its verdict, which is why the culprit lookup above is
- * worth having. An EXTERNAL run-wide cancellation takes the gate with it, and
- * then the explanation comes from the recovery workflow instead — see
+ * itself runs. A job timeout leaves the gate running (`if: always()`), so the
+ * culprit lookup above explains it here. An EXTERNAL run-wide cancellation
+ * takes the gate with it, and then the explanation comes from the recovery
+ * workflow instead — see
* "External cancellation and one automatic retry" in docs/GITHUB_ACTIONS.md.
*
* Reads `CI_GATE_RESULT_<JOB>` out of the environment rather than taking
@@ -132,7 +129,7 @@ export function summarizeGateResults(
`❌ ${label}: selected CI jobs did not pass.`,
`Failed jobs: ${failed.join(', ')}`,
...(cancelled.length
- ? [`Cancelled alongside the failure (expected — the fail-fast step stops siblings): ${cancelled.join(', ')}`]
+ ? [`Also cancelled: ${cancelled.join(', ')}`]
: []),
finished(),
],
@@ -144,8 +141,8 @@ export function summarizeGateResults(
return {
verdict: 'failure',
lines: [
- `❌ ${label}: a job FAILED. Its conclusion reads cancelled because the fail-fast`
- + ' step stopped the run before GitHub recorded the failure.',
+ `❌ ${label}: a job FAILED. Its conclusion reads cancelled because the run`
+ + ' was stopped before GitHub recorded the failure.',
...culprits.map((job) => `Failing job: ${formatFailedJob(job)}`),
`Cancelled jobs: ${cancelled.join(', ')}`,
finished(),
@@ -162,25 +159,19 @@ export function summarizeGateResults(
finished(),
// This distinction is load-bearing. The old line said "No test failed",
// and twice (#7482, #7571) that sent a reader straight to the billing
- // dashboard while a real assertion sat unread in a job log: a job that
- // fails is cancelled by scripts/cancel-current-ci-run.js before its own
- // reporting step runs, so its conclusion is `cancelled` too and NOTHING
- // in the run metadata reads `failure`. That is now CHECKED rather than
+ // dashboard while a real assertion sat unread in a job log: a job
+ // cancelled mid-step concludes `cancelled` even when a step failed, so
+ // NOTHING in the run metadata reads `failure`. That is now CHECKED rather than
// guessed — but only when the lookup ran, so the two states say
// different things instead of sharing the more alarming wording.
...(culprits
? ["Checked every job's STEP conclusions too, and none failed — so this"
- + ' is not the usual fail-fast cancel wearing a cancelled conclusion.']
+ + ' is not a failed step hidden behind a cancelled conclusion.']
: ["This run's step conclusions could NOT be read, so a cancelled"
- + ' conclusion here does NOT mean no test failed: PortOS fail-fasts,'
- + ' the job that failed is cancelled out from under its own reporting'
- + ' step, and a real red build then looks exactly like this.'
- + ' READ THE JOB LOGS FIRST — and note that'
- + ' `gh run view --job <id> --log-failed` prints NOTHING here, because'
- + ' no step is marked failed. Use the full `--log` and grep for'
- + ' `🛑 Requested cancellation` (written only when a job really'
- + ' failed), `FAIL `, or `AssertionError`; the culprit is usually the'
- + ' job that completed EARLIEST.']),
+ + ' conclusion here does NOT mean no test failed: a job cancelled'
+ + ' mid-step hides a failed step behind a cancelled conclusion.'
+ + ' READ THE JOB LOGS FIRST — use the full `--log` and grep for'
+ + ' `FAIL ` or `AssertionError`.']),
// Every ci.yml job sets its own `timeout-minutes`, so the limit that
// usually bites is the job's (20 minutes for DB tests), not GitHub's
// 6-hour default. GitHub records either as `cancelled`, so a slow apt
@@ -222,7 +213,7 @@ export function summarizeGateResults(
*
* `null` on an unusable environment or a failed lookup, kept distinct from the
* empty array: both keep the cancelled verdict, but only "looked and found
- * none" earns the wording that rules a fail-fast cancel out. A failed lookup
+ * none" earns the wording that rules a hidden failed step out. A failed lookup
* must never invent a failure the run may not have, nor vouch for its absence.
*
* @param {NodeJS.ProcessEnv} env
diff --git a/scripts/ci-gate-report.test.js b/scripts/ci-gate-report.test.js
index 7a5fd90043..8034e83e4e 100644
--- a/scripts/ci-gate-report.test.js
+++ b/scripts/ci-gate-report.test.js
@@ -58,7 +58,7 @@ describe('summarizeGateResults', () => {
const text = summary.lines.join('\n');
expect(text).toContain('CANCELLED, not failed');
expect(text).toContain('no job REPORTED a failure');
- // The gate reads job CONCLUSIONS, and a fail-fast run hides a real
+ // The gate reads job CONCLUSIONS, and a cancelled job can hide a real
// failure behind the same conclusions (#7482, #7571). The summary must
// therefore send the reader to the logs, never imply the tree is green.
expect(text).toContain('READ THE JOB LOGS FIRST');
@@ -140,7 +140,7 @@ describe('summarizeGateResults', () => {
});
it('names the culprit instead of reporting a cancel when a job has a failed step', () => {
- // The shape of every fail-fast run (7574): the failing job's own conclusion
+ // A job cancelled mid-step (7574): the failing job's own conclusion
// is `cancelled` because the cancel beat GitHub to writing it, so this is
// the ONLY input that can tell the reader a test actually failed.
const summary = summarizeGateResults(
diff --git a/scripts/ci-retry-cancelled-run.js b/scripts/ci-retry-cancelled-run.js
index 6bc7433bd8..35c64f8a8a 100644
--- a/scripts/ci-retry-cancelled-run.js
+++ b/scripts/ci-retry-cancelled-run.js
@@ -21,9 +21,8 @@
* event payload, because the payload is a snapshot: a human can re-run
* the same run, or the event can be redelivered, while this job is
* still queued behind the very backlog it exists to survive.
- * - No job may have concluded `failure`/`timed_out`. This repository cancels
- * its own run from a failing job (scripts/cancel-current-ci-run.js), which
- * makes the RUN read `cancelled` while a job really did fail.
+ * - No job may have concluded `failure`/`timed_out`, or carry a failed step.
+ * A run cancelled while one job was already red reads `cancelled` too.
* - No job may have hit its own `timeout-minutes`. GitHub reports such a
* job as `cancelled`, so only its check-run annotation tells it apart
* from an external cancel (#11910).
diff --git a/scripts/ci-retry-cancelled-run.test.js b/scripts/ci-retry-cancelled-run.test.js
index 5417b82f3b..8a32a077b5 100644
--- a/scripts/ci-retry-cancelled-run.test.js
+++ b/scripts/ci-retry-cancelled-run.test.js
@@ -4,7 +4,7 @@
* looping forever, or fighting a legitimate `cancel-in-progress` supersession.
*
* The API-base and repository validation live in lib/githubActionsApi.test.js,
- * shared with scripts/cancel-current-ci-run.js.
+ * shared with scripts/ci-gate-report.js.
*/
import { readFileSync } from 'fs';
import { dirname, join } from 'path';
@@ -145,10 +145,10 @@ describe('retryCancelledCiRun guards', () => {
});
it.each(['failure', 'timed_out'])(
- 'never retries a run whose job concluded %s — this repo self-cancels on a real red',
+ 'never retries a run whose job concluded %s',
async (conclusion) => {
- // scripts/cancel-current-ci-run.js makes the RUN read `cancelled` while a
- // job really failed. Retrying that re-runs the whole suite on a red tree.
+ // A run cancelled while one job was already red reads `cancelled`.
+ // Retrying that re-runs the whole suite on a red tree.
const fetchStub = stubFetch({
jobs: [{ name: 'Server tests', conclusion }, { name: 'Client', conclusion: 'cancelled' }],
});
@@ -182,9 +182,9 @@ describe('retryCancelledCiRun guards', () => {
});
it('retries a job whose STEP failed but whose conclusion reads cancelled', async () => {
- // The fail-fast self-cancel can land while the failing job is still in its
- // post-steps, and GitHub then records that job as `cancelled`. Trusting the
- // job conclusion alone would re-run a genuinely red tree.
+ // A cancel can land while the failing job is still in its post-steps, and
+ // GitHub then records that job as `cancelled`. Trusting the job conclusion
+ // alone would re-run a genuinely red tree.
const fetchStub = stubFetch({
jobs: [{
name: 'Server tests',
@@ -461,9 +461,8 @@ describe('the wait before re-dispatching', () => {
});
it('catches a job whose failure only settles DURING the wait', async () => {
- // The fail-fast self-cancel can land while the failing job is still in its
- // post-steps, so the pre-wait listing can legitimately read clean. The
- // post-wait re-read is the more reliable one, and must block the retry.
+ // A cancel can land while the failing job is still in its post-steps, so
+ // the pre-wait listing can legitimately read clean. The post-wait re-read is the more reliable one, and must block the retry.
let stepsSettled = false;
const impl = vi.fn(async (url) => {
if (url.includes('/jobs?')) {
diff --git a/scripts/ci-test-diagnostics.test.js b/scripts/ci-test-diagnostics.test.js
index f2432b9462..7436c30791 100644
--- a/scripts/ci-test-diagnostics.test.js
+++ b/scripts/ci-test-diagnostics.test.js
@@ -138,15 +138,13 @@ describe('independent CI test failure recovery (#11552)', () => {
it.each([
['windows-server', 'Run server tests on Windows', 'Preserve Windows test failure diagnostics', 'run-ci-tests.js'],
- ['database', 'Run cross-workspace browser suites', 'Preserve browser test failure diagnostics', 'watch-ci-tests.js'],
- ])('%s collects after failure and before sibling cancellation with bounded, non-masking failure handling', (jobId, testName, collectorName, launcherName) => {
+ ['browser', 'Run cross-workspace browser suites', 'Preserve browser test failure diagnostics', 'watch-ci-tests.js'],
+ ])('%s collects after failure with bounded, non-masking failure handling', (jobId, testName, collectorName, launcherName) => {
const workflow = readFileSync(new URL('../.github/workflows/ci.yml', import.meta.url), 'utf8');
const job = workflowJobs(workflow)[jobId];
const start = job.indexOf(` - name: ${collectorName}`);
expect(start).toBeGreaterThan(job.indexOf(`run: node scripts/${launcherName} server`));
- const end = job.indexOf(' - name: Report the failure, then cancel sibling CI jobs');
- expect(end).toBeGreaterThan(start);
- const step = job.slice(start, end);
+ const step = job.slice(start);
expect(step).toContain('if: failure()');
expect(step).toContain('continue-on-error: true');
expect(step).toContain('timeout-minutes: 1');
@@ -160,6 +158,5 @@ describe('independent CI test failure recovery (#11552)', () => {
expect(testMinutes).toBeGreaterThan(0);
expect(testMinutes).toBeLessThan(jobMinutes - 1);
expect(testStep).not.toContain('continue-on-error');
- expect(job.slice(end)).toContain("if: failure() && github.event_name == 'pull_request'");
});
});
diff --git a/scripts/lib/browserFidelity.js b/scripts/lib/browserFidelity.js
index 1d68fbdb6a..989bc460c3 100644
--- a/scripts/lib/browserFidelity.js
+++ b/scripts/lib/browserFidelity.js
@@ -45,19 +45,4 @@ export async function browserFidelityTarget(repository, pr, request, expected =
return { repository, pr, head, base, merge: SHA.test(merge) ? merge : null, tree, profile: browserFidelityProfileDigest() };
}
-export function assertFidelityStatus(identity, statuses) {
- if (!Array.isArray(statuses)) throw Error('Browser fidelity statuses unavailable.');
- const context = browserFidelityContext(identity);
- // GitHub returns newest first; numeric IDs also disambiguate same-second posts.
- const matching = statuses.filter(status => status?.context === context);
- if (matching.some(status => !Number.isSafeInteger(status.id))) throw Error('Invalid browser fidelity status identity.');
- matching.sort((a, b) => b.id - a.id);
- const latest = matching[0];
- if (!latest || latest.state !== 'success' || latest.creator?.login?.toLowerCase() !== browserFidelityActor(identity.repository)
- || !/^receipt sha256=[a-f0-9]{64}$/.test(latest.description ?? '')) {
- throw Error('Current trusted browser fidelity receipt is missing, pending, or failed.');
- }
- return { context, receiptDigest: latest.description.slice('receipt sha256='.length) };
-}
-
export const browserReceiptDigest = receipt => digest(JSON.stringify(receipt));
diff --git a/scripts/lib/githubActionsApi.js b/scripts/lib/githubActionsApi.js
index bb77e7eda5..aa73cc1b62 100644
--- a/scripts/lib/githubActionsApi.js
+++ b/scripts/lib/githubActionsApi.js
@@ -1,8 +1,8 @@
/**
* Shared GitHub Actions REST plumbing for the CI gates in `scripts/`.
*
- * Extracted because two scripts now talk to the Actions API from inside a
- * workflow — `cancel-current-ci-run.js` and `ci-retry-cancelled-run.js` — and
+ * Extracted because two scripts talk to the Actions API from inside a
+ * workflow — `ci-gate-report.js` and `ci-retry-cancelled-run.js` — and
* the piece they share is a SECURITY check, not boilerplate: `resolveApiBase`
* decides which host a step-scoped `GITHUB_TOKEN` is sent to. Two copies of
* that meant a hardening fix could land on one and silently miss the other.
@@ -123,10 +123,8 @@ export function isSuccess(response) {
/**
* The run this step is part of, plus the token to read it with — or null.
*
- * The single derivation of "which run am I, and with what token". Both the
- * fail-fast reporter and the gate reporter need it, and this module exists
- * precisely so a hardening fix to that reasoning cannot land on one copy and
- * miss the other.
+ * The single derivation of "which run am I, and with what token", kept here
+ * beside the API-host check it depends on.
*
* @param {NodeJS.ProcessEnv} env
* @returns {{token: string, repoPath: string, runId: string}|null}
@@ -174,9 +172,8 @@ const FAILING_CONCLUSIONS = new Set(['failure', 'timed_out']);
* The steps of one job that actually failed.
*
* A cancelled job KEEPS its steps' conclusions, which is the whole reason this
- * exists: `cancel-current-ci-run.js` cancels the run from inside the failing
- * job, so GitHub can record the job as `cancelled` while one of its steps is
- * `failure`. Nothing that reads only job conclusions — `gh pr checks`,
+ * exists: a job cancelled mid-step (a run-wide cancel, or its own timeout) is
+ * recorded as `cancelled` while one of its steps is `failure`. Nothing that reads only job conclusions — `gh pr checks`,
* `gh run view --json jobs`, `gh run view --log-failed` — can see that (7574).
*/
const failedSteps = (job) => (Array.isArray(job?.steps) ? job.steps : [])
diff --git a/scripts/lib/githubActionsApi.test.js b/scripts/lib/githubActionsApi.test.js
index 9d6462823a..f934e21ecb 100644
--- a/scripts/lib/githubActionsApi.test.js
+++ b/scripts/lib/githubActionsApi.test.js
@@ -122,7 +122,7 @@ describe('fetchJobsWithFailedSteps', () => {
const jobs = (value) => vi.fn().mockResolvedValue({ ok: true, status: 200, json: async () => value });
it('names the failed steps of a job GitHub recorded as cancelled', () => {
- // The fail-fast cancel lands before the job's `failure` conclusion is
+ // A mid-step cancel lands before the job's `failure` conclusion is
// written, so the step conclusions are the only surviving evidence (7574).
return expect(fetchJobsWithFailedSteps(jobs({
jobs: [
diff --git a/scripts/pre-install-entrypoints.test.js b/scripts/pre-install-entrypoints.test.js
index 30bae5d72d..470f73b390 100644
--- a/scripts/pre-install-entrypoints.test.js
+++ b/scripts/pre-install-entrypoints.test.js
@@ -43,9 +43,6 @@ const PRE_INSTALL_ENTRYPOINTS = [
* `import()` inside the database probe for the same reason, which is why it
* does not show up in this static walk.
*
- * `cancel-current-ci-run.js` is here because it runs from an `if: failure()`
- * workflow step that may fire before or during a failed dependency install.
- *
* `ci-gate-report.js` runs on the two gate jobs, which check out the repo for
* this one script and deliberately install nothing; `ci-retry-cancelled-run.js`
* is the whole body of the `workflow_run` recovery workflow, which does the
@@ -53,7 +50,6 @@ const PRE_INSTALL_ENTRYPOINTS = [
* required check — or the thing that recovers a cancelled one — unloadable.
*/
const BARE_CHECKOUT_SCRIPTS = [
- 'scripts/cancel-current-ci-run.js',
'scripts/ci-gate-report.js',
'scripts/ci-retry-cancelled-run.js',
'scripts/doctor.js',
diff --git a/scripts/pregate.js b/scripts/pregate.js
index b98eab9d55..e36b664f4b 100644
--- a/scripts/pregate.js
+++ b/scripts/pregate.js
@@ -59,21 +59,15 @@ export const UNCOVERED_SUITES = Object.freeze({
/**
* Plan lines for the cross-workspace browser selection CI consumes
- * (`browser_files`). `receipt` is the native fidelity receipt state when known
- * ('valid' | 'absent'); the receipt is a PR-head commit status, so a local plan
- * normally cannot see it and says CI verifies it.
+ * (`browser_files`). PRs run the smoke profile; the `[fidelity]` cases run in
+ * hosted CI on nightly, release and dispatched runs, and locally on request.
*/
-export function describeBrowserSelection(files, receipt = null) {
- if (files.length === 0) return ['ℹ️ Browser suites: none selected — no native browser-fidelity receipt required.'];
+export function describeBrowserSelection(files) {
+ if (files.length === 0) return ['ℹ️ Browser suites: none selected.'];
const lines = [`ℹ️ Browser suites: ${files.length} selected (${files.join(', ')}).`];
- if (!needsBrowserFidelity(files)) {
- lines.push('ℹ️ Native browser-fidelity receipt: not required for this selection.');
- } else if (receipt === 'valid') {
- lines.push('ℹ️ Native browser-fidelity receipt: required — present and valid.');
- } else if (receipt === 'absent') {
- lines.push('⚠️ Native browser-fidelity receipt: required — ABSENT; CI will fail until the trusted receipt for the PR head is posted.');
- } else {
- lines.push('⚠️ Native browser-fidelity receipt: required — not checkable locally; CI verifies a trusted receipt for the exact PR head.');
+ if (needsBrowserFidelity(files)) {
+ lines.push('ℹ️ Fidelity cases are selected: CI runs them nightly and on the release PR,'
+ + ' not on this PR. To check them now, run `node scripts/browser-fidelity.js --pr NUMBER`.');
}
return lines;
}
diff --git a/scripts/pregate.test.js b/scripts/pregate.test.js
index 8662e2d62a..34064bf5ec 100644
--- a/scripts/pregate.test.js
+++ b/scripts/pregate.test.js
@@ -533,18 +533,13 @@ describe('describeBrowserSelection', () => {
const selected = BROWSER_SUITES.slice(0, 10);
const fidelity = BROWSER_FIDELITY_FILES[0];
- it('distinguishes no selection from a selection that needs a receipt', () => {
- const none = describeBrowserSelection([]).join('\n');
- expect(none).toContain('none selected');
- expect(none).toContain('no native browser-fidelity receipt required');
+ it('distinguishes no selection from a selection with fidelity cases', () => {
+ expect(describeBrowserSelection([]).join('\n')).toContain('none selected');
const some = describeBrowserSelection([fidelity, ...selected]).join('\n');
expect(some).toContain('selected');
- expect(some).toContain('receipt: required');
- });
-
- it('reports absent and valid receipt state', () => {
- expect(describeBrowserSelection([fidelity], 'absent').join('\n')).toContain('ABSENT');
- expect(describeBrowserSelection([fidelity], 'valid').join('\n')).toContain('present and valid');
+ expect(some).toContain('nightly and on the release PR');
+ expect(describeBrowserSelection(selected.filter((file) => !BROWSER_FIDELITY_FILES.includes(file))).join('\n'))
+ .not.toContain('Fidelity cases');
});
it('shows the selection in --plan-only output', () => {
diff --git a/scripts/run-ci-tests.js b/scripts/run-ci-tests.js
index e75206690f..3042da88ba 100644
--- a/scripts/run-ci-tests.js
+++ b/scripts/run-ci-tests.js
@@ -9,7 +9,7 @@ import { fileURLToPath } from 'url';
import { killProcessTree, prepareCliSpawn } from '../server/lib/bufferedSpawn.js';
import { isDirectlyInvoked } from './lib/directInvocation.js';
import { sweepStaleRunRoots, writeOwnerFile } from './lib/vitestStaleRunRoots.js';
-import { writeStepEnv, writeStepSummary } from './lib/githubOutput.js';
+import { formatErrorAnnotation, writeStepSummary } from './lib/githubOutput.js';
import { reportWindowsCrashDiagnostics } from './ci-windows-crash-diagnostics.js';
import { diagnosticTestFile, persistTestDiagnostic } from './ci-test-diagnostics.js';
import { browserTestProfileArgs, browserTestProfileFiles, assertBrowserProfileExecuted, BROWSER_FIDELITY_ONLY_FILES } from './lib/browserTestProfiles.js';
@@ -347,7 +347,7 @@ export async function spawnNpm(scope, script, extraArgs, label, {
const plan = planCrashRetry(scope, result.output);
if (!plan.retry) {
- if (plan.crashedPath) writeStepEnv('CI_CRASHED_TEST_FILE', plan.crashedPath);
+ if (plan.crashedPath) console.error(formatErrorAnnotation('Vitest worker crashed', `The worker crashed natively on ${plan.crashedPath}`));
return result.status;
}
const { relPath, selector } = plan;
@@ -374,7 +374,7 @@ export async function spawnNpm(scope, script, extraArgs, label, {
if (extractCrashedTestFile(retry.output)) {
await diagnose(retryStartedAt);
console.error(`❌ ${relPath} crashed again on retry — this reproduces, not a runner fluke`);
- writeStepEnv('CI_CRASHED_TEST_FILE', relPath);
+ console.error(formatErrorAnnotation('Vitest worker crashed', `The worker crashed natively on ${relPath}, again on retry`));
} else {
// The retry failed for a different reason (an assertion, a real error) —
// report it as an ordinary test failure, not a repeat native crash.
diff --git a/scripts/run-ci-tests.test.js b/scripts/run-ci-tests.test.js
index 025c2ba94a..d93928f483 100644
--- a/scripts/run-ci-tests.test.js
+++ b/scripts/run-ci-tests.test.js
@@ -24,11 +24,11 @@ import { killProcessTree } from '../server/lib/bufferedSpawn.js';
import { OWNER_FILE, STALE_ROOT_AGE_MS } from './lib/vitestStaleRunRoots.js';
import { workflowJobs } from './lib/workflowJobs.js';
-// Synthetic crash/retry cases must not export a fabricated crash filename to
-// the Actions job's real GITHUB_ENV. Duration/summary tests retain real I/O.
+// Synthetic crash/retry cases must not print a fabricated crash as a real
+// Actions annotation. Duration/summary tests retain real I/O.
vi.mock('./lib/githubOutput.js', async (importOriginal) => ({
...await importOriginal(),
- writeStepEnv: vi.fn(),
+ formatErrorAnnotation: vi.fn((title, message) => `${title}: ${message}`),
}));
// Keep nested Vitest's own scratch under its separately owned pvt-* root,
@@ -101,11 +101,11 @@ describe('ci.yml shard wiring', () => {
const runners = Object.entries(workflowJobs(WORKFLOW)).filter(([, body]) => /(?:run|watch)-ci-tests\.js/.test(body));
it('builds every test runner matrix from the planner and hands each slice to the runner', () => {
- expect(runners.map(([id]) => id).sort()).toEqual(['client', 'database', 'server', 'windows-server']);
- // `database` runs run-ci-tests.js only in `files` mode, for the
- // cross-workspace browser suites the planner selected (#10312): an exact
- // file list has nothing to shard, so it is the one runner without a matrix.
- for (const [id, body] of runners.filter(([id]) => id !== 'database')) {
+ expect(runners.map(([id]) => id).sort()).toEqual(['browser', 'client', 'server', 'windows-server']);
+ // `browser` runs the planner's cross-workspace browser suites in `files`
+ // mode (#10312): an exact file list has nothing to shard, so it is the one
+ // runner without a matrix.
+ for (const [id, body] of runners.filter(([id]) => id !== 'browser')) {
// The fan-out is decided by the impact job, never hardcoded here: a
// scoped plan must collapse to one runner, and a job-level `if` cannot
// read `matrix` to skip the extra shards itself.
@@ -117,9 +117,9 @@ describe('ci.yml shard wiring', () => {
}
});
- it('runs the planner-selected browser suites on the database job with prerequisites required (#10312)', () => {
- const body = workflowJobs(WORKFLOW).database;
- expect(body).toContain("if: needs.impact.outputs.db == 'true' || needs.impact.outputs.browser_files != '[]'");
+ it('runs the planner-selected browser suites on their own job with prerequisites required (#10312)', () => {
+ const body = workflowJobs(WORKFLOW).browser;
+ expect(body).toContain("if: needs.impact.outputs.browser_files != '[]'");
const start = body.indexOf('- name: Run cross-workspace browser suites');
expect(start).toBeGreaterThan(0);
const step = body.slice(start, body.indexOf('- name:', start + 1));
@@ -131,16 +131,13 @@ describe('ci.yml shard wiring', () => {
expect(body.indexOf('npm ci --prefix client')).toBeLessThan(start);
});
- it('deduplicates only Linux browser ownership and admits PR fidelity separately', () => {
+ it('deduplicates only Linux browser ownership and runs PR browser suites as smoke', () => {
const jobs = workflowJobs(WORKFLOW);
expect(jobs.server).toContain('PORTOS_CI_BROWSER_OWNER: unit');
expect(jobs['windows-server']).not.toContain('PORTOS_CI_BROWSER_OWNER');
- expect(jobs.database).toContain("github.event_name == 'pull_request' && github.base_ref != 'release' && inputs.full != true && 'smoke' || 'full'");
- expect(jobs.gate.indexOf('node scripts/ci-gate-report.js')).toBeLessThan(
- jobs.gate.indexOf('node scripts/verify-browser-fidelity.js'));
- for (const name of ['CI_BROWSER_FILES', 'CI_EXPECTED_HEAD']) {
- expect(jobs.gate).toContain(`${name}:`);
- }
+ expect(jobs.browser).toContain("github.event_name == 'pull_request' && github.base_ref != 'release' && inputs.full != true && 'smoke' || 'full'");
+ expect(jobs.database).not.toContain('Run cross-workspace browser suites');
+ expect(jobs.gate).toContain('CI_GATE_RESULT_BROWSER: ${{ needs.browser.result }}');
expect(jobs['full-gate']).toContain("(github.event_name != 'pull_request' || github.base_ref == 'release' || inputs.full == true)");
expect(jobs.gate).toContain('CI_GATE_RESULT_LINUX_CONTAINMENT:');
expect(jobs.gate).toContain('CI_GATE_RESULT_WINDOWS_SERVER:');
@@ -168,11 +165,9 @@ describe('ci.yml shard wiring', () => {
expect(jobs.lint).toBeTruthy();
expect(jobs.lint).toContain('run: node scripts/run-ci-lint.js');
expect(jobs.lint).not.toContain('strategy:');
- // Exactly one `matrix.shard`: the shared fail-fast step's CI_FAILED_SHARD,
- // which renders empty on an unsharded job. A second occurrence — a cache
- // key, a step name, above all a shard-pinned `if:` — is the regression.
- expect(jobs.lint.match(/matrix\.shard/g)).toHaveLength(1);
- expect(jobs.lint).toContain('CI_FAILED_SHARD: ${{ matrix.shard }}');
+ // No `matrix.shard` at all: lint is not a matrix, so a cache key, a step
+ // name, above all a shard-pinned `if:` naming it is the regression.
+ expect(jobs.lint).not.toMatch(/matrix\.shard/);
// A scoped plan emits `client_shards: [1]`, so a later-shard pin would skip
// lint on every impact-scoped pull request. Its gate is the plan's lint mode.
expect(jobs.lint).toContain("if: needs.impact.outputs.lint_mode != 'skip'");
diff --git a/scripts/verify-browser-fidelity.js b/scripts/verify-browser-fidelity.js
deleted file mode 100644
index 19eb6c7aaa..0000000000
--- a/scripts/verify-browser-fidelity.js
+++ /dev/null
@@ -1,41 +0,0 @@
-#!/usr/bin/env node
-
-import { readFileSync } from 'node:fs';
-import { browserFidelityTarget, assertFidelityStatus, needsBrowserFidelity } from './lib/browserFidelity.js';
-import { githubJson, repoApiPath } from './lib/githubActionsApi.js';
-import { isDirectlyInvoked } from './lib/directInvocation.js';
-
-export async function verifyBrowserFidelity({ env = process.env, fetchImpl = fetch } = {}) {
- const files = JSON.parse(env.CI_BROWSER_FILES);
- if (!needsBrowserFidelity(files)) return { required: false };
- const repository = env.GITHUB_REPOSITORY;
- const repoPath = repoApiPath(env), token = env.GITHUB_TOKEN;
- let pr = Number(env.CI_PULL_REQUEST);
- if (!pr && env.GITHUB_EVENT_PATH) pr = Number(JSON.parse(readFileSync(env.GITHUB_EVENT_PATH, 'utf8')).number);
- if (!repoPath || !token) throw Error('Browser fidelity verification credentials unavailable.');
- const request = async path => {
- const prefix = `repos/${repository}`;
- if (!path.startsWith(`${prefix}/`)) throw Error('Invalid fidelity API target.');
- const body = await githubJson(fetchImpl, `${repoPath}${path.slice(prefix.length)}`, token);
- if (body === null) throw Error('Browser fidelity API evidence unavailable.');
- return body;
- };
- if (!/^[a-f0-9]{40}$/.test(env.CI_EXPECTED_HEAD ?? '')) {
- throw Error('CI browser fidelity checkout identity unavailable.');
- }
- const target = await browserFidelityTarget(repository, pr, request, {
- head: env.CI_EXPECTED_HEAD,
- });
- const statuses = await request(`repos/${repository}/commits/${target.head}/statuses?per_page=100`);
- return { required: true, ...assertFidelityStatus(target, statuses) };
-}
-
-if (isDirectlyInvoked(import.meta.url)) {
- try {
- const result = await verifyBrowserFidelity();
- console.log(result.required ? '✅ Current trusted browser fidelity receipt verified.' : '✅ No deferred browser fidelity selected.');
- } catch {
- console.error('❌ Browser fidelity admission failed: current trusted exact-head/tree/profile evidence is required.');
- process.exitCode = 1;
- }
-}
diff --git a/scripts/verify-browser-fidelity.test.js b/scripts/verify-browser-fidelity.test.js
deleted file mode 100644
index a80de9346f..0000000000
--- a/scripts/verify-browser-fidelity.test.js
+++ /dev/null
@@ -1,74 +0,0 @@
-import { describe, expect, it, vi } from 'vitest';
-import { verifyBrowserFidelity } from './verify-browser-fidelity.js';
-import { browserFidelityContext, browserFidelityProfileDigest } from './lib/browserFidelity.js';
-import { BROWSER_FIDELITY_FILES } from './lib/browserTestProfiles.js';
-
-const target = { repository: 'fixture/PortOS', pr: 12, head: 'a'.repeat(40), base: 'b'.repeat(40),
- merge: 'c'.repeat(40), tree: 'd'.repeat(40), profile: browserFidelityProfileDigest() };
-const status = (overrides = {}) => ({ id: 1, context: browserFidelityContext(target), state: 'success',
- description: `receipt sha256=${'e'.repeat(64)}`, creator: { login: 'fixture' }, ...overrides });
-
-function fixture({ statuses = [status()], expected = {}, mergeTree = target.tree, forkHead = false } = {}) {
- const env = { GITHUB_REPOSITORY: target.repository, GITHUB_TOKEN: 'synthetic token', CI_PULL_REQUEST: '12',
- CI_BROWSER_FILES: JSON.stringify(BROWSER_FIDELITY_FILES), CI_EXPECTED_HEAD: target.head,
- CI_EXPECTED_BASE: target.base, CI_EXPECTED_MERGE: target.merge, ...expected };
- const fetchImpl = vi.fn(async url => {
- let body;
- if (url.endsWith('/pulls/12')) body = { state: 'open', head: { sha: target.head, repo: { full_name: forkHead ? 'contributor/PortOS' : target.repository } },
- base: { sha: target.base, repo: { full_name: target.repository } }, merge_commit_sha: target.merge };
- else if (url.includes('/git/commits/')) body = { sha: target.head, tree: { sha: mergeTree },
- parents: [{ sha: target.base }, { sha: target.head }] };
- else if (url.includes('/statuses?')) body = statuses;
- else throw Error('Unexpected synthetic API URL');
- return new Response(JSON.stringify(body), { status: 200 });
- });
- return { env, fetchImpl };
-}
-
-describe('required browser fidelity admission', () => {
- it('accepts the personal repository owner and does not imply a remotely stored receipt', async () => {
- const f = fixture();
- await expect(verifyBrowserFidelity(f)).resolves.toEqual({ required: true,
- context: status().context, receiptDigest: 'e'.repeat(64) });
- expect(f.fetchImpl.mock.calls.at(-1)[0]).toContain(`/commits/${target.head}/statuses`);
- });
-
- it('accepts a fork-origin PR only through the base repository owner status', async () => {
- await expect(verifyBrowserFidelity(fixture({ forkHead: true }))).resolves.toMatchObject({ required: true });
- await expect(verifyBrowserFidelity(fixture({ forkHead: true,
- statuses: [status({ creator: { login: 'contributor' } })] }))).rejects.toThrow('receipt');
- });
-
- it('rejects a stale PR head', async () => {
- await expect(verifyBrowserFidelity(fixture({ expected: { CI_EXPECTED_HEAD: 'f'.repeat(40) } }))).rejects.toThrow('stale');
- });
-
- it('retains head-only fidelity when only the observed base and merge advance', async () => {
- const f = fixture({ expected: { CI_EXPECTED_BASE: 'f'.repeat(40), CI_EXPECTED_MERGE: 'f'.repeat(40) } });
- await expect(verifyBrowserFidelity(f)).resolves.toMatchObject({ required: true });
- expect(browserFidelityContext({ ...target, base: 'f'.repeat(40), merge: 'f'.repeat(40) })).toBe(status().context);
- });
-
- it.each([
- [],
- [status({ context: browserFidelityContext({ ...target, profile: '0'.repeat(64) }) })],
- [status({ creator: { login: 'untrusted' } })],
- [status(), status({ id: 2, state: 'pending' })],
- [status(), status({ id: 2, state: 'failure' })],
- ].map(statuses => ({ statuses })))('fails closed on missing, wrong-profile, untrusted or superseded evidence: %j', async ({ statuses }) => {
- await expect(verifyBrowserFidelity(fixture({ statuses }))).rejects.toThrow('receipt');
- });
-
- it('rejects a receipt for a different head tree', async () => {
- await expect(verifyBrowserFidelity(fixture({ mergeTree: 'f'.repeat(40) }))).rejects.toThrow('receipt');
- });
-
- it('requires selection input, but does not request receipts for smoke-only suites', async () => {
- const f = fixture();
- delete f.env.CI_BROWSER_FILES;
- await expect(verifyBrowserFidelity(f)).rejects.toThrow();
- f.env.CI_BROWSER_FILES = '["server/lib/filmLook.browser.test.js"]';
- await expect(verifyBrowserFidelity(f)).resolves.toEqual({ required: false });
- expect(f.fetchImpl).not.toHaveBeenCalled();
- });
-});
diff --git a/scripts/workspaceCaches.test.js b/scripts/workspaceCaches.test.js
index d74bc8af1f..4ff421fa3b 100644
--- a/scripts/workspaceCaches.test.js
+++ b/scripts/workspaceCaches.test.js
@@ -11,7 +11,7 @@ describe('checkout-owned server Vite caches with shared dependencies', () => {
const workflow = readFileSync(join(REPO, '.github/workflows/ci.yml'), 'utf8');
const caches = workflow.split('- name: Cache Vitest transform artifacts').slice(1)
.map(block => block.split('\n - name:')[0]);
- expect(caches).toHaveLength(4);
+ expect(caches).toHaveLength(5);
for (const cache of caches) {
expect(cache).not.toContain('node_modules/.vite');
if (cache.includes('key: vitest-server-')) {