Skip to content

The ultimate harness for coding agents and beyond. All your agents. All your machines. One command center. Start with code, then follow your curiosity and build across disciplines: CAD, circuits, robots, games and music.

ci: PRs run only their components' fast Linux checks; the queue reruns none (#991)

autonomousdevops47a2e77aee4f
diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
index f90995ddb..8fed92f8b 100644
--- a/.github/workflows/ci.yml
+++ b/.github/workflows/ci.yml
@@ -1,6 +1,8 @@
name: CI
-# Automatic PR and merge-candidate validation use the same coarse suite planner.
+# Fast PR checks: each changed component runs its own Linux unit tests. Native,
+# macOS, browser and end-to-end acceptance run in the release workflows. Merge
+# candidates rerun only the plan and process checks; the PR run covered the code.
# Contributor code runs with a read-only token and no release credentials.
on:
pull_request:
@@ -13,21 +15,14 @@ on:
scope:
description: 'Affected component; full for core suites, all for every component'
type: choice
- options: [full, cli, tui, backend, desktop, process, all, companions, website, experience, authoring, home-assistant, desktop-logging, os, provider, firmware, mobile, daemons]
+ options: [full, cli, tui, backend, desktop, process, all, companions, website, os, provider, firmware, mobile, daemons]
default: full
- # Callable as a reusable workflow. release.yml used to gate on this and take the Node pin from
- # its output; it no longer does (see the note above `publish` there) and pins the same version
- # itself, so the output below is kept only for any future caller.
workflow_call:
inputs:
scope:
description: 'full, cli, tui, backend, desktop (VM tests), or process (repository tooling only)'
type: string
default: full
- outputs:
- node-version:
- description: The Node the suite ran on. release.yml bundles the release with it.
- value: ${{ jobs.typecheck-test.outputs.node-version }}
permissions:
contents: read
@@ -138,7 +133,7 @@ jobs:
strategy:
fail-fast: false
matrix:
- os: [ubuntu-22.04, macos-15]
+ os: [ubuntu-22.04]
shard: [1, 2, 3, 4]
steps:
- uses: actions/checkout@v7
@@ -184,117 +179,6 @@ jobs:
overwrite: true
retention-days: 7
- desktop-test-summary:
- needs: [plan, desktop-tests]
- if: ${{ !cancelled() && needs.plan.outputs.ready == 'true' && needs.plan.outputs.desktop == 'true' }}
- runs-on: ubuntu-22.04
- timeout-minutes: 3
- steps:
- - uses: actions/checkout@v7
- with:
- ref: ${{ env.CI_SOURCE_SHA }}
- sparse-checkout: |
- /desktop/test/
- /scripts/
- /.gitignore
- sparse-checkout-cone-mode: false
- - name: Verify declared source checkout
- run: python3 scripts/ci-source-inputs.py desktop --subset
- - uses: actions/download-artifact@v4
- with:
- pattern: desktop-shard-*
- path: ${{ runner.temp }}/desktop-shards
- - name: Verify every VM file and its complete raw reporter on both platforms
- env:
- DESKTOP_TEST_RESULT: ${{ needs.desktop-tests.result }}
- run: >-
- python3 scripts/verify-desktop-test-shards.py "$RUNNER_TEMP/desktop-shards"
- --root "$GITHUB_WORKSPACE" --shards 4 --tests-result "$DESKTOP_TEST_RESULT"
- --output "$RUNNER_TEMP/desktop-test-summary.json"
- - uses: actions/upload-artifact@v4
- with:
- name: desktop-test-summary
- path: ${{ runner.temp }}/desktop-test-summary.json
- if-no-files-found: error
- overwrite: true
- retention-days: 7
-
- # tty.ReadStream uses libuv's native TTY handle. Exercise descriptor ownership,
- # binary backpressure and unplug behavior on both supported OSes and runtimes.
- # These are disposable PTYs; no USB device or existing daemon is touched.
- serial-native:
- concurrency:
- group: ci-${{ github.event_name }}-${{ github.event.pull_request.number || github.run_id }}-serial-native-${{ strategy.job-index }}
- cancel-in-progress: ${{ github.event_name == 'pull_request' }}
- needs: plan
- if: needs.plan.outputs.ready == 'true' && needs.plan.outputs.cli == 'true'
- strategy:
- fail-fast: false
- matrix:
- os: [ubuntu-latest, macos-latest]
- node: [20.19.0, 22.23.2]
- runs-on: ${{ matrix.os }}
- timeout-minutes: 10
- defaults:
- run:
- working-directory: cli
- steps:
- - uses: actions/checkout@v7
- with:
- ref: ${{ env.CI_SOURCE_SHA }}
- - uses: actions/setup-node@v7
- with:
- node-version: ${{ matrix.node }}
- cache: npm
- cache-dependency-path: cli/package-lock.json
- - run: npm ci --no-audit --no-fund
- - name: Require native PTY support instead of silently skipping it
- run: python3 -c 'import pty'
- - name: Serial lifecycle, binary transfer and descriptor cleanup
- run: npx vitest run src/cable/serial.spec.ts src/cable/serial.real.spec.ts --maxWorkers=1
- - name: Agent startup through the system zsh login shell
- run: npx vitest run src/lib/engineShellStartup.spec.ts
-
- # Build both slices and execute each host architecture. All process/tmux
- # fixtures are private; no installed agent or running session is controlled.
- process-images-native:
- concurrency:
- group: ci-${{ github.event_name }}-${{ github.event.pull_request.number || github.run_id }}-process-images-native-${{ strategy.job-index }}
- cancel-in-progress: ${{ github.event_name == 'pull_request' }}
- needs: plan
- if: needs.plan.outputs.ready == 'true' && needs.plan.outputs.cli == 'true'
- strategy:
- fail-fast: false
- matrix:
- os: [macos-15, macos-15-intel]
- runs-on: ${{ matrix.os }}
- timeout-minutes: 10
- defaults:
- run:
- working-directory: cli
- steps:
- - uses: actions/checkout@v7
- with:
- ref: ${{ env.CI_SOURCE_SHA }}
- - uses: actions/setup-node@v7
- with:
- node-version: 22.23.2
- cache: npm
- cache-dependency-path: cli/package-lock.json
- - run: npm ci --no-audit --no-fund
- - run: command -v tmux || brew install tmux
- - name: Read-only kernel probe, universal signing and byte-safe protocol
- run: python3 native/test_process_images.py
- - name: Build the embedded universal artifact
- run: python3 scripts/build-process-images.py --output "$RUNNER_TEMP/process-images.json"
- - name: Bundled native discovery, exec replacement and private tmux cleanup
- env:
- HARNESS_PROCESS_IMAGES_ARTIFACT: ${{ runner.temp }}/process-images.json
- RUN_DARWIN_PROCESS_IMAGES: '1'
- run: |
- node scripts/test-process-images-bundled.mjs
- npx vitest run src/lib/nativeProcessImages.spec.ts src/lib/processImages.spec.ts src/lib/processImages.real.spec.ts src/lib/periodicCpu.integration.spec.ts --maxWorkers=1
-
# The optional product has its own dependencies, bundle and suites. CLI-only CI
# deliberately checks core without installing or building this package.
companion-subsystems:
@@ -329,109 +213,14 @@ jobs:
- run: npx vitest run src/shared/guard.fuzz.spec.ts --maxWorkers=1
- run: npm test -- --exclude src/shared/guard.fuzz.spec.ts --maxWorkers=3
- # Typecheck and contract checks run alongside the default suite's four shards.
- cli-contracts:
+ cli-typecheck:
concurrency:
- group: ci-${{ github.event_name }}-${{ github.event.pull_request.number || github.run_id }}-cli-contracts
+ group: ci-${{ github.event_name }}-${{ github.event.pull_request.number || github.run_id }}-cli-typecheck
cancel-in-progress: ${{ github.event_name == 'pull_request' }}
needs: plan
if: needs.plan.outputs.ready == 'true' && needs.plan.outputs.cli == 'true'
runs-on: ubuntu-latest
- timeout-minutes: 20
- outputs:
- node-version: ${{ steps.pin.outputs.node-version }}
- defaults:
- run:
- working-directory: cli
- steps:
- - uses: actions/checkout@v7
- with:
- ref: ${{ env.CI_SOURCE_SHA }}
-
- # THE runtime pin — release.yml's `publish` job carries the same value; bump them together.
- #
- # It is not a taste in Node versions: it is the version of the MANAGED runtime the shipped CLI
- # actually executes on. `harness/runtime/metadata.json` in the release bucket serves
- # node v22.23.2 for darwin-arm64, darwin-x64, linux-x64 and linux-arm64, `install.sh` unpacks
- # it under ~/.harness/runtime, and the launcher names that binary absolutely — so the user's
- # own node, if any, never runs this bundle. Testing on anything else measures a runtime no
- # machine in the field has. Bump this together with `make upload-node-runtime` in
- # autonomous-harness-desktop, never on its own.
- #
- # (package.json's `engines` says >=20, which is the floor a THIRD PARTY may run it on. A floor
- # is not a pin: a floating 22.x turns an unrelated upstream release into a red build on an
- # untouched commit.)
- - id: pin
- run: echo "node-version=$CLI_NODE_VERSION" >> "$GITHUB_OUTPUT"
-
- - uses: actions/setup-node@v7
- with:
- node-version: ${{ steps.pin.outputs.node-version }}
- cache: npm
- # Relative to the repo root, not to `defaults.working-directory` — this is an action
- # input, not a `run:`.
- cache-dependency-path: cli/package-lock.json
-
- # TWO lockfiles are tracked here, and only one of them has ever been guarded.
- #
- # `package-lock.json` is the one the released bytes are built from — every build path in this
- # repo shells out to `npm run bundle` (install-cli.sh, upload-cli.sh, remote-machine.sh) — and
- # `npm ci` below refuses to run when it disagrees with package.json, so npm's half checks
- # itself.
- #
- # `pnpm-lock.yaml` is checked by nothing, and on 2026-09-09 that cost a maintainer an
- # afternoon: `bonjour-service` was added to package.json with only package-lock.json updated,
- # main went green, and `make install-cli` then died at `Could not resolve "bonjour-service"`
- # on every machine whose node_modules came from pnpm. Deleting the file would also fix that;
- # keeping it and checking it is the choice made here, because contributors do use it.
- #
- # `--lockfile-only` resolves without installing (~5s) and writes nothing under --frozen-lockfile.
- # Verified against this repo: it exits 1 on the stale lockfile that shipped in c4de318, naming
- # the added dependency, and 0 once it is regenerated.
- - name: Check pnpm-lock.yaml still matches package.json
- run: |
- npm i -g pnpm@11 >/dev/null
- pnpm install --frozen-lockfile --lockfile-only
-
- # `npm ci`, not `npm install`: the lockfile is the input, not a suggestion.
- - run: npm ci
-
- - run: npx tsc --noEmit
-
- - name: Native helper build-input and portable bundle contracts
- run: node --test scripts/test-process-image-bundle.mjs
-
- - name: Concurrent registry discovery, hooks and durable close intents
- run: node scripts/registry-observation-e2e.mjs "$RUNNER_TEMP/registry-observation-e2e"
-
- - name: Verify hn launcher repair in an existing installation
- run: |
- npm run bundle
- node scripts/test-hn-upgrade.mjs
-
- - name: Verify core harness rollout in the release bundle
- run: node scripts/test-core-harness-upgrade.mjs dist/cli.js
-
- - name: Require complete harness updater coverage
- run: npm run test:dsh-updates
-
- # The daemon's supervisor and its core modules are held to 100% in every file
- # (docs/design/2026-10-03-harnessd.md#testing).
- - name: Require complete harnessd and core coverage
- run: npm run test:harnessd && npm run test:core
-
- # The other per-file 100% coverage gates in cli/package.json, beside cli-contracts' harnessd and core
- # ones, on a runner of their own so the CI run takes no longer. They used to run only when somebody
- # remembered: test:resume had slipped to 93% of stoppedAgents.ts and test:orchestrator below 100% by
- # 5 October 2026 without anyone noticing. One step per gate, so a red run names the gate.
- cli-coverage-gates:
- concurrency:
- group: ci-${{ github.event_name }}-${{ github.event.pull_request.number || github.run_id }}-cli-coverage-gates
- cancel-in-progress: ${{ github.event_name == 'pull_request' }}
- needs: plan
- if: needs.plan.outputs.ready == 'true' && needs.plan.outputs.cli == 'true'
- runs-on: ubuntu-latest
- timeout-minutes: 15
+ timeout-minutes: 10
defaults:
run:
working-directory: cli
@@ -445,26 +234,7 @@ jobs:
cache: npm
cache-dependency-path: cli/package-lock.json
- run: npm ci --no-audit --no-fund
- - name: Stopped harnesses, stop and resume
- run: npm run test:resume
- - name: Orchestrator
- run: npm run test:orchestrator
- - name: Sharing
- run: npm run test:sharing
- - name: Remote viewers
- run: npm run test:remote-viewers
- - name: Harness portability
- run: npm run test:portability
- - name: Local models and bundled harnesses
- run: npm run test:local-models -- --coverage.reporter=json --coverage.reportOnFailure
- - name: Preserve local-model coverage details
- if: always()
- uses: actions/upload-artifact@v4
- with:
- name: cli-local-models-coverage
- path: cli/coverage/local-models/*.json
- if-no-files-found: ignore
- retention-days: 7
+ - run: npx tsc --noEmit
cli-tests:
concurrency:
@@ -510,61 +280,15 @@ jobs:
overwrite: true
retention-days: 7
- # Preserve the original aggregate check/output for callers. A missing, failed
- # or skipped shard/contract job cannot turn into a green aggregate check.
- typecheck-test:
- needs: [plan, cli-contracts, cli-tests]
- if: ${{ !cancelled() && needs.plan.outputs.ready == 'true' && needs.plan.outputs.cli == 'true' }}
- runs-on: ubuntu-latest
- timeout-minutes: 3
- outputs:
- node-version: ${{ needs.cli-contracts.outputs.node-version }}
- steps:
- - uses: actions/checkout@v7
- with:
- ref: ${{ env.CI_SOURCE_SHA }}
- # This job reads only the shard reports and this stdlib-only verifier.
- # Avoid fetching/materializing the app, fixtures and performance assets again.
- sparse-checkout: scripts/verify-cli-test-shards.py
- sparse-checkout-cone-mode: false
- - uses: actions/download-artifact@v4
- with:
- pattern: cli-shard-*
- path: ${{ runner.temp }}/cli-shards
- merge-multiple: true
- - name: Require every discovered file exactly once and all checks passing
- env:
- CLI_TEST_RESULT: ${{ needs.cli-tests.result }}
- CLI_CONTRACT_RESULT: ${{ needs.cli-contracts.result }}
- run: >-
- python3 scripts/verify-cli-test-shards.py "$RUNNER_TEMP/cli-shards"
- --root "$GITHUB_WORKSPACE/cli" --shards 4
- --tests-result "$CLI_TEST_RESULT" --contracts-result "$CLI_CONTRACT_RESULT"
- --output "$RUNNER_TEMP/cli-test-summary.json"
- - uses: actions/upload-artifact@v4
- with:
- name: cli-test-summary
- path: ${{ runner.temp }}/cli-test-summary.json
- if-no-files-found: error
- overwrite: true
- retention-days: 7
-
- # Native shell lifecycle checks on the Linux platforms used over SSH. This workflow
- # remains on demand; it builds and tests without publishing or installing a release.
+ # Unit tests only. Native fixtures and connected-shell checks run in the release workflow.
tui-test:
concurrency:
- group: ci-${{ github.event_name }}-${{ github.event.pull_request.number || github.run_id }}-tui-test-${{ strategy.job-index }}
+ group: ci-${{ github.event_name }}-${{ github.event.pull_request.number || github.run_id }}-tui-test
cancel-in-progress: ${{ github.event_name == 'pull_request' }}
needs: plan
if: needs.plan.outputs.ready == 'true' && needs.plan.outputs.tui == 'true'
- strategy:
- fail-fast: false
- matrix:
- include:
- - { runner: ubuntu-latest, target: x86_64-unknown-linux-musl }
- - { runner: ubuntu-24.04-arm, target: aarch64-unknown-linux-musl }
- runs-on: ${{ matrix.runner }}
- timeout-minutes: 25
+ runs-on: ubuntu-latest
+ timeout-minutes: 15
defaults:
run:
working-directory: tui
@@ -573,88 +297,17 @@ jobs:
with:
ref: ${{ env.CI_SOURCE_SHA }}
- uses: dtolnay/rust-toolchain@stable
- with:
- targets: ${{ matrix.target }}
- uses: Swatinem/rust-cache@6323deb102c322ba6fcbdcafc7e3dddab59af2b6 # v2.9.2
with:
workspaces: tui
- shared-key: tui-${{ matrix.target }}
+ shared-key: tui-pr
cache-bin: false
- - uses: actions/setup-node@v7
- with:
- node-version: 22.23.2
- cache: npm
- cache-dependency-path: cli/package-lock.json
- - name: Install isolated test dependencies
- working-directory: .
- run: |
- sudo apt-get update
- sudo apt-get install -y tmux musl-tools zsh
- npm ci --prefix cli --no-audit --no-fund
- - name: Real tmux question polling and answer routing
- working-directory: cli
- run: node --import tsx scripts/question-poll-native.ts "$RUNNER_TEMP/question-poll.json"
- - name: Native fixture process cleanup and ownership isolation
- working-directory: cli
- run: node --import tsx scripts/check-native-fixture-git.mjs
- - name: Real tmux byte streaming, paste and resize
- working-directory: cli
- env:
- RUN_REAL_TMUX_STREAM: '1'
- run: npx vitest run src/lib/tmuxStream.real.spec.ts
- - name: Codex startup timeout recovery and cancellation
- working-directory: cli
- env:
- RUN_REAL_TMUX_CODEX_STARTUP: '1'
- run: npx vitest run src/lib/codexStartupRetry.real.spec.ts --maxWorkers=1
- - name: Release unit tests
- # Use the shipped target for both commands so Cargo can reuse the
- # dependencies, instead of compiling once for glibc and again for musl.
+ - run: sudo apt-get update && sudo apt-get install -y tmux zsh
+ - name: Unit tests
env:
TERM: xterm-256color
COLORTERM: truecolor
- run: env -u NO_COLOR cargo test --locked --release --target ${{ matrix.target }}
- - name: Build static Linux binary
- run: cargo build --locked --release --target ${{ matrix.target }}
- - name: Build native comparison reference
- id: reference-tmux
- uses: ./.github/actions/reference-tmux
- - name: Complete native fixtures with bounded parallelism
- working-directory: .
- env:
- REFERENCE_TMUX_BIN: ${{ steps.reference-tmux.outputs.bin }}
- run: PATH="$REFERENCE_TMUX_BIN:$PATH" python3 scripts/validate-tui-native.py tui/target/${{ matrix.target }}/release/harness-tui
- - name: Connected hn local and remote shells through the real service
- working-directory: cli
- env:
- HN_TUI_BINARY: ${{ github.workspace }}/tui/target/${{ matrix.target }}/release/harness-tui
- E2E_BUNDLE: '1'
- E2E_ARTIFACTS_DIR: ${{ github.workspace }}/.harness/validation/connected-shells
- run: npm run test:e2e -- e2e/tuiShell.e2e.ts e2e/shell.e2e.ts
- - name: Real daemon composer and recursive folder search
- working-directory: .
- env:
- HN_SHELL_TEST_BINARY: ${{ github.workspace }}/tui/target/${{ matrix.target }}/release/harness-tui
- HN_SHELL_TEST_CLI: ${{ github.workspace }}/cli/dist/cli.js
- HN_SHELL_TEST_FZF: '0'
- run: |
- npm run build --prefix cli
- HN_SHELL_TEST_COMPOSER=1 HN_SHELL_TEST_COMPOSER_ONLY=1 python3 -u tui/tests/shell-first.py
- HN_SHELL_TEST_FOLDERS=1 HN_SHELL_TEST_LARGE_FOLDERS=1 HN_SHELL_TEST_FOLDERS_ONLY=1 python3 -u tui/tests/shell-first.py
- python3 -u tui/tests/shell-first.py
- - name: Show native fixture failures
- if: failure()
- working-directory: .
- run: |
- find .harness/validation -name '*.log' -type f -exec sh -c 'echo "::group::$1"; cat "$1"; echo "::endgroup::"' sh {} \;
- - uses: actions/upload-artifact@v4
- if: always()
- with:
- name: tui-validation-${{ matrix.target }}
- path: .harness/validation/
- include-hidden-files: true
- if-no-files-found: ignore
- retention-days: 14
+ run: env -u NO_COLOR cargo test --locked
backend-desk:
concurrency:
@@ -688,38 +341,6 @@ jobs:
if: needs.plan.outputs.ready == 'true' && needs.plan.outputs.website == 'true'
uses: ./.github/workflows/website-checks.yml
- experience-checks:
- concurrency:
- group: ci-${{ github.event_name }}-${{ github.event.pull_request.number || github.run_id }}-experience-checks
- cancel-in-progress: ${{ github.event_name == 'pull_request' }}
- needs: plan
- if: needs.plan.outputs.ready == 'true' && needs.plan.outputs.experience == 'true'
- uses: ./.github/workflows/experience-checks.yml
-
- authoring-checks:
- concurrency:
- group: ci-${{ github.event_name }}-${{ github.event.pull_request.number || github.run_id }}-authoring-checks
- cancel-in-progress: ${{ github.event_name == 'pull_request' }}
- needs: plan
- if: needs.plan.outputs.ready == 'true' && needs.plan.outputs.authoring == 'true'
- uses: ./.github/workflows/authoring-browser-checks.yml
-
- home-assistant-checks:
- concurrency:
- group: ci-${{ github.event_name }}-${{ github.event.pull_request.number || github.run_id }}-home-assistant-checks
- cancel-in-progress: ${{ github.event_name == 'pull_request' }}
- needs: plan
- if: needs.plan.outputs.ready == 'true' && needs.plan.outputs.home-assistant == 'true'
- uses: ./.github/workflows/home-assistant-checks.yml
-
- desktop-logging-checks:
- concurrency:
- group: ci-${{ github.event_name }}-${{ github.event.pull_request.number || github.run_id }}-desktop-logging-checks
- cancel-in-progress: ${{ github.event_name == 'pull_request' }}
- needs: plan
- if: needs.plan.outputs.ready == 'true' && needs.plan.outputs.desktop-logging == 'true'
- uses: ./.github/workflows/desktop-logging-checks.yml
-
os-checks:
concurrency:
group: ci-${{ github.event_name }}-${{ github.event.pull_request.number || github.run_id }}-os-checks
@@ -858,10 +479,8 @@ jobs:
required:
name: ci/required
- needs: [plan, process-checks, desktop-tests, desktop-test-summary, serial-native, process-images-native,
- companion-subsystems, cli-contracts, cli-coverage-gates, cli-tests, typecheck-test, tui-test,
- backend-desk, website-checks, experience-checks, authoring-checks, home-assistant-checks,
- desktop-logging-checks, os-checks, provider-checks, firmware-checks, mobile-checks, daemons-checks]
+ needs: [plan, process-checks, desktop-tests, companion-subsystems, cli-typecheck, cli-tests, tui-test,
+ backend-desk, website-checks, os-checks, provider-checks, firmware-checks, mobile-checks, daemons-checks]
if: always()
runs-on: ubuntu-latest
timeout-minutes: 3
diff --git a/.github/workflows/cli-e2e.yml b/.github/workflows/cli-e2e.yml
index 5199c57cb..b9895ac36 100644
--- a/.github/workflows/cli-e2e.yml
+++ b/.github/workflows/cli-e2e.yml
@@ -10,16 +10,10 @@ name: CLI end to end
# merge-pr`) refuses to merge anything GitHub does not report clean: a per-PR run that failed would
# block every CLI merge. The suite had only ever run on macOS, and its first Linux runs found real
# failures (zsh's new-user menu took every agent's pane, engineLaunch.ts zshNewUserGuard) beside
-# runner-specific ones. So it runs after a merge to main, every night, and on demand from a branch
-# (Actions -> CLI end to end -> Run workflow) when a change needs it. Promote it to pull requests,
-# and then to a required check collected by record-ci-validation.py, once it has stayed green.
+# runner-specific ones. So it runs every night and on demand from a branch (Actions -> CLI end to
+# end -> Run workflow) when a change needs it. It no longer runs on every merge to main: each run
+# held nine of the account's twenty concurrent runners, and the next merge usually cancelled it.
on:
- push:
- branches: [main]
- paths:
- - 'cli/**'
- - '.github/workflows/cli-e2e.yml'
- - 'scripts/verify-cli-test-shards.py'
schedule:
- cron: '17 6 * * *'
workflow_dispatch:
diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md
index 33645c624..80175a75f 100644
--- a/CONTRIBUTING.md
+++ b/CONTRIBUTING.md
@@ -198,11 +198,10 @@ while developing. Shared state, authentication, protocols, dependencies, and cha
with an unclear impact also need the full CLI suite. A passing CI run on the tested
source satisfies that full-suite check; do not run it again locally before release.
Manual CI offers `cli`, `tui`, `backend`, `desktop`, and `full` scopes; `full` remains the default.
-CLI scope includes the supported OS/Node shell and serial matrix. Cross-component
-changes need all affected scopes or `full`.
-Desktop scope runs the complete VM suite in four shards on both macOS and Linux,
-with a verified coverage summary. Changed Dart analysis, browser tests and native
-integration checks remain separate requirements when relevant.
+CLI scope runs the typecheck and the default Vitest suite in four Linux shards.
+Desktop scope runs the VM suite in four Linux shards. PR CI runs only the suites of
+the components a PR changes; native, macOS, coverage-gate and end-to-end checks run in
+the release workflows.
Two further suites exist and are
**opt-in**, because they need software the machine may not have — they skip themselves rather than
diff --git a/docs/validation-and-release.md b/docs/validation-and-release.md
index 1e888ce6d..13049db5f 100644
--- a/docs/validation-and-release.md
+++ b/docs/validation-and-release.md
@@ -25,38 +25,34 @@ compiling, or starting a broad suite. Do not clean other sessions' files to make
Run independent checks concurrently, with enough capacity for their workers;
two commands each spawning every CPU is not useful parallelism.
-Manual **CI → Run workflow** accepts `scope`: `cli`, `tui`, `backend`, `desktop`, or `full`
-(the default). `cli` includes typecheck, all CLI tests, updater coverage, release
-bundle checks, the serial/login-shell OS/Node matrix, and the per-file 100% coverage
-gates (`test:core`, `test:harnessd`, `test:resume`, `test:orchestrator`, `test:sharing`,
-`test:remote-viewers`, `test:portability`). `tui` includes its native
-CLI integration tests. Select `full` for cross-component changes or uncertain impact.
-Automatic PR and merge-group runs select all affected component scopes; manual scopes remain available.
-
-`desktop` runs all Desktop VM files on both macOS and Linux, split into four
-isolated runners per platform. It uses the release's pinned Flutter SDK and shared
-dependency caches. Each shard has four test workers and the bounded runner's
-ten-minute budget; startup recovery retains the same narrow rules below. `full`
-includes this matrix alongside the existing component checks. Changed Dart
-analysis and browser/native integration checks remain separate where relevant.
-
-The Desktop aggregate reads each raw test log, verifies its hash and complete-file
-result, and compares every shard's inventory with the checked-out test files.
-Every file must finish exactly once per platform. Failed jobs, missing or duplicate
-files, changed source/environment, malformed reports and incomplete tests fail it.
-Platform-specific skips and any pre-test loader recovery remain explicit in the
-summary. Logs, receipts and the summary are retained for seven days. Collect the
-result with `scripts/record-ci-validation.py RUN_ID --scope desktop --pr PR_NUMBER`.
+PR CI is deliberately fast. Each changed component runs only its own Linux suite:
+
+| Changed directory | PR check |
+| --- | --- |
+| `cli/`, `tests/` | `cli-typecheck` and `cli-tests` (default Vitest suite, four shards) |
+| `desktop/` | `desktop-tests` (VM suite, four Linux shards) |
+| `tui/` | `tui-test` (`cargo test`) |
+| `backend/`, `companions/`, `website/`, `os/`, `provider/`, `mobile/`, `daemons/` | that component's check |
+| `devices/` | `firmware-checks` |
+| anything else (docs, workflows, scripts, store) | `process-checks` only |
+
+`plan` (actionlint and suite selection), `process-checks` and `ci/required` always
+run. Merge-queue runs rerun only those three jobs: the PR run already tested the
+change. macOS, native process and serial, per-file coverage gates, browser
+acceptance and CLI end-to-end checks run in the release workflows, on the nightly
+CLI end-to-end run, or on demand from their own workflows. A release failure is a
+code failure.
+
+Manual **CI → Run workflow** accepts `scope`: `cli`, `tui`, `backend`, `desktop`,
+`full` (the default, all five core components), `all`, or one extra component.
+
+`desktop-tests` uses the release's pinned Flutter SDK and shared dependency caches.
+Each shard has four test workers and the bounded runner's ten-minute budget. Logs
+and receipts are retained for seven days.
CLI's default Vitest suite runs as four file shards on separate runners, retaining
-its worker cap and isolation. Typecheck, lockfile checks, guard fuzz, registry and
-release-bundle integration, and updater coverage run alongside them. Guard fuzz
-keeps its own process and timing budget. The existing `typecheck-test` job is the
-aggregate: it requires passing shard/contract jobs and verifies that their JSON
-reports cover every discovered file exactly once. Missing, duplicated, failed or
-unfinished results fail it. Review the complete workflow result, including the
-serial/login-shell matrix, rather than one early finishing job. Shard reports,
-inventories and the combined summary are retained as artifacts for seven days.
+its worker cap and isolation. Shard reports and inventories are retained as
+artifacts for seven days.
CI's `vitest.ci.config.ts` uses the slow-file timing hints in `cli/ci-test-durations.json`
to distribute estimated work across the same four runners. The complete discovered
@@ -70,8 +66,8 @@ the CI assignment locally. The aggregate still requires every discovered file
exactly once, independent of these estimates.
The CLI's end-to-end suite (`cli/e2e`) runs in its own workflow, **CLI end to end**
-(`.github/workflows/cli-e2e.yml`): after each merge to `main` that touches `cli/`, nightly,
-and on demand from a branch (Actions → CLI end to end → Run workflow).
+(`.github/workflows/cli-e2e.yml`): nightly and on demand from a branch
+(Actions → CLI end to end → Run workflow).
Eight Linux runners each take a shard, planned from `cli/ci-e2e-durations.json` the same
way (`--config vitest.e2e.ci.config.ts --shard=N/8` reproduces one locally). Each shard
installs tmux, zsh, tcsh and dash, starts every daemon from one bundle (`E2E_BUNDLE=1`) and
diff --git a/scripts/ci-plan.py b/scripts/ci-plan.py
index f59c56851..69f33f123 100644
--- a/scripts/ci-plan.py
+++ b/scripts/ci-plan.py
@@ -1,12 +1,11 @@
#!/usr/bin/env python3
-"""Select coarse CI suites from a complete Git diff and verify their final results.
+"""Select fast CI suites from a complete Git diff and verify their final results.
-PRs test their exact head; merge groups test their exact combined candidate.
-Manual scopes retain their existing meaning. Native/engine/hardware acceptance
-selected during review remains additional to these automatic checks.
+A PR runs the unit suite of each component it changes, and nothing else. Merge
+groups run only the plan and process checks: the PR run already covered the code,
+and the release workflows carry native, macOS, browser and end-to-end acceptance.
"""
import argparse
-import fnmatch
import json
import os
from pathlib import Path
@@ -15,26 +14,20 @@ import subprocess
CORE = {"cli", "desktop", "tui", "backend", "companions"}
-EXTRA = {"website", "experience", "authoring", "home-assistant", "desktop-logging",
- "os", "provider", "firmware", "mobile", "daemons"}
+EXTRA = {"website", "os", "provider", "firmware", "mobile", "daemons"}
SUITES = CORE | EXTRA
JOBS = {
- "cli": {"serial-native", "process-images-native", "cli-contracts", "cli-coverage-gates", "cli-tests", "typecheck-test"},
- "desktop": {"desktop-tests", "desktop-test-summary"},
+ "cli": {"cli-typecheck", "cli-tests"},
+ "desktop": {"desktop-tests"},
"tui": {"tui-test"}, "backend": {"backend-desk"}, "companions": {"companion-subsystems"},
**{name: {name + "-checks"} for name in EXTRA},
}
MANUAL = {"full": CORE, "all": SUITES, "process": set(), **{name: {name} for name in SUITES}}
-AUTHORING = {"generative-art", "music-studio", "creative-direction", "voxel-worlds",
- "drone-pilot", "game-master", "lab-bench", "data-studio"}
-LOGGING = ["desktop/lib/logging/*", "desktop/test/app_log_test.dart", "desktop/test/buffered_log_test.dart",
- "desktop/test/crash_log_test.dart", "desktop/test/dial_log_tail_test.dart", "desktop/test/log_*_test.dart",
- "desktop/test/export_logs_dialog_test.dart", "desktop/test/cli_transcript_test.dart", "desktop/tool/log_append_probe.dart"]
-SPECIAL_WORKFLOWS = {"website-checks.yml": "website", "experience-checks.yml": "experience",
- "authoring-browser-checks.yml": "authoring", "home-assistant-checks.yml": "home-assistant",
- "desktop-logging-checks.yml": "desktop-logging"}
-ROOT_DOCS = {"AGENTS.md", "CHANGELOG.md", "CLAUDE.md", "CONTEXT.md", "CONTRIBUTING.md", "HANDOFF.md",
- "LICENSE", "README.md", "SECURITY.md", "claude_research.md"}
+# Each component's own directory selects its suite. Everything else (docs, workflows,
+# scripts, store, fixtures) is covered by the plan's actionlint and process checks.
+COMPONENTS = {"cli": "cli", "tests": "cli", "desktop": "desktop", "tui": "tui", "backend": "backend",
+ "companions": "companions", "website": "website", "os": "os", "provider": "provider",
+ "devices": "firmware", "mobile": "mobile", "daemons": "daemons"}
def git(root, *args):
@@ -48,61 +41,14 @@ def sha(value):
def select(paths):
- selected, reasons, unknown = set(), {}, []
+ selected, reasons = set(), {}
for path in paths:
if not isinstance(path, str) or not path or path.startswith("/") or ".." in path.split("/"):
raise ValueError("invalid changed path")
- scopes = set()
- root = path.split("/", 1)[0]
- if path in ROOT_DOCS or (root == "docs" and not path.startswith("docs/images/")):
- pass
- elif root in {".github", "scripts"} or path in {"Makefile", ".gitattributes", ".gitignore", ".gitmodules"}:
- # Keep existing source-input contracts conservative. Workflow/helper
- # changes invalidate all core suites until narrower contracts exist.
- scopes |= CORE
- if path in {".github/workflows/ci.yml", "scripts/ci-plan.py"}:
- scopes |= SUITES # The shared planner/job graph controls every component.
- workflow = path.removeprefix(".github/workflows/")
- if workflow in SPECIAL_WORKFLOWS:
- scopes.add(SPECIAL_WORKFLOWS[workflow])
- if workflow.startswith("os-") or workflow == "os.yml":
- scopes.add("os")
- elif root == "cli":
- scopes |= CORE # Desktop/TUI/companions consume CLI inputs.
- scopes.add("mobile") # Phone protocol contracts read the CLI's frame definitions.
- elif root == "desktop":
- scopes.add("desktop")
- if path.startswith("desktop/assets/engine-icons/"):
- scopes.add("experience")
- if any(fnmatch.fnmatchcase(path, pattern) for pattern in LOGGING):
- scopes.add("desktop-logging")
- elif root in {"tui", "backend", "companions", "website", "os", "provider"}:
- scopes.add(root)
- elif root == "devices":
- scopes |= {"firmware", "cli", "desktop"}
- elif root == "mobile":
- scopes.add("mobile")
- if path == "mobile/pubspec.lock":
- scopes.add("desktop")
- elif root in {"tests", "daemons"}:
- scopes |= CORE | {"mobile", "daemons"}
- elif root == "store":
- scopes |= {"desktop", "experience"}
- if (path.startswith(("store/tools/", "store/viewers/web-viewer/"))
- or any(path.startswith(f"store/agents/{name}/") for name in AUTHORING)):
- scopes.add("authoring")
- if path.startswith("store/agents/home-assistant/") or path in {"store/tools/runtimes.sh", "store/tools/sync-runtimes.mjs"}:
- scopes.add("home-assistant")
- elif path.startswith("docs/images/"):
- scopes.add("desktop")
- elif root in {"artifacts", "output", "work"}:
- scopes |= CORE
- else:
- unknown.append(path)
+ suite = COMPONENTS.get(path.split("/", 1)[0])
+ scopes = {suite} if suite else set()
selected |= scopes
reasons[path] = sorted(scopes)
- if unknown:
- raise ValueError("register CI coverage for these paths before merging: " + ", ".join(unknown))
return selected, reasons
@@ -135,6 +81,8 @@ def make_plan(root, event_name, event, scope="full", source=None):
changed = subprocess.check_output(["git", "diff", "--no-renames", "--name-only", "-z", base, head], cwd=root, timeout=120)
paths = sorted(p.decode() for p in changed.split(b"\0") if p)
suites, reasons = select(paths)
+ if event_name == "merge_group":
+ suites = set() # The PR run tested these changes; releases test the rest.
else:
if scope not in MANUAL:
raise ValueError("invalid manual CI scope")
diff --git a/scripts/tests/test_ci_plan.py b/scripts/tests/test_ci_plan.py
index 9e9a8141d..0f1dfa275 100644
--- a/scripts/tests/test_ci_plan.py
+++ b/scripts/tests/test_ci_plan.py
@@ -12,36 +12,26 @@ spec.loader.exec_module(planner)
class SelectionTests(unittest.TestCase):
- def test_docs_still_require_process_checks_and_assets_are_inputs(self):
- self.assertEqual(planner.select(["docs/guide.md", "README.md"])[0], set())
- self.assertEqual(planner.select(["docs/images/poster.png"])[0], {"desktop"})
-
- def test_cli_and_process_changes_keep_conservative_core_coverage(self):
- for path in ["scripts/merge-validated-pr.py", "Makefile"]:
+ def test_docs_workflows_scripts_and_unknown_paths_need_only_process_checks(self):
+ for path in ["docs/guide.md", "README.md", "docs/images/poster.png", "scripts/ci-plan.py",
+ ".github/workflows/ci.yml", "Makefile", "store/tools/runtimes.sh", "surprise.py"]:
with self.subTest(path=path):
- self.assertEqual(planner.select([path])[0], planner.CORE)
-
- def test_cli_changes_include_phone_protocol_contracts(self):
- self.assertEqual(planner.select(["cli/src/lib/relayFrames.ts"])[0], planner.CORE | {"mobile"})
-
- def test_shared_planner_or_job_graph_changes_validate_every_component(self):
- for path in ["scripts/ci-plan.py", ".github/workflows/ci.yml"]:
- self.assertEqual(planner.select([path])[0], planner.SUITES)
+ self.assertEqual(planner.select([path])[0], set())
- def test_known_component_and_browser_inputs(self):
+ def test_each_component_selects_only_its_own_suite(self):
for path, expected in {
+ "cli/src/lib/relayFrames.ts": {"cli"}, "tests/fixture.ts": {"cli"},
+ "desktop/test/log_redact_test.dart": {"desktop"}, "tui/src/main.rs": {"tui"},
+ "backend/src/app.ts": {"backend"}, "companions/src/a.ts": {"companions"},
"website/app/page.tsx": {"website"}, "os/root/usr/lib/session": {"os"},
- "mobile/pubspec.lock": {"mobile", "desktop"}, "tui/src/main.rs": {"tui"},
- "provider/spec.md": {"provider"}, "devices/harness-device/firmware/main/main.c": {"firmware", "cli", "desktop"},
- "store/agents/home-assistant/src/a.py": {"desktop", "experience", "home-assistant"},
- "store/tools/runtimes.sh": {"desktop", "experience", "home-assistant", "authoring"},
- "desktop/test/log_redact_test.dart": {"desktop", "desktop-logging"},
+ "provider/spec.md": {"provider"}, "devices/harness-device/firmware/main/main.c": {"firmware"},
+ "mobile/pubspec.lock": {"mobile"}, "daemons/tools/card.mjs": {"daemons"},
}.items():
with self.subTest(path=path):
self.assertEqual(planner.select([path])[0], expected)
- def test_unknown_paths_fail_instead_of_getting_an_empty_green_plan(self):
- for path in ["new-component/code.ts", "surprise.py", "../cli/code.ts", "/absolute"]:
+ def test_invalid_paths_fail(self):
+ for path in ["../cli/code.ts", "/absolute", ""]:
with self.subTest(path=path), self.assertRaises(ValueError):
planner.select([path])
@@ -86,7 +76,7 @@ class GitPlanTests(unittest.TestCase):
self.assertFalse(ready["draft"])
self.assertEqual(draft["suites"], ready["suites"])
self.assertEqual(draft["required_jobs"], ready["required_jobs"])
- self.assertIn("mobile-checks", draft["required_jobs"])
+ self.assertIn("cli-tests", draft["required_jobs"])
with self.assertRaisesRegex(ValueError, "invalid PR draft state"):
self.plan(draft="false")
@@ -119,13 +109,15 @@ class GitPlanTests(unittest.TestCase):
self.commit()
self.assertEqual(self.plan(base=main)["suites"], ["website"])
- def test_group_includes_every_change_ahead_of_this_pr(self):
+ def test_group_records_every_change_but_reruns_only_process_checks(self):
self.write("website/new.txt")
self.commit()
self.write("os/new.txt")
self.commit()
plan = self.plan("merge_group")
- self.assertEqual(set(plan["suites"]), {"os", "website"})
+ self.assertEqual(plan["paths"], ["os/new.txt", "website/new.txt"])
+ self.assertEqual(plan["suites"], [])
+ self.assertEqual(plan["required_jobs"], ["process-checks"])
self.assertEqual(plan["base"], self.base)
def test_wrong_head_event_or_nonancestor_group_fails(self):