Skip to content

Claude Code. Any Model. The most powerful AI coding agent now speaks every language.

ci: run the hermetic suite as balanced slices at the same time

Jack Rudenko8dcea3444915
diff --git a/.github/workflows/test-experiment.yml b/.github/workflows/test-experiment.yml
deleted file mode 100644
index a6dd578f..00000000
--- a/.github/workflows/test-experiment.yml
+++ /dev/null
@@ -1,73 +0,0 @@
-name: Tests experiment
-
-# MEASUREMENT ONLY, deleted before merge. Answers two questions for the hermetic-job split:
-# 1. Which test files pass on Linux runners (20 concurrent jobs on the free plan, against 5 macOS)?
-# 2. How long each group takes when the three groups run concurrently on ONE macOS runner.
-# Both write junit, so per-file and per-test times come from the reports, not log timestamps.
-
-on:
- pull_request:
- branches: [main]
-
-jobs:
- linux-probe:
- name: linux probe · ${{ matrix.group }}
- if: startsWith(github.head_ref, 'ci/hermetic-under-4min')
- strategy:
- fail-fast: false
- matrix:
- group: [pane, idle, rest]
- runs-on: ubuntu-latest
- env:
- FORCE_JAVASCRIPT_ACTIONS_TO_NODE24: true
- steps:
- - uses: actions/checkout@v5
- - uses: oven-sh/setup-bun@v2
- with:
- bun-version: "1.3.10"
- - run: bun install --frozen-lockfile
- - uses: ./.github/actions/install-magmux
- with:
- token: ${{ github.token }}
- - name: Tests (${{ matrix.group }})
- working-directory: packages/cli
- continue-on-error: true
- run: bun scripts/ci-tests.ts --group ${{ matrix.group }} -- --reporter=junit --reporter-outfile=junit-{group}.xml
- timeout-minutes: 40
- env:
- CLAUDISH_SKIP_LIVE_E2E: "1"
- - uses: actions/upload-artifact@v5
- if: always()
- with:
- name: junit-linux-${{ matrix.group }}
- path: packages/cli/junit-*.xml
-
- mac-lanes:
- name: macOS lanes · pane + idle + rest concurrently
- if: startsWith(github.head_ref, 'ci/hermetic-under-4min')
- runs-on: macos-latest
- env:
- FORCE_JAVASCRIPT_ACTIONS_TO_NODE24: true
- steps:
- - uses: actions/checkout@v5
- - uses: oven-sh/setup-bun@v2
- with:
- bun-version: "1.3.10"
- - run: bun install --frozen-lockfile
- - uses: ./.github/actions/install-magmux
- with:
- token: ${{ github.token }}
- - name: sysctl
- run: sysctl -n hw.ncpu hw.memsize machdep.cpu.brand_string
- - name: Tests (three groups at once)
- working-directory: packages/cli
- continue-on-error: true
- run: bun scripts/ci-tests.ts --group pane,idle,rest -- --reporter=junit --reporter-outfile=junit-{group}.xml
- timeout-minutes: 40
- env:
- CLAUDISH_SKIP_LIVE_E2E: "1"
- - uses: actions/upload-artifact@v5
- if: always()
- with:
- name: junit-mac-lanes
- path: packages/cli/junit-*.xml
diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml
index 6b7840d4..10f87093 100644
--- a/.github/workflows/test.yml
+++ b/.github/workflows/test.yml
@@ -31,117 +31,120 @@ concurrency:
cancel-in-progress: true
jobs:
- test:
- name: typecheck · lint · hermetic tests
- # macOS, not ubuntu — deliberate. Measured on ubuntu-latest (run
- # 31173939734): the credential-equivalence suite and the magmux e2e fail
- # wholesale because claudish's credential authority reaches for macOS
- # facilities (keychain, `ioreg` screen-lock probe) and the magmux bridge
- # binary is a macOS artifact the release workflow downloads separately.
- # The suite has only ever been developed and run on darwin. Gating on the
- # platform it is actually written for gives a HONEST signal; gating on
- # ubuntu would mean permanently-red checks people learn to ignore.
- # Linux portability of the test suite is a separate piece of work.
- runs-on: macos-latest
+ # The hermetic suite is cut into slices that run at the same time, so the gate takes as
+ # long as its slowest slice, not as long as the whole suite (about 25 minutes run whole).
+ # packages/cli/scripts/ci-tests.ts owns the file list, the groups and the slices; the
+ # slices are balanced by scripts/ci-test-times.json (refresh it from these jobs' junit:
+ # `bun scripts/ci-tests.ts --update-times junit-*.xml`).
+ #
+ # WHERE EACH GROUP RUNS, measured on PR #328 (run 39545203694):
+ # - idle (recovery ladders, the grok refresh ceiling) passes 54/54 on ubuntu and is CPU-idle,
+ # so it runs there, in three slices.
+ # - pane (real magmux panes) and rest stay on macOS: on ubuntu-latest pane fails 23 tests
+ # and rest 11 (credential, keychain and platform paths written for darwin).
+ # The org is on the free plan: 5 macOS jobs at once across every repo, against 20 Linux
+ # ones. So macOS runs two runners, each with several slices as concurrent processes; the
+ # pane slices mostly wait on magmux's fixed send windows, not on CPU.
+ #
+ # Every bun is the PINNED 1.3.10, not `latest`. Measured on run 31174096977: a newer bun's
+ # `bun install` REWRITES packages/cli/package.json (expanding the `files` array), which
+ # biome then fails as a format error. `latest` also lets a bun release break CI overnight.
+ #
+ # CLAUDISH_SKIP_LIVE_E2E is REQUIRED wherever tests run. mcp-session/e2e-sessions.test.ts
+ # probes for a usable Claude Code at MODULE LOAD by spawning `claude --version` and
+ # awaiting only its `exit` event. A runner has no `claude`, so spawn emits `error`, the
+ # promise never settles, and the suite hangs until the job is killed (run 31172281077).
+ static:
+ name: typecheck · lint
+ runs-on: ubuntu-latest
env:
FORCE_JAVASCRIPT_ACTIONS_TO_NODE24: true
steps:
- uses: actions/checkout@v5
-
- # PINNED, not `latest`. Measured on run 31174096977: a newer bun's
- # `bun install` REWRITES packages/cli/package.json (expanding the `files`
- # array), which biome then fails as a format error — a red gate caused by
- # an upstream release rather than by the code under test. `latest` also
- # means a bun release can break CI overnight with no commit to blame.
- uses: oven-sh/setup-bun@v2
with:
bun-version: "1.3.10"
-
- # Root install — `workspaces: ["packages/*"]`, so this covers cli AND
- # macos-bridge. The root scripts below span both packages; running
- # `--cwd packages/cli` instead would silently drop macos-bridge.
- - name: Install dependencies
- run: bun install --frozen-lockfile
-
- # The pane suites run against the SAME magmux the release ships: the tag and
- # checksums in .github/magmux.env, which release.yml reads too. The Homebrew
- # tap served whatever was current, so CI could go green against 0.15 while
- # 0.14.0 shipped untested. Installed to /usr/local/bin, a path
- # findMagmuxForTest() and the launcher both search (no Homebrew copy exists on
- # the runner, so nothing earlier in the search shadows it).
- - name: Install the pinned magmux (pane suites)
- run: |
- set -a; . .github/magmux.env; set +a
- ARCH=$(uname -m); case "$ARCH" in (arm64|aarch64) ARCH=arm64;; (x86_64) ARCH=amd64;; esac
- ASSET="magmux_darwin_${ARCH}.tar.gz"
- WANT=$(eval echo "\$MAGMUX_SHA256_darwin_${ARCH}")
- gh release download "${MAGMUX_TAG}" --repo MadAppGang/magmux --pattern "${ASSET}" --dir /tmp
- echo "${WANT} /tmp/${ASSET}" | shasum -a 256 -c -
- tar xzf "/tmp/${ASSET}" -C /tmp
- sudo install -m 0755 /tmp/magmux /usr/local/bin/magmux
- magmux --version | grep -q "${MAGMUX_TAG#v}" || { echo "installed magmux is not ${MAGMUX_TAG}"; magmux --version; exit 1; }
- env:
- GH_TOKEN: ${{ github.token }}
-
+ # Root install — `workspaces: ["packages/*"]`, so this covers cli AND macos-bridge.
+ - run: bun install --frozen-lockfile
- name: Typecheck
run: bun run typecheck
-
- # Fails on ERRORS only. The suite currently carries ~797 warnings
- # (noExplicitAny, noExcessiveCognitiveComplexity) which are pre-existing
- # style debt, not defects — biome exits 0 for those, so this gate catches
- # genuine regressions without demanding a 300-file cleanup first.
+ # Fails on ERRORS only. The ~800 warnings (noExplicitAny,
+ # noExcessiveCognitiveComplexity) are pre-existing style debt, which biome exits 0 for.
- name: Lint
run: bun run lint
- # CLAUDISH_SKIP_LIVE_E2E is REQUIRED here, and not just an optimisation.
- # mcp-session/e2e-sessions.test.ts probes for a usable Claude Code at MODULE
- # LOAD by spawning `claude --version` and awaiting only its `exit` event.
- # A runner has no `claude` binary, so spawn emits `error` instead, the
- # promise never settles, and the whole suite hangs until the job is
- # killed (observed: >12 min on run 31172281077 before cancellation).
- # The file's own flag short-circuits that probe.
- #
- # The timeout is the backstop: a hang should fail in 25 minutes, not burn a
- # 6-hour runner slot. "A healthy run is ~60s" stopped being true long ago:
- # measured locally (M-series, 2026-10-06) packages/cli alone takes 697 s
- # before the pane suites and 832-846 s with them (bun 1.4.0 and the pinned
- # 1.3.10), already past 10 minutes; the recovery-contract black-box suites
- # and the real-magmux pane suites dominate. On the CI runner main took 1088 s
- # (2026-10-20, run 39383777717) and the gated-delivery suite for magmux's input
- # gate adds ~310 s, so ~26 minutes; 40 minutes leaves ~1.5x headroom.
- #
- # src/hooks/blind is EXCLUDED here and runs in the blind-hooks job below.
- # This is the root `bun run test` spelled out, because Bun 1.3.10 has no way
- # to ignore a path: no --path-ignore-patterns flag, and bunfig's
- # test.pathIgnorePatterns is not read (both measured on 1.3.10). So the cli
- # files are listed explicitly, with Bun's own test-file name rule, and the
- # list must not come back empty. They are passed WITHOUT a leading `./`: Bun
- # then reads them as filters on its own discovery and keeps a bare `bun
- # test`'s file order (measured: same 431 files, same order). `./` paths run
- # in argument order instead, and a few suites share process state across
- # files, so a new order is a new suite.
- - name: Hermetic tests
- shell: bash
+ idle:
+ name: hermetic · idle ${{ matrix.slice }}/3
+ runs-on: ubuntu-latest
+ strategy:
+ fail-fast: false
+ matrix:
+ slice: [1, 2, 3]
+ env:
+ FORCE_JAVASCRIPT_ACTIONS_TO_NODE24: true
+ CLAUDISH_SKIP_LIVE_E2E: "1"
+ steps:
+ - uses: actions/checkout@v5
+ - uses: oven-sh/setup-bun@v2
+ with:
+ bun-version: "1.3.10"
+ - run: bun install --frozen-lockfile
+ - name: Tests
+ working-directory: packages/cli
+ run: bun scripts/ci-tests.ts --run idle@${{ matrix.slice }}/3 -- --reporter=junit --reporter-outfile=junit-{label}.xml
+ timeout-minutes: 10
+ - uses: actions/upload-artifact@v5
+ if: always()
+ with:
+ name: junit-idle-${{ matrix.slice }}
+ path: packages/cli/junit-*.xml
+
+ hermetic:
+ name: hermetic · macOS ${{ matrix.runner }}/2
+ runs-on: macos-latest
+ strategy:
+ fail-fast: false
+ matrix:
+ include:
+ - runner: 1
+ runs: pane@1/6,pane@2/6,pane@3/6,rest
+ - runner: 2
+ runs: pane@4/6,pane@5/6,pane@6/6
+ env:
+ FORCE_JAVASCRIPT_ACTIONS_TO_NODE24: true
+ CLAUDISH_SKIP_LIVE_E2E: "1"
+ steps:
+ - uses: actions/checkout@v5
+ - uses: oven-sh/setup-bun@v2
+ with:
+ bun-version: "1.3.10"
+ - run: bun install --frozen-lockfile
+ # The pane suites run against the SAME magmux the release ships (.github/magmux.env).
+ - uses: ./.github/actions/install-magmux
+ with:
+ token: ${{ github.token }}
+ - name: Tests (${{ matrix.runs }})
+ working-directory: packages/cli
+ run: bun scripts/ci-tests.ts --run ${{ matrix.runs }} -- --reporter=junit --reporter-outfile=junit-{label}.xml
+ timeout-minutes: 15
+ - name: macos-bridge and script tests
+ if: matrix.runner == 2
run: |
- cd packages/cli
- FILES=$(find . -name node_modules -prune -o -path ./src/hooks/blind -prune -o -type f -print \
- | grep -E '(\.test|_test_|\.spec|_spec_)[^/]*\.[cm]?[jt]sx?$' | sed 's|^\./||')
- [ -n "$FILES" ] || { echo "no cli test files found"; exit 1; }
- # shellcheck disable=SC2086 # one argument per file; no path holds a space
- bun test $FILES
- cd ../..
bun run --cwd packages/macos-bridge test
bun test ./scripts/*.test.ts
- timeout-minutes: 40
- env:
- CLAUDISH_SKIP_LIVE_E2E: "1"
+ timeout-minutes: 5
+ - uses: actions/upload-artifact@v5
+ if: always()
+ with:
+ name: junit-macos-${{ matrix.runner }}
+ path: packages/cli/junit-*.xml
# The blind hook tests, on Bun 1.4.0 instead of the pinned 1.3.10. Two of them
# fail on 1.3.10 because of bugs in Bun's TEST client, not in claudish, and pass
# 20 of 20 on 1.4.0 (reproduced on 1.3.10 with no claudish code in the loop):
# C-17, a fetch reader.cancel() that never closes the connection, and X-12, a
# paused-then-resumed node:child_process reader that drops every byte. The
- # blind TYPECHECK stays in the job above, under the pinned Bun. This job does
+ # blind TYPECHECK stays in the static job, under the pinned Bun. This job does
# not lint, so a newer bun rewriting packages/cli/package.json on install does
# not matter here.
blind-hooks:
@@ -159,21 +162,10 @@ jobs:
- name: Install dependencies
run: bun install --frozen-lockfile
- # Same pinned magmux as the job above, so the blind tests run in the same
- # environment the hermetic job gives the rest of the suite.
- - name: Install the pinned magmux (pane suites)
- run: |
- set -a; . .github/magmux.env; set +a
- ARCH=$(uname -m); case "$ARCH" in (arm64|aarch64) ARCH=arm64;; (x86_64) ARCH=amd64;; esac
- ASSET="magmux_darwin_${ARCH}.tar.gz"
- WANT=$(eval echo "\$MAGMUX_SHA256_darwin_${ARCH}")
- gh release download "${MAGMUX_TAG}" --repo MadAppGang/magmux --pattern "${ASSET}" --dir /tmp
- echo "${WANT} /tmp/${ASSET}" | shasum -a 256 -c -
- tar xzf "/tmp/${ASSET}" -C /tmp
- sudo install -m 0755 /tmp/magmux /usr/local/bin/magmux
- magmux --version | grep -q "${MAGMUX_TAG#v}" || { echo "installed magmux is not ${MAGMUX_TAG}"; magmux --version; exit 1; }
- env:
- GH_TOKEN: ${{ github.token }}
+ # Same pinned magmux as the hermetic jobs, so the blind tests run in the same environment.
+ - uses: ./.github/actions/install-magmux
+ with:
+ token: ${{ github.token }}
- name: Blind hook tests
working-directory: packages/cli
@@ -184,7 +176,7 @@ jobs:
# S1: a user hook module (.ts type import, .ts value import through the
# Bun.plugin virtual module, .js) loads from a `--target bun` bundle and from a
- # `--compile` binary. The hermetic job above runs the same test on macOS with
+ # `--compile` binary. The hermetic jobs run the same test on macOS with
# the pinned 1.3.10; this job runs it on every OS/arch release.yml builds a
# binary for, with the Bun release.yml builds with (`latest`), because that is
# the binary magus installs. The test imports no dependency, so there is no
diff --git a/packages/cli/scripts/ci-test-times.json b/packages/cli/scripts/ci-test-times.json
new file mode 100644
index 00000000..4eb9ebec
--- /dev/null
+++ b/packages/cli/scripts/ci-test-times.json
@@ -0,0 +1,452 @@
+{
+ "scripts/smoke.test.ts": 0.1,
+ "src/adapters/anthropic-inline-system.test.ts": 0,
+ "src/adapters/catalog-contract-payload.test.ts": 0,
+ "src/adapters/effort-mapping.test.ts": 0,
+ "src/adapters/effort-not-silently-dropped.test.ts": 0,
+ "src/adapters/effort-override.test.ts": 0,
+ "src/adapters/gemini-api-format-claude-budget.test.ts": 0,
+ "src/adapters/grok-effort-support.test.ts": 0.1,
+ "src/adapters/litellm-api-format.test.ts": 0,
+ "src/adapters/model-catalog-vision.test.ts": 0.1,
+ "src/adapters/model-catalog.test.ts": 0,
+ "src/adapters/openrouter-api-format.test.ts": 0,
+ "src/adapters/qwen-model-dialect.test.ts": 0,
+ "src/adapters/reasoning-cache.test.ts": 0,
+ "src/adapters/reasoning-effort.test.ts": 0,
+ "src/adapters/sampling-params-forwarded.test.ts": 0,
+ "src/adapters/tool-choice-mapping.test.ts": 0,
+ "src/adapters/tool-name-codec.test.ts": 0,
+ "src/adapters/variant-presets.test.ts": 0,
+ "src/advisor-startup.test.ts": 0,
+ "src/auth/antigravity-client-version-regression.test.ts": 0,
+ "src/auth/antigravity-oauth.test.ts": 0,
+ "src/auth/antigravity-token.test.ts": 0,
+ "src/auth/antigravity-user.test.ts": 0,
+ "src/auth/auth-commands.test.ts": 0.3,
+ "src/auth/credentials/api-key-credential.test.ts": 0,
+ "src/auth/credentials/authority.test.ts": 0,
+ "src/auth/credentials/billing-probe.test.ts": 0.1,
+ "src/auth/credentials/equivalence.test.ts": 0,
+ "src/auth/credentials/keychain-source.test.ts": 0,
+ "src/auth/credentials/kimi-credential.test.ts": 0.1,
+ "src/auth/credentials/muse-code-credential.test.ts": 0,
+ "src/auth/credentials/op-source.test.ts": 0,
+ "src/auth/credentials/prehydrate.test.ts": 0,
+ "src/auth/credentials/prompt-role.test.ts": 0,
+ "src/auth/credentials/readiness.test.ts": 0,
+ "src/auth/credentials/types.blackbox.test.ts": 0,
+ "src/auth/credentials/types.test.ts": 0,
+ "src/auth/grok-oauth.test.ts": 0,
+ "src/auth/meta-muse-oauth.test.ts": 0,
+ "src/auth/oauth-manager.test.ts": 0,
+ "src/auth/quota/quota.test.ts": 0,
+ "src/auth/quota/sources/grok.test.ts": 0,
+ "src/auth/vertex-access-denied.test.ts": 0,
+ "src/auth/vertex-auth-mode.test.ts": 0,
+ "src/classifier-passthrough.integration.test.ts": 0,
+ "src/classifier-passthrough.test.ts": 0,
+ "src/claude-runner-advisor.test.ts": 0,
+ "src/claude-runner-child-env.test.ts": 0,
+ "src/claude-runner-child-stderr.test.ts": 0,
+ "src/claude-runner-context-window.test.ts": 0,
+ "src/claude-runner-token-file.test.ts": 0,
+ "src/cli-native-model-normalization.test.ts": 0,
+ "src/cli-passthrough-walker.test.ts": 0,
+ "src/cli-passthrough.test.ts": 0,
+ "src/cli-provider-shortcuts.test.ts": 0,
+ "src/cli.test.ts": 0,
+ "src/config-override.test.ts": 0,
+ "src/config-schema.test.ts": 0,
+ "src/cwd-env-control.test.ts": 1.4,
+ "src/default-provider.test.ts": 0.8,
+ "src/force-login-method.test.ts": 0,
+ "src/format-translation.test.ts": 0,
+ "src/glm-adapter.test.ts": 0,
+ "src/handlers/advisor-capture.test.ts": 0,
+ "src/handlers/advisor-decorator.test.ts": 0,
+ "src/handlers/advisor-failures.test.ts": 0,
+ "src/handlers/advisor-panel-prompt.test.ts": 0,
+ "src/handlers/advisor-routing-and-state.test.ts": 0.3,
+ "src/handlers/composed-handler-connection-error.test.ts": 0,
+ "src/handlers/composed-handler-error.test.ts": 0,
+ "src/handlers/composed-handler-hooks.test.ts": 0,
+ "src/handlers/composed-handler-raw.test.ts": 3.2,
+ "src/handlers/composed-handler-recovery.test.ts": 2.3,
+ "src/handlers/composed-handler-request-headers.test.ts": 0,
+ "src/handlers/composed-handler-seam.test.ts": 0,
+ "src/handlers/composed-handler-wire.test.ts": 3,
+ "src/handlers/composed-handler.muse-code.test.ts": 1.5,
+ "src/handlers/composed-handler.test.ts": 0,
+ "src/handlers/default-provider-e2e.test.ts": 0.9,
+ "src/handlers/explicit-spec-no-credential.test.ts": 0,
+ "src/handlers/fallback-handler.muse-code.test.ts": 0,
+ "src/handlers/fallback-handler.subscription-only.test.ts": 0,
+ "src/handlers/fallback-handler.test.ts": 0,
+ "src/handlers/native-handler-advisor.test.ts": 0,
+ "src/handlers/native-handler-hooks.test.ts": 0,
+ "src/handlers/param-rejection-recovery.test.ts": 0,
+ "src/handlers/recovery-status.test.ts": 2.7,
+ "src/handlers/shared/anthropic-error.test.ts": 0,
+ "src/handlers/shared/auth-failure.test.ts": 0,
+ "src/handlers/shared/collect-sse-message.test.ts": 0.1,
+ "src/handlers/shared/connection-error.test.ts": 0,
+ "src/handlers/shared/format/openai-tools.test.ts": 0,
+ "src/handlers/shared/gemini-schema.test.ts": 0,
+ "src/handlers/shared/model-unsupported.blackbox.test.ts": 0,
+ "src/handlers/shared/model-unsupported.test.ts": 0,
+ "src/handlers/shared/quota-exhaustion.test.ts": 0,
+ "src/handlers/shared/refresh-deadline.test.ts": 0,
+ "src/handlers/shared/request-shape.test.ts": 0,
+ "src/handlers/shared/schema-validate.test.ts": 0,
+ "src/handlers/shared/stream-parsers/anthropic-sse-index-clamping.test.ts": 0,
+ "src/handlers/shared/stream-parsers/block-nesting.test.ts": 0,
+ "src/handlers/shared/stream-parsers/block-writer.test.ts": 0,
+ "src/handlers/shared/stream-parsers/contentless-turn.test.ts": 0,
+ "src/handlers/shared/stream-parsers/ending-taxonomy.test.ts": 0,
+ "src/handlers/shared/stream-parsers/openai-responses-sse.test.ts": 2.3,
+ "src/handlers/shared/stream-parsers/sse-bare-data-lines.test.ts": 0,
+ "src/handlers/shared/stream-parsers/sse-line.test.ts": 0,
+ "src/handlers/shared/stream-parsers/sse-log-payload.test.ts": 0,
+ "src/handlers/shared/stream-parsers/tool-block-input.test.ts": 0,
+ "src/handlers/shared/stream-parsers/tool-name-mangling.test.ts": 0,
+ "src/handlers/shared/stream-parsers/usage-cache-split.test.ts": 0,
+ "src/handlers/shared/stream-parsers/usage-regressions.test.ts": 0,
+ "src/handlers/shared/thinking-signature.test.ts": 0,
+ "src/handlers/shared/token-tracker.test.ts": 0,
+ "src/handlers/shared/tool-call-recovery.test.ts": 0,
+ "src/handlers/shared/transient-retry-slow-net.test.ts": 1.6,
+ "src/handlers/shared/transient-retry.test.ts": 1.1,
+ "src/handlers/shared/upstream-error-capture.test.ts": 0,
+ "src/handlers/stream-head-unreachable.test.ts": 12,
+ "src/handlers/tool-name-encode-decode-pairing.test.ts": 0,
+ "src/handlers/unregistered-credential-name.test.ts": 0,
+ "src/hooks-command.followup.test.ts": 0.2,
+ "src/hooks-command.test.ts": 2.8,
+ "src/hooks-docs.test.ts": 1.3,
+ "src/hooks/actions.test.ts": 0,
+ "src/hooks/builtins/observer.test.ts": 0,
+ "src/hooks/builtins/plan-mode.test.ts": 0,
+ "src/hooks/builtins/session-context.test.ts": 0,
+ "src/hooks/config.test.ts": 0,
+ "src/hooks/engine.test.ts": 0.3,
+ "src/hooks/harness.test.ts": 0,
+ "src/hooks/injection/in-process-sink.test.ts": 0.6,
+ "src/hooks/injection/inbox.test.ts": 1,
+ "src/hooks/injection/queue.test.ts": 0,
+ "src/hooks/injection/sinks.test.ts": 0,
+ "src/hooks/journal.test.ts": 0.5,
+ "src/hooks/loader.test.ts": 0.7,
+ "src/hooks/mcp-isolation.test.ts": 0.9,
+ "src/hooks/observer/corpus.test.ts": 0,
+ "src/hooks/output-capture.test.ts": 1.5,
+ "src/hooks/public.test.ts": 0,
+ "src/hooks/raw-request-payload.test.ts": 0,
+ "src/hooks/raw.test.ts": 0.5,
+ "src/hooks/registry.test.ts": 0,
+ "src/hooks/request-stage.test.ts": 0,
+ "src/hooks/response-seam.followup.test.ts": 1.2,
+ "src/hooks/response-seam.test.ts": 0,
+ "src/hooks/s1-probe/s1-loading.test.ts": 0,
+ "src/hooks/seam.test.ts": 1.4,
+ "src/hooks/session/binding.test.ts": 1.3,
+ "src/hooks/session/cross-process.integration.test.ts": 19.9,
+ "src/hooks/session/interactive-driver.integration.test.ts": 57.7,
+ "src/hooks/session/interactive-launch.test.ts": 0,
+ "src/hooks/sse-codec.test.ts": 0.1,
+ "src/hooks/telemetry.test.ts": 0,
+ "src/hooks/text-rewrite.test.ts": 0,
+ "src/launch-env.test.ts": 0,
+ "src/launcher-bun-detect.test.ts": 0,
+ "src/launcher/catalog-warm.test.ts": 0,
+ "src/launcher/magmux-wrapper.test.ts": 0,
+ "src/logger.test.ts": 0,
+ "src/mcp-blackbox/channel-sessions.blackbox.test.ts": 18.6,
+ "src/mcp-blackbox/resize.blackbox.test.ts": 2.3,
+ "src/mcp-blackbox/run-prompt.blackbox.test.ts": 5.8,
+ "src/mcp-blackbox/team-contract.blackbox.test.ts": 34.6,
+ "src/mcp-blackbox/tool-surface.blackbox.test.ts": 18.5,
+ "src/mcp-child-flags.test.ts": 0,
+ "src/mcp-contract.e2e.test.ts": 6.4,
+ "src/mcp-preflight-not-a-precondition.test.ts": 0,
+ "src/mcp-preflight-row.test.ts": 2.3,
+ "src/mcp-session/e2e-sessions.test.ts": 0,
+ "src/mcp-session/home-dir.test.ts": 0,
+ "src/mcp-session/host-pid.contract.test.ts": 2.5,
+ "src/mcp-session/parent-session.contract.test.ts": 0,
+ "src/mcp-session/scrollback-buffer.test.ts": 0,
+ "src/mcp-session/session-create-options.test.ts": 1.9,
+ "src/mcp-session/session-manager.test.ts": 49.4,
+ "src/mcp-session/session-records.contract.test.ts": 22.6,
+ "src/mcp-session/session-state-records.contract.test.ts": 44.9,
+ "src/mcp-session/session-timeout.test.ts": 1.1,
+ "src/mcp-session/session-wire-format.test.ts": 0.8,
+ "src/mcp-session/spawn-record.contract.test.ts": 38.4,
+ "src/mcp-session/team-run-record.contract.test.ts": 0,
+ "src/mcp-shutdown.e2e.test.ts": 9.8,
+ "src/mcp/nested-mode.test.ts": 0,
+ "src/mcp/progress-heartbeat.test.ts": 1.2,
+ "src/mcp/run-prompt-claude-code.env.test.ts": 0,
+ "src/model-catalog.test.ts": 0,
+ "src/model-loader-v3.test.ts": 0,
+ "src/model-loader.muse-code.test.ts": 0,
+ "src/model-loader.test.ts": 0,
+ "src/model-params.test.ts": 0,
+ "src/model-selector-antigravity-order.test.ts": 0,
+ "src/model-selector-discovery.test.ts": 0,
+ "src/model-selector-picker.test.ts": 0,
+ "src/model-selector.muse-code.test.ts": 0,
+ "src/model-selector.test.ts": 0,
+ "src/native-anthropic-mapping.test.ts": 0,
+ "src/pane/accounting.test.ts": 0,
+ "src/pane/box-locate.test.ts": 0,
+ "src/pane/child-env.test.ts": 3,
+ "src/pane/child-flags.test.ts": 0,
+ "src/pane/input-gate.test.ts": 0,
+ "src/pane/magmux-client-reason.test.ts": 0,
+ "src/pane/magmux-client.test.ts": 0.8,
+ "src/pane/own-pane.test.ts": 0,
+ "src/pane/pane-child-real-claudish.integration.test.ts": 4.3,
+ "src/pane/pane-exit-code.followup.test.ts": 20.2,
+ "src/pane/pane-gated-delivery.integration.test.ts": 92.1,
+ "src/pane/pane-gated-retype.integration.test.ts": 105.5,
+ "src/pane/pane-gated-settle.integration.test.ts": 111.7,
+ "src/pane/pane-hooks.integration.test.ts": 19.6,
+ "src/pane/pane-launch-version-probe.test.ts": 0.5,
+ "src/pane/pane-launch.test.ts": 0.8,
+ "src/pane/pane-live-claude.integration.test.ts": 0,
+ "src/pane/pane-registry.test.ts": 31.7,
+ "src/pane/pane-resize.integration.test.ts": 36.2,
+ "src/pane/pane-session-liveness-exit.test.ts": 5.1,
+ "src/pane/pane-session-resize.test.ts": 1.3,
+ "src/pane/pane-session.integration.test.ts": 19,
+ "src/pane/pane-size.test.ts": 0,
+ "src/pane/process-identity.test.ts": 0.4,
+ "src/pane/process-table-trusted.test.ts": 0,
+ "src/pane/prompt-delivery.test.ts": 0,
+ "src/pane/reason-detail.test.ts": 0,
+ "src/pane/repl-wrap.test.ts": 0,
+ "src/pane/screen-classifier-sizes.test.ts": 0,
+ "src/pane/screen-classifier.test.ts": 0,
+ "src/pane/screen-model.test.ts": 0,
+ "src/pane/session-registry.test.ts": 0,
+ "src/pane/settle-rule.test.ts": 0.4,
+ "src/pane/slot-state.test.ts": 0,
+ "src/pane/slot-steps.test.ts": 0,
+ "src/pane/sweep-untrusted-table.test.ts": 0.6,
+ "src/pane/test-fixtures/fixtures-redaction.test.ts": 0.1,
+ "src/pane/transcript-follower-human.test.ts": 0,
+ "src/pane/transcript-follower.test.ts": 0.5,
+ "src/pane/transcript-recent.test.ts": 0,
+ "src/picker/import-direction.test.ts": 0,
+ "src/picker/layout.test.ts": 0,
+ "src/picker/ModelPicker-provider-order.test.tsx": 0.2,
+ "src/picker/ModelPicker.test.tsx": 4,
+ "src/picker/no-terminal-writes.test.ts": 0,
+ "src/picker/row-semantics.test.ts": 0,
+ "src/probe/probe-chain.test.ts": 3.1,
+ "src/probe/probe-results-printer.test.ts": 0,
+ "src/process-tree.test.ts": 3.3,
+ "src/profile-config-recovery.test.ts": 0,
+ "src/profile-config-scoped.test.ts": 0,
+ "src/providers/accept-served-model.test.ts": 0.4,
+ "src/providers/all-models-cache-separation.test.ts": 0.1,
+ "src/providers/all-models-cache.muse-code.test.ts": 0,
+ "src/providers/all-models-cache.test.ts": 0.1,
+ "src/providers/antigravity-model-discovery.test.ts": 0,
+ "src/providers/catalog-client-v3-reader-gaps.test.ts": 0,
+ "src/providers/catalog-client.muse-code.test.ts": 0,
+ "src/providers/catalog-client.test.ts": 4.2,
+ "src/providers/catalog-query.test.ts": 0,
+ "src/providers/catalog-route-bindings.test.ts": 0,
+ "src/providers/claude-code-aliases.test.ts": 0,
+ "src/providers/connection-price.test.ts": 0,
+ "src/providers/connection-resolver.muse-code.test.ts": 0,
+ "src/providers/connection-resolver.test.ts": 0.2,
+ "src/providers/connection-wire.test.ts": 0,
+ "src/providers/content-use.test.ts": 0,
+ "src/providers/custom-endpoints-loader.test.ts": 0.2,
+ "src/providers/declared-endpoints.test.ts": 3,
+ "src/providers/declared-wire.test.ts": 0,
+ "src/providers/devin/devin-credentials.test.ts": 0,
+ "src/providers/devin/devin-request.test.ts": 0,
+ "src/providers/devin/devin-system-prompt.test.ts": 0,
+ "src/providers/devin/model-id-resolver.test.ts": 0,
+ "src/providers/devin/proto-codec.test.ts": 0,
+ "src/providers/devin/tool-descriptions.test.ts": 0,
+ "src/providers/endpoint-registration.test.ts": 0,
+ "src/providers/grok/grok-credentials.test.ts": 0,
+ "src/providers/grok/grok-refresh-timeout.test.ts": 1,
+ "src/providers/keychain.test.ts": 0,
+ "src/providers/latest-opus.test.ts": 0,
+ "src/providers/local-liveness.test.ts": 0.3,
+ "src/providers/meta-muse/muse-credentials.test.ts": 0,
+ "src/providers/meta-muse/muse-keychain.test.ts": 0.2,
+ "src/providers/meta-muse/muse-launch.test.ts": 0,
+ "src/providers/meta-muse/muse-mint.test.ts": 0,
+ "src/providers/meta-muse/muse-models.test.ts": 0,
+ "src/providers/meta-muse/muse-reporting.test.ts": 0.5,
+ "src/providers/meta-muse/muse-store.test.ts": 0,
+ "src/providers/missing-credential-hint.test.ts": 0,
+ "src/providers/missing-output-limit.test.ts": 0.1,
+ "src/providers/model-availability.test.ts": 0,
+ "src/providers/model-catalog-reasoning.test.ts": 0,
+ "src/providers/model-catalog.test.ts": 0,
+ "src/providers/model-chain.test.ts": 0,
+ "src/providers/model-discovery-models-catalog.test.ts": 0,
+ "src/providers/model-discovery-registry.test.ts": 0.6,
+ "src/providers/model-discovery-seam.test.ts": 0,
+ "src/providers/model-discovery.muse-code.test.ts": 0,
+ "src/providers/model-discovery.test.ts": 0.1,
+ "src/providers/model-parser.test.ts": 0,
+ "src/providers/model-resolvers/devin.test.ts": 0.4,
+ "src/providers/model-resolvers/registry.test.ts": 0,
+ "src/providers/native-route.test.ts": 0,
+ "src/providers/no-name-rules.test.ts": 0.2,
+ "src/providers/onepassword-config.test.ts": 0,
+ "src/providers/onepassword-handshake-lock.test.ts": 0.2,
+ "src/providers/onepassword-wasm.test.ts": 0,
+ "src/providers/onepassword.test.ts": 1.2,
+ "src/providers/op-source-entry.test.ts": 0,
+ "src/providers/opencode-tier-keys.test.ts": 0,
+ "src/providers/predefined-activation.test.ts": 0,
+ "src/providers/predefined-baseurl.test.ts": 0,
+ "src/providers/predefined-catalog.test.ts": 0,
+ "src/providers/predefined-collision.test.ts": 0,
+ "src/providers/predefined-containment.test.ts": 0,
+ "src/providers/predefined-gating.test.ts": 0,
+ "src/providers/predefined-override.test.ts": 0,
+ "src/providers/probe-catalog.muse-code.test.ts": 0.2,
+ "src/providers/probe-catalog.test.ts": 0.1,
+ "src/providers/probe-effort.test.ts": 0,
+ "src/providers/probe-live-classify.test.ts": 0,
+ "src/providers/probe-live-provider-effort.test.ts": 0,
+ "src/providers/probe-live.muse-code.test.ts": 0.1,
+ "src/providers/probe-live.test.ts": 0,
+ "src/providers/probe-runner.test.ts": 0,
+ "src/providers/provider-definitions.muse-code.test.ts": 0,
+ "src/providers/provider-definitions.test.ts": 0,
+ "src/providers/provider-profiles.muse-code.test.ts": 0.1,
+ "src/providers/provider-profiles.test.ts": 0,
+ "src/providers/provider-resolver-rescue.test.ts": 0,
+ "src/providers/provider-routing.test.ts": 0,
+ "src/providers/provider-slug-resolve.test.ts": 0,
+ "src/providers/qwen-coding-plan.test.ts": 0,
+ "src/providers/remote-provider-registry.test.ts": 0,
+ "src/providers/route-candidates.test.ts": 0,
+ "src/providers/route-description.test.ts": 0,
+ "src/providers/routing-rules.muse-code.test.ts": 0.1,
+ "src/providers/routing-rules.test.ts": 5.5,
+ "src/providers/swe-routing.test.ts": 0,
+ "src/providers/transport/anthropic-compat-kimi-oauth.test.ts": 0,
+ "src/providers/transport/anthropic-compat.test.ts": 0,
+ "src/providers/transport/antigravity-modelid.test.ts": 0,
+ "src/providers/transport/antigravity-probe-discovery.test.ts": 0,
+ "src/providers/transport/antigravity-terminality.test.ts": 0,
+ "src/providers/transport/connect-evidence.test.ts": 0,
+ "src/providers/transport/local-recovery.test.ts": 1,
+ "src/providers/transport/meta-muse-code.test.ts": 0.1,
+ "src/providers/transport/openai-codex-cache-key.test.ts": 0,
+ "src/providers/transport/openai-codex-oauth.test.ts": 0,
+ "src/providers/transport/openai-codex-refresh-outage.test.ts": 0.1,
+ "src/providers/transport/openai.test.ts": 4,
+ "src/providers/transport/opencode-zen.test.ts": 0,
+ "src/providers/transport/probe-discovery-catalog-chat-capability.test.ts": 0,
+ "src/providers/transport/probe-discovery-modalities.test.ts": 0,
+ "src/providers/transport/probe-discovery.test.ts": 0,
+ "src/providers/transport/vertex-discovery.access-denied.test.ts": 0,
+ "src/providers/transport/vertex-discovery.test.ts": 0,
+ "src/providers/transport/vertex-oauth.test.ts": 0,
+ "src/providers/wire-handler-pending.test.ts": 0.4,
+ "src/providers/wire-handler.test.ts": 0.1,
+ "src/providers/wire-markers.test.ts": 0,
+ "src/providers/wire-request.test.ts": 0,
+ "src/providers/wire-try-in-turn.test.ts": 0,
+ "src/proxy-placeholder-auth.test.ts": 0,
+ "src/proxy-server.muse-code.test.ts": 0.3,
+ "src/proxy-server.test.ts": 0.1,
+ "src/recommendation-tiers.test.ts": 0,
+ "src/recommended-capabilities.test.ts": 0,
+ "src/recommended-listing.test.ts": 0,
+ "src/recovery-contract/recovery-chain-blackbox.test.ts": 16.7,
+ "src/recovery-contract/recovery-completes-blackbox.test.ts": 33,
+ "src/recovery-contract/recovery-deadline-blackbox.test.ts": 35.1,
+ "src/recovery-contract/recovery-disconnect-blackbox.test.ts": 53.6,
+ "src/recovery-contract/recovery-exhaustion-blackbox.test.ts": 15,
+ "src/recovery-contract/recovery-ladder-blackbox.test.ts": 67.2,
+ "src/recovery-contract/recovery-process-blackbox.test.ts": 33.2,
+ "src/recovery-contract/recovery-slow-fault-blackbox.test.ts": 45.1,
+ "src/recovery-contract/recovery-switches-blackbox.test.ts": 20.1,
+ "src/recovery/clock.test.ts": 0.1,
+ "src/recovery/coordinator.test.ts": 0.6,
+ "src/recovery/magmux-ui.test.ts": 24.5,
+ "src/recovery/retry-watchdog-owner.test.ts": 0,
+ "src/recovery/settings.test.ts": 0.5,
+ "src/redact.test.ts": 0,
+ "src/session-events/event-translator.test.ts": 0,
+ "src/session-events/pro-injection.test.ts": 0,
+ "src/session-events/registry.test.ts": 0,
+ "src/session-events/session-state.test.ts": 0,
+ "src/session-events/transcript-tailer.test.ts": 0.1,
+ "src/session/ansi-viz.test.ts": 0,
+ "src/session/baseline-pricing.test.ts": 0,
+ "src/session/conversation.test.ts": 0.1,
+ "src/session/resume-picker-contrast.test.ts": 0,
+ "src/session/session-discovery.test.ts": 0.1,
+ "src/session/session-stats.test.ts": 0,
+ "src/session/session-summary.test.ts": 0,
+ "src/spawn-claudish.test.ts": 0,
+ "src/startup-trace.test.ts": 0,
+ "src/stats-buffer.test.ts": 0,
+ "src/stats-otlp.test.ts": 0,
+ "src/stats.test.ts": 0,
+ "src/status-line-context.test.ts": 0.6,
+ "src/team-cancel.test.ts": 15,
+ "src/team-cli-run-mode.test.ts": 6.1,
+ "src/team-cli-signal.test.ts": 3.8,
+ "src/team-grid.e2e.test.ts": 10.4,
+ "src/team-heartbeat-survival.test.ts": 23.2,
+ "src/team-input-file.test.ts": 0,
+ "src/team-list-contract.test.ts": 27,
+ "src/team-liveness-exited-slot.test.ts": 4,
+ "src/team-mcp-shape-contract.e2e.test.ts": 12.1,
+ "src/team-multibyte-answer.test.ts": 3.1,
+ "src/team-orchestrator-settle.contract.test.ts": 7.9,
+ "src/team-orchestrator.test.ts": 4,
+ "src/team-output-classification.test.ts": 0,
+ "src/team-response-write.test.ts": 3.4,
+ "src/team-result-card.test.ts": 0,
+ "src/team-run-mcp.contract.test.ts": 18.9,
+ "src/team-run-settles-once.test.ts": 4.4,
+ "src/team-run-summary.contract.test.ts": 0,
+ "src/team-shutdown.test.ts": 1.2,
+ "src/team-start-failure.test.ts": 3.3,
+ "src/team-status-payload.test.ts": 0,
+ "src/telemetry.test.ts": 0.1,
+ "src/terminal-isolation.test.ts": 0,
+ "src/test-fixtures/extract-sse-from-log.test.ts": 0,
+ "src/test-fixtures/parser-goldens/parser-goldens.test.ts": 0,
+ "src/test-helpers/stub-upstream.test.ts": 0.2,
+ "src/theme/ansi.test.ts": 0,
+ "src/theme/theme-mode.test.ts": 0,
+ "src/transform.test.ts": 0,
+ "src/tui/App.probe-status.test.ts": 0,
+ "src/tui/components/footer-keys.test.ts": 0,
+ "src/tui/components/OnepasswordModal.test.ts": 0,
+ "src/tui/components/ProviderDetail.test.ts": 0,
+ "src/tui/index.gcloud-command.test.ts": 0,
+ "src/tui/probe-outcome.test.ts": 0,
+ "src/tui/providers.muse-code.test.ts": 0.1,
+ "src/tui/providers.test.ts": 0,
+ "src/tui/providers.vertex.test.ts": 0,
+ "src/tui/routing-explanation.test.ts": 0.1,
+ "src/tui/runtime/should-open-picker.test.ts": 0,
+ "src/tui/runtime/shutdown.test.ts": 1.8,
+ "src/tui/theme-contrast.test.ts": 0,
+ "src/tui/viz/color.test.ts": 0.2,
+ "src/tui/viz/widgets.test.tsx": 0.2,
+ "src/update-checker.test.ts": 1.3,
+ "src/update-command-help.test.ts": 0.6,
+ "src/version-contract.test.ts": 0,
+ "src/zai-glm.e2e.test.ts": 0
+}
diff --git a/packages/cli/scripts/ci-tests.ts b/packages/cli/scripts/ci-tests.ts
index 6d550127..8b93b40b 100644
--- a/packages/cli/scripts/ci-tests.ts
+++ b/packages/cli/scripts/ci-tests.ts
@@ -1,12 +1,20 @@
/**
- * The ONE list of the hermetic test files, split into named groups, and the runner for them.
+ * The ONE list of the hermetic test files, split into named groups and balanced slices, and the
+ * runner for them.
*
* bun scripts/ci-tests.ts --list
- * bun scripts/ci-tests.ts --group pane,idle [-- <bun test args>]
+ * bun scripts/ci-tests.ts --plan pane@6
+ * bun scripts/ci-tests.ts --run pane@1/6,pane@2/6,rest [-- <bun test args>]
+ * bun scripts/ci-tests.ts --update-times <junit.xml>…
*
- * Several groups run as concurrent `bun test` processes, each line prefixed with its group.
- * `{group}` in a passed-through argument becomes the group name, so each process can write its
- * own `--reporter-outfile`. The exit code is non-zero when any group fails.
+ * `--run` takes groups (`rest`) and slices (`pane@2/6`: the 2nd of 6 slices of `pane`). Several
+ * run as concurrent `bun test` processes, each output line prefixed with its label. `{label}` in
+ * a passed-through argument becomes the label (`pane-2of6`), so each process can write its own
+ * `--reporter-outfile`. The exit code is non-zero when any process fails.
+ *
+ * Slices are balanced by the measured seconds in ci-test-times.json, which `--update-times`
+ * rewrites from Bun junit reports. A file missing from the table weighs 1 s, so a stale table
+ * only unbalances the slices: every file of a group is still in exactly one of its slices.
*
* Files are passed WITHOUT a leading `./`: Bun then reads them as filters on its own discovery
* and keeps a bare `bun test`'s file order. `./` paths run in argument order instead, and a few
@@ -14,18 +22,22 @@
*
* src/hooks/blind is not here: it runs under Bun 1.4.0 in its own job (test.yml).
*/
+import { readFileSync, writeFileSync } from "node:fs";
import { join } from "node:path";
const CLI = join(import.meta.dir, "..");
+const TIMES = join(import.meta.dir, "ci-test-times.json");
/** Bun's own test-file name rule. */
const TEST_FILE = /(\.test|_test_|\.spec|_spec_)[^/]*\.[cm]?[jt]sx?$/;
const EXCLUDED = ["node_modules/", "src/hooks/blind/"];
+const UNMEASURED_SECONDS = 1;
/**
* Each file belongs to the first group whose prefix it starts with; `rest` takes the others.
* - idle: CPU-idle waits on real retry ladders and token-refresh ceilings, with no magmux.
- * - pane: real magmux panes and the MCP session machinery on top of them.
+ * Passes on Linux.
+ * - pane: real magmux panes and the MCP session machinery on top of them. macOS only.
*/
const GROUPS: { name: string; prefixes: string[] }[] = [
{ name: "idle", prefixes: ["src/recovery-contract/", "src/providers/grok/grok-refresh-timeout"] },
@@ -43,6 +55,9 @@ const GROUPS: { name: string; prefixes: string[] }[] = [
];
const REST = "rest";
+/** Files that share process state across files: a slice takes all of them or none. */
+const TOGETHER = ["src/recovery/"];
+
export function discover(): string[] {
const files: string[] = [];
for (const path of new Bun.Glob("**/*").scanSync({ cwd: CLI, onlyFiles: true })) {
@@ -65,6 +80,101 @@ function byGroup(files: string[]): Map<string, string[]> {
return out;
}
+function readTimes(): Record<string, number> {
+ try {
+ return JSON.parse(readFileSync(TIMES, "utf8")) as Record<string, number>;
+ } catch {
+ return {};
+ }
+}
+
+/**
+ * Split `files` into `n` slices of about equal measured time: longest unit first, each to the
+ * lightest slice. Each slice keeps the discovery order.
+ */
+export function slices(
+ files: string[],
+ n: number,
+ times: Record<string, number>
+): { files: string[]; seconds: number }[] {
+ const units = new Map<string, string[]>();
+ for (const f of files) {
+ const key = TOGETHER.find((p) => f.startsWith(p)) ?? f;
+ units.set(key, [...(units.get(key) ?? []), f]);
+ }
+ const weigh = (fs: string[]) => fs.reduce((s, f) => s + (times[f] ?? UNMEASURED_SECONDS), 0);
+ const out = Array.from({ length: n }, () => ({ files: [] as string[], seconds: 0 }));
+ for (const unit of [...units.values()].sort((a, b) => weigh(b) - weigh(a))) {
+ const lightest = out.reduce((min, s) => (s.seconds < min.seconds ? s : min));
+ lightest.files.push(...unit);
+ lightest.seconds += weigh(unit);
+ }
+ for (const s of out) s.files.sort();
+ return out;
+}
+
+/** `rest` or `pane@2/6` → the label and the files it names. */
+function resolve(
+ spec: string,
+ groups: Map<string, string[]>,
+ times: Record<string, number>
+): { label: string; files: string[] } {
+ const m = spec.match(/^([a-z]+)(?:@(\d+)\/(\d+))?$/);
+ if (!m) throw new Error(`"${spec}" is not a group or a slice like pane@2/6`);
+ const [, name, index, count] = m;
+ const files = groups.get(name);
+ if (!files) throw new Error(`unknown group "${name}"; groups: ${[...groups.keys()].join(", ")}`);
+ if (!index) return { label: name, files };
+ const i = Number(index);
+ const n = Number(count);
+ if (i < 1 || i > n) throw new Error(`slice ${i} of ${n} does not exist`);
+ return { label: `${name}-${i}of${n}`, files: slices(files, n, times)[i - 1]?.files ?? [] };
+}
+
+/**
+ * Per-file wall seconds from Bun junit: the sum of each file's top-level describes and tests.
+ * Bun reports a `describe.concurrent` as 0 s; such a file's wall time is its longest test.
+ */
+export function junitSeconds(xml: string): Record<string, number> {
+ const w: JunitWalk = { depth: 0, file: "", sums: new Map(), longest: new Map() };
+ for (const [tag] of xml.matchAll(/<\/testsuite>|<testsuite\b[^>]*>|<testcase\b[^>]*>/g)) {
+ if (tag === "</testsuite>") w.depth--;
+ else if (tag.startsWith("<testsuite")) enterSuite(w, tag);
+ else visitCase(w, tag);
+ }
+ const files = new Set([...w.sums.keys(), ...w.longest.keys()]);
+ return Object.fromEntries(
+ [...files].map((f) => [f, Math.max(w.sums.get(f) ?? 0, w.longest.get(f) ?? 0)])
+ );
+}
+
+interface JunitWalk {
+ depth: number;
+ file: string;
+ /** a file's direct children: its top-level describes, and tests outside any describe */
+ sums: Map<string, number>;
+ longest: Map<string, number>;
+}
+
+function enterSuite(w: JunitWalk, tag: string): void {
+ w.depth++;
+ if (w.depth === 1) w.file = xmlAttr(tag, "file") || xmlAttr(tag, "name");
+ if (w.depth === 2) w.sums.set(w.file, (w.sums.get(w.file) ?? 0) + seconds(tag));
+}
+
+function visitCase(w: JunitWalk, tag: string): void {
+ if (w.depth === 1) w.sums.set(w.file, (w.sums.get(w.file) ?? 0) + seconds(tag));
+ w.longest.set(w.file, Math.max(w.longest.get(w.file) ?? 0, seconds(tag)));
+}
+
+function seconds(tag: string): number {
+ return Number(xmlAttr(tag, "time") || 0);
+}
+
+function xmlAttr(tag: string, name: string): string {
+ return tag.match(new RegExp(`\\b${name}="([^"]*)"`))?.[1] ?? "";
+}
+
async function pipe(stream: ReadableStream<Uint8Array>, prefix: string, sink: NodeJS.WriteStream) {
const decoder = new TextDecoder();
let carry = "";
@@ -77,12 +187,12 @@ async function pipe(stream: ReadableStream<Uint8Array>, prefix: string, sink: No
}
async function run(
- group: string,
+ label: string,
files: string[],
bunArgs: string[],
prefixed: boolean
): Promise<number> {
- const args = bunArgs.map((a) => a.replaceAll("{group}", group));
+ const args = bunArgs.map((a) => a.replaceAll("{label}", label));
const started = performance.now();
const proc = Bun.spawn([process.execPath, "test", ...args, ...files], {
cwd: CLI,
@@ -91,39 +201,58 @@ async function run(
});
if (prefixed) {
await Promise.all([
- pipe(proc.stdout as ReadableStream<Uint8Array>, `[${group}] `, process.stdout),
- pipe(proc.stderr as ReadableStream<Uint8Array>, `[${group}] `, process.stderr),
+ pipe(proc.stdout as ReadableStream<Uint8Array>, `[${label}] `, process.stdout),
+ pipe(proc.stderr as ReadableStream<Uint8Array>, `[${label}] `, process.stderr),
]);
}
const code = await proc.exited;
const secs = ((performance.now() - started) / 1000).toFixed(0);
- console.log(`[ci-tests] ${group}: ${files.length} files, exit ${code}, ${secs} s`);
+ console.log(`[ci-tests] ${label}: ${files.length} files, exit ${code}, ${secs} s`);
return code;
}
+const USAGE =
+ "usage: ci-tests.ts --list | --plan <group>@<n> | --run <group|group@i/n>[,…] [-- <bun test args>] | --update-times <junit.xml>…";
+
if (import.meta.main) {
const argv = process.argv.slice(2);
const dash = argv.indexOf("--");
const own = dash === -1 ? argv : argv.slice(0, dash);
const bunArgs = dash === -1 ? [] : argv.slice(dash + 1);
const groups = byGroup(discover());
+ const times = readTimes();
+ const [command, value] = own;
- if (own.includes("--list")) {
- for (const [name, files] of groups) console.log(`${name}\t${files.length}`);
- process.exit(0);
+ if (command === "--list") {
+ for (const [name, files] of groups) {
+ const secs = files.reduce((s, f) => s + (times[f] ?? UNMEASURED_SECONDS), 0);
+ console.log(`${name}\t${files.length} files\t${secs.toFixed(0)} s measured`);
+ }
+ } else if (command === "--plan" && value) {
+ const [name, n] = value.split("@");
+ for (const [i, s] of slices(groups.get(name ?? "") ?? [], Number(n), times).entries())
+ console.log(`${name}@${i + 1}/${n}\t${s.files.length} files\t${s.seconds.toFixed(0)} s`);
+ } else if (command === "--update-times" && own.length > 1) {
+ const next = { ...times };
+ for (const path of own.slice(1)) Object.assign(next, junitSeconds(readFileSync(path, "utf8")));
+ const known = new Set(discover());
+ const sorted = Object.fromEntries(
+ Object.entries(next)
+ .filter(([f]) => known.has(f))
+ .sort(([a], [b]) => a.localeCompare(b))
+ .map(([f, s]) => [f, Math.round(s * 10) / 10])
+ );
+ writeFileSync(TIMES, `${JSON.stringify(sorted, null, 2)}\n`);
+ console.log(`[ci-tests] ${Object.keys(sorted).length} files timed in ${TIMES}`);
+ } else if (command === "--run" && value) {
+ const wanted = value.split(",").filter(Boolean);
+ const runs = wanted.map((spec) => resolve(spec, groups, times));
+ for (const r of runs) if (r.files.length === 0) throw new Error(`${r.label} has no test files`);
+ const codes = await Promise.all(
+ runs.map((r) => run(r.label, r.files, bunArgs, runs.length > 1))
+ );
+ process.exit(codes.some((c) => c !== 0) ? 1 : 0);
+ } else {
+ throw new Error(USAGE);
}
- const i = own.indexOf("--group");
- const wanted = i === -1 ? [] : (own[i + 1] ?? "").split(",").filter(Boolean);
- if (wanted.length === 0)
- throw new Error("usage: ci-tests.ts --list | --group <name>[,<name>…] [-- <bun test args>]");
- for (const name of wanted) {
- const files = groups.get(name);
- if (!files)
- throw new Error(`unknown group "${name}"; groups: ${[...groups.keys()].join(", ")}`);
- if (files.length === 0) throw new Error(`group "${name}" has no test files`);
- }
- const codes = await Promise.all(
- wanted.map((name) => run(name, groups.get(name) ?? [], bunArgs, wanted.length > 1))
- );
- process.exit(codes.some((c) => c !== 0) ? 1 : 0);
}