Claude Code. Any Model. The most powerful AI coding agent now speaks every language.
ci: one list of hermetic test files in named groups, and a Linux/lanes measurement
Jack Rudenko8bee644b30f8
diff --git a/.github/actions/install-magmux/action.yml b/.github/actions/install-magmux/action.ymlnew file mode 100644index 00000000..64c33495--- /dev/null+++ b/.github/actions/install-magmux/action.yml@@ -0,0 +1,26 @@+name: Install the pinned magmux+description: >+ Installs the magmux release pinned in .github/magmux.env (the one release.yml ships) to+ /usr/local/bin, after checking its sha256 against the pin. A path findMagmuxForTest() and the+ launcher both search; nothing earlier in that search exists on a runner.+inputs:+ token:+ description: GitHub token for gh release download+ required: true+runs:+ using: composite+ steps:+ - shell: bash+ env:+ GH_TOKEN: ${{ inputs.token }}+ run: |+ set -a; . .github/magmux.env; set +a+ OS=$(uname -s | tr '[:upper:]' '[:lower:]')+ ARCH=$(uname -m); case "$ARCH" in (arm64|aarch64) ARCH=arm64;; (x86_64) ARCH=amd64;; esac+ ASSET="magmux_${OS}_${ARCH}.tar.gz"+ WANT=$(eval echo "\$MAGMUX_SHA256_${OS}_${ARCH}")+ gh release download "${MAGMUX_TAG}" --repo MadAppGang/magmux --pattern "${ASSET}" --dir /tmp+ echo "${WANT} /tmp/${ASSET}" | shasum -a 256 -c -+ tar xzf "/tmp/${ASSET}" -C /tmp+ sudo install -m 0755 /tmp/magmux /usr/local/bin/magmux+ magmux --version | grep -q "${MAGMUX_TAG#v}" || { echo "installed magmux is not ${MAGMUX_TAG}"; magmux --version; exit 1; }diff --git a/.github/workflows/test-experiment.yml b/.github/workflows/test-experiment.ymlnew file mode 100644index 00000000..a6dd578f--- /dev/null+++ b/.github/workflows/test-experiment.yml@@ -0,0 +1,73 @@+name: Tests experiment++# MEASUREMENT ONLY, deleted before merge. Answers two questions for the hermetic-job split:+# 1. Which test files pass on Linux runners (20 concurrent jobs on the free plan, against 5 macOS)?+# 2. How long each group takes when the three groups run concurrently on ONE macOS runner.+# Both write junit, so per-file and per-test times come from the reports, not log timestamps.++on:+ pull_request:+ branches: [main]++jobs:+ linux-probe:+ name: linux probe · ${{ matrix.group }}+ if: startsWith(github.head_ref, 'ci/hermetic-under-4min')+ strategy:+ fail-fast: false+ matrix:+ group: [pane, idle, rest]+ runs-on: ubuntu-latest+ env:+ FORCE_JAVASCRIPT_ACTIONS_TO_NODE24: true+ steps:+ - uses: actions/checkout@v5+ - uses: oven-sh/setup-bun@v2+ with:+ bun-version: "1.3.10"+ - run: bun install --frozen-lockfile+ - uses: ./.github/actions/install-magmux+ with:+ token: ${{ github.token }}+ - name: Tests (${{ matrix.group }})+ working-directory: packages/cli+ continue-on-error: true+ run: bun scripts/ci-tests.ts --group ${{ matrix.group }} -- --reporter=junit --reporter-outfile=junit-{group}.xml+ timeout-minutes: 40+ env:+ CLAUDISH_SKIP_LIVE_E2E: "1"+ - uses: actions/upload-artifact@v5+ if: always()+ with:+ name: junit-linux-${{ matrix.group }}+ path: packages/cli/junit-*.xml++ mac-lanes:+ name: macOS lanes · pane + idle + rest concurrently+ if: startsWith(github.head_ref, 'ci/hermetic-under-4min')+ runs-on: macos-latest+ env:+ FORCE_JAVASCRIPT_ACTIONS_TO_NODE24: true+ steps:+ - uses: actions/checkout@v5+ - uses: oven-sh/setup-bun@v2+ with:+ bun-version: "1.3.10"+ - run: bun install --frozen-lockfile+ - uses: ./.github/actions/install-magmux+ with:+ token: ${{ github.token }}+ - name: sysctl+ run: sysctl -n hw.ncpu hw.memsize machdep.cpu.brand_string+ - name: Tests (three groups at once)+ working-directory: packages/cli+ continue-on-error: true+ run: bun scripts/ci-tests.ts --group pane,idle,rest -- --reporter=junit --reporter-outfile=junit-{group}.xml+ timeout-minutes: 40+ env:+ CLAUDISH_SKIP_LIVE_E2E: "1"+ - uses: actions/upload-artifact@v5+ if: always()+ with:+ name: junit-mac-lanes+ path: packages/cli/junit-*.xmldiff --git a/packages/cli/scripts/ci-tests.ts b/packages/cli/scripts/ci-tests.tsnew file mode 100644index 00000000..6d550127--- /dev/null+++ b/packages/cli/scripts/ci-tests.ts@@ -0,0 +1,129 @@+/**+ * The ONE list of the hermetic test files, split into named groups, and the runner for them.+ *+ * bun scripts/ci-tests.ts --list+ * bun scripts/ci-tests.ts --group pane,idle [-- <bun test args>]+ *+ * Several groups run as concurrent `bun test` processes, each line prefixed with its group.+ * `{group}` in a passed-through argument becomes the group name, so each process can write its+ * own `--reporter-outfile`. The exit code is non-zero when any group fails.+ *+ * Files are passed WITHOUT a leading `./`: Bun then reads them as filters on its own discovery+ * and keeps a bare `bun test`'s file order. `./` paths run in argument order instead, and a few+ * suites share process state across files, so a new order is a new suite.+ *+ * src/hooks/blind is not here: it runs under Bun 1.4.0 in its own job (test.yml).+ */+import { join } from "node:path";++const CLI = join(import.meta.dir, "..");++/** Bun's own test-file name rule. */+const TEST_FILE = /(\.test|_test_|\.spec|_spec_)[^/]*\.[cm]?[jt]sx?$/;+const EXCLUDED = ["node_modules/", "src/hooks/blind/"];++/**+ * Each file belongs to the first group whose prefix it starts with; `rest` takes the others.+ * - idle: CPU-idle waits on real retry ladders and token-refresh ceilings, with no magmux.+ * - pane: real magmux panes and the MCP session machinery on top of them.+ */+const GROUPS: { name: string; prefixes: string[] }[] = [+ { name: "idle", prefixes: ["src/recovery-contract/", "src/providers/grok/grok-refresh-timeout"] },+ {+ name: "pane",+ prefixes: [+ "src/pane/",+ "src/mcp-session/",+ "src/mcp-blackbox/",+ "src/hooks/session/",+ "src/recovery/",+ "src/team",+ ],+ },+];+const REST = "rest";++export function discover(): string[] {+ const files: string[] = [];+ for (const path of new Bun.Glob("**/*").scanSync({ cwd: CLI, onlyFiles: true })) {+ if (EXCLUDED.some((dir) => path.startsWith(dir) || path.includes(`/${dir}`))) continue;+ if (TEST_FILE.test(path)) files.push(path);+ }+ return files.sort();+}++export function groupOf(file: string): string {+ return GROUPS.find((g) => g.prefixes.some((p) => file.startsWith(p)))?.name ?? REST;+}++function byGroup(files: string[]): Map<string, string[]> {+ const out = new Map<string, string[]>([+ ...GROUPS.map((g) => [g.name, []] as [string, string[]]),+ [REST, []],+ ]);+ for (const f of files) out.get(groupOf(f))?.push(f);+ return out;+}++async function pipe(stream: ReadableStream<Uint8Array>, prefix: string, sink: NodeJS.WriteStream) {+ const decoder = new TextDecoder();+ let carry = "";+ for await (const chunk of stream) {+ const lines = (carry + decoder.decode(chunk, { stream: true })).split("\n");+ carry = lines.pop() ?? "";+ for (const line of lines) sink.write(`${prefix}${line}\n`);+ }+ if (carry) sink.write(`${prefix}${carry}\n`);+}++async function run(+ group: string,+ files: string[],+ bunArgs: string[],+ prefixed: boolean+): Promise<number> {+ const args = bunArgs.map((a) => a.replaceAll("{group}", group));+ const started = performance.now();+ const proc = Bun.spawn([process.execPath, "test", ...args, ...files], {+ cwd: CLI,+ stdout: prefixed ? "pipe" : "inherit",+ stderr: prefixed ? "pipe" : "inherit",+ });+ if (prefixed) {+ await Promise.all([+ pipe(proc.stdout as ReadableStream<Uint8Array>, `[${group}] `, process.stdout),+ pipe(proc.stderr as ReadableStream<Uint8Array>, `[${group}] `, process.stderr),+ ]);+ }+ const code = await proc.exited;+ const secs = ((performance.now() - started) / 1000).toFixed(0);+ console.log(`[ci-tests] ${group}: ${files.length} files, exit ${code}, ${secs} s`);+ return code;+}++if (import.meta.main) {+ const argv = process.argv.slice(2);+ const dash = argv.indexOf("--");+ const own = dash === -1 ? argv : argv.slice(0, dash);+ const bunArgs = dash === -1 ? [] : argv.slice(dash + 1);+ const groups = byGroup(discover());++ if (own.includes("--list")) {+ for (const [name, files] of groups) console.log(`${name}\t${files.length}`);+ process.exit(0);+ }+ const i = own.indexOf("--group");+ const wanted = i === -1 ? [] : (own[i + 1] ?? "").split(",").filter(Boolean);+ if (wanted.length === 0)+ throw new Error("usage: ci-tests.ts --list | --group <name>[,<name>…] [-- <bun test args>]");+ for (const name of wanted) {+ const files = groups.get(name);+ if (!files)+ throw new Error(`unknown group "${name}"; groups: ${[...groups.keys()].join(", ")}`);+ if (files.length === 0) throw new Error(`group "${name}" has no test files`);+ }+ const codes = await Promise.all(+ wanted.map((name) => run(name, groups.get(name) ?? [], bunArgs, wanted.length > 1))+ );+ process.exit(codes.some((c) => c !== 0) ? 1 : 0);+}