Skip to content

Claude Code. Any Model. The most powerful AI coding agent now speaks every language.

ci: one list of hermetic test files in named groups, and a Linux/lanes measurement

Jack Rudenko8bee644b30f8
diff --git a/.github/actions/install-magmux/action.yml b/.github/actions/install-magmux/action.yml
new file mode 100644
index 00000000..64c33495
--- /dev/null
+++ b/.github/actions/install-magmux/action.yml
@@ -0,0 +1,26 @@
+name: Install the pinned magmux
+description: >
+ Installs the magmux release pinned in .github/magmux.env (the one release.yml ships) to
+ /usr/local/bin, after checking its sha256 against the pin. A path findMagmuxForTest() and the
+ launcher both search; nothing earlier in that search exists on a runner.
+inputs:
+ token:
+ description: GitHub token for gh release download
+ required: true
+runs:
+ using: composite
+ steps:
+ - shell: bash
+ env:
+ GH_TOKEN: ${{ inputs.token }}
+ run: |
+ set -a; . .github/magmux.env; set +a
+ OS=$(uname -s | tr '[:upper:]' '[:lower:]')
+ ARCH=$(uname -m); case "$ARCH" in (arm64|aarch64) ARCH=arm64;; (x86_64) ARCH=amd64;; esac
+ ASSET="magmux_${OS}_${ARCH}.tar.gz"
+ WANT=$(eval echo "\$MAGMUX_SHA256_${OS}_${ARCH}")
+ gh release download "${MAGMUX_TAG}" --repo MadAppGang/magmux --pattern "${ASSET}" --dir /tmp
+ echo "${WANT} /tmp/${ASSET}" | shasum -a 256 -c -
+ tar xzf "/tmp/${ASSET}" -C /tmp
+ sudo install -m 0755 /tmp/magmux /usr/local/bin/magmux
+ magmux --version | grep -q "${MAGMUX_TAG#v}" || { echo "installed magmux is not ${MAGMUX_TAG}"; magmux --version; exit 1; }
diff --git a/.github/workflows/test-experiment.yml b/.github/workflows/test-experiment.yml
new file mode 100644
index 00000000..a6dd578f
--- /dev/null
+++ b/.github/workflows/test-experiment.yml
@@ -0,0 +1,73 @@
+name: Tests experiment
+
+# MEASUREMENT ONLY, deleted before merge. Answers two questions for the hermetic-job split:
+# 1. Which test files pass on Linux runners (20 concurrent jobs on the free plan, against 5 macOS)?
+# 2. How long each group takes when the three groups run concurrently on ONE macOS runner.
+# Both write junit, so per-file and per-test times come from the reports, not log timestamps.
+
+on:
+ pull_request:
+ branches: [main]
+
+jobs:
+ linux-probe:
+ name: linux probe · ${{ matrix.group }}
+ if: startsWith(github.head_ref, 'ci/hermetic-under-4min')
+ strategy:
+ fail-fast: false
+ matrix:
+ group: [pane, idle, rest]
+ runs-on: ubuntu-latest
+ env:
+ FORCE_JAVASCRIPT_ACTIONS_TO_NODE24: true
+ steps:
+ - uses: actions/checkout@v5
+ - uses: oven-sh/setup-bun@v2
+ with:
+ bun-version: "1.3.10"
+ - run: bun install --frozen-lockfile
+ - uses: ./.github/actions/install-magmux
+ with:
+ token: ${{ github.token }}
+ - name: Tests (${{ matrix.group }})
+ working-directory: packages/cli
+ continue-on-error: true
+ run: bun scripts/ci-tests.ts --group ${{ matrix.group }} -- --reporter=junit --reporter-outfile=junit-{group}.xml
+ timeout-minutes: 40
+ env:
+ CLAUDISH_SKIP_LIVE_E2E: "1"
+ - uses: actions/upload-artifact@v5
+ if: always()
+ with:
+ name: junit-linux-${{ matrix.group }}
+ path: packages/cli/junit-*.xml
+
+ mac-lanes:
+ name: macOS lanes · pane + idle + rest concurrently
+ if: startsWith(github.head_ref, 'ci/hermetic-under-4min')
+ runs-on: macos-latest
+ env:
+ FORCE_JAVASCRIPT_ACTIONS_TO_NODE24: true
+ steps:
+ - uses: actions/checkout@v5
+ - uses: oven-sh/setup-bun@v2
+ with:
+ bun-version: "1.3.10"
+ - run: bun install --frozen-lockfile
+ - uses: ./.github/actions/install-magmux
+ with:
+ token: ${{ github.token }}
+ - name: sysctl
+ run: sysctl -n hw.ncpu hw.memsize machdep.cpu.brand_string
+ - name: Tests (three groups at once)
+ working-directory: packages/cli
+ continue-on-error: true
+ run: bun scripts/ci-tests.ts --group pane,idle,rest -- --reporter=junit --reporter-outfile=junit-{group}.xml
+ timeout-minutes: 40
+ env:
+ CLAUDISH_SKIP_LIVE_E2E: "1"
+ - uses: actions/upload-artifact@v5
+ if: always()
+ with:
+ name: junit-mac-lanes
+ path: packages/cli/junit-*.xml
diff --git a/packages/cli/scripts/ci-tests.ts b/packages/cli/scripts/ci-tests.ts
new file mode 100644
index 00000000..6d550127
--- /dev/null
+++ b/packages/cli/scripts/ci-tests.ts
@@ -0,0 +1,129 @@
+/**
+ * The ONE list of the hermetic test files, split into named groups, and the runner for them.
+ *
+ * bun scripts/ci-tests.ts --list
+ * bun scripts/ci-tests.ts --group pane,idle [-- <bun test args>]
+ *
+ * Several groups run as concurrent `bun test` processes, each line prefixed with its group.
+ * `{group}` in a passed-through argument becomes the group name, so each process can write its
+ * own `--reporter-outfile`. The exit code is non-zero when any group fails.
+ *
+ * Files are passed WITHOUT a leading `./`: Bun then reads them as filters on its own discovery
+ * and keeps a bare `bun test`'s file order. `./` paths run in argument order instead, and a few
+ * suites share process state across files, so a new order is a new suite.
+ *
+ * src/hooks/blind is not here: it runs under Bun 1.4.0 in its own job (test.yml).
+ */
+import { join } from "node:path";
+
+const CLI = join(import.meta.dir, "..");
+
+/** Bun's own test-file name rule. */
+const TEST_FILE = /(\.test|_test_|\.spec|_spec_)[^/]*\.[cm]?[jt]sx?$/;
+const EXCLUDED = ["node_modules/", "src/hooks/blind/"];
+
+/**
+ * Each file belongs to the first group whose prefix it starts with; `rest` takes the others.
+ * - idle: CPU-idle waits on real retry ladders and token-refresh ceilings, with no magmux.
+ * - pane: real magmux panes and the MCP session machinery on top of them.
+ */
+const GROUPS: { name: string; prefixes: string[] }[] = [
+ { name: "idle", prefixes: ["src/recovery-contract/", "src/providers/grok/grok-refresh-timeout"] },
+ {
+ name: "pane",
+ prefixes: [
+ "src/pane/",
+ "src/mcp-session/",
+ "src/mcp-blackbox/",
+ "src/hooks/session/",
+ "src/recovery/",
+ "src/team",
+ ],
+ },
+];
+const REST = "rest";
+
+export function discover(): string[] {
+ const files: string[] = [];
+ for (const path of new Bun.Glob("**/*").scanSync({ cwd: CLI, onlyFiles: true })) {
+ if (EXCLUDED.some((dir) => path.startsWith(dir) || path.includes(`/${dir}`))) continue;
+ if (TEST_FILE.test(path)) files.push(path);
+ }
+ return files.sort();
+}
+
+export function groupOf(file: string): string {
+ return GROUPS.find((g) => g.prefixes.some((p) => file.startsWith(p)))?.name ?? REST;
+}
+
+function byGroup(files: string[]): Map<string, string[]> {
+ const out = new Map<string, string[]>([
+ ...GROUPS.map((g) => [g.name, []] as [string, string[]]),
+ [REST, []],
+ ]);
+ for (const f of files) out.get(groupOf(f))?.push(f);
+ return out;
+}
+
+async function pipe(stream: ReadableStream<Uint8Array>, prefix: string, sink: NodeJS.WriteStream) {
+ const decoder = new TextDecoder();
+ let carry = "";
+ for await (const chunk of stream) {
+ const lines = (carry + decoder.decode(chunk, { stream: true })).split("\n");
+ carry = lines.pop() ?? "";
+ for (const line of lines) sink.write(`${prefix}${line}\n`);
+ }
+ if (carry) sink.write(`${prefix}${carry}\n`);
+}
+
+async function run(
+ group: string,
+ files: string[],
+ bunArgs: string[],
+ prefixed: boolean
+): Promise<number> {
+ const args = bunArgs.map((a) => a.replaceAll("{group}", group));
+ const started = performance.now();
+ const proc = Bun.spawn([process.execPath, "test", ...args, ...files], {
+ cwd: CLI,
+ stdout: prefixed ? "pipe" : "inherit",
+ stderr: prefixed ? "pipe" : "inherit",
+ });
+ if (prefixed) {
+ await Promise.all([
+ pipe(proc.stdout as ReadableStream<Uint8Array>, `[${group}] `, process.stdout),
+ pipe(proc.stderr as ReadableStream<Uint8Array>, `[${group}] `, process.stderr),
+ ]);
+ }
+ const code = await proc.exited;
+ const secs = ((performance.now() - started) / 1000).toFixed(0);
+ console.log(`[ci-tests] ${group}: ${files.length} files, exit ${code}, ${secs} s`);
+ return code;
+}
+
+if (import.meta.main) {
+ const argv = process.argv.slice(2);
+ const dash = argv.indexOf("--");
+ const own = dash === -1 ? argv : argv.slice(0, dash);
+ const bunArgs = dash === -1 ? [] : argv.slice(dash + 1);
+ const groups = byGroup(discover());
+
+ if (own.includes("--list")) {
+ for (const [name, files] of groups) console.log(`${name}\t${files.length}`);
+ process.exit(0);
+ }
+ const i = own.indexOf("--group");
+ const wanted = i === -1 ? [] : (own[i + 1] ?? "").split(",").filter(Boolean);
+ if (wanted.length === 0)
+ throw new Error("usage: ci-tests.ts --list | --group <name>[,<name>…] [-- <bun test args>]");
+ for (const name of wanted) {
+ const files = groups.get(name);
+ if (!files)
+ throw new Error(`unknown group "${name}"; groups: ${[...groups.keys()].join(", ")}`);
+ if (files.length === 0) throw new Error(`group "${name}" has no test files`);
+ }
+ const codes = await Promise.all(
+ wanted.map((name) => run(name, groups.get(name) ?? [], bunArgs, wanted.length > 1))
+ );
+ process.exit(codes.some((c) => c !== 0) ? 1 : 0);
+}