Skip to content

OpenAI's Codex Security CLI and TypeScript SDK for finding, validating, and fixing security vulnerabilities. npm: https://www.npmjs.com/package/@openai/codex-security

ci: balance MCP test shards by measured file duration (#1970)

Michael D'Angelod0e077370fb2
diff --git a/.github/workflows/node-ci.yml b/.github/workflows/node-ci.yml
index add9e828b..f88a9e593 100644
--- a/.github/workflows/node-ci.yml
+++ b/.github/workflows/node-ci.yml
@@ -293,11 +293,9 @@ jobs:
run: tar -xzf dist/*.tgz --strip-components=1 -C sdk/typescript package/_bundled_plugin
- name: Test MCP app
working-directory: plugins/codex-security/mcp-app
- # Keep Node options before the glob; trailing shard flags run every file.
run: >-
- node --experimental-strip-types --test --test-concurrency=2
- --test-shard=${{ matrix.shard }}/${{ strategy.job-total }}
- --test-reporter=./scripts/test_reporter.mts "tests/test_*.ts"
+ node --experimental-strip-types ../../../sdk/typescript/scripts/run-mcp-test-shard.mts
+ ${{ matrix.shard }} ${{ strategy.job-total }}
- name: Upload MCP test reports
if: always()
continue-on-error: true
diff --git a/docs/ci.md b/docs/ci.md
index c65614b23..7ce66538c 100644
--- a/docs/ci.md
+++ b/docs/ci.md
@@ -17,6 +17,11 @@ are discovered from the source tree, including new files without timing data.
Platform-specific filesystem, credential, process, cancellation, and resumed
worker cases remain enabled.
+The two Linux MCP jobs discover the same test files as local MCP testing. CI
+assigns files by observed runtimes and starts the longest files first on each
+job's two Node workers. Refresh the
+slow-file estimates in `run-mcp-test-shard.mts` when the jobs drift apart.
+
Pull requests limited to `.ts` files under `sdk/typescript/src/`,
`sdk/typescript/tests-ts/`, `plugins/codex-security/mcp-app/src/`, and
`plugins/codex-security/mcp-app/tests/`, optionally mixed with ordinary docs,
diff --git a/plugins/codex-security/mcp-app/scripts/test_reporter.mts b/plugins/codex-security/mcp-app/scripts/test_reporter.mts
index ec79aed40..730d02423 100644
--- a/plugins/codex-security/mcp-app/scripts/test_reporter.mts
+++ b/plugins/codex-security/mcp-app/scripts/test_reporter.mts
@@ -1,4 +1,5 @@
import { mkdir, writeFile } from "node:fs/promises";
+import { ReadableStream } from "node:stream/web";
import { junit, tap, type TestEvent } from "node:test/reporters";
export default async function* report(source: AsyncIterable<TestEvent>) {
diff --git a/sdk/typescript/TESTING.md b/sdk/typescript/TESTING.md
index 0eec2daed..71caaee73 100644
--- a/sdk/typescript/TESTING.md
+++ b/sdk/typescript/TESTING.md
@@ -156,9 +156,12 @@ Source qualification runs the full Bun suite once per OS under Node 22: six
file shards on Linux and macOS, and fourteen on Windows. GitHub runs the shards
as separate jobs; each job uses Bun's default serial runner. The other selected
Node versions run the installed-package checks instead of repeating the same
-Bun suite. MCP tests run in two required jobs using Node's native file sharding,
-with two isolated test file processes per job. Local and release `test:mcp`
-commands still run the full suite. Python tests run separately and use four
+Bun suite. MCP CI assigns files by measured duration and starts the longest
+files first, using two isolated Node test processes per job. From
+`plugins/codex-security/mcp-app`, replay shard 1
+with `node --experimental-strip-types ../../../sdk/typescript/scripts/run-mcp-test-shard.mts 1 2`.
+Local and release `test:mcp` commands run the full suite. Python tests run
+separately and use four
isolated pytest-xdist workers with work stealing; worker crashes fail the run
without automatic restarts.
diff --git a/sdk/typescript/scripts/run-mcp-test-shard.mts b/sdk/typescript/scripts/run-mcp-test-shard.mts
new file mode 100644
index 000000000..39938b4ee
--- /dev/null
+++ b/sdk/typescript/scripts/run-mcp-test-shard.mts
@@ -0,0 +1,73 @@
+import { globSync } from "node:fs";
+import { pipeline } from "node:stream/promises";
+import { run } from "node:test";
+import report from "../../../plugins/codex-security/mcp-app/scripts/test_reporter.mts";
+import { partitionTests } from "./test-shards.mts";
+
+// Rounded Linux JUnit seconds from main run 40152086428. The remaining
+// files had a median under one second; discovery also picks up new files.
+const slowFiles: Record<string, number> = {
+ "test_artifact_discovery.ts": 11,
+ "test_artifact_scan_draft_recovery.ts": 26,
+ "test_artifact_scan_identity.ts": 68,
+ "test_artifact_scan_publication_recovery.ts": 14,
+ "test_artifact_storage.ts": 19,
+ "test_artifact_storage_regressions.ts": 34,
+ "test_compact_artifact_server.ts": 84,
+ "test_coverage_provenance.ts": 60,
+ "test_deep_reducer_paging_eval.ts": 80,
+ "test_deep_scan_coordinator.ts": 18,
+ "test_deep_scan_executor.ts": 40,
+ "test_deep_scan_publication_integration.ts": 153,
+ "test_deep_scan_stdio_lifecycle.ts": 23,
+ "test_deep_scan_store_integration.ts": 51,
+ "test_deep_scan_terminal_race.ts": 22,
+ "test_mcp_app_smoke.ts": 45,
+ "test_retry_coverage_dispositions.ts": 96,
+ "test_retry_coverage_receipts_namespaces.ts": 135,
+ "test_retry_coverage_receipts_ownership.ts": 145,
+ "test_retry_coverage_receipts_shared.ts": 150,
+ "test_workbench_database.ts": 10,
+};
+
+const [shard, count] = process.argv.slice(2).map(Number);
+if (
+ !shard ||
+ !count ||
+ !Number.isInteger(shard) ||
+ !Number.isInteger(count) ||
+ shard < 1 ||
+ shard > count
+) {
+ throw new Error("Expected a shard number and count");
+}
+const files = globSync("tests/test_*.ts");
+const timings = Object.fromEntries(
+ files.map((file) => [
+ file,
+ { linux: slowFiles[file.slice("tests/".length)] ?? 1 },
+ ]),
+);
+const selected = partitionTests(files, count, timings, "linux")[shard - 1]!;
+if (!selected.length) throw new Error(`Empty MCP test shard ${shard}/${count}`);
+// Start long files while both workers are available. Equal estimates keep
+// the lexical order returned by the partitioner.
+selected.sort((left, right) => timings[right]!.linux - timings[left]!.linux);
+console.error(`MCP test shard ${shard}/${count}: ${selected.join(" ")}`);
+// Pass literal files: the Node CLI interprets explicit arguments as globs again.
+const controller = new AbortController();
+const cancel = () => {
+ process.exitCode = 1;
+ controller.abort();
+};
+process.once("SIGINT", cancel);
+process.once("SIGTERM", cancel);
+const events = run({
+ files: selected,
+ concurrency: 2,
+ signal: controller.signal,
+});
+events.on("test:summary", ({ success }) => {
+ if (!success) process.exitCode = 1;
+});
+await pipeline(events, report, process.stdout, { end: false });
diff --git a/sdk/typescript/tests-ts/test-shards.test.ts b/sdk/typescript/tests-ts/test-shards.test.ts
index d2c77dbff..0ff4129ec 100644
--- a/sdk/typescript/tests-ts/test-shards.test.ts
+++ b/sdk/typescript/tests-ts/test-shards.test.ts
@@ -348,3 +348,99 @@ test("isolated timeout", async () => {
await rm(directory, { recursive: true, force: true });
}
});
+
+function runMcpShard(root: string, shard: number, count: number) {
+ return readSubprocess(
+ Bun.spawn({
+ cmd: [
+ Bun.which("node")!,
+ "--experimental-strip-types",
+ join(import.meta.dir, "../scripts/run-mcp-test-shard.mts"),
+ String(shard),
+ String(count),
+ ],
+ cwd: root,
+ stdout: "pipe",
+ stderr: "pipe",
+ }),
+ );
+}
+
+test("MCP shards run each literal test filename once and propagate failures", async () => {
+ const root = await temporaryDirectory("mcp-shard-");
+ const tests = join(root, "tests");
+ await mkdir(tests);
+ const files = ["test_route_[ab].ts", "test_route_a.ts", "test_route_b.ts"];
+ try {
+ for (const file of files) {
+ await writeFile(
+ join(tests, file),
+ [
+ 'import { appendFileSync } from "node:fs";',
+ 'import { test } from "node:test";',
+ `test(${JSON.stringify(file)}, () => {`,
+ ` appendFileSync("executed.txt", ${JSON.stringify(file + "\n")});`,
+ file === files[0]
+ ? ' throw new Error("intentional test failure");'
+ : "",
+ "});",
+ ].join("\n"),
+ );
+ }
+ const statuses = [];
+ for (const shard of [1, 2]) {
+ const { status } = await runMcpShard(root, shard, 2);
+ statuses.push(status);
+ }
+ expect(statuses.sort()).toEqual([0, 1]);
+ expect(
+ (await readFile(join(root, "executed.txt"), "utf8"))
+ .trim()
+ .split("\n")
+ .sort(),
+ ).toEqual(files.sort());
+ } finally {
+ await rm(root, { recursive: true, force: true });
+ }
+});
+
+test.skipIf(process.platform !== "linux")(
+ "terminating an MCP shard stops its active file worker",
+ async () => {
+ const root = await temporaryDirectory("mcp-shard-cancel-");
+ const pid = Bun.file(join(root, "worker.pid"));
+ const stopped = Bun.file(join(root, "stopped"));
+ try {
+ await mkdir(join(root, "tests"));
+ await writeFile(
+ join(root, "tests/test_cancel.ts"),
+ `
+ import { writeFileSync } from "node:fs";
+ import { test } from "node:test";
+ import { setTimeout } from "node:timers/promises";
+ test("cancel the runner once this worker is ready", async () => {
+ process.once("SIGTERM", () => {
+ writeFileSync("stopped", "SIGTERM");
+ process.exit(0);
+ });
+ writeFileSync("worker.pid", String(process.pid));
+ process.kill(process.ppid, "SIGTERM");
+ await setTimeout(20_000);
+ });
+ `,
+ );
+ const { status } = await runMcpShard(root, 1, 1);
+ expect(await stopped.text()).toBe("SIGTERM");
+ expect(status).not.toBe(0);
+ } finally {
+ if (!(await stopped.exists()) && (await pid.exists())) {
+ try {
+ process.kill(Number(await pid.text()), "SIGKILL");
+ } catch (error) {
+ if ((error as NodeJS.ErrnoException).code !== "ESRCH") throw error;
+ }
+ }
+ await rm(root, { recursive: true, force: true });
+ }
+ },
+);
diff --git a/sdk/typescript/tsconfig.json b/sdk/typescript/tsconfig.json
index 641b4fe4d..66f3a4a52 100644
--- a/sdk/typescript/tsconfig.json
+++ b/sdk/typescript/tsconfig.json
@@ -8,6 +8,7 @@
"../../examples/custom-validation/*.mts",
"scripts/compare-test-reports.mts",
"scripts/run-ci-tests.mts",
+ "scripts/run-mcp-test-shard.mts",
"scripts/update-test-timings.mts"
],
"compilerOptions": {