OpenAI's Codex Security CLI and TypeScript SDK for finding, validating, and fixing security vulnerabilities. npm: https://www.npmjs.com/package/@openai/codex-security
TypeScript ai-securityapplication-securityclicode-scanningcodexcodex-securitycybersecuritydevsecopsnodejsnpmopenaisdksecuritytypescriptvulnerability-scanning
ci: balance MCP test shards by measured file duration (#1970)
Michael D'Angelod0e077370fb2
diff --git a/.github/workflows/node-ci.yml b/.github/workflows/node-ci.ymlindex add9e828b..f88a9e593 100644--- a/.github/workflows/node-ci.yml+++ b/.github/workflows/node-ci.yml@@ -293,11 +293,9 @@ jobs:run: tar -xzf dist/*.tgz --strip-components=1 -C sdk/typescript package/_bundled_plugin- name: Test MCP appworking-directory: plugins/codex-security/mcp-app- # Keep Node options before the glob; trailing shard flags run every file.run: >-- node --experimental-strip-types --test --test-concurrency=2- --test-shard=${{ matrix.shard }}/${{ strategy.job-total }}- --test-reporter=./scripts/test_reporter.mts "tests/test_*.ts"+ node --experimental-strip-types ../../../sdk/typescript/scripts/run-mcp-test-shard.mts+ ${{ matrix.shard }} ${{ strategy.job-total }}- name: Upload MCP test reportsif: always()continue-on-error: truediff --git a/docs/ci.md b/docs/ci.mdindex c65614b23..7ce66538c 100644--- a/docs/ci.md+++ b/docs/ci.md@@ -17,6 +17,11 @@ are discovered from the source tree, including new files without timing data.Platform-specific filesystem, credential, process, cancellation, and resumedworker cases remain enabled.+The two Linux MCP jobs discover the same test files as local MCP testing. CI+assigns files by observed runtimes and starts the longest files first on each+job's two Node workers. Refresh the+slow-file estimates in `run-mcp-test-shard.mts` when the jobs drift apart.+Pull requests limited to `.ts` files under `sdk/typescript/src/`,`sdk/typescript/tests-ts/`, `plugins/codex-security/mcp-app/src/`, and`plugins/codex-security/mcp-app/tests/`, optionally mixed with ordinary docs,diff --git a/plugins/codex-security/mcp-app/scripts/test_reporter.mts b/plugins/codex-security/mcp-app/scripts/test_reporter.mtsindex ec79aed40..730d02423 100644--- a/plugins/codex-security/mcp-app/scripts/test_reporter.mts+++ b/plugins/codex-security/mcp-app/scripts/test_reporter.mts@@ -1,4 +1,5 @@import { mkdir, writeFile } from "node:fs/promises";+import { ReadableStream } from "node:stream/web";import { junit, tap, type TestEvent } from "node:test/reporters";export default async function* report(source: AsyncIterable<TestEvent>) {diff --git a/sdk/typescript/TESTING.md b/sdk/typescript/TESTING.mdindex 0eec2daed..71caaee73 100644--- a/sdk/typescript/TESTING.md+++ b/sdk/typescript/TESTING.md@@ -156,9 +156,12 @@ Source qualification runs the full Bun suite once per OS under Node 22: sixfile shards on Linux and macOS, and fourteen on Windows. GitHub runs the shardsas separate jobs; each job uses Bun's default serial runner. The other selectedNode versions run the installed-package checks instead of repeating the same-Bun suite. MCP tests run in two required jobs using Node's native file sharding,-with two isolated test file processes per job. Local and release `test:mcp`-commands still run the full suite. Python tests run separately and use four+Bun suite. MCP CI assigns files by measured duration and starts the longest+files first, using two isolated Node test processes per job. From+`plugins/codex-security/mcp-app`, replay shard 1+with `node --experimental-strip-types ../../../sdk/typescript/scripts/run-mcp-test-shard.mts 1 2`.+Local and release `test:mcp` commands run the full suite. Python tests run+separately and use fourisolated pytest-xdist workers with work stealing; worker crashes fail the runwithout automatic restarts.diff --git a/sdk/typescript/scripts/run-mcp-test-shard.mts b/sdk/typescript/scripts/run-mcp-test-shard.mtsnew file mode 100644index 000000000..39938b4ee--- /dev/null+++ b/sdk/typescript/scripts/run-mcp-test-shard.mts@@ -0,0 +1,73 @@+import { globSync } from "node:fs";+import { pipeline } from "node:stream/promises";+import { run } from "node:test";+import report from "../../../plugins/codex-security/mcp-app/scripts/test_reporter.mts";+import { partitionTests } from "./test-shards.mts";++// Rounded Linux JUnit seconds from main run 40152086428. The remaining+// files had a median under one second; discovery also picks up new files.+const slowFiles: Record<string, number> = {+ "test_artifact_discovery.ts": 11,+ "test_artifact_scan_draft_recovery.ts": 26,+ "test_artifact_scan_identity.ts": 68,+ "test_artifact_scan_publication_recovery.ts": 14,+ "test_artifact_storage.ts": 19,+ "test_artifact_storage_regressions.ts": 34,+ "test_compact_artifact_server.ts": 84,+ "test_coverage_provenance.ts": 60,+ "test_deep_reducer_paging_eval.ts": 80,+ "test_deep_scan_coordinator.ts": 18,+ "test_deep_scan_executor.ts": 40,+ "test_deep_scan_publication_integration.ts": 153,+ "test_deep_scan_stdio_lifecycle.ts": 23,+ "test_deep_scan_store_integration.ts": 51,+ "test_deep_scan_terminal_race.ts": 22,+ "test_mcp_app_smoke.ts": 45,+ "test_retry_coverage_dispositions.ts": 96,+ "test_retry_coverage_receipts_namespaces.ts": 135,+ "test_retry_coverage_receipts_ownership.ts": 145,+ "test_retry_coverage_receipts_shared.ts": 150,+ "test_workbench_database.ts": 10,+};++const [shard, count] = process.argv.slice(2).map(Number);+if (+ !shard ||+ !count ||+ !Number.isInteger(shard) ||+ !Number.isInteger(count) ||+ shard < 1 ||+ shard > count+) {+ throw new Error("Expected a shard number and count");+}+const files = globSync("tests/test_*.ts");+const timings = Object.fromEntries(+ files.map((file) => [+ file,+ { linux: slowFiles[file.slice("tests/".length)] ?? 1 },+ ]),+);+const selected = partitionTests(files, count, timings, "linux")[shard - 1]!;+if (!selected.length) throw new Error(`Empty MCP test shard ${shard}/${count}`);+// Start long files while both workers are available. Equal estimates keep+// the lexical order returned by the partitioner.+selected.sort((left, right) => timings[right]!.linux - timings[left]!.linux);+console.error(`MCP test shard ${shard}/${count}: ${selected.join(" ")}`);+// Pass literal files: the Node CLI interprets explicit arguments as globs again.+const controller = new AbortController();+const cancel = () => {+ process.exitCode = 1;+ controller.abort();+};+process.once("SIGINT", cancel);+process.once("SIGTERM", cancel);+const events = run({+ files: selected,+ concurrency: 2,+ signal: controller.signal,+});+events.on("test:summary", ({ success }) => {+ if (!success) process.exitCode = 1;+});+await pipeline(events, report, process.stdout, { end: false });diff --git a/sdk/typescript/tests-ts/test-shards.test.ts b/sdk/typescript/tests-ts/test-shards.test.tsindex d2c77dbff..0ff4129ec 100644--- a/sdk/typescript/tests-ts/test-shards.test.ts+++ b/sdk/typescript/tests-ts/test-shards.test.ts@@ -348,3 +348,99 @@ test("isolated timeout", async () => {await rm(directory, { recursive: true, force: true });}});++function runMcpShard(root: string, shard: number, count: number) {+ return readSubprocess(+ Bun.spawn({+ cmd: [+ Bun.which("node")!,+ "--experimental-strip-types",+ join(import.meta.dir, "../scripts/run-mcp-test-shard.mts"),+ String(shard),+ String(count),+ ],+ cwd: root,+ stdout: "pipe",+ stderr: "pipe",+ }),+ );+}++test("MCP shards run each literal test filename once and propagate failures", async () => {+ const root = await temporaryDirectory("mcp-shard-");+ const tests = join(root, "tests");+ await mkdir(tests);+ const files = ["test_route_[ab].ts", "test_route_a.ts", "test_route_b.ts"];+ try {+ for (const file of files) {+ await writeFile(+ join(tests, file),+ [+ 'import { appendFileSync } from "node:fs";',+ 'import { test } from "node:test";',+ `test(${JSON.stringify(file)}, () => {`,+ ` appendFileSync("executed.txt", ${JSON.stringify(file + "\n")});`,+ file === files[0]+ ? ' throw new Error("intentional test failure");'+ : "",+ "});",+ ].join("\n"),+ );+ }+ const statuses = [];+ for (const shard of [1, 2]) {+ const { status } = await runMcpShard(root, shard, 2);+ statuses.push(status);+ }+ expect(statuses.sort()).toEqual([0, 1]);+ expect(+ (await readFile(join(root, "executed.txt"), "utf8"))+ .trim()+ .split("\n")+ .sort(),+ ).toEqual(files.sort());+ } finally {+ await rm(root, { recursive: true, force: true });+ }+});++test.skipIf(process.platform !== "linux")(+ "terminating an MCP shard stops its active file worker",+ async () => {+ const root = await temporaryDirectory("mcp-shard-cancel-");+ const pid = Bun.file(join(root, "worker.pid"));+ const stopped = Bun.file(join(root, "stopped"));+ try {+ await mkdir(join(root, "tests"));+ await writeFile(+ join(root, "tests/test_cancel.ts"),+ `+ import { writeFileSync } from "node:fs";+ import { test } from "node:test";+ import { setTimeout } from "node:timers/promises";+ test("cancel the runner once this worker is ready", async () => {+ process.once("SIGTERM", () => {+ writeFileSync("stopped", "SIGTERM");+ process.exit(0);+ });+ writeFileSync("worker.pid", String(process.pid));+ process.kill(process.ppid, "SIGTERM");+ await setTimeout(20_000);+ });+ `,+ );+ const { status } = await runMcpShard(root, 1, 1);+ expect(await stopped.text()).toBe("SIGTERM");+ expect(status).not.toBe(0);+ } finally {+ if (!(await stopped.exists()) && (await pid.exists())) {+ try {+ process.kill(Number(await pid.text()), "SIGKILL");+ } catch (error) {+ if ((error as NodeJS.ErrnoException).code !== "ESRCH") throw error;+ }+ }+ await rm(root, { recursive: true, force: true });+ }+ },+);diff --git a/sdk/typescript/tsconfig.json b/sdk/typescript/tsconfig.jsonindex 641b4fe4d..66f3a4a52 100644--- a/sdk/typescript/tsconfig.json+++ b/sdk/typescript/tsconfig.json@@ -8,6 +8,7 @@"../../examples/custom-validation/*.mts","scripts/compare-test-reports.mts","scripts/run-ci-tests.mts",+ "scripts/run-mcp-test-shard.mts","scripts/update-test-timings.mts"],"compilerOptions": {