Skip to content

OpenAI's Codex Security CLI and TypeScript SDK for finding, validating, and fixing security vulnerabilities. npm: https://www.npmjs.com/package/@openai/codex-security

ci(tests): use Bun native sharding (#1479)

mldangelo-oaia7dfc3c5c14b
diff --git a/sdk/typescript/TESTING.md b/sdk/typescript/TESTING.md
index ea985d934..4a6d08d2a 100644
--- a/sdk/typescript/TESTING.md
+++ b/sdk/typescript/TESTING.md
@@ -120,11 +120,10 @@ package checks instead of repeating the same Bun suite. MCP and Python tests
run in separate required jobs. Python uses four isolated pytest-xdist workers
with work stealing; worker crashes fail the run without automatic restarts.
-`scripts/run-ci-tests.mts` assigns the longest measured files first. Its
-`ci-test-durations.json` records per-file seconds from CI reports.
-Every new test file is included automatically with a one-second estimate.
-Refresh those estimates from the uploaded reports when adding or splitting
-expensive files; estimates affect scheduling, never whether a test runs.
+`scripts/run-ci-tests.mts` discovers every shared test file and passes the
+inventory to Bun's native `--shard` option. Bun sorts the file paths and assigns
+them round-robin across the shards. New files are included automatically;
+there is no timing table to update when adding or splitting tests.
To reproduce one Windows shard locally after building the plugin, run
`node --experimental-strip-types scripts/run-ci-tests.mts 3/7 --seed=12345`.
@@ -149,9 +148,6 @@ that breaks the Ink UI tests under isolation. Keep the trial pin until a newer
release passes the full SDK suite in every mode. Required CI and the mutation
trial remain on Bun 1.3.14.
-Keep the measured file-balanced runner until the native runner has
-matching inventories and acceptable Windows timings. Before promotion, compare
-native and file-balanced shards using the same commit and Bun version.
Keep the machine-policy test serial. Do not replace the full required suite
with `--changed`: Python files, schemas, fixtures, and workflows loaded at
runtime are not necessarily part of Bun's import graph.
diff --git a/sdk/typescript/scripts/ci-test-durations.json b/sdk/typescript/scripts/ci-test-durations.json
deleted file mode 100644
index 58c03dd5b..000000000
--- a/sdk/typescript/scripts/ci-test-durations.json
+++ /dev/null
@@ -1,348 +0,0 @@
-{
- "sources": [
- "https://github.com/openai/codex-security/actions/runs/33215429477",
- "https://github.com/openai/codex-security/actions/runs/33227512538",
- "https://github.com/openai/codex-security/actions/runs/33233690024",
- "https://github.com/openai/codex-security/actions/runs/37420072376",
- "https://github.com/openai/codex-security/actions/runs/37472306800",
- "https://github.com/openai/codex-security/actions/runs/37782856773"
- ],
- "unix": {
- "api-attribution-concurrency.test.ts": 1.552,
- "api-credentials.test.ts": 3.703,
- "api-cyber-access.test.ts": 0.12,
- "api-environment.test.ts": 0.0,
- "api-events.test.ts": 9.883,
- "api-integration.test.ts": 0.0,
- "api-policy.test.ts": 40.924,
- "api-post-scan.test.ts": 22.923,
- "api-preflight-config.test.ts": 2.059,
- "api.test.ts": 50.539,
- "artifact-export.test.ts": 2.289,
- "attack-path-tool-name.test.ts": 0.305,
- "auth.test.ts": 2.604,
- "azure-pipelines-example.test.ts": 0.137,
- "bind-repo-scopes.test.ts": 4.878,
- "build-plugin.test.ts": 6.903,
- "bulk-scan-discovery.test.ts": 0.034,
- "candidate-normalizer.test.ts": 15.489,
- "check-markdown.test.ts": 2.334,
- "classify-scan-severity.test.ts": 25.587,
- "classify-severity.test.ts": 0.055,
- "cli-authentication.test.ts": 12.265,
- "cli-classify-severity.test.ts": 0.036,
- "cli-cloud-publish.test.ts": 0.261,
- "cli-dedupe.test.ts": 0.125,
- "cli-deep-scan-summary.test.ts": 0.055,
- "cli-diagnostics.test.ts": 0.153,
- "cli-export.test.ts": 9.881,
- "cli-feedback.test.ts": 3.787,
- "cli-help.test.ts": 0.192,
- "cli-history-paths.test.ts": 0.0,
- "cli-launcher.test.ts": 1.887,
- "cli-patch-results.test.ts": 2.697,
- "cli-patch-trust.test.ts": 2.484,
- "cli-patch.test.ts": 5.118,
- "cli-policy.test.ts": 10.654,
- "cli-project-config.test.ts": 0.766,
- "cli-publish.test.ts": 0.427,
- "cli-scan-import.test.ts": 0.096,
- "cli-scan-logs.test.ts": 10.883,
- "cli-scan-patch-settings.test.ts": 0.32,
- "cli-scan-prompts.test.ts": 0.503,
- "cli-serve.test.ts": 11.424,
- "cli-show-cost.test.ts": 0.12,
- "cli-signals.test.ts": 0.075,
- "cli-skill-chatgpt-provider-auth.test.ts": 2.381,
- "cli-skill-native-config.test.ts": 7.617,
- "cli-skills.test.ts": 4.946,
- "cli-suggest-owners.test.ts": 0.338,
- "cli-verify-fix.test.ts": 0.109,
- "cli-workbench.test.ts": 0.276,
- "cli.test.ts": 3.987,
- "cloud-publish.test.ts": 2.566,
- "codex-review.test.ts": 4.444,
- "compact-diff-scan.test.ts": 26.361,
- "completed-scan-handoff.test.ts": 0.011,
- "component-scan.test.ts": 16.369,
- "config.test.ts": 3.029,
- "container-ci-workflow.test.ts": 0.3,
- "container-entrypoint.test.ts": 0.256,
- "contract.test.ts": 12.909,
- "cost-context.test.ts": 0.011,
- "cost-model.property.test.ts": 0.02,
- "cost.test.ts": 1.231,
- "custom-publish.test.ts": 0.594,
- "custom-validation-example.test.ts": 0.835,
- "custom-validation.test.ts": 14.506,
- "dedupe-records.test.ts": 1.457,
- "deep-config.test.ts": 0.04,
- "deep-progress.test.ts": 0.002,
- "deep-review-input.test.ts": 15.767,
- "deep-scan-parent-denials.test.ts": 0.048,
- "deep-scan-reducer-recovery.test.ts": 0.407,
- "deep-scan-windows-executable.test.ts": 0.033,
- "deep-scan-workbench.test.ts": 27.622,
- "deep-scan-worker-shutdown.test.ts": 0.019,
- "diff-rank-input.test.ts": 1.378,
- "effort-forwarding.test.ts": 0.342,
- "errors.property.test.ts": 0.004,
- "errors.test.ts": 0.002,
- "feedback.test.ts": 0.461,
- "finding-catalogue.test.ts": 21.247,
- "finding-deduplication.test.ts": 6.609,
- "finding-embeddings.test.ts": 3.056,
- "finding-preview.test.ts": 0.399,
- "finding-workflow-integration.test.ts": 131.052,
- "finding-workflow-publication.test.ts": 10.601,
- "finding-workflow.test.ts": 1.61,
- "findings-client-retry.test.ts": 0.004,
- "findings-dashboard.test.ts": 0.001,
- "findings-import.test.ts": 1.76,
- "findings-server.test.ts": 47.498,
- "github-actions-example.test.ts": 0.08,
- "github.test.ts": 0.082,
- "import-scan.test.ts": 11.019,
- "invoice-desk-workflow.test.ts": 0.811,
- "knowledge-base.test.ts": 1.885,
- "linear-comment-pagination.test.ts": 0.002,
- "linear-pagination.test.ts": 0.004,
- "linear.test.ts": 0.002,
- "mcp-launcher.test.ts": 1.17,
- "mock-scan.test.ts": 18.596,
- "multiscan.test.ts": 29.727,
- "package-provenance.test.ts": 0.001,
- "package-public-content.test.ts": 0.009,
- "package-smoke-timeouts.test.ts": 0.0,
- "package-tar-listing.test.ts": 0.0,
- "patch-risk-contract.test.ts": 19.551,
- "patch-tui.test.ts": 2.839,
- "plugin-apps.test.ts": 0.006,
- "plugin-finding-detail-contract.test.ts": 3.533,
- "plugin-report-limits.test.ts": 0.756,
- "project-config.test.ts": 6.125,
- "prompt-only-start-timeout.test.ts": 0.019,
- "publication-check.test.ts": 0.005,
- "publication-events.test.ts": 0.003,
- "publication-integration.test.ts": 26.289,
- "publication-store.test.ts": 27.125,
- "publication.test.ts": 1.542,
- "publish.test.ts": 1.957,
- "published-install-smoke.test.ts": 0.019,
- "rank-pool.test.ts": 35.165,
- "rank-shards.test.ts": 24.268,
- "release-automation.test.ts": 18.603,
- "release-cut-workflow.test.ts": 0.0,
- "release-pr.test.ts": 36.676,
- "repository-findings.test.ts": 0.245,
- "result.test.ts": 0.008,
- "runtime-plugin-reuse.test.ts": 0.764,
- "runtime.test.ts": 45.386,
- "sandbox-probe.test.ts": 0.066,
- "sarif.test.ts": 0.89,
- "scan-activity.test.ts": 0.005,
- "scan-comparison.property.test.ts": 0.562,
- "scan-comparison.test.ts": 2.817,
- "scan-dashboard.test.ts": 0.103,
- "scan-history-renderer.test.ts": 0.011,
- "scan-logs.test.ts": 0.173,
- "scan-matching-e2e.test.ts": 8.781,
- "scan-recipe.test.ts": 2.5,
- "scan-recovery.test.ts": 36.092,
- "scan-resume.test.ts": 102.599,
- "script-entrypoints.test.ts": 3.564,
- "sdk-project-config.test.ts": 0.5,
- "sdk-scan-prompts.test.ts": 0.738,
- "security-policy-helper.test.ts": 6.905,
- "security-policy.test.ts": 23.727,
- "shell.test.ts": 0.082,
- "skeleton.test.ts": 0.08,
- "stopped-scan-results.test.ts": 15.197,
- "suggest-owners.test.ts": 3.669,
- "targets-large-output.test.ts": 1.285,
- "targets.test.ts": 3.605,
- "test-reports.test.ts": 4.004,
- "test-shards.test.ts": 2.342,
- "trusted-executable.test.ts": 0.356,
- "update-notice.test.ts": 0.125,
- "workbench-canonical-paths.test.ts": 0.873,
- "workbench-nested-git-utf8.test.ts": 0.154,
- "workbench-progress.test.ts": 1.183,
- "workbench-scan-history.test.ts": 3.351,
- "workbench-scan-root-alias.test.ts": 0.0,
- "workbench-windows-compatibility.test.ts": 0.939,
- "worker-progress.property.test.ts": 0.018,
- "worker-progress.test.ts": 0.005,
- "workflow-quality.test.ts": 0.197
- },
- "windows": {
- "api-attribution-concurrency.test.ts": 27.364,
- "api-credentials.test.ts": 76.223,
- "api-cyber-access.test.ts": 0.442,
- "api-environment.test.ts": 0.0,
- "api-events.test.ts": 17.652,
- "api-integration.test.ts": 0.0,
- "api-policy.test.ts": 101.385,
- "api-post-scan.test.ts": 22.571,
- "api-preflight-config.test.ts": 1.602,
- "api-provider-isolation.test.ts": 320.294,
- "api.test.ts": 137.276,
- "artifact-export.test.ts": 2.514,
- "attack-path-tool-name.test.ts": 0.283,
- "auth.test.ts": 3.266,
- "azure-pipelines-example.test.ts": 0.721,
- "bind-repo-scopes.test.ts": 13.787,
- "build-plugin.test.ts": 7.354,
- "bulk-scan-discovery.test.ts": 0.066,
- "candidate-normalizer.test.ts": 13.829,
- "check-markdown.test.ts": 2.666,
- "classify-scan-severity.test.ts": 38.077,
- "classify-severity.test.ts": 0.073,
- "cli-authentication.test.ts": 227.941,
- "cli-classify-severity.test.ts": 0.106,
- "cli-cloud-publish.test.ts": 0.413,
- "cli-dedupe.test.ts": 0.1,
- "cli-deep-scan-summary.test.ts": 0.054,
- "cli-diagnostics.test.ts": 0.1,
- "cli-export.test.ts": 15.871,
- "cli-feedback.test.ts": 2.578,
- "cli-help.test.ts": 0.198,
- "cli-history-paths.test.ts": 0.009,
- "cli-launcher.test.ts": 1.319,
- "cli-patch-results.test.ts": 5.409,
- "cli-patch-trust.test.ts": 9.679,
- "cli-patch.test.ts": 21.549,
- "cli-policy.test.ts": 22.256,
- "cli-project-config.test.ts": 4.89,
- "cli-publish.test.ts": 0.402,
- "cli-scan-import.test.ts": 0.102,
- "cli-scan-logs.test.ts": 7.695,
- "cli-scan-patch-settings.test.ts": 0.77,
- "cli-scan-prompts.test.ts": 0.901,
- "cli-serve.test.ts": 13.64,
- "cli-show-cost.test.ts": 0.085,
- "cli-signals.test.ts": 0.037,
- "cli-skill-chatgpt-provider-auth.test.ts": 269.629,
- "cli-skill-native-config.test.ts": 49.542,
- "cli-skills.test.ts": 3.198,
- "cli-suggest-owners.test.ts": 0.419,
- "cli-verify-fix.test.ts": 0.193,
- "cli-workbench.test.ts": 0.232,
- "cli.test.ts": 7.41,
- "cloud-publish.test.ts": 7.371,
- "codex-review.test.ts": 12.619,
- "compact-diff-scan.test.ts": 38.57,
- "completed-scan-handoff.test.ts": 0.004,
- "component-scan.test.ts": 19.591,
- "config.test.ts": 7.622,
- "container-ci-workflow.test.ts": 1.191,
- "container-entrypoint.test.ts": 0.0,
- "contract.test.ts": 10.574,
- "cost-context.test.ts": 0.003,
- "cost-model.property.test.ts": 0.021,
- "cost.test.ts": 1.515,
- "custom-publish.test.ts": 0.581,
- "custom-validation-example.test.ts": 0.444,
- "custom-validation.test.ts": 23.262,
- "dedupe-records.test.ts": 0.861,
- "deep-config.test.ts": 0.492,
- "deep-progress.test.ts": 0.005,
- "deep-review-input.test.ts": 15.492,
- "deep-scan-parent-denials.test.ts": 0.034,
- "deep-scan-reducer-recovery.test.ts": 0.544,
- "deep-scan-windows-executable.test.ts": 0.033,
- "deep-scan-workbench.test.ts": 29.766,
- "deep-scan-worker-shutdown.test.ts": 0.024,
- "diff-rank-input.test.ts": 2.272,
- "effort-forwarding.test.ts": 0.172,
- "errors.property.test.ts": 0.005,
- "errors.test.ts": 0.001,
- "feedback.test.ts": 1.226,
- "finding-catalogue.test.ts": 11.43,
- "finding-deduplication.test.ts": 5.169,
- "finding-embeddings.test.ts": 2.258,
- "finding-preview.test.ts": 0.284,
- "finding-workflow-integration.test.ts": 166.129,
- "finding-workflow-publication.test.ts": 14.351,
- "finding-workflow.test.ts": 4.138,
- "findings-client-retry.test.ts": 0.003,
- "findings-dashboard.test.ts": 0.001,
- "findings-import.test.ts": 1.886,
- "findings-server.test.ts": 58.875,
- "github-actions-example.test.ts": 0.327,
- "github.test.ts": 0.05,
- "import-scan.test.ts": 15.313,
- "invoice-desk-workflow.test.ts": 1.181,
- "knowledge-base.test.ts": 7.043,
- "linear-comment-pagination.test.ts": 0.002,
- "linear-pagination.test.ts": 0.003,
- "linear.test.ts": 0.003,
- "local-dedupe.test.ts": 120.007,
- "mcp-launcher.test.ts": 3.656,
- "mock-scan.test.ts": 28.689,
- "multiscan.test.ts": 71.201,
- "package-provenance.test.ts": 0.001,
- "package-public-content.test.ts": 0.003,
- "package-smoke-timeouts.test.ts": 0.0,
- "package-tar-listing.test.ts": 0.0,
- "patch-risk-contract.test.ts": 38.98,
- "patch-tui.test.ts": 4.659,
- "plugin-apps.test.ts": 0.001,
- "plugin-finding-detail-contract.test.ts": 3.659,
- "plugin-report-limits.test.ts": 2.249,
- "project-config.test.ts": 2.719,
- "prompt-only-start-timeout.test.ts": 0.012,
- "publication-check.test.ts": 0.008,
- "publication-events.test.ts": 0.01,
- "publication-integration.test.ts": 24.468,
- "publication-store.test.ts": 33.269,
- "publication.test.ts": 1.771,
- "publish.test.ts": 1.622,
- "published-install-smoke.test.ts": 0.068,
- "rank-pool.test.ts": 32.82,
- "rank-shards.test.ts": 15.229,
- "release-automation.test.ts": 35.575,
- "release-cut-workflow.test.ts": 0.0,
- "release-pr.test.ts": 91.813,
- "repository-findings.test.ts": 0.359,
- "result.test.ts": 0.006,
- "runtime-plugin-reuse.test.ts": 0.846,
- "runtime.test.ts": 157.186,
- "sandbox-probe.test.ts": 0.001,
- "sarif.test.ts": 1.437,
- "scan-activity.test.ts": 0.007,
- "scan-comparison.property.test.ts": 0.65,
- "scan-comparison.test.ts": 8.199,
- "scan-dashboard.test.ts": 0.096,
- "scan-history-renderer.test.ts": 0.006,
- "scan-logs.test.ts": 0.35,
- "scan-matching-e2e.test.ts": 16.789,
- "scan-recipe.test.ts": 5.424,
- "scan-recovery.test.ts": 76.061,
- "scan-resume.test.ts": 151.317,
- "script-entrypoints.test.ts": 3.367,
- "sdk-project-config.test.ts": 0.939,
- "sdk-scan-prompts.test.ts": 1.712,
- "security-policy-helper.test.ts": 11.885,
- "security-policy.test.ts": 63.859,
- "shell.test.ts": 0.087,
- "skeleton.test.ts": 0.098,
- "stopped-scan-results.test.ts": 17.338,
- "suggest-owners.test.ts": 23.573,
- "targets-large-output.test.ts": 8.455,
- "targets.test.ts": 8.794,
- "test-reports.test.ts": 6.265,
- "test-shards.test.ts": 3.448,
- "trusted-executable.test.ts": 0.091,
- "update-notice.test.ts": 0.034,
- "workbench-canonical-paths.test.ts": 4.027,
- "workbench-nested-git-utf8.test.ts": 1.221,
- "workbench-progress.test.ts": 0.807,
- "workbench-scan-history.test.ts": 4.08,
- "workbench-scan-root-alias.test.ts": 0.298,
- "workbench-windows-compatibility.test.ts": 1.935,
- "worker-progress.property.test.ts": 0.013,
- "worker-progress.test.ts": 0.003,
- "workflow-quality.test.ts": 0.0
- }
-}
diff --git a/sdk/typescript/scripts/run-ci-tests.mts b/sdk/typescript/scripts/run-ci-tests.mts
index 9060bc2b8..73ff9a706 100644
--- a/sdk/typescript/scripts/run-ci-tests.mts
+++ b/sdk/typescript/scripts/run-ci-tests.mts
@@ -1,7 +1,6 @@
import { spawn } from "node:child_process";
-import { mkdir, readFile, readdir } from "node:fs/promises";
+import { mkdir, readdir } from "node:fs/promises";
import { parseArgs } from "node:util";
-import { shardTestFiles } from "./test-shards.mts";
const selection = /^([1-9]\d*)\/([1-9]\d*)$/.exec(process.argv[2] ?? "");
const shard = Number(selection?.[1]);
@@ -11,27 +10,21 @@ if (!Number.isSafeInteger(count) || shard > count) {
"Usage: node --experimental-strip-types scripts/run-ci-tests.mts <shard>/<count> [bun test options]",
);
}
-const tests = (await readdir(new URL("../tests-ts/", import.meta.url))).filter(
- (file) =>
- file.endsWith(".test.ts") && file !== "windows-machine-policy.test.ts",
-);
-const timings = JSON.parse(
- await readFile(new URL("./ci-test-durations.json", import.meta.url), "utf8"),
-);
-const windows = process.platform === "win32";
-const defaultTestTimeoutMs = windows ? "120000" : "30000";
-const assignments = shardTestFiles(
- tests,
- count,
- timings[windows ? "windows" : "unix"],
-);
-const selected = assignments[shard - 1]!;
-if (selected.files.length === 0) {
+const tests = (await readdir(new URL("../tests-ts/", import.meta.url)))
+ .filter(
+ (file) =>
+ file.endsWith(".test.ts") && file !== "windows-machine-policy.test.ts",
+ )
+ .sort();
+// Match Bun 1.3.14's sorted round-robin assignment for startup diagnostics.
+// https://github.com/oven-sh/bun/blob/bun-v1.3.14/src/cli/test_command.zig
+const selected = tests.filter((_, index) => index % count === shard - 1);
+if (selected.length === 0) {
throw new Error(`Test shard ${shard}/${count} is empty.`);
}
-console.log(
- `Test shard ${shard}/${count} (estimated ${selected.seconds.toFixed(1)}s): ${selected.files.join(" ")}`,
-);
+const windows = process.platform === "win32";
+const defaultTestTimeoutMs = windows ? "120000" : "30000";
+console.log(`Test shard ${shard}/${count}: ${selected.join(" ")}`);
await mkdir(new URL("../reports/", import.meta.url), { recursive: true }).catch(
() => {
// Bun warns about report write failures without changing the test result.
@@ -39,12 +32,13 @@ await mkdir(new URL("../reports/", import.meta.url), { recursive: true }).catch(
);
const testArguments = [
"test",
+ `--shard=${shard}/${count}`,
"--timeout",
defaultTestTimeoutMs,
"--reporter=junit",
`--reporter-outfile=reports/junit-${shard}.xml`,
...process.argv.slice(3),
- ...selected.files.map((file) => `./tests-ts/${file}`),
+ ...tests.map((file) => `./tests-ts/${file}`),
];
// Keep arities aligned with the pinned Bun's test_params:
// https://github.com/oven-sh/bun/blob/bun-v1.3.14/src/cli/Arguments.zig
diff --git a/sdk/typescript/scripts/test-shards.mts b/sdk/typescript/scripts/test-shards.mts
deleted file mode 100644
index bf6db5906..000000000
--- a/sdk/typescript/scripts/test-shards.mts
+++ /dev/null
@@ -1,25 +0,0 @@
-/** Place the longest measured files first, without omitting new test files. */
-export function shardTestFiles(
- files: readonly string[],
- count: number,
- durations: Readonly<Record<string, number>> = {},
-) {
- const shards = Array.from({ length: count }, () => ({
- files: [] as string[],
- seconds: 0,
- }));
- const duration = (file: string) => durations[file] ?? 1;
- const ordered = [...files].sort(
- (left, right) =>
- duration(right) - duration(left) ||
- (left < right ? -1 : left > right ? 1 : 0),
- );
- for (const file of ordered) {
- const shard = shards.reduce((shortest, candidate) =>
- candidate.seconds < shortest.seconds ? candidate : shortest,
- );
- shard.files.push(file);
- shard.seconds += duration(file);
- }
- return shards;
-}
diff --git a/sdk/typescript/tests-ts/test-shards.test.ts b/sdk/typescript/tests-ts/test-shards.test.ts
index c2f4cbf86..ebf27282c 100644
--- a/sdk/typescript/tests-ts/test-shards.test.ts
+++ b/sdk/typescript/tests-ts/test-shards.test.ts
@@ -2,7 +2,6 @@ import {
copyFile,
mkdir,
mkdtemp,
- readdir,
readFile,
rm,
writeFile,
@@ -10,55 +9,127 @@ import {
import { tmpdir } from "node:os";
import { delimiter, dirname, join } from "node:path";
import { expect, test } from "bun:test";
-import { shardTestFiles } from "../scripts/test-shards.mts";
import { readSubprocess } from "./support/shell.js";
import { temporaryDirectory } from "./support/temporary-directories.js";
-test("balances measured work and includes new files without mutating the inventory", () => {
- const files = [
- "new.test.ts",
- "small.test.ts",
- "medium.test.ts",
- "slow.test.ts",
- ];
- const shards = shardTestFiles(files, 2, {
- "slow.test.ts": 100,
- "medium.test.ts": 70,
- "small.test.ts": 30,
- });
- expect(shards.map(({ seconds }) => seconds)).toEqual([101, 100]);
- expect(shards.flatMap(({ files }) => files).sort()).toEqual(
- [...files].sort(),
- );
- expect(files[0]).toBe("new.test.ts");
- expect(
- shardTestFiles([...files].reverse(), 2, {
- "slow.test.ts": 100,
- "medium.test.ts": 70,
- "small.test.ts": 30,
- }),
- ).toEqual(shards);
-});
+test.each([3, 7])(
+ "runs each eligible file once across %i native shards, including new files",
+ async (count) => {
+ const node = Bun.which("node");
+ expect(node).not.toBeNull();
+ const root = await temporaryDirectory("codex-security-native-shards-");
+ try {
+ await mkdir(join(root, "scripts"));
+ await mkdir(join(root, "tests-ts"));
+ await copyFile(
+ new URL("../scripts/run-ci-tests.mts", import.meta.url),
+ join(root, "scripts", "run-ci-tests.mts"),
+ );
+ const files = [
+ ...Array.from({ length: 20 }, (_, index) => `probe-${index}.test.ts`),
+ "newly-added.test.ts",
+ ];
+ const executionLog = join(root, "executed.txt");
+ for (const file of files) {
+ await writeFile(
+ join(root, "tests-ts", file),
+ `import { appendFileSync } from "node:fs";
+import { test } from "bun:test";
+test(${JSON.stringify(file)}, () => {
+ appendFileSync(${JSON.stringify(executionLog)}, ${JSON.stringify(`${file}\n`)});
+});\n`,
+ );
+ }
+ await writeFile(
+ join(root, "tests-ts", "windows-machine-policy.test.ts"),
+ 'throw new Error("Machine policy test must run separately");\n',
+ );
+ const executed: string[] = [];
+ for (let shard = 1; shard <= count; shard++) {
+ await writeFile(executionLog, "");
+ const child = Bun.spawn({
+ cmd: [
+ node!,
+ "--experimental-strip-types",
+ join(root, "scripts", "run-ci-tests.mts"),
+ `${shard}/${count}`,
+ ],
+ env: {
+ ...process.env,
+ PATH: `${dirname(process.execPath)}${delimiter}${process.env["PATH"] ?? ""}`,
+ },
+ stdin: "ignore",
+ stdout: "pipe",
+ stderr: "pipe",
+ timeout: 30_000,
+ windowsHide: true,
+ });
+ const { status, stdout, stderr } = await readSubprocess(child);
+ expect(status, stderr).toBe(0);
+ const shardFiles = (await readFile(executionLog, "utf8"))
+ .trim()
+ .split("\n");
+ expect(stdout).toContain(
+ `Test shard ${shard}/${count}: ${shardFiles.sort().join(" ")}`,
+ );
+ executed.push(...shardFiles);
+ }
+ expect(executed.sort()).toEqual(files.sort());
+ } finally {
+ await rm(root, { recursive: true, force: true });
+ }
+ },
+);
-test.each([
- ["unix", 3],
- ["windows", 7],
-] as const)("covers every shared test once on %s", async (platform, count) => {
- const tests = (await readdir(new URL("./", import.meta.url)))
- .filter(
- (file) =>
- file.endsWith(".test.ts") && file !== "windows-machine-policy.test.ts",
- )
- .sort();
- const timings = JSON.parse(
- await readFile(
- new URL("../scripts/ci-test-durations.json", import.meta.url),
- "utf8",
- ),
- );
- const shards = shardTestFiles(tests, count, timings[platform]);
- expect(shards.every(({ files }) => files.length > 0)).toBe(true);
- expect(shards.flatMap(({ files }) => files).sort()).toEqual(tests);
+test("rejects an empty shard while allowing a nonempty shard with the same count", async () => {
+ const node = Bun.which("node");
+ expect(node).not.toBeNull();
+ const root = await temporaryDirectory("codex-security-empty-shard-");
+ try {
+ await mkdir(join(root, "scripts"));
+ await mkdir(join(root, "tests-ts"));
+ await copyFile(
+ new URL("../scripts/run-ci-tests.mts", import.meta.url),
+ join(root, "scripts", "run-ci-tests.mts"),
+ );
+ await writeFile(
+ join(root, "tests-ts", "probe.test.ts"),
+ 'import { test } from "bun:test"; test("synthetic shard probe", () => {});\n',
+ );
+ await writeFile(
+ join(root, "tests-ts", "windows-machine-policy.test.ts"),
+ 'throw new Error("Machine policy test must run separately");\n',
+ );
+ for (const shard of [1, 2]) {
+ const child = Bun.spawn({
+ cmd: [
+ node!,
+ "--experimental-strip-types",
+ join(root, "scripts", "run-ci-tests.mts"),
+ `${shard}/2`,
+ ],
+ env: {
+ ...process.env,
+ PATH: `${dirname(process.execPath)}${delimiter}${process.env["PATH"] ?? ""}`,
+ },
+ stdin: "ignore",
+ stdout: "pipe",
+ stderr: "pipe",
+ timeout: 30_000,
+ windowsHide: true,
+ });
+ const { status, stdout, stderr } = await readSubprocess(child);
+ expect(status, stderr).toBe(shard === 1 ? 0 : 1);
+ if (shard === 1) {
+ expect(stdout).toContain("Test shard 1/2: probe.test.ts");
+ expect(stderr).toContain("synthetic shard probe");
+ } else {
+ expect(stderr).toContain("Test shard 2/2 is empty.");
+ }
+ }
+ } finally {
+ await rm(root, { recursive: true, force: true });
+ }
});
const defaultTimeoutMs = process.platform === "win32" ? "120000" : "30000";
@@ -96,16 +167,10 @@ test.each([
try {
await mkdir(join(root, "scripts"));
await mkdir(join(root, "tests-ts"));
- for (const file of [
- "run-ci-tests.mts",
- "test-shards.mts",
- "ci-test-durations.json",
- ]) {
- await copyFile(
- new URL(`../scripts/${file}`, import.meta.url),
- join(root, "scripts", file),
- );
- }
+ await copyFile(
+ new URL("../scripts/run-ci-tests.mts", import.meta.url),
+ join(root, "scripts", "run-ci-tests.mts"),
+ );
await writeFile(
join(root, "tests-ts", "probe.test.ts"),
`import { expect, test } from "bun:test";
@@ -137,7 +202,7 @@ test("synthetic report probe --timeout=5000", () => {
});
const { status, stdout, stderr } = await readSubprocess(child);
expect(status, stderr).toBe(fail ? 1 : 0);
- expect(stdout).toContain("probe.test.ts");
+ expect(stdout).toContain("Test shard 1/1: probe.test.ts");
expect(stderr).toContain("synthetic report probe");
if (report === "available") {
const xml = await readFile(