Skip to content

OpenAI's Codex Security CLI and TypeScript SDK for finding, validating, and fixing security vulnerabilities. npm: https://www.npmjs.com/package/@openai/codex-security

test(ci): balance test shards by recorded duration (#1856)

Michael D'Angelo8b248f00c185
diff --git a/sdk/typescript/TESTING.md b/sdk/typescript/TESTING.md
index 44566ac03..1c3ccdf51 100644
--- a/sdk/typescript/TESTING.md
+++ b/sdk/typescript/TESTING.md
@@ -147,13 +147,26 @@ package checks instead of repeating the same Bun suite. MCP and Python tests
run in separate required jobs. Python uses four isolated pytest-xdist workers
with work stealing; worker crashes fail the run without automatic restarts.
-`scripts/run-ci-tests.mts` discovers every shared test file and passes the
-inventory to Bun's native `--shard` option. Bun sorts the file paths and assigns
-them round-robin across the shards. New files are included automatically;
-there is no timing table to update when adding or splitting tests.
+`scripts/run-ci-tests.mts` discovers every shared test file and balances whole
+files using the platform's recorded durations in `scripts/test-timings.json`.
+It assigns longer files first to the shard with the least estimated work, then
+passes each shard's files directly to Bun. Every job uses the same checked-in
+snapshot. New files are included automatically with the mean known-file duration;
+deleted files in the timing snapshot are ignored. Timing hints never select or
+exclude tests, and adding or splitting tests does not require updating them.
To reproduce one Windows shard locally after building the plugin, run
`node --experimental-strip-types scripts/run-ci-tests.mts 3/7 --seed=12345`.
+Refresh timing hints from successful CI JUnit artifacts when shard runtimes
+become uneven. From the SDK directory, pass reports for one platform to
+`node --experimental-strip-types scripts/update-test-timings.mts win32 <junit.xml> ...`
+(use `linux` or `darwin` for the other platforms). The updater sums testcase
+durations per file and uses their median across reports, preserving the other
+platforms. Review and commit the generated snapshot. The initial hints use runs
+`39521539826`, `39694506447`, and `39694510011`. Keep runner versions, coverage,
+timeouts, seeds, and platform-specific checks unchanged when comparing scheduling
+changes; compare complete JUnit inventories and outcomes as well as wall time.
+
Every Bun lane uploads JUnit; Linux lanes also upload LCOV per shard. Python
reports include case durations, and the MCP runner can upload its JUnit report.
Coverage is diagnostic and split across shards, not a combined percentage.
diff --git a/sdk/typescript/scripts/run-ci-tests.mts b/sdk/typescript/scripts/run-ci-tests.mts
index 73ff9a706..dba9b5bbb 100644
--- a/sdk/typescript/scripts/run-ci-tests.mts
+++ b/sdk/typescript/scripts/run-ci-tests.mts
@@ -1,6 +1,7 @@
import { spawn } from "node:child_process";
-import { mkdir, readdir } from "node:fs/promises";
+import { mkdir, readFile, readdir } from "node:fs/promises";
import { parseArgs } from "node:util";
+import { partitionTests, type TestTimings } from "./test-shards.mts";
const selection = /^([1-9]\d*)\/([1-9]\d*)$/.exec(process.argv[2] ?? "");
const shard = Number(selection?.[1]);
@@ -16,9 +17,14 @@ const tests = (await readdir(new URL("../tests-ts/", import.meta.url)))
file.endsWith(".test.ts") && file !== "windows-machine-policy.test.ts",
)
.sort();
-// Match Bun 1.3.14's sorted round-robin assignment for startup diagnostics.
-// https://github.com/oven-sh/bun/blob/bun-v1.3.14/src/cli/test_command.zig
-const selected = tests.filter((_, index) => index % count === shard - 1);
+// The checked-in snapshot keeps every job in this run on the same assignment.
+// Discovery remains authoritative, including files without timing history.
+const timings = JSON.parse(
+ await readFile(new URL("./test-timings.json", import.meta.url), "utf8"),
+) as TestTimings;
+const selected = partitionTests(tests, count, timings, process.platform)[
+ shard - 1
+]!;
if (selected.length === 0) {
throw new Error(`Test shard ${shard}/${count} is empty.`);
}
@@ -32,13 +38,12 @@ await mkdir(new URL("../reports/", import.meta.url), { recursive: true }).catch(
);
const testArguments = [
"test",
- `--shard=${shard}/${count}`,
"--timeout",
defaultTestTimeoutMs,
"--reporter=junit",
`--reporter-outfile=reports/junit-${shard}.xml`,
...process.argv.slice(3),
- ...tests.map((file) => `./tests-ts/${file}`),
+ ...selected.map((file) => `./tests-ts/${file}`),
];
// Keep arities aligned with the pinned Bun's test_params:
// https://github.com/oven-sh/bun/blob/bun-v1.3.14/src/cli/Arguments.zig
diff --git a/sdk/typescript/scripts/test-shards.mts b/sdk/typescript/scripts/test-shards.mts
new file mode 100644
index 000000000..9628e28e5
--- /dev/null
+++ b/sdk/typescript/scripts/test-shards.mts
@@ -0,0 +1,34 @@
+export type TestTimings = Record<
+ string,
+ Partial<Record<NodeJS.Platform, number>>
+>;
+
+export function partitionTests(
+ files: readonly string[],
+ count: number,
+ timings: TestTimings,
+ platform: NodeJS.Platform,
+): string[][] {
+ const known = files.flatMap((file) => {
+ const seconds = timings[file]?.[platform];
+ return seconds === undefined ? [] : [seconds];
+ });
+ const fallback = known.length
+ ? known.reduce((total, seconds) => total + seconds, 0) / known.length
+ : 1;
+ const duration = (file: string) => timings[file]?.[platform] ?? fallback;
+ const shards = Array.from({ length: count }, () => ({
+ files: [] as string[],
+ seconds: 0,
+ }));
+ for (const file of [...files]
+ .sort()
+ .sort((left, right) => duration(right) - duration(left))) {
+ const shard = shards.reduce((least, candidate) =>
+ candidate.seconds < least.seconds ? candidate : least,
+ );
+ shard.files.push(file);
+ shard.seconds += duration(file);
+ }
+ return shards.map((shard) => shard.files.sort());
+}
diff --git a/sdk/typescript/scripts/test-timings.json b/sdk/typescript/scripts/test-timings.json
new file mode 100644
index 000000000..1c280766e
--- /dev/null
+++ b/sdk/typescript/scripts/test-timings.json
@@ -0,0 +1,1072 @@
+{
+ "action-release-workflow.test.ts": {
+ "linux": 2.9,
+ "darwin": 10.7,
+ "win32": 53.4
+ },
+ "api-attribution-concurrency.test.ts": {
+ "linux": 1.9,
+ "darwin": 5.5,
+ "win32": 134.7
+ },
+ "api-audit-admission.test.ts": {
+ "linux": 0.2,
+ "darwin": 0.2,
+ "win32": 0.6
+ },
+ "api-credentials.test.ts": {
+ "linux": 2.0,
+ "darwin": 4.5,
+ "win32": 142.5
+ },
+ "api-cyber-access.test.ts": {
+ "linux": 0.1,
+ "darwin": 0.3,
+ "win32": 1.7
+ },
+ "api-deep-scan-advisory-routing.test.ts": {
+ "linux": 0.2,
+ "darwin": 0.1,
+ "win32": 4.4
+ },
+ "api-deep-scan-runtime-routing.test.ts": {
+ "linux": 0.1,
+ "darwin": 0.1,
+ "win32": 0.6
+ },
+ "api-direct-deep-scan.test.ts": {
+ "linux": 0.5,
+ "darwin": 1.7,
+ "win32": 12.6
+ },
+ "api-environment.test.ts": {
+ "linux": 0.1,
+ "darwin": 0.1,
+ "win32": 0.1
+ },
+ "api-events.test.ts": {
+ "linux": 6.3,
+ "darwin": 11.2,
+ "win32": 17.9
+ },
+ "api-integration.test.ts": {
+ "linux": 0.1,
+ "darwin": 0.1,
+ "win32": 0.1
+ },
+ "api-login.test.ts": {
+ "linux": 0.1,
+ "darwin": 0.1,
+ "win32": 2.7
+ },
+ "api-operation-registration.test.ts": {
+ "linux": 0.1,
+ "darwin": 0.1,
+ "win32": 0.1
+ },
+ "api-patch.test.ts": {
+ "linux": 0.2,
+ "darwin": 0.4,
+ "win32": 6.7
+ },
+ "api-policy.test.ts": {
+ "linux": 25.3,
+ "darwin": 26.8,
+ "win32": 191.6
+ },
+ "api-post-scan.test.ts": {
+ "linux": 10.2,
+ "darwin": 12.8,
+ "win32": 25.5
+ },
+ "api-preflight-config.test.ts": {
+ "linux": 0.1,
+ "darwin": 0.2,
+ "win32": 0.9
+ },
+ "api-provider-isolation.test.ts": {
+ "linux": 3.6,
+ "darwin": 5.4,
+ "win32": 150.2
+ },
+ "api-runtime-storage.test.ts": {
+ "linux": 0.1,
+ "darwin": 0.3,
+ "win32": 62.8
+ },
+ "api-time-budget.test.ts": {
+ "linux": 0.5,
+ "darwin": 0.7,
+ "win32": 2.1
+ },
+ "api-usage-finalization.test.ts": {
+ "linux": 0.4,
+ "darwin": 0.4,
+ "win32": 1.9
+ },
+ "api-usage.test.ts": {
+ "linux": 0.7,
+ "darwin": 1.1,
+ "win32": 2.1
+ },
+ "api.test.ts": {
+ "linux": 62.1,
+ "darwin": 116.1,
+ "win32": 445.9
+ },
+ "artifact-export.test.ts": {
+ "linux": 3.5,
+ "darwin": 3.5,
+ "win32": 2.5
+ },
+ "attack-path-tool-name.test.ts": {
+ "linux": 0.1,
+ "darwin": 0.2,
+ "win32": 0.2
+ },
+ "auth-status-concurrency.test.ts": {
+ "linux": 1.5,
+ "darwin": 2.8,
+ "win32": 62.8
+ },
+ "auth.test.ts": {
+ "linux": 1.9,
+ "darwin": 2.6,
+ "win32": 3.7
+ },
+ "azure-pipelines-example.test.ts": {
+ "linux": 0.1,
+ "darwin": 0.1,
+ "win32": 0.7
+ },
+ "bind-repo-scopes.test.ts": {
+ "linux": 3.0,
+ "darwin": 4.5,
+ "win32": 9.5
+ },
+ "build-plugin.test.ts": {
+ "linux": 10.6,
+ "darwin": 8.7,
+ "win32": 15.9
+ },
+ "bulk-scan-discovery.test.ts": {
+ "linux": 0.1,
+ "darwin": 0.1,
+ "win32": 0.1
+ },
+ "candidate-normalizer.test.ts": {
+ "linux": 11.3,
+ "darwin": 19.4,
+ "win32": 17.3
+ },
+ "check-markdown.test.ts": {
+ "linux": 2.3,
+ "darwin": 2.4,
+ "win32": 4.7
+ },
+ "classify-scan-severity.test.ts": {
+ "linux": 33.0,
+ "darwin": 48.0,
+ "win32": 74.4
+ },
+ "classify-severity-records.test.ts": {
+ "linux": 0.1,
+ "darwin": 0.1,
+ "win32": 0.1
+ },
+ "classify-severity.test.ts": {
+ "linux": 0.1,
+ "darwin": 0.1,
+ "win32": 0.1
+ },
+ "cli-authentication.test.ts": {
+ "linux": 12.3,
+ "darwin": 17.2,
+ "win32": 308.7
+ },
+ "cli-classify-severity.test.ts": {
+ "linux": 0.1,
+ "darwin": 0.1,
+ "win32": 0.2
+ },
+ "cli-cloud-publish.test.ts": {
+ "linux": 1.5,
+ "darwin": 0.7,
+ "win32": 1.9
+ },
+ "cli-daybreak.test.ts": {
+ "linux": 0.1,
+ "darwin": 0.1,
+ "win32": 0.1
+ },
+ "cli-dedupe.test.ts": {
+ "linux": 0.2,
+ "darwin": 0.2,
+ "win32": 0.3
+ },
+ "cli-deep-scan-summary.test.ts": {
+ "linux": 0.1,
+ "darwin": 0.1,
+ "win32": 0.1
+ },
+ "cli-diagnostics.test.ts": {
+ "linux": 0.4,
+ "darwin": 0.4,
+ "win32": 0.5
+ },
+ "cli-error-diagnostics.test.ts": {
+ "linux": 0.4,
+ "darwin": 0.7,
+ "win32": 1.0
+ },
+ "cli-export.test.ts": {
+ "linux": 7.6,
+ "darwin": 6.2,
+ "win32": 16.0
+ },
+ "cli-feedback.test.ts": {
+ "linux": 1.3,
+ "darwin": 2.4,
+ "win32": 4.4
+ },
+ "cli-help.test.ts": {
+ "linux": 0.2,
+ "darwin": 0.3,
+ "win32": 0.4
+ },
+ "cli-history-paths.test.ts": {
+ "linux": 0.1,
+ "darwin": 0.1,
+ "win32": 0.1
+ },
+ "cli-launcher.test.ts": {
+ "linux": 1.5,
+ "darwin": 2.2,
+ "win32": 2.0
+ },
+ "cli-patch-results.test.ts": {
+ "linux": 3.2,
+ "darwin": 8.7,
+ "win32": 89.9
+ },
+ "cli-patch-trust.test.ts": {
+ "linux": 1.0,
+ "darwin": 2.4,
+ "win32": 4.7
+ },
+ "cli-patch.test.ts": {
+ "linux": 2.7,
+ "darwin": 8.9,
+ "win32": 23.3
+ },
+ "cli-policy.test.ts": {
+ "linux": 4.6,
+ "darwin": 5.9,
+ "win32": 23.6
+ },
+ "cli-project-config.test.ts": {
+ "linux": 0.9,
+ "darwin": 1.7,
+ "win32": 4.2
+ },
+ "cli-publish.test.ts": {
+ "linux": 1.1,
+ "darwin": 1.8,
+ "win32": 1.7
+ },
+ "cli-saved-finding-evidence.test.ts": {
+ "linux": 6.2,
+ "darwin": 6.7,
+ "win32": 14.7
+ },
+ "cli-scan-failures.test.ts": {
+ "linux": 1.1,
+ "darwin": 0.8,
+ "win32": 1.6
+ },
+ "cli-scan-import.test.ts": {
+ "linux": 0.2,
+ "darwin": 0.3,
+ "win32": 0.3
+ },
+ "cli-scan-logs.test.ts": {
+ "linux": 6.4,
+ "darwin": 5.7,
+ "win32": 8.8
+ },
+ "cli-scan-navigation.test.ts": {
+ "linux": 2.8,
+ "darwin": 2.7,
+ "win32": 5.9
+ },
+ "cli-scan-patch-settings.test.ts": {
+ "linux": 1.5,
+ "darwin": 1.9,
+ "win32": 3.4
+ },
+ "cli-scan-prompts.test.ts": {
+ "linux": 0.3,
+ "darwin": 0.4,
+ "win32": 1.1
+ },
+ "cli-show-cost.test.ts": {
+ "linux": 0.3,
+ "darwin": 0.3,
+ "win32": 0.5
+ },
+ "cli-signals.test.ts": {
+ "linux": 0.1,
+ "darwin": 0.1,
+ "win32": 0.1
+ },
+ "cli-skill-chatgpt-provider-auth.test.ts": {
+ "linux": 2.3,
+ "darwin": 2.7,
+ "win32": 271.3
+ },
+ "cli-skill-native-config.test.ts": {
+ "linux": 16.3,
+ "darwin": 27.2,
+ "win32": 223.9
+ },
+ "cli-skills.test.ts": {
+ "linux": 4.3,
+ "darwin": 5.2,
+ "win32": 5.1
+ },
+ "cli-suggest-owners.test.ts": {
+ "linux": 0.3,
+ "darwin": 0.2,
+ "win32": 0.4
+ },
+ "cli-verify-fix.test.ts": {
+ "linux": 0.4,
+ "darwin": 0.2,
+ "win32": 0.7
+ },
+ "cli-workbench.test.ts": {
+ "linux": 0.6,
+ "darwin": 0.6,
+ "win32": 1.1
+ },
+ "cli.test.ts": {
+ "linux": 5.8,
+ "darwin": 5.9,
+ "win32": 13.0
+ },
+ "cloud-publish.test.ts": {
+ "linux": 1.8,
+ "darwin": 1.6,
+ "win32": 6.9
+ },
+ "codeowners.test.ts": {
+ "linux": 0.1,
+ "darwin": 0.3,
+ "win32": 0.2
+ },
+ "codex-process-cancellation.test.ts": {
+ "linux": 2.2,
+ "darwin": 3.8,
+ "win32": 3.0
+ },
+ "codex-review.test.ts": {
+ "linux": 4.7,
+ "darwin": 6.7,
+ "win32": 12.7
+ },
+ "compact-diff-scan.test.ts": {
+ "linux": 12.7,
+ "darwin": 29.1,
+ "win32": 47.2
+ },
+ "completed-scan-handoff.test.ts": {
+ "linux": 0.1,
+ "darwin": 0.1,
+ "win32": 0.1
+ },
+ "component-scan.test.ts": {
+ "linux": 13.2,
+ "darwin": 17.2,
+ "win32": 88.6
+ },
+ "config-approval-policy.test.ts": {
+ "linux": 0.1,
+ "darwin": 0.1,
+ "win32": 0.1
+ },
+ "config.test.ts": {
+ "linux": 0.5,
+ "darwin": 3.3,
+ "win32": 27.1
+ },
+ "container-ci-workflow.test.ts": {
+ "linux": 0.1,
+ "darwin": 0.2,
+ "win32": 2.0
+ },
+ "container-entrypoint.test.ts": {
+ "linux": 0.1,
+ "darwin": 0.5,
+ "win32": 0.1
+ },
+ "container-release-workflow.test.ts": {
+ "linux": 0.5,
+ "darwin": 2.4,
+ "win32": 14.6
+ },
+ "contract.test.ts": {
+ "linux": 12.2,
+ "darwin": 11.2,
+ "win32": 21.5
+ },
+ "cost-context.test.ts": {
+ "linux": 0.1,
+ "darwin": 0.1,
+ "win32": 0.1
+ },
+ "cost-model.property.test.ts": {
+ "linux": 0.1,
+ "darwin": 0.1,
+ "win32": 0.1
+ },
+ "cost.test.ts": {
+ "linux": 1.3,
+ "darwin": 1.5,
+ "win32": 2.1
+ },
+ "custom-publish.test.ts": {
+ "linux": 0.4,
+ "darwin": 0.5,
+ "win32": 0.7
+ },
+ "custom-validation-example.test.ts": {
+ "linux": 0.2,
+ "darwin": 0.6,
+ "win32": 0.3
+ },
+ "custom-validation.test.ts": {
+ "linux": 16.1,
+ "darwin": 19.0,
+ "win32": 44.7
+ },
+ "dedupe-records.test.ts": {
+ "linux": 0.7,
+ "darwin": 1.2,
+ "win32": 1.4
+ },
+ "deduplication-retry.test.ts": {
+ "linux": 0.2,
+ "darwin": 0.3,
+ "win32": 0.9
+ },
+ "deep-config.test.ts": {
+ "linux": 0.1,
+ "darwin": 0.1,
+ "win32": 0.2
+ },
+ "deep-progress.test.ts": {
+ "linux": 0.1,
+ "darwin": 0.1,
+ "win32": 0.1
+ },
+ "deep-review-input.test.ts": {
+ "linux": 6.2,
+ "darwin": 9.4,
+ "win32": 14.6
+ },
+ "deep-scan-parent-denials.test.ts": {
+ "linux": 0.1,
+ "darwin": 0.1,
+ "win32": 0.1
+ },
+ "deep-scan-reducer-recovery.test.ts": {
+ "linux": 0.3,
+ "darwin": 0.4,
+ "win32": 0.5
+ },
+ "deep-scan-windows-executable.test.ts": {
+ "linux": 0.1,
+ "darwin": 0.1,
+ "win32": 0.1
+ },
+ "deep-scan-workbench.test.ts": {
+ "linux": 13.8,
+ "darwin": 19.2,
+ "win32": 38.0
+ },
+ "deep-scan-worker-shutdown.test.ts": {
+ "linux": 32.1,
+ "darwin": 72.1,
+ "win32": 33.8
+ },
+ "devcontainer-cooldown.test.ts": {
+ "linux": 0.1,
+ "darwin": 0.1,
+ "win32": 0.1
+ },
+ "diff-rank-input.test.ts": {
+ "linux": 0.8,
+ "darwin": 1.7,
+ "win32": 3.4
+ },
+ "direct-deep-scan.test.ts": {
+ "linux": 0.1,
+ "darwin": 0.4,
+ "win32": 10.5
+ },
+ "effort-forwarding.test.ts": {
+ "linux": 0.1,
+ "darwin": 0.2,
+ "win32": 0.2
+ },
+ "errors.property.test.ts": {
+ "linux": 0.1,
+ "darwin": 0.1,
+ "win32": 0.1
+ },
+ "errors.test.ts": {
+ "linux": 0.1,
+ "darwin": 0.1,
+ "win32": 0.1
+ },
+ "external-findings-publish.test.ts": {
+ "linux": 6.0,
+ "darwin": 10.5,
+ "win32": 25.7
+ },
+ "feedback.test.ts": {
+ "linux": 0.4,
+ "darwin": 0.5,
+ "win32": 1.8
+ },
+ "finding-catalogue.test.ts": {
+ "linux": 21.4,
+ "darwin": 10.3,
+ "win32": 11.7
+ },
+ "finding-deduplication.test.ts": {
+ "linux": 6.2,
+ "darwin": 5.6,
+ "win32": 6.5
+ },
+ "finding-embeddings.test.ts": {
+ "linux": 2.6,
+ "darwin": 2.5,
+ "win32": 3.3
+ },
+ "finding-preview.test.ts": {
+ "linux": 8.9,
+ "darwin": 16.1,
+ "win32": 50.4
+ },
+ "finding-workflow-integration.test.ts": {
+ "linux": 76.6,
+ "darwin": 151.4,
+ "win32": 163.6
+ },
+ "finding-workflow-publication.test.ts": {
+ "linux": 11.1,
+ "darwin": 10.2,
+ "win32": 14.3
+ },
+ "finding-workflow.test.ts": {
+ "linux": 0.6,
+ "darwin": 0.9,
+ "win32": 4.6
+ },
+ "findings-client-retry.test.ts": {
+ "linux": 0.1,
+ "darwin": 0.1,
+ "win32": 0.1
+ },
+ "findings-import-literal-fields.test.ts": {
+ "linux": 0.1,
+ "darwin": 0.1,
+ "win32": 0.1
+ },
+ "findings-import-record-endings.test.ts": {
+ "linux": 0.2,
+ "darwin": 0.2,
+ "win32": 0.6
+ },
+ "findings-import.test.ts": {
+ "linux": 1.7,
+ "darwin": 3.2,
+ "win32": 3.6
+ },
+ "findings-store.test.ts": {
+ "linux": 9.0,
+ "darwin": 12.8,
+ "win32": 16.0
+ },
+ "git-remote.test.ts": {
+ "linux": 0.1,
+ "darwin": 0.1,
+ "win32": 0.1
+ },
+ "github-actions-example.test.ts": {
+ "linux": 0.1,
+ "darwin": 0.1,
+ "win32": 0.4
+ },
+ "github.test.ts": {
+ "linux": 0.2,
+ "darwin": 0.4,
+ "win32": 0.4
+ },
+ "host-protocol-node.test.ts": {
+ "linux": 7.5,
+ "darwin": 6.2,
+ "win32": 12.8
+ },
+ "hosted-remediation.test.ts": {
+ "linux": 1.2,
+ "darwin": 3.3,
+ "win32": 7.4
+ },
+ "hosted-scan.test.ts": {
+ "linux": 39.7,
+ "darwin": 76.2,
+ "win32": 174.1
+ },
+ "import-scan.test.ts": {
+ "linux": 30.5,
+ "darwin": 34.4,
+ "win32": 55.5
+ },
+ "invoice-desk-workflow.test.ts": {
+ "linux": 0.2,
+ "darwin": 0.5,
+ "win32": 1.4
+ },
+ "knowledge-base.test.ts": {
+ "linux": 1.7,
+ "darwin": 2.0,
+ "win32": 8.6
+ },
+ "linear-comment-pagination.test.ts": {
+ "linux": 0.1,
+ "darwin": 0.1,
+ "win32": 0.1
+ },
+ "linear-pagination.test.ts": {
+ "linux": 0.1,
+ "darwin": 0.1,
+ "win32": 0.1
+ },
+ "linear.test.ts": {
+ "linux": 0.1,
+ "darwin": 0.1,
+ "win32": 0.1
+ },
+ "local-dedupe.test.ts": {
+ "linux": 71.6,
+ "darwin": 75.9,
+ "win32": 143.2
+ },
+ "mcp-launcher.test.ts": {
+ "linux": 1.3,
+ "darwin": 2.6,
+ "win32": 7.6
+ },
+ "mock-scan.test.ts": {
+ "linux": 10.8,
+ "darwin": 17.6,
+ "win32": 44.1
+ },
+ "multiscan.test.ts": {
+ "linux": 42.9,
+ "darwin": 67.3,
+ "win32": 216.6
+ },
+ "package-provenance.test.ts": {
+ "linux": 0.1,
+ "darwin": 0.1,
+ "win32": 0.1
+ },
+ "package-public-content.test.ts": {
+ "linux": 0.9,
+ "darwin": 1.2,
+ "win32": 1.1
+ },
+ "package-smoke-timeouts.test.ts": {
+ "linux": 0.1,
+ "darwin": 0.1,
+ "win32": 0.1
+ },
+ "package-tar-entries.test.ts": {
+ "linux": 0.1,
+ "darwin": 0.1,
+ "win32": 0.1
+ },
+ "package-tar-listing.test.ts": {
+ "linux": 19.8,
+ "darwin": 45.3,
+ "win32": 91.7
+ },
+ "patch-publication.test.ts": {
+ "linux": 0.1,
+ "darwin": 0.2,
+ "win32": 0.9
+ },
+ "patch-risk-contract.test.ts": {
+ "linux": 14.4,
+ "darwin": 29.8,
+ "win32": 48.1
+ },
+ "patch-tui.test.ts": {
+ "linux": 2.0,
+ "darwin": 3.1,
+ "win32": 16.2
+ },
+ "plugin-apps.test.ts": {
+ "linux": 0.1,
+ "darwin": 0.1,
+ "win32": 0.1
+ },
+ "plugin-finding-detail-contract.test.ts": {
+ "linux": 0.2,
+ "darwin": 0.3,
+ "win32": 0.5
+ },
+ "plugin-report-limits.test.ts": {
+ "linux": 0.5,
+ "darwin": 0.7,
+ "win32": 0.7
+ },
+ "project-config.test.ts": {
+ "linux": 0.1,
+ "darwin": 0.1,
+ "win32": 0.1
+ },
+ "prompt-only-start-timeout.test.ts": {
+ "linux": 0.1,
+ "darwin": 0.1,
+ "win32": 0.1
+ },
+ "provider-profile.test.ts": {
+ "linux": 1.5,
+ "darwin": 2.9,
+ "win32": 9.9
+ },
+ "publication-check.test.ts": {
+ "linux": 0.1,
+ "darwin": 0.1,
+ "win32": 0.1
+ },
+ "publication-events.test.ts": {
+ "linux": 0.1,
+ "darwin": 0.1,
+ "win32": 0.1
+ },
+ "publication-integration.test.ts": {
+ "linux": 15.1,
+ "darwin": 21.4,
+ "win32": 31.7
+ },
+ "publication-store.test.ts": {
+ "linux": 22.2,
+ "darwin": 25.1,
+ "win32": 44.6
+ },
+ "publication.test.ts": {
+ "linux": 2.1,
+ "darwin": 2.0,
+ "win32": 2.6
+ },
+ "publish.test.ts": {
+ "linux": 3.5,
+ "darwin": 4.9,
+ "win32": 6.5
+ },
+ "published-install-smoke.test.ts": {
+ "linux": 0.2,
+ "darwin": 0.2,
+ "win32": 0.6
+ },
+ "release-automation.test.ts": {
+ "linux": 3.6,
+ "darwin": 11.6,
+ "win32": 25.3
+ },
+ "release-cut-workflow.test.ts": {
+ "linux": 0.1,
+ "darwin": 0.1,
+ "win32": 0.1
+ },
+ "release-pr.test.ts": {
+ "linux": 27.5,
+ "darwin": 53.6,
+ "win32": 226.9
+ },
+ "repository-findings.test.ts": {
+ "linux": 0.1,
+ "darwin": 0.3,
+ "win32": 0.4
+ },
+ "request-metadata.test.ts": {
+ "linux": 0.1,
+ "darwin": 0.8,
+ "win32": 0.7
+ },
+ "result.test.ts": {
+ "linux": 0.1,
+ "darwin": 0.1,
+ "win32": 0.1
+ },
+ "runtime-plugin-reuse.test.ts": {
+ "linux": 0.4,
+ "darwin": 1.9,
+ "win32": 1.4
+ },
+ "runtime-zip-descriptor.test.ts": {
+ "linux": 0.4,
+ "darwin": 0.5,
+ "win32": 0.7
+ },
+ "runtime.test.ts": {
+ "linux": 39.7,
+ "darwin": 41.6,
+ "win32": 195.4
+ },
+ "sandbox-probe.test.ts": {
+ "linux": 0.1,
+ "darwin": 0.1,
+ "win32": 0.1
+ },
+ "sarif.test.ts": {
+ "linux": 0.8,
+ "darwin": 1.2,
+ "win32": 1.2
+ },
+ "saved-scan-bootstrap.test.ts": {
+ "linux": 118.8,
+ "darwin": 119.0,
+ "win32": 103.7
+ },
+ "scan-activity.test.ts": {
+ "linux": 0.1,
+ "darwin": 0.1,
+ "win32": 0.1
+ },
+ "scan-comparison.property.test.ts": {
+ "linux": 0.4,
+ "darwin": 0.3,
+ "win32": 0.5
+ },
+ "scan-comparison.test.ts": {
+ "linux": 4.6,
+ "darwin": 5.0,
+ "win32": 127.7
+ },
+ "scan-dashboard.test.ts": {
+ "linux": 4.5,
+ "darwin": 21.5,
+ "win32": 40.1
+ },
+ "scan-history-renderer.test.ts": {
+ "linux": 0.1,
+ "darwin": 0.1,
+ "win32": 0.1
+ },
+ "scan-logs-node.test.ts": {
+ "linux": 0.1,
+ "darwin": 0.2,
+ "win32": 0.2
+ },
+ "scan-logs.test.ts": {
+ "linux": 0.1,
+ "darwin": 0.3,
+ "win32": 0.3
+ },
+ "scan-matching-e2e.test.ts": {
+ "linux": 7.4,
+ "darwin": 12.4,
+ "win32": 17.5
+ },
+ "scan-mcp-startup.test.ts": {
+ "linux": 1.9,
+ "darwin": 2.1,
+ "win32": 8.7
+ },
+ "scan-owner-suggestions.test.ts": {
+ "linux": 0.3,
+ "darwin": 1.3,
+ "win32": 9.4
+ },
+ "scan-progress.test.ts": {
+ "linux": 0.1,
+ "darwin": 0.1,
+ "win32": 0.1
+ },
+ "scan-recipe.test.ts": {
+ "linux": 12.2,
+ "darwin": 12.5,
+ "win32": 35.5
+ },
+ "scan-recovery.test.ts": {
+ "linux": 15.2,
+ "darwin": 32.8,
+ "win32": 66.1
+ },
+ "scan-resume.test.ts": {
+ "linux": 97.0,
+ "darwin": 164.8,
+ "win32": 231.9
+ },
+ "scan-skill-paths.test.ts": {
+ "linux": 0.1,
+ "darwin": 0.1,
+ "win32": 0.1
+ },
+ "scanner-import.test.ts": {
+ "linux": 0.1,
+ "darwin": 0.1,
+ "win32": 0.1
+ },
+ "script-entrypoints.test.ts": {
+ "linux": 3.1,
+ "darwin": 6.1,
+ "win32": 7.0
+ },
+ "sdk-project-config.test.ts": {
+ "linux": 0.2,
+ "darwin": 0.3,
+ "win32": 1.4
+ },
+ "sdk-scan-prompts.test.ts": {
+ "linux": 0.9,
+ "darwin": 1.1,
+ "win32": 3.3
+ },
+ "security-policy-diff.test.ts": {
+ "linux": 0.1,
+ "darwin": 0.1,
+ "win32": 0.1
+ },
+ "security-policy-helper.test.ts": {
+ "linux": 8.3,
+ "darwin": 9.5,
+ "win32": 9.7
+ },
+ "security-policy.test.ts": {
+ "linux": 8.4,
+ "darwin": 24.5,
+ "win32": 82.5
+ },
+ "severity-review.test.ts": {
+ "linux": 0.1,
+ "darwin": 0.1,
+ "win32": 0.1
+ },
+ "shell.test.ts": {
+ "linux": 0.1,
+ "darwin": 0.1,
+ "win32": 0.1
+ },
+ "skeleton.test.ts": {
+ "linux": 0.1,
+ "darwin": 0.1,
+ "win32": 0.4
+ },
+ "stopped-scan-results.test.ts": {
+ "linux": 8.0,
+ "darwin": 10.9,
+ "win32": 19.3
+ },
+ "suggest-owners.test.ts": {
+ "linux": 2.9,
+ "darwin": 11.8,
+ "win32": 70.5
+ },
+ "targets-large-output.test.ts": {
+ "linux": 0.4,
+ "darwin": 1.3,
+ "win32": 4.8
+ },
+ "targets.test.ts": {
+ "linux": 0.8,
+ "darwin": 3.9,
+ "win32": 12.0
+ },
+ "test-reports.test.ts": {
+ "linux": 3.1,
+ "darwin": 4.6,
+ "win32": 6.0
+ },
+ "test-shards.test.ts": {
+ "linux": 2.3,
+ "darwin": 5.3,
+ "win32": 6.1
+ },
+ "trusted-executable.test.ts": {
+ "linux": 0.6,
+ "darwin": 1.1,
+ "win32": 0.1
+ },
+ "update-notice.test.ts": {
+ "linux": 0.1,
+ "darwin": 0.1,
+ "win32": 0.1
+ },
+ "validation-records.test.ts": {
+ "linux": 0.1,
+ "darwin": 0.3,
+ "win32": 0.3
+ },
+ "windows-scan-output.test.ts": {
+ "linux": 0.1,
+ "darwin": 0.1,
+ "win32": 16.6
+ },
+ "wiz-findings.test.ts": {
+ "linux": 4.4,
+ "darwin": 5.4,
+ "win32": 10.6
+ },
+ "workbench-arguments.test.ts": {
+ "linux": 0.1,
+ "darwin": 0.1,
+ "win32": 0.1
+ },
+ "workbench-canonical-paths.test.ts": {
+ "linux": 0.8,
+ "darwin": 2.3,
+ "win32": 4.9
+ },
+ "workbench-nested-git-utf8.test.ts": {
+ "linux": 0.1,
+ "darwin": 0.2,
+ "win32": 1.6
+ },
+ "workbench-scan-history.test.ts": {
+ "linux": 1.5,
+ "darwin": 2.3,
+ "win32": 3.9
+ },
+ "workbench-scan-root-alias.test.ts": {
+ "linux": 0.2,
+ "darwin": 0.3,
+ "win32": 0.6
+ },
+ "workbench-windows-compatibility.test.ts": {
+ "linux": 0.5,
+ "darwin": 1.1,
+ "win32": 2.0
+ },
+ "worker-progress.property.test.ts": {
+ "linux": 0.1,
+ "darwin": 0.1,
+ "win32": 0.1
+ },
+ "worker-progress.test.ts": {
+ "linux": 0.1,
+ "darwin": 0.1,
+ "win32": 0.1
+ },
+ "workflow-quality.test.ts": {
+ "linux": 0.1,
+ "darwin": 0.2,
+ "win32": 0.1
+ }
+}
diff --git a/sdk/typescript/scripts/update-test-timings.mts b/sdk/typescript/scripts/update-test-timings.mts
new file mode 100644
index 000000000..ee30cf0bb
--- /dev/null
+++ b/sdk/typescript/scripts/update-test-timings.mts
@@ -0,0 +1,86 @@
+import { readFile, writeFile } from "node:fs/promises";
+import { resolve } from "node:path";
+import { fileURLToPath } from "node:url";
+import { SaxesParser } from "saxes";
+import type { TestTimings } from "./test-shards.mts";
+
+export function updateTestTimings(
+ previous: TestTimings,
+ platform: "linux" | "darwin" | "win32",
+ reports: readonly string[],
+): TestTimings {
+ const samples = new Map<string, number[]>();
+ for (const report of reports) {
+ const totals = new Map<string, number>();
+ const parser = new SaxesParser({ xmlns: false });
+ parser.on("opentag", ({ name, attributes }) => {
+ if (
+ name === "failure" ||
+ name === "error" ||
+ Number(attributes["failures"] ?? 0) > 0 ||
+ Number(attributes["errors"] ?? 0) > 0
+ ) {
+ throw new Error(
+ "Use successful JUnit reports to refresh test timings.",
+ );
+ }
+ if (name !== "testcase") return;
+ const file = attributes["file"]?.replaceAll("\\", "/").split("/").at(-1);
+ const seconds = Number(attributes["time"] ?? 0);
+ if (!file || !Number.isFinite(seconds) || seconds < 0) {
+ throw new Error(
+ "JUnit testcase needs a file and a nonnegative duration.",
+ );
+ }
+ totals.set(file, (totals.get(file) ?? 0) + seconds);
+ });
+ parser.write(report).close();
+ if (!totals.size) throw new Error("JUnit report contains no test cases.");
+ for (const [file, seconds] of totals) {
+ const values = samples.get(file) ?? [];
+ values.push(seconds);
+ samples.set(file, values);
+ }
+ }
+
+ const updated = { ...previous };
+ for (const [file, values] of samples) {
+ values.sort((left, right) => left - right);
+ const middle = Math.floor(values.length / 2);
+ const median =
+ values.length % 2
+ ? values[middle]!
+ : (values[middle - 1]! + values[middle]!) / 2;
+ updated[file] = {
+ ...previous[file],
+ [platform]: Math.max(0.1, Math.round(median * 10) / 10),
+ };
+ }
+ return Object.fromEntries(
+ Object.keys(updated)
+ .sort()
+ .map((file) => [file, updated[file]!]),
+ );
+}
+
+if (
+ process.argv[1] &&
+ fileURLToPath(import.meta.url) === resolve(process.argv[1])
+) {
+ const [platform, ...paths] = process.argv.slice(2);
+ if (
+ !paths.length ||
+ (platform !== "linux" && platform !== "darwin" && platform !== "win32")
+ ) {
+ throw new Error(
+ "Usage: node --experimental-strip-types scripts/update-test-timings.mts <linux|darwin|win32> <junit.xml> [...]",
+ );
+ }
+ const target = new URL("./test-timings.json", import.meta.url);
+ const previous = JSON.parse(await readFile(target, "utf8")) as TestTimings;
+ const reports = await Promise.all(
+ paths.map((path) => readFile(path, "utf8")),
+ );
+ const updated = updateTestTimings(previous, platform, reports);
+ await writeFile(target, `${JSON.stringify(updated, null, 2)}\n`);
+}
diff --git a/sdk/typescript/tests-ts/test-shards.test.ts b/sdk/typescript/tests-ts/test-shards.test.ts
index ebf27282c..8d55cdd5d 100644
--- a/sdk/typescript/tests-ts/test-shards.test.ts
+++ b/sdk/typescript/tests-ts/test-shards.test.ts
@@ -9,9 +9,62 @@ import {
import { tmpdir } from "node:os";
import { delimiter, dirname, join } from "node:path";
import { expect, test } from "bun:test";
+import { partitionTests, type TestTimings } from "../scripts/test-shards.mts";
import { readSubprocess } from "./support/shell.js";
import { temporaryDirectory } from "./support/temporary-directories.js";
+async function copyRunner(root: string, timings: TestTimings = {}) {
+ for (const name of ["run-ci-tests.mts", "test-shards.mts"]) {
+ await copyFile(
+ new URL(`../scripts/${name}`, import.meta.url),
+ join(root, "scripts", name),
+ );
+ }
+ await writeFile(
+ join(root, "scripts", "test-timings.json"),
+ JSON.stringify(timings),
+ );
+}
+
+test.each(["linux", "darwin", "win32"] as const)(
+ "balances whole files using %s durations without depending on discovery order",
+ (platform) => {
+ const files = ["a", "b", "c", "d", "e", "f"];
+ const timings = Object.fromEntries(
+ files.map((file, index) => [file, { [platform]: 100 - index * 10 }]),
+ );
+ const shards = partitionTests(files, 3, timings, platform);
+ expect(shards).toEqual([
+ ["a", "f"],
+ ["b", "e"],
+ ["c", "d"],
+ ]);
+ expect(partitionTests(files.toReversed(), 3, timings, platform)).toEqual(
+ shards,
+ );
+ expect(files).toEqual(["a", "b", "c", "d", "e", "f"]);
+ },
+);
+
+test("uses each platform's history and includes new files without selecting deleted files", () => {
+ const files = ["a", "b", "c", "new"];
+ const timings = {
+ a: { linux: 100, win32: 1 },
+ b: { linux: 1, win32: 100 },
+ c: { linux: 1, win32: 1 },
+ deleted: { linux: 1_000, win32: 1_000 },
+ };
+ expect(partitionTests(files, 2, timings, "linux")).toEqual([
+ ["a"],
+ ["b", "c", "new"],
+ ]);
+ expect(partitionTests(files, 2, timings, "win32")).toEqual([
+ ["b"],
+ ["a", "c", "new"],
+ ]);
+ expect(partitionTests(files, 2, {}, "darwin").flat().sort()).toEqual(files);
+});
+
test.each([3, 7])(
"runs each eligible file once across %i native shards, including new files",
async (count) => {
@@ -21,10 +74,11 @@ test.each([3, 7])(
try {
await mkdir(join(root, "scripts"));
await mkdir(join(root, "tests-ts"));
- await copyFile(
- new URL("../scripts/run-ci-tests.mts", import.meta.url),
- join(root, "scripts", "run-ci-tests.mts"),
- );
+ await copyRunner(root, {
+ "probe-0.test.ts": { linux: 100, darwin: 100, win32: 100 },
+ "probe-1.test.ts": { linux: 80, darwin: 80, win32: 80 },
+ "deleted.test.ts": { linux: 1_000, darwin: 1_000, win32: 1_000 },
+ });
const files = [
...Array.from({ length: 20 }, (_, index) => `probe-${index}.test.ts`),
"newly-added.test.ts",
@@ -88,10 +142,7 @@ test("rejects an empty shard while allowing a nonempty shard with the same count
try {
await mkdir(join(root, "scripts"));
await mkdir(join(root, "tests-ts"));
- await copyFile(
- new URL("../scripts/run-ci-tests.mts", import.meta.url),
- join(root, "scripts", "run-ci-tests.mts"),
- );
+ await copyRunner(root);
await writeFile(
join(root, "tests-ts", "probe.test.ts"),
'import { test } from "bun:test"; test("synthetic shard probe", () => {});\n',
@@ -167,10 +218,7 @@ test.each([
try {
await mkdir(join(root, "scripts"));
await mkdir(join(root, "tests-ts"));
- await copyFile(
- new URL("../scripts/run-ci-tests.mts", import.meta.url),
- join(root, "scripts", "run-ci-tests.mts"),
- );
+ await copyRunner(root);
await writeFile(
join(root, "tests-ts", "probe.test.ts"),
`import { expect, test } from "bun:test";
diff --git a/sdk/typescript/tests-ts/test-timings.test.ts b/sdk/typescript/tests-ts/test-timings.test.ts
new file mode 100644
index 000000000..02f253da1
--- /dev/null
+++ b/sdk/typescript/tests-ts/test-timings.test.ts
@@ -0,0 +1,72 @@
+import { expect, test } from "bun:test";
+import { updateTestTimings } from "../scripts/update-test-timings.mts";
+
+function report(cases: string) {
+ return `<testsuites failures="0"><testsuite>${cases}</testsuite></testsuites>`;
+}
+
+test("refreshes per-file medians while retaining other platforms and undiscovered files", () => {
+ const previous = {
+ "slow.test.ts": { linux: 100, darwin: 20, win32: 50 },
+ "stale.test.ts": { linux: 2, darwin: 3 },
+ };
+ const updated = updateTestTimings(previous, "linux", [
+ report(`
+ <testcase file="tests-ts/slow.test.ts" time="2" />
+ <testcase file="tests-ts/slow.test.ts" time="4" />
+ <testcase file="tests-ts/new.test.ts" time="0.24" />
+ `),
+ report('<testcase file="tests-ts/slow.test.ts" time="100" />'),
+ report('<testcase file="tests-ts/slow.test.ts" time="10" />'),
+ ]);
+ expect(updated).toEqual({
+ "new.test.ts": { linux: 0.2 },
+ "slow.test.ts": { linux: 10, darwin: 20, win32: 50 },
+ "stale.test.ts": { linux: 2, darwin: 3 },
+ });
+ expect(previous["slow.test.ts"].linux).toBe(100);
+ expect(Object.keys(updated)).toEqual([
+ "new.test.ts",
+ "slow.test.ts",
+ "stale.test.ts",
+ ]);
+});
+
+test("handles XML-escaped and Windows file paths and zero-duration skipped cases", () => {
+ const updated = updateTestTimings({}, "win32", [
+ report(String.raw`
+ <testcase file="C:\checkout\tests-ts\names&amp;paths.test.ts" time="1.2" />
+ <testcase file="tests-ts/names&amp;paths.test.ts" time="0.5" />
+ <testcase file="tests-ts/skipped.test.ts"><skipped /></testcase>
+ `),
+ report('<testcase file="tests-ts/names&amp;paths.test.ts" time="3.1" />'),
+ ]);
+ expect(updated).toEqual({
+ "names&paths.test.ts": { win32: 2.4 },
+ "skipped.test.ts": { win32: 0.1 },
+ });
+});
+
+test.each([
+ '<testsuites failures="1"><testcase file="tests-ts/example.test.ts" time="3" /></testsuites>',
+ report(
+ '<testcase file="tests-ts/example.test.ts" time="3"><failure /></testcase>',
+ ),
+ report('<testcase file="tests-ts/example.test.ts" time="NaN" />'),
+ report('<testcase file="tests-ts/example.test.ts" time="-1" />'),
+ report('<testcase time="1" />'),
+ report(""),
+ "<testsuites>",
+])(
+ "rejects unusable reports without altering the timing snapshot: %s",
+ (invalid) => {
+ const previous = { "example.test.ts": { darwin: 5 } };
+ expect(() =>
+ updateTestTimings(previous, "darwin", [
+ report('<testcase file="tests-ts/example.test.ts" time="10" />'),
+ invalid,
+ ]),
+ ).toThrow();
+ expect(previous).toEqual({ "example.test.ts": { darwin: 5 } });
+ },
+);
diff --git a/sdk/typescript/tsconfig.json b/sdk/typescript/tsconfig.json
index ad25f8e4f..641b4fe4d 100644
--- a/sdk/typescript/tsconfig.json
+++ b/sdk/typescript/tsconfig.json
@@ -7,7 +7,8 @@
"../../plugins/codex-security/native/*.mts",
"../../examples/custom-validation/*.mts",
"scripts/compare-test-reports.mts",
- "scripts/run-ci-tests.mts"
+ "scripts/run-ci-tests.mts",
+ "scripts/update-test-timings.mts"
],
"compilerOptions": {
"allowJs": true,