OpenAI's Codex Security CLI and TypeScript SDK for finding, validating, and fixing security vulnerabilities. npm: https://www.npmjs.com/package/@openai/codex-security
TypeScript ai-securityapplication-securityclicode-scanningcodexcodex-securitycybersecuritydevsecopsnodejsnpmopenaisdksecuritytypescriptvulnerability-scanning
test(ci): balance test shards by recorded duration (#1856)
Michael D'Angelo8b248f00c185
diff --git a/sdk/typescript/TESTING.md b/sdk/typescript/TESTING.mdindex 44566ac03..1c3ccdf51 100644--- a/sdk/typescript/TESTING.md+++ b/sdk/typescript/TESTING.md@@ -147,13 +147,26 @@ package checks instead of repeating the same Bun suite. MCP and Python testsrun in separate required jobs. Python uses four isolated pytest-xdist workerswith work stealing; worker crashes fail the run without automatic restarts.-`scripts/run-ci-tests.mts` discovers every shared test file and passes the-inventory to Bun's native `--shard` option. Bun sorts the file paths and assigns-them round-robin across the shards. New files are included automatically;-there is no timing table to update when adding or splitting tests.+`scripts/run-ci-tests.mts` discovers every shared test file and balances whole+files using the platform's recorded durations in `scripts/test-timings.json`.+It assigns longer files first to the shard with the least estimated work, then+passes each shard's files directly to Bun. Every job uses the same checked-in+snapshot. New files are included automatically with the mean known-file duration;+deleted files in the timing snapshot are ignored. Timing hints never select or+exclude tests, and adding or splitting tests does not require updating them.To reproduce one Windows shard locally after building the plugin, run`node --experimental-strip-types scripts/run-ci-tests.mts 3/7 --seed=12345`.+Refresh timing hints from successful CI JUnit artifacts when shard runtimes+become uneven. From the SDK directory, pass reports for one platform to+`node --experimental-strip-types scripts/update-test-timings.mts win32 <junit.xml> ...`+(use `linux` or `darwin` for the other platforms). The updater sums testcase+durations per file and uses their median across reports, preserving the other+platforms. Review and commit the generated snapshot. The initial hints use runs+`39521539826`, `39694506447`, and `39694510011`. Keep runner versions, coverage,+timeouts, seeds, and platform-specific checks unchanged when comparing scheduling+changes; compare complete JUnit inventories and outcomes as well as wall time.+Every Bun lane uploads JUnit; Linux lanes also upload LCOV per shard. Pythonreports include case durations, and the MCP runner can upload its JUnit report.Coverage is diagnostic and split across shards, not a combined percentage.diff --git a/sdk/typescript/scripts/run-ci-tests.mts b/sdk/typescript/scripts/run-ci-tests.mtsindex 73ff9a706..dba9b5bbb 100644--- a/sdk/typescript/scripts/run-ci-tests.mts+++ b/sdk/typescript/scripts/run-ci-tests.mts@@ -1,6 +1,7 @@import { spawn } from "node:child_process";-import { mkdir, readdir } from "node:fs/promises";+import { mkdir, readFile, readdir } from "node:fs/promises";import { parseArgs } from "node:util";+import { partitionTests, type TestTimings } from "./test-shards.mts";const selection = /^([1-9]\d*)\/([1-9]\d*)$/.exec(process.argv[2] ?? "");const shard = Number(selection?.[1]);@@ -16,9 +17,14 @@ const tests = (await readdir(new URL("../tests-ts/", import.meta.url)))file.endsWith(".test.ts") && file !== "windows-machine-policy.test.ts",).sort();-// Match Bun 1.3.14's sorted round-robin assignment for startup diagnostics.-// https://github.com/oven-sh/bun/blob/bun-v1.3.14/src/cli/test_command.zig-const selected = tests.filter((_, index) => index % count === shard - 1);+// The checked-in snapshot keeps every job in this run on the same assignment.+// Discovery remains authoritative, including files without timing history.+const timings = JSON.parse(+ await readFile(new URL("./test-timings.json", import.meta.url), "utf8"),+) as TestTimings;+const selected = partitionTests(tests, count, timings, process.platform)[+ shard - 1+]!;if (selected.length === 0) {throw new Error(`Test shard ${shard}/${count} is empty.`);}@@ -32,13 +38,12 @@ await mkdir(new URL("../reports/", import.meta.url), { recursive: true }).catch();const testArguments = ["test",- `--shard=${shard}/${count}`,"--timeout",defaultTestTimeoutMs,"--reporter=junit",`--reporter-outfile=reports/junit-${shard}.xml`,...process.argv.slice(3),- ...tests.map((file) => `./tests-ts/${file}`),+ ...selected.map((file) => `./tests-ts/${file}`),];// Keep arities aligned with the pinned Bun's test_params:// https://github.com/oven-sh/bun/blob/bun-v1.3.14/src/cli/Arguments.zigdiff --git a/sdk/typescript/scripts/test-shards.mts b/sdk/typescript/scripts/test-shards.mtsnew file mode 100644index 000000000..9628e28e5--- /dev/null+++ b/sdk/typescript/scripts/test-shards.mts@@ -0,0 +1,34 @@+export type TestTimings = Record<+ string,+ Partial<Record<NodeJS.Platform, number>>+>;++export function partitionTests(+ files: readonly string[],+ count: number,+ timings: TestTimings,+ platform: NodeJS.Platform,+): string[][] {+ const known = files.flatMap((file) => {+ const seconds = timings[file]?.[platform];+ return seconds === undefined ? [] : [seconds];+ });+ const fallback = known.length+ ? known.reduce((total, seconds) => total + seconds, 0) / known.length+ : 1;+ const duration = (file: string) => timings[file]?.[platform] ?? fallback;+ const shards = Array.from({ length: count }, () => ({+ files: [] as string[],+ seconds: 0,+ }));+ for (const file of [...files]+ .sort()+ .sort((left, right) => duration(right) - duration(left))) {+ const shard = shards.reduce((least, candidate) =>+ candidate.seconds < least.seconds ? candidate : least,+ );+ shard.files.push(file);+ shard.seconds += duration(file);+ }+ return shards.map((shard) => shard.files.sort());+}diff --git a/sdk/typescript/scripts/test-timings.json b/sdk/typescript/scripts/test-timings.jsonnew file mode 100644index 000000000..1c280766e--- /dev/null+++ b/sdk/typescript/scripts/test-timings.json@@ -0,0 +1,1072 @@+{+ "action-release-workflow.test.ts": {+ "linux": 2.9,+ "darwin": 10.7,+ "win32": 53.4+ },+ "api-attribution-concurrency.test.ts": {+ "linux": 1.9,+ "darwin": 5.5,+ "win32": 134.7+ },+ "api-audit-admission.test.ts": {+ "linux": 0.2,+ "darwin": 0.2,+ "win32": 0.6+ },+ "api-credentials.test.ts": {+ "linux": 2.0,+ "darwin": 4.5,+ "win32": 142.5+ },+ "api-cyber-access.test.ts": {+ "linux": 0.1,+ "darwin": 0.3,+ "win32": 1.7+ },+ "api-deep-scan-advisory-routing.test.ts": {+ "linux": 0.2,+ "darwin": 0.1,+ "win32": 4.4+ },+ "api-deep-scan-runtime-routing.test.ts": {+ "linux": 0.1,+ "darwin": 0.1,+ "win32": 0.6+ },+ "api-direct-deep-scan.test.ts": {+ "linux": 0.5,+ "darwin": 1.7,+ "win32": 12.6+ },+ "api-environment.test.ts": {+ "linux": 0.1,+ "darwin": 0.1,+ "win32": 0.1+ },+ "api-events.test.ts": {+ "linux": 6.3,+ "darwin": 11.2,+ "win32": 17.9+ },+ "api-integration.test.ts": {+ "linux": 0.1,+ "darwin": 0.1,+ "win32": 0.1+ },+ "api-login.test.ts": {+ "linux": 0.1,+ "darwin": 0.1,+ "win32": 2.7+ },+ "api-operation-registration.test.ts": {+ "linux": 0.1,+ "darwin": 0.1,+ "win32": 0.1+ },+ "api-patch.test.ts": {+ "linux": 0.2,+ "darwin": 0.4,+ "win32": 6.7+ },+ "api-policy.test.ts": {+ "linux": 25.3,+ "darwin": 26.8,+ "win32": 191.6+ },+ "api-post-scan.test.ts": {+ "linux": 10.2,+ "darwin": 12.8,+ "win32": 25.5+ },+ "api-preflight-config.test.ts": {+ "linux": 0.1,+ "darwin": 0.2,+ "win32": 0.9+ },+ "api-provider-isolation.test.ts": {+ "linux": 3.6,+ "darwin": 5.4,+ "win32": 150.2+ },+ "api-runtime-storage.test.ts": {+ "linux": 0.1,+ "darwin": 0.3,+ "win32": 62.8+ },+ "api-time-budget.test.ts": {+ "linux": 0.5,+ "darwin": 0.7,+ "win32": 2.1+ },+ "api-usage-finalization.test.ts": {+ "linux": 0.4,+ "darwin": 0.4,+ "win32": 1.9+ },+ "api-usage.test.ts": {+ "linux": 0.7,+ "darwin": 1.1,+ "win32": 2.1+ },+ "api.test.ts": {+ "linux": 62.1,+ "darwin": 116.1,+ "win32": 445.9+ },+ "artifact-export.test.ts": {+ "linux": 3.5,+ "darwin": 3.5,+ "win32": 2.5+ },+ "attack-path-tool-name.test.ts": {+ "linux": 0.1,+ "darwin": 0.2,+ "win32": 0.2+ },+ "auth-status-concurrency.test.ts": {+ "linux": 1.5,+ "darwin": 2.8,+ "win32": 62.8+ },+ "auth.test.ts": {+ "linux": 1.9,+ "darwin": 2.6,+ "win32": 3.7+ },+ "azure-pipelines-example.test.ts": {+ "linux": 0.1,+ "darwin": 0.1,+ "win32": 0.7+ },+ "bind-repo-scopes.test.ts": {+ "linux": 3.0,+ "darwin": 4.5,+ "win32": 9.5+ },+ "build-plugin.test.ts": {+ "linux": 10.6,+ "darwin": 8.7,+ "win32": 15.9+ },+ "bulk-scan-discovery.test.ts": {+ "linux": 0.1,+ "darwin": 0.1,+ "win32": 0.1+ },+ "candidate-normalizer.test.ts": {+ "linux": 11.3,+ "darwin": 19.4,+ "win32": 17.3+ },+ "check-markdown.test.ts": {+ "linux": 2.3,+ "darwin": 2.4,+ "win32": 4.7+ },+ "classify-scan-severity.test.ts": {+ "linux": 33.0,+ "darwin": 48.0,+ "win32": 74.4+ },+ "classify-severity-records.test.ts": {+ "linux": 0.1,+ "darwin": 0.1,+ "win32": 0.1+ },+ "classify-severity.test.ts": {+ "linux": 0.1,+ "darwin": 0.1,+ "win32": 0.1+ },+ "cli-authentication.test.ts": {+ "linux": 12.3,+ "darwin": 17.2,+ "win32": 308.7+ },+ "cli-classify-severity.test.ts": {+ "linux": 0.1,+ "darwin": 0.1,+ "win32": 0.2+ },+ "cli-cloud-publish.test.ts": {+ "linux": 1.5,+ "darwin": 0.7,+ "win32": 1.9+ },+ "cli-daybreak.test.ts": {+ "linux": 0.1,+ "darwin": 0.1,+ "win32": 0.1+ },+ "cli-dedupe.test.ts": {+ "linux": 0.2,+ "darwin": 0.2,+ "win32": 0.3+ },+ "cli-deep-scan-summary.test.ts": {+ "linux": 0.1,+ "darwin": 0.1,+ "win32": 0.1+ },+ "cli-diagnostics.test.ts": {+ "linux": 0.4,+ "darwin": 0.4,+ "win32": 0.5+ },+ "cli-error-diagnostics.test.ts": {+ "linux": 0.4,+ "darwin": 0.7,+ "win32": 1.0+ },+ "cli-export.test.ts": {+ "linux": 7.6,+ "darwin": 6.2,+ "win32": 16.0+ },+ "cli-feedback.test.ts": {+ "linux": 1.3,+ "darwin": 2.4,+ "win32": 4.4+ },+ "cli-help.test.ts": {+ "linux": 0.2,+ "darwin": 0.3,+ "win32": 0.4+ },+ "cli-history-paths.test.ts": {+ "linux": 0.1,+ "darwin": 0.1,+ "win32": 0.1+ },+ "cli-launcher.test.ts": {+ "linux": 1.5,+ "darwin": 2.2,+ "win32": 2.0+ },+ "cli-patch-results.test.ts": {+ "linux": 3.2,+ "darwin": 8.7,+ "win32": 89.9+ },+ "cli-patch-trust.test.ts": {+ "linux": 1.0,+ "darwin": 2.4,+ "win32": 4.7+ },+ "cli-patch.test.ts": {+ "linux": 2.7,+ "darwin": 8.9,+ "win32": 23.3+ },+ "cli-policy.test.ts": {+ "linux": 4.6,+ "darwin": 5.9,+ "win32": 23.6+ },+ "cli-project-config.test.ts": {+ "linux": 0.9,+ "darwin": 1.7,+ "win32": 4.2+ },+ "cli-publish.test.ts": {+ "linux": 1.1,+ "darwin": 1.8,+ "win32": 1.7+ },+ "cli-saved-finding-evidence.test.ts": {+ "linux": 6.2,+ "darwin": 6.7,+ "win32": 14.7+ },+ "cli-scan-failures.test.ts": {+ "linux": 1.1,+ "darwin": 0.8,+ "win32": 1.6+ },+ "cli-scan-import.test.ts": {+ "linux": 0.2,+ "darwin": 0.3,+ "win32": 0.3+ },+ "cli-scan-logs.test.ts": {+ "linux": 6.4,+ "darwin": 5.7,+ "win32": 8.8+ },+ "cli-scan-navigation.test.ts": {+ "linux": 2.8,+ "darwin": 2.7,+ "win32": 5.9+ },+ "cli-scan-patch-settings.test.ts": {+ "linux": 1.5,+ "darwin": 1.9,+ "win32": 3.4+ },+ "cli-scan-prompts.test.ts": {+ "linux": 0.3,+ "darwin": 0.4,+ "win32": 1.1+ },+ "cli-show-cost.test.ts": {+ "linux": 0.3,+ "darwin": 0.3,+ "win32": 0.5+ },+ "cli-signals.test.ts": {+ "linux": 0.1,+ "darwin": 0.1,+ "win32": 0.1+ },+ "cli-skill-chatgpt-provider-auth.test.ts": {+ "linux": 2.3,+ "darwin": 2.7,+ "win32": 271.3+ },+ "cli-skill-native-config.test.ts": {+ "linux": 16.3,+ "darwin": 27.2,+ "win32": 223.9+ },+ "cli-skills.test.ts": {+ "linux": 4.3,+ "darwin": 5.2,+ "win32": 5.1+ },+ "cli-suggest-owners.test.ts": {+ "linux": 0.3,+ "darwin": 0.2,+ "win32": 0.4+ },+ "cli-verify-fix.test.ts": {+ "linux": 0.4,+ "darwin": 0.2,+ "win32": 0.7+ },+ "cli-workbench.test.ts": {+ "linux": 0.6,+ "darwin": 0.6,+ "win32": 1.1+ },+ "cli.test.ts": {+ "linux": 5.8,+ "darwin": 5.9,+ "win32": 13.0+ },+ "cloud-publish.test.ts": {+ "linux": 1.8,+ "darwin": 1.6,+ "win32": 6.9+ },+ "codeowners.test.ts": {+ "linux": 0.1,+ "darwin": 0.3,+ "win32": 0.2+ },+ "codex-process-cancellation.test.ts": {+ "linux": 2.2,+ "darwin": 3.8,+ "win32": 3.0+ },+ "codex-review.test.ts": {+ "linux": 4.7,+ "darwin": 6.7,+ "win32": 12.7+ },+ "compact-diff-scan.test.ts": {+ "linux": 12.7,+ "darwin": 29.1,+ "win32": 47.2+ },+ "completed-scan-handoff.test.ts": {+ "linux": 0.1,+ "darwin": 0.1,+ "win32": 0.1+ },+ "component-scan.test.ts": {+ "linux": 13.2,+ "darwin": 17.2,+ "win32": 88.6+ },+ "config-approval-policy.test.ts": {+ "linux": 0.1,+ "darwin": 0.1,+ "win32": 0.1+ },+ "config.test.ts": {+ "linux": 0.5,+ "darwin": 3.3,+ "win32": 27.1+ },+ "container-ci-workflow.test.ts": {+ "linux": 0.1,+ "darwin": 0.2,+ "win32": 2.0+ },+ "container-entrypoint.test.ts": {+ "linux": 0.1,+ "darwin": 0.5,+ "win32": 0.1+ },+ "container-release-workflow.test.ts": {+ "linux": 0.5,+ "darwin": 2.4,+ "win32": 14.6+ },+ "contract.test.ts": {+ "linux": 12.2,+ "darwin": 11.2,+ "win32": 21.5+ },+ "cost-context.test.ts": {+ "linux": 0.1,+ "darwin": 0.1,+ "win32": 0.1+ },+ "cost-model.property.test.ts": {+ "linux": 0.1,+ "darwin": 0.1,+ "win32": 0.1+ },+ "cost.test.ts": {+ "linux": 1.3,+ "darwin": 1.5,+ "win32": 2.1+ },+ "custom-publish.test.ts": {+ "linux": 0.4,+ "darwin": 0.5,+ "win32": 0.7+ },+ "custom-validation-example.test.ts": {+ "linux": 0.2,+ "darwin": 0.6,+ "win32": 0.3+ },+ "custom-validation.test.ts": {+ "linux": 16.1,+ "darwin": 19.0,+ "win32": 44.7+ },+ "dedupe-records.test.ts": {+ "linux": 0.7,+ "darwin": 1.2,+ "win32": 1.4+ },+ "deduplication-retry.test.ts": {+ "linux": 0.2,+ "darwin": 0.3,+ "win32": 0.9+ },+ "deep-config.test.ts": {+ "linux": 0.1,+ "darwin": 0.1,+ "win32": 0.2+ },+ "deep-progress.test.ts": {+ "linux": 0.1,+ "darwin": 0.1,+ "win32": 0.1+ },+ "deep-review-input.test.ts": {+ "linux": 6.2,+ "darwin": 9.4,+ "win32": 14.6+ },+ "deep-scan-parent-denials.test.ts": {+ "linux": 0.1,+ "darwin": 0.1,+ "win32": 0.1+ },+ "deep-scan-reducer-recovery.test.ts": {+ "linux": 0.3,+ "darwin": 0.4,+ "win32": 0.5+ },+ "deep-scan-windows-executable.test.ts": {+ "linux": 0.1,+ "darwin": 0.1,+ "win32": 0.1+ },+ "deep-scan-workbench.test.ts": {+ "linux": 13.8,+ "darwin": 19.2,+ "win32": 38.0+ },+ "deep-scan-worker-shutdown.test.ts": {+ "linux": 32.1,+ "darwin": 72.1,+ "win32": 33.8+ },+ "devcontainer-cooldown.test.ts": {+ "linux": 0.1,+ "darwin": 0.1,+ "win32": 0.1+ },+ "diff-rank-input.test.ts": {+ "linux": 0.8,+ "darwin": 1.7,+ "win32": 3.4+ },+ "direct-deep-scan.test.ts": {+ "linux": 0.1,+ "darwin": 0.4,+ "win32": 10.5+ },+ "effort-forwarding.test.ts": {+ "linux": 0.1,+ "darwin": 0.2,+ "win32": 0.2+ },+ "errors.property.test.ts": {+ "linux": 0.1,+ "darwin": 0.1,+ "win32": 0.1+ },+ "errors.test.ts": {+ "linux": 0.1,+ "darwin": 0.1,+ "win32": 0.1+ },+ "external-findings-publish.test.ts": {+ "linux": 6.0,+ "darwin": 10.5,+ "win32": 25.7+ },+ "feedback.test.ts": {+ "linux": 0.4,+ "darwin": 0.5,+ "win32": 1.8+ },+ "finding-catalogue.test.ts": {+ "linux": 21.4,+ "darwin": 10.3,+ "win32": 11.7+ },+ "finding-deduplication.test.ts": {+ "linux": 6.2,+ "darwin": 5.6,+ "win32": 6.5+ },+ "finding-embeddings.test.ts": {+ "linux": 2.6,+ "darwin": 2.5,+ "win32": 3.3+ },+ "finding-preview.test.ts": {+ "linux": 8.9,+ "darwin": 16.1,+ "win32": 50.4+ },+ "finding-workflow-integration.test.ts": {+ "linux": 76.6,+ "darwin": 151.4,+ "win32": 163.6+ },+ "finding-workflow-publication.test.ts": {+ "linux": 11.1,+ "darwin": 10.2,+ "win32": 14.3+ },+ "finding-workflow.test.ts": {+ "linux": 0.6,+ "darwin": 0.9,+ "win32": 4.6+ },+ "findings-client-retry.test.ts": {+ "linux": 0.1,+ "darwin": 0.1,+ "win32": 0.1+ },+ "findings-import-literal-fields.test.ts": {+ "linux": 0.1,+ "darwin": 0.1,+ "win32": 0.1+ },+ "findings-import-record-endings.test.ts": {+ "linux": 0.2,+ "darwin": 0.2,+ "win32": 0.6+ },+ "findings-import.test.ts": {+ "linux": 1.7,+ "darwin": 3.2,+ "win32": 3.6+ },+ "findings-store.test.ts": {+ "linux": 9.0,+ "darwin": 12.8,+ "win32": 16.0+ },+ "git-remote.test.ts": {+ "linux": 0.1,+ "darwin": 0.1,+ "win32": 0.1+ },+ "github-actions-example.test.ts": {+ "linux": 0.1,+ "darwin": 0.1,+ "win32": 0.4+ },+ "github.test.ts": {+ "linux": 0.2,+ "darwin": 0.4,+ "win32": 0.4+ },+ "host-protocol-node.test.ts": {+ "linux": 7.5,+ "darwin": 6.2,+ "win32": 12.8+ },+ "hosted-remediation.test.ts": {+ "linux": 1.2,+ "darwin": 3.3,+ "win32": 7.4+ },+ "hosted-scan.test.ts": {+ "linux": 39.7,+ "darwin": 76.2,+ "win32": 174.1+ },+ "import-scan.test.ts": {+ "linux": 30.5,+ "darwin": 34.4,+ "win32": 55.5+ },+ "invoice-desk-workflow.test.ts": {+ "linux": 0.2,+ "darwin": 0.5,+ "win32": 1.4+ },+ "knowledge-base.test.ts": {+ "linux": 1.7,+ "darwin": 2.0,+ "win32": 8.6+ },+ "linear-comment-pagination.test.ts": {+ "linux": 0.1,+ "darwin": 0.1,+ "win32": 0.1+ },+ "linear-pagination.test.ts": {+ "linux": 0.1,+ "darwin": 0.1,+ "win32": 0.1+ },+ "linear.test.ts": {+ "linux": 0.1,+ "darwin": 0.1,+ "win32": 0.1+ },+ "local-dedupe.test.ts": {+ "linux": 71.6,+ "darwin": 75.9,+ "win32": 143.2+ },+ "mcp-launcher.test.ts": {+ "linux": 1.3,+ "darwin": 2.6,+ "win32": 7.6+ },+ "mock-scan.test.ts": {+ "linux": 10.8,+ "darwin": 17.6,+ "win32": 44.1+ },+ "multiscan.test.ts": {+ "linux": 42.9,+ "darwin": 67.3,+ "win32": 216.6+ },+ "package-provenance.test.ts": {+ "linux": 0.1,+ "darwin": 0.1,+ "win32": 0.1+ },+ "package-public-content.test.ts": {+ "linux": 0.9,+ "darwin": 1.2,+ "win32": 1.1+ },+ "package-smoke-timeouts.test.ts": {+ "linux": 0.1,+ "darwin": 0.1,+ "win32": 0.1+ },+ "package-tar-entries.test.ts": {+ "linux": 0.1,+ "darwin": 0.1,+ "win32": 0.1+ },+ "package-tar-listing.test.ts": {+ "linux": 19.8,+ "darwin": 45.3,+ "win32": 91.7+ },+ "patch-publication.test.ts": {+ "linux": 0.1,+ "darwin": 0.2,+ "win32": 0.9+ },+ "patch-risk-contract.test.ts": {+ "linux": 14.4,+ "darwin": 29.8,+ "win32": 48.1+ },+ "patch-tui.test.ts": {+ "linux": 2.0,+ "darwin": 3.1,+ "win32": 16.2+ },+ "plugin-apps.test.ts": {+ "linux": 0.1,+ "darwin": 0.1,+ "win32": 0.1+ },+ "plugin-finding-detail-contract.test.ts": {+ "linux": 0.2,+ "darwin": 0.3,+ "win32": 0.5+ },+ "plugin-report-limits.test.ts": {+ "linux": 0.5,+ "darwin": 0.7,+ "win32": 0.7+ },+ "project-config.test.ts": {+ "linux": 0.1,+ "darwin": 0.1,+ "win32": 0.1+ },+ "prompt-only-start-timeout.test.ts": {+ "linux": 0.1,+ "darwin": 0.1,+ "win32": 0.1+ },+ "provider-profile.test.ts": {+ "linux": 1.5,+ "darwin": 2.9,+ "win32": 9.9+ },+ "publication-check.test.ts": {+ "linux": 0.1,+ "darwin": 0.1,+ "win32": 0.1+ },+ "publication-events.test.ts": {+ "linux": 0.1,+ "darwin": 0.1,+ "win32": 0.1+ },+ "publication-integration.test.ts": {+ "linux": 15.1,+ "darwin": 21.4,+ "win32": 31.7+ },+ "publication-store.test.ts": {+ "linux": 22.2,+ "darwin": 25.1,+ "win32": 44.6+ },+ "publication.test.ts": {+ "linux": 2.1,+ "darwin": 2.0,+ "win32": 2.6+ },+ "publish.test.ts": {+ "linux": 3.5,+ "darwin": 4.9,+ "win32": 6.5+ },+ "published-install-smoke.test.ts": {+ "linux": 0.2,+ "darwin": 0.2,+ "win32": 0.6+ },+ "release-automation.test.ts": {+ "linux": 3.6,+ "darwin": 11.6,+ "win32": 25.3+ },+ "release-cut-workflow.test.ts": {+ "linux": 0.1,+ "darwin": 0.1,+ "win32": 0.1+ },+ "release-pr.test.ts": {+ "linux": 27.5,+ "darwin": 53.6,+ "win32": 226.9+ },+ "repository-findings.test.ts": {+ "linux": 0.1,+ "darwin": 0.3,+ "win32": 0.4+ },+ "request-metadata.test.ts": {+ "linux": 0.1,+ "darwin": 0.8,+ "win32": 0.7+ },+ "result.test.ts": {+ "linux": 0.1,+ "darwin": 0.1,+ "win32": 0.1+ },+ "runtime-plugin-reuse.test.ts": {+ "linux": 0.4,+ "darwin": 1.9,+ "win32": 1.4+ },+ "runtime-zip-descriptor.test.ts": {+ "linux": 0.4,+ "darwin": 0.5,+ "win32": 0.7+ },+ "runtime.test.ts": {+ "linux": 39.7,+ "darwin": 41.6,+ "win32": 195.4+ },+ "sandbox-probe.test.ts": {+ "linux": 0.1,+ "darwin": 0.1,+ "win32": 0.1+ },+ "sarif.test.ts": {+ "linux": 0.8,+ "darwin": 1.2,+ "win32": 1.2+ },+ "saved-scan-bootstrap.test.ts": {+ "linux": 118.8,+ "darwin": 119.0,+ "win32": 103.7+ },+ "scan-activity.test.ts": {+ "linux": 0.1,+ "darwin": 0.1,+ "win32": 0.1+ },+ "scan-comparison.property.test.ts": {+ "linux": 0.4,+ "darwin": 0.3,+ "win32": 0.5+ },+ "scan-comparison.test.ts": {+ "linux": 4.6,+ "darwin": 5.0,+ "win32": 127.7+ },+ "scan-dashboard.test.ts": {+ "linux": 4.5,+ "darwin": 21.5,+ "win32": 40.1+ },+ "scan-history-renderer.test.ts": {+ "linux": 0.1,+ "darwin": 0.1,+ "win32": 0.1+ },+ "scan-logs-node.test.ts": {+ "linux": 0.1,+ "darwin": 0.2,+ "win32": 0.2+ },+ "scan-logs.test.ts": {+ "linux": 0.1,+ "darwin": 0.3,+ "win32": 0.3+ },+ "scan-matching-e2e.test.ts": {+ "linux": 7.4,+ "darwin": 12.4,+ "win32": 17.5+ },+ "scan-mcp-startup.test.ts": {+ "linux": 1.9,+ "darwin": 2.1,+ "win32": 8.7+ },+ "scan-owner-suggestions.test.ts": {+ "linux": 0.3,+ "darwin": 1.3,+ "win32": 9.4+ },+ "scan-progress.test.ts": {+ "linux": 0.1,+ "darwin": 0.1,+ "win32": 0.1+ },+ "scan-recipe.test.ts": {+ "linux": 12.2,+ "darwin": 12.5,+ "win32": 35.5+ },+ "scan-recovery.test.ts": {+ "linux": 15.2,+ "darwin": 32.8,+ "win32": 66.1+ },+ "scan-resume.test.ts": {+ "linux": 97.0,+ "darwin": 164.8,+ "win32": 231.9+ },+ "scan-skill-paths.test.ts": {+ "linux": 0.1,+ "darwin": 0.1,+ "win32": 0.1+ },+ "scanner-import.test.ts": {+ "linux": 0.1,+ "darwin": 0.1,+ "win32": 0.1+ },+ "script-entrypoints.test.ts": {+ "linux": 3.1,+ "darwin": 6.1,+ "win32": 7.0+ },+ "sdk-project-config.test.ts": {+ "linux": 0.2,+ "darwin": 0.3,+ "win32": 1.4+ },+ "sdk-scan-prompts.test.ts": {+ "linux": 0.9,+ "darwin": 1.1,+ "win32": 3.3+ },+ "security-policy-diff.test.ts": {+ "linux": 0.1,+ "darwin": 0.1,+ "win32": 0.1+ },+ "security-policy-helper.test.ts": {+ "linux": 8.3,+ "darwin": 9.5,+ "win32": 9.7+ },+ "security-policy.test.ts": {+ "linux": 8.4,+ "darwin": 24.5,+ "win32": 82.5+ },+ "severity-review.test.ts": {+ "linux": 0.1,+ "darwin": 0.1,+ "win32": 0.1+ },+ "shell.test.ts": {+ "linux": 0.1,+ "darwin": 0.1,+ "win32": 0.1+ },+ "skeleton.test.ts": {+ "linux": 0.1,+ "darwin": 0.1,+ "win32": 0.4+ },+ "stopped-scan-results.test.ts": {+ "linux": 8.0,+ "darwin": 10.9,+ "win32": 19.3+ },+ "suggest-owners.test.ts": {+ "linux": 2.9,+ "darwin": 11.8,+ "win32": 70.5+ },+ "targets-large-output.test.ts": {+ "linux": 0.4,+ "darwin": 1.3,+ "win32": 4.8+ },+ "targets.test.ts": {+ "linux": 0.8,+ "darwin": 3.9,+ "win32": 12.0+ },+ "test-reports.test.ts": {+ "linux": 3.1,+ "darwin": 4.6,+ "win32": 6.0+ },+ "test-shards.test.ts": {+ "linux": 2.3,+ "darwin": 5.3,+ "win32": 6.1+ },+ "trusted-executable.test.ts": {+ "linux": 0.6,+ "darwin": 1.1,+ "win32": 0.1+ },+ "update-notice.test.ts": {+ "linux": 0.1,+ "darwin": 0.1,+ "win32": 0.1+ },+ "validation-records.test.ts": {+ "linux": 0.1,+ "darwin": 0.3,+ "win32": 0.3+ },+ "windows-scan-output.test.ts": {+ "linux": 0.1,+ "darwin": 0.1,+ "win32": 16.6+ },+ "wiz-findings.test.ts": {+ "linux": 4.4,+ "darwin": 5.4,+ "win32": 10.6+ },+ "workbench-arguments.test.ts": {+ "linux": 0.1,+ "darwin": 0.1,+ "win32": 0.1+ },+ "workbench-canonical-paths.test.ts": {+ "linux": 0.8,+ "darwin": 2.3,+ "win32": 4.9+ },+ "workbench-nested-git-utf8.test.ts": {+ "linux": 0.1,+ "darwin": 0.2,+ "win32": 1.6+ },+ "workbench-scan-history.test.ts": {+ "linux": 1.5,+ "darwin": 2.3,+ "win32": 3.9+ },+ "workbench-scan-root-alias.test.ts": {+ "linux": 0.2,+ "darwin": 0.3,+ "win32": 0.6+ },+ "workbench-windows-compatibility.test.ts": {+ "linux": 0.5,+ "darwin": 1.1,+ "win32": 2.0+ },+ "worker-progress.property.test.ts": {+ "linux": 0.1,+ "darwin": 0.1,+ "win32": 0.1+ },+ "worker-progress.test.ts": {+ "linux": 0.1,+ "darwin": 0.1,+ "win32": 0.1+ },+ "workflow-quality.test.ts": {+ "linux": 0.1,+ "darwin": 0.2,+ "win32": 0.1+ }+}diff --git a/sdk/typescript/scripts/update-test-timings.mts b/sdk/typescript/scripts/update-test-timings.mtsnew file mode 100644index 000000000..ee30cf0bb--- /dev/null+++ b/sdk/typescript/scripts/update-test-timings.mts@@ -0,0 +1,86 @@+import { readFile, writeFile } from "node:fs/promises";+import { resolve } from "node:path";+import { fileURLToPath } from "node:url";+import { SaxesParser } from "saxes";+import type { TestTimings } from "./test-shards.mts";++export function updateTestTimings(+ previous: TestTimings,+ platform: "linux" | "darwin" | "win32",+ reports: readonly string[],+): TestTimings {+ const samples = new Map<string, number[]>();+ for (const report of reports) {+ const totals = new Map<string, number>();+ const parser = new SaxesParser({ xmlns: false });+ parser.on("opentag", ({ name, attributes }) => {+ if (+ name === "failure" ||+ name === "error" ||+ Number(attributes["failures"] ?? 0) > 0 ||+ Number(attributes["errors"] ?? 0) > 0+ ) {+ throw new Error(+ "Use successful JUnit reports to refresh test timings.",+ );+ }+ if (name !== "testcase") return;+ const file = attributes["file"]?.replaceAll("\\", "/").split("/").at(-1);+ const seconds = Number(attributes["time"] ?? 0);+ if (!file || !Number.isFinite(seconds) || seconds < 0) {+ throw new Error(+ "JUnit testcase needs a file and a nonnegative duration.",+ );+ }+ totals.set(file, (totals.get(file) ?? 0) + seconds);+ });+ parser.write(report).close();+ if (!totals.size) throw new Error("JUnit report contains no test cases.");+ for (const [file, seconds] of totals) {+ const values = samples.get(file) ?? [];+ values.push(seconds);+ samples.set(file, values);+ }+ }++ const updated = { ...previous };+ for (const [file, values] of samples) {+ values.sort((left, right) => left - right);+ const middle = Math.floor(values.length / 2);+ const median =+ values.length % 2+ ? values[middle]!+ : (values[middle - 1]! + values[middle]!) / 2;+ updated[file] = {+ ...previous[file],+ [platform]: Math.max(0.1, Math.round(median * 10) / 10),+ };+ }+ return Object.fromEntries(+ Object.keys(updated)+ .sort()+ .map((file) => [file, updated[file]!]),+ );+}++if (+ process.argv[1] &&+ fileURLToPath(import.meta.url) === resolve(process.argv[1])+) {+ const [platform, ...paths] = process.argv.slice(2);+ if (+ !paths.length ||+ (platform !== "linux" && platform !== "darwin" && platform !== "win32")+ ) {+ throw new Error(+ "Usage: node --experimental-strip-types scripts/update-test-timings.mts <linux|darwin|win32> <junit.xml> [...]",+ );+ }+ const target = new URL("./test-timings.json", import.meta.url);+ const previous = JSON.parse(await readFile(target, "utf8")) as TestTimings;+ const reports = await Promise.all(+ paths.map((path) => readFile(path, "utf8")),+ );+ const updated = updateTestTimings(previous, platform, reports);+ await writeFile(target, `${JSON.stringify(updated, null, 2)}\n`);+}diff --git a/sdk/typescript/tests-ts/test-shards.test.ts b/sdk/typescript/tests-ts/test-shards.test.tsindex ebf27282c..8d55cdd5d 100644--- a/sdk/typescript/tests-ts/test-shards.test.ts+++ b/sdk/typescript/tests-ts/test-shards.test.ts@@ -9,9 +9,62 @@ import {import { tmpdir } from "node:os";import { delimiter, dirname, join } from "node:path";import { expect, test } from "bun:test";+import { partitionTests, type TestTimings } from "../scripts/test-shards.mts";import { readSubprocess } from "./support/shell.js";import { temporaryDirectory } from "./support/temporary-directories.js";+async function copyRunner(root: string, timings: TestTimings = {}) {+ for (const name of ["run-ci-tests.mts", "test-shards.mts"]) {+ await copyFile(+ new URL(`../scripts/${name}`, import.meta.url),+ join(root, "scripts", name),+ );+ }+ await writeFile(+ join(root, "scripts", "test-timings.json"),+ JSON.stringify(timings),+ );+}++test.each(["linux", "darwin", "win32"] as const)(+ "balances whole files using %s durations without depending on discovery order",+ (platform) => {+ const files = ["a", "b", "c", "d", "e", "f"];+ const timings = Object.fromEntries(+ files.map((file, index) => [file, { [platform]: 100 - index * 10 }]),+ );+ const shards = partitionTests(files, 3, timings, platform);+ expect(shards).toEqual([+ ["a", "f"],+ ["b", "e"],+ ["c", "d"],+ ]);+ expect(partitionTests(files.toReversed(), 3, timings, platform)).toEqual(+ shards,+ );+ expect(files).toEqual(["a", "b", "c", "d", "e", "f"]);+ },+);++test("uses each platform's history and includes new files without selecting deleted files", () => {+ const files = ["a", "b", "c", "new"];+ const timings = {+ a: { linux: 100, win32: 1 },+ b: { linux: 1, win32: 100 },+ c: { linux: 1, win32: 1 },+ deleted: { linux: 1_000, win32: 1_000 },+ };+ expect(partitionTests(files, 2, timings, "linux")).toEqual([+ ["a"],+ ["b", "c", "new"],+ ]);+ expect(partitionTests(files, 2, timings, "win32")).toEqual([+ ["b"],+ ["a", "c", "new"],+ ]);+ expect(partitionTests(files, 2, {}, "darwin").flat().sort()).toEqual(files);+});+test.each([3, 7])("runs each eligible file once across %i native shards, including new files",async (count) => {@@ -21,10 +74,11 @@ test.each([3, 7])(try {await mkdir(join(root, "scripts"));await mkdir(join(root, "tests-ts"));- await copyFile(- new URL("../scripts/run-ci-tests.mts", import.meta.url),- join(root, "scripts", "run-ci-tests.mts"),- );+ await copyRunner(root, {+ "probe-0.test.ts": { linux: 100, darwin: 100, win32: 100 },+ "probe-1.test.ts": { linux: 80, darwin: 80, win32: 80 },+ "deleted.test.ts": { linux: 1_000, darwin: 1_000, win32: 1_000 },+ });const files = [...Array.from({ length: 20 }, (_, index) => `probe-${index}.test.ts`),"newly-added.test.ts",@@ -88,10 +142,7 @@ test("rejects an empty shard while allowing a nonempty shard with the same counttry {await mkdir(join(root, "scripts"));await mkdir(join(root, "tests-ts"));- await copyFile(- new URL("../scripts/run-ci-tests.mts", import.meta.url),- join(root, "scripts", "run-ci-tests.mts"),- );+ await copyRunner(root);await writeFile(join(root, "tests-ts", "probe.test.ts"),'import { test } from "bun:test"; test("synthetic shard probe", () => {});\n',@@ -167,10 +218,7 @@ test.each([try {await mkdir(join(root, "scripts"));await mkdir(join(root, "tests-ts"));- await copyFile(- new URL("../scripts/run-ci-tests.mts", import.meta.url),- join(root, "scripts", "run-ci-tests.mts"),- );+ await copyRunner(root);await writeFile(join(root, "tests-ts", "probe.test.ts"),`import { expect, test } from "bun:test";diff --git a/sdk/typescript/tests-ts/test-timings.test.ts b/sdk/typescript/tests-ts/test-timings.test.tsnew file mode 100644index 000000000..02f253da1--- /dev/null+++ b/sdk/typescript/tests-ts/test-timings.test.ts@@ -0,0 +1,72 @@+import { expect, test } from "bun:test";+import { updateTestTimings } from "../scripts/update-test-timings.mts";++function report(cases: string) {+ return `<testsuites failures="0"><testsuite>${cases}</testsuite></testsuites>`;+}++test("refreshes per-file medians while retaining other platforms and undiscovered files", () => {+ const previous = {+ "slow.test.ts": { linux: 100, darwin: 20, win32: 50 },+ "stale.test.ts": { linux: 2, darwin: 3 },+ };+ const updated = updateTestTimings(previous, "linux", [+ report(`+ <testcase file="tests-ts/slow.test.ts" time="2" />+ <testcase file="tests-ts/slow.test.ts" time="4" />+ <testcase file="tests-ts/new.test.ts" time="0.24" />+ `),+ report('<testcase file="tests-ts/slow.test.ts" time="100" />'),+ report('<testcase file="tests-ts/slow.test.ts" time="10" />'),+ ]);+ expect(updated).toEqual({+ "new.test.ts": { linux: 0.2 },+ "slow.test.ts": { linux: 10, darwin: 20, win32: 50 },+ "stale.test.ts": { linux: 2, darwin: 3 },+ });+ expect(previous["slow.test.ts"].linux).toBe(100);+ expect(Object.keys(updated)).toEqual([+ "new.test.ts",+ "slow.test.ts",+ "stale.test.ts",+ ]);+});++test("handles XML-escaped and Windows file paths and zero-duration skipped cases", () => {+ const updated = updateTestTimings({}, "win32", [+ report(String.raw`+ <testcase file="C:\checkout\tests-ts\names&paths.test.ts" time="1.2" />+ <testcase file="tests-ts/names&paths.test.ts" time="0.5" />+ <testcase file="tests-ts/skipped.test.ts"><skipped /></testcase>+ `),+ report('<testcase file="tests-ts/names&paths.test.ts" time="3.1" />'),+ ]);+ expect(updated).toEqual({+ "names&paths.test.ts": { win32: 2.4 },+ "skipped.test.ts": { win32: 0.1 },+ });+});++test.each([+ '<testsuites failures="1"><testcase file="tests-ts/example.test.ts" time="3" /></testsuites>',+ report(+ '<testcase file="tests-ts/example.test.ts" time="3"><failure /></testcase>',+ ),+ report('<testcase file="tests-ts/example.test.ts" time="NaN" />'),+ report('<testcase file="tests-ts/example.test.ts" time="-1" />'),+ report('<testcase time="1" />'),+ report(""),+ "<testsuites>",+])(+ "rejects unusable reports without altering the timing snapshot: %s",+ (invalid) => {+ const previous = { "example.test.ts": { darwin: 5 } };+ expect(() =>+ updateTestTimings(previous, "darwin", [+ report('<testcase file="tests-ts/example.test.ts" time="10" />'),+ invalid,+ ]),+ ).toThrow();+ expect(previous).toEqual({ "example.test.ts": { darwin: 5 } });+ },+);diff --git a/sdk/typescript/tsconfig.json b/sdk/typescript/tsconfig.jsonindex ad25f8e4f..641b4fe4d 100644--- a/sdk/typescript/tsconfig.json+++ b/sdk/typescript/tsconfig.json@@ -7,7 +7,8 @@"../../plugins/codex-security/native/*.mts","../../examples/custom-validation/*.mts","scripts/compare-test-reports.mts",- "scripts/run-ci-tests.mts"+ "scripts/run-ci-tests.mts",+ "scripts/update-test-timings.mts"],"compilerOptions": {"allowJs": true,