OpenAI's Codex Security CLI and TypeScript SDK for finding, validating, and fixing security vulnerabilities. npm: https://www.npmjs.com/package/@openai/codex-security
TypeScript ai-securityapplication-securityclicode-scanningcodexcodex-securitycybersecuritydevsecopsnodejsnpmopenaisdksecuritytypescriptvulnerability-scanning
ci: reuse one package and balance test jobs (#734)
mldangelo-oaie2d329744d27
diff --git a/.github/workflows/node-ci.yml b/.github/workflows/node-ci.ymlindex 6171278df..2db2c9f07 100644--- a/.github/workflows/node-ci.yml+++ b/.github/workflows/node-ci.yml@@ -120,33 +120,73 @@ jobs:pnpm --dir sdk/typescript exec prettier --check "${files[@]}"fi- test:- name: tests / ${{ matrix.os }} / node-${{ matrix.node == '22.13.0' && '22' || matrix.node }}+ package:+ name: build and check packageneeds: validate-titleif: needs.validate-title.outputs.ci-mode == 'full'+ runs-on: ubuntu-latest+ timeout-minutes: 20+ steps:+ - name: Checkout repository+ uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2+ with:+ persist-credentials: false+ - name: Check plugin source boundary+ run: node sdk/typescript/scripts/check-plugin-source.mjs+ - name: Set up pnpm+ uses: pnpm/action-setup@0ebf47130e4866e96fce0953f49152a61190b271 # v6.0.9+ with:+ package_json_file: sdk/typescript/package.json+ cache: true+ cache_dependency_path: |+ sdk/typescript/pnpm-lock.yaml+ plugins/codex-security/mcp-app/pnpm-lock.yaml+ - name: Set up Node.js+ uses: actions/setup-node@53b83947a5a98c8d113130e565377fae1a50d02f # v6+ with:+ node-version: "22.13.0"+ - name: Install dependencies+ run: |+ pnpm --dir sdk/typescript install --frozen-lockfile+ pnpm --dir plugins/codex-security/mcp-app install --frozen-lockfile+ - name: Audit production dependencies+ continue-on-error: true+ run: pnpm --dir sdk/typescript run audit:prod+ - name: Typecheck+ run: pnpm --dir sdk/typescript run types+ - name: Check formatting+ run: pnpm --dir sdk/typescript run format+ - name: Pack+ working-directory: sdk/typescript+ run: pnpm pack --pack-destination ../../dist+ - name: Inspect archive contents+ working-directory: sdk/typescript+ run: node scripts/check-package.mjs ../../dist/*.tgz ../../plugins/codex-security/plugin-files.json+ - name: Upload package for this commit+ uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1+ with:+ name: package-${{ github.sha }}+ overwrite: true+ path: dist/*.tgz+ if-no-files-found: error+ retention-days: 7++ test:+ name: tests / ${{ matrix.os }} / shard-${{ matrix.shard }}+ needs: [validate-title, package]+ if: needs.validate-title.outputs.ci-mode == 'full'runs-on: ${{ matrix.os }}timeout-minutes: 20strategy:fail-fast: falsematrix:os: [ubuntu-latest, macos-latest]- node: ["22.13.0"]- include:- - os: ubuntu-latest- node: "24.0.0"- - os: ubuntu-latest- node: "24"- - os: ubuntu-latest- node: "26.0.0"- - os: ubuntu-latest- node: "26"-+ shard: [1, 2, 3]steps:- name: Checkout repositoryuses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2with:persist-credentials: false-- name: Set up pnpmuses: pnpm/action-setup@0ebf47130e4866e96fce0953f49152a61190b271 # v6.0.9with:@@ -155,79 +195,132 @@ jobs:cache_dependency_path: |sdk/typescript/pnpm-lock.yamlplugins/codex-security/mcp-app/pnpm-lock.yaml-- name: Set up Node.jsuses: actions/setup-node@53b83947a5a98c8d113130e565377fae1a50d02f # v6with:- node-version: ${{ matrix.node }}-- - name: Check plugin source boundary- if: matrix.os == 'ubuntu-latest' && matrix.node == '22.13.0'- run: node sdk/typescript/scripts/check-plugin-source.mjs-+ node-version: "22.13.0"- name: Set up Bunuses: oven-sh/setup-bun@0c5077e51419868618aeaa5fe8019c62421857d6 # v2with:bun-version: "1.3.14"-- name: Install dependenciesrun: |pnpm --dir sdk/typescript install --frozen-lockfilepnpm --dir plugins/codex-security/mcp-app install --frozen-lockfile-+ - name: Download package for this commit+ uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1+ with:+ name: package-${{ github.sha }}+ path: dist+ - name: Restore bundled plugin+ run: tar -xzf dist/*.tgz --strip-components=1 -C sdk/typescript package/_bundled_plugin- name: Install ripgrep- if: matrix.os == 'ubuntu-latest' && matrix.node == '22.13.0'+ if: matrix.os == 'ubuntu-latest'run: |sudo apt-get updatesudo apt-get install --yes ripgrep-- - name: Audit production dependencies- if: matrix.os == 'ubuntu-latest' && matrix.node == '22.13.0'- continue-on-error: true- run: pnpm --dir sdk/typescript run audit:prod-- - name: Typecheck- if: matrix.os == 'ubuntu-latest' && matrix.node == '22.13.0'- run: pnpm --dir sdk/typescript run types-- name: Testtimeout-minutes: 10+ working-directory: sdk/typescriptenv:TEMP: ${{ runner.temp }}TMP: ${{ runner.temp }}TMPDIR: ${{ runner.temp }}CODEX_SECURITY_ALLOW_MACHINE_POLICY_TEST: "false"- run: pnpm --dir sdk/typescript run ${{ matrix.os == 'ubuntu-latest' && matrix.node == '22.13.0' && 'test:ci' || 'test' }}-- - name: Test MCP app- if: matrix.os == 'ubuntu-latest' && matrix.node == '22.13.0'- run: pnpm --dir sdk/typescript run test:mcp-+ run: node scripts/run-ci-tests.mjs ${{ matrix.shard }}/3 ${{ matrix.os == 'ubuntu-latest' && '--coverage --coverage-reporter=text --coverage-reporter=lcov' || '' }}- name: Upload test reports- if: always() && matrix.os == 'ubuntu-latest' && matrix.node == '22.13.0'+ if: always()continue-on-error: trueuses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1with:- name: node-22-test-reports+ name: bun-${{ runner.os }}-${{ matrix.shard }}overwrite: truepath: |- sdk/typescript/reports/junit.xml+ sdk/typescript/reports/junit-*.xmlsdk/typescript/coverage/lcov.infoif-no-files-found: warnretention-days: 14- - name: Check formatting- if: matrix.os == 'ubuntu-latest' && matrix.node == '22.13.0'- run: pnpm --dir sdk/typescript run format-- - name: Pack+ compatibility:+ name: installed package / ${{ matrix.os }} / node-${{ matrix.node }}+ needs: [validate-title, package]+ if: needs.validate-title.outputs.ci-mode == 'full'+ runs-on: ${{ matrix.os }}+ timeout-minutes: 20+ strategy:+ fail-fast: false+ matrix:+ os: [ubuntu-latest]+ node: ["22.13.0", "24.0.0", "24", "26.0.0", "26"]+ include:+ - os: macos-latest+ node: "22.13.0"+ steps:+ - name: Checkout repository+ uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2+ with:+ persist-credentials: false+ - name: Set up Node.js+ uses: actions/setup-node@53b83947a5a98c8d113130e565377fae1a50d02f # v6+ with:+ node-version: ${{ matrix.node }}+ cache: npm+ cache-dependency-path: sdk/typescript/pnpm-lock.yaml+ - name: Download package for this commit+ uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1+ with:+ name: package-${{ github.sha }}+ path: dist+ - name: Inspect installed packageworking-directory: sdk/typescript- run: pnpm pack --pack-destination ../../dist+ run: node scripts/check-package.mjs ../../dist/*.tgz- - name: Inspect package- working-directory: sdk/typescript- shell: bash- run: pnpm run check:package ../../dist/*.tgz+ mcp:+ name: MCP app tests+ needs: [validate-title, package]+ if: needs.validate-title.outputs.ci-mode == 'full'+ runs-on: ubuntu-latest+ timeout-minutes: 20+ steps:+ - name: Checkout repository+ uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2+ with:+ persist-credentials: false+ - name: Set up pnpm+ uses: pnpm/action-setup@0ebf47130e4866e96fce0953f49152a61190b271 # v6.0.9+ with:+ package_json_file: plugins/codex-security/mcp-app/package.json+ cache: true+ cache_dependency_path: plugins/codex-security/mcp-app/pnpm-lock.yaml+ - name: Set up Node.js+ uses: actions/setup-node@53b83947a5a98c8d113130e565377fae1a50d02f # v6+ with:+ node-version: "22.13.0"+ - name: Install dependencies+ run: pnpm --dir plugins/codex-security/mcp-app install --frozen-lockfile+ - name: Install ripgrep+ run: |+ sudo apt-get update+ sudo apt-get install --yes ripgrep+ - name: Download package for this commit+ uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1+ with:+ name: package-${{ github.sha }}+ path: dist+ - name: Restore bundled plugin+ run: tar -xzf dist/*.tgz --strip-components=1 -C sdk/typescript package/_bundled_plugin+ - name: Test MCP app+ run: pnpm --dir plugins/codex-security/mcp-app run test:mcp+ - name: Upload MCP test reports+ if: always()+ continue-on-error: true+ uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1+ with:+ name: mcp-test-reports+ overwrite: true+ path: plugins/codex-security/mcp-app/reports/junit.xml+ if-no-files-found: warn+ retention-days: 14plugin-source:name: plugin source contracts@@ -235,105 +328,103 @@ jobs:if: needs.validate-title.outputs.ci-mode == 'full'runs-on: ubuntu-latesttimeout-minutes: 30-steps:- name: Checkout repositoryuses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2with:persist-credentials: false-- name: Set up Pythonuses: actions/setup-python@ece7cb06caefa5fff74198d8649806c4678c61a1 # v6with:python-version: "3.12"cache: pipcache-dependency-path: plugins/codex-security/pyproject.toml-- name: Install Python test dependenciesrun: python -m pip install --disable-pip-version-check --no-input -e 'plugins/codex-security[test]'-- name: Install ripgreprun: |sudo apt-get updatesudo apt-get install --yes ripgrep-- name: Check plugin source compatibilityrun: |python -m ruff check --config plugins/codex-security/pyproject.toml .github/scripts/check_plugin_source_compatibility.py .github/scripts/test_check_plugin_source_compatibility.py plugins/codex-securitypython -m ruff format --check --config plugins/codex-security/pyproject.toml .github/scripts/check_plugin_source_compatibility.py .github/scripts/test_check_plugin_source_compatibility.py plugins/codex-securitypython .github/scripts/check_plugin_source_compatibility.py-- name: Test Python source contracts- run: python -m pytest .github/scripts/test_check_plugin_source_compatibility.py plugins/codex-security/tests -q-+ run: python -m pytest .github/scripts/test_check_plugin_source_compatibility.py plugins/codex-security/tests -q -n 4 --dist worksteal --max-worker-restart 0 --durations=30 --junitxml=reports/python.xml+ - name: Upload Python test reports+ if: always()+ continue-on-error: true+ uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1+ with:+ name: python-test-reports+ overwrite: true+ path: reports/python.xml+ if-no-files-found: warn+ retention-days: 14- name: Set up Node.jsuses: actions/setup-node@53b83947a5a98c8d113130e565377fae1a50d02f # v6with:node-version: "22.13.0"-- name: Test deterministic triage eval contractsworking-directory: plugins/codex-security/skills/triage-finding/evalsrun: node --run test:deterministicrequired-test:name: ${{ matrix.os }} / node-22- needs: [validate-title, test, plugin-source]+ needs: [validate-title, package, test, compatibility, mcp, plugin-source]if: always()runs-on: ubuntu-lateststrategy:fail-fast: falsematrix:os: [ubuntu-latest, macos-latest]-steps:- name: Require every Unix coverage job- if: needs.validate-title.result != 'success' || (needs.validate-title.outputs.ci-mode == 'full' && (needs.test.result != 'success' || needs.plugin-source.result != 'success')) || (needs.validate-title.outputs.ci-mode != 'full' && needs.validate-title.outputs.ci-mode != 'markdown')+ if: needs.validate-title.result != 'success' || (needs.validate-title.outputs.ci-mode == 'full' && (needs.package.result != 'success' || needs.test.result != 'success' || needs.compatibility.result != 'success' || needs.mcp.result != 'success' || needs.plugin-source.result != 'success')) || (needs.validate-title.outputs.ci-mode != 'full' && needs.validate-title.outputs.ci-mode != 'markdown')run: exit 1windows-test:- name: windows-latest / node-${{ matrix.node == '22.13.0' && '22' || matrix.node }} / tests-${{ matrix.shard }}- needs: validate-title+ name: windows-latest / node-22 / tests-${{ matrix.shard }}+ needs: [validate-title, package]if: needs.validate-title.outputs.ci-mode == 'full'runs-on: windows-latesttimeout-minutes: 20strategy:fail-fast: falsematrix:- node: ["22.13.0", "24"]shard: [1, 2, 3, 4, 5, 6, 7]-steps:- name: Checkout repositoryuses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2with:persist-credentials: false-- name: Set up Node.jsuses: actions/setup-node@53b83947a5a98c8d113130e565377fae1a50d02f # v6with:- node-version: ${{ matrix.node }}-+ node-version: "22.13.0"- name: Set up pnpmrun: npm install --global [email protected] --no-audit --no-fund-- name: Set up Bunuses: oven-sh/setup-bun@0c5077e51419868618aeaa5fe8019c62421857d6 # v2with:bun-version: "1.3.14"-- name: Install dependenciesrun: |pnpm --dir sdk/typescript install --frozen-lockfilepnpm --dir plugins/codex-security/mcp-app install --frozen-lockfile-- - name: Generate bundled plugin- run: pnpm --dir sdk/typescript run build:plugin-+ - name: Download package for this commit+ uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1+ with:+ name: package-${{ github.sha }}+ path: dist+ - name: Restore bundled plugin+ shell: bash+ run: tar -xzf dist/*.tgz --strip-components=1 -C sdk/typescript package/_bundled_plugin- name: Prepare private Windows test rootid: windows-tempshell: pwshrun: ./sdk/typescript/scripts/prepare-windows-test-root.ps1-- name: Test shard ${{ matrix.shard }}timeout-minutes: 10env:@@ -341,8 +432,17 @@ jobs:TMP: ${{ steps.windows-temp.outputs.path }}TMPDIR: ${{ steps.windows-temp.outputs.path }}CODEX_SECURITY_ALLOW_MACHINE_POLICY_TEST: "false"- run: node sdk/typescript/scripts/run-windows-ci-tests.mjs ${{ matrix.shard }}-+ run: node sdk/typescript/scripts/run-ci-tests.mjs ${{ matrix.shard }}/7+ - name: Upload Windows test reports+ if: always()+ continue-on-error: true+ uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1+ with:+ name: bun-Windows-${{ matrix.shard }}+ overwrite: true+ path: sdk/typescript/reports/junit-*.xml+ if-no-files-found: warn+ retention-days: 14- name: Test machine-wide PowerShell policyif: matrix.shard == 3 && runner.environment == 'github-hosted'timeout-minutes: 5@@ -356,7 +456,7 @@ jobs:windows-verify:name: windows-latest / node-${{ matrix.node == '22.13.0' && '22' || matrix.node }} / verify- needs: validate-title+ needs: [validate-title, package]if: needs.validate-title.outputs.ci-mode == 'full'runs-on: windows-latesttimeout-minutes: 20@@ -364,34 +464,26 @@ jobs:fail-fast: falsematrix:node: ["22.13.0", "24"]-steps:- name: Checkout repositoryuses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2with:persist-credentials: false-- name: Set up Node.jsuses: actions/setup-node@53b83947a5a98c8d113130e565377fae1a50d02f # v6with:node-version: ${{ matrix.node }}-- - name: Set up pnpm- run: npm install --global [email protected] --no-audit --no-fund-- - name: Install dependencies- run: |- pnpm --dir sdk/typescript install --frozen-lockfile- pnpm --dir plugins/codex-security/mcp-app install --frozen-lockfile-- - name: Pack- working-directory: sdk/typescript- run: pnpm pack --pack-destination ../../dist-- - name: Inspect package+ cache: npm+ cache-dependency-path: sdk/typescript/pnpm-lock.yaml+ - name: Download package for this commit+ uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1+ with:+ name: package-${{ github.sha }}+ path: dist+ - name: Inspect installed packageworking-directory: sdk/typescriptshell: bash- run: pnpm run check:package ../../dist/*.tgz+ run: node scripts/check-package.mjs ../../dist/*.tgzwindows:name: windows-latest / node-${{ matrix.node == '22.13.0' && '22' || matrix.node }}@@ -402,7 +494,6 @@ jobs:fail-fast: falsematrix:node: ["22.13.0", "24"]-steps:- name: Require every Windows coverage jobif: needs.validate-title.result != 'success' || (needs.validate-title.outputs.ci-mode == 'full' && (needs.windows-test.result != 'success' || needs.windows-verify.result != 'success')) || (needs.validate-title.outputs.ci-mode != 'full' && needs.validate-title.outputs.ci-mode != 'markdown')diff --git a/.gitignore b/.gitignoreindex fe166502f..d6b8b94b5 100644--- a/.gitignore+++ b/.gitignore@@ -1,4 +1,5 @@node_modules/+/reports/dist/__pycache__/.pytest_cache/diff --git a/plugins/codex-security/pyproject.toml b/plugins/codex-security/pyproject.tomlindex 85f487962..ae53a7b2b 100644--- a/plugins/codex-security/pyproject.toml+++ b/plugins/codex-security/pyproject.toml@@ -8,6 +8,7 @@ test = ["jsonschema[format-nongpl]==4.26.0","packaging==26.3","pytest==9.0.3",+ "pytest-xdist==3.8.0","PyYAML==6.0.3","ruff==0.16.1",]diff --git a/plugins/codex-security/tests/test_finalize_scan_contract.py b/plugins/codex-security/tests/test_finalize_scan_contract.pyindex bb5dd5010..c9a50e501 100644--- a/plugins/codex-security/tests/test_finalize_scan_contract.py+++ b/plugins/codex-security/tests/test_finalize_scan_contract.py@@ -1202,7 +1202,7 @@ The extraction root is not enforced.parent_alias.symlink_to(self.scan_dir.parent, target_is_directory=True)for alias in (direct_alias, parent_alias / self.scan_dir.name):- with self.subTest(alias=alias):+ with self.subTest(alias=str(alias)):output.write_bytes(protected_contents)result = subprocess.run([@@ -2559,7 +2559,7 @@ The extraction root is not enforced.def test_recovery_rejects_malformed_severity_change_condition_lists(self) -> None:for change_conditions in ([], [" "], ["Valid condition.", 1], ["\ud800"]):- with self.subTest(change_conditions=change_conditions):+ with self.subTest(change_conditions=repr(change_conditions)):self.findings["findings"][0]["severity"]["changeConditions"] = change_conditionsself.write_scan()warnings: list[str] = []diff --git a/sdk/typescript/TESTING.md b/sdk/typescript/TESTING.mdindex 187a5b8fc..261ce99c6 100644--- a/sdk/typescript/TESTING.md+++ b/sdk/typescript/TESTING.md@@ -82,11 +82,32 @@ default to 100 cases; filesystem contract properties default to 20.## GitHub Actions`node-ci` retains the required `ubuntu-latest / node-22`,-`macos-latest / node-22`, and `windows-latest / node-22` checks. Its Ubuntu-Node 22 job runs static checks and uploads JUnit and LCOV. All supported runtime-lanes still test and inspect an installed package. Package inspection includes-a strict NodeNext TypeScript consumer and the actual installed CLI. Failed-tests block CI; a failed diagnostic upload does not.+`macos-latest / node-22`, and `windows-latest / node-22` checks. It builds and+inspects one package archive, then passes that archive to jobs in the same+workflow run, using the commit SHA in the artifact name. Every supported Node+runtime still installs and inspects the package, including a strict NodeNext+TypeScript consumer, the actual CLI, credential locking, dashboard assets, and+a nested Codex worker. Native plugin-build tests remain in the shared Bun suite.++The full Bun suite runs once per OS under Node 22: three file shards on Linux+and macOS, and seven on Windows. The other Node versions run the installed+package checks instead of repeating the same Bun suite. MCP and Python tests+run in separate required jobs. Python uses four isolated pytest-xdist workers+with work stealing; worker crashes fail the run without automatic restarts.++`scripts/run-ci-tests.mjs` assigns the longest measured files first. Its+`ci-test-durations.json` records per-file seconds from CI reports.+Every new test file is included automatically with a one-second estimate.+Refresh those estimates from the uploaded reports when adding or splitting+expensive files; estimates affect scheduling, never whether a test runs.+To reproduce one Windows shard locally after building the plugin, run+`node scripts/run-ci-tests.mjs 3/7 --seed=12345`.++Every Bun lane uploads JUnit; Linux lanes also upload LCOV per shard. Python+reports include case durations, and the MCP runner can upload its JUnit report.+Coverage is diagnostic and split across shards, not a combined percentage.+Failed tests and missing package artifacts block CI; failed diagnostic uploads+do not. Use `python -m pytest -n 0` to reproduce Python failures serially.The separate `test-quality` workflow runs weekly, can be dispatched manually,and runs on pull requests that change its workflow file. It compares Bun's@@ -103,7 +124,7 @@ that breaks the Ink UI tests under isolation. Keep the trial pin until a newerrelease passes the full SDK suite in every mode. Required CI and the mutationtrial remain on Bun 1.3.14.-Keep the current file-balanced Windows runner until the native runner has+Keep the measured file-balanced runner until the native runner hasmatching inventories and acceptable Windows timings. Before promotion, comparenative and file-balanced shards using the same commit and Bun version.Keep the machine-policy test serial. Do not replace the full required suitediff --git a/sdk/typescript/scripts/ci-test-durations.json b/sdk/typescript/scripts/ci-test-durations.jsonnew file mode 100644index 000000000..40c7b63d3--- /dev/null+++ b/sdk/typescript/scripts/ci-test-durations.json@@ -0,0 +1,231 @@+{+ "sources": [+ "https://github.com/openai/codex-security/actions/runs/33215429477",+ "https://github.com/openai/codex-security/actions/runs/33227512538",+ "https://github.com/openai/codex-security/actions/runs/33233690024"+ ],+ "unix": {+ "api-attribution-concurrency.test.ts": 0.372,+ "api-credentials.test.ts": 0.892,+ "api-environment.test.ts": 0.0,+ "api-events.test.ts": 2.277,+ "api-integration.test.ts": 0.0,+ "api-post-scan.test.ts": 7.161,+ "api-preflight-config.test.ts": 1.053,+ "api-surface.test.ts": 0.124,+ "api.test.ts": 16.055,+ "attack-path-tool-name.test.ts": 0.237,+ "auth.test.ts": 1.425,+ "build-plugin.test.ts": 3.973,+ "bulk-scan-discovery.test.ts": 0.021,+ "cli-authentication.test.ts": 5.757,+ "cli-cloud-publish.test.ts": 0.354,+ "cli-dedupe.test.ts": 0.061,+ "cli-deep-scan-summary.test.ts": 0.06,+ "cli-export.test.ts": 2.058,+ "cli-history-paths.test.ts": 0.0,+ "cli-launcher.test.ts": 8.154,+ "cli-patch-trust.test.ts": 0.74,+ "cli-patch.test.ts": 0.953,+ "cli-publish.test.ts": 0.494,+ "cli-scan-prompts.test.ts": 0.255,+ "cli-serve.test.ts": 5.033,+ "cli-signals.test.ts": 0.05,+ "cli-skills.test.ts": 2.954,+ "cli-verify-fix.test.ts": 0.158,+ "cli-workbench.test.ts": 0.208,+ "cli.test.ts": 3.071,+ "cloud-publish.test.ts": 2.111,+ "codex-review.test.ts": 0.246,+ "compact-diff-scan.test.ts": 12.898,+ "completed-scan-handoff.test.ts": 0.004,+ "component-scan.test.ts": 0.396,+ "config.test.ts": 0.403,+ "container-entrypoint.test.ts": 0.051,+ "contract.test.ts": 7.989,+ "cost-model.property.test.ts": 0.01,+ "cost.test.ts": 1.0,+ "custom-publish.test.ts": 0.463,+ "custom-validation.test.ts": 8.264,+ "deep-progress.test.ts": 0.001,+ "deep-scan-parent-denials.test.ts": 0.013,+ "deep-scan-reducer-recovery.test.ts": 0.504,+ "deep-scan-timestamp-compat.test.ts": 1.176,+ "deep-scan-windows-executable.test.ts": 0.008,+ "deep-scan-workbench.test.ts": 15.302,+ "deep-scan-worker-shutdown.test.ts": 0.009,+ "diff-rank-input.test.ts": 0.413,+ "errors.property.test.ts": 0.033,+ "errors.test.ts": 0.004,+ "finding-deduplication.test.ts": 0.389,+ "finding-embeddings.test.ts": 1.297,+ "finding-preview.test.ts": 0.234,+ "finding-workflow-integration.test.ts": 51.852,+ "finding-workflow-publication.test.ts": 1.44,+ "finding-workflow.test.ts": 0.455,+ "findings-dashboard.test.ts": 0.001,+ "findings-server.test.ts": 20.208,+ "github.test.ts": 0.064,+ "knowledge-base.test.ts": 1.665,+ "linear.test.ts": 0.002,+ "mcp-launcher.test.ts": 0.222,+ "multiscan.test.ts": 2.819,+ "package-provenance.test.ts": 0.001,+ "package-smoke-timeouts.test.ts": 0.0,+ "package-tar-listing.test.ts": 0.0,+ "patch-risk-contract.test.ts": 2.318,+ "patch-tui.test.ts": 4.739,+ "plugin-apps.test.ts": 0.0,+ "plugin-finding-detail-contract.test.ts": 2.077,+ "plugin-report-limits.test.ts": 0.709,+ "prompt-only-start-timeout.test.ts": 0.002,+ "publication-check.test.ts": 0.004,+ "publication-events.test.ts": 0.002,+ "publication-integration.test.ts": 9.618,+ "publication-store.test.ts": 17.125,+ "publication.test.ts": 1.359,+ "publish.test.ts": 1.395,+ "release-automation.test.ts": 6.993,+ "release-cut-workflow.test.ts": 0.0,+ "repository-findings.test.ts": 0.107,+ "result.test.ts": 0.002,+ "runtime.test.ts": 27.803,+ "sarif.test.ts": 0.79,+ "scan-activity.test.ts": 0.006,+ "scan-comparison.property.test.ts": 0.127,+ "scan-comparison.test.ts": 0.14,+ "scan-dashboard.test.ts": 0.058,+ "scan-history-renderer.test.ts": 0.004,+ "scan-logs.test.ts": 0.039,+ "scan-recovery.test.ts": 25.336,+ "skeleton.test.ts": 0.024,+ "stopped-scan-results.test.ts": 7.685,+ "targets-large-output.test.ts": 0.486,+ "targets.test.ts": 0.855,+ "test-reports.test.ts": 0.439,+ "trusted-executable.test.ts": 0.266,+ "update-notice.test.ts": 0.037,+ "windows-machine-policy.test.ts": 0.0,+ "workbench-canonical-paths.test.ts": 0.692,+ "workbench-nested-git-utf8.test.ts": 0.063,+ "workbench-patch.test.ts": 0.198,+ "workbench-progress.test.ts": 0.493,+ "workbench-remediation-timestamp.test.ts": 0.364,+ "workbench-scan-history.test.ts": 0.199,+ "workbench-scan-root-alias.test.ts": 0.0,+ "workbench-windows-compatibility.test.ts": 0.417,+ "worker-progress.property.test.ts": 0.009,+ "worker-progress.test.ts": 0.003+ },+ "windows": {+ "api-attribution-concurrency.test.ts": 18.599,+ "api-credentials.test.ts": 56.214,+ "api-environment.test.ts": 0.0,+ "api-events.test.ts": 2.999,+ "api-post-scan.test.ts": 11.43,+ "api-preflight-config.test.ts": 1.84,+ "api-surface.test.ts": 0.482,+ "api.test.ts": 100.166,+ "attack-path-tool-name.test.ts": 0.326,+ "auth.test.ts": 2.932,+ "build-plugin.test.ts": 4.531,+ "bulk-scan-discovery.test.ts": 0.054,+ "cli-authentication.test.ts": 45.757,+ "cli-cloud-publish.test.ts": 0.494,+ "cli-dedupe.test.ts": 0.059,+ "cli-deep-scan-summary.test.ts": 0.072,+ "cli-export.test.ts": 6.323,+ "cli-history-paths.test.ts": 0.009,+ "cli-launcher.test.ts": 9.241,+ "cli-patch-trust.test.ts": 3.655,+ "cli-patch.test.ts": 8.605,+ "cli-publish.test.ts": 0.583,+ "cli-scan-prompts.test.ts": 0.919,+ "cli-serve.test.ts": 7.776,+ "cli-signals.test.ts": 0.061,+ "cli-skills.test.ts": 2.774,+ "cli-verify-fix.test.ts": 0.173,+ "cli-workbench.test.ts": 0.241,+ "cli.test.ts": 6.814,+ "cloud-publish.test.ts": 9.022,+ "codex-review.test.ts": 0.642,+ "compact-diff-scan.test.ts": 33.15,+ "completed-scan-handoff.test.ts": 0.003,+ "component-scan.test.ts": 3.831,+ "config.test.ts": 5.1,+ "contract.test.ts": 9.095,+ "cost-model.property.test.ts": 0.012,+ "cost.test.ts": 1.574,+ "custom-publish.test.ts": 0.582,+ "custom-validation.test.ts": 20.415,+ "deep-progress.test.ts": 0.007,+ "deep-scan-parent-denials.test.ts": 0.013,+ "deep-scan-reducer-recovery.test.ts": 0.527,+ "deep-scan-timestamp-compat.test.ts": 1.981,+ "deep-scan-windows-executable.test.ts": 0.031,+ "deep-scan-workbench.test.ts": 29.291,+ "deep-scan-worker-shutdown.test.ts": 0.009,+ "diff-rank-input.test.ts": 2.241,+ "errors.property.test.ts": 0.032,+ "errors.test.ts": 0.006,+ "finding-deduplication.test.ts": 0.608,+ "finding-embeddings.test.ts": 1.515,+ "finding-preview.test.ts": 0.368,+ "finding-workflow-integration.test.ts": 58.802,+ "finding-workflow-publication.test.ts": 3.961,+ "finding-workflow.test.ts": 1.472,+ "findings-dashboard.test.ts": 0.001,+ "findings-server.test.ts": 36.179,+ "github.test.ts": 0.094,+ "knowledge-base.test.ts": 13.919,+ "linear.test.ts": 0.002,+ "mcp-launcher.test.ts": 0.284,+ "multiscan.test.ts": 31.671,+ "package-provenance.test.ts": 0.0,+ "package-smoke-timeouts.test.ts": 0.0,+ "package-tar-listing.test.ts": 0.0,+ "patch-risk-contract.test.ts": 4.074,+ "patch-tui.test.ts": 5.4,+ "plugin-apps.test.ts": 0.001,+ "plugin-finding-detail-contract.test.ts": 3.221,+ "plugin-report-limits.test.ts": 2.521,+ "prompt-only-start-timeout.test.ts": 0.009,+ "publication-check.test.ts": 0.005,+ "publication-events.test.ts": 0.005,+ "publication-integration.test.ts": 15.871,+ "publication-store.test.ts": 31.854,+ "publication.test.ts": 1.7,+ "publish.test.ts": 7.768,+ "release-automation.test.ts": 41.22,+ "release-cut-workflow.test.ts": 0.0,+ "repository-findings.test.ts": 0.277,+ "result.test.ts": 0.008,+ "runtime.test.ts": 134.065,+ "sarif.test.ts": 1.416,+ "scan-activity.test.ts": 0.007,+ "scan-comparison.property.test.ts": 0.16,+ "scan-comparison.test.ts": 10.704,+ "scan-dashboard.test.ts": 0.061,+ "scan-history-renderer.test.ts": 0.005,+ "scan-logs.test.ts": 0.139,+ "scan-recovery.test.ts": 62.302,+ "skeleton.test.ts": 0.03,+ "stopped-scan-results.test.ts": 15.814,+ "targets-large-output.test.ts": 11.48,+ "targets.test.ts": 8.454,+ "test-reports.test.ts": 1.222,+ "trusted-executable.test.ts": 0.064,+ "update-notice.test.ts": 0.052,+ "windows-machine-policy.test.ts": 3.448,+ "workbench-canonical-paths.test.ts": 1.853,+ "workbench-nested-git-utf8.test.ts": 0.364,+ "workbench-patch.test.ts": 0.323,+ "workbench-progress.test.ts": 0.802,+ "workbench-remediation-timestamp.test.ts": 0.587,+ "workbench-scan-history.test.ts": 0.547,+ "workbench-scan-root-alias.test.ts": 0.276,+ "workbench-windows-compatibility.test.ts": 1.479,+ "worker-progress.property.test.ts": 0.013,+ "worker-progress.test.ts": 0.003+ }+}diff --git a/sdk/typescript/scripts/run-ci-tests.mjs b/sdk/typescript/scripts/run-ci-tests.mjsnew file mode 100644index 000000000..477be4ad5--- /dev/null+++ b/sdk/typescript/scripts/run-ci-tests.mjs@@ -0,0 +1,63 @@+import { spawn } from "node:child_process";+import { mkdir, readFile, readdir } from "node:fs/promises";+import { fileURLToPath } from "node:url";+import { shardTestFiles } from "./test-shards.mjs";++const selection = /^([1-9]\d*)\/([1-9]\d*)$/.exec(process.argv[2] ?? "");+const shard = Number(selection?.[1]);+const count = Number(selection?.[2]);+if (+ !Number.isSafeInteger(shard) ||+ !Number.isSafeInteger(count) ||+ shard > count+) {+ throw new Error(+ "Usage: node scripts/run-ci-tests.mjs <shard>/<count> [bun test options]",+ );+}+const packageDirectory = fileURLToPath(new URL("../", import.meta.url));+const tests = (await readdir(new URL("../tests-ts/", import.meta.url))).filter(+ (file) =>+ file.endsWith(".test.ts") && file !== "windows-machine-policy.test.ts",+);+const timings = JSON.parse(+ await readFile(new URL("./ci-test-durations.json", import.meta.url), "utf8"),+);+const windows = process.platform === "win32";+const assignments = shardTestFiles(+ tests,+ count,+ timings[windows ? "windows" : "unix"],+);+const selected = assignments[shard - 1];+if (selected.files.length === 0) {+ throw new Error(`Test shard ${shard}/${count} is empty.`);+}+console.log(+ `Test shard ${shard}/${count} (estimated ${selected.seconds.toFixed(1)}s): ${selected.files.join(" ")}`,+);+await mkdir(new URL("../reports/", import.meta.url), { recursive: true }).catch(+ () => {+ // Bun warns about report write failures without changing the test result.+ },+);+const child = spawn(+ "bun",+ [+ "test",+ "--timeout",+ windows ? "120000" : "30000",+ "--reporter=junit",+ `--reporter-outfile=reports/junit-${shard}.xml`,+ ...process.argv.slice(3),+ ...selected.files.map((file) => `./tests-ts/${file}`),+ ],+ { cwd: packageDirectory, stdio: "inherit", windowsHide: true },+);+child.once("error", (error) => {+ console.error(error);+ process.exitCode = 1;+});+child.once("close", (code) => {+ process.exitCode = code ?? 1;+});diff --git a/sdk/typescript/scripts/run-windows-ci-tests.mjs b/sdk/typescript/scripts/run-windows-ci-tests.mjsdeleted file mode 100644index 83b792e81..000000000--- a/sdk/typescript/scripts/run-windows-ci-tests.mjs+++ /dev/null@@ -1,91 +0,0 @@-import { spawn } from "node:child_process";-import { readdir } from "node:fs/promises";-import { fileURLToPath } from "node:url";--const testsDirectory = new URL("../tests-ts/", import.meta.url);-const packageDirectory = fileURLToPath(new URL("../", import.meta.url));-const tests = (await readdir(testsDirectory))- .filter(- (file) =>- file.endsWith(".test.ts") && file !== "windows-machine-policy.test.ts",- )- .sort();-const shardSeeds = [- ["api-credentials.test.ts"],- ["api.test.ts"],- ["runtime.test.ts"],- ["cli-authentication.test.ts"],- ["scan-recovery.test.ts"],- [],- [],-];-const assigned = new Set(shardSeeds.flat());-for (const file of assigned) {- if (!tests.includes(file)) {- throw new Error("Windows CI test shard references a missing file: " + file);- }-}-const unassigned = tests.filter((file) => !assigned.has(file));-const slowRemainderFiles = new Set([- "deep-scan-workbench.test.ts",- "release-automation.test.ts",- "scan-comparison.test.ts",-]);-for (const [index, file] of unassigned.entries()) {- shardSeeds[slowRemainderFiles.has(file) ? 6 : 5 + (index % 2)].push(file);-}--const assignments = shardSeeds.flat();-if (- assignments.length !== tests.length ||- new Set(assignments).size !== tests.length-) {- throw new Error("Windows CI test shards must run every test file once.");-}--const requestedShard =- process.argv[2] === undefined ? undefined : Number(process.argv[2]);-if (- requestedShard !== undefined &&- (!Number.isSafeInteger(requestedShard) ||- requestedShard < 1 ||- requestedShard > shardSeeds.length)-) {- throw new Error("Usage: node scripts/run-windows-ci-tests.mjs [1-7]");-}-const selectedShards =- requestedShard === undefined- ? shardSeeds.map((files, index) => ({ files, index }))- : [{ files: shardSeeds[requestedShard - 1], index: requestedShard - 1 }];--const results = await Promise.all(- selectedShards.map(- ({ files, index }) =>- new Promise((resolve, reject) => {- const paths = files.map((file) => "./tests-ts/" + file);- console.log(- "Windows CI test shard " +- (index + 1) +- "/" +- shardSeeds.length +- ": " +- paths.join(" "),- );- // Native Windows credential and document checks can exceed 30 seconds.- // The workflow still bounds each complete shard to ten minutes.- const child = spawn("bun", ["test", "--timeout", "120000", ...paths], {- cwd: packageDirectory,- stdio: "inherit",- windowsHide: true,- });- child.once("error", reject);- child.once("close", (code) => {- resolve(code ?? 1);- });- }),- ),-);--if (results.some((code) => code !== 0)) {- process.exitCode = 1;-}diff --git a/sdk/typescript/scripts/test-shards.d.mts b/sdk/typescript/scripts/test-shards.d.mtsnew file mode 100644index 000000000..51d7be683--- /dev/null+++ b/sdk/typescript/scripts/test-shards.d.mts@@ -0,0 +1,5 @@+export function shardTestFiles(+ files: readonly string[],+ count: number,+ durations?: Readonly<Record<string, number>>,+): Array<{ files: string[]; seconds: number }>;diff --git a/sdk/typescript/scripts/test-shards.mjs b/sdk/typescript/scripts/test-shards.mjsnew file mode 100644index 000000000..e6e72e427--- /dev/null+++ b/sdk/typescript/scripts/test-shards.mjs@@ -0,0 +1,24 @@+/** Place the longest measured files first, without omitting new test files. */+export function shardTestFiles(files, count, durations = {}) {+ if (!Number.isSafeInteger(count) || count < 1) {+ throw new Error("The test shard count must be a positive integer.");+ }+ const shards = Array.from({ length: count }, () => ({+ files: [],+ seconds: 0,+ }));+ const duration = (file) => durations[file] ?? 1;+ const ordered = [...files].sort(+ (left, right) =>+ duration(right) - duration(left) ||+ (left < right ? -1 : left > right ? 1 : 0),+ );+ for (const file of ordered) {+ const shard = shards.reduce((shortest, candidate) =>+ candidate.seconds < shortest.seconds ? candidate : shortest,+ );+ shard.files.push(file);+ shard.seconds += duration(file);+ }+ return shards;+}diff --git a/sdk/typescript/tests-ts/attack-path-tool-name.test.ts b/sdk/typescript/tests-ts/attack-path-tool-name.test.tsindex 03a007f4d..c11a599f2 100644--- a/sdk/typescript/tests-ts/attack-path-tool-name.test.ts+++ b/sdk/typescript/tests-ts/attack-path-tool-name.test.ts@@ -1,30 +1,38 @@-import { spawnSync } from "node:child_process";import { mkdtempSync, rmSync } from "node:fs";import { tmpdir } from "node:os";import { join } from "node:path";import { expect, test } from "bun:test";import { PLUGIN_ROOT } from "./plugin-root.js";-test("keeps model-visible attack-path tool names within the Codex limit", () => {+test("keeps model-visible attack-path tool names within the Codex limit", async () => {const node = Bun.which("node");expect(node).not.toBeNull();const state = mkdtempSync(join(tmpdir(), "codex-security-mcp-tools-"));try {- const server = spawnSync(node!, [join(PLUGIN_ROOT, "mcp", "server.mjs")], {- encoding: "utf8",+ const server = Bun.spawn({+ cmd: [node!, join(PLUGIN_ROOT, "mcp", "server.mjs")],env: { ...process.env, CODEX_SECURITY_STATE_DIR: state },- input: [- '{"jsonrpc":"2.0","id":1,"method":"initialize","params":{"protocolVersion":"2025-11-25","capabilities":{},"clientInfo":{"name":"codex-security-test","version":"1.0.0"}}}',- '{"jsonrpc":"2.0","method":"notifications/initialized","params":{}}',- '{"jsonrpc":"2.0","id":2,"method":"tools/list","params":{}}',- "",- ].join("\n"),+ stdin: new TextEncoder().encode(+ [+ '{"jsonrpc":"2.0","id":1,"method":"initialize","params":{"protocolVersion":"2025-11-25","capabilities":{},"clientInfo":{"name":"codex-security-test","version":"1.0.0"}}}',+ '{"jsonrpc":"2.0","method":"notifications/initialized","params":{}}',+ '{"jsonrpc":"2.0","id":2,"method":"tools/list","params":{}}',+ "",+ ].join("\n"),+ ),+ stdout: "pipe",+ stderr: "pipe",timeout: 30_000,});- expect(server.status, server.stderr).toBe(0);+ const [status, stdout, stderr] = await Promise.all([+ server.exited,+ new Response(server.stdout).text(),+ new Response(server.stderr).text(),+ ]);+ expect(status, stderr).toBe(0);- const tools = server.stdout+ const tools = stdout.trim().split("\n").map((line) => JSON.parse(line))diff --git a/sdk/typescript/tests-ts/cli-launcher.test.ts b/sdk/typescript/tests-ts/cli-launcher.test.tsindex d427bb05b..5a0e6df2b 100644--- a/sdk/typescript/tests-ts/cli-launcher.test.ts+++ b/sdk/typescript/tests-ts/cli-launcher.test.ts@@ -27,14 +27,22 @@ describe("CLI launcher", () => {if (process.platform !== "win32") {await symlink(launcher, bin);}- const child = spawnSync(process.execPath, [bin, "--version"], {- encoding: "utf8",+ const child = Bun.spawn({+ cmd: [process.execPath, bin, "--version"],+ stdin: "ignore",+ stdout: "pipe",+ stderr: "pipe",timeout: 30_000,});+ const [status, stdout, stderr] = await Promise.all([+ child.exited,+ new Response(child.stdout).text(),+ new Response(child.stderr).text(),+ ]);- expect(child.status).toBe(0);- expect(child.stderr).toBe("");- expect(child.stdout).toBe(`${VERSION}\n`);+ expect(status, stderr).toBe(0);+ expect(stderr).toBe("");+ expect(stdout).toBe(`${VERSION}\n`);} finally {await rm(root, { recursive: true, force: true });}@@ -48,15 +56,28 @@ describe("CLI launcher", () => {preload,`Object.defineProperty(process, "cwd", { value() { throw new Error(${JSON.stringify(`working directory is unavailable: ${SYNTHETIC_CREDENTIALS}`)}); } });\n`,);- const child = spawnSync(- process.execPath,- ["--preload", preload, join(packageRoot, "src", "cli.ts"), "scan"],- { encoding: "utf8", timeout: 30_000 },- );+ const child = Bun.spawn({+ cmd: [+ process.execPath,+ "--preload",+ preload,+ join(packageRoot, "src", "cli.ts"),+ "scan",+ ],+ stdin: "ignore",+ stdout: "pipe",+ stderr: "pipe",+ timeout: 30_000,+ });+ const [status, stdout, stderr] = await Promise.all([+ child.exited,+ new Response(child.stdout).text(),+ new Response(child.stderr).text(),+ ]);- expect(child.status).toBe(2);- expect(child.stdout).toBe("");- expect(child.stderr).toBe(+ expect(status, stderr).toBe(2);+ expect(stdout).toBe("");+ expect(stderr).toBe(`working directory is unavailable: ${SYNTHETIC_CREDENTIALS}\n`,);} finally {diff --git a/sdk/typescript/tests-ts/release-automation.test.ts b/sdk/typescript/tests-ts/release-automation.test.tsindex d1900db39..0d9d42f56 100644--- a/sdk/typescript/tests-ts/release-automation.test.ts+++ b/sdk/typescript/tests-ts/release-automation.test.ts@@ -3958,7 +3958,10 @@ describe("GitHub release workflow safeguards", () => {const fullCiCondition = "needs.validate-title.outputs.ci-mode == 'full'";for (const job of [+ "package","test",+ "compatibility",+ "mcp","plugin-source","windows-test","windows-verify",@@ -3987,7 +3990,7 @@ describe("GitHub release workflow safeguards", () => {expect(workflow.jobs["required-test"]?.if).toBe(requiredJobCondition);expect(workflow.jobs["windows"]?.if).toBe(requiredJobCondition);expect(workflow.jobs["required-test"]?.steps[0]?.if).toBe(- "needs.validate-title.result != 'success' || (needs.validate-title.outputs.ci-mode == 'full' && (needs.test.result != 'success' || needs.plugin-source.result != 'success')) || (needs.validate-title.outputs.ci-mode != 'full' && needs.validate-title.outputs.ci-mode != 'markdown')",+ "needs.validate-title.result != 'success' || (needs.validate-title.outputs.ci-mode == 'full' && (needs.package.result != 'success' || needs.test.result != 'success' || needs.compatibility.result != 'success' || needs.mcp.result != 'success' || needs.plugin-source.result != 'success')) || (needs.validate-title.outputs.ci-mode != 'full' && needs.validate-title.outputs.ci-mode != 'markdown')",);expect(workflow.jobs["windows"]?.steps[0]?.if).toBe("needs.validate-title.result != 'success' || (needs.validate-title.outputs.ci-mode == 'full' && (needs.windows-test.result != 'success' || needs.windows-verify.result != 'success')) || (needs.validate-title.outputs.ci-mode != 'full' && needs.validate-title.outputs.ci-mode != 'markdown')",@@ -4002,6 +4005,9 @@ describe("GitHub release workflow safeguards", () => {] as const) {const values = {"needs.plugin-source.result": upstream,+ "needs.package.result": upstream,+ "needs.compatibility.result": upstream,+ "needs.mcp.result": upstream,"needs.test.result": upstream,"needs.validate-title.outputs.ci-mode": ciMode,"needs.validate-title.result": validation,@@ -4019,6 +4025,35 @@ describe("GitHub release workflow safeguards", () => {}}+ const dependencies = [+ "package",+ "test",+ "compatibility",+ "mcp",+ "plugin-source",+ ];+ for (const dependency of dependencies) {+ for (const result of ["failure", "cancelled", "skipped"]) {+ expect(+ evaluateWorkflowCondition(+ workflow.jobs["required-test"]?.steps[0]?.if ?? "",+ {+ "needs.validate-title.result": "success",+ "needs.validate-title.outputs.ci-mode": "full",+ ...Object.fromEntries(+ dependencies.map((job) => [+ `needs.${job}.result`,+ job === dependency ? result : "success",+ ]),+ ),+ },+ ),+ `${dependency}: ${result}`,+ ).toBe(true);+ }+ }+ expect(workflow.jobs["required-test"]?.steps[0]?.run).toBe("exit 1");+const renderName = (template: string, values: Record<string, string>) => {let name = template;for (const [key, value] of Object.entries(values)) {diff --git a/sdk/typescript/tests-ts/skeleton.test.ts b/sdk/typescript/tests-ts/skeleton.test.tsindex 62c2aaafb..27fda31d6 100644--- a/sdk/typescript/tests-ts/skeleton.test.ts+++ b/sdk/typescript/tests-ts/skeleton.test.ts@@ -101,35 +101,48 @@ describe("TypeScript package skeleton", () => {).toEqual([]);});- test("pins each Node.js minimum and preserves protected and latest LTS checks", async () => {+ test("runs Bun once per OS and checks every supported Node runtime with the installed package", async () => {const { jobs } = await workflow("node-ci.yml");- expect(jobs["test"]?.name).toContain(- "tests / ${{ matrix.os }} / node-${{ matrix.node == '22.13.0' && '22' || matrix.node }}",- );expect(jobs["test"]?.strategy?.matrix).toEqual({os: ["ubuntu-latest", "macos-latest"],- node: ["22.13.0"],- include: ["24.0.0", "24", "26.0.0", "26"].map((node) => ({- os: "ubuntu-latest",- node,- })),+ shard: [1, 2, 3],});- expect(jobs["required-test"]?.name).toContain("${{ matrix.os }} / node-22");- expect(jobs["required-test"]?.strategy?.matrix).toEqual({- os: ["ubuntu-latest", "macos-latest"],+ expect(jobs["compatibility"]?.strategy?.matrix).toEqual({+ os: ["ubuntu-latest"],+ node: ["22.13.0", "24.0.0", "24", "26.0.0", "26"],+ include: [{ os: "macos-latest", node: "22.13.0" }],});- expect(jobs["windows"]?.name).toContain(- "windows-latest / node-${{ matrix.node == '22.13.0' && '22' || matrix.node }}",- );+ expect(jobs["windows-test"]?.strategy?.matrix).toEqual({+ shard: [1, 2, 3, 4, 5, 6, 7],+ });+ expect(jobs["windows-verify"]?.strategy?.matrix["node"]).toEqual([+ "22.13.0",+ "24",+ ]);+ expect(jobs["required-test"]?.name).toBe("${{ matrix.os }} / node-22");+ expect(jobs["required-test"]?.needs).toEqual([+ "validate-title",+ "package",+ "test",+ "compatibility",+ "mcp",+ "plugin-source",+ ]);expect(jobs["windows"]?.needs).toEqual(["validate-title","windows-test","windows-verify",]);- expect(jobs["windows-test"]?.strategy?.matrix["node"]).toEqual([- "22.13.0",- "24",- ]);+ for (const name of ["compatibility", "windows-verify"]) {+ expect(jobs[name]?.steps).toContainEqual(+ expect.objectContaining({+ run: "node scripts/check-package.mjs ../../dist/*.tgz",+ }),+ );+ expect(jobs[name]?.steps.some(({ name }) => name === "Set up Bun")).toBe(+ false,+ );+ }});test("randomizes tests and keeps the default and Windows CI timeouts", async () => {@@ -140,7 +153,6 @@ describe("TypeScript package skeleton", () => {const bunConfig = parse(await readFile(new URL("../bunfig.toml", import.meta.url), "utf8"),);-expect(packageJson.scripts.test).toBe("node --run build:plugin && bun test --timeout 30000 ./tests-ts",);@@ -148,50 +160,73 @@ describe("TypeScript package skeleton", () => {expect(packageJson.scripts["test:ci"]).toContain("pnpm run test ");expect(jobs["windows-test"]?.steps).toContainEqual(expect.objectContaining({- run: "node sdk/typescript/scripts/run-windows-ci-tests.mjs ${{ matrix.shard }}",+ run: "node sdk/typescript/scripts/run-ci-tests.mjs ${{ matrix.shard }}/7",}),);});- test("generates the bundled plugin before Windows test shards", async () => {+ test("checks one archive and restores its plugin before every test shard", async () => {const { jobs } = await workflow("node-ci.yml");- const steps = jobs["windows-test"]!.steps;- const dependencyInstallation = steps.findIndex(- (step) => step.name === "Install dependencies",- );- const pluginGeneration = steps.findIndex(- (step) => step.name === "Generate bundled plugin",+ const uploads = jobs["package"]!.steps;+ const inspection = uploads.findIndex(+ ({ name }) => name === "Inspect archive contents",);- const shard = steps.findIndex(- (step) => step.name === "Test shard ${{ matrix.shard }}",+ const upload = uploads.findIndex(+ ({ name }) => name === "Upload package for this commit",);-- expect(steps[dependencyInstallation]?.run).toContain(- "pnpm --dir plugins/codex-security/mcp-app install --frozen-lockfile",- );- expect(steps[pluginGeneration]?.run).toBe(- "pnpm --dir sdk/typescript run build:plugin",- );- expect(dependencyInstallation).toBeLessThan(pluginGeneration);- expect(pluginGeneration).toBeLessThan(shard);+ expect(inspection).toBeGreaterThanOrEqual(0);+ expect(inspection).toBeLessThan(upload);+ expect(uploads[upload]?.with).toMatchObject({+ name: "package-${{ github.sha }}",+ "if-no-files-found": "error",+ });+ expect(uploads[upload]).not.toHaveProperty("continue-on-error");+ for (const name of [+ "test",+ "windows-test",+ "mcp",+ "compatibility",+ "windows-verify",+ ]) {+ const job = jobs[name]!;+ expect(job.needs).toContain("package");+ expect(+ job.steps.find(+ ({ name }) => name === "Download package for this commit",+ )?.with,+ ).toEqual({ name: "package-${{ github.sha }}", path: "dist" });+ }+ for (const name of ["test", "windows-test", "mcp"]) {+ const steps = jobs[name]!.steps;+ const restore = steps.findIndex(+ ({ name }) => name === "Restore bundled plugin",+ );+ const testStep = steps.findIndex(+ ({ name }) =>+ name === "Test" ||+ name === "Test shard ${{ matrix.shard }}" ||+ name === "Test MCP app",+ );+ expect(restore).toBeGreaterThanOrEqual(0);+ expect(restore).toBeLessThan(testStep);+ expect(steps[restore]?.run).toContain("package/_bundled_plugin");+ expect(steps[testStep]).not.toHaveProperty("continue-on-error");+ }});- test("installs ripgrep before the MCP app tests", async () => {+ test("installs ripgrep before the independent MCP job", async () => {const { jobs } = await workflow("node-ci.yml");- const steps = jobs["test"]!.steps;- const ripgrepInstallation = steps.findIndex(- (step) => step.name === "Install ripgrep",- );- const mcpTests = steps.findIndex((step) => step.name === "Test MCP app");-- expect(steps[ripgrepInstallation]).toMatchObject({- if: "matrix.os == 'ubuntu-latest' && matrix.node == '22.13.0'",- run: expect.stringContaining("apt-get install --yes ripgrep"),- });- expect(ripgrepInstallation).toBeLessThan(mcpTests);+ const steps = jobs["mcp"]!.steps;+ const ripgrep = steps.findIndex(({ name }) => name === "Install ripgrep");+ const tests = steps.findIndex(({ name }) => name === "Test MCP app");+ expect(steps[ripgrep]?.run).toContain("apt-get install --yes ripgrep");+ expect(ripgrep).toBeLessThan(tests);+ expect(+ jobs["test"]!.steps.some(({ name }) => name === "Test MCP app"),+ ).toBe(false);});- test("runs shared static checks once and keeps report upload non-blocking", async () => {+ test("runs shared static checks once and keeps every diagnostic upload non-blocking", async () => {const { jobs } = await workflow("node-ci.yml");const steps = Object.values(jobs).flatMap((job) => job.steps);for (const name of [@@ -199,27 +234,29 @@ describe("TypeScript package skeleton", () => {"Typecheck","Check formatting",]) {- expect(steps.filter((step) => step.name === name)).toEqual([- expect.objectContaining({- if: "matrix.os == 'ubuntu-latest' && matrix.node == '22.13.0'",- }),- ]);- }- expect(steps.find((step) => step.name === "Test")).not.toHaveProperty(- "continue-on-error",- );- expect(- steps.find((step) => step.name === "Upload test reports"),- ).toMatchObject({- "continue-on-error": true,- });- for (const name of ["test", "windows-verify"]) {- expect(jobs[name]?.steps).toContainEqual(- expect.objectContaining({- run: "pnpm run check:package ../../dist/*.tgz",- }),+ expect(steps.filter((step) => step.name === name)).toHaveLength(1);+ expect(jobs["package"]!.steps.some((step) => step.name === name)).toBe(+ true,);}+ for (const name of [+ "Upload test reports",+ "Upload Windows test reports",+ "Upload MCP test reports",+ "Upload Python test reports",+ ]) {+ expect(steps.find((step) => step.name === name)).toMatchObject({+ if: "always()",+ "continue-on-error": true,+ });+ }+ expect(+ jobs["plugin-source"]!.steps.find(+ ({ name }) => name === "Test Python source contracts",+ )?.run,+ ).toContain(+ "-n 4 --dist worksteal --max-worker-restart 0 --durations=30 --junitxml=reports/python.xml",+ );});test("keeps machine-wide policy changes out of parallel and experimental runs", async () => {@@ -297,7 +334,6 @@ describe("TypeScript package skeleton", () => {const uploads = [...Object.values(ci.jobs), ...Object.values(quality.jobs)].flatMap((job) => job.steps).filter((step) => step.uses?.startsWith("actions/upload-artifact@"));- expect(uploads).toHaveLength(3);for (const upload of uploads) {expect(upload.with?.["overwrite"]).toBe(true);}diff --git a/sdk/typescript/tests-ts/test-shards.test.ts b/sdk/typescript/tests-ts/test-shards.test.tsnew file mode 100644index 000000000..95b5289eb--- /dev/null+++ b/sdk/typescript/tests-ts/test-shards.test.ts@@ -0,0 +1,138 @@+import {+ copyFile,+ mkdir,+ mkdtemp,+ readdir,+ readFile,+ rm,+ writeFile,+} from "node:fs/promises";+import { tmpdir } from "node:os";+import { delimiter, dirname, join } from "node:path";+import { expect, test } from "bun:test";+import { shardTestFiles } from "../scripts/test-shards.mjs";++test("balances measured work and includes new files without mutating the inventory", () => {+ const files = [+ "new.test.ts",+ "small.test.ts",+ "medium.test.ts",+ "slow.test.ts",+ ];+ const shards = shardTestFiles(files, 2, {+ "slow.test.ts": 100,+ "medium.test.ts": 70,+ "small.test.ts": 30,+ });+ expect(shards.map(({ seconds }) => seconds)).toEqual([101, 100]);+ expect(shards.flatMap(({ files }) => files).sort()).toEqual(+ [...files].sort(),+ );+ expect(files[0]).toBe("new.test.ts");+ expect(+ shardTestFiles([...files].reverse(), 2, {+ "slow.test.ts": 100,+ "medium.test.ts": 70,+ "small.test.ts": 30,+ }),+ ).toEqual(shards);+});++test.each([0, -1, 1.5, Number.NaN])(+ "rejects an invalid shard count %p",+ (count) => {+ expect(() => shardTestFiles(["a.test.ts"], count)).toThrow(+ "positive integer",+ );+ },+);++test.each([+ ["unix", 3],+ ["windows", 7],+] as const)("covers every shared test once on %s", async (platform, count) => {+ const tests = (await readdir(new URL("./", import.meta.url)))+ .filter(+ (file) =>+ file.endsWith(".test.ts") && file !== "windows-machine-policy.test.ts",+ )+ .sort();+ const timings = JSON.parse(+ await readFile(+ new URL("../scripts/ci-test-durations.json", import.meta.url),+ "utf8",+ ),+ );+ const shards = shardTestFiles(tests, count, timings[platform]);+ expect(shards.every(({ files }) => files.length > 0)).toBe(true);+ expect(shards.flatMap(({ files }) => files).sort()).toEqual(tests);+});++test.each([+ ["available", false],+ ["available", true],+ ["blocked directory", false],+ ["blocked directory", true],+] as const)(+ "preserves the test result with %s reports and failure=%p",+ async (report, fail) => {+ const node = Bun.which("node");+ expect(node).not.toBeNull();+ const root = await mkdtemp(join(tmpdir(), "codex-security-shard-report-"));+ try {+ await mkdir(join(root, "scripts"));+ await mkdir(join(root, "tests-ts"));+ for (const file of [+ "run-ci-tests.mjs",+ "test-shards.mjs",+ "ci-test-durations.json",+ ]) {+ await copyFile(+ new URL(`../scripts/${file}`, import.meta.url),+ join(root, "scripts", file),+ );+ }+ await writeFile(+ join(root, "tests-ts", "probe.test.ts"),+ `import { expect, test } from "bun:test"; test("synthetic report probe", () => expect(true).toBe(${!fail}));\n`,+ );+ if (report === "blocked directory") {+ await writeFile(join(root, "reports"), "synthetic blocker");+ }+ const child = Bun.spawn({+ cmd: [node!, join(root, "scripts", "run-ci-tests.mjs"), "1/1"],+ env: {+ ...process.env,+ PATH: `${dirname(process.execPath)}${delimiter}${process.env["PATH"] ?? ""}`,+ },+ stdin: "ignore",+ stdout: "pipe",+ stderr: "pipe",+ timeout: 30_000,+ });+ const [status, stdout, stderr] = await Promise.all([+ child.exited,+ new Response(child.stdout).text(),+ new Response(child.stderr).text(),+ ]);+ expect(status, stderr).toBe(fail ? 1 : 0);+ expect(stdout).toContain("probe.test.ts");+ expect(stderr).toContain("synthetic report probe");+ if (report === "available") {+ const xml = await readFile(+ join(root, "reports", "junit-1.xml"),+ "utf8",+ );+ expect(xml).toContain("<testcase ");+ expect(xml.includes("<failure ")).toBe(fail);+ } else {+ expect(stderr).toContain("JUnitReportFailed");+ expect(await readFile(join(root, "reports"), "utf8")).toBe(+ "synthetic blocker",+ );+ }+ } finally {+ await rm(root, { recursive: true, force: true });+ }+ },+);