Skip to content

OpenAI's Codex Security CLI and TypeScript SDK for finding, validating, and fixing security vulnerabilities. npm: https://www.npmjs.com/package/@openai/codex-security

ci: reuse one package and balance test jobs (#734)

mldangelo-oaie2d329744d27
diff --git a/.github/workflows/node-ci.yml b/.github/workflows/node-ci.yml
index 6171278df..2db2c9f07 100644
--- a/.github/workflows/node-ci.yml
+++ b/.github/workflows/node-ci.yml
@@ -120,33 +120,73 @@ jobs:
pnpm --dir sdk/typescript exec prettier --check "${files[@]}"
fi
- test:
- name: tests / ${{ matrix.os }} / node-${{ matrix.node == '22.13.0' && '22' || matrix.node }}
+ package:
+ name: build and check package
needs: validate-title
if: needs.validate-title.outputs.ci-mode == 'full'
+ runs-on: ubuntu-latest
+ timeout-minutes: 20
+ steps:
+ - name: Checkout repository
+ uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
+ with:
+ persist-credentials: false
+ - name: Check plugin source boundary
+ run: node sdk/typescript/scripts/check-plugin-source.mjs
+ - name: Set up pnpm
+ uses: pnpm/action-setup@0ebf47130e4866e96fce0953f49152a61190b271 # v6.0.9
+ with:
+ package_json_file: sdk/typescript/package.json
+ cache: true
+ cache_dependency_path: |
+ sdk/typescript/pnpm-lock.yaml
+ plugins/codex-security/mcp-app/pnpm-lock.yaml
+ - name: Set up Node.js
+ uses: actions/setup-node@53b83947a5a98c8d113130e565377fae1a50d02f # v6
+ with:
+ node-version: "22.13.0"
+ - name: Install dependencies
+ run: |
+ pnpm --dir sdk/typescript install --frozen-lockfile
+ pnpm --dir plugins/codex-security/mcp-app install --frozen-lockfile
+ - name: Audit production dependencies
+ continue-on-error: true
+ run: pnpm --dir sdk/typescript run audit:prod
+ - name: Typecheck
+ run: pnpm --dir sdk/typescript run types
+ - name: Check formatting
+ run: pnpm --dir sdk/typescript run format
+ - name: Pack
+ working-directory: sdk/typescript
+ run: pnpm pack --pack-destination ../../dist
+ - name: Inspect archive contents
+ working-directory: sdk/typescript
+ run: node scripts/check-package.mjs ../../dist/*.tgz ../../plugins/codex-security/plugin-files.json
+ - name: Upload package for this commit
+ uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
+ with:
+ name: package-${{ github.sha }}
+ overwrite: true
+ path: dist/*.tgz
+ if-no-files-found: error
+ retention-days: 7
+
+ test:
+ name: tests / ${{ matrix.os }} / shard-${{ matrix.shard }}
+ needs: [validate-title, package]
+ if: needs.validate-title.outputs.ci-mode == 'full'
runs-on: ${{ matrix.os }}
timeout-minutes: 20
strategy:
fail-fast: false
matrix:
os: [ubuntu-latest, macos-latest]
- node: ["22.13.0"]
- include:
- - os: ubuntu-latest
- node: "24.0.0"
- - os: ubuntu-latest
- node: "24"
- - os: ubuntu-latest
- node: "26.0.0"
- - os: ubuntu-latest
- node: "26"
-
+ shard: [1, 2, 3]
steps:
- name: Checkout repository
uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
with:
persist-credentials: false
-
- name: Set up pnpm
uses: pnpm/action-setup@0ebf47130e4866e96fce0953f49152a61190b271 # v6.0.9
with:
@@ -155,79 +195,132 @@ jobs:
cache_dependency_path: |
sdk/typescript/pnpm-lock.yaml
plugins/codex-security/mcp-app/pnpm-lock.yaml
-
- name: Set up Node.js
uses: actions/setup-node@53b83947a5a98c8d113130e565377fae1a50d02f # v6
with:
- node-version: ${{ matrix.node }}
-
- - name: Check plugin source boundary
- if: matrix.os == 'ubuntu-latest' && matrix.node == '22.13.0'
- run: node sdk/typescript/scripts/check-plugin-source.mjs
-
+ node-version: "22.13.0"
- name: Set up Bun
uses: oven-sh/setup-bun@0c5077e51419868618aeaa5fe8019c62421857d6 # v2
with:
bun-version: "1.3.14"
-
- name: Install dependencies
run: |
pnpm --dir sdk/typescript install --frozen-lockfile
pnpm --dir plugins/codex-security/mcp-app install --frozen-lockfile
-
+ - name: Download package for this commit
+ uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1
+ with:
+ name: package-${{ github.sha }}
+ path: dist
+ - name: Restore bundled plugin
+ run: tar -xzf dist/*.tgz --strip-components=1 -C sdk/typescript package/_bundled_plugin
- name: Install ripgrep
- if: matrix.os == 'ubuntu-latest' && matrix.node == '22.13.0'
+ if: matrix.os == 'ubuntu-latest'
run: |
sudo apt-get update
sudo apt-get install --yes ripgrep
-
- - name: Audit production dependencies
- if: matrix.os == 'ubuntu-latest' && matrix.node == '22.13.0'
- continue-on-error: true
- run: pnpm --dir sdk/typescript run audit:prod
-
- - name: Typecheck
- if: matrix.os == 'ubuntu-latest' && matrix.node == '22.13.0'
- run: pnpm --dir sdk/typescript run types
-
- name: Test
timeout-minutes: 10
+ working-directory: sdk/typescript
env:
TEMP: ${{ runner.temp }}
TMP: ${{ runner.temp }}
TMPDIR: ${{ runner.temp }}
CODEX_SECURITY_ALLOW_MACHINE_POLICY_TEST: "false"
- run: pnpm --dir sdk/typescript run ${{ matrix.os == 'ubuntu-latest' && matrix.node == '22.13.0' && 'test:ci' || 'test' }}
-
- - name: Test MCP app
- if: matrix.os == 'ubuntu-latest' && matrix.node == '22.13.0'
- run: pnpm --dir sdk/typescript run test:mcp
-
+ run: node scripts/run-ci-tests.mjs ${{ matrix.shard }}/3 ${{ matrix.os == 'ubuntu-latest' && '--coverage --coverage-reporter=text --coverage-reporter=lcov' || '' }}
- name: Upload test reports
- if: always() && matrix.os == 'ubuntu-latest' && matrix.node == '22.13.0'
+ if: always()
continue-on-error: true
uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
with:
- name: node-22-test-reports
+ name: bun-${{ runner.os }}-${{ matrix.shard }}
overwrite: true
path: |
- sdk/typescript/reports/junit.xml
+ sdk/typescript/reports/junit-*.xml
sdk/typescript/coverage/lcov.info
if-no-files-found: warn
retention-days: 14
- - name: Check formatting
- if: matrix.os == 'ubuntu-latest' && matrix.node == '22.13.0'
- run: pnpm --dir sdk/typescript run format
-
- - name: Pack
+ compatibility:
+ name: installed package / ${{ matrix.os }} / node-${{ matrix.node }}
+ needs: [validate-title, package]
+ if: needs.validate-title.outputs.ci-mode == 'full'
+ runs-on: ${{ matrix.os }}
+ timeout-minutes: 20
+ strategy:
+ fail-fast: false
+ matrix:
+ os: [ubuntu-latest]
+ node: ["22.13.0", "24.0.0", "24", "26.0.0", "26"]
+ include:
+ - os: macos-latest
+ node: "22.13.0"
+ steps:
+ - name: Checkout repository
+ uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
+ with:
+ persist-credentials: false
+ - name: Set up Node.js
+ uses: actions/setup-node@53b83947a5a98c8d113130e565377fae1a50d02f # v6
+ with:
+ node-version: ${{ matrix.node }}
+ cache: npm
+ cache-dependency-path: sdk/typescript/pnpm-lock.yaml
+ - name: Download package for this commit
+ uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1
+ with:
+ name: package-${{ github.sha }}
+ path: dist
+ - name: Inspect installed package
working-directory: sdk/typescript
- run: pnpm pack --pack-destination ../../dist
+ run: node scripts/check-package.mjs ../../dist/*.tgz
- - name: Inspect package
- working-directory: sdk/typescript
- shell: bash
- run: pnpm run check:package ../../dist/*.tgz
+ mcp:
+ name: MCP app tests
+ needs: [validate-title, package]
+ if: needs.validate-title.outputs.ci-mode == 'full'
+ runs-on: ubuntu-latest
+ timeout-minutes: 20
+ steps:
+ - name: Checkout repository
+ uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
+ with:
+ persist-credentials: false
+ - name: Set up pnpm
+ uses: pnpm/action-setup@0ebf47130e4866e96fce0953f49152a61190b271 # v6.0.9
+ with:
+ package_json_file: plugins/codex-security/mcp-app/package.json
+ cache: true
+ cache_dependency_path: plugins/codex-security/mcp-app/pnpm-lock.yaml
+ - name: Set up Node.js
+ uses: actions/setup-node@53b83947a5a98c8d113130e565377fae1a50d02f # v6
+ with:
+ node-version: "22.13.0"
+ - name: Install dependencies
+ run: pnpm --dir plugins/codex-security/mcp-app install --frozen-lockfile
+ - name: Install ripgrep
+ run: |
+ sudo apt-get update
+ sudo apt-get install --yes ripgrep
+ - name: Download package for this commit
+ uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1
+ with:
+ name: package-${{ github.sha }}
+ path: dist
+ - name: Restore bundled plugin
+ run: tar -xzf dist/*.tgz --strip-components=1 -C sdk/typescript package/_bundled_plugin
+ - name: Test MCP app
+ run: pnpm --dir plugins/codex-security/mcp-app run test:mcp
+ - name: Upload MCP test reports
+ if: always()
+ continue-on-error: true
+ uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
+ with:
+ name: mcp-test-reports
+ overwrite: true
+ path: plugins/codex-security/mcp-app/reports/junit.xml
+ if-no-files-found: warn
+ retention-days: 14
plugin-source:
name: plugin source contracts
@@ -235,105 +328,103 @@ jobs:
if: needs.validate-title.outputs.ci-mode == 'full'
runs-on: ubuntu-latest
timeout-minutes: 30
-
steps:
- name: Checkout repository
uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
with:
persist-credentials: false
-
- name: Set up Python
uses: actions/setup-python@ece7cb06caefa5fff74198d8649806c4678c61a1 # v6
with:
python-version: "3.12"
cache: pip
cache-dependency-path: plugins/codex-security/pyproject.toml
-
- name: Install Python test dependencies
run: python -m pip install --disable-pip-version-check --no-input -e 'plugins/codex-security[test]'
-
- name: Install ripgrep
run: |
sudo apt-get update
sudo apt-get install --yes ripgrep
-
- name: Check plugin source compatibility
run: |
python -m ruff check --config plugins/codex-security/pyproject.toml .github/scripts/check_plugin_source_compatibility.py .github/scripts/test_check_plugin_source_compatibility.py plugins/codex-security
python -m ruff format --check --config plugins/codex-security/pyproject.toml .github/scripts/check_plugin_source_compatibility.py .github/scripts/test_check_plugin_source_compatibility.py plugins/codex-security
python .github/scripts/check_plugin_source_compatibility.py
-
- name: Test Python source contracts
- run: python -m pytest .github/scripts/test_check_plugin_source_compatibility.py plugins/codex-security/tests -q
-
+ run: python -m pytest .github/scripts/test_check_plugin_source_compatibility.py plugins/codex-security/tests -q -n 4 --dist worksteal --max-worker-restart 0 --durations=30 --junitxml=reports/python.xml
+ - name: Upload Python test reports
+ if: always()
+ continue-on-error: true
+ uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
+ with:
+ name: python-test-reports
+ overwrite: true
+ path: reports/python.xml
+ if-no-files-found: warn
+ retention-days: 14
- name: Set up Node.js
uses: actions/setup-node@53b83947a5a98c8d113130e565377fae1a50d02f # v6
with:
node-version: "22.13.0"
-
- name: Test deterministic triage eval contracts
working-directory: plugins/codex-security/skills/triage-finding/evals
run: node --run test:deterministic
required-test:
name: ${{ matrix.os }} / node-22
- needs: [validate-title, test, plugin-source]
+ needs: [validate-title, package, test, compatibility, mcp, plugin-source]
if: always()
runs-on: ubuntu-latest
strategy:
fail-fast: false
matrix:
os: [ubuntu-latest, macos-latest]
-
steps:
- name: Require every Unix coverage job
- if: needs.validate-title.result != 'success' || (needs.validate-title.outputs.ci-mode == 'full' && (needs.test.result != 'success' || needs.plugin-source.result != 'success')) || (needs.validate-title.outputs.ci-mode != 'full' && needs.validate-title.outputs.ci-mode != 'markdown')
+ if: needs.validate-title.result != 'success' || (needs.validate-title.outputs.ci-mode == 'full' && (needs.package.result != 'success' || needs.test.result != 'success' || needs.compatibility.result != 'success' || needs.mcp.result != 'success' || needs.plugin-source.result != 'success')) || (needs.validate-title.outputs.ci-mode != 'full' && needs.validate-title.outputs.ci-mode != 'markdown')
run: exit 1
windows-test:
- name: windows-latest / node-${{ matrix.node == '22.13.0' && '22' || matrix.node }} / tests-${{ matrix.shard }}
- needs: validate-title
+ name: windows-latest / node-22 / tests-${{ matrix.shard }}
+ needs: [validate-title, package]
if: needs.validate-title.outputs.ci-mode == 'full'
runs-on: windows-latest
timeout-minutes: 20
strategy:
fail-fast: false
matrix:
- node: ["22.13.0", "24"]
shard: [1, 2, 3, 4, 5, 6, 7]
-
steps:
- name: Checkout repository
uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
with:
persist-credentials: false
-
- name: Set up Node.js
uses: actions/setup-node@53b83947a5a98c8d113130e565377fae1a50d02f # v6
with:
- node-version: ${{ matrix.node }}
-
+ node-version: "22.13.0"
- name: Set up pnpm
run: npm install --global [email protected] --no-audit --no-fund
-
- name: Set up Bun
uses: oven-sh/setup-bun@0c5077e51419868618aeaa5fe8019c62421857d6 # v2
with:
bun-version: "1.3.14"
-
- name: Install dependencies
run: |
pnpm --dir sdk/typescript install --frozen-lockfile
pnpm --dir plugins/codex-security/mcp-app install --frozen-lockfile
-
- - name: Generate bundled plugin
- run: pnpm --dir sdk/typescript run build:plugin
-
+ - name: Download package for this commit
+ uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1
+ with:
+ name: package-${{ github.sha }}
+ path: dist
+ - name: Restore bundled plugin
+ shell: bash
+ run: tar -xzf dist/*.tgz --strip-components=1 -C sdk/typescript package/_bundled_plugin
- name: Prepare private Windows test root
id: windows-temp
shell: pwsh
run: ./sdk/typescript/scripts/prepare-windows-test-root.ps1
-
- name: Test shard ${{ matrix.shard }}
timeout-minutes: 10
env:
@@ -341,8 +432,17 @@ jobs:
TMP: ${{ steps.windows-temp.outputs.path }}
TMPDIR: ${{ steps.windows-temp.outputs.path }}
CODEX_SECURITY_ALLOW_MACHINE_POLICY_TEST: "false"
- run: node sdk/typescript/scripts/run-windows-ci-tests.mjs ${{ matrix.shard }}
-
+ run: node sdk/typescript/scripts/run-ci-tests.mjs ${{ matrix.shard }}/7
+ - name: Upload Windows test reports
+ if: always()
+ continue-on-error: true
+ uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
+ with:
+ name: bun-Windows-${{ matrix.shard }}
+ overwrite: true
+ path: sdk/typescript/reports/junit-*.xml
+ if-no-files-found: warn
+ retention-days: 14
- name: Test machine-wide PowerShell policy
if: matrix.shard == 3 && runner.environment == 'github-hosted'
timeout-minutes: 5
@@ -356,7 +456,7 @@ jobs:
windows-verify:
name: windows-latest / node-${{ matrix.node == '22.13.0' && '22' || matrix.node }} / verify
- needs: validate-title
+ needs: [validate-title, package]
if: needs.validate-title.outputs.ci-mode == 'full'
runs-on: windows-latest
timeout-minutes: 20
@@ -364,34 +464,26 @@ jobs:
fail-fast: false
matrix:
node: ["22.13.0", "24"]
-
steps:
- name: Checkout repository
uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
with:
persist-credentials: false
-
- name: Set up Node.js
uses: actions/setup-node@53b83947a5a98c8d113130e565377fae1a50d02f # v6
with:
node-version: ${{ matrix.node }}
-
- - name: Set up pnpm
- run: npm install --global [email protected] --no-audit --no-fund
-
- - name: Install dependencies
- run: |
- pnpm --dir sdk/typescript install --frozen-lockfile
- pnpm --dir plugins/codex-security/mcp-app install --frozen-lockfile
-
- - name: Pack
- working-directory: sdk/typescript
- run: pnpm pack --pack-destination ../../dist
-
- - name: Inspect package
+ cache: npm
+ cache-dependency-path: sdk/typescript/pnpm-lock.yaml
+ - name: Download package for this commit
+ uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1
+ with:
+ name: package-${{ github.sha }}
+ path: dist
+ - name: Inspect installed package
working-directory: sdk/typescript
shell: bash
- run: pnpm run check:package ../../dist/*.tgz
+ run: node scripts/check-package.mjs ../../dist/*.tgz
windows:
name: windows-latest / node-${{ matrix.node == '22.13.0' && '22' || matrix.node }}
@@ -402,7 +494,6 @@ jobs:
fail-fast: false
matrix:
node: ["22.13.0", "24"]
-
steps:
- name: Require every Windows coverage job
if: needs.validate-title.result != 'success' || (needs.validate-title.outputs.ci-mode == 'full' && (needs.windows-test.result != 'success' || needs.windows-verify.result != 'success')) || (needs.validate-title.outputs.ci-mode != 'full' && needs.validate-title.outputs.ci-mode != 'markdown')
diff --git a/.gitignore b/.gitignore
index fe166502f..d6b8b94b5 100644
--- a/.gitignore
+++ b/.gitignore
@@ -1,4 +1,5 @@
node_modules/
+/reports/
dist/
__pycache__/
.pytest_cache/
diff --git a/plugins/codex-security/pyproject.toml b/plugins/codex-security/pyproject.toml
index 85f487962..ae53a7b2b 100644
--- a/plugins/codex-security/pyproject.toml
+++ b/plugins/codex-security/pyproject.toml
@@ -8,6 +8,7 @@ test = [
"jsonschema[format-nongpl]==4.26.0",
"packaging==26.3",
"pytest==9.0.3",
+ "pytest-xdist==3.8.0",
"PyYAML==6.0.3",
"ruff==0.16.1",
]
diff --git a/plugins/codex-security/tests/test_finalize_scan_contract.py b/plugins/codex-security/tests/test_finalize_scan_contract.py
index bb5dd5010..c9a50e501 100644
--- a/plugins/codex-security/tests/test_finalize_scan_contract.py
+++ b/plugins/codex-security/tests/test_finalize_scan_contract.py
@@ -1202,7 +1202,7 @@ The extraction root is not enforced.
parent_alias.symlink_to(self.scan_dir.parent, target_is_directory=True)
for alias in (direct_alias, parent_alias / self.scan_dir.name):
- with self.subTest(alias=alias):
+ with self.subTest(alias=str(alias)):
output.write_bytes(protected_contents)
result = subprocess.run(
[
@@ -2559,7 +2559,7 @@ The extraction root is not enforced.
def test_recovery_rejects_malformed_severity_change_condition_lists(self) -> None:
for change_conditions in ([], [" "], ["Valid condition.", 1], ["\ud800"]):
- with self.subTest(change_conditions=change_conditions):
+ with self.subTest(change_conditions=repr(change_conditions)):
self.findings["findings"][0]["severity"]["changeConditions"] = change_conditions
self.write_scan()
warnings: list[str] = []
diff --git a/sdk/typescript/TESTING.md b/sdk/typescript/TESTING.md
index 187a5b8fc..261ce99c6 100644
--- a/sdk/typescript/TESTING.md
+++ b/sdk/typescript/TESTING.md
@@ -82,11 +82,32 @@ default to 100 cases; filesystem contract properties default to 20.
## GitHub Actions
`node-ci` retains the required `ubuntu-latest / node-22`,
-`macos-latest / node-22`, and `windows-latest / node-22` checks. Its Ubuntu
-Node 22 job runs static checks and uploads JUnit and LCOV. All supported runtime
-lanes still test and inspect an installed package. Package inspection includes
-a strict NodeNext TypeScript consumer and the actual installed CLI. Failed
-tests block CI; a failed diagnostic upload does not.
+`macos-latest / node-22`, and `windows-latest / node-22` checks. It builds and
+inspects one package archive, then passes that archive to jobs in the same
+workflow run, using the commit SHA in the artifact name. Every supported Node
+runtime still installs and inspects the package, including a strict NodeNext
+TypeScript consumer, the actual CLI, credential locking, dashboard assets, and
+a nested Codex worker. Native plugin-build tests remain in the shared Bun suite.
+
+The full Bun suite runs once per OS under Node 22: three file shards on Linux
+and macOS, and seven on Windows. The other Node versions run the installed
+package checks instead of repeating the same Bun suite. MCP and Python tests
+run in separate required jobs. Python uses four isolated pytest-xdist workers
+with work stealing; worker crashes fail the run without automatic restarts.
+
+`scripts/run-ci-tests.mjs` assigns the longest measured files first. Its
+`ci-test-durations.json` records per-file seconds from CI reports.
+Every new test file is included automatically with a one-second estimate.
+Refresh those estimates from the uploaded reports when adding or splitting
+expensive files; estimates affect scheduling, never whether a test runs.
+To reproduce one Windows shard locally after building the plugin, run
+`node scripts/run-ci-tests.mjs 3/7 --seed=12345`.
+
+Every Bun lane uploads JUnit; Linux lanes also upload LCOV per shard. Python
+reports include case durations, and the MCP runner can upload its JUnit report.
+Coverage is diagnostic and split across shards, not a combined percentage.
+Failed tests and missing package artifacts block CI; failed diagnostic uploads
+do not. Use `python -m pytest -n 0` to reproduce Python failures serially.
The separate `test-quality` workflow runs weekly, can be dispatched manually,
and runs on pull requests that change its workflow file. It compares Bun's
@@ -103,7 +124,7 @@ that breaks the Ink UI tests under isolation. Keep the trial pin until a newer
release passes the full SDK suite in every mode. Required CI and the mutation
trial remain on Bun 1.3.14.
-Keep the current file-balanced Windows runner until the native runner has
+Keep the measured file-balanced runner until the native runner has
matching inventories and acceptable Windows timings. Before promotion, compare
native and file-balanced shards using the same commit and Bun version.
Keep the machine-policy test serial. Do not replace the full required suite
diff --git a/sdk/typescript/scripts/ci-test-durations.json b/sdk/typescript/scripts/ci-test-durations.json
new file mode 100644
index 000000000..40c7b63d3
--- /dev/null
+++ b/sdk/typescript/scripts/ci-test-durations.json
@@ -0,0 +1,231 @@
+{
+ "sources": [
+ "https://github.com/openai/codex-security/actions/runs/33215429477",
+ "https://github.com/openai/codex-security/actions/runs/33227512538",
+ "https://github.com/openai/codex-security/actions/runs/33233690024"
+ ],
+ "unix": {
+ "api-attribution-concurrency.test.ts": 0.372,
+ "api-credentials.test.ts": 0.892,
+ "api-environment.test.ts": 0.0,
+ "api-events.test.ts": 2.277,
+ "api-integration.test.ts": 0.0,
+ "api-post-scan.test.ts": 7.161,
+ "api-preflight-config.test.ts": 1.053,
+ "api-surface.test.ts": 0.124,
+ "api.test.ts": 16.055,
+ "attack-path-tool-name.test.ts": 0.237,
+ "auth.test.ts": 1.425,
+ "build-plugin.test.ts": 3.973,
+ "bulk-scan-discovery.test.ts": 0.021,
+ "cli-authentication.test.ts": 5.757,
+ "cli-cloud-publish.test.ts": 0.354,
+ "cli-dedupe.test.ts": 0.061,
+ "cli-deep-scan-summary.test.ts": 0.06,
+ "cli-export.test.ts": 2.058,
+ "cli-history-paths.test.ts": 0.0,
+ "cli-launcher.test.ts": 8.154,
+ "cli-patch-trust.test.ts": 0.74,
+ "cli-patch.test.ts": 0.953,
+ "cli-publish.test.ts": 0.494,
+ "cli-scan-prompts.test.ts": 0.255,
+ "cli-serve.test.ts": 5.033,
+ "cli-signals.test.ts": 0.05,
+ "cli-skills.test.ts": 2.954,
+ "cli-verify-fix.test.ts": 0.158,
+ "cli-workbench.test.ts": 0.208,
+ "cli.test.ts": 3.071,
+ "cloud-publish.test.ts": 2.111,
+ "codex-review.test.ts": 0.246,
+ "compact-diff-scan.test.ts": 12.898,
+ "completed-scan-handoff.test.ts": 0.004,
+ "component-scan.test.ts": 0.396,
+ "config.test.ts": 0.403,
+ "container-entrypoint.test.ts": 0.051,
+ "contract.test.ts": 7.989,
+ "cost-model.property.test.ts": 0.01,
+ "cost.test.ts": 1.0,
+ "custom-publish.test.ts": 0.463,
+ "custom-validation.test.ts": 8.264,
+ "deep-progress.test.ts": 0.001,
+ "deep-scan-parent-denials.test.ts": 0.013,
+ "deep-scan-reducer-recovery.test.ts": 0.504,
+ "deep-scan-timestamp-compat.test.ts": 1.176,
+ "deep-scan-windows-executable.test.ts": 0.008,
+ "deep-scan-workbench.test.ts": 15.302,
+ "deep-scan-worker-shutdown.test.ts": 0.009,
+ "diff-rank-input.test.ts": 0.413,
+ "errors.property.test.ts": 0.033,
+ "errors.test.ts": 0.004,
+ "finding-deduplication.test.ts": 0.389,
+ "finding-embeddings.test.ts": 1.297,
+ "finding-preview.test.ts": 0.234,
+ "finding-workflow-integration.test.ts": 51.852,
+ "finding-workflow-publication.test.ts": 1.44,
+ "finding-workflow.test.ts": 0.455,
+ "findings-dashboard.test.ts": 0.001,
+ "findings-server.test.ts": 20.208,
+ "github.test.ts": 0.064,
+ "knowledge-base.test.ts": 1.665,
+ "linear.test.ts": 0.002,
+ "mcp-launcher.test.ts": 0.222,
+ "multiscan.test.ts": 2.819,
+ "package-provenance.test.ts": 0.001,
+ "package-smoke-timeouts.test.ts": 0.0,
+ "package-tar-listing.test.ts": 0.0,
+ "patch-risk-contract.test.ts": 2.318,
+ "patch-tui.test.ts": 4.739,
+ "plugin-apps.test.ts": 0.0,
+ "plugin-finding-detail-contract.test.ts": 2.077,
+ "plugin-report-limits.test.ts": 0.709,
+ "prompt-only-start-timeout.test.ts": 0.002,
+ "publication-check.test.ts": 0.004,
+ "publication-events.test.ts": 0.002,
+ "publication-integration.test.ts": 9.618,
+ "publication-store.test.ts": 17.125,
+ "publication.test.ts": 1.359,
+ "publish.test.ts": 1.395,
+ "release-automation.test.ts": 6.993,
+ "release-cut-workflow.test.ts": 0.0,
+ "repository-findings.test.ts": 0.107,
+ "result.test.ts": 0.002,
+ "runtime.test.ts": 27.803,
+ "sarif.test.ts": 0.79,
+ "scan-activity.test.ts": 0.006,
+ "scan-comparison.property.test.ts": 0.127,
+ "scan-comparison.test.ts": 0.14,
+ "scan-dashboard.test.ts": 0.058,
+ "scan-history-renderer.test.ts": 0.004,
+ "scan-logs.test.ts": 0.039,
+ "scan-recovery.test.ts": 25.336,
+ "skeleton.test.ts": 0.024,
+ "stopped-scan-results.test.ts": 7.685,
+ "targets-large-output.test.ts": 0.486,
+ "targets.test.ts": 0.855,
+ "test-reports.test.ts": 0.439,
+ "trusted-executable.test.ts": 0.266,
+ "update-notice.test.ts": 0.037,
+ "windows-machine-policy.test.ts": 0.0,
+ "workbench-canonical-paths.test.ts": 0.692,
+ "workbench-nested-git-utf8.test.ts": 0.063,
+ "workbench-patch.test.ts": 0.198,
+ "workbench-progress.test.ts": 0.493,
+ "workbench-remediation-timestamp.test.ts": 0.364,
+ "workbench-scan-history.test.ts": 0.199,
+ "workbench-scan-root-alias.test.ts": 0.0,
+ "workbench-windows-compatibility.test.ts": 0.417,
+ "worker-progress.property.test.ts": 0.009,
+ "worker-progress.test.ts": 0.003
+ },
+ "windows": {
+ "api-attribution-concurrency.test.ts": 18.599,
+ "api-credentials.test.ts": 56.214,
+ "api-environment.test.ts": 0.0,
+ "api-events.test.ts": 2.999,
+ "api-post-scan.test.ts": 11.43,
+ "api-preflight-config.test.ts": 1.84,
+ "api-surface.test.ts": 0.482,
+ "api.test.ts": 100.166,
+ "attack-path-tool-name.test.ts": 0.326,
+ "auth.test.ts": 2.932,
+ "build-plugin.test.ts": 4.531,
+ "bulk-scan-discovery.test.ts": 0.054,
+ "cli-authentication.test.ts": 45.757,
+ "cli-cloud-publish.test.ts": 0.494,
+ "cli-dedupe.test.ts": 0.059,
+ "cli-deep-scan-summary.test.ts": 0.072,
+ "cli-export.test.ts": 6.323,
+ "cli-history-paths.test.ts": 0.009,
+ "cli-launcher.test.ts": 9.241,
+ "cli-patch-trust.test.ts": 3.655,
+ "cli-patch.test.ts": 8.605,
+ "cli-publish.test.ts": 0.583,
+ "cli-scan-prompts.test.ts": 0.919,
+ "cli-serve.test.ts": 7.776,
+ "cli-signals.test.ts": 0.061,
+ "cli-skills.test.ts": 2.774,
+ "cli-verify-fix.test.ts": 0.173,
+ "cli-workbench.test.ts": 0.241,
+ "cli.test.ts": 6.814,
+ "cloud-publish.test.ts": 9.022,
+ "codex-review.test.ts": 0.642,
+ "compact-diff-scan.test.ts": 33.15,
+ "completed-scan-handoff.test.ts": 0.003,
+ "component-scan.test.ts": 3.831,
+ "config.test.ts": 5.1,
+ "contract.test.ts": 9.095,
+ "cost-model.property.test.ts": 0.012,
+ "cost.test.ts": 1.574,
+ "custom-publish.test.ts": 0.582,
+ "custom-validation.test.ts": 20.415,
+ "deep-progress.test.ts": 0.007,
+ "deep-scan-parent-denials.test.ts": 0.013,
+ "deep-scan-reducer-recovery.test.ts": 0.527,
+ "deep-scan-timestamp-compat.test.ts": 1.981,
+ "deep-scan-windows-executable.test.ts": 0.031,
+ "deep-scan-workbench.test.ts": 29.291,
+ "deep-scan-worker-shutdown.test.ts": 0.009,
+ "diff-rank-input.test.ts": 2.241,
+ "errors.property.test.ts": 0.032,
+ "errors.test.ts": 0.006,
+ "finding-deduplication.test.ts": 0.608,
+ "finding-embeddings.test.ts": 1.515,
+ "finding-preview.test.ts": 0.368,
+ "finding-workflow-integration.test.ts": 58.802,
+ "finding-workflow-publication.test.ts": 3.961,
+ "finding-workflow.test.ts": 1.472,
+ "findings-dashboard.test.ts": 0.001,
+ "findings-server.test.ts": 36.179,
+ "github.test.ts": 0.094,
+ "knowledge-base.test.ts": 13.919,
+ "linear.test.ts": 0.002,
+ "mcp-launcher.test.ts": 0.284,
+ "multiscan.test.ts": 31.671,
+ "package-provenance.test.ts": 0.0,
+ "package-smoke-timeouts.test.ts": 0.0,
+ "package-tar-listing.test.ts": 0.0,
+ "patch-risk-contract.test.ts": 4.074,
+ "patch-tui.test.ts": 5.4,
+ "plugin-apps.test.ts": 0.001,
+ "plugin-finding-detail-contract.test.ts": 3.221,
+ "plugin-report-limits.test.ts": 2.521,
+ "prompt-only-start-timeout.test.ts": 0.009,
+ "publication-check.test.ts": 0.005,
+ "publication-events.test.ts": 0.005,
+ "publication-integration.test.ts": 15.871,
+ "publication-store.test.ts": 31.854,
+ "publication.test.ts": 1.7,
+ "publish.test.ts": 7.768,
+ "release-automation.test.ts": 41.22,
+ "release-cut-workflow.test.ts": 0.0,
+ "repository-findings.test.ts": 0.277,
+ "result.test.ts": 0.008,
+ "runtime.test.ts": 134.065,
+ "sarif.test.ts": 1.416,
+ "scan-activity.test.ts": 0.007,
+ "scan-comparison.property.test.ts": 0.16,
+ "scan-comparison.test.ts": 10.704,
+ "scan-dashboard.test.ts": 0.061,
+ "scan-history-renderer.test.ts": 0.005,
+ "scan-logs.test.ts": 0.139,
+ "scan-recovery.test.ts": 62.302,
+ "skeleton.test.ts": 0.03,
+ "stopped-scan-results.test.ts": 15.814,
+ "targets-large-output.test.ts": 11.48,
+ "targets.test.ts": 8.454,
+ "test-reports.test.ts": 1.222,
+ "trusted-executable.test.ts": 0.064,
+ "update-notice.test.ts": 0.052,
+ "windows-machine-policy.test.ts": 3.448,
+ "workbench-canonical-paths.test.ts": 1.853,
+ "workbench-nested-git-utf8.test.ts": 0.364,
+ "workbench-patch.test.ts": 0.323,
+ "workbench-progress.test.ts": 0.802,
+ "workbench-remediation-timestamp.test.ts": 0.587,
+ "workbench-scan-history.test.ts": 0.547,
+ "workbench-scan-root-alias.test.ts": 0.276,
+ "workbench-windows-compatibility.test.ts": 1.479,
+ "worker-progress.property.test.ts": 0.013,
+ "worker-progress.test.ts": 0.003
+ }
+}
diff --git a/sdk/typescript/scripts/run-ci-tests.mjs b/sdk/typescript/scripts/run-ci-tests.mjs
new file mode 100644
index 000000000..477be4ad5
--- /dev/null
+++ b/sdk/typescript/scripts/run-ci-tests.mjs
@@ -0,0 +1,63 @@
+import { spawn } from "node:child_process";
+import { mkdir, readFile, readdir } from "node:fs/promises";
+import { fileURLToPath } from "node:url";
+import { shardTestFiles } from "./test-shards.mjs";
+
+const selection = /^([1-9]\d*)\/([1-9]\d*)$/.exec(process.argv[2] ?? "");
+const shard = Number(selection?.[1]);
+const count = Number(selection?.[2]);
+if (
+ !Number.isSafeInteger(shard) ||
+ !Number.isSafeInteger(count) ||
+ shard > count
+) {
+ throw new Error(
+ "Usage: node scripts/run-ci-tests.mjs <shard>/<count> [bun test options]",
+ );
+}
+const packageDirectory = fileURLToPath(new URL("../", import.meta.url));
+const tests = (await readdir(new URL("../tests-ts/", import.meta.url))).filter(
+ (file) =>
+ file.endsWith(".test.ts") && file !== "windows-machine-policy.test.ts",
+);
+const timings = JSON.parse(
+ await readFile(new URL("./ci-test-durations.json", import.meta.url), "utf8"),
+);
+const windows = process.platform === "win32";
+const assignments = shardTestFiles(
+ tests,
+ count,
+ timings[windows ? "windows" : "unix"],
+);
+const selected = assignments[shard - 1];
+if (selected.files.length === 0) {
+ throw new Error(`Test shard ${shard}/${count} is empty.`);
+}
+console.log(
+ `Test shard ${shard}/${count} (estimated ${selected.seconds.toFixed(1)}s): ${selected.files.join(" ")}`,
+);
+await mkdir(new URL("../reports/", import.meta.url), { recursive: true }).catch(
+ () => {
+ // Bun warns about report write failures without changing the test result.
+ },
+);
+const child = spawn(
+ "bun",
+ [
+ "test",
+ "--timeout",
+ windows ? "120000" : "30000",
+ "--reporter=junit",
+ `--reporter-outfile=reports/junit-${shard}.xml`,
+ ...process.argv.slice(3),
+ ...selected.files.map((file) => `./tests-ts/${file}`),
+ ],
+ { cwd: packageDirectory, stdio: "inherit", windowsHide: true },
+);
+child.once("error", (error) => {
+ console.error(error);
+ process.exitCode = 1;
+});
+child.once("close", (code) => {
+ process.exitCode = code ?? 1;
+});
diff --git a/sdk/typescript/scripts/run-windows-ci-tests.mjs b/sdk/typescript/scripts/run-windows-ci-tests.mjs
deleted file mode 100644
index 83b792e81..000000000
--- a/sdk/typescript/scripts/run-windows-ci-tests.mjs
+++ /dev/null
@@ -1,91 +0,0 @@
-import { spawn } from "node:child_process";
-import { readdir } from "node:fs/promises";
-import { fileURLToPath } from "node:url";
-
-const testsDirectory = new URL("../tests-ts/", import.meta.url);
-const packageDirectory = fileURLToPath(new URL("../", import.meta.url));
-const tests = (await readdir(testsDirectory))
- .filter(
- (file) =>
- file.endsWith(".test.ts") && file !== "windows-machine-policy.test.ts",
- )
- .sort();
-const shardSeeds = [
- ["api-credentials.test.ts"],
- ["api.test.ts"],
- ["runtime.test.ts"],
- ["cli-authentication.test.ts"],
- ["scan-recovery.test.ts"],
- [],
- [],
-];
-const assigned = new Set(shardSeeds.flat());
-for (const file of assigned) {
- if (!tests.includes(file)) {
- throw new Error("Windows CI test shard references a missing file: " + file);
- }
-}
-const unassigned = tests.filter((file) => !assigned.has(file));
-const slowRemainderFiles = new Set([
- "deep-scan-workbench.test.ts",
- "release-automation.test.ts",
- "scan-comparison.test.ts",
-]);
-for (const [index, file] of unassigned.entries()) {
- shardSeeds[slowRemainderFiles.has(file) ? 6 : 5 + (index % 2)].push(file);
-}
-
-const assignments = shardSeeds.flat();
-if (
- assignments.length !== tests.length ||
- new Set(assignments).size !== tests.length
-) {
- throw new Error("Windows CI test shards must run every test file once.");
-}
-
-const requestedShard =
- process.argv[2] === undefined ? undefined : Number(process.argv[2]);
-if (
- requestedShard !== undefined &&
- (!Number.isSafeInteger(requestedShard) ||
- requestedShard < 1 ||
- requestedShard > shardSeeds.length)
-) {
- throw new Error("Usage: node scripts/run-windows-ci-tests.mjs [1-7]");
-}
-const selectedShards =
- requestedShard === undefined
- ? shardSeeds.map((files, index) => ({ files, index }))
- : [{ files: shardSeeds[requestedShard - 1], index: requestedShard - 1 }];
-
-const results = await Promise.all(
- selectedShards.map(
- ({ files, index }) =>
- new Promise((resolve, reject) => {
- const paths = files.map((file) => "./tests-ts/" + file);
- console.log(
- "Windows CI test shard " +
- (index + 1) +
- "/" +
- shardSeeds.length +
- ": " +
- paths.join(" "),
- );
- // Native Windows credential and document checks can exceed 30 seconds.
- // The workflow still bounds each complete shard to ten minutes.
- const child = spawn("bun", ["test", "--timeout", "120000", ...paths], {
- cwd: packageDirectory,
- stdio: "inherit",
- windowsHide: true,
- });
- child.once("error", reject);
- child.once("close", (code) => {
- resolve(code ?? 1);
- });
- }),
- ),
-);
-
-if (results.some((code) => code !== 0)) {
- process.exitCode = 1;
-}
diff --git a/sdk/typescript/scripts/test-shards.d.mts b/sdk/typescript/scripts/test-shards.d.mts
new file mode 100644
index 000000000..51d7be683
--- /dev/null
+++ b/sdk/typescript/scripts/test-shards.d.mts
@@ -0,0 +1,5 @@
+export function shardTestFiles(
+ files: readonly string[],
+ count: number,
+ durations?: Readonly<Record<string, number>>,
+): Array<{ files: string[]; seconds: number }>;
diff --git a/sdk/typescript/scripts/test-shards.mjs b/sdk/typescript/scripts/test-shards.mjs
new file mode 100644
index 000000000..e6e72e427
--- /dev/null
+++ b/sdk/typescript/scripts/test-shards.mjs
@@ -0,0 +1,24 @@
+/** Place the longest measured files first, without omitting new test files. */
+export function shardTestFiles(files, count, durations = {}) {
+ if (!Number.isSafeInteger(count) || count < 1) {
+ throw new Error("The test shard count must be a positive integer.");
+ }
+ const shards = Array.from({ length: count }, () => ({
+ files: [],
+ seconds: 0,
+ }));
+ const duration = (file) => durations[file] ?? 1;
+ const ordered = [...files].sort(
+ (left, right) =>
+ duration(right) - duration(left) ||
+ (left < right ? -1 : left > right ? 1 : 0),
+ );
+ for (const file of ordered) {
+ const shard = shards.reduce((shortest, candidate) =>
+ candidate.seconds < shortest.seconds ? candidate : shortest,
+ );
+ shard.files.push(file);
+ shard.seconds += duration(file);
+ }
+ return shards;
+}
diff --git a/sdk/typescript/tests-ts/attack-path-tool-name.test.ts b/sdk/typescript/tests-ts/attack-path-tool-name.test.ts
index 03a007f4d..c11a599f2 100644
--- a/sdk/typescript/tests-ts/attack-path-tool-name.test.ts
+++ b/sdk/typescript/tests-ts/attack-path-tool-name.test.ts
@@ -1,30 +1,38 @@
-import { spawnSync } from "node:child_process";
import { mkdtempSync, rmSync } from "node:fs";
import { tmpdir } from "node:os";
import { join } from "node:path";
import { expect, test } from "bun:test";
import { PLUGIN_ROOT } from "./plugin-root.js";
-test("keeps model-visible attack-path tool names within the Codex limit", () => {
+test("keeps model-visible attack-path tool names within the Codex limit", async () => {
const node = Bun.which("node");
expect(node).not.toBeNull();
const state = mkdtempSync(join(tmpdir(), "codex-security-mcp-tools-"));
try {
- const server = spawnSync(node!, [join(PLUGIN_ROOT, "mcp", "server.mjs")], {
- encoding: "utf8",
+ const server = Bun.spawn({
+ cmd: [node!, join(PLUGIN_ROOT, "mcp", "server.mjs")],
env: { ...process.env, CODEX_SECURITY_STATE_DIR: state },
- input: [
- '{"jsonrpc":"2.0","id":1,"method":"initialize","params":{"protocolVersion":"2025-11-25","capabilities":{},"clientInfo":{"name":"codex-security-test","version":"1.0.0"}}}',
- '{"jsonrpc":"2.0","method":"notifications/initialized","params":{}}',
- '{"jsonrpc":"2.0","id":2,"method":"tools/list","params":{}}',
- "",
- ].join("\n"),
+ stdin: new TextEncoder().encode(
+ [
+ '{"jsonrpc":"2.0","id":1,"method":"initialize","params":{"protocolVersion":"2025-11-25","capabilities":{},"clientInfo":{"name":"codex-security-test","version":"1.0.0"}}}',
+ '{"jsonrpc":"2.0","method":"notifications/initialized","params":{}}',
+ '{"jsonrpc":"2.0","id":2,"method":"tools/list","params":{}}',
+ "",
+ ].join("\n"),
+ ),
+ stdout: "pipe",
+ stderr: "pipe",
timeout: 30_000,
});
- expect(server.status, server.stderr).toBe(0);
+ const [status, stdout, stderr] = await Promise.all([
+ server.exited,
+ new Response(server.stdout).text(),
+ new Response(server.stderr).text(),
+ ]);
+ expect(status, stderr).toBe(0);
- const tools = server.stdout
+ const tools = stdout
.trim()
.split("\n")
.map((line) => JSON.parse(line))
diff --git a/sdk/typescript/tests-ts/cli-launcher.test.ts b/sdk/typescript/tests-ts/cli-launcher.test.ts
index d427bb05b..5a0e6df2b 100644
--- a/sdk/typescript/tests-ts/cli-launcher.test.ts
+++ b/sdk/typescript/tests-ts/cli-launcher.test.ts
@@ -27,14 +27,22 @@ describe("CLI launcher", () => {
if (process.platform !== "win32") {
await symlink(launcher, bin);
}
- const child = spawnSync(process.execPath, [bin, "--version"], {
- encoding: "utf8",
+ const child = Bun.spawn({
+ cmd: [process.execPath, bin, "--version"],
+ stdin: "ignore",
+ stdout: "pipe",
+ stderr: "pipe",
timeout: 30_000,
});
+ const [status, stdout, stderr] = await Promise.all([
+ child.exited,
+ new Response(child.stdout).text(),
+ new Response(child.stderr).text(),
+ ]);
- expect(child.status).toBe(0);
- expect(child.stderr).toBe("");
- expect(child.stdout).toBe(`${VERSION}\n`);
+ expect(status, stderr).toBe(0);
+ expect(stderr).toBe("");
+ expect(stdout).toBe(`${VERSION}\n`);
} finally {
await rm(root, { recursive: true, force: true });
}
@@ -48,15 +56,28 @@ describe("CLI launcher", () => {
preload,
`Object.defineProperty(process, "cwd", { value() { throw new Error(${JSON.stringify(`working directory is unavailable: ${SYNTHETIC_CREDENTIALS}`)}); } });\n`,
);
- const child = spawnSync(
- process.execPath,
- ["--preload", preload, join(packageRoot, "src", "cli.ts"), "scan"],
- { encoding: "utf8", timeout: 30_000 },
- );
+ const child = Bun.spawn({
+ cmd: [
+ process.execPath,
+ "--preload",
+ preload,
+ join(packageRoot, "src", "cli.ts"),
+ "scan",
+ ],
+ stdin: "ignore",
+ stdout: "pipe",
+ stderr: "pipe",
+ timeout: 30_000,
+ });
+ const [status, stdout, stderr] = await Promise.all([
+ child.exited,
+ new Response(child.stdout).text(),
+ new Response(child.stderr).text(),
+ ]);
- expect(child.status).toBe(2);
- expect(child.stdout).toBe("");
- expect(child.stderr).toBe(
+ expect(status, stderr).toBe(2);
+ expect(stdout).toBe("");
+ expect(stderr).toBe(
`working directory is unavailable: ${SYNTHETIC_CREDENTIALS}\n`,
);
} finally {
diff --git a/sdk/typescript/tests-ts/release-automation.test.ts b/sdk/typescript/tests-ts/release-automation.test.ts
index d1900db39..0d9d42f56 100644
--- a/sdk/typescript/tests-ts/release-automation.test.ts
+++ b/sdk/typescript/tests-ts/release-automation.test.ts
@@ -3958,7 +3958,10 @@ describe("GitHub release workflow safeguards", () => {
const fullCiCondition = "needs.validate-title.outputs.ci-mode == 'full'";
for (const job of [
+ "package",
"test",
+ "compatibility",
+ "mcp",
"plugin-source",
"windows-test",
"windows-verify",
@@ -3987,7 +3990,7 @@ describe("GitHub release workflow safeguards", () => {
expect(workflow.jobs["required-test"]?.if).toBe(requiredJobCondition);
expect(workflow.jobs["windows"]?.if).toBe(requiredJobCondition);
expect(workflow.jobs["required-test"]?.steps[0]?.if).toBe(
- "needs.validate-title.result != 'success' || (needs.validate-title.outputs.ci-mode == 'full' && (needs.test.result != 'success' || needs.plugin-source.result != 'success')) || (needs.validate-title.outputs.ci-mode != 'full' && needs.validate-title.outputs.ci-mode != 'markdown')",
+ "needs.validate-title.result != 'success' || (needs.validate-title.outputs.ci-mode == 'full' && (needs.package.result != 'success' || needs.test.result != 'success' || needs.compatibility.result != 'success' || needs.mcp.result != 'success' || needs.plugin-source.result != 'success')) || (needs.validate-title.outputs.ci-mode != 'full' && needs.validate-title.outputs.ci-mode != 'markdown')",
);
expect(workflow.jobs["windows"]?.steps[0]?.if).toBe(
"needs.validate-title.result != 'success' || (needs.validate-title.outputs.ci-mode == 'full' && (needs.windows-test.result != 'success' || needs.windows-verify.result != 'success')) || (needs.validate-title.outputs.ci-mode != 'full' && needs.validate-title.outputs.ci-mode != 'markdown')",
@@ -4002,6 +4005,9 @@ describe("GitHub release workflow safeguards", () => {
] as const) {
const values = {
"needs.plugin-source.result": upstream,
+ "needs.package.result": upstream,
+ "needs.compatibility.result": upstream,
+ "needs.mcp.result": upstream,
"needs.test.result": upstream,
"needs.validate-title.outputs.ci-mode": ciMode,
"needs.validate-title.result": validation,
@@ -4019,6 +4025,35 @@ describe("GitHub release workflow safeguards", () => {
}
}
+ const dependencies = [
+ "package",
+ "test",
+ "compatibility",
+ "mcp",
+ "plugin-source",
+ ];
+ for (const dependency of dependencies) {
+ for (const result of ["failure", "cancelled", "skipped"]) {
+ expect(
+ evaluateWorkflowCondition(
+ workflow.jobs["required-test"]?.steps[0]?.if ?? "",
+ {
+ "needs.validate-title.result": "success",
+ "needs.validate-title.outputs.ci-mode": "full",
+ ...Object.fromEntries(
+ dependencies.map((job) => [
+ `needs.${job}.result`,
+ job === dependency ? result : "success",
+ ]),
+ ),
+ },
+ ),
+ `${dependency}: ${result}`,
+ ).toBe(true);
+ }
+ }
+ expect(workflow.jobs["required-test"]?.steps[0]?.run).toBe("exit 1");
+
const renderName = (template: string, values: Record<string, string>) => {
let name = template;
for (const [key, value] of Object.entries(values)) {
diff --git a/sdk/typescript/tests-ts/skeleton.test.ts b/sdk/typescript/tests-ts/skeleton.test.ts
index 62c2aaafb..27fda31d6 100644
--- a/sdk/typescript/tests-ts/skeleton.test.ts
+++ b/sdk/typescript/tests-ts/skeleton.test.ts
@@ -101,35 +101,48 @@ describe("TypeScript package skeleton", () => {
).toEqual([]);
});
- test("pins each Node.js minimum and preserves protected and latest LTS checks", async () => {
+ test("runs Bun once per OS and checks every supported Node runtime with the installed package", async () => {
const { jobs } = await workflow("node-ci.yml");
- expect(jobs["test"]?.name).toContain(
- "tests / ${{ matrix.os }} / node-${{ matrix.node == '22.13.0' && '22' || matrix.node }}",
- );
expect(jobs["test"]?.strategy?.matrix).toEqual({
os: ["ubuntu-latest", "macos-latest"],
- node: ["22.13.0"],
- include: ["24.0.0", "24", "26.0.0", "26"].map((node) => ({
- os: "ubuntu-latest",
- node,
- })),
+ shard: [1, 2, 3],
});
- expect(jobs["required-test"]?.name).toContain("${{ matrix.os }} / node-22");
- expect(jobs["required-test"]?.strategy?.matrix).toEqual({
- os: ["ubuntu-latest", "macos-latest"],
+ expect(jobs["compatibility"]?.strategy?.matrix).toEqual({
+ os: ["ubuntu-latest"],
+ node: ["22.13.0", "24.0.0", "24", "26.0.0", "26"],
+ include: [{ os: "macos-latest", node: "22.13.0" }],
});
- expect(jobs["windows"]?.name).toContain(
- "windows-latest / node-${{ matrix.node == '22.13.0' && '22' || matrix.node }}",
- );
+ expect(jobs["windows-test"]?.strategy?.matrix).toEqual({
+ shard: [1, 2, 3, 4, 5, 6, 7],
+ });
+ expect(jobs["windows-verify"]?.strategy?.matrix["node"]).toEqual([
+ "22.13.0",
+ "24",
+ ]);
+ expect(jobs["required-test"]?.name).toBe("${{ matrix.os }} / node-22");
+ expect(jobs["required-test"]?.needs).toEqual([
+ "validate-title",
+ "package",
+ "test",
+ "compatibility",
+ "mcp",
+ "plugin-source",
+ ]);
expect(jobs["windows"]?.needs).toEqual([
"validate-title",
"windows-test",
"windows-verify",
]);
- expect(jobs["windows-test"]?.strategy?.matrix["node"]).toEqual([
- "22.13.0",
- "24",
- ]);
+ for (const name of ["compatibility", "windows-verify"]) {
+ expect(jobs[name]?.steps).toContainEqual(
+ expect.objectContaining({
+ run: "node scripts/check-package.mjs ../../dist/*.tgz",
+ }),
+ );
+ expect(jobs[name]?.steps.some(({ name }) => name === "Set up Bun")).toBe(
+ false,
+ );
+ }
});
test("randomizes tests and keeps the default and Windows CI timeouts", async () => {
@@ -140,7 +153,6 @@ describe("TypeScript package skeleton", () => {
const bunConfig = parse(
await readFile(new URL("../bunfig.toml", import.meta.url), "utf8"),
);
-
expect(packageJson.scripts.test).toBe(
"node --run build:plugin && bun test --timeout 30000 ./tests-ts",
);
@@ -148,50 +160,73 @@ describe("TypeScript package skeleton", () => {
expect(packageJson.scripts["test:ci"]).toContain("pnpm run test ");
expect(jobs["windows-test"]?.steps).toContainEqual(
expect.objectContaining({
- run: "node sdk/typescript/scripts/run-windows-ci-tests.mjs ${{ matrix.shard }}",
+ run: "node sdk/typescript/scripts/run-ci-tests.mjs ${{ matrix.shard }}/7",
}),
);
});
- test("generates the bundled plugin before Windows test shards", async () => {
+ test("checks one archive and restores its plugin before every test shard", async () => {
const { jobs } = await workflow("node-ci.yml");
- const steps = jobs["windows-test"]!.steps;
- const dependencyInstallation = steps.findIndex(
- (step) => step.name === "Install dependencies",
- );
- const pluginGeneration = steps.findIndex(
- (step) => step.name === "Generate bundled plugin",
+ const uploads = jobs["package"]!.steps;
+ const inspection = uploads.findIndex(
+ ({ name }) => name === "Inspect archive contents",
);
- const shard = steps.findIndex(
- (step) => step.name === "Test shard ${{ matrix.shard }}",
+ const upload = uploads.findIndex(
+ ({ name }) => name === "Upload package for this commit",
);
-
- expect(steps[dependencyInstallation]?.run).toContain(
- "pnpm --dir plugins/codex-security/mcp-app install --frozen-lockfile",
- );
- expect(steps[pluginGeneration]?.run).toBe(
- "pnpm --dir sdk/typescript run build:plugin",
- );
- expect(dependencyInstallation).toBeLessThan(pluginGeneration);
- expect(pluginGeneration).toBeLessThan(shard);
+ expect(inspection).toBeGreaterThanOrEqual(0);
+ expect(inspection).toBeLessThan(upload);
+ expect(uploads[upload]?.with).toMatchObject({
+ name: "package-${{ github.sha }}",
+ "if-no-files-found": "error",
+ });
+ expect(uploads[upload]).not.toHaveProperty("continue-on-error");
+ for (const name of [
+ "test",
+ "windows-test",
+ "mcp",
+ "compatibility",
+ "windows-verify",
+ ]) {
+ const job = jobs[name]!;
+ expect(job.needs).toContain("package");
+ expect(
+ job.steps.find(
+ ({ name }) => name === "Download package for this commit",
+ )?.with,
+ ).toEqual({ name: "package-${{ github.sha }}", path: "dist" });
+ }
+ for (const name of ["test", "windows-test", "mcp"]) {
+ const steps = jobs[name]!.steps;
+ const restore = steps.findIndex(
+ ({ name }) => name === "Restore bundled plugin",
+ );
+ const testStep = steps.findIndex(
+ ({ name }) =>
+ name === "Test" ||
+ name === "Test shard ${{ matrix.shard }}" ||
+ name === "Test MCP app",
+ );
+ expect(restore).toBeGreaterThanOrEqual(0);
+ expect(restore).toBeLessThan(testStep);
+ expect(steps[restore]?.run).toContain("package/_bundled_plugin");
+ expect(steps[testStep]).not.toHaveProperty("continue-on-error");
+ }
});
- test("installs ripgrep before the MCP app tests", async () => {
+ test("installs ripgrep before the independent MCP job", async () => {
const { jobs } = await workflow("node-ci.yml");
- const steps = jobs["test"]!.steps;
- const ripgrepInstallation = steps.findIndex(
- (step) => step.name === "Install ripgrep",
- );
- const mcpTests = steps.findIndex((step) => step.name === "Test MCP app");
-
- expect(steps[ripgrepInstallation]).toMatchObject({
- if: "matrix.os == 'ubuntu-latest' && matrix.node == '22.13.0'",
- run: expect.stringContaining("apt-get install --yes ripgrep"),
- });
- expect(ripgrepInstallation).toBeLessThan(mcpTests);
+ const steps = jobs["mcp"]!.steps;
+ const ripgrep = steps.findIndex(({ name }) => name === "Install ripgrep");
+ const tests = steps.findIndex(({ name }) => name === "Test MCP app");
+ expect(steps[ripgrep]?.run).toContain("apt-get install --yes ripgrep");
+ expect(ripgrep).toBeLessThan(tests);
+ expect(
+ jobs["test"]!.steps.some(({ name }) => name === "Test MCP app"),
+ ).toBe(false);
});
- test("runs shared static checks once and keeps report upload non-blocking", async () => {
+ test("runs shared static checks once and keeps every diagnostic upload non-blocking", async () => {
const { jobs } = await workflow("node-ci.yml");
const steps = Object.values(jobs).flatMap((job) => job.steps);
for (const name of [
@@ -199,27 +234,29 @@ describe("TypeScript package skeleton", () => {
"Typecheck",
"Check formatting",
]) {
- expect(steps.filter((step) => step.name === name)).toEqual([
- expect.objectContaining({
- if: "matrix.os == 'ubuntu-latest' && matrix.node == '22.13.0'",
- }),
- ]);
- }
- expect(steps.find((step) => step.name === "Test")).not.toHaveProperty(
- "continue-on-error",
- );
- expect(
- steps.find((step) => step.name === "Upload test reports"),
- ).toMatchObject({
- "continue-on-error": true,
- });
- for (const name of ["test", "windows-verify"]) {
- expect(jobs[name]?.steps).toContainEqual(
- expect.objectContaining({
- run: "pnpm run check:package ../../dist/*.tgz",
- }),
+ expect(steps.filter((step) => step.name === name)).toHaveLength(1);
+ expect(jobs["package"]!.steps.some((step) => step.name === name)).toBe(
+ true,
);
}
+ for (const name of [
+ "Upload test reports",
+ "Upload Windows test reports",
+ "Upload MCP test reports",
+ "Upload Python test reports",
+ ]) {
+ expect(steps.find((step) => step.name === name)).toMatchObject({
+ if: "always()",
+ "continue-on-error": true,
+ });
+ }
+ expect(
+ jobs["plugin-source"]!.steps.find(
+ ({ name }) => name === "Test Python source contracts",
+ )?.run,
+ ).toContain(
+ "-n 4 --dist worksteal --max-worker-restart 0 --durations=30 --junitxml=reports/python.xml",
+ );
});
test("keeps machine-wide policy changes out of parallel and experimental runs", async () => {
@@ -297,7 +334,6 @@ describe("TypeScript package skeleton", () => {
const uploads = [...Object.values(ci.jobs), ...Object.values(quality.jobs)]
.flatMap((job) => job.steps)
.filter((step) => step.uses?.startsWith("actions/upload-artifact@"));
- expect(uploads).toHaveLength(3);
for (const upload of uploads) {
expect(upload.with?.["overwrite"]).toBe(true);
}
diff --git a/sdk/typescript/tests-ts/test-shards.test.ts b/sdk/typescript/tests-ts/test-shards.test.ts
new file mode 100644
index 000000000..95b5289eb
--- /dev/null
+++ b/sdk/typescript/tests-ts/test-shards.test.ts
@@ -0,0 +1,138 @@
+import {
+ copyFile,
+ mkdir,
+ mkdtemp,
+ readdir,
+ readFile,
+ rm,
+ writeFile,
+} from "node:fs/promises";
+import { tmpdir } from "node:os";
+import { delimiter, dirname, join } from "node:path";
+import { expect, test } from "bun:test";
+import { shardTestFiles } from "../scripts/test-shards.mjs";
+
+test("balances measured work and includes new files without mutating the inventory", () => {
+ const files = [
+ "new.test.ts",
+ "small.test.ts",
+ "medium.test.ts",
+ "slow.test.ts",
+ ];
+ const shards = shardTestFiles(files, 2, {
+ "slow.test.ts": 100,
+ "medium.test.ts": 70,
+ "small.test.ts": 30,
+ });
+ expect(shards.map(({ seconds }) => seconds)).toEqual([101, 100]);
+ expect(shards.flatMap(({ files }) => files).sort()).toEqual(
+ [...files].sort(),
+ );
+ expect(files[0]).toBe("new.test.ts");
+ expect(
+ shardTestFiles([...files].reverse(), 2, {
+ "slow.test.ts": 100,
+ "medium.test.ts": 70,
+ "small.test.ts": 30,
+ }),
+ ).toEqual(shards);
+});
+
+test.each([0, -1, 1.5, Number.NaN])(
+ "rejects an invalid shard count %p",
+ (count) => {
+ expect(() => shardTestFiles(["a.test.ts"], count)).toThrow(
+ "positive integer",
+ );
+ },
+);
+
+test.each([
+ ["unix", 3],
+ ["windows", 7],
+] as const)("covers every shared test once on %s", async (platform, count) => {
+ const tests = (await readdir(new URL("./", import.meta.url)))
+ .filter(
+ (file) =>
+ file.endsWith(".test.ts") && file !== "windows-machine-policy.test.ts",
+ )
+ .sort();
+ const timings = JSON.parse(
+ await readFile(
+ new URL("../scripts/ci-test-durations.json", import.meta.url),
+ "utf8",
+ ),
+ );
+ const shards = shardTestFiles(tests, count, timings[platform]);
+ expect(shards.every(({ files }) => files.length > 0)).toBe(true);
+ expect(shards.flatMap(({ files }) => files).sort()).toEqual(tests);
+});
+
+test.each([
+ ["available", false],
+ ["available", true],
+ ["blocked directory", false],
+ ["blocked directory", true],
+] as const)(
+ "preserves the test result with %s reports and failure=%p",
+ async (report, fail) => {
+ const node = Bun.which("node");
+ expect(node).not.toBeNull();
+ const root = await mkdtemp(join(tmpdir(), "codex-security-shard-report-"));
+ try {
+ await mkdir(join(root, "scripts"));
+ await mkdir(join(root, "tests-ts"));
+ for (const file of [
+ "run-ci-tests.mjs",
+ "test-shards.mjs",
+ "ci-test-durations.json",
+ ]) {
+ await copyFile(
+ new URL(`../scripts/${file}`, import.meta.url),
+ join(root, "scripts", file),
+ );
+ }
+ await writeFile(
+ join(root, "tests-ts", "probe.test.ts"),
+ `import { expect, test } from "bun:test"; test("synthetic report probe", () => expect(true).toBe(${!fail}));\n`,
+ );
+ if (report === "blocked directory") {
+ await writeFile(join(root, "reports"), "synthetic blocker");
+ }
+ const child = Bun.spawn({
+ cmd: [node!, join(root, "scripts", "run-ci-tests.mjs"), "1/1"],
+ env: {
+ ...process.env,
+ PATH: `${dirname(process.execPath)}${delimiter}${process.env["PATH"] ?? ""}`,
+ },
+ stdin: "ignore",
+ stdout: "pipe",
+ stderr: "pipe",
+ timeout: 30_000,
+ });
+ const [status, stdout, stderr] = await Promise.all([
+ child.exited,
+ new Response(child.stdout).text(),
+ new Response(child.stderr).text(),
+ ]);
+ expect(status, stderr).toBe(fail ? 1 : 0);
+ expect(stdout).toContain("probe.test.ts");
+ expect(stderr).toContain("synthetic report probe");
+ if (report === "available") {
+ const xml = await readFile(
+ join(root, "reports", "junit-1.xml"),
+ "utf8",
+ );
+ expect(xml).toContain("<testcase ");
+ expect(xml.includes("<failure ")).toBe(fail);
+ } else {
+ expect(stderr).toContain("JUnitReportFailed");
+ expect(await readFile(join(root, "reports"), "utf8")).toBe(
+ "synthetic blocker",
+ );
+ }
+ } finally {
+ await rm(root, { recursive: true, force: true });
+ }
+ },
+);