Skip to content

OpenAI's Codex Security CLI and TypeScript SDK for finding, validating, and fixing security vulnerabilities. npm: https://www.npmjs.com/package/@openai/codex-security

ci: report test quality and exercise native Bun runners (#508)

mldangelo-oai9e061f251cc6
diff --git a/.github/workflows/node-ci.yml b/.github/workflows/node-ci.yml
index c6398851d..b8704c94c 100644
--- a/.github/workflows/node-ci.yml
+++ b/.github/workflows/node-ci.yml
@@ -66,6 +66,7 @@ jobs:
run: pnpm --dir sdk/typescript run audit:prod
- name: Typecheck
+ if: matrix.os == 'ubuntu-latest' && matrix.node == '22.13.0'
run: pnpm --dir sdk/typescript run types
- name: Test
@@ -75,9 +76,23 @@ jobs:
TMP: ${{ runner.temp }}
TMPDIR: ${{ runner.temp }}
CODEX_SECURITY_ALLOW_MACHINE_POLICY_TEST: "false"
- run: pnpm --dir sdk/typescript run test
+ run: pnpm --dir sdk/typescript run ${{ matrix.os == 'ubuntu-latest' && matrix.node == '22.13.0' && 'test:ci' || 'test' }}
+
+ - name: Upload test reports
+ if: always() && matrix.os == 'ubuntu-latest' && matrix.node == '22.13.0'
+ continue-on-error: true
+ uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
+ with:
+ name: node-22-test-reports
+ overwrite: true
+ path: |
+ sdk/typescript/reports/junit.xml
+ sdk/typescript/coverage/lcov.info
+ if-no-files-found: warn
+ retention-days: 14
- name: Check formatting
+ if: matrix.os == 'ubuntu-latest' && matrix.node == '22.13.0'
run: pnpm --dir sdk/typescript run format
- name: Pack
@@ -89,21 +104,6 @@ jobs:
shell: bash
run: pnpm run check:package ../../dist/*.tgz
- - name: Smoke-test Node.js runtime
- working-directory: sdk/typescript
- shell: bash
- run: |
- set -euo pipefail
- node --input-type=module --eval '
- import { CodexSecurity } from "@openai/codex-security";
-
- if (typeof CodexSecurity !== "function") {
- throw new Error("The SDK does not export CodexSecurity.");
- }
- '
- node bin/codex-security.mjs --version
- node bin/codex-security.mjs --help
-
windows-test:
name: windows-latest / node-${{ matrix.node == '22.13.0' && '22' || matrix.node }} / tests-${{ matrix.shard }}
runs-on: windows-latest
@@ -162,16 +162,6 @@ jobs:
CODEX_SECURITY_ALLOW_MACHINE_POLICY_TEST: "true"
run: bun test --timeout 120000 ./tests-ts/windows-machine-policy.test.ts
- - name: Typecheck
- if: matrix.shard == 7
- working-directory: sdk/typescript
- run: pnpm run types
-
- - name: Check formatting
- if: matrix.shard == 7
- working-directory: sdk/typescript
- run: pnpm run format
-
windows-verify:
name: windows-latest / node-${{ matrix.node == '22.13.0' && '22' || matrix.node }} / verify
runs-on: windows-latest
@@ -210,21 +200,6 @@ jobs:
shell: bash
run: pnpm run check:package ../../dist/*.tgz
- - name: Smoke-test Node.js runtime
- working-directory: sdk/typescript
- shell: bash
- run: |
- set -euo pipefail
- node --input-type=module --eval '
- import { CodexSecurity } from "@openai/codex-security";
-
- if (typeof CodexSecurity !== "function") {
- throw new Error("The SDK does not export CodexSecurity.");
- }
- '
- node bin/codex-security.mjs --version
- node bin/codex-security.mjs --help
-
windows:
name: windows-latest / node-${{ matrix.node == '22.13.0' && '22' || matrix.node }}
runs-on: ubuntu-latest
diff --git a/.github/workflows/test-quality.yml b/.github/workflows/test-quality.yml
new file mode 100644
index 000000000..ff70a17ed
--- /dev/null
+++ b/.github/workflows/test-quality.yml
@@ -0,0 +1,161 @@
+name: test-quality
+
+on:
+ pull_request:
+ paths:
+ - .github/workflows/test-quality.yml
+ workflow_dispatch:
+ schedule:
+ - cron: "23 9 * * 1"
+
+permissions:
+ contents: read
+
+concurrency:
+ group: ${{ github.workflow }}-${{ github.ref }}
+ cancel-in-progress: true
+
+env:
+ CODEX_SECURITY_INTEGRATION: "0"
+ CODEX_SECURITY_ALLOW_MACHINE_POLICY_TEST: "false"
+
+jobs:
+ runner:
+ name: ${{ matrix.os }} / ${{ matrix.mode }}
+ runs-on: ${{ matrix.os }}
+ timeout-minutes: 20
+ env:
+ CODEX_SECURITY_PROPERTY_SEED: ${{ github.event_name == 'pull_request' && 1 || github.run_number }}
+ strategy:
+ fail-fast: false
+ matrix:
+ os: [ubuntu-latest, windows-latest]
+ mode: [baseline, isolated, parallel]
+ include:
+ - mode: baseline
+ args: ""
+ - mode: isolated
+ args: --isolate
+ - mode: parallel
+ args: --parallel=2
+ - os: windows-latest
+ mode: shard-1
+ args: --shard=1/7
+ - os: windows-latest
+ mode: shard-2
+ args: --shard=2/7
+ - os: windows-latest
+ mode: shard-3
+ args: --shard=3/7
+ - os: windows-latest
+ mode: shard-4
+ args: --shard=4/7
+ - os: windows-latest
+ mode: shard-5
+ args: --shard=5/7
+ - os: windows-latest
+ mode: shard-6
+ args: --shard=6/7
+ - os: windows-latest
+ mode: shard-7
+ args: --shard=7/7
+ steps:
+ - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
+ with:
+ persist-credentials: false
+ - uses: actions/setup-node@53b83947a5a98c8d113130e565377fae1a50d02f # v6
+ with:
+ node-version: "22.13.0"
+ - name: Set up pnpm
+ uses: pnpm/action-setup@0ebf47130e4866e96fce0953f49152a61190b271 # v6.0.9
+ with:
+ package_json_file: sdk/typescript/package.json
+ cache: true
+ cache_dependency_path: sdk/typescript/pnpm-lock.yaml
+ - uses: oven-sh/setup-bun@0c5077e51419868618aeaa5fe8019c62421857d6 # v2
+ with:
+ bun-version: "1.3.13"
+ - name: Install dependencies
+ run: pnpm --dir sdk/typescript install --frozen-lockfile
+ - name: Prepare private Windows test root
+ if: runner.os == 'Windows'
+ id: windows-temp
+ shell: pwsh
+ run: ./sdk/typescript/scripts/prepare-windows-test-root.ps1
+ - name: Test runner mode
+ env:
+ TEMP: ${{ steps.windows-temp.outputs.path || runner.temp }}
+ TMP: ${{ steps.windows-temp.outputs.path || runner.temp }}
+ TMPDIR: ${{ steps.windows-temp.outputs.path || runner.temp }}
+ run: |
+ node -e "require('node:fs').mkdirSync('sdk/typescript/reports',{recursive:true})"
+ pnpm --dir sdk/typescript run test ${{ runner.os == 'Windows' && '--timeout=120000' || '' }} ${{ matrix.args }} --seed=${{ env.CODEX_SECURITY_PROPERTY_SEED }} --reporter=junit --reporter-outfile=reports/runner-${{ matrix.os }}-${{ matrix.mode }}.xml
+ - name: Upload runner report
+ if: always()
+ uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
+ with:
+ name: runner-${{ matrix.os }}-${{ matrix.mode }}
+ overwrite: true
+ path: sdk/typescript/reports/runner-*.xml
+ if-no-files-found: error
+ retention-days: 14
+
+ compare:
+ name: Runner inventory and timing
+ if: always() && !cancelled()
+ needs: [runner]
+ runs-on: ubuntu-latest
+ steps:
+ - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
+ with:
+ persist-credentials: false
+ - uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1
+ with:
+ pattern: runner-*
+ merge-multiple: true
+ path: reports
+ - name: Compare inventories and outcomes
+ shell: bash
+ run: |
+ comparison_status=0
+ for os in ubuntu-latest windows-latest; do
+ for mode in isolated parallel; do
+ python3 sdk/typescript/scripts/compare-test-reports.py "reports/runner-$os-baseline.xml" "reports/runner-$os-$mode.xml" >> "$GITHUB_STEP_SUMMARY" || comparison_status=1
+ done
+ done
+ python3 sdk/typescript/scripts/compare-test-reports.py reports/runner-windows-latest-baseline.xml 'reports/runner-windows-latest-shard-*.xml' >> "$GITHUB_STEP_SUMMARY" || comparison_status=1
+ exit "$comparison_status"
+
+ mutation:
+ name: Pure-module mutation trial
+ runs-on: ubuntu-latest
+ timeout-minutes: 20
+ steps:
+ - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
+ with:
+ persist-credentials: false
+ - uses: actions/setup-node@53b83947a5a98c8d113130e565377fae1a50d02f # v6
+ with:
+ node-version: "22.13.0"
+ - uses: pnpm/action-setup@0ebf47130e4866e96fce0953f49152a61190b271 # v6.0.9
+ with:
+ package_json_file: sdk/typescript/package.json
+ cache: true
+ cache_dependency_path: sdk/typescript/pnpm-lock.yaml
+ - uses: oven-sh/setup-bun@0c5077e51419868618aeaa5fe8019c62421857d6 # v2
+ with:
+ bun-version: "1.3.14"
+ - name: Install dependencies
+ run: pnpm --dir sdk/typescript install --frozen-lockfile
+ - name: Run mutation trial
+ run: pnpm --dir sdk/typescript run test:mutation
+ - name: Upload mutation report
+ if: always()
+ continue-on-error: true
+ uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
+ with:
+ name: mutation-report
+ overwrite: true
+ path: sdk/typescript/reports/mutation/
+ if-no-files-found: warn
+ retention-days: 14
diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md
index 303eb270a..e4e6015f7 100644
--- a/CONTRIBUTING.md
+++ b/CONTRIBUTING.md
@@ -47,3 +47,6 @@ Maintainers update package dependencies and the committed lockfile in the
canonical repository. The public release workflow installs that locked graph,
tests the package, and publishes a verified artifact with npm provenance.
GitHub Actions dependencies are maintained separately in this repository.
+
+See the [SDK testing guide](sdk/typescript/TESTING.md) for local checks,
+test conventions, and the required and experimental CI jobs.
diff --git a/sdk/typescript/.gitignore b/sdk/typescript/.gitignore
index d39addef6..350fb263f 100644
--- a/sdk/typescript/.gitignore
+++ b/sdk/typescript/.gitignore
@@ -1,5 +1,6 @@
/dist/
/node_modules
+/coverage/
/reports/
/.stryker-tmp/
/private_release/dist/
diff --git a/sdk/typescript/TESTING.md b/sdk/typescript/TESTING.md
new file mode 100644
index 000000000..3fb473480
--- /dev/null
+++ b/sdk/typescript/TESTING.md
@@ -0,0 +1,102 @@
+# Testing the SDK and CLI
+
+Use the pnpm version in `package.json` and Bun 1.3.14, matching required CI.
+Run these commands from `sdk/typescript`:
+
+```sh
+pnpm install --frozen-lockfile
+bun test --timeout 30000 ./tests-ts/worker-progress.test.ts
+pnpm run types
+pnpm run format
+pnpm run test
+pnpm run test:ci
+pnpm pack --pack-destination ../../dist
+pnpm run test:package
+```
+
+For CI's full archive inspection, pass the exact `.tgz` path printed by
+`pnpm pack` to `pnpm run check:package`.
+
+Tests run in random order by default. To replay a failure, pass the seed from
+Bun's summary to `pnpm run test --seed 12345`.
+
+The local test commands pass a 30-second per-test timeout explicitly. Windows
+CI and the Windows runner experiment allow 120 seconds for slower native
+credential and document checks. `test:ci` writes `reports/junit.xml` and
+`coverage/lcov.info`. Coverage measures loaded
+JavaScript and TypeScript, not the Python helpers or child processes. It is
+diagnostic for now. Use several successful CI runs to establish a baseline
+before proposing a coverage floor.
+
+## Writing tests
+
+- Test observable results, failures, cancellation, and cleanup. Prefer a
+ regression case that fails before a fix over assertions about private calls
+ or exact prose.
+- Keep fixtures synthetic and independent. Use real temporary directories,
+ Git repositories, SQLite databases, and installed packages when those
+ boundaries are the behavior under test. Do not use live model credentials.
+- Use the typed `TestClient` and `createApiTestFixtures` helpers for API tests.
+ Do not add a production abstraction solely to support a mock.
+- Restore spies, timers, and environment changes. Tests that change the process
+ cwd or install persistent ESM module mocks use `runTestInSubprocess`.
+ Per-file Bun isolation does not isolate process-wide state inside one file.
+- Keep shared behavior enabled on Linux, macOS, and Windows. The constrained
+ PowerShell test changes machine-wide policy and runs alone, only on an
+ explicitly enabled GitHub-hosted Windows runner.
+- Add property tests for meaningful invariants, with accepted and rejected
+ inputs. Keep example-based regression tests for readable failure cases.
+- Give parameterized cases distinct names. Use `%p`, `%j`, or `%#` for values
+ that are not strings. The JUnit comparison rejects duplicate identities.
+
+Property tests use a fixed default seed. Fast-check prints the seed, shrink
+path, and counterexample on failure. To replay one property, select its file
+and test name, then set `CODEX_SECURITY_PROPERTY_SEED` and
+`CODEX_SECURITY_PROPERTY_PATH` to the reported values. Set
+`CODEX_SECURITY_PROPERTY_RUNS` to increase the case count. Pure properties
+default to 100 cases; filesystem contract properties default to 20.
+
+## GitHub Actions
+
+`node-ci` retains the required `ubuntu-latest / node-22`,
+`macos-latest / node-22`, and `windows-latest / node-22` checks. Its Ubuntu
+Node 22 job runs static checks and uploads JUnit and LCOV. All supported runtime
+lanes still test and inspect an installed package. Package inspection includes
+a strict NodeNext TypeScript consumer and the actual installed CLI. Failed
+tests block CI; a failed diagnostic upload does not.
+
+The separate `test-quality` workflow runs weekly, can be dispatched manually,
+and runs on pull requests that change its workflow file. It compares Bun's
+default runner, `--isolate`, `--parallel=2`, and seven-way Windows sharding.
+Every mode uses the same seed for property cases and test ordering: pull
+requests replay seed 1; scheduled and manual runs use the workflow run number.
+The workflow compares test identities and outcomes against the unsharded
+default run and records timings, including failed shards, in the job summary.
+It is not a required check or part of the release trigger.
+
+Runner trials use Bun 1.3.13 to avoid the
+[async-module initialization bug in 1.3.14](https://github.com/oven-sh/bun/issues/31410)
+that breaks the Ink UI tests under isolation. Keep the trial pin until a newer
+release passes the full SDK suite in every mode. Required CI and the mutation
+trial remain on Bun 1.3.14.
+
+Keep the current file-balanced Windows runner until the native runner has
+matching inventories and acceptable Windows timings. Before promotion, compare
+native and file-balanced shards using the same commit and Bun version.
+Keep the machine-policy test serial. Do not replace the full required suite
+with `--changed`: Python files, schemas, fixtures, and workflows loaded at
+runtime are not necessarily part of Bun's import graph.
+
+## Mutation testing
+
+```sh
+pnpm run test:mutation
+pnpm exec stryker run --mutate src/worker-progress.ts
+```
+
+The initial Stryker trial covers progress parsing, safe error messages, and
+pure cost arithmetic. It uses the fixed default property-test seed and writes
+HTML and JSON under `reports/mutation`. Review surviving mutants for missing
+behavior assertions or equivalent changes. There is no score gate yet; set
+one only after the trial has a stable, useful baseline. Do not hide surviving
+mutants with assertions about implementation details.
diff --git a/sdk/typescript/bunfig.toml b/sdk/typescript/bunfig.toml
index d1639b19a..0ff58afce 100644
--- a/sdk/typescript/bunfig.toml
+++ b/sdk/typescript/bunfig.toml
@@ -1,2 +1,4 @@
[test]
randomize = true
+coverageSkipTestFiles = true
+coveragePathIgnorePatterns = ["tests-ts/**", "**/node_modules/**", "dist/**"]
diff --git a/sdk/typescript/package.json b/sdk/typescript/package.json
index 91efc2769..4eb87ef6f 100644
--- a/sdk/typescript/package.json
+++ b/sdk/typescript/package.json
@@ -49,6 +49,7 @@
"lint": "tsc --noEmit",
"prepack": "node --run build",
"test": "bun test --timeout 30000 ./tests-ts",
+ "test:ci": "node -e \"require('node:fs').mkdirSync('reports',{recursive:true})\" && pnpm run test --coverage --coverage-reporter=text --coverage-reporter=lcov --reporter=junit --reporter-outfile=reports/junit.xml",
"test:mutation": "stryker run",
"test:package": "node scripts/smoke-package.mjs",
"types": "pnpm run generate:models:check && tsc --noEmit"
diff --git a/sdk/typescript/scripts/compare-test-reports.py b/sdk/typescript/scripts/compare-test-reports.py
new file mode 100644
index 000000000..6f735eed4
--- /dev/null
+++ b/sdk/typescript/scripts/compare-test-reports.py
@@ -0,0 +1,84 @@
+"""Compare Bun JUnit inventories before changing the required CI runner."""
+
+import argparse
+from collections import Counter
+from glob import glob
+from pathlib import Path
+import sys
+import xml.etree.ElementTree as ET
+
+
+def read_report(path: Path) -> tuple[Counter, float, bool]:
+ root = ET.parse(path).getroot()
+ cases = {}
+ for case in root.iter("testcase"):
+ status = "passed"
+ if case.find("skipped") is not None:
+ status = "skipped"
+ if case.find("failure") is not None or case.find("error") is not None:
+ status = "failed"
+ identity = (
+ case.get("file", "").replace("\\", "/").removeprefix("./"),
+ case.get("classname", ""),
+ case.get("name", ""),
+ )
+ if identity in cases:
+ raise ValueError(f"{path}: duplicate test identity: {' > '.join(identity)}")
+ cases[identity] = status
+ if not cases:
+ raise ValueError(f"{path}: no test cases")
+ if int(root.get("tests", str(len(cases)))) != len(cases):
+ raise ValueError(f"{path}: reported test count does not match test cases")
+ failed = "failed" in cases.values() or any(
+ int(node.get(field, "0"))
+ for node in root.iter()
+ if node.tag in ("testsuite", "testsuites")
+ for field in ("failures", "errors")
+ )
+ if failed:
+ print(f"{path}: test run failed", file=sys.stderr)
+ seconds = float(root.get("time", "0"))
+ skipped = sum(status == "skipped" for status in cases.values())
+ print(f"| {path.name} | {len(cases)} | {skipped} | {seconds:.2f} |")
+ return (
+ Counter((*identity, status) for identity, status in cases.items()),
+ seconds,
+ failed,
+ )
+
+
+def main() -> int:
+ parser = argparse.ArgumentParser(description=__doc__)
+ parser.add_argument("baseline", type=Path)
+ parser.add_argument("candidates", nargs="+", help="JUnit files or glob patterns")
+ args = parser.parse_args()
+ print("| Report | Cases | Skipped | Seconds |")
+ print("| --- | ---: | ---: | ---: |")
+ baseline, _, failed = read_report(args.baseline)
+ candidates = Counter()
+ durations = []
+ for pattern in args.candidates:
+ paths = sorted(glob(pattern))
+ if not paths:
+ raise ValueError(f"No reports match {pattern}")
+ for path in paths:
+ cases, seconds, report_failed = read_report(Path(path))
+ failed = failed or report_failed
+ candidates.update(cases)
+ durations.append(seconds)
+ missing, extra = baseline - candidates, candidates - baseline
+ if failed or missing or extra:
+ for label, difference in (("Missing", missing), ("Extra", extra)):
+ for identity, count in sorted(difference.items()):
+ print(f"{label} ({count}): {' > '.join(identity)}", file=sys.stderr)
+ return 1
+ print(f"\nIdentical test inventory and outcomes. Slowest candidate: {max(durations):.2f}s; combined test time: {sum(durations):.2f}s.\n")
+ return 0
+
+
+if __name__ == "__main__":
+ try:
+ sys.exit(main())
+ except (OSError, ValueError, ET.ParseError) as error:
+ print(error, file=sys.stderr)
+ sys.exit(1)
diff --git a/sdk/typescript/tests-ts/cli-deep-scan-summary.test.ts b/sdk/typescript/tests-ts/cli-deep-scan-summary.test.ts
index b4b358c69..382fe33d7 100644
--- a/sdk/typescript/tests-ts/cli-deep-scan-summary.test.ts
+++ b/sdk/typescript/tests-ts/cli-deep-scan-summary.test.ts
@@ -112,7 +112,7 @@ describe("deep scan completion summary", () => {
});
test.each([false, true])(
- "keeps the result when stop details are unavailable (read fails: %s)",
+ "keeps the result when stop details are unavailable (read fails: %p)",
async (fails) => {
const text = await summary({
result: fakeResult(),
diff --git a/sdk/typescript/tests-ts/cli.test.ts b/sdk/typescript/tests-ts/cli.test.ts
index ea5cd9846..fbf7520e1 100644
--- a/sdk/typescript/tests-ts/cli.test.ts
+++ b/sdk/typescript/tests-ts/cli.test.ts
@@ -1849,7 +1849,7 @@ describe("CLI", () => {
});
test.each([false, true])(
- "subscribes to session details only with TTY stdin: %s",
+ "subscribes to session details only with TTY stdin: %p",
async (stdinTTY) => {
const descriptor = Object.getOwnPropertyDescriptor(
process.stdin,
diff --git a/sdk/typescript/tests-ts/cost.test.ts b/sdk/typescript/tests-ts/cost.test.ts
index 50b46d27a..3d0fbe317 100644
--- a/sdk/typescript/tests-ts/cost.test.ts
+++ b/sdk/typescript/tests-ts/cost.test.ts
@@ -175,7 +175,7 @@ describe("scan cost", () => {
[{ cache_write_input_tokens: 0, cache_write_tokens: 15 }, 15],
[{ cache_write_input_tokens: 0, cache_write_tokens: 80 }, 0],
] as const)(
- "keeps workbench cache-write normalization aligned with SDK usage",
+ "keeps workbench cache-write normalization aligned with SDK usage for %j as %p tokens",
async (cacheWrites, expectedCacheWrites) => {
const { PLUGIN_ROOT } = await import("./plugin-root.js");
const python = Bun.which("python3") ?? Bun.which("python");
diff --git a/sdk/typescript/tests-ts/deep-scan-workbench.test.ts b/sdk/typescript/tests-ts/deep-scan-workbench.test.ts
index 262de005a..02c66b8d6 100644
--- a/sdk/typescript/tests-ts/deep-scan-workbench.test.ts
+++ b/sdk/typescript/tests-ts/deep-scan-workbench.test.ts
@@ -392,7 +392,7 @@ describe("deep scan workbench ownership", () => {
[0.5, 0.5],
[96, 96],
] as const)(
- "resolves the configured discovery deadline %s as %s hours",
+ "resolves the configured discovery deadline %p as %p hours",
async (configuredHours, expectedHours) => {
const root = await realpath(
await mkdtemp(join(tmpdir(), "codex-security-deep-deadline-config-")),
@@ -509,7 +509,7 @@ describe("deep scan workbench ownership", () => {
});
test.each([false, true] as const)(
- "backfills and repairs discovery deadline migration when already recorded: %s",
+ "backfills and repairs discovery deadline migration when already recorded: %p",
(migrationRecorded) => {
const python = Bun.which("python3") ?? Bun.which("python");
expect(python).not.toBeNull();
@@ -565,7 +565,7 @@ describe("deep scan workbench ownership", () => {
["scoped_path", false],
["repository", true],
] as const)(
- "returns an honest partial %s report with existing deferred work %s when saturated discovery exceeds its cost limit",
+ "returns an honest partial %s report with existing deferred work %p when saturated discovery exceeds its cost limit",
async (inventoryStrategy, existingDeferred) => {
const root = await realpath(
await mkdtemp(join(tmpdir(), "codex-security-deep-budget-recovery-")),
diff --git a/sdk/typescript/tests-ts/multiscan.test.ts b/sdk/typescript/tests-ts/multiscan.test.ts
index 390988965..8fd8ac9ba 100644
--- a/sdk/typescript/tests-ts/multiscan.test.ts
+++ b/sdk/typescript/tests-ts/multiscan.test.ts
@@ -385,7 +385,7 @@ describe("multiscan", () => {
});
test.each([false, true])(
- "continues scanning when a progress observer fails %s",
+ "continues scanning when a progress observer fails %p",
async (asynchronous) => {
const paths = await fixture();
const source = await repository(paths.root, "observer-failure");
diff --git a/sdk/typescript/tests-ts/release-automation.test.ts b/sdk/typescript/tests-ts/release-automation.test.ts
index 69126b1c9..e69ac613a 100644
--- a/sdk/typescript/tests-ts/release-automation.test.ts
+++ b/sdk/typescript/tests-ts/release-automation.test.ts
@@ -1,7 +1,6 @@
import { spawnSync } from "node:child_process";
import { createHash } from "node:crypto";
import {
- existsSync,
mkdirSync,
mkdtempSync,
readFileSync,
@@ -9,9 +8,10 @@ import {
writeFileSync,
} from "node:fs";
import { tmpdir } from "node:os";
-import { dirname, join } from "node:path";
+import { join } from "node:path";
import { fileURLToPath } from "node:url";
import { describe, expect, test } from "bun:test";
+import { bashCommand } from "./support/shell.js";
type ReleaseMetadata = Record<string, unknown>;
@@ -142,15 +142,7 @@ const releaseRun = "30481596229";
const releaseRepository = "openai/codex-security";
const releaseTagTimeout = process.platform === "win32" ? 20_000 : 10_000;
-function releaseShell(): string {
- if (process.platform !== "win32") return "bash";
- const git = Bun.which("git");
- if (git === null) return "bash";
- const gitBash = join(dirname(dirname(git)), "bin", "bash.exe");
- return existsSync(gitBash) ? gitBash : "bash";
-}
-
-const bash = releaseShell();
+const bash = bashCommand();
const jqMock = [
"jq() {",
' node -e \'const fs=require("node:fs");const filter=process.argv.at(-1);const value=JSON.parse(fs.readFileSync(0,"utf8"));if(filter==="[.object.type, .object.sha] | @tsv"){const fields=[value.object?.type,value.object?.sha];if(fields.some((field)=>typeof field!=="string"))process.exit(1);process.stdout.write(fields.join("\\t")+"\\n");}else if(filter===".status // empty"){if(value.status!=null)process.stdout.write(String(value.status)+"\\n");}else process.exit(64);\' -- "$@"',
diff --git a/sdk/typescript/tests-ts/sarif.test.ts b/sdk/typescript/tests-ts/sarif.test.ts
index 56c0d94a6..b98676f40 100644
--- a/sdk/typescript/tests-ts/sarif.test.ts
+++ b/sdk/typescript/tests-ts/sarif.test.ts
@@ -142,7 +142,7 @@ describe("SARIF presentation", () => {
["high", 0, undefined],
["high", 6.25, "6.25"],
["critical", 10, "10"],
- ] as const)("maps %s severity with score %s", (level, score, expected) => {
+ ] as const)("maps %s severity with score %p", (level, score, expected) => {
const severity =
score === undefined
? { level }
diff --git a/sdk/typescript/tests-ts/skeleton.test.ts b/sdk/typescript/tests-ts/skeleton.test.ts
index d6020a41f..1ccf9b86f 100644
--- a/sdk/typescript/tests-ts/skeleton.test.ts
+++ b/sdk/typescript/tests-ts/skeleton.test.ts
@@ -10,6 +10,37 @@ import {
} from "../src/index.js";
import { main } from "../src/cli.js";
+interface WorkflowStep {
+ name?: string;
+ uses?: string;
+ with?: Record<string, unknown>;
+ run?: string;
+ if?: string;
+ env?: Record<string, unknown>;
+ "continue-on-error"?: boolean;
+}
+
+interface WorkflowJob {
+ name?: string;
+ needs?: string[];
+ env?: Record<string, unknown>;
+ strategy?: { matrix: Record<string, unknown> };
+ steps: WorkflowStep[];
+}
+
+async function workflow(name: string) {
+ return Bun.YAML.parse(
+ await readFile(
+ new URL(`../../../.github/workflows/${name}`, import.meta.url),
+ "utf8",
+ ),
+ ) as {
+ on: Record<string, unknown>;
+ env?: Record<string, unknown>;
+ jobs: Record<string, WorkflowJob>;
+ };
+}
+
function capture(): {
stream: Pick<NodeJS.WriteStream, "write">;
text: () => string;
@@ -71,28 +102,33 @@ describe("TypeScript package skeleton", () => {
});
test("pins each Node.js minimum and preserves protected and latest LTS checks", async () => {
- const ciWorkflow = await readFile(
- new URL("../../../.github/workflows/node-ci.yml", import.meta.url),
- "utf8",
+ const { jobs } = await workflow("node-ci.yml");
+ expect(jobs["test"]?.name).toBe(
+ "${{ matrix.os }} / node-${{ matrix.node == '22.13.0' && '22' || matrix.node }}",
);
-
- expect(ciWorkflow).toContain(
- "${{ matrix.node == '22.13.0' && '22' || matrix.node }}",
+ expect(jobs["test"]?.strategy?.matrix).toEqual({
+ os: ["ubuntu-latest", "macos-latest"],
+ node: ["22.13.0"],
+ include: ["24.0.0", "24", "26.0.0", "26"].map((node) => ({
+ os: "ubuntu-latest",
+ node,
+ })),
+ });
+ expect(jobs["windows"]?.name).toBe(
+ "windows-latest / node-${{ matrix.node == '22.13.0' && '22' || matrix.node }}",
);
- expect(ciWorkflow).toContain('node: ["22.13.0"]');
- for (const version of ["24.0.0", "24", "26.0.0", "26"]) {
- expect(ciWorkflow).toContain(`node: "${version}"`);
- }
+ expect(jobs["windows"]?.needs).toEqual(["windows-test", "windows-verify"]);
+ expect(jobs["windows-test"]?.strategy?.matrix["node"]).toEqual([
+ "22.13.0",
+ "24",
+ ]);
});
test("randomizes tests and keeps the default and Windows CI timeouts", async () => {
const packageJson = JSON.parse(
await readFile(new URL("../package.json", import.meta.url), "utf8"),
);
- const ciWorkflow = await readFile(
- new URL("../../../.github/workflows/node-ci.yml", import.meta.url),
- "utf8",
- );
+ const { jobs } = await workflow("node-ci.yml");
const bunConfig = parse(
await readFile(new URL("../bunfig.toml", import.meta.url), "utf8"),
);
@@ -101,17 +137,131 @@ describe("TypeScript package skeleton", () => {
"bun test --timeout 30000 ./tests-ts",
);
expect(bunConfig).toMatchObject({ test: { randomize: true } });
- expect(ciWorkflow).toContain(
- "run: node sdk/typescript/scripts/run-windows-ci-tests.mjs ${{ matrix.shard }}",
+ expect(packageJson.scripts["test:ci"]).toContain("pnpm run test ");
+ expect(jobs["windows-test"]?.steps).toContainEqual(
+ expect.objectContaining({
+ run: "node sdk/typescript/scripts/run-windows-ci-tests.mjs ${{ matrix.shard }}",
+ }),
+ );
+ });
+
+ test("runs shared static checks once and keeps report upload non-blocking", async () => {
+ const { jobs } = await workflow("node-ci.yml");
+ const steps = Object.values(jobs).flatMap((job) => job.steps);
+ for (const name of ["Typecheck", "Check formatting"]) {
+ expect(steps.filter((step) => step.name === name)).toEqual([
+ expect.objectContaining({
+ if: "matrix.os == 'ubuntu-latest' && matrix.node == '22.13.0'",
+ }),
+ ]);
+ }
+ expect(steps.find((step) => step.name === "Test")).not.toHaveProperty(
+ "continue-on-error",
);
- expect(ciWorkflow).toContain(
- "run: bun test --timeout 120000 ./tests-ts/windows-machine-policy.test.ts",
+ expect(
+ steps.find((step) => step.name === "Upload test reports"),
+ ).toMatchObject({
+ "continue-on-error": true,
+ });
+ for (const name of ["test", "windows-verify"]) {
+ expect(jobs[name]?.steps).toContainEqual(
+ expect.objectContaining({
+ run: "pnpm run check:package ../../dist/*.tgz",
+ }),
+ );
+ }
+ });
+
+ test("keeps machine-wide policy changes out of parallel and experimental runs", async () => {
+ const ci = await workflow("node-ci.yml");
+ const windows = ci.jobs["windows-test"]!.steps;
+ expect(
+ windows.find((step) => step.name === "Test shard ${{ matrix.shard }}")
+ ?.env?.["CODEX_SECURITY_ALLOW_MACHINE_POLICY_TEST"],
+ ).toBe("false");
+ expect(
+ windows.find(
+ (step) => step.name === "Test machine-wide PowerShell policy",
+ ),
+ ).toMatchObject({
+ if: "matrix.shard == 3 && runner.environment == 'github-hosted'",
+ env: { CODEX_SECURITY_ALLOW_MACHINE_POLICY_TEST: "true" },
+ run: "bun test --timeout 120000 ./tests-ts/windows-machine-policy.test.ts",
+ });
+ const quality = await workflow("test-quality.yml");
+ expect(Object.keys(quality.on).sort()).toEqual([
+ "pull_request",
+ "schedule",
+ "workflow_dispatch",
+ ]);
+ expect(quality.on["pull_request"]).toEqual({
+ paths: [".github/workflows/test-quality.yml"],
+ });
+ expect(quality.env?.["CODEX_SECURITY_ALLOW_MACHINE_POLICY_TEST"]).toBe(
+ "false",
);
- expect(ciWorkflow).toContain(
- "name: windows-latest / node-${{ matrix.node == '22.13.0' && '22' || matrix.node }}",
+ expect(quality.env?.["CODEX_SECURITY_INTEGRATION"]).toBe("0");
+ for (let shard = 1; shard <= 7; shard += 1) {
+ expect(
+ quality.jobs["runner"]?.strategy?.matrix["include"],
+ ).toContainEqual({
+ os: "windows-latest",
+ mode: `shard-${shard}`,
+ args: `--shard=${shard}/7`,
+ });
+ }
+ });
+
+ test("keeps runner modes reproducible and report uploads rerunnable", async () => {
+ const ci = await workflow("node-ci.yml");
+ const quality = await workflow("test-quality.yml");
+ const runner = quality.jobs["runner"]!;
+ const seed =
+ "${{ github.event_name == 'pull_request' && 1 || github.run_number }}";
+ expect(quality.env).not.toHaveProperty("CODEX_SECURITY_PROPERTY_SEED");
+ expect(runner.env?.["CODEX_SECURITY_PROPERTY_SEED"]).toBe(seed);
+ expect(runner.strategy?.matrix["mode"]).toEqual([
+ "baseline",
+ "isolated",
+ "parallel",
+ ]);
+ for (const [mode, args] of [
+ ["baseline", ""],
+ ["isolated", "--isolate"],
+ ["parallel", "--parallel=2"],
+ ] as const) {
+ expect(runner.strategy?.matrix["include"]).toContainEqual({
+ mode,
+ args,
+ });
+ }
+ const command = runner.steps.find(
+ (step) => step.name === "Test runner mode",
+ )?.run;
+ expect(command).toContain(
+ "${{ runner.os == 'Windows' && '--timeout=120000' || '' }}",
);
- expect(ciWorkflow).toContain("run: pnpm --dir sdk/typescript run test");
- expect(ciWorkflow).not.toContain("--timeout 60000");
+ expect(command).toContain("${{ matrix.args }}");
+ expect(command).toContain("--seed=${{ env.CODEX_SECURITY_PROPERTY_SEED }}");
+
+ const uploads = [...Object.values(ci.jobs), ...Object.values(quality.jobs)]
+ .flatMap((job) => job.steps)
+ .filter((step) => step.uses?.startsWith("actions/upload-artifact@"));
+ expect(uploads).toHaveLength(3);
+ for (const upload of uploads) {
+ expect(upload.with?.["overwrite"]).toBe(true);
+ }
+ expect(
+ uploads.find((step) => step.name === "Upload mutation report"),
+ ).toMatchObject({ "continue-on-error": true });
+ expect(
+ uploads.find((step) => step.name === "Upload runner report"),
+ ).not.toHaveProperty("continue-on-error");
+ expect(
+ quality.jobs["mutation"]?.steps.find(
+ (step) => step.name === "Run mutation trial",
+ ),
+ ).not.toHaveProperty("continue-on-error");
});
test("builds packages without a preinstalled package manager and provides a production audit", async () => {
@@ -130,14 +280,17 @@ describe("TypeScript package skeleton", () => {
test("keeps production dependency audits non-blocking in CI and releases", async () => {
for (const workflowName of ["node-ci.yml", "node-release.yml"]) {
- const workflow = await readFile(
- new URL(`../../../.github/workflows/${workflowName}`, import.meta.url),
- "utf8",
- );
-
- expect(workflow).toMatch(
- /- name: Audit production dependencies\n(?:\s+if: [^\n]+\n)?\s+continue-on-error: true\n\s+run: (?:sfw )?pnpm --dir sdk\/typescript run audit:prod/u,
- );
+ const { jobs } = await workflow(workflowName);
+ const audits = Object.values(jobs)
+ .flatMap((job) => job.steps)
+ .filter((step) => step.name === "Audit production dependencies");
+ expect(audits.length).toBeGreaterThan(0);
+ for (const audit of audits) {
+ expect(audit["continue-on-error"]).toBe(true);
+ expect(audit.run).toMatch(
+ /^(?:sfw )?pnpm --dir sdk\/typescript run audit:prod$/u,
+ );
+ }
}
});
diff --git a/sdk/typescript/tests-ts/support/shell.ts b/sdk/typescript/tests-ts/support/shell.ts
new file mode 100644
index 000000000..31427daa1
--- /dev/null
+++ b/sdk/typescript/tests-ts/support/shell.ts
@@ -0,0 +1,10 @@
+import { existsSync } from "node:fs";
+import { dirname, join } from "node:path";
+
+export function bashCommand(): string {
+ if (process.platform !== "win32") return "bash";
+ const git = Bun.which("git");
+ if (git === null) return "bash";
+ const gitBash = join(dirname(dirname(git)), "bin", "bash.exe");
+ return existsSync(gitBash) ? gitBash : "bash";
+}
diff --git a/sdk/typescript/tests-ts/test-reports.test.ts b/sdk/typescript/tests-ts/test-reports.test.ts
new file mode 100644
index 000000000..27eda4015
--- /dev/null
+++ b/sdk/typescript/tests-ts/test-reports.test.ts
@@ -0,0 +1,196 @@
+import { spawnSync } from "node:child_process";
+import { mkdtemp, readFile, rm, writeFile } from "node:fs/promises";
+import { tmpdir } from "node:os";
+import { join } from "node:path";
+import { fileURLToPath } from "node:url";
+import { afterEach, describe, expect, test } from "bun:test";
+import { bashCommand } from "./support/shell.js";
+
+const bash = bashCommand();
+const directories: string[] = [];
+afterEach(async () => {
+ await Promise.all(
+ directories
+ .splice(0)
+ .map((path) => rm(path, { recursive: true, force: true })),
+ );
+});
+
+async function fixtures() {
+ const root = await mkdtemp(join(tmpdir(), "codex-security-test-reports-"));
+ directories.push(root);
+ return {
+ root,
+ async report(
+ name: string,
+ cases: string[],
+ failures = 0,
+ count = cases.length,
+ ) {
+ const path = join(root, name);
+ await writeFile(
+ path,
+ `<testsuites tests="${count}" failures="${failures}" time="1.25"><testsuite>${cases.join("")}</testsuite></testsuites>`,
+ );
+ return path;
+ },
+ };
+}
+
+function testcase(name: string, status = "") {
+ return `<testcase file="tests-ts/example.test.ts" classname="example" name="${name}">${status}</testcase>`;
+}
+
+async function compare(baseline: string, ...candidates: string[]) {
+ const python = Bun.which("python3") ?? Bun.which("python") ?? Bun.which("py");
+ if (python === null) throw new Error("A Python interpreter is required.");
+ const child = Bun.spawn({
+ cmd: [
+ python,
+ "-I",
+ "-B",
+ fileURLToPath(
+ new URL("../scripts/compare-test-reports.py", import.meta.url),
+ ),
+ baseline,
+ ...candidates,
+ ],
+ stdin: "ignore",
+ stdout: "pipe",
+ stderr: "pipe",
+ });
+ const [status, stdout, stderr] = await Promise.all([
+ child.exited,
+ new Response(child.stdout).text(),
+ new Response(child.stderr).text(),
+ ]);
+ return { status, stdout, stderr };
+}
+
+describe("JUnit inventory comparison", () => {
+ test("runs every workflow comparison before reporting a mismatch", async () => {
+ const fixture = await fixtures();
+ const workflow = Bun.YAML.parse(
+ await readFile(
+ new URL("../../../.github/workflows/test-quality.yml", import.meta.url),
+ "utf8",
+ ),
+ ) as {
+ jobs: { compare: { steps: Array<{ name?: string; run?: string }> } };
+ };
+ const script = workflow.jobs.compare.steps.find(
+ (step) => step.name === "Compare inventories and outcomes",
+ )!.run!;
+ const expected = [
+ ...["ubuntu-latest", "windows-latest"].flatMap((os) =>
+ ["isolated", "parallel"].map(
+ (mode) => `reports/runner-${os}-${mode}.xml`,
+ ),
+ ),
+ "reports/runner-windows-latest-shard-*.xml",
+ ];
+ const mock = `python3() {
+ printf '%s\\n' "$3"
+ [[ "$3" != "$CODEX_SECURITY_TEST_FAIL_REPORT" ]]
+}`;
+ const summary = join(fixture.root, "summary.md");
+ for (const failedReport of ["", expected[0]!]) {
+ await writeFile(summary, "");
+ const result = spawnSync(
+ bash,
+ ["-e", "-o", "pipefail", "-c", `${mock}\n${script}`],
+ {
+ cwd: fixture.root,
+ encoding: "utf8",
+ env: {
+ ...process.env,
+ GITHUB_STEP_SUMMARY: "summary.md",
+ CODEX_SECURITY_TEST_FAIL_REPORT: failedReport,
+ },
+ timeout: 10_000,
+ },
+ );
+ expect(result.status, result.stderr).toBe(failedReport === "" ? 0 : 1);
+ expect((await readFile(summary, "utf8")).trim().split(/\r?\n/u)).toEqual(
+ expected,
+ );
+ }
+ });
+
+ test("merges native shards without depending on test order", async () => {
+ const fixture = await fixtures();
+ const passed = testcase("accepts &amp; preserves");
+ const skipped = testcase("platform-only", "<skipped/>");
+ const baseline = await fixture.report("baseline.xml", [passed, skipped]);
+ await fixture.report("shard-1.xml", [skipped]);
+ await fixture.report("shard-2.xml", [passed]);
+ const result = await compare(baseline, join(fixture.root, "shard-*.xml"));
+ expect(result.status, result.stderr).toBe(0);
+ expect(result.stdout).toContain("combined test time: 2.50s");
+ });
+
+ test("rejects ambiguous test identities even when totals match", async () => {
+ const fixture = await fixtures();
+ const first = testcase("same parameterized name");
+ for (const [name, repeated] of [
+ ["same-outcome", first],
+ ["different-outcome", testcase("same parameterized name", "<skipped/>")],
+ ] as const) {
+ const baseline = await fixture.report(`${name}-baseline.xml`, [
+ first,
+ repeated,
+ ]);
+ const candidate = await fixture.report(`${name}-candidate.xml`, [
+ first,
+ repeated,
+ ]);
+ const result = await compare(baseline, candidate);
+ expect(result.status).toBe(1);
+ expect(result.stderr).toContain("duplicate test identity");
+ }
+ });
+
+ test("reports every shard's timing even when an earlier shard fails", async () => {
+ const fixture = await fixtures();
+ const first = testcase("first");
+ const second = testcase("second");
+ const baseline = await fixture.report("baseline.xml", [first, second]);
+ const failed = await fixture.report("shard-1.xml", [
+ testcase("first", "<failure/>"),
+ ]);
+ const passed = await fixture.report("shard-2.xml", [second]);
+ const result = await compare(baseline, failed, passed);
+
+ expect(result.status).toBe(1);
+ expect(result.stderr).toContain("test run failed");
+ expect(result.stdout).toContain("shard-1.xml");
+ expect(result.stdout).toContain("shard-2.xml");
+ });
+
+ test("rejects dropped, duplicated, skipped, failed, or incomplete results", async () => {
+ const fixture = await fixtures();
+ const first = testcase("first");
+ const second = testcase("second");
+ const baseline = await fixture.report("baseline.xml", [first, second]);
+ for (const [name, cases, failures, count] of [
+ ["missing", [first], 0, 1],
+ ["duplicate", [first, second, second], 0, 3],
+ ["skipped", [first, testcase("second", "<skipped/>")], 0, 2],
+ ["failed", [first, testcase("second", "<failure/>")], 1, 2],
+ ["summary-failed", [first, second], 1, 2],
+ ["incomplete", [first], 0, 2],
+ ["empty", [], 0, 0],
+ ] as const) {
+ const candidate = await fixture.report(
+ `${name}.xml`,
+ [...cases],
+ failures,
+ count,
+ );
+ expect((await compare(baseline, candidate)).status, name).toBe(1);
+ }
+ expect(
+ (await compare(baseline, join(fixture.root, "absent-*.xml"))).status,
+ ).toBe(1);
+ });
+});