OpenAI's Codex Security CLI and TypeScript SDK for finding, validating, and fixing security vulnerabilities. npm: https://www.npmjs.com/package/@openai/codex-security
TypeScript ai-securityapplication-securityclicode-scanningcodexcodex-securitycybersecuritydevsecopsnodejsnpmopenaisdksecuritytypescriptvulnerability-scanning
ci: report test quality and exercise native Bun runners (#508)
mldangelo-oai9e061f251cc6
diff --git a/.github/workflows/node-ci.yml b/.github/workflows/node-ci.ymlindex c6398851d..b8704c94c 100644--- a/.github/workflows/node-ci.yml+++ b/.github/workflows/node-ci.yml@@ -66,6 +66,7 @@ jobs:run: pnpm --dir sdk/typescript run audit:prod- name: Typecheck+ if: matrix.os == 'ubuntu-latest' && matrix.node == '22.13.0'run: pnpm --dir sdk/typescript run types- name: Test@@ -75,9 +76,23 @@ jobs:TMP: ${{ runner.temp }}TMPDIR: ${{ runner.temp }}CODEX_SECURITY_ALLOW_MACHINE_POLICY_TEST: "false"- run: pnpm --dir sdk/typescript run test+ run: pnpm --dir sdk/typescript run ${{ matrix.os == 'ubuntu-latest' && matrix.node == '22.13.0' && 'test:ci' || 'test' }}++ - name: Upload test reports+ if: always() && matrix.os == 'ubuntu-latest' && matrix.node == '22.13.0'+ continue-on-error: true+ uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1+ with:+ name: node-22-test-reports+ overwrite: true+ path: |+ sdk/typescript/reports/junit.xml+ sdk/typescript/coverage/lcov.info+ if-no-files-found: warn+ retention-days: 14- name: Check formatting+ if: matrix.os == 'ubuntu-latest' && matrix.node == '22.13.0'run: pnpm --dir sdk/typescript run format- name: Pack@@ -89,21 +104,6 @@ jobs:shell: bashrun: pnpm run check:package ../../dist/*.tgz- - name: Smoke-test Node.js runtime- working-directory: sdk/typescript- shell: bash- run: |- set -euo pipefail- node --input-type=module --eval '- import { CodexSecurity } from "@openai/codex-security";-- if (typeof CodexSecurity !== "function") {- throw new Error("The SDK does not export CodexSecurity.");- }- '- node bin/codex-security.mjs --version- node bin/codex-security.mjs --help-windows-test:name: windows-latest / node-${{ matrix.node == '22.13.0' && '22' || matrix.node }} / tests-${{ matrix.shard }}runs-on: windows-latest@@ -162,16 +162,6 @@ jobs:CODEX_SECURITY_ALLOW_MACHINE_POLICY_TEST: "true"run: bun test --timeout 120000 ./tests-ts/windows-machine-policy.test.ts- - name: Typecheck- if: matrix.shard == 7- working-directory: sdk/typescript- run: pnpm run types-- - name: Check formatting- if: matrix.shard == 7- working-directory: sdk/typescript- run: pnpm run format-windows-verify:name: windows-latest / node-${{ matrix.node == '22.13.0' && '22' || matrix.node }} / verifyruns-on: windows-latest@@ -210,21 +200,6 @@ jobs:shell: bashrun: pnpm run check:package ../../dist/*.tgz- - name: Smoke-test Node.js runtime- working-directory: sdk/typescript- shell: bash- run: |- set -euo pipefail- node --input-type=module --eval '- import { CodexSecurity } from "@openai/codex-security";-- if (typeof CodexSecurity !== "function") {- throw new Error("The SDK does not export CodexSecurity.");- }- '- node bin/codex-security.mjs --version- node bin/codex-security.mjs --help-windows:name: windows-latest / node-${{ matrix.node == '22.13.0' && '22' || matrix.node }}runs-on: ubuntu-latestdiff --git a/.github/workflows/test-quality.yml b/.github/workflows/test-quality.ymlnew file mode 100644index 000000000..ff70a17ed--- /dev/null+++ b/.github/workflows/test-quality.yml@@ -0,0 +1,161 @@+name: test-quality++on:+ pull_request:+ paths:+ - .github/workflows/test-quality.yml+ workflow_dispatch:+ schedule:+ - cron: "23 9 * * 1"++permissions:+ contents: read++concurrency:+ group: ${{ github.workflow }}-${{ github.ref }}+ cancel-in-progress: true++env:+ CODEX_SECURITY_INTEGRATION: "0"+ CODEX_SECURITY_ALLOW_MACHINE_POLICY_TEST: "false"++jobs:+ runner:+ name: ${{ matrix.os }} / ${{ matrix.mode }}+ runs-on: ${{ matrix.os }}+ timeout-minutes: 20+ env:+ CODEX_SECURITY_PROPERTY_SEED: ${{ github.event_name == 'pull_request' && 1 || github.run_number }}+ strategy:+ fail-fast: false+ matrix:+ os: [ubuntu-latest, windows-latest]+ mode: [baseline, isolated, parallel]+ include:+ - mode: baseline+ args: ""+ - mode: isolated+ args: --isolate+ - mode: parallel+ args: --parallel=2+ - os: windows-latest+ mode: shard-1+ args: --shard=1/7+ - os: windows-latest+ mode: shard-2+ args: --shard=2/7+ - os: windows-latest+ mode: shard-3+ args: --shard=3/7+ - os: windows-latest+ mode: shard-4+ args: --shard=4/7+ - os: windows-latest+ mode: shard-5+ args: --shard=5/7+ - os: windows-latest+ mode: shard-6+ args: --shard=6/7+ - os: windows-latest+ mode: shard-7+ args: --shard=7/7+ steps:+ - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2+ with:+ persist-credentials: false+ - uses: actions/setup-node@53b83947a5a98c8d113130e565377fae1a50d02f # v6+ with:+ node-version: "22.13.0"+ - name: Set up pnpm+ uses: pnpm/action-setup@0ebf47130e4866e96fce0953f49152a61190b271 # v6.0.9+ with:+ package_json_file: sdk/typescript/package.json+ cache: true+ cache_dependency_path: sdk/typescript/pnpm-lock.yaml+ - uses: oven-sh/setup-bun@0c5077e51419868618aeaa5fe8019c62421857d6 # v2+ with:+ bun-version: "1.3.13"+ - name: Install dependencies+ run: pnpm --dir sdk/typescript install --frozen-lockfile+ - name: Prepare private Windows test root+ if: runner.os == 'Windows'+ id: windows-temp+ shell: pwsh+ run: ./sdk/typescript/scripts/prepare-windows-test-root.ps1+ - name: Test runner mode+ env:+ TEMP: ${{ steps.windows-temp.outputs.path || runner.temp }}+ TMP: ${{ steps.windows-temp.outputs.path || runner.temp }}+ TMPDIR: ${{ steps.windows-temp.outputs.path || runner.temp }}+ run: |+ node -e "require('node:fs').mkdirSync('sdk/typescript/reports',{recursive:true})"+ pnpm --dir sdk/typescript run test ${{ runner.os == 'Windows' && '--timeout=120000' || '' }} ${{ matrix.args }} --seed=${{ env.CODEX_SECURITY_PROPERTY_SEED }} --reporter=junit --reporter-outfile=reports/runner-${{ matrix.os }}-${{ matrix.mode }}.xml+ - name: Upload runner report+ if: always()+ uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1+ with:+ name: runner-${{ matrix.os }}-${{ matrix.mode }}+ overwrite: true+ path: sdk/typescript/reports/runner-*.xml+ if-no-files-found: error+ retention-days: 14++ compare:+ name: Runner inventory and timing+ if: always() && !cancelled()+ needs: [runner]+ runs-on: ubuntu-latest+ steps:+ - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2+ with:+ persist-credentials: false+ - uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1+ with:+ pattern: runner-*+ merge-multiple: true+ path: reports+ - name: Compare inventories and outcomes+ shell: bash+ run: |+ comparison_status=0+ for os in ubuntu-latest windows-latest; do+ for mode in isolated parallel; do+ python3 sdk/typescript/scripts/compare-test-reports.py "reports/runner-$os-baseline.xml" "reports/runner-$os-$mode.xml" >> "$GITHUB_STEP_SUMMARY" || comparison_status=1+ done+ done+ python3 sdk/typescript/scripts/compare-test-reports.py reports/runner-windows-latest-baseline.xml 'reports/runner-windows-latest-shard-*.xml' >> "$GITHUB_STEP_SUMMARY" || comparison_status=1+ exit "$comparison_status"++ mutation:+ name: Pure-module mutation trial+ runs-on: ubuntu-latest+ timeout-minutes: 20+ steps:+ - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2+ with:+ persist-credentials: false+ - uses: actions/setup-node@53b83947a5a98c8d113130e565377fae1a50d02f # v6+ with:+ node-version: "22.13.0"+ - uses: pnpm/action-setup@0ebf47130e4866e96fce0953f49152a61190b271 # v6.0.9+ with:+ package_json_file: sdk/typescript/package.json+ cache: true+ cache_dependency_path: sdk/typescript/pnpm-lock.yaml+ - uses: oven-sh/setup-bun@0c5077e51419868618aeaa5fe8019c62421857d6 # v2+ with:+ bun-version: "1.3.14"+ - name: Install dependencies+ run: pnpm --dir sdk/typescript install --frozen-lockfile+ - name: Run mutation trial+ run: pnpm --dir sdk/typescript run test:mutation+ - name: Upload mutation report+ if: always()+ continue-on-error: true+ uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1+ with:+ name: mutation-report+ overwrite: true+ path: sdk/typescript/reports/mutation/+ if-no-files-found: warn+ retention-days: 14diff --git a/CONTRIBUTING.md b/CONTRIBUTING.mdindex 303eb270a..e4e6015f7 100644--- a/CONTRIBUTING.md+++ b/CONTRIBUTING.md@@ -47,3 +47,6 @@ Maintainers update package dependencies and the committed lockfile in thecanonical repository. The public release workflow installs that locked graph,tests the package, and publishes a verified artifact with npm provenance.GitHub Actions dependencies are maintained separately in this repository.++See the [SDK testing guide](sdk/typescript/TESTING.md) for local checks,+test conventions, and the required and experimental CI jobs.diff --git a/sdk/typescript/.gitignore b/sdk/typescript/.gitignoreindex d39addef6..350fb263f 100644--- a/sdk/typescript/.gitignore+++ b/sdk/typescript/.gitignore@@ -1,5 +1,6 @@/dist//node_modules+/coverage//reports//.stryker-tmp//private_release/dist/diff --git a/sdk/typescript/TESTING.md b/sdk/typescript/TESTING.mdnew file mode 100644index 000000000..3fb473480--- /dev/null+++ b/sdk/typescript/TESTING.md@@ -0,0 +1,102 @@+# Testing the SDK and CLI++Use the pnpm version in `package.json` and Bun 1.3.14, matching required CI.+Run these commands from `sdk/typescript`:++```sh+pnpm install --frozen-lockfile+bun test --timeout 30000 ./tests-ts/worker-progress.test.ts+pnpm run types+pnpm run format+pnpm run test+pnpm run test:ci+pnpm pack --pack-destination ../../dist+pnpm run test:package+```++For CI's full archive inspection, pass the exact `.tgz` path printed by+`pnpm pack` to `pnpm run check:package`.++Tests run in random order by default. To replay a failure, pass the seed from+Bun's summary to `pnpm run test --seed 12345`.++The local test commands pass a 30-second per-test timeout explicitly. Windows+CI and the Windows runner experiment allow 120 seconds for slower native+credential and document checks. `test:ci` writes `reports/junit.xml` and+`coverage/lcov.info`. Coverage measures loaded+JavaScript and TypeScript, not the Python helpers or child processes. It is+diagnostic for now. Use several successful CI runs to establish a baseline+before proposing a coverage floor.++## Writing tests++- Test observable results, failures, cancellation, and cleanup. Prefer a+ regression case that fails before a fix over assertions about private calls+ or exact prose.+- Keep fixtures synthetic and independent. Use real temporary directories,+ Git repositories, SQLite databases, and installed packages when those+ boundaries are the behavior under test. Do not use live model credentials.+- Use the typed `TestClient` and `createApiTestFixtures` helpers for API tests.+ Do not add a production abstraction solely to support a mock.+- Restore spies, timers, and environment changes. Tests that change the process+ cwd or install persistent ESM module mocks use `runTestInSubprocess`.+ Per-file Bun isolation does not isolate process-wide state inside one file.+- Keep shared behavior enabled on Linux, macOS, and Windows. The constrained+ PowerShell test changes machine-wide policy and runs alone, only on an+ explicitly enabled GitHub-hosted Windows runner.+- Add property tests for meaningful invariants, with accepted and rejected+ inputs. Keep example-based regression tests for readable failure cases.+- Give parameterized cases distinct names. Use `%p`, `%j`, or `%#` for values+ that are not strings. The JUnit comparison rejects duplicate identities.++Property tests use a fixed default seed. Fast-check prints the seed, shrink+path, and counterexample on failure. To replay one property, select its file+and test name, then set `CODEX_SECURITY_PROPERTY_SEED` and+`CODEX_SECURITY_PROPERTY_PATH` to the reported values. Set+`CODEX_SECURITY_PROPERTY_RUNS` to increase the case count. Pure properties+default to 100 cases; filesystem contract properties default to 20.++## GitHub Actions++`node-ci` retains the required `ubuntu-latest / node-22`,+`macos-latest / node-22`, and `windows-latest / node-22` checks. Its Ubuntu+Node 22 job runs static checks and uploads JUnit and LCOV. All supported runtime+lanes still test and inspect an installed package. Package inspection includes+a strict NodeNext TypeScript consumer and the actual installed CLI. Failed+tests block CI; a failed diagnostic upload does not.++The separate `test-quality` workflow runs weekly, can be dispatched manually,+and runs on pull requests that change its workflow file. It compares Bun's+default runner, `--isolate`, `--parallel=2`, and seven-way Windows sharding.+Every mode uses the same seed for property cases and test ordering: pull+requests replay seed 1; scheduled and manual runs use the workflow run number.+The workflow compares test identities and outcomes against the unsharded+default run and records timings, including failed shards, in the job summary.+It is not a required check or part of the release trigger.++Runner trials use Bun 1.3.13 to avoid the+[async-module initialization bug in 1.3.14](https://github.com/oven-sh/bun/issues/31410)+that breaks the Ink UI tests under isolation. Keep the trial pin until a newer+release passes the full SDK suite in every mode. Required CI and the mutation+trial remain on Bun 1.3.14.++Keep the current file-balanced Windows runner until the native runner has+matching inventories and acceptable Windows timings. Before promotion, compare+native and file-balanced shards using the same commit and Bun version.+Keep the machine-policy test serial. Do not replace the full required suite+with `--changed`: Python files, schemas, fixtures, and workflows loaded at+runtime are not necessarily part of Bun's import graph.++## Mutation testing++```sh+pnpm run test:mutation+pnpm exec stryker run --mutate src/worker-progress.ts+```++The initial Stryker trial covers progress parsing, safe error messages, and+pure cost arithmetic. It uses the fixed default property-test seed and writes+HTML and JSON under `reports/mutation`. Review surviving mutants for missing+behavior assertions or equivalent changes. There is no score gate yet; set+one only after the trial has a stable, useful baseline. Do not hide surviving+mutants with assertions about implementation details.diff --git a/sdk/typescript/bunfig.toml b/sdk/typescript/bunfig.tomlindex d1639b19a..0ff58afce 100644--- a/sdk/typescript/bunfig.toml+++ b/sdk/typescript/bunfig.toml@@ -1,2 +1,4 @@[test]randomize = true+coverageSkipTestFiles = true+coveragePathIgnorePatterns = ["tests-ts/**", "**/node_modules/**", "dist/**"]diff --git a/sdk/typescript/package.json b/sdk/typescript/package.jsonindex 91efc2769..4eb87ef6f 100644--- a/sdk/typescript/package.json+++ b/sdk/typescript/package.json@@ -49,6 +49,7 @@"lint": "tsc --noEmit","prepack": "node --run build","test": "bun test --timeout 30000 ./tests-ts",+ "test:ci": "node -e \"require('node:fs').mkdirSync('reports',{recursive:true})\" && pnpm run test --coverage --coverage-reporter=text --coverage-reporter=lcov --reporter=junit --reporter-outfile=reports/junit.xml","test:mutation": "stryker run","test:package": "node scripts/smoke-package.mjs","types": "pnpm run generate:models:check && tsc --noEmit"diff --git a/sdk/typescript/scripts/compare-test-reports.py b/sdk/typescript/scripts/compare-test-reports.pynew file mode 100644index 000000000..6f735eed4--- /dev/null+++ b/sdk/typescript/scripts/compare-test-reports.py@@ -0,0 +1,84 @@+"""Compare Bun JUnit inventories before changing the required CI runner."""++import argparse+from collections import Counter+from glob import glob+from pathlib import Path+import sys+import xml.etree.ElementTree as ET+++def read_report(path: Path) -> tuple[Counter, float, bool]:+ root = ET.parse(path).getroot()+ cases = {}+ for case in root.iter("testcase"):+ status = "passed"+ if case.find("skipped") is not None:+ status = "skipped"+ if case.find("failure") is not None or case.find("error") is not None:+ status = "failed"+ identity = (+ case.get("file", "").replace("\\", "/").removeprefix("./"),+ case.get("classname", ""),+ case.get("name", ""),+ )+ if identity in cases:+ raise ValueError(f"{path}: duplicate test identity: {' > '.join(identity)}")+ cases[identity] = status+ if not cases:+ raise ValueError(f"{path}: no test cases")+ if int(root.get("tests", str(len(cases)))) != len(cases):+ raise ValueError(f"{path}: reported test count does not match test cases")+ failed = "failed" in cases.values() or any(+ int(node.get(field, "0"))+ for node in root.iter()+ if node.tag in ("testsuite", "testsuites")+ for field in ("failures", "errors")+ )+ if failed:+ print(f"{path}: test run failed", file=sys.stderr)+ seconds = float(root.get("time", "0"))+ skipped = sum(status == "skipped" for status in cases.values())+ print(f"| {path.name} | {len(cases)} | {skipped} | {seconds:.2f} |")+ return (+ Counter((*identity, status) for identity, status in cases.items()),+ seconds,+ failed,+ )+++def main() -> int:+ parser = argparse.ArgumentParser(description=__doc__)+ parser.add_argument("baseline", type=Path)+ parser.add_argument("candidates", nargs="+", help="JUnit files or glob patterns")+ args = parser.parse_args()+ print("| Report | Cases | Skipped | Seconds |")+ print("| --- | ---: | ---: | ---: |")+ baseline, _, failed = read_report(args.baseline)+ candidates = Counter()+ durations = []+ for pattern in args.candidates:+ paths = sorted(glob(pattern))+ if not paths:+ raise ValueError(f"No reports match {pattern}")+ for path in paths:+ cases, seconds, report_failed = read_report(Path(path))+ failed = failed or report_failed+ candidates.update(cases)+ durations.append(seconds)+ missing, extra = baseline - candidates, candidates - baseline+ if failed or missing or extra:+ for label, difference in (("Missing", missing), ("Extra", extra)):+ for identity, count in sorted(difference.items()):+ print(f"{label} ({count}): {' > '.join(identity)}", file=sys.stderr)+ return 1+ print(f"\nIdentical test inventory and outcomes. Slowest candidate: {max(durations):.2f}s; combined test time: {sum(durations):.2f}s.\n")+ return 0+++if __name__ == "__main__":+ try:+ sys.exit(main())+ except (OSError, ValueError, ET.ParseError) as error:+ print(error, file=sys.stderr)+ sys.exit(1)diff --git a/sdk/typescript/tests-ts/cli-deep-scan-summary.test.ts b/sdk/typescript/tests-ts/cli-deep-scan-summary.test.tsindex b4b358c69..382fe33d7 100644--- a/sdk/typescript/tests-ts/cli-deep-scan-summary.test.ts+++ b/sdk/typescript/tests-ts/cli-deep-scan-summary.test.ts@@ -112,7 +112,7 @@ describe("deep scan completion summary", () => {});test.each([false, true])(- "keeps the result when stop details are unavailable (read fails: %s)",+ "keeps the result when stop details are unavailable (read fails: %p)",async (fails) => {const text = await summary({result: fakeResult(),diff --git a/sdk/typescript/tests-ts/cli.test.ts b/sdk/typescript/tests-ts/cli.test.tsindex ea5cd9846..fbf7520e1 100644--- a/sdk/typescript/tests-ts/cli.test.ts+++ b/sdk/typescript/tests-ts/cli.test.ts@@ -1849,7 +1849,7 @@ describe("CLI", () => {});test.each([false, true])(- "subscribes to session details only with TTY stdin: %s",+ "subscribes to session details only with TTY stdin: %p",async (stdinTTY) => {const descriptor = Object.getOwnPropertyDescriptor(process.stdin,diff --git a/sdk/typescript/tests-ts/cost.test.ts b/sdk/typescript/tests-ts/cost.test.tsindex 50b46d27a..3d0fbe317 100644--- a/sdk/typescript/tests-ts/cost.test.ts+++ b/sdk/typescript/tests-ts/cost.test.ts@@ -175,7 +175,7 @@ describe("scan cost", () => {[{ cache_write_input_tokens: 0, cache_write_tokens: 15 }, 15],[{ cache_write_input_tokens: 0, cache_write_tokens: 80 }, 0],] as const)(- "keeps workbench cache-write normalization aligned with SDK usage",+ "keeps workbench cache-write normalization aligned with SDK usage for %j as %p tokens",async (cacheWrites, expectedCacheWrites) => {const { PLUGIN_ROOT } = await import("./plugin-root.js");const python = Bun.which("python3") ?? Bun.which("python");diff --git a/sdk/typescript/tests-ts/deep-scan-workbench.test.ts b/sdk/typescript/tests-ts/deep-scan-workbench.test.tsindex 262de005a..02c66b8d6 100644--- a/sdk/typescript/tests-ts/deep-scan-workbench.test.ts+++ b/sdk/typescript/tests-ts/deep-scan-workbench.test.ts@@ -392,7 +392,7 @@ describe("deep scan workbench ownership", () => {[0.5, 0.5],[96, 96],] as const)(- "resolves the configured discovery deadline %s as %s hours",+ "resolves the configured discovery deadline %p as %p hours",async (configuredHours, expectedHours) => {const root = await realpath(await mkdtemp(join(tmpdir(), "codex-security-deep-deadline-config-")),@@ -509,7 +509,7 @@ describe("deep scan workbench ownership", () => {});test.each([false, true] as const)(- "backfills and repairs discovery deadline migration when already recorded: %s",+ "backfills and repairs discovery deadline migration when already recorded: %p",(migrationRecorded) => {const python = Bun.which("python3") ?? Bun.which("python");expect(python).not.toBeNull();@@ -565,7 +565,7 @@ describe("deep scan workbench ownership", () => {["scoped_path", false],["repository", true],] as const)(- "returns an honest partial %s report with existing deferred work %s when saturated discovery exceeds its cost limit",+ "returns an honest partial %s report with existing deferred work %p when saturated discovery exceeds its cost limit",async (inventoryStrategy, existingDeferred) => {const root = await realpath(await mkdtemp(join(tmpdir(), "codex-security-deep-budget-recovery-")),diff --git a/sdk/typescript/tests-ts/multiscan.test.ts b/sdk/typescript/tests-ts/multiscan.test.tsindex 390988965..8fd8ac9ba 100644--- a/sdk/typescript/tests-ts/multiscan.test.ts+++ b/sdk/typescript/tests-ts/multiscan.test.ts@@ -385,7 +385,7 @@ describe("multiscan", () => {});test.each([false, true])(- "continues scanning when a progress observer fails %s",+ "continues scanning when a progress observer fails %p",async (asynchronous) => {const paths = await fixture();const source = await repository(paths.root, "observer-failure");diff --git a/sdk/typescript/tests-ts/release-automation.test.ts b/sdk/typescript/tests-ts/release-automation.test.tsindex 69126b1c9..e69ac613a 100644--- a/sdk/typescript/tests-ts/release-automation.test.ts+++ b/sdk/typescript/tests-ts/release-automation.test.ts@@ -1,7 +1,6 @@import { spawnSync } from "node:child_process";import { createHash } from "node:crypto";import {- existsSync,mkdirSync,mkdtempSync,readFileSync,@@ -9,9 +8,10 @@ import {writeFileSync,} from "node:fs";import { tmpdir } from "node:os";-import { dirname, join } from "node:path";+import { join } from "node:path";import { fileURLToPath } from "node:url";import { describe, expect, test } from "bun:test";+import { bashCommand } from "./support/shell.js";type ReleaseMetadata = Record<string, unknown>;@@ -142,15 +142,7 @@ const releaseRun = "30481596229";const releaseRepository = "openai/codex-security";const releaseTagTimeout = process.platform === "win32" ? 20_000 : 10_000;-function releaseShell(): string {- if (process.platform !== "win32") return "bash";- const git = Bun.which("git");- if (git === null) return "bash";- const gitBash = join(dirname(dirname(git)), "bin", "bash.exe");- return existsSync(gitBash) ? gitBash : "bash";-}--const bash = releaseShell();+const bash = bashCommand();const jqMock = ["jq() {",' node -e \'const fs=require("node:fs");const filter=process.argv.at(-1);const value=JSON.parse(fs.readFileSync(0,"utf8"));if(filter==="[.object.type, .object.sha] | @tsv"){const fields=[value.object?.type,value.object?.sha];if(fields.some((field)=>typeof field!=="string"))process.exit(1);process.stdout.write(fields.join("\\t")+"\\n");}else if(filter===".status // empty"){if(value.status!=null)process.stdout.write(String(value.status)+"\\n");}else process.exit(64);\' -- "$@"',diff --git a/sdk/typescript/tests-ts/sarif.test.ts b/sdk/typescript/tests-ts/sarif.test.tsindex 56c0d94a6..b98676f40 100644--- a/sdk/typescript/tests-ts/sarif.test.ts+++ b/sdk/typescript/tests-ts/sarif.test.ts@@ -142,7 +142,7 @@ describe("SARIF presentation", () => {["high", 0, undefined],["high", 6.25, "6.25"],["critical", 10, "10"],- ] as const)("maps %s severity with score %s", (level, score, expected) => {+ ] as const)("maps %s severity with score %p", (level, score, expected) => {const severity =score === undefined? { level }diff --git a/sdk/typescript/tests-ts/skeleton.test.ts b/sdk/typescript/tests-ts/skeleton.test.tsindex d6020a41f..1ccf9b86f 100644--- a/sdk/typescript/tests-ts/skeleton.test.ts+++ b/sdk/typescript/tests-ts/skeleton.test.ts@@ -10,6 +10,37 @@ import {} from "../src/index.js";import { main } from "../src/cli.js";+interface WorkflowStep {+ name?: string;+ uses?: string;+ with?: Record<string, unknown>;+ run?: string;+ if?: string;+ env?: Record<string, unknown>;+ "continue-on-error"?: boolean;+}++interface WorkflowJob {+ name?: string;+ needs?: string[];+ env?: Record<string, unknown>;+ strategy?: { matrix: Record<string, unknown> };+ steps: WorkflowStep[];+}++async function workflow(name: string) {+ return Bun.YAML.parse(+ await readFile(+ new URL(`../../../.github/workflows/${name}`, import.meta.url),+ "utf8",+ ),+ ) as {+ on: Record<string, unknown>;+ env?: Record<string, unknown>;+ jobs: Record<string, WorkflowJob>;+ };+}+function capture(): {stream: Pick<NodeJS.WriteStream, "write">;text: () => string;@@ -71,28 +102,33 @@ describe("TypeScript package skeleton", () => {});test("pins each Node.js minimum and preserves protected and latest LTS checks", async () => {- const ciWorkflow = await readFile(- new URL("../../../.github/workflows/node-ci.yml", import.meta.url),- "utf8",+ const { jobs } = await workflow("node-ci.yml");+ expect(jobs["test"]?.name).toBe(+ "${{ matrix.os }} / node-${{ matrix.node == '22.13.0' && '22' || matrix.node }}",);-- expect(ciWorkflow).toContain(- "${{ matrix.node == '22.13.0' && '22' || matrix.node }}",+ expect(jobs["test"]?.strategy?.matrix).toEqual({+ os: ["ubuntu-latest", "macos-latest"],+ node: ["22.13.0"],+ include: ["24.0.0", "24", "26.0.0", "26"].map((node) => ({+ os: "ubuntu-latest",+ node,+ })),+ });+ expect(jobs["windows"]?.name).toBe(+ "windows-latest / node-${{ matrix.node == '22.13.0' && '22' || matrix.node }}",);- expect(ciWorkflow).toContain('node: ["22.13.0"]');- for (const version of ["24.0.0", "24", "26.0.0", "26"]) {- expect(ciWorkflow).toContain(`node: "${version}"`);- }+ expect(jobs["windows"]?.needs).toEqual(["windows-test", "windows-verify"]);+ expect(jobs["windows-test"]?.strategy?.matrix["node"]).toEqual([+ "22.13.0",+ "24",+ ]);});test("randomizes tests and keeps the default and Windows CI timeouts", async () => {const packageJson = JSON.parse(await readFile(new URL("../package.json", import.meta.url), "utf8"),);- const ciWorkflow = await readFile(- new URL("../../../.github/workflows/node-ci.yml", import.meta.url),- "utf8",- );+ const { jobs } = await workflow("node-ci.yml");const bunConfig = parse(await readFile(new URL("../bunfig.toml", import.meta.url), "utf8"),);@@ -101,17 +137,131 @@ describe("TypeScript package skeleton", () => {"bun test --timeout 30000 ./tests-ts",);expect(bunConfig).toMatchObject({ test: { randomize: true } });- expect(ciWorkflow).toContain(- "run: node sdk/typescript/scripts/run-windows-ci-tests.mjs ${{ matrix.shard }}",+ expect(packageJson.scripts["test:ci"]).toContain("pnpm run test ");+ expect(jobs["windows-test"]?.steps).toContainEqual(+ expect.objectContaining({+ run: "node sdk/typescript/scripts/run-windows-ci-tests.mjs ${{ matrix.shard }}",+ }),+ );+ });++ test("runs shared static checks once and keeps report upload non-blocking", async () => {+ const { jobs } = await workflow("node-ci.yml");+ const steps = Object.values(jobs).flatMap((job) => job.steps);+ for (const name of ["Typecheck", "Check formatting"]) {+ expect(steps.filter((step) => step.name === name)).toEqual([+ expect.objectContaining({+ if: "matrix.os == 'ubuntu-latest' && matrix.node == '22.13.0'",+ }),+ ]);+ }+ expect(steps.find((step) => step.name === "Test")).not.toHaveProperty(+ "continue-on-error",);- expect(ciWorkflow).toContain(- "run: bun test --timeout 120000 ./tests-ts/windows-machine-policy.test.ts",+ expect(+ steps.find((step) => step.name === "Upload test reports"),+ ).toMatchObject({+ "continue-on-error": true,+ });+ for (const name of ["test", "windows-verify"]) {+ expect(jobs[name]?.steps).toContainEqual(+ expect.objectContaining({+ run: "pnpm run check:package ../../dist/*.tgz",+ }),+ );+ }+ });++ test("keeps machine-wide policy changes out of parallel and experimental runs", async () => {+ const ci = await workflow("node-ci.yml");+ const windows = ci.jobs["windows-test"]!.steps;+ expect(+ windows.find((step) => step.name === "Test shard ${{ matrix.shard }}")+ ?.env?.["CODEX_SECURITY_ALLOW_MACHINE_POLICY_TEST"],+ ).toBe("false");+ expect(+ windows.find(+ (step) => step.name === "Test machine-wide PowerShell policy",+ ),+ ).toMatchObject({+ if: "matrix.shard == 3 && runner.environment == 'github-hosted'",+ env: { CODEX_SECURITY_ALLOW_MACHINE_POLICY_TEST: "true" },+ run: "bun test --timeout 120000 ./tests-ts/windows-machine-policy.test.ts",+ });+ const quality = await workflow("test-quality.yml");+ expect(Object.keys(quality.on).sort()).toEqual([+ "pull_request",+ "schedule",+ "workflow_dispatch",+ ]);+ expect(quality.on["pull_request"]).toEqual({+ paths: [".github/workflows/test-quality.yml"],+ });+ expect(quality.env?.["CODEX_SECURITY_ALLOW_MACHINE_POLICY_TEST"]).toBe(+ "false",);- expect(ciWorkflow).toContain(- "name: windows-latest / node-${{ matrix.node == '22.13.0' && '22' || matrix.node }}",+ expect(quality.env?.["CODEX_SECURITY_INTEGRATION"]).toBe("0");+ for (let shard = 1; shard <= 7; shard += 1) {+ expect(+ quality.jobs["runner"]?.strategy?.matrix["include"],+ ).toContainEqual({+ os: "windows-latest",+ mode: `shard-${shard}`,+ args: `--shard=${shard}/7`,+ });+ }+ });++ test("keeps runner modes reproducible and report uploads rerunnable", async () => {+ const ci = await workflow("node-ci.yml");+ const quality = await workflow("test-quality.yml");+ const runner = quality.jobs["runner"]!;+ const seed =+ "${{ github.event_name == 'pull_request' && 1 || github.run_number }}";+ expect(quality.env).not.toHaveProperty("CODEX_SECURITY_PROPERTY_SEED");+ expect(runner.env?.["CODEX_SECURITY_PROPERTY_SEED"]).toBe(seed);+ expect(runner.strategy?.matrix["mode"]).toEqual([+ "baseline",+ "isolated",+ "parallel",+ ]);+ for (const [mode, args] of [+ ["baseline", ""],+ ["isolated", "--isolate"],+ ["parallel", "--parallel=2"],+ ] as const) {+ expect(runner.strategy?.matrix["include"]).toContainEqual({+ mode,+ args,+ });+ }+ const command = runner.steps.find(+ (step) => step.name === "Test runner mode",+ )?.run;+ expect(command).toContain(+ "${{ runner.os == 'Windows' && '--timeout=120000' || '' }}",);- expect(ciWorkflow).toContain("run: pnpm --dir sdk/typescript run test");- expect(ciWorkflow).not.toContain("--timeout 60000");+ expect(command).toContain("${{ matrix.args }}");+ expect(command).toContain("--seed=${{ env.CODEX_SECURITY_PROPERTY_SEED }}");++ const uploads = [...Object.values(ci.jobs), ...Object.values(quality.jobs)]+ .flatMap((job) => job.steps)+ .filter((step) => step.uses?.startsWith("actions/upload-artifact@"));+ expect(uploads).toHaveLength(3);+ for (const upload of uploads) {+ expect(upload.with?.["overwrite"]).toBe(true);+ }+ expect(+ uploads.find((step) => step.name === "Upload mutation report"),+ ).toMatchObject({ "continue-on-error": true });+ expect(+ uploads.find((step) => step.name === "Upload runner report"),+ ).not.toHaveProperty("continue-on-error");+ expect(+ quality.jobs["mutation"]?.steps.find(+ (step) => step.name === "Run mutation trial",+ ),+ ).not.toHaveProperty("continue-on-error");});test("builds packages without a preinstalled package manager and provides a production audit", async () => {@@ -130,14 +280,17 @@ describe("TypeScript package skeleton", () => {test("keeps production dependency audits non-blocking in CI and releases", async () => {for (const workflowName of ["node-ci.yml", "node-release.yml"]) {- const workflow = await readFile(- new URL(`../../../.github/workflows/${workflowName}`, import.meta.url),- "utf8",- );-- expect(workflow).toMatch(- /- name: Audit production dependencies\n(?:\s+if: [^\n]+\n)?\s+continue-on-error: true\n\s+run: (?:sfw )?pnpm --dir sdk\/typescript run audit:prod/u,- );+ const { jobs } = await workflow(workflowName);+ const audits = Object.values(jobs)+ .flatMap((job) => job.steps)+ .filter((step) => step.name === "Audit production dependencies");+ expect(audits.length).toBeGreaterThan(0);+ for (const audit of audits) {+ expect(audit["continue-on-error"]).toBe(true);+ expect(audit.run).toMatch(+ /^(?:sfw )?pnpm --dir sdk\/typescript run audit:prod$/u,+ );+ }}});diff --git a/sdk/typescript/tests-ts/support/shell.ts b/sdk/typescript/tests-ts/support/shell.tsnew file mode 100644index 000000000..31427daa1--- /dev/null+++ b/sdk/typescript/tests-ts/support/shell.ts@@ -0,0 +1,10 @@+import { existsSync } from "node:fs";+import { dirname, join } from "node:path";++export function bashCommand(): string {+ if (process.platform !== "win32") return "bash";+ const git = Bun.which("git");+ if (git === null) return "bash";+ const gitBash = join(dirname(dirname(git)), "bin", "bash.exe");+ return existsSync(gitBash) ? gitBash : "bash";+}diff --git a/sdk/typescript/tests-ts/test-reports.test.ts b/sdk/typescript/tests-ts/test-reports.test.tsnew file mode 100644index 000000000..27eda4015--- /dev/null+++ b/sdk/typescript/tests-ts/test-reports.test.ts@@ -0,0 +1,196 @@+import { spawnSync } from "node:child_process";+import { mkdtemp, readFile, rm, writeFile } from "node:fs/promises";+import { tmpdir } from "node:os";+import { join } from "node:path";+import { fileURLToPath } from "node:url";+import { afterEach, describe, expect, test } from "bun:test";+import { bashCommand } from "./support/shell.js";++const bash = bashCommand();+const directories: string[] = [];+afterEach(async () => {+ await Promise.all(+ directories+ .splice(0)+ .map((path) => rm(path, { recursive: true, force: true })),+ );+});++async function fixtures() {+ const root = await mkdtemp(join(tmpdir(), "codex-security-test-reports-"));+ directories.push(root);+ return {+ root,+ async report(+ name: string,+ cases: string[],+ failures = 0,+ count = cases.length,+ ) {+ const path = join(root, name);+ await writeFile(+ path,+ `<testsuites tests="${count}" failures="${failures}" time="1.25"><testsuite>${cases.join("")}</testsuite></testsuites>`,+ );+ return path;+ },+ };+}++function testcase(name: string, status = "") {+ return `<testcase file="tests-ts/example.test.ts" classname="example" name="${name}">${status}</testcase>`;+}++async function compare(baseline: string, ...candidates: string[]) {+ const python = Bun.which("python3") ?? Bun.which("python") ?? Bun.which("py");+ if (python === null) throw new Error("A Python interpreter is required.");+ const child = Bun.spawn({+ cmd: [+ python,+ "-I",+ "-B",+ fileURLToPath(+ new URL("../scripts/compare-test-reports.py", import.meta.url),+ ),+ baseline,+ ...candidates,+ ],+ stdin: "ignore",+ stdout: "pipe",+ stderr: "pipe",+ });+ const [status, stdout, stderr] = await Promise.all([+ child.exited,+ new Response(child.stdout).text(),+ new Response(child.stderr).text(),+ ]);+ return { status, stdout, stderr };+}++describe("JUnit inventory comparison", () => {+ test("runs every workflow comparison before reporting a mismatch", async () => {+ const fixture = await fixtures();+ const workflow = Bun.YAML.parse(+ await readFile(+ new URL("../../../.github/workflows/test-quality.yml", import.meta.url),+ "utf8",+ ),+ ) as {+ jobs: { compare: { steps: Array<{ name?: string; run?: string }> } };+ };+ const script = workflow.jobs.compare.steps.find(+ (step) => step.name === "Compare inventories and outcomes",+ )!.run!;+ const expected = [+ ...["ubuntu-latest", "windows-latest"].flatMap((os) =>+ ["isolated", "parallel"].map(+ (mode) => `reports/runner-${os}-${mode}.xml`,+ ),+ ),+ "reports/runner-windows-latest-shard-*.xml",+ ];+ const mock = `python3() {+ printf '%s\\n' "$3"+ [[ "$3" != "$CODEX_SECURITY_TEST_FAIL_REPORT" ]]+}`;+ const summary = join(fixture.root, "summary.md");+ for (const failedReport of ["", expected[0]!]) {+ await writeFile(summary, "");+ const result = spawnSync(+ bash,+ ["-e", "-o", "pipefail", "-c", `${mock}\n${script}`],+ {+ cwd: fixture.root,+ encoding: "utf8",+ env: {+ ...process.env,+ GITHUB_STEP_SUMMARY: "summary.md",+ CODEX_SECURITY_TEST_FAIL_REPORT: failedReport,+ },+ timeout: 10_000,+ },+ );+ expect(result.status, result.stderr).toBe(failedReport === "" ? 0 : 1);+ expect((await readFile(summary, "utf8")).trim().split(/\r?\n/u)).toEqual(+ expected,+ );+ }+ });++ test("merges native shards without depending on test order", async () => {+ const fixture = await fixtures();+ const passed = testcase("accepts & preserves");+ const skipped = testcase("platform-only", "<skipped/>");+ const baseline = await fixture.report("baseline.xml", [passed, skipped]);+ await fixture.report("shard-1.xml", [skipped]);+ await fixture.report("shard-2.xml", [passed]);+ const result = await compare(baseline, join(fixture.root, "shard-*.xml"));+ expect(result.status, result.stderr).toBe(0);+ expect(result.stdout).toContain("combined test time: 2.50s");+ });++ test("rejects ambiguous test identities even when totals match", async () => {+ const fixture = await fixtures();+ const first = testcase("same parameterized name");+ for (const [name, repeated] of [+ ["same-outcome", first],+ ["different-outcome", testcase("same parameterized name", "<skipped/>")],+ ] as const) {+ const baseline = await fixture.report(`${name}-baseline.xml`, [+ first,+ repeated,+ ]);+ const candidate = await fixture.report(`${name}-candidate.xml`, [+ first,+ repeated,+ ]);+ const result = await compare(baseline, candidate);+ expect(result.status).toBe(1);+ expect(result.stderr).toContain("duplicate test identity");+ }+ });++ test("reports every shard's timing even when an earlier shard fails", async () => {+ const fixture = await fixtures();+ const first = testcase("first");+ const second = testcase("second");+ const baseline = await fixture.report("baseline.xml", [first, second]);+ const failed = await fixture.report("shard-1.xml", [+ testcase("first", "<failure/>"),+ ]);+ const passed = await fixture.report("shard-2.xml", [second]);+ const result = await compare(baseline, failed, passed);++ expect(result.status).toBe(1);+ expect(result.stderr).toContain("test run failed");+ expect(result.stdout).toContain("shard-1.xml");+ expect(result.stdout).toContain("shard-2.xml");+ });++ test("rejects dropped, duplicated, skipped, failed, or incomplete results", async () => {+ const fixture = await fixtures();+ const first = testcase("first");+ const second = testcase("second");+ const baseline = await fixture.report("baseline.xml", [first, second]);+ for (const [name, cases, failures, count] of [+ ["missing", [first], 0, 1],+ ["duplicate", [first, second, second], 0, 3],+ ["skipped", [first, testcase("second", "<skipped/>")], 0, 2],+ ["failed", [first, testcase("second", "<failure/>")], 1, 2],+ ["summary-failed", [first, second], 1, 2],+ ["incomplete", [first], 0, 2],+ ["empty", [], 0, 0],+ ] as const) {+ const candidate = await fixture.report(+ `${name}.xml`,+ [...cases],+ failures,+ count,+ );+ expect((await compare(baseline, candidate)).status, name).toBe(1);+ }+ expect(+ (await compare(baseline, join(fixture.root, "absent-*.xml"))).status,+ ).toBe(1);+ });+});