Skip to content

OpenAI's Codex Security CLI and TypeScript SDK for finding, validating, and fixing security vulnerabilities. npm: https://www.npmjs.com/package/@openai/codex-security

ci: require checks for changed Action and eval components (#1931)

Michael D'Angeloe53b6c0757d3
diff --git a/.github/scripts/check-ci-results.mjs b/.github/scripts/check-ci-results.mjs
index 2c9a865e8..dc55d6036 100644
--- a/.github/scripts/check-ci-results.mjs
+++ b/.github/scripts/check-ci-results.mjs
@@ -5,26 +5,30 @@ export function failedChecks(needs) {
return ["classify-changes"];
const selected = needs["classify-changes"].outputs;
const required = ["static-checks"];
- if (selected["source-required"] === "true") {
- required.push(
- "native",
- "package",
+ const groups = {
+ "package-required": ["native", "package", "compatibility"],
+ "source-required": [
"test",
- "compatibility",
"mcp",
"plugin-host",
"plugin-source",
"windows-test",
"windows-verify",
- );
- } else if (selected["source-required"] !== "false") {
- return ["source-required classification"];
+ ],
+ "workflow-quality": ["workflow-quality"],
+ "container-validate": ["container-validate"],
+ "action-ci": ["action-ci"],
+ "eval-contracts": ["eval-contracts"],
+ };
+ for (const [name, checks] of Object.entries(groups)) {
+ if (selected[name] === "true") required.push(...checks);
+ else if (selected[name] !== "false") return [`${name} classification`];
}
- for (const check of ["workflow-quality", "container-validate"]) {
- if (selected[check] === "true") required.push(check);
- else if (selected[check] !== "false") return [`${check} classification`];
- }
- return required.filter((name) => needs[name]?.result !== "success");
+ return required.filter(
+ (name) =>
+ needs[name]?.result !== "success" ||
+ (name === "action-ci" && needs[name].outputs?.passed !== "true"),
+ );
}
if (
diff --git a/.github/scripts/select-ci-checks.mjs b/.github/scripts/select-ci-checks.mjs
index 5734ea589..002e99bbc 100644
--- a/.github/scripts/select-ci-checks.mjs
+++ b/.github/scripts/select-ci-checks.mjs
@@ -6,17 +6,35 @@ const documentation =
/^(?:README\.md|CONTRIBUTING\.md|CODE_OF_CONDUCT\.md|docs\/.*\.md|github-action\/README\.md|docker\/README\.md)$/;
const ordinarySource =
/^(?:sdk\/typescript\/(?:src|tests-ts)|plugins\/codex-security\/mcp-app\/(?:src|tests))\/.*\.ts$/;
+const actionFiles =
+ /^(?:action\.yml|github-action\/(?:README\.md|(?:runtime\/)?package(?:-lock)?\.json|dist\/(?:index|post)\.cjs|(?:src|scripts|tests)\/[^/]+\.(?:mjs|ts)))$/;
+const evalFiles =
+ /^evals\/(?:standard-time-budget\/(?:[^/]+\.mts|experiment\.json|README\.md)|triage-finding\/(?:package\.json|pnpm-lock\.yaml|README\.md|(?:assertions|scripts)\/[^/]+\.mts))$/;
+
+function scopeFor(paths, event, baseChanged) {
+ if (event !== "pull_request" || baseChanged || !paths.length) return "full";
+ if (paths.every((path) => documentation.test(path))) return "docs";
+ if (paths.every((path) => actionFiles.test(path))) return "action";
+ if (paths.every((path) => evalFiles.test(path))) return "evals";
+ if (
+ paths.every((path) => ordinarySource.test(path) || documentation.test(path))
+ )
+ return "source";
+ return "full";
+}
export function selectChecks(paths, { event, baseChanged = false }) {
- // Only known documentation and TypeScript source paths can narrow qualification.
- const full =
- event !== "pull_request" ||
- baseChanged ||
- paths.length === 0 ||
+ const scope = scopeFor(paths, event, baseChanged);
+ const full = scope === "full";
+ const source = full || scope === "source";
+ const action =
+ full ||
+ scope === "action" ||
paths.some(
- (path) => !documentation.test(path) && !ordinarySource.test(path),
+ (path) =>
+ path === "github-action/README.md" ||
+ path === "sdk/typescript/src/git-remote.ts",
);
- const source = full || !paths.every((path) => documentation.test(path));
const compatibility = ["22.13.0", "24.0.0", "24", "26.0.0", "26"]
.filter((version) => full || version !== "26")
.map((version) => ({ os: "ubuntu-latest", node: version }));
@@ -42,9 +60,17 @@ export function selectChecks(paths, { event, baseChanged = false }) {
];
return {
"source-required": String(source),
+ "package-required": String(source || scope === "action"),
+ "action-ci": String(action),
+ "eval-contracts": String(source || scope === "evals"),
"workflow-quality": String(full),
"container-validate": String(source),
- "compatibility-matrix": JSON.stringify({ include: compatibility }),
+ "compatibility-matrix": JSON.stringify({
+ include:
+ scope === "action"
+ ? [{ os: "ubuntu-latest", node: "24" }]
+ : compatibility,
+ }),
"python-matrix": JSON.stringify({ include: pythonMatrix }),
"windows-node-matrix": JSON.stringify({
node: full ? ["22.13.0", "24"] : ["22.13.0"],
diff --git a/.github/scripts/test-ci-selection.mjs b/.github/scripts/test-ci-selection.mjs
index ab86b340e..034844ed5 100644
--- a/.github/scripts/test-ci-selection.mjs
+++ b/.github/scripts/test-ci-selection.mjs
@@ -9,6 +9,9 @@ const select = (paths, options = {}) =>
const regularSource = {
"source-required": "true",
+ "package-required": "true",
+ "action-ci": "false",
+ "eval-contracts": "true",
"workflow-quality": "false",
"container-validate": "true",
"compatibility-matrix": JSON.stringify({
@@ -38,7 +41,8 @@ const regularSource = {
"windows-node-matrix": JSON.stringify({ node: ["22.13.0"] }),
};
const fullQualification = {
- "source-required": "true",
+ ...regularSource,
+ "action-ci": "true",
"workflow-quality": "true",
"container-validate": "true",
"compatibility-matrix": JSON.stringify({
@@ -71,12 +75,16 @@ const fullQualification = {
"windows-node-matrix": JSON.stringify({ node: ["22.13.0", "24"] }),
};
-test("only ordinary documentation can skip source qualification", () => {
- assert.deepEqual(select(["README.md", "docs/bedrock.md"]), {
- ...regularSource,
- "source-required": "false",
- "container-validate": "false",
- });
+const docs = {
+ ...regularSource,
+ "source-required": "false",
+ "package-required": "false",
+ "eval-contracts": "false",
+ "container-validate": "false",
+};
+
+test("ordinary documentation selects static checks", () => {
+ assert.deepEqual(select(["README.md", "docs/bedrock.md"]), docs);
for (const path of [
"plugins/codex-security/skills/security-scan/SKILL.md",
"plugins/codex-security/references/core-scan.md",
@@ -94,6 +102,106 @@ test("only ordinary documentation can skip source qualification", () => {
);
});
+test("the Action README also requires Action documentation and release contracts", () => {
+ const expected = { ...docs, "action-ci": "true" };
+ assert.deepEqual(select(["github-action/README.md"]), expected);
+ assert.deepEqual(select(["docs/ci.md", "github-action/README.md"]), expected);
+ assert.deepEqual(
+ select(["sdk/typescript/src/cost.ts", "github-action/README.md"]),
+ { ...regularSource, "action-ci": "true" },
+ );
+ assert.deepEqual(
+ failedChecks({
+ "classify-changes": { result: "success", outputs: expected },
+ "static-checks": { result: "success" },
+ }),
+ ["action-ci"],
+ );
+});
+
+test("Action-only changes require Action CI and integration against the built package", () => {
+ const expected = {
+ ...docs,
+ "package-required": "true",
+ "action-ci": "true",
+ "compatibility-matrix": JSON.stringify({
+ include: [{ os: "ubuntu-latest", node: "24" }],
+ }),
+ };
+ for (const paths of [
+ ["action.yml", "github-action/README.md"],
+ ["github-action/src/results.ts", "github-action/dist/index.cjs"],
+ ["github-action/dist/post.cjs"],
+ ["github-action/package.json", "github-action/package-lock.json"],
+ [
+ "github-action/runtime/package.json",
+ "github-action/runtime/package-lock.json",
+ ],
+ [
+ "github-action/tests/release.test.mjs",
+ "github-action/scripts/release.mjs",
+ ],
+ ]) {
+ assert.deepEqual(select(paths), expected, paths.join(", "));
+ }
+ const needs = {
+ "classify-changes": { result: "success", outputs: expected },
+ "static-checks": { result: "success" },
+ "action-ci": { result: "success", outputs: { passed: "true" } },
+ };
+ assert.deepEqual(failedChecks(needs), ["native", "package", "compatibility"]);
+ for (const name of ["native", "package", "compatibility"])
+ needs[name] = { result: "success" };
+ assert.deepEqual(failedChecks(needs), []);
+ for (const action of [
+ { result: "failure", outputs: { passed: "true" } },
+ { result: "success", outputs: { passed: "false" } },
+ { result: "success" },
+ ]) {
+ assert.deepEqual(failedChecks({ ...needs, "action-ci": action }), [
+ "action-ci",
+ ]);
+ }
+ assert.deepEqual(select(["sdk/typescript/src/git-remote.ts"]), {
+ ...regularSource,
+ "action-ci": "true",
+ });
+});
+
+test("Standard and triage eval changes require all deterministic eval contracts", () => {
+ const expected = { ...docs, "eval-contracts": "true" };
+ for (const paths of [
+ [
+ "evals/standard-time-budget/fixtures.mts",
+ "evals/standard-time-budget/test_production.mts",
+ ],
+ [
+ "evals/standard-time-budget/experiment.json",
+ "evals/standard-time-budget/README.md",
+ ],
+ [
+ "evals/triage-finding/package.json",
+ "evals/triage-finding/pnpm-lock.yaml",
+ ],
+ [
+ "evals/triage-finding/assertions/output.mts",
+ "evals/triage-finding/scripts/test-triage-io.mts",
+ ],
+ ]) {
+ assert.deepEqual(select(paths), expected, paths.join(", "));
+ }
+ const needs = {
+ "classify-changes": { result: "success", outputs: expected },
+ "static-checks": { result: "success" },
+ "eval-contracts": { result: "skipped" },
+ };
+ assert.deepEqual(failedChecks(needs), ["eval-contracts"]);
+ assert.deepEqual(
+ failedChecks({ ...needs, "eval-contracts": { result: "success" } }),
+ [],
+ );
+});
+
test("packaged SDK documentation keeps package validation selected", () => {
const manifest = JSON.parse(
readFileSync(
@@ -169,6 +277,17 @@ test("unknown paths and dependency configuration receive full qualification", ()
["sdk/typescript/pnpm-workspace.yaml"],
["plugins/codex-security/mcp-app/pnpm-workspace.yaml"],
["evals/triage-finding/pnpm-workspace.yaml"],
+ ["evals/standard-time-budget/fixtures.mts", "sdk/typescript/src/cost.ts"],
+ ["evals/triage-finding/fixtures/repo/package.json"],
+ [
+ "evals/triage-finding/pnpm-lock.yaml",
+ "github-action/runtime/package-lock.json",
+ ],
+ ["github-action/src/main.ts", "sdk/typescript/src/git-remote.ts"],
+ ["github-action/src/main.ts", "github-action/src/nested/unknown.ts"],
+ ["github-action/tsconfig.json"],
+ ["github-action/package.json", "github-action/.npmrc"],
+ ["plugins/codex-security/examples/completed-scan/findings.jsonl"],
["README.md", "new-runtime/file"],
["sdk/typescript/src/cost.ts", "new-runtime/file"],
// --no-renames reports both sides when source becomes an unknown input.
@@ -188,6 +307,7 @@ test("tooling and non-TypeScript product inputs receive full qualification", ()
"plugins/codex-security/native/src/lib.rs",
"sdk/typescript/scripts/build-plugin.mjs",
".github/workflows/node-ci.yml",
+ ".github/workflows/action-ci.yml",
"plugins/codex-security/scripts/workbench_db.py",
"plugins/codex-security/tests/test_database.py",
"plugins/codex-security/requirements-test.txt",
@@ -217,6 +337,8 @@ test("the final gate rejects every missing or unsuccessful selected check", () =
const needs = Object.fromEntries(
[
"static-checks",
+ "action-ci",
+ "eval-contracts",
"native",
"package",
"test",
@@ -230,6 +352,7 @@ test("the final gate rejects every missing or unsuccessful selected check", () =
"container-validate",
].map((name) => [name, { result: "success" }]),
);
+ needs["action-ci"].outputs = { passed: "true" };
needs["classify-changes"] = { result: "success", outputs };
assert.deepEqual(failedChecks(needs), []);
for (const name of Object.keys(needs)) {
@@ -255,6 +378,9 @@ test("documentation requires static checks and explicit selection, without skipp
);
for (const key of [
"source-required",
+ "package-required",
+ "action-ci",
+ "eval-contracts",
"workflow-quality",
"container-validate",
]) {
diff --git a/.github/workflows/action-ci.yml b/.github/workflows/action-ci.yml
index 4687efce4..29f0baf6b 100644
--- a/.github/workflows/action-ci.yml
+++ b/.github/workflows/action-ci.yml
@@ -1,21 +1,38 @@
name: Action CI
on:
- push: &action-changes
+ workflow_call:
+ inputs:
+ sdk-contracts:
+ description: Run the Action release contracts when the full SDK suite is skipped.
+ type: boolean
+ default: false
+ outputs:
+ passed:
+ description: Validation, dependency audits and eligible hosted smoke tests passed.
+ value: >-
+ ${{ jobs.validate.outputs.result == 'success' &&
+ jobs.dependencies.outputs.result == 'success' &&
+ (jobs.smoke.outputs.result || 'skipped') == (github.event_name == 'pull_request' &&
+ github.event.pull_request.head.repo.full_name != github.repository && 'skipped' || 'success') }}
+ # node-ci owns PRs and main pushes. Fork branches and tags can exercise the Action directly.
+ push:
+ branches-ignore: [main]
+ tags: ["**"]
paths:
- action.yml
- github-action/**
- sdk/typescript/src/git-remote.ts
- plugins/codex-security/examples/completed-scan/**
- .github/workflows/action-ci.yml
- pull_request: *action-changes
workflow_dispatch:
permissions:
contents: read
concurrency:
- group: action-ci-${{ github.ref }}
+ # Keep each required main result. Superseded fork pushes can be cancelled.
+ group: action-ci-${{ github.event_name == 'push' && github.ref != 'refs/heads/main' && github.ref || github.run_id }}
cancel-in-progress: true
defaults:
@@ -30,6 +47,8 @@ jobs:
github.repository != 'openai/codex-security'
runs-on: ubuntu-24.04
timeout-minutes: 15
+ outputs: &action-result
+ result: ${{ job.status }}
steps:
- &actions-checkout
uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
@@ -39,6 +58,7 @@ jobs:
uses: actions/setup-node@949feb2413d6458794dcd2491c4babbbce0c15c1 # v7.1.0
with:
node-version: "24"
+ package-manager-cache: false
- &set-up-the-ubuntu-sandbox
name: Set up the Ubuntu sandbox
run: |
@@ -50,6 +70,16 @@ jobs:
sudo apparmor_parser -r /usr/share/apparmor/extra-profiles/bwrap-userns-restrict
- run: npm ci --ignore-scripts --no-audit --no-fund
- run: npm run validate
+ - name: Set up Bun for Action release contracts
+ if: inputs.sdk-contracts
+ uses: oven-sh/setup-bun@0c5077e51419868618aeaa5fe8019c62421857d6 # v2
+ with:
+ bun-version: "1.3.14"
+ no-cache: true
+ - name: Test Action release contracts
+ if: inputs.sdk-contracts
+ working-directory: sdk/typescript
+ run: bun test --timeout 30000 tests-ts/action-release-workflow.test.ts
- name: Install the pinned CLI for integration tests
run: npm ci --prefix runtime --ignore-scripts --no-audit --no-fund
- name: Published CLI result and SARIF compatibility (no model calls)
@@ -74,6 +104,7 @@ jobs:
github.event.pull_request.head.repo.full_name == github.repository)
runs-on: ubuntu-24.04
timeout-minutes: 15
+ outputs: *action-result
steps:
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
with:
@@ -121,6 +152,7 @@ jobs:
if: *run-action-ci
runs-on: ubuntu-24.04
timeout-minutes: 10
+ outputs: *action-result
steps:
- *actions-checkout
- *actions-setup-node
diff --git a/.github/workflows/node-ci.yml b/.github/workflows/node-ci.yml
index 3b6907883..11294e47c 100644
--- a/.github/workflows/node-ci.yml
+++ b/.github/workflows/node-ci.yml
@@ -25,6 +25,9 @@ jobs:
timeout-minutes: 10
outputs:
source-required: ${{ steps.scope.outputs.source-required }}
+ package-required: ${{ steps.scope.outputs.package-required }}
+ action-ci: ${{ steps.scope.outputs.action-ci }}
+ eval-contracts: ${{ steps.scope.outputs.eval-contracts }}
container-validate: ${{ steps.scope.outputs.container-validate }}
workflow-quality: ${{ steps.scope.outputs.workflow-quality }}
compatibility-matrix: ${{ steps.scope.outputs.compatibility-matrix }}
@@ -48,6 +51,13 @@ jobs:
if: needs.classify-changes.outputs.workflow-quality == 'true'
uses: ./.github/workflows/workflow-quality.yml
+ action-ci:
+ needs: classify-changes
+ if: needs.classify-changes.outputs.action-ci == 'true'
+ uses: ./.github/workflows/action-ci.yml
+ with:
+ sdk-contracts: ${{ needs.classify-changes.outputs.source-required == 'false' }}
+
static-checks:
name: typecheck and formatting
needs: classify-changes
@@ -90,7 +100,7 @@ jobs:
run: pnpm --dir sdk/typescript run audit:prod
native:
- if: needs.classify-changes.outputs.source-required == 'true'
+ if: needs.classify-changes.outputs.package-required == 'true'
needs: classify-changes
uses: ./.github/workflows/native-artifacts.yml
with:
@@ -102,7 +112,7 @@ jobs:
uses: ./.github/workflows/container-validate.yml
package:
- if: needs.classify-changes.outputs.source-required == 'true'
+ if: needs.classify-changes.outputs.package-required == 'true'
name: build and check package
needs: [classify-changes, native]
runs-on: ubuntu-latest
@@ -192,7 +202,7 @@ jobs:
retention-days: 14
compatibility:
- if: needs.classify-changes.outputs.source-required == 'true'
+ if: needs.classify-changes.outputs.package-required == 'true'
name: installed package / ${{ matrix.os }} / node-${{ matrix.node }}
needs: [classify-changes, package]
runs-on: ${{ matrix.os }}
@@ -372,7 +382,14 @@ jobs:
python -m pytest plugins/codex-security/tests -m "$PYTEST_MARKERS" -q -n 4 --dist worksteal --max-worker-restart 0 --durations=30 --junitxml=reports/python.xml
- name: Require native platform coverage
if: runner.os != 'Linux'
- run: python -c "import pathlib, xml.etree.ElementTree as ET; assert not any(ET.parse(report).findall('.//testcase/skipped') for report in pathlib.Path('reports').glob('python*.xml')), 'Native platform tests must run without skips'"
+ shell: python
+ run: |
+ from xml.etree import ElementTree
+ report = ElementTree.parse("reports/python.xml")
+ if report.find(".//testcase") is None:
+ raise SystemExit("No native platform tests ran")
+ if report.find(".//testcase/skipped") is not None:
+ raise SystemExit("Native platform tests must run without skips")
- name: Upload Python test reports
if: always()
continue-on-error: true
@@ -383,31 +400,55 @@ jobs:
path: reports/python*.xml
if-no-files-found: warn
retention-days: 14
- - name: Set up Node.js for triage evals
- if: matrix.os == 'ubuntu-latest' && matrix.python == '3.12'
+
+ eval-contracts:
+ name: deterministic eval contracts
+ needs: classify-changes
+ if: needs.classify-changes.outputs.eval-contracts == 'true'
+ runs-on: ubuntu-latest
+ timeout-minutes: 20
+ steps:
+ - *checkout-repository
+ - *set-up-bun
+ - name: Check the Codex version across the SDK, MCP app and evals
+ run: bun test sdk/typescript/tests-ts/codex-dependency-pins.test.ts
+ - uses: pnpm/action-setup@ea17c68df8912ef543352723c149a84f56e3d413 # v6.1.0
+ with:
+ package_json_file: package.json
+ - name: Set up eval Node.js and pnpm cache
uses: actions/setup-node@949feb2413d6458794dcd2491c4babbbce0c15c1 # v7.1.0
with:
node-version: "24.15.0"
- - name: Set up triage eval dependencies and host runtime
- if: matrix.os == 'ubuntu-latest' && matrix.python == '3.12'
- run: pnpm --dir evals/triage-finding run setup
+ cache: pnpm
+ cache-dependency-path: |
+ sdk/typescript/pnpm-lock.yaml
+ plugins/codex-security/mcp-app/pnpm-lock.yaml
+ evals/triage-finding/pnpm-lock.yaml
+ - &set-up-python
+ name: Set up Python
+ uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0
+ with:
+ python-version: "3.12"
+ - name: Install ripgrep
+ run: |
+ sudo apt-get update
+ sudo apt-get install --yes ripgrep
+ - name: Set up eval dependencies and host runtime
+ run: |
+ pnpm --dir sdk/typescript install --frozen-lockfile
+ pnpm --dir evals/triage-finding run setup
- name: Test deterministic triage eval contracts
- if: matrix.os == 'ubuntu-latest' && matrix.python == '3.12'
working-directory: evals/triage-finding
run: node --run test:deterministic
- name: Test secret discovery eval contracts
- if: matrix.os == 'ubuntu-latest' && matrix.python == '3.12'
run: node --experimental-strip-types --test evals/secret-discovery/test_*.mts
- name: Build host plugin for Standard eval integration
- if: matrix.os == 'ubuntu-latest' && matrix.python == '3.12'
run: |
node plugins/codex-security/mcp-app/scripts/build_mcp_app.mjs --output plugins/codex-security/mcp --native host
ln -s ../../plugins/codex-security sdk/typescript/_bundled_plugin
- name: Test Standard time-budget eval contracts
- if: matrix.os == 'ubuntu-latest' && matrix.python == '3.12'
run: node --experimental-strip-types --test evals/standard-time-budget/test_*.mts
- name: Test workflow routing eval contracts
- if: matrix.os == 'ubuntu-latest' && matrix.python == '3.12'
run: node --experimental-strip-types --test evals/workflow-routing/test_*.mts
ci:
@@ -416,6 +457,8 @@ jobs:
[
classify-changes,
static-checks,
+ action-ci,
+ eval-contracts,
native,
package,
test,
@@ -463,11 +506,7 @@ jobs:
shard: [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14]
steps:
- *checkout-repository
- - &set-up-python
- name: Set up Python
- uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0
- with:
- python-version: "3.12"
+ - *set-up-python
- *download-native
- *set-up-typescript-tools
- *set-up-bun
diff --git a/docs/ci.md b/docs/ci.md
index dc98c7ab2..e35421dc0 100644
--- a/docs/ci.md
+++ b/docs/ci.md
@@ -2,10 +2,12 @@
The `node-ci` workflow selects checks from the complete pull request diff. An
ordinary documentation-only change runs static checks and the final `ci` gate.
+Static checks format the tracked Markdown in this route. The Action README also
+runs Action CI to check generated input documentation and usage examples.
Plugin skills, references, schemas, examples, and unrecognized paths receive
-full qualification: plugin Markdown is part of the product. SDK README and
+full qualification because plugin Markdown is part of the product. SDK README and
documentation changes also receive full qualification because those files are
-required package contents; deleting or renaming them must pass archive checks.
+required package contents. Deleting or renaming them must pass archive checks.
`RELEASING.md` also receives full qualification because SDK tests validate its
canonical release recovery example.
@@ -23,11 +25,30 @@ current Node 24 for the Action adapter, macOS installation, and Windows Node 22
installation. Python runs the full suite on 3.12 and the native subsets on macOS
and Windows. All source changes still select container validation.
-Every other path selects full qualification, including workflow checks,
+PRs limited to Action metadata, wrapper implementation, tests, npm manifests,
+locks, or checked-in bundles run Action CI. It verifies a clean rebuild, runs
+the published CLI integration, audits the wrapper and pinned runtime, and
+exercises the local Action on eligible events. Fork PRs run validation and
+both audits. Same-repository PRs also run the hosted dry-run smoke. Changes to
+these Action files keep the Linux Node 24 integration against the package built
+from the PR. The focused SDK Action release contracts run when the rest of the
+SDK suite is skipped. Changes to the shared SDK Git remote parser also select
+Action CI.
+
+PRs limited to Standard time-budget eval modules, experiment configuration or
+README, or to the triage eval's scripts, assertions, README, manifest or lock,
+run the deterministic eval job and static checks. The eval job builds the SDK
+and host plugin, checks the Codex version across the SDK, MCP and triage
+lockfiles, and runs the triage, secret-discovery, Standard time-budget and
+workflow-routing suites. This job also runs on source changes, independently
+of the Python matrix.
+
+Inputs outside these routes select full qualification, including workflow checks,
containers, current Node 26, Windows Node 24 installation, and Python 3.10 and
3.14 Linux suites. This includes dependency configuration such as `.npmrc` and
`pnpm-workspace.yaml`, build tooling, Python and native code, and unknown paths
-even when mixed with ordinary source or documentation.
+even when mixed with ordinary source or documentation. Mixing Action
+implementation or eval changes with product code also selects full qualification.
Every push to `main`, nightly run, manual `node-ci` dispatch, and pull-request
retarget runs the complete version matrices. Release automation still requires
diff --git a/github-action/tests/ci-events.test.mjs b/github-action/tests/ci-events.test.mjs
new file mode 100644
index 000000000..8fe4ee16e
--- /dev/null
+++ b/github-action/tests/ci-events.test.mjs
@@ -0,0 +1,51 @@
+import assert from 'node:assert/strict';
+import { readFile } from 'node:fs/promises';
+import { test } from 'node:test';
+import { parse } from 'yaml';
+
+const workflow = parse(await readFile(new URL('../../.github/workflows/action-ci.yml', import.meta.url), 'utf8'));
+const upstream = 'openai/codex-security';
+const fork = 'example/fork';
+
+test('node-ci owns pull requests and main pushes; forks can qualify their own pushes', () => {
+ assert.ok('workflow_call' in workflow.on);
+ assert.ok('workflow_dispatch' in workflow.on);
+ assert.ok(!('pull_request' in workflow.on));
+ assert.ok(workflow.on.push['branches-ignore'].includes('main'));
+ assert.ok(workflow.on.push.tags.includes('**'));
+
+ for (const [event, repository, ref, headRepo, validate, smoke] of [
+ ['pull_request', upstream, 'refs/pull/1/merge', upstream, true, true],
+ ['pull_request', upstream, 'refs/pull/1/merge', fork, true, false],
+ ['push', upstream, 'refs/heads/main', undefined, true, true],
+ ['push', upstream, 'refs/heads/topic', undefined, false, false],
+ ['push', fork, 'refs/heads/topic', undefined, true, true],
+ ['push', fork, 'refs/tags/test', undefined, true, true],
+ ['workflow_dispatch', upstream, 'refs/heads/main', undefined, true, true],
+ ['schedule', upstream, 'refs/heads/main', undefined, true, true],
+ ]) {
+ const github = { event_name: event, repository, ref, event: { pull_request: { head: { repo: { full_name: headRepo } } } } };
+ for (const [id, expected] of [['validate', validate], ['dependencies', validate], ['smoke', smoke]]) {
+ const condition = workflow.jobs[id].if ?? 'true';
+ assert.equal(new Function('github', `return (${condition})`)(github), expected, `${event} ${repository} ${ref}: ${id}`);
+ }
+ }
+});
+
+test('the called workflow requires every eligible job to finish successfully', () => {
+ const expression = workflow.on.workflow_call.outputs.passed.value;
+ const passed = new Function('jobs', 'github', `return (${expression.slice(3, -2)})`);
+ for (const headRepo of [upstream, fork]) {
+ const github = { event_name: 'pull_request', repository: upstream, event: { pull_request: { head: { repo: { full_name: headRepo } } } } };
+ // A skipped job never sets its outputs. Only fork PR smoke may be skipped.
+ const expected = { validate: 'success', dependencies: 'success', smoke: headRepo === fork ? '' : 'success' };
+ for (const [id, expectedStatus] of Object.entries(expected)) {
+ assert.equal(workflow.jobs[id].outputs.result, '${{ job.status }}');
+ for (const status of ['success', 'failure', 'cancelled', '']) {
+ const jobs = Object.fromEntries(Object.entries({ ...expected, [id]: status }).map(([job, result]) =>
+ [job, { outputs: result ? { result } : {} }]));
+ assert.equal(passed(jobs, github), status === expectedStatus, `${headRepo}: ${id} ${status || 'skipped'}`);
+ }
+ }
+ }
+});
diff --git a/sdk/typescript/scripts/check-markdown.mjs b/sdk/typescript/scripts/check-markdown.mjs
index e709d5985..c74e1a770 100644
--- a/sdk/typescript/scripts/check-markdown.mjs
+++ b/sdk/typescript/scripts/check-markdown.mjs
@@ -10,6 +10,12 @@ const files = execFileSync(
"ls-files",
"-z",
"--",
+ "README.md",
+ "CONTRIBUTING.md",
+ "CODE_OF_CONDUCT.md",
+ ":(glob)docs/**/*.md",
+ "github-action/README.md",
+ "docker/README.md",
".github/release-notes.md",
":(glob)sdk/typescript/**/*.md",
":(glob)examples/custom-validation/*.md",
diff --git a/sdk/typescript/tests-ts/check-markdown.test.ts b/sdk/typescript/tests-ts/check-markdown.test.ts
index 611e0bf7c..1423c93f1 100644
--- a/sdk/typescript/tests-ts/check-markdown.test.ts
+++ b/sdk/typescript/tests-ts/check-markdown.test.ts
@@ -8,7 +8,7 @@ import {
writeFile,
} from "node:fs/promises";
import { tmpdir } from "node:os";
-import { join } from "node:path";
+import { dirname, join } from "node:path";
import { fileURLToPath } from "node:url";
import { gitText, runCommand } from "./support/shell.js";
@@ -31,18 +31,23 @@ test("checks tracked Markdown including leading-dash paths and skips deleted, un
);
await writeFile(join(root, ".gitignore"), "node_modules/\n");
await writeFile(join(sdk, "README.md"), "# Readme\n");
- // The previous format command checks these folders, not all repository Markdown.
- await mkdir(join(root, "github-action"));
- await writeFile(join(root, "github-action", "README.md"), invalid);
- const outsideGuides = [
- join(root, "examples", "custom-validation", "guide.md"),
- join(root, "plugins", "codex-security", "native", "guide.md"),
+ const guides = [
+ "README.md",
+ "CONTRIBUTING.md",
+ "CODE_OF_CONDUCT.md",
+ "docs/nested/guide.md",
+ "github-action/README.md",
+ "docker/README.md",
+ "examples/custom-validation/guide.md",
+ "plugins/codex-security/native/guide.md",
];
- for (const path of outsideGuides) {
- await mkdir(join(path, ".."), { recursive: true });
- await writeFile(path, invalid);
- await mkdir(join(path, "..", "nested"));
- await writeFile(join(path, "..", "nested", "outside.md"), invalid);
+ for (const path of [
+ ...guides,
+ "examples/custom-validation/nested/outside.md",
+ "plugins/codex-security/native/nested/outside.md",
+ ]) {
+ await mkdir(dirname(join(root, path)), { recursive: true });
+ await writeFile(join(root, path), invalid);
}
await writeFile(join(sdk, "--plugin=fixture.md"), "# Markdown fixture\n");
// The combined paths exceed the Windows process command-line limit.
@@ -68,13 +73,11 @@ test("checks tracked Markdown including leading-dash paths and skips deleted, un
});
expect(failing.status).toBe(1);
expect(failing.stdout + failing.stderr).toContain("guide with # spaces.md");
- for (const path of outsideGuides)
- expect(failing.stdout + failing.stderr).toContain(
- path.replaceAll("\\", "/").slice(root.length + 1),
- );
+ for (const path of guides)
+ expect(failing.stdout + failing.stderr).toContain(path);
await writeFile(guide, "# Title\n");
- for (const path of outsideGuides) await writeFile(path, "# Guide\n");
+ for (const path of guides) await writeFile(join(root, path), "# Guide\n");
const passing = await runCommand("node", [script], {
cwd: sdk,
timeout: 30_000,
diff --git a/sdk/typescript/tests-ts/codex-dependency-pins.test.ts b/sdk/typescript/tests-ts/codex-dependency-pins.test.ts
new file mode 100644
index 000000000..d68141a7e
--- /dev/null
+++ b/sdk/typescript/tests-ts/codex-dependency-pins.test.ts
@@ -0,0 +1,47 @@
+import { readFile } from "node:fs/promises";
+import { expect, test } from "bun:test";
+
+test("pins one Codex version across the CLI, MCP app, and evals", async () => {
+ const directories = [
+ "sdk/typescript",
+ "plugins/codex-security/mcp-app",
+ "evals/triage-finding",
+ ];
+ const manifests = await Promise.all(
+ directories.map(async (directory) =>
+ JSON.parse(
+ await readFile(
+ new URL(`../../../${directory}/package.json`, import.meta.url),
+ "utf8",
+ ),
+ ),
+ ),
+ );
+ const version = manifests[0].dependencies["@openai/codex"];
+ expect(version).toBeTruthy();
+ for (const [index, directory] of directories.entries()) {
+ expect(manifests[index].dependencies["@openai/codex-sdk"]).toBe(version);
+ const lockfile = Bun.YAML.parse(
+ await readFile(
+ new URL(`../../../${directory}/pnpm-lock.yaml`, import.meta.url),
+ "utf8",
+ ),
+ ) as {
+ packages: Record<string, { os?: string[]; cpu?: string[] }>;
+ };
+ expect(
+ Object.keys(lockfile.packages).filter((name) =>
+ name.startsWith("@openai/codex-sdk@"),
+ ),
+ ).toEqual([`@openai/codex-sdk@${version}`]);
+ for (const [name, metadata] of Object.entries(lockfile.packages)) {
+ if (name.startsWith("@openai/codex@")) {
+ const platform =
+ metadata.os && metadata.cpu
+ ? `-${metadata.os[0]}-${metadata.cpu[0]}`
+ : "";
+ expect(name).toBe(`@openai/codex@${version}${platform}`);
+ }
+ }
+ }
+});
diff --git a/sdk/typescript/tests-ts/container-ci-workflow.test.ts b/sdk/typescript/tests-ts/container-ci-workflow.test.ts
index 61251efcf..c4a94af03 100644
--- a/sdk/typescript/tests-ts/container-ci-workflow.test.ts
+++ b/sdk/typescript/tests-ts/container-ci-workflow.test.ts
@@ -134,6 +134,7 @@ test("the final CI gate executes selected container failures and permits unselec
const classified = {
...needs,
"classify-changes": { result: "success", outputs },
+ "action-ci": { result: "success", outputs: { passed: "true" } },
};
for (const result of ["success", "failure", "cancelled", "skipped"]) {
const execution = spawnSync(
diff --git a/sdk/typescript/tests-ts/release-automation.test.ts b/sdk/typescript/tests-ts/release-automation.test.ts
index 33b475672..61716faaf 100644
--- a/sdk/typescript/tests-ts/release-automation.test.ts
+++ b/sdk/typescript/tests-ts/release-automation.test.ts
@@ -3827,6 +3827,8 @@ describe("GitHub release workflow safeguards", () => {
name: string;
if?: string;
needs?: string | string[];
+ uses?: string;
+ outputs?: Record<string, string>;
"timeout-minutes"?: number;
steps: Array<{
if?: string;
@@ -3857,42 +3859,54 @@ describe("GitHub release workflow safeguards", () => {
});
expect(checkout).not.toHaveProperty("if");
- expect(workflow.jobs["static-checks"]?.if).toBeUndefined();
- for (const job of [
- "native",
- "package",
- "test",
- "compatibility",
- "mcp",
- "plugin-host",
- "plugin-source",
- "windows-test",
- "windows-verify",
- ]) {
- expect(workflow.jobs[job]?.if).toBe(
- "needs.classify-changes.outputs.source-required == 'true'",
- );
- expect(workflow.jobs["ci"]?.needs).toContain(job);
+ const aggregates = ["ci", "required-test", "windows"];
+ const checks = Object.entries(workflow.jobs).filter(
+ ([name]) => name !== "classify-changes" && !aggregates.includes(name),
+ );
+ expect(new Set([workflow.jobs["ci"]!.needs].flat())).toEqual(
+ new Set(["classify-changes", ...checks.map(([name]) => name)]),
+ );
+ for (const job of aggregates) {
+ expect(workflow.jobs[job]?.if).toContain("always()");
}
- for (const [name, job] of Object.entries(workflow.jobs)) {
- if (
- ["classify-changes", "ci", "required-test", "windows"].includes(name)
- ) {
- continue;
- }
- expect([job.needs].flat(), name).toContain("classify-changes");
+ expect(workflow.jobs["action-ci"]).toMatchObject({
+ uses: "./.github/workflows/action-ci.yml",
+ with: {
+ "sdk-contracts":
+ "${{ needs.classify-changes.outputs.source-required == 'false' }}",
+ },
+ });
+ for (const [name, job] of checks) {
expect(job.if ?? "", name).not.toMatch(/\b(always|failure|cancelled)\(/u);
}
- for (const job of ["ci", "required-test", "windows"]) {
- expect(workflow.jobs[job]?.if).toContain("always()");
- }
- for (const job of [
- "classify-changes",
- "static-checks",
- "workflow-quality",
- "container-validate",
- ]) {
- expect(workflow.jobs["ci"]?.needs).toContain(job);
+
+ // With no job results, the gate must report exactly the jobs enabled in YAML.
+ const { failedChecks } = (await import(
+ new URL("../../../.github/scripts/check-ci-results.mjs", import.meta.url)
+ .href
+ )) as { failedChecks(needs: unknown): string[] };
+ const classifierOutputs = classifier.outputs!;
+ const flags = Object.keys(classifierOutputs).filter(
+ (key) => !key.endsWith("-matrix"),
+ );
+ for (const enabled of flags) {
+ expect(classifierOutputs[enabled]).toBe(
+ `\${{ steps.scope.outputs.${enabled} }}`,
+ );
+ const outputs = Object.fromEntries(
+ flags.map((key) => [key, String(key === enabled)]),
+ );
+ const scheduled = checks.filter(
+ ([, job]) =>
+ !job.if ||
+ job.if === `needs.classify-changes.outputs.${enabled} == 'true'`,
+ );
+ expect(
+ new Set(
+ failedChecks({ "classify-changes": { result: "success", outputs } }),
+ ),
+ enabled,
+ ).toEqual(new Set(scheduled.map(([name]) => name)));
}
expect(
workflow.jobs["ci"]?.steps.find(
@@ -3906,7 +3920,9 @@ describe("GitHub release workflow safeguards", () => {
const conditionCases = [];
for (const name of ["required-test", "windows"]) {
const job = workflow.jobs[name]!;
- expect(job.needs).toEqual(["classify-changes", "ci"]);
+ expect(new Set([job.needs].flat())).toEqual(
+ new Set(["classify-changes", "ci"]),
+ );
const gate = job.steps.find(({ name }) => name === "Require CI gate");
expect(gate?.run).toBe("exit 1");
for (const result of ["success", "failure", "cancelled", "skipped"]) {
diff --git a/sdk/typescript/tests-ts/skeleton.test.ts b/sdk/typescript/tests-ts/skeleton.test.ts
index df914c043..c4aa6eab0 100644
--- a/sdk/typescript/tests-ts/skeleton.test.ts
+++ b/sdk/typescript/tests-ts/skeleton.test.ts
@@ -1,4 +1,4 @@
-import { mkdtemp, readFile, rm } from "node:fs/promises";
+import { mkdir, mkdtemp, readFile, rm, writeFile } from "node:fs/promises";
import { tmpdir } from "node:os";
import { join } from "node:path";
import { describe, expect, test } from "bun:test";
@@ -12,7 +12,7 @@ import {
} from "../src/index.js";
import { main } from "../src/cli.js";
import { capture } from "./cli-fixtures.js";
-import { bashCommand, runCommand } from "./support/shell.js";
+import { bashCommand, pythonExecutable, runCommand } from "./support/shell.js";
const { selectChecks } = (await import(
new URL("../../../.github/scripts/select-ci-checks.mjs", import.meta.url).href
@@ -33,6 +33,7 @@ interface WorkflowStep {
with?: Record<string, unknown>;
run?: string;
if?: string;
+ shell?: string;
env?: Record<string, unknown>;
"continue-on-error"?: boolean;
}
@@ -59,51 +60,6 @@ async function workflow(name: string) {
}
describe("TypeScript package skeleton", () => {
- test("pins one Codex version across the CLI, MCP app, and evals", async () => {
- const directories = [
- "sdk/typescript",
- "plugins/codex-security/mcp-app",
- "evals/triage-finding",
- ];
- const manifests = await Promise.all(
- directories.map(async (directory) =>
- JSON.parse(
- await readFile(
- new URL(`../../../${directory}/package.json`, import.meta.url),
- "utf8",
- ),
- ),
- ),
- );
- const version = manifests[0].dependencies["@openai/codex"];
- expect(version).toBeTruthy();
- for (const [index, directory] of directories.entries()) {
- expect(manifests[index].dependencies["@openai/codex-sdk"]).toBe(version);
- const lockfile = Bun.YAML.parse(
- await readFile(
- new URL(`../../../${directory}/pnpm-lock.yaml`, import.meta.url),
- "utf8",
- ),
- ) as {
- packages: Record<string, { os?: string[]; cpu?: string[] }>;
- };
- expect(
- Object.keys(lockfile.packages).filter((name) =>
- name.startsWith("@openai/codex-sdk@"),
- ),
- ).toEqual([`@openai/codex-sdk@${version}`]);
- for (const [name, metadata] of Object.entries(lockfile.packages)) {
- if (name.startsWith("@openai/codex@")) {
- const platform =
- metadata.os && metadata.cpu
- ? `-${metadata.os[0]}-${metadata.cpu[0]}`
- : "";
- expect(name).toBe(`@openai/codex@${version}${platform}`);
- }
- }
- }
- });
-
test("exports typed attack-path aliases", () => {
const dataflow: AttackPathDataflow = {
transformations: ["decode archive entry"],
@@ -199,24 +155,6 @@ describe("TypeScript package skeleton", () => {
verificationSteps[pythonSetup]?.with?.["update-environment"],
);
expect(jobs["required-test"]?.name).toEndWith("${{ matrix.os }} / node-22");
- expect(jobs["ci"]?.needs).toEqual([
- "classify-changes",
- "static-checks",
- "native",
- "package",
- "test",
- "compatibility",
- "mcp",
- "plugin-host",
- "plugin-source",
- "container-validate",
- "workflow-quality",
- "windows-test",
- "windows-verify",
- ]);
- for (const name of ["required-test", "windows"]) {
- expect(jobs[name]?.needs).toEqual(["classify-changes", "ci"]);
- }
for (const name of ["compatibility", "windows-verify"]) {
expect(jobs[name]?.steps).toContainEqual(
expect.objectContaining({
@@ -361,24 +299,80 @@ describe("TypeScript package skeleton", () => {
);
expect(testStep).not.toHaveProperty("if");
expect(testStep).not.toHaveProperty("continue-on-error");
- const nativeCoverageStep = job.steps!.find(
+ expect(
+ job.steps!.find(({ name }) => name === "Install plugin dependencies")?.if,
+ ).toBe("matrix.os == 'ubuntu-latest' && matrix.python == '3.12'");
+ });
+
+ test("requires a nonempty native Python report without skips", async () => {
+ const { jobs } = await workflow("node-ci.yml");
+ const step = jobs["plugin-source"]!.steps!.find(
({ name }) => name === "Require native platform coverage",
)!;
- expect(nativeCoverageStep.if).toBe("runner.os != 'Linux'");
- expect(nativeCoverageStep.run).toContain(".//testcase/skipped");
- expect(nativeCoverageStep).not.toHaveProperty("continue-on-error");
- for (const name of [
- "Install plugin dependencies",
- "Set up Node.js for triage evals",
- "Set up triage eval dependencies and host runtime",
+ expect(step).toMatchObject({ if: "runner.os != 'Linux'", shell: "python" });
+ expect(step).not.toHaveProperty("continue-on-error");
+ const directory = await mkdtemp(join(tmpdir(), "native-python-report-"));
+ try {
+ const reports = join(directory, "reports");
+ await mkdir(reports);
+ const passing = "<testsuite><testcase/></testsuite>";
+ await writeFile(join(reports, "python-stale.xml"), passing);
+ for (const [report, expected] of [
+ [null, 1],
+ ["", 1],
+ ["<testsuite/>", 1],
+ [
+ "<testsuite><testcase/><testcase><skipped/></testcase></testsuite>",
+ 1,
+ ],
+ [passing, 0],
+ ] as const) {
+ if (report !== null)
+ await writeFile(join(reports, "python.xml"), report);
+ const result = await runCommand(
+ pythonExecutable()!,
+ ["-c", step.run!],
+ { cwd: directory },
+ );
+ expect(result.status, result.stderr).toBe(expected);
+ }
+ } finally {
+ await rm(directory, { recursive: true, force: true });
+ }
+ });
+
+ test("requires the deterministic eval suites and their cross-project dependency pin", async () => {
+ const { jobs } = await workflow("node-ci.yml");
+ const requiredSteps = jobs["eval-contracts"]!.steps!.filter(
+ (step) => !step.if && !step["continue-on-error"],
+ );
+ expect(requiredSteps).toContainEqual(
+ expect.objectContaining({
+ "working-directory": "evals/triage-finding",
+ run: "node --run test:deterministic",
+ }),
+ );
+ const commands = requiredSteps.map((step) => step.run).join("\n");
+ for (const suite of [
+ "secret-discovery",
+ "standard-time-budget",
+ "workflow-routing",
]) {
- expect(job.steps!.find((step) => step.name === name)?.if).toBe(
- "matrix.os == 'ubuntu-latest' && matrix.python == '3.12'",
- );
+ expect(commands).toContain(`--test evals/${suite}/test_*.mts`);
}
- expect(jobs["ci"]?.needs).toContain("plugin-source");
- expect(jobs["required-test"]?.needs).toContain("ci");
- expect(jobs["windows"]?.needs).toContain("ci");
+ expect(commands).toContain(
+ "bun test sdk/typescript/tests-ts/codex-dependency-pins.test.ts",
+ );
+ const cachePaths = requiredSteps.find(
+ (step) => step.with?.["cache"] === "pnpm",
+ )?.with?.["cache-dependency-path"];
+ expect(String(cachePaths).trim().split("\n")).toEqual(
+ expect.arrayContaining([
+ "evals/triage-finding/pnpm-lock.yaml",
+ "plugins/codex-security/mcp-app/pnpm-lock.yaml",
+ "sdk/typescript/pnpm-lock.yaml",
+ ]),
+ );
});
test("checks one archive and restores its plugin before every test shard", async () => {