Skip to content

OpenAI's Codex Security CLI and TypeScript SDK for finding, validating, and fixing security vulnerabilities. npm: https://www.npmjs.com/package/@openai/codex-security

ci: run rotating properties with source fixtures (#2022)

Michael D'Angelo5124c5e9b042
diff --git a/.github/workflows/test-quality.yml b/.github/workflows/test-quality.yml
index 41e794204..ffca3bfed 100644
--- a/.github/workflows/test-quality.yml
+++ b/.github/workflows/test-quality.yml
@@ -30,10 +30,8 @@ env:
jobs:
native:
- if: ${{ !inputs.native-artifacts-ready }}
+ if: inputs.compare-layouts && !inputs.native-artifacts-ready
uses: ./.github/workflows/native-artifacts.yml
- with:
- reuse-verified: ${{ !inputs.compare-layouts }}
runner:
needs: native
@@ -89,7 +87,7 @@ jobs:
- { os: windows-latest, shard: 12, count: 14, mode: expanded }
- { os: windows-latest, shard: 13, count: 14, mode: expanded }
- { os: windows-latest, shard: 14, count: 14, mode: expanded }
- steps: &sdk-test-steps
+ steps:
- &actions-checkout
uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
with:
@@ -116,7 +114,8 @@ jobs:
run: |
pnpm --dir sdk/typescript install --frozen-lockfile
pnpm --dir plugins/codex-security/mcp-app install --frozen-lockfile
- - name: Prepare private Windows test root
+ - &prepare-windows-test-root
+ name: Prepare private Windows test root
if: runner.os == 'Windows'
id: windows-temp
shell: pwsh
@@ -127,23 +126,14 @@ jobs:
if: runner.os == 'Linux'
run: .github/scripts/install-ripgrep.sh
- name: Test shard layout
- if: inputs.compare-layouts
working-directory: sdk/typescript
env: &test-temp
TEMP: ${{ steps.windows-temp.outputs.path || runner.temp }}
TMP: ${{ steps.windows-temp.outputs.path || runner.temp }}
TMPDIR: ${{ steps.windows-temp.outputs.path || runner.temp }}
run: node --experimental-strip-types scripts/run-ci-tests.mts ${{ env.SHARD }} ${{ runner.os == 'Linux' && '--coverage --coverage-reporter=lcov' || '' }} --coverage-dir=reports/coverage-${{ env.REPORT_ID }} --seed=${{ env.CODEX_SECURITY_PROPERTY_SEED }} --reporter-outfile=reports/runner-${{ env.REPORT_ID }}.xml
- - name: Test rotating properties
- if: ${{ !inputs.compare-layouts }}
- working-directory: sdk/typescript
- shell: bash
- env: *test-temp
- run: |
- git grep -lzF 'support/property' -- 'tests-ts/*.test.ts' > "$RUNNER_TEMP/property-tests"
- mkdir -p reports
- xargs -0 bun test --timeout="$CODEX_SECURITY_TEST_TIMEOUT_MS" --seed="$CODEX_SECURITY_PROPERTY_SEED" --reporter=junit --reporter-outfile=reports/properties.xml < "$RUNNER_TEMP/property-tests"
- - name: Upload runner report
+ - &upload-runner-report
+ name: Upload runner report
if: always()
uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2
with:
@@ -155,8 +145,7 @@ jobs:
properties:
name: ${{ matrix.os }} / rotating properties
- needs: native
- if: ${{ !cancelled() && !inputs.compare-layouts && (inputs.native-artifacts-ready || needs.native.result == 'success') }}
+ if: ${{ !inputs.compare-layouts }}
runs-on: ${{ matrix.os }}
timeout-minutes: 20
env:
@@ -167,7 +156,27 @@ jobs:
fail-fast: false
matrix:
os: [ubuntu-latest, macos-latest, windows-latest]
- steps: *sdk-test-steps
+ steps:
+ - *actions-checkout
+ - &set-up-sdk-tools
+ name: Set up TypeScript tools
+ uses: ./.github/actions/setup-tools
+ - *set-up-bun
+ - &install-sdk-dependencies
+ name: Install dependencies
+ run: pnpm --dir sdk/typescript install --frozen-lockfile
+ - *prepare-windows-test-root
+ - name: Test rotating properties
+ working-directory: sdk/typescript
+ shell: bash
+ env: *test-temp
+ run: |
+ git grep -lzF 'support/property' -- 'tests-ts/*.test.ts' > "$RUNNER_TEMP/property-tests"
+ mkdir -p reports _bundled_plugin/examples
+ cp -R ../../plugins/codex-security/examples/completed-scan _bundled_plugin/examples/
+ cp -R ../../plugins/codex-security/schemas _bundled_plugin/
+ xargs -0 bun test --timeout="$CODEX_SECURITY_TEST_TIMEOUT_MS" --seed="$CODEX_SECURITY_PROPERTY_SEED" --reporter=junit --reporter-outfile=reports/properties.xml < "$RUNNER_TEMP/property-tests"
+ - *upload-runner-report
compare:
name: Runner inventory and timing
@@ -176,10 +185,8 @@ jobs:
runs-on: ubuntu-latest
steps:
- *actions-checkout
- - name: Set up TypeScript tools
- uses: ./.github/actions/setup-tools
- - name: Install dependencies
- run: pnpm --dir sdk/typescript install --frozen-lockfile
+ - *set-up-sdk-tools
+ - *install-sdk-dependencies
- uses: actions/download-artifact@9000827ccba6bdab643e8b6fd33ac0654aef8333 # v8.0.2
with:
pattern: runner-*
@@ -223,8 +230,7 @@ jobs:
# Stryker 10's Babel 8 requires Node 22.18+ or 24.11+.
node-version: "24.15.0"
- *set-up-bun
- - name: Install dependencies
- run: pnpm --dir sdk/typescript install --frozen-lockfile
+ - *install-sdk-dependencies
- name: Run mutation trial
run: pnpm --dir sdk/typescript run test:mutation
- name: Upload mutation report
diff --git a/sdk/typescript/TESTING.md b/sdk/typescript/TESTING.md
index 2ed9733cf..59da9ed72 100644
--- a/sdk/typescript/TESTING.md
+++ b/sdk/typescript/TESTING.md
@@ -199,7 +199,8 @@ do not. Use `python -m pytest -n 0` to reproduce Python failures serially.
Each week `test-quality` runs the mutation trial on Linux. On Linux, macOS, and
Windows it runs the test files that import the shared property options, using
-the workflow run number as the property seed. Each platform uploads its JUnit
+the workflow run number as the property seed. The filesystem properties read
+the plugin's checked-in schemas and example scan. Each platform uploads its JUnit
report. To replay the weekly jobs, dispatch with `compare-layouts` disabled.
Ordinary source pull requests run focused contracts for the shard runner, timing
metadata, report comparator, and Windows test-root helper in the required SDK suite.
diff --git a/sdk/typescript/tests-ts/native-artifact-cache.test.ts b/sdk/typescript/tests-ts/native-artifact-cache.test.ts
index 4a856ec2b..6ec501193 100644
--- a/sdk/typescript/tests-ts/native-artifact-cache.test.ts
+++ b/sdk/typescript/tests-ts/native-artifact-cache.test.ts
@@ -246,16 +246,13 @@ test.each([
expect(checked.status === 0, checked.stderr).toBe(result.passes);
});
-test("source CI and rotating properties opt in to native artifact reuse", () => {
+test("source CI opts in to native artifact reuse", () => {
expect(native.on.workflow_call?.inputs["reuse-verified"]?.default).toBe(
false,
);
expect(workflow("node-ci").jobs["native"]?.with?.["reuse-verified"]).toBe(
true,
);
- expect(
- workflow("test-quality").jobs["native"]?.with?.["reuse-verified"],
- ).toBe("${{ !inputs.compare-layouts }}");
for (const caller of ["node-release", "container-release", "container-ci"]) {
expect(
workflow(caller).jobs["native"]?.with?.["reuse-verified"],