Skip to content

OpenAI's Codex Security CLI and TypeScript SDK for finding, validating, and fixing security vulnerabilities. npm: https://www.npmjs.com/package/@openai/codex-security

test: cover the Python runtime floor and native platforms (#1265)

mldangelo-oaie0b90bff6a37
diff --git a/.github/workflows/node-ci.yml b/.github/workflows/node-ci.yml
index 0b500d6d3..3f41e68ef 100644
--- a/.github/workflows/node-ci.yml
+++ b/.github/workflows/node-ci.yml
@@ -419,21 +419,42 @@ jobs:
run: node --experimental-strip-types --test plugins/codex-security/mcp-app/scripts/test_host_build.mts
plugin-source:
- name: plugin source contracts
+ name: plugin source contracts / ${{ matrix.os }} / python-${{ matrix.python }}
needs: validate-title
if: needs.validate-title.outputs.ci-mode == 'full'
- runs-on: ubuntu-latest
+ runs-on: ${{ matrix.os }}
timeout-minutes: 30
+ strategy:
+ fail-fast: false
+ matrix:
+ include:
+ - os: ubuntu-latest
+ python: "3.10"
+ tests: plugins/codex-security/tests
+ - os: ubuntu-latest
+ python: "3.12"
+ tests: plugins/codex-security/tests
+ - os: ubuntu-latest
+ python: "3.14"
+ tests: plugins/codex-security/tests
+ - os: macos-latest
+ python: "3.12"
+ tests: plugins/codex-security/tests/test_workbench_scan_usage.py
+ - os: windows-latest
+ python: "3.12"
+ tests: plugins/codex-security/tests/test_windows_scan_local_files.py
steps:
- name: Checkout repository
uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
with:
persist-credentials: false
- name: Set up Node.js
+ if: matrix.os == 'ubuntu-latest' && matrix.python == '3.12'
uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 # v7.0.0
with:
node-version: "22.13.0"
- name: Set up pnpm
+ if: matrix.os == 'ubuntu-latest' && matrix.python == '3.12'
uses: ./.github/actions/setup-tools
with:
setup-node: false
@@ -442,44 +463,57 @@ jobs:
sdk/typescript/pnpm-lock.yaml
plugins/codex-security/mcp-app/pnpm-lock.yaml
- name: Install plugin dependencies
+ if: matrix.os == 'ubuntu-latest' && matrix.python == '3.12'
run: pnpm --dir plugins/codex-security/mcp-app install --frozen-lockfile
- name: Set up Python
uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0
with:
- python-version: "3.12"
+ python-version: ${{ matrix.python }}
cache: pip
cache-dependency-path: plugins/codex-security/pyproject.toml
- name: Install Python test dependencies
run: python -m pip install --disable-pip-version-check --no-input -e 'plugins/codex-security[test]'
- name: Install ripgrep
+ if: runner.os == 'Linux'
run: |
sudo apt-get update
sudo apt-get install --yes ripgrep
- name: Check plugin source compatibility
+ if: matrix.os == 'ubuntu-latest' && matrix.python == '3.12'
run: |
python -m ruff check --config plugins/codex-security/pyproject.toml plugins/codex-security
python -m ruff format --check --config plugins/codex-security/pyproject.toml plugins/codex-security
node .github/scripts/check_plugin_source_compatibility.mjs
- name: Test source compatibility checker
+ if: matrix.os == 'ubuntu-latest' && matrix.python == '3.12'
run: node --test .github/scripts/test_check_plugin_source_compatibility.mjs
- name: Test Python source contracts
- run: python -m pytest plugins/codex-security/tests -q -n 4 --dist worksteal --max-worker-restart 0 --durations=30 --junitxml=reports/python.xml
+ shell: bash
+ env:
+ PYTHON_TEST_PATH: ${{ matrix.tests }}
+ run: python -m pytest "$PYTHON_TEST_PATH" -q -n 4 --dist worksteal --max-worker-restart 0 --durations=30 --junitxml=reports/python.xml
+ - name: Require native platform coverage
+ if: runner.os != 'Linux'
+ run: python -c "import xml.etree.ElementTree as ET; assert not ET.parse('reports/python.xml').findall('.//testcase/skipped'), 'Native platform tests must run without skips'"
- name: Upload Python test reports
if: always()
continue-on-error: true
uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
with:
- name: python-test-reports
+ name: python-test-reports-${{ matrix.os }}-${{ matrix.python }}
overwrite: true
path: reports/python.xml
if-no-files-found: warn
retention-days: 14
- name: Build SDK and type-check eval tooling
+ if: matrix.os == 'ubuntu-latest' && matrix.python == '3.12'
run: pnpm --dir sdk/typescript run build:evals
- name: Test deterministic triage eval contracts
+ if: matrix.os == 'ubuntu-latest' && matrix.python == '3.12'
working-directory: evals/triage-finding
run: node --run test:deterministic
- name: Test secret discovery eval contracts
+ if: matrix.os == 'ubuntu-latest' && matrix.python == '3.12'
run: node --experimental-strip-types --test evals/secret-discovery/test_*.mts
required-test:
@@ -631,12 +665,19 @@ jobs:
runs-on: ubuntu-latest
if: always()
needs:
- [validate-title, static-checks, plugin-host, windows-test, windows-verify]
+ [
+ validate-title,
+ static-checks,
+ plugin-host,
+ plugin-source,
+ windows-test,
+ windows-verify,
+ ]
strategy:
fail-fast: false
matrix:
node: ["22.13.0", "24"]
steps:
- name: Require every Windows coverage job
- if: needs.validate-title.result != 'success' || (needs.validate-title.outputs.ci-mode == 'full' && (needs.static-checks.result != 'success' || needs.plugin-host.result != 'success' || needs.windows-test.result != 'success' || needs.windows-verify.result != 'success')) || (needs.validate-title.outputs.ci-mode != 'full' && needs.validate-title.outputs.ci-mode != 'markdown')
+ if: needs.validate-title.result != 'success' || (needs.validate-title.outputs.ci-mode == 'full' && (needs.static-checks.result != 'success' || needs.plugin-host.result != 'success' || needs.plugin-source.result != 'success' || needs.windows-test.result != 'success' || needs.windows-verify.result != 'success')) || (needs.validate-title.outputs.ci-mode != 'full' && needs.validate-title.outputs.ci-mode != 'markdown')
run: exit 1
diff --git a/plugins/codex-security/pyproject.toml b/plugins/codex-security/pyproject.toml
index 8deafb8e3..501a11d31 100644
--- a/plugins/codex-security/pyproject.toml
+++ b/plugins/codex-security/pyproject.toml
@@ -1,7 +1,7 @@
[project]
name = "codex-security"
version = "0.1.0"
-requires-python = ">=3.12"
+requires-python = ">=3.10"
[project.optional-dependencies]
test = [
@@ -19,7 +19,7 @@ testpaths = ["tests"]
[tool.ruff]
line-length = 100
required-version = "==0.16.9"
-target-version = "py312"
+target-version = "py310"
[tool.ruff.lint]
select = [
diff --git a/plugins/codex-security/tests/test_finalize_scan_contract.py b/plugins/codex-security/tests/test_finalize_scan_contract.py
index 2b458330e..cbf4d619c 100644
--- a/plugins/codex-security/tests/test_finalize_scan_contract.py
+++ b/plugins/codex-security/tests/test_finalize_scan_contract.py
@@ -263,8 +263,9 @@ The extraction root is not enforced.
completed_at = manifest["scan"]["completedAt"]
self.assertEqual(manifest["scan"]["startedAt"], started_at)
self.assertTrue(completed_at.endswith("Z"))
- self.assertGreaterEqual(datetime.fromisoformat(completed_at), before)
- self.assertLessEqual(datetime.fromisoformat(completed_at), datetime.now(timezone.utc))
+ completed_datetime = datetime.fromisoformat(completed_at.replace("Z", "+00:00"))
+ self.assertGreaterEqual(completed_datetime, before)
+ self.assertLessEqual(completed_datetime, datetime.now(timezone.utc))
self.assertEqual(manifest["scan"]["sealedAt"], completed_at)
def test_headless_finalization_rejects_invalid_authoritative_start(self) -> None:
diff --git a/plugins/codex-security/tests/test_generate_rank_input.py b/plugins/codex-security/tests/test_generate_rank_input.py
index fdcf39a6f..07ca2d187 100644
--- a/plugins/codex-security/tests/test_generate_rank_input.py
+++ b/plugins/codex-security/tests/test_generate_rank_input.py
@@ -26,7 +26,7 @@ def run_cli(*args: str, check: bool = True) -> subprocess.CompletedProcess[str]:
def test_cli_loads_preview_helper_with_safe_path() -> None:
result = subprocess.run(
- [sys.executable, "-P", str(SCRIPT), "--help"],
+ [sys.executable, "-I", str(SCRIPT), "--help"],
check=True,
capture_output=True,
text=True,
diff --git a/plugins/codex-security/tests/test_rank_preview.py b/plugins/codex-security/tests/test_rank_preview.py
index 4eb740368..829800144 100644
--- a/plugins/codex-security/tests/test_rank_preview.py
+++ b/plugins/codex-security/tests/test_rank_preview.py
@@ -717,12 +717,14 @@ second_runtime_line()
"prefix", ["", "def visible():\n pass\n"], ids=["sampled-source", "simple-outline"]
)
def test_python_preview_falls_back_on_ast_recursion(tmp_path: Path, prefix: str) -> None:
- source = prefix + "value = " + " + ".join(["x"] * 10000) + "\n"
+ source = prefix + "value = 1\n"
path = tmp_path / "generated.py"
data = source.encode("utf-8")
path.write_bytes(data)
- preview, is_binary = preview_for(path, DEFAULT_PREVIEW_BYTES)
+ with patch("rank_preview.ast.parse", side_effect=RecursionError):
+ preview, is_binary = preview_for(path, DEFAULT_PREVIEW_BYTES)
+ assert preview_for_bytes(path, data, DEFAULT_PREVIEW_BYTES) == (preview, False)
assert not is_binary
assert preview
@@ -731,7 +733,6 @@ def test_python_preview_falls_back_on_ast_recursion(tmp_path: Path, prefix: str)
assert preview == "function visible"
else:
assert source.startswith(preview)
- assert preview_for_bytes(path, data, DEFAULT_PREVIEW_BYTES) == (preview, False)
def test_fallback_preview_uses_head_and_evenly_sampled_nonblank_lines(tmp_path: Path) -> None:
diff --git a/plugins/codex-security/tests/test_workbench_scan_usage.py b/plugins/codex-security/tests/test_workbench_scan_usage.py
index 06b7ea425..7d76b3cad 100644
--- a/plugins/codex-security/tests/test_workbench_scan_usage.py
+++ b/plugins/codex-security/tests/test_workbench_scan_usage.py
@@ -99,7 +99,7 @@ def _start_scan(tmp_path: Path, *, mode: str = "standard") -> ScanFixture:
target,
str(started["scanId"]),
Path(str(started["scanDir"])),
- datetime.fromisoformat(row[0]),
+ datetime.fromisoformat(row[0].replace("Z", "+00:00")),
environment,
mode,
diff_target,
@@ -561,7 +561,7 @@ def test_completion_counts_deep_sdk_workers_and_descendants(tmp_path: Path) -> N
target,
scan_id,
scan_dir,
- datetime.fromisoformat(row[0]),
+ datetime.fromisoformat(row[0].replace("Z", "+00:00")),
environment,
"deep",
)
diff --git a/plugins/codex-security/tests/test_workbench_target.py b/plugins/codex-security/tests/test_workbench_target.py
index 94d6871e3..8c9e24051 100644
--- a/plugins/codex-security/tests/test_workbench_target.py
+++ b/plugins/codex-security/tests/test_workbench_target.py
@@ -27,6 +27,17 @@ def initialize_unborn_git_repository(target: Path) -> None:
subprocess.run(["git", "init", "-q"], cwd=target, check=True)
+def set_default_subprocess_encoding(monkeypatch: pytest.MonkeyPatch, encoding: str) -> None:
+ run = subprocess.run
+
+ def run_with_default_encoding(*args: Any, **kwargs: Any):
+ if kwargs.get("text") and kwargs.get("encoding") is None:
+ kwargs["encoding"] = encoding
+ return run(*args, **kwargs)
+
+ monkeypatch.setattr(subprocess, "run", run_with_default_encoding)
+
+
def test_stale_git_binding_does_not_spawn(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None:
monkeypatch.setenv("CODEX_SECURITY_GIT", str(tmp_path / "missing-git"))
@@ -61,7 +72,7 @@ def test_git_metadata_preserves_unicode_commit_subject(
subprocess.run(
["git", "config", "i18n.logOutputEncoding", log_encoding], cwd=target, check=True
)
- monkeypatch.setattr(subprocess, "_text_encoding", lambda: encoding)
+ set_default_subprocess_encoding(monkeypatch, encoding)
assert WORKBENCH_TARGET["git_target_metadata"](target)["commitSubject"] == subject
assert WORKBENCH_TARGET["git_bytes"](
@@ -75,7 +86,7 @@ def test_git_output_decodes_repository_paths_as_utf8(
) -> None:
target = tmp_path / "Jos\u00e9-\u65e5\u672c\u8a9e-\ud55c\uad6d\uc5b4"
initialize_git_repository(target)
- monkeypatch.setattr(subprocess, "_text_encoding", lambda: encoding)
+ set_default_subprocess_encoding(monkeypatch, encoding)
output = WORKBENCH_TARGET["git_output"](target, "rev-parse", "--show-toplevel")
assert Path(output) == target
diff --git a/sdk/typescript/tests-ts/release-automation.test.ts b/sdk/typescript/tests-ts/release-automation.test.ts
index 73cea1aaf..d06a8a9dc 100644
--- a/sdk/typescript/tests-ts/release-automation.test.ts
+++ b/sdk/typescript/tests-ts/release-automation.test.ts
@@ -4089,6 +4089,7 @@ describe("GitHub release workflow safeguards", () => {
windows: [
"static-checks",
"plugin-host",
+ "plugin-source",
"windows-test",
"windows-verify",
],
diff --git a/sdk/typescript/tests-ts/skeleton.test.ts b/sdk/typescript/tests-ts/skeleton.test.ts
index bd9dc2510..505910734 100644
--- a/sdk/typescript/tests-ts/skeleton.test.ts
+++ b/sdk/typescript/tests-ts/skeleton.test.ts
@@ -190,6 +190,7 @@ describe("TypeScript package skeleton", () => {
"validate-title",
"static-checks",
"plugin-host",
+ "plugin-source",
"windows-test",
"windows-verify",
]);
@@ -225,6 +226,45 @@ describe("TypeScript package skeleton", () => {
);
});
+ test("covers the Python runtime floor and native platform paths", async () => {
+ const { jobs } = await workflow("node-ci.yml");
+ const job = jobs["plugin-source"]!;
+ expect(job.strategy?.matrix["include"]).toEqual([
+ ...["3.10", "3.12", "3.14"].map((python) => ({
+ os: "ubuntu-latest",
+ python,
+ tests: "plugins/codex-security/tests",
+ })),
+ {
+ os: "macos-latest",
+ python: "3.12",
+ tests: "plugins/codex-security/tests/test_workbench_scan_usage.py",
+ },
+ {
+ os: "windows-latest",
+ python: "3.12",
+ tests: "plugins/codex-security/tests/test_windows_scan_local_files.py",
+ },
+ ]);
+ const testStep = job.steps!.find(
+ ({ name }) => name === "Test Python source contracts",
+ )!;
+ expect(testStep.env?.["PYTHON_TEST_PATH"]).toBe("${{ matrix.tests }}");
+ expect(testStep.run).toContain('python -m pytest "$PYTHON_TEST_PATH"');
+ expect(testStep).not.toHaveProperty("if");
+ expect(testStep).not.toHaveProperty("continue-on-error");
+ for (const name of [
+ "Install plugin dependencies",
+ "Build SDK and type-check eval tooling",
+ ]) {
+ expect(job.steps!.find((step) => step.name === name)?.if).toBe(
+ "matrix.os == 'ubuntu-latest' && matrix.python == '3.12'",
+ );
+ }
+ expect(jobs["required-test"]?.needs).toContain("plugin-source");
+ expect(jobs["windows"]?.needs).toContain("plugin-source");
+ });
+
test("checks one archive and restores its plugin before every test shard", async () => {
const { jobs } = await workflow("node-ci.yml");
const uploads = jobs["package"]!.steps!;