Skip to content

OpenAI's Codex Security CLI and TypeScript SDK for finding, validating, and fixing security vulnerabilities. npm: https://www.npmjs.com/package/@openai/codex-security

test: check native Python reports in the Python matrix (#1950)

Michael D'Angelo65b96e8ead95
diff --git a/.github/scripts/require_native_python_report.py b/.github/scripts/require_native_python_report.py
new file mode 100644
index 000000000..54e04ff40
--- /dev/null
+++ b/.github/scripts/require_native_python_report.py
@@ -0,0 +1,7 @@
+from xml.etree import ElementTree
+
+report = ElementTree.parse("reports/python.xml")
+if report.find(".//testcase") is None:
+ raise SystemExit("No native platform tests ran")
+if report.find(".//testcase/skipped") is not None:
+ raise SystemExit("Native platform tests must run without skips")
diff --git a/.github/scripts/test_require_native_python_report.py b/.github/scripts/test_require_native_python_report.py
new file mode 100644
index 000000000..2b1e3b0ff
--- /dev/null
+++ b/.github/scripts/test_require_native_python_report.py
@@ -0,0 +1,33 @@
+from contextlib import nullcontext
+from pathlib import Path
+from runpy import run_path
+from xml.etree.ElementTree import ParseError
+
+import pytest
+
+CHECK = Path(__file__).with_name("require_native_python_report.py")
+PASSING = "<testsuite><testcase/></testsuite>"
+
+
+ ("report", "outcome"),
+ [
+ (None, pytest.raises(FileNotFoundError, match="python.xml")),
+ ("", pytest.raises(ParseError)),
+ ("<testsuite/>", pytest.raises(SystemExit, match="No native platform tests ran")),
+ (
+ "<testsuite><testcase/><testcase><skipped/></testcase></testsuite>",
+ pytest.raises(SystemExit, match="Native platform tests must run without skips"),
+ ),
+ (PASSING, nullcontext()),
+ ],
+)
+def test_native_python_report(tmp_path, monkeypatch, report, outcome):
+ monkeypatch.chdir(tmp_path)
+ reports = Path("reports")
+ reports.mkdir()
+ (reports / "python-stale.xml").write_text(PASSING)
+ if report is not None:
+ (reports / "python.xml").write_text(report)
+ with outcome:
+ run_path(str(CHECK), run_name="__main__")
diff --git a/.github/workflows/node-ci.yml b/.github/workflows/node-ci.yml
index edc2bd1af..71213c9cc 100644
--- a/.github/workflows/node-ci.yml
+++ b/.github/workflows/node-ci.yml
@@ -371,6 +371,9 @@ jobs:
- name: Test source compatibility checker
if: matrix.os == 'ubuntu-latest' && matrix.python == '3.12'
run: node --test .github/scripts/test_check_plugin_source_compatibility.mjs
+ - name: Test the native Python report gate
+ if: matrix.python == '3.12'
+ run: python -m pytest -q .github/scripts/test_require_native_python_report.py
- name: Prepare private Windows test root
if: runner.os == 'Windows'
id: windows-temp
@@ -388,14 +391,7 @@ jobs:
python -m pytest plugins/codex-security/tests -m "$PYTEST_MARKERS" -q -n 4 --dist worksteal --max-worker-restart 0 --durations=30 --junitxml=reports/python.xml
- name: Require native platform coverage
if: runner.os != 'Linux'
- shell: python
- run: |
- from xml.etree import ElementTree
- report = ElementTree.parse("reports/python.xml")
- if report.find(".//testcase") is None:
- raise SystemExit("No native platform tests ran")
- if report.find(".//testcase/skipped") is not None:
- raise SystemExit("Native platform tests must run without skips")
+ run: python .github/scripts/require_native_python_report.py
- name: Upload Python test reports
if: always()
continue-on-error: true
diff --git a/sdk/typescript/tests-ts/skeleton.test.ts b/sdk/typescript/tests-ts/skeleton.test.ts
index bf4b7952c..fab89eeb2 100644
--- a/sdk/typescript/tests-ts/skeleton.test.ts
+++ b/sdk/typescript/tests-ts/skeleton.test.ts
@@ -1,4 +1,4 @@
-import { mkdir, mkdtemp, readFile, rm, writeFile } from "node:fs/promises";
+import { mkdtemp, readFile, rm } from "node:fs/promises";
import { tmpdir } from "node:os";
import { join } from "node:path";
import { describe, expect, test } from "bun:test";
@@ -12,7 +12,7 @@ import {
} from "../src/index.js";
import { main } from "../src/cli.js";
import { capture } from "./cli-fixtures.js";
-import { bashCommand, pythonExecutable, runCommand } from "./support/shell.js";
+import { bashCommand, runCommand } from "./support/shell.js";
const { selectChecks } = (await import(
new URL("../../../.github/scripts/select-ci-checks.mjs", import.meta.url).href
@@ -312,36 +312,11 @@ describe("TypeScript package skeleton", () => {
const step = jobs["plugin-source"]!.steps!.find(
({ name }) => name === "Require native platform coverage",
)!;
- expect(step).toMatchObject({ if: "runner.os != 'Linux'", shell: "python" });
+ expect(step).toMatchObject({
+ if: "runner.os != 'Linux'",
+ run: "python .github/scripts/require_native_python_report.py",
+ });
expect(step).not.toHaveProperty("continue-on-error");
- const directory = await mkdtemp(join(tmpdir(), "native-python-report-"));
- try {
- const reports = join(directory, "reports");
- await mkdir(reports);
- const passing = "<testsuite><testcase/></testsuite>";
- await writeFile(join(reports, "python-stale.xml"), passing);
- for (const [report, expected] of [
- [null, 1],
- ["", 1],
- ["<testsuite/>", 1],
- [
- "<testsuite><testcase/><testcase><skipped/></testcase></testsuite>",
- 1,
- ],
- [passing, 0],
- ] as const) {
- if (report !== null)
- await writeFile(join(reports, "python.xml"), report);
- const result = await runCommand(
- pythonExecutable()!,
- ["-c", step.run!],
- { cwd: directory },
- );
- expect(result.status, result.stderr).toBe(expected);
- }
- } finally {
- await rm(directory, { recursive: true, force: true });
- }
});
test("requires the deterministic eval suites and their cross-project dependency pin", async () => {