Skip to content

OpenAI's Codex Security CLI and TypeScript SDK for finding, validating, and fixing security vulnerabilities. npm: https://www.npmjs.com/package/@openai/codex-security

test(plugin): simplify contract and inventory fixtures (#1969)

Michael D'Angelo8397396426b7
diff --git a/plugins/codex-security/tests/LAYER1_FOLLOWUPS.md b/plugins/codex-security/tests/LAYER1_FOLLOWUPS.md
index 64cf9d451..aa7833b74 100644
--- a/plugins/codex-security/tests/LAYER1_FOLLOWUPS.md
+++ b/plugins/codex-security/tests/LAYER1_FOLLOWUPS.md
@@ -8,8 +8,8 @@ This file is the source of truth for portable deterministic, no-model Codex Secu
- **Schema validation:** Draft 2020-12 validators and checked-in completed-scan examples are validated with `jsonschema`, including RFC 3339 formats.
- **Plugin metadata:** the plugin manifest, referenced assets, skill frontmatter, and agent interface YAML are parsed and checked for required fields and file consistency.
- **Prompt linting:** agent default prompts are required to be non-empty.
-- **CLI and harness dry runs:** every Python script supports `--help`; major command paths run against temporary repositories and scan bundles without a model call.
-- **Static checks:** Ruff checks the plugin scripts and tests; the `--help` smoke test exercises top-level Python script imports.
+- **CLI and harness dry runs:** Python command entry points support `--help`; major command paths run against temporary repositories and scan bundles without a model call.
+- **Static checks:** Ruff checks the plugin scripts and tests. An isolated Python interpreter imports every module in `scripts/`.
- **MCP dependencies and templates:** CI installs the MCP package with its frozen lockfile; its deterministic tests exercise Deep Scan prompt-template rendering.
diff --git a/plugins/codex-security/tests/finalize_sarif_cases.py b/plugins/codex-security/tests/finalize_sarif_cases.py
index c84bf5fb6..0cc1e31e6 100644
--- a/plugins/codex-security/tests/finalize_sarif_cases.py
+++ b/plugins/codex-security/tests/finalize_sarif_cases.py
@@ -377,15 +377,17 @@ class FinalizerSarifCases:
self.assertNotIn("primaryLocationLineHash", result["partialFingerprints"])
def test_sarif_line_hash_streams_source(self) -> None:
- source_path = self.scan_dir / "extract.py"
- source_path.write_text("line 1\nline 2\n", encoding="utf-8")
- with mock.patch.object(Path, "read_text", side_effect=AssertionError("not streamed")):
- with source_path.open("r", encoding="utf-8") as handle:
- line_hashes = FINALIZER._github_line_hashes(handle)
- self.assertEqual(line_hashes[1], "a4060029c94de37e:1")
+ class ShortReads(io.StringIO):
+ def read(self, size: int = -1) -> str:
+ if size <= 0:
+ raise AssertionError("expected a bounded source read")
+ return super().read(min(size, 3))
+
+ with ShortReads("line 1\nline 2\n") as source:
+ self.assertEqual(FINALIZER._github_line_hashes(source)[1], "a4060029c94de37e:1")
def test_sarif_line_hash_includes_lines_beyond_previous_limit(self) -> None:
- source = io.StringIO("line\n" * 100_001)
+ source = io.StringIO("\n" * 100_000 + "source\n")
self.assertIn(100_001, FINALIZER._github_line_hashes(source, {100_001}))
@@ -396,7 +398,8 @@ class FinalizerSarifCases:
findings = []
for index in range(2):
relative_path = f"src/extract-{index}.py"
- (source_root / relative_path).write_text(" " * (6 * 1024 * 1024) + "x")
+ # The two files together exceed the former 10 MiB source budget.
+ (source_root / relative_path).write_text(" " * (5 * 1024 * 1024) + "x")
finding = self.sarif_finding(f"fingerprint-{index}", str(index))
finding["locations"][0] = {"path": relative_path, "startLine": 1}
findings.append(finding)
diff --git a/plugins/codex-security/tests/test_cli_dry_runs.py b/plugins/codex-security/tests/test_cli_dry_runs.py
index 27767c7e1..f51fbdfea 100644
--- a/plugins/codex-security/tests/test_cli_dry_runs.py
+++ b/plugins/codex-security/tests/test_cli_dry_runs.py
@@ -20,13 +20,38 @@ def run_script(name: str, *args: str, isolated: bool = False) -> subprocess.Comp
)
-def test_all_scripts_support_help() -> None:
- for script in sorted(SCRIPT_DIR.glob("*.py")):
- if script.name == "workbench_coverage_receipts.py":
- continue
- result = run_script(script.name, "--help")
- assert result.returncode == 0, f"{script.name}: {result.stderr}"
- assert "usage:" in result.stdout.lower(), script.name
+ "name",
+ (
+ "deep_scan_config.py",
+ "finalize_scan_contract.py",
+ "generate_in_scope_files.py",
+ "generate_rank_input.py",
+ "snapshot_sqlite.py",
+ "threat_model_projection.py",
+ "validate_scan_contract.py",
+ "validate_tracking_source.py",
+ "workbench_db.py",
+ ),
+)
+def test_commands_support_help(name: str) -> None:
+ result = run_script(name, "--help")
+ assert result.returncode == 0, result.stderr
+ assert "usage:" in result.stdout.lower()
+
+
+def test_script_modules_import_in_a_fresh_interpreter() -> None:
+ modules = sorted(path.stem for path in SCRIPT_DIR.glob("*.py"))
+ program = (
+ "import importlib, sys\n"
+ "sys.path.insert(0, sys.argv[1])\n"
+ "for name in sys.argv[2:]:\n"
+ " importlib.import_module(name)\n"
+ )
+ subprocess.run(
+ [sys.executable, "-I", "-c", program, str(SCRIPT_DIR), *modules],
+ check=True,
+ )
@pytest.mark.parametrize(
diff --git a/plugins/codex-security/tests/test_finalize_scan_contract.py b/plugins/codex-security/tests/test_finalize_scan_contract.py
index b1385f64d..27d52ad9b 100644
--- a/plugins/codex-security/tests/test_finalize_scan_contract.py
+++ b/plugins/codex-security/tests/test_finalize_scan_contract.py
@@ -25,7 +25,8 @@ CANONICAL_FILES = ("scan-manifest.json", "findings.json", "coverage.json")
class FinalizeScanContractTest(FinalizerSarifCases, ScanFixtureTestCase):
def setUp(self) -> None:
self.temp_dir = tempfile.TemporaryDirectory()
- self.scan_dir = Path(self.temp_dir.name).resolve()
+ self.scan_dir = Path(self.temp_dir.name).resolve() / "scan"
+ self.scan_dir.mkdir(mode=0o700)
self.manifest = json.loads((EXAMPLE_DIR / "scan-manifest.json").read_text(encoding="utf-8"))
scan = self.manifest["scan"]
scan["id"] = "scan_001"
@@ -203,12 +204,10 @@ The extraction root is not enforced.
0
]["results"][0]
- def export_csv(self) -> csv.DictReader:
- self.findings["findings"] = [copy.deepcopy(self.finding)]
- self.write_sealed_scan()
- result = self.run_finalizer("--export-format", "csv")
- self.assertEqual(result.returncode, 0, result.stderr)
- return csv.DictReader(io.StringIO(result.stdout, newline=""))
+ def project_csv(self) -> csv.DictReader:
+ finding = {**self.finding, "findingId": "csf_example", "occurrenceId": "occ_example"}
+ contents = FINALIZER.build_csv_projection({"findings": [finding]}, self.coverage)
+ return csv.DictReader(io.StringIO(contents.decode("utf-8"), newline=""))
def write_sealed_receipt_scan(self, receipt_ref: str) -> Path:
self.coverage["surfaces"][0]["receiptRefs"] = [receipt_ref]
@@ -487,16 +486,6 @@ The extraction root is not enforced.
):
FINALIZER.finalize_scan(self.scan_dir)
- def test_sealed_rerun_accepts_duplicate_legacy_evidence_ids(self) -> None:
- self.write_sealed_scan()
- findings = self.read_json("findings.json")
- findings["findings"][0]["code_evidence"] = [
- {"id": "legacy-duplicate", "code": "first_legacy_source()"},
- {"id": "legacy-duplicate", "code": "second_legacy_source()"},
- ]
- with self.preserving_sealed_findings(findings):
- pass
-
@pytest.mark.cross_platform
def test_sealed_rerun_ignores_non_string_legacy_validation_scalars(self) -> None:
self.write_sealed_scan()
@@ -758,23 +747,27 @@ The extraction root is not enforced.
manifest = self.read_json("scan-manifest.json")
self.assertEqual(manifest["scan"]["sealedAt"], "2026-05-31T18:09:00Z")
- def test_sarif_only_entrypoint_exports_a_sealed_scan_without_mutating_it(self) -> None:
+ def test_sarif_cli_alias_exports_to_stdout_or_external_file_from_read_only_scan(self) -> None:
self.write_sealed_scan()
sarif_path = self.scan_dir / "exports" / "results.sarif"
+ expected = sarif_path.read_bytes()
sarif_path.unlink()
+ sarif_path.parent.rmdir()
canonical = self.scan_file_bytes(*CANONICAL_FILES, "report.md")
+ output = self.scan_dir.parent / "results.sarif"
+ self.scan_dir.chmod(0o500)
+ try:
+ stdout = self.run_finalizer("--sarif-only")
+ to_file = self.run_finalizer("--sarif-only", "--sarif-output", str(output))
+ finally:
+ self.scan_dir.chmod(0o700)
- result = self.run_finalizer("--sarif-only")
-
- self.assertEqual(result.returncode, 0, result.stderr)
- sarif = json.loads(result.stdout)
- self.assertEqual(sarif["version"], "2.1.0")
- rule = sarif["runs"][0]["tool"]["driver"]["rules"][0]
- self.assertEqual(rule["id"], self.finding["ruleId"])
- self.assertIn(self.finding["remediation"], rule["help"]["markdown"])
- self.assertEqual(rule["properties"]["security-severity"], "8.1")
- self.assertIn("external/cwe/cwe-022", rule["properties"]["tags"])
- self.assertFalse(sarif_path.exists())
+ self.assertEqual(stdout.returncode, 0, stdout.stderr)
+ self.assertEqual(json.loads(stdout.stdout), json.loads(expected))
+ self.assertEqual(to_file.returncode, 0, to_file.stderr)
+ self.assertEqual(to_file.stdout, "")
+ self.assertEqual(output.read_bytes(), expected)
+ self.assertFalse(sarif_path.parent.exists())
self.assertEqual(canonical, self.scan_file_bytes(*canonical))
def test_export_entrypoint_writes_json_csv_and_sarif_without_mutating_a_sealed_scan(
@@ -834,7 +827,13 @@ The extraction root is not enforced.
self.assertEqual(row["start_line"], "41")
self.assertEqual(row["end_line"], "44")
else:
- self.assertEqual(json.loads(result.stdout)["version"], "2.1.0")
+ sarif = json.loads(result.stdout)
+ self.assertEqual(sarif["version"], "2.1.0")
+ rule = sarif["runs"][0]["tool"]["driver"]["rules"][0]
+ self.assertEqual(rule["id"], self.finding["ruleId"])
+ self.assertIn(self.finding["remediation"], rule["help"]["markdown"])
+ self.assertEqual(rule["properties"]["security-severity"], "8.1")
+ self.assertIn("external/cwe/cwe-022", rule["properties"]["tags"])
self.assertEqual(
canonical,
self.scan_file_bytes(*canonical),
@@ -843,7 +842,7 @@ The extraction root is not enforced.
def test_deep_csv_export_includes_the_canonical_candidate_id(self) -> None:
self.coverage["mode"] = "deep_repository"
self.finding["extensions"] = {"candidateId": "DSS-145", "reportId": "DSS-145-rogue"}
- reader = self.export_csv()
+ reader = self.project_csv()
self.assertIn("candidate_id", reader.fieldnames or [])
row = next(reader)
self.assertEqual(row["candidate_id"], "DSS-145")
@@ -853,7 +852,7 @@ The extraction root is not enforced.
self.coverage["mode"] = "scoped_path"
self.finding["extensions"] = {"candidateId": "DSS-146", "reportId": "DSS-146-rogue"}
self.finding["locations"] = [{"path": "src/extract.py", "startLine": 41, "role": "sink"}]
- reader = self.export_csv()
+ reader = self.project_csv()
self.assertIn("candidate_id", reader.fieldnames or [])
row = next(reader)
self.assertEqual(row["candidate_id"], "DSS-146")
@@ -868,7 +867,7 @@ The extraction root is not enforced.
with self.subTest(field=field):
self.coverage["mode"] = coverage_mode
self.finding["extensions"] = {field: f"legacy-{field}"}
- reader = self.export_csv()
+ reader = self.project_csv()
self.assertIn("candidate_id", reader.fieldnames or [])
self.assertEqual(next(reader)["candidate_id"], f"legacy-{field}")
@@ -890,40 +889,50 @@ The extraction root is not enforced.
"ledgerRowId": ledger,
}
self.assertEqual(FINALIZER.finding_candidate_id(self.finding), identity)
- self.assertEqual(next(self.export_csv())["candidate_id"], exported)
+ self.assertEqual(next(self.project_csv())["candidate_id"], exported)
def test_scoped_csv_export_does_not_classify_ordinary_ledger_ids_as_deep(self) -> None:
self.coverage["mode"] = "scoped_path"
self.finding["extensions"] = {"ledgerRowId": "SCAN-001-parser"}
- reader = self.export_csv()
+ reader = self.project_csv()
self.assertNotIn("candidate_id", reader.fieldnames or [])
def test_csv_export_escapes_newline_and_full_width_formula_prefixes(self) -> None:
self.finding["title"] = "\n=1+1"
self.finding["summary"] = "=1+1"
self.finding["remediation"] = " \t+1+1"
- row = next(self.export_csv())
+ row = next(self.project_csv())
self.assertEqual(row["title"], "'\n=1+1")
self.assertEqual(row["summary"], "'=1+1")
self.assertEqual(row["remediation"], "' \t+1+1")
- def test_export_entrypoint_rejects_unsealed_and_tampered_scans(self) -> None:
+ def test_exports_reject_unsealed_and_tampered_scans(self) -> None:
self.write_scan()
for export_format in ("json", "csv"):
with self.subTest(export_format=export_format, state="unsealed"):
- result = self.run_finalizer("--export-format", export_format)
- self.assertNotEqual(result.returncode, 0)
- self.assertIn(
- f"{export_format.upper()} export requires a sealed scan", result.stderr
- )
+ with self.assertRaisesRegex(
+ FINALIZER.ContractError,
+ f"{export_format.upper()} export requires a sealed scan",
+ ):
+ FINALIZER.build_findings_export(self.scan_dir, export_format)
+
+ sarif_path = self.scan_dir / "exports" / "results.sarif"
+ result = self.run_finalizer("--export-format", "sarif", "--export-output", str(sarif_path))
+ self.assertNotEqual(result.returncode, 0)
+ self.assertIn("SARIF projection requires a sealed scan", result.stderr)
+ self.assertFalse(sarif_path.parent.exists())
FINALIZER.finalize_scan(self.scan_dir)
+ sarif_path.unlink()
self.write_json("findings.json", {"changed": True})
for export_format in ("json", "csv"):
with self.subTest(export_format=export_format, state="tampered"):
- result = self.run_finalizer("--export-format", export_format)
- self.assertNotEqual(result.returncode, 0)
- self.assertIn("sealed artifact changed or is missing", result.stderr)
+ with self.assertRaisesRegex(FINALIZER.ContractError, "sealed artifact changed"):
+ FINALIZER.build_findings_export(self.scan_dir, export_format)
+ result = self.run_finalizer("--export-format", "sarif")
+ self.assertNotEqual(result.returncode, 0)
+ self.assertIn("sealed artifact changed or is missing", result.stderr)
+ self.assertFalse(sarif_path.exists())
def test_export_entrypoint_writes_external_output_and_rejects_artifact_overwrite(self) -> None:
self.write_sealed_scan()
@@ -932,7 +941,9 @@ The extraction root is not enforced.
output = self.scan_dir.parent / "findings.csv"
exported = self.run_finalizer("--export-format", "csv", "--export-output", str(output))
- rejected = self.run_finalizer("--export-format", "json", "--export-output", str(findings))
+ rejected = self.run_finalizer(
+ "--export-format", "json", "--export-output", findings.name, cwd=self.scan_dir
+ )
self.assertEqual(exported.returncode, 0, exported.stderr)
self.assertTrue(output.read_text().startswith("occurrence_id,finding_id,title,summary"))
@@ -1098,125 +1109,52 @@ The extraction root is not enforced.
self.assertIn("symbolic link", result.stderr)
self.assertEqual(output.read_bytes(), protected_contents)
- def test_export_entrypoint_rejects_orphaned_or_mismatched_output_flags(
- self,
- ) -> None:
- for arguments in (
- ("--export-output", "exports/findings.csv"),
- ("--export-format", "csv", "--sarif-output", "exports/results.sarif"),
+ def test_export_entrypoint_rejects_orphaned_or_conflicting_flags(self) -> None:
+ self.write_scan()
+ before = self.scan_file_bytes(*CANONICAL_FILES)
+ for arguments, error in (
+ (
+ ("--export-output", "exports/findings.csv"),
+ "--export-output requires --export-format",
+ ),
+ (
+ ("--export-format", "csv", "--sarif-output", "output"),
+ "--sarif-output requires --sarif-only",
+ ),
+ (
+ ("--sarif-only", "--export-format", "sarif"),
+ "--sarif-only cannot be combined with --export-format",
+ ),
):
with self.subTest(arguments=arguments):
- self.write_scan()
- before = self.scan_file_bytes(*CANONICAL_FILES)
-
result = self.run_finalizer(*arguments)
-
self.assertNotEqual(result.returncode, 0)
- self.assertIn("requires", result.stderr)
- self.assertEqual(
- before,
- self.scan_file_bytes(*before),
- )
+ self.assertIn(error, result.stderr)
+ self.assertEqual(before, self.scan_file_bytes(*before))
- def test_sarif_only_entrypoint_rejects_changed_sealed_artifact(self) -> None:
- self.write_sealed_scan()
- self.write_json("findings.json", {"changed": True})
- sarif_path = self.scan_dir / "exports" / "results.sarif"
- sarif_path.unlink()
-
- result = self.run_finalizer("--sarif-only")
-
- self.assertNotEqual(result.returncode, 0)
- self.assertIn("sealed artifact changed or is missing", result.stderr)
- self.assertFalse(sarif_path.exists())
-
- def test_sarif_only_entrypoint_rejects_invalid_source_root(self) -> None:
+ def test_sarif_export_rejects_invalid_source_root(self) -> None:
self.write_sealed_scan()
source_file = self.scan_dir / "source-file"
source_file.write_text("not a directory\n", encoding="utf-8")
-
for source_root in (self.scan_dir / "missing-source", source_file):
- for export_args in (("--sarif-only",), ("--export-format", "sarif")):
- with self.subTest(source_root=str(source_root), export_args=export_args):
- result = self.run_finalizer(*export_args, "--source-root", str(source_root))
-
- self.assertNotEqual(result.returncode, 0)
- self.assertIn("source root: expected an existing directory", result.stderr)
- self.assertEqual(result.stdout, "")
-
- def test_sarif_only_entrypoint_writes_external_output_atomically(self) -> None:
- self.write_sealed_scan()
- sarif_path = self.scan_dir / "exports" / "results.sarif"
- sarif_path.unlink()
- output = self.scan_dir.parent / "results.sarif"
-
- result = self.run_finalizer("--sarif-only", "--sarif-output", str(output))
-
- self.assertEqual(result.returncode, 0, result.stderr)
- self.assertEqual(result.stdout, "")
- self.assertEqual(json.loads(output.read_text(encoding="utf-8"))["version"], "2.1.0")
- self.assertFalse(sarif_path.exists())
-
- def test_failed_sarif_only_entrypoint_does_not_create_exports_directory(self) -> None:
- self.write_scan()
- output = self.scan_dir / "exports" / "results.sarif"
-
- result = self.run_finalizer("--sarif-only", "--sarif-output", str(output))
-
- self.assertNotEqual(result.returncode, 0)
- self.assertIn("SARIF projection requires a sealed scan", result.stderr)
- self.assertFalse(output.parent.exists())
-
- def test_sarif_only_entrypoint_rejects_sealed_artifact_output(self) -> None:
- self.write_sealed_scan()
- findings = self.scan_dir / "findings.json"
- before = findings.read_bytes()
-
- result = subprocess.run(
- [
- sys.executable,
- str(Path(FINALIZER.__file__)),
- "--scan-dir",
- str(self.scan_dir),
- "--sarif-only",
- "--sarif-output",
- "findings.json",
- ],
- capture_output=True,
- text=True,
- check=False,
- cwd=self.scan_dir,
- )
-
- self.assertNotEqual(result.returncode, 0)
- self.assertIn("cannot overwrite a scan artifact", result.stderr)
- self.assertEqual(findings.read_bytes(), before)
+ with self.subTest(source_root=source_root):
+ result = self.run_finalizer(
+ "--export-format", "sarif", "--source-root", str(source_root)
+ )
+ self.assertNotEqual(result.returncode, 0)
+ self.assertIn("source root: expected an existing directory", result.stderr)
+ self.assertEqual(result.stdout, "")
@skipIf(os.name == "nt", "backslash is a path separator on Windows")
- def test_sarif_only_entrypoint_accepts_posix_backslash_output_name(self) -> None:
+ def test_sarif_export_accepts_posix_backslash_output_name(self) -> None:
self.write_sealed_scan()
output = self.scan_dir.parent / "results\\v1.sarif"
- result = self.run_finalizer("--sarif-only", "--sarif-output", str(output))
+ result = self.run_finalizer("--export-format", "sarif", "--export-output", str(output))
self.assertEqual(result.returncode, 0, result.stderr)
self.assertEqual(json.loads(output.read_text(encoding="utf-8"))["version"], "2.1.0")
- def test_sarif_only_entrypoint_accepts_read_only_scan_directory(self) -> None:
- self.write_sealed_scan()
- sarif_path = self.scan_dir / "exports" / "results.sarif"
- sarif_path.unlink()
- sarif_path.parent.rmdir()
- self.scan_dir.chmod(0o500)
- try:
- result = self.run_finalizer("--sarif-only")
- finally:
- self.scan_dir.chmod(0o700)
-
- self.assertEqual(result.returncode, 0, result.stderr)
- self.assertEqual(json.loads(result.stdout)["version"], "2.1.0")
- self.assertFalse(sarif_path.exists())
-
def test_sarif_projection_rejects_symlinked_scan_directory(self) -> None:
outside_dir = self.scan_dir / "outside"
outside_dir.mkdir()
@@ -1247,11 +1185,9 @@ The extraction root is not enforced.
original_dir = self.scan_dir / "nested"
original_dir.mkdir()
(original_dir / "payload.json").write_text('{"source": "original"}\n', encoding="utf-8")
- outside_dir = Path(self.temp_dir.name).parent / f"{self.scan_dir.name}-outside"
+ outside_dir = self.scan_dir.parent / "outside"
outside_dir.mkdir()
- self.addCleanup(lambda: outside_dir.rmdir())
(outside_dir / "payload.json").write_text('{"source": "outside"}\n', encoding="utf-8")
- self.addCleanup(lambda: (outside_dir / "payload.json").unlink())
moved_dir = self.scan_dir / "nested-original"
open_directory = FINALIZER._open_scan_local_directory
diff --git a/plugins/codex-security/tests/test_finding_preview.py b/plugins/codex-security/tests/test_finding_preview.py
index 40e1324bd..b19925c95 100644
--- a/plugins/codex-security/tests/test_finding_preview.py
+++ b/plugins/codex-security/tests/test_finding_preview.py
@@ -86,12 +86,10 @@ def test_verbose_finding_metadata_retains_diagnostics_and_report(
)
[email protected]("length", [2_000, 20_000])
@pytest.mark.parametrize("extra_metadata", [False, True])
-def test_multiple_verbose_metadata_sections_each_retain_a_preview(
- length: int, extra_metadata: bool
-) -> None:
+def test_multiple_verbose_metadata_sections_each_retain_a_preview(extra_metadata: bool) -> None:
preview = runpy.run_path(str(FINDING_PREVIEW_SCRIPT))
+ length = 20_000
finding = {
"severity": {"level": "high", "rationale": "s" * length},
"confidence": {"level": "high", "rationale": "c" * length},
@@ -445,12 +443,10 @@ def test_large_code_preview_bounds_sizing_probes(monkeypatch: pytest.MonkeyPatch
bounded = project(finding)
assert bounded == expected
- assert list(bounded) == list(expected)
encoded = json.dumps(bounded, separators=(",", ":")).encode("ascii")
assert encoded == json.dumps(expected, separators=(",", ":")).encode("ascii")
assert len(encoded) <= preview["FINDING_DETAILS_PREVIEW_BYTES"]
assert finding == original
- assert probe_lengths
assert 1_498 in probe_lengths
assert max(probe_lengths) <= 1_500
diff --git a/plugins/codex-security/tests/test_generate_in_scope_files.py b/plugins/codex-security/tests/test_generate_in_scope_files.py
index ebd9962ea..d25ce685b 100644
--- a/plugins/codex-security/tests/test_generate_in_scope_files.py
+++ b/plugins/codex-security/tests/test_generate_in_scope_files.py
@@ -135,7 +135,9 @@ def test_inventory_matches_the_existing_standard_command(tmp_path: Path, scope:
assert {"./ignored/secret.py", "./app/ignored.skip"}.isdisjoint(rows)
[email protected]("name", [f"{'a' * 251}.txt", f"{'文' * 83}.txt"])
+ "name", [f"{'a' * 251}.txt", f"{'文' * 83}.txt"], ids=["ascii", "multibyte"]
+)
def test_inventory_replaces_outputs_with_long_filenames(tmp_path: Path, name: str) -> None:
repository = make_repository(tmp_path)
output = tmp_path / name
@@ -303,24 +305,41 @@ def test_inventory_excludes_the_git_file_of_a_linked_worktree(tmp_path: Path, ne
@pytest.mark.skipif(os.name == "nt", reason="Windows does not allow CR/LF in filenames")
[email protected]("separator", ["\n", "\r", "\r\n"])
[email protected]("ignored_tracked", [False, True])
[email protected]("scope_kind", ["repository", "directory", "file"])
+ ("scope_kind", "separator"),
+ [
+ pytest.param("ignored-tracked", "\r", id="git-ignored-tracked-cr"),
+ pytest.param("directory", "\n", id="ripgrep-directory-lf"),
+ pytest.param("file", "\r\n", id="explicit-file-crlf"),
+ pytest.param("local-diff", "\n", id="diff-path-lf"),
+ ],
+)
def test_inventory_rejects_line_breaks_before_serializing_paths(
- tmp_path: Path, separator: str, ignored_tracked: bool, scope_kind: str
+ tmp_path: Path, scope_kind: str, separator: str
) -> None:
- repository = make_repository(tmp_path)
- directory = "ignored" if ignored_tracked else "app"
+ repository = tmp_path / "repository"
+ repository.mkdir()
+ git(repository, "init", "-q")
+ directory = "ignored" if scope_kind == "ignored-tracked" else "app"
name = f"{directory}/concealed.py{separator}phantom.py"
write_file(repository, name)
- if ignored_tracked:
+ scope = "."
+ arguments = None
+ if scope_kind == "ignored-tracked":
+ write_file(repository, ".gitignore", b"ignored/\n")
git(repository, "add", "--force", "--", name)
- scope = {"repository": ".", "directory": f"./{directory}", "file": name}[scope_kind]
+ elif scope_kind == "directory":
+ scope = f"./{directory}"
+ elif scope_kind == "file":
+ scope = name
+ else:
+ git(repository, "commit", "--allow-empty", "-qm", "base")
+ arguments = ["--diff-base", "HEAD", "--diff-mode", "local-patch"]
output = tmp_path / "in_scope_files.txt"
previous = b"previous.py\n"
output.write_bytes(previous)
- result = run_inventory(repository, scope, output)
+ result = run_inventory(repository, scope, output, arguments=arguments)
assert result.returncode == 2
assert "path that cannot fit in the file inventory" in result.stderr
diff --git a/plugins/codex-security/tests/test_generate_rank_input.py b/plugins/codex-security/tests/test_generate_rank_input.py
index 3353884f4..e984dd08e 100644
--- a/plugins/codex-security/tests/test_generate_rank_input.py
+++ b/plugins/codex-security/tests/test_generate_rank_input.py
@@ -83,44 +83,6 @@ def test_rank_input_keeps_large_generated_python(tmp_path: Path) -> None:
assert rows[1]["preview"] == "value = 1"
-def test_revision_previews_use_the_worktree_read_budget(tmp_path: Path) -> None:
- repo = tmp_path / "repo"
- repo.mkdir()
- initialize_repo(repo)
- git(repo, "commit", "--allow-empty", "-qm", "base")
- base = git(repo, "rev-parse", "HEAD")
- (repo / "large.py").write_text(
- "def visible():\n pass\n" + "# comment\n" * 10000 + "def outside():\n pass\n",
- encoding="utf-8",
- )
- git(repo, "add", ".")
- git(repo, "commit", "-qm", "change")
- output = tmp_path / "rank.jsonl"
-
- run_repo_cli("make-diff-rank-input", repo, output, "--base", base, "--head", "HEAD")
-
- preview = read_jsonl(output)[0]["preview"]
- assert preview.startswith("def visible():\n pass\n# comment")
- assert len(preview.encode("utf-8")) <= 1024
- assert "outside" not in preview
-
-
-def test_revision_rank_input_classifies_bytes_beyond_the_preview_sample(tmp_path: Path) -> None:
- repo = tmp_path / "repo"
- repo.mkdir()
- initialize_repo(repo)
- git(repo, "commit", "--allow-empty", "-qm", "base")
- base = git(repo, "rev-parse", "HEAD")
- (repo / "binary.py").write_bytes(b"# comment\n" * 10_000 + b"\0")
- git(repo, "add", ".")
- git(repo, "commit", "-qm", "change")
- output = tmp_path / "rank.jsonl"
-
- run_repo_cli("make-diff-rank-input", repo, output, "--base", base, "--head", "HEAD")
-
- assert read_jsonl(output) == []
-
-
def test_rank_input_streams_classification_and_bounds_previews(
tmp_path: Path, monkeypatch: pytest.MonkeyPatch
) -> None:
@@ -775,65 +737,69 @@ def test_revision_preview_uses_the_same_read_window_as_local_files(tmp_path: Pat
"--head",
"HEAD",
)
- rows = read_jsonl(output)
- assert "first" in rows[0]["preview"]
- assert "outside_window" not in rows[0]["preview"]
+ preview = read_jsonl(output)[0]["preview"]
+ assert preview.startswith("def first():\n pass")
+ assert "outside_window" not in preview
+ assert len(preview.encode("utf-8")) <= 1024
[email protected]("mode", ["staged", "unstaged", "untracked", "deleted", "revisions"])
[email protected]("command", ["make-diff-rank-input", "generate-in-scope-files"])
-def test_local_diff_inventory_excludes_svn_but_revision_diff_retains_it(
- tmp_path: Path, mode: str, command: str
-) -> None:
+def test_local_diff_inventory_excludes_svn_but_revision_diff_retains_it(tmp_path: Path) -> None:
repo = tmp_path / "repo"
repo.mkdir()
initialize_repo(repo)
- names = [".svn/wc.db", "vendor/.svn/entries", "source.py", "vendor/.svn-source.py"]
- (repo / "base.py").write_text("base\n", encoding="utf-8")
- if mode != "untracked":
- for name in names:
- source = repo / name
- source.parent.mkdir(parents=True, exist_ok=True)
- source.write_text("before\n", encoding="utf-8")
+ changes = {}
+ sources = []
+ for state in ("staged", "unstaged", "untracked", "deleted"):
+ ordinary = [f"{state}.py", f"vendor/.svn-{state}.py"]
+ sources.extend(ordinary)
+ changes[state] = [*ordinary, f".svn/{state}", f"vendor/.svn/{state}"]
+
+ def write_source(name: str, version: str) -> None:
+ source = repo / name
+ source.parent.mkdir(parents=True, exist_ok=True)
+ source.write_text(f"{name}: {version}\n", encoding="utf-8")
+
+ for state in ("staged", "unstaged", "deleted"):
+ for name in changes[state]:
+ write_source(name, "before")
git(repo, "add", ".")
git(repo, "commit", "-qm", "base")
base = git(repo, "rev-parse", "HEAD")
- for name in names:
- source = repo / name
- source.parent.mkdir(parents=True, exist_ok=True)
- if mode == "deleted":
- source.unlink()
- else:
- source.write_text("after\n", encoding="utf-8")
- if mode in {"staged", "revisions"}:
- git(repo, "add", ".")
- if mode == "revisions":
- git(repo, "commit", "-qm", "changed")
- diff_mode = "revisions" if mode == "revisions" else "local-patch"
- output = tmp_path / "inventory"
- if command == "make-diff-rank-input":
- run_repo_cli(command, repo, output, "--base", base, "--mode", diff_mode)
- paths = [row["path"] for row in read_jsonl(output)]
- else:
+
+ for state in ("staged", "unstaged", "untracked"):
+ for name in changes[state]:
+ write_source(name, "after")
+ git(repo, "add", "--", *changes["staged"])
+ for name in changes["deleted"]:
+ (repo / name).unlink()
+
+ def assert_inventories(mode: str, expected: list[str]) -> None:
+ rank = tmp_path / f"{mode}.jsonl"
+ run_repo_cli("make-diff-rank-input", repo, rank, "--base", base, "--mode", mode)
+ assert [row["path"] for row in read_jsonl(rank)] == sorted(expected), mode
+ inventory = tmp_path / f"{mode}.txt"
subprocess.run(
[
sys.executable,
- str(PLUGIN_ROOT / "scripts" / "generate_in_scope_files.py"),
+ str(SCRIPT.with_name("generate_in_scope_files.py")),
"--repo",
str(repo),
"--scope",
".",
"--out",
- str(output),
+ str(inventory),
"--diff-base",
base,
"--diff-mode",
- diff_mode,
+ mode,
],
check=True,
capture_output=True,
text=True,
)
- paths = output.read_text(encoding="utf-8").splitlines()
- expected = names if mode == "revisions" else names[2:]
- assert sorted(paths) == sorted(expected)
+ assert inventory.read_text(encoding="utf-8").splitlines() == sorted(expected), mode
+
+ assert_inventories("local-patch", sources)
+ git(repo, "add", "-A")
+ git(repo, "commit", "-qm", "change all four paths in each group")
+ assert_inventories("revisions", [name for names in changes.values() for name in names])
diff --git a/plugins/codex-security/tests/test_git_inventory_regressions.py b/plugins/codex-security/tests/test_git_inventory_regressions.py
index 7cb3d660d..385d676b0 100644
--- a/plugins/codex-security/tests/test_git_inventory_regressions.py
+++ b/plugins/codex-security/tests/test_git_inventory_regressions.py
@@ -292,21 +292,23 @@ def test_diff_inventories_exclude_non_file_type_changes(
git(target, "add", ".")
git(target, "commit", "-qm", "Add source fixture")
base = git(target, "rev-parse", "HEAD").decode()
+ source.unlink()
if replacement == "symlink":
- source.unlink()
try:
source.symlink_to("README.md")
except OSError:
pytest.skip("symlinks are unavailable")
+ git(target, "add", "--", source.name)
else:
- origin = tmp_path / "origin"
- initialize_git_repository(origin)
- git(target, "rm", "-f", source.name)
- git(
- target, "-c", "protocol.file.allow=always", "submodule", "add", str(origin), source.name
+ # The diff consumes the gitlink's index mode, so it needs no child clone.
+ source.mkdir()
+ git(target, "update-index", "--add", "--cacheinfo", f"160000,{base},{source.name}")
+ (target / ".gitmodules").write_text(
+ f'[submodule "fixture"]\n\tpath = {source.name}\n\turl = ../fixture\n'
)
+ git(target, "add", ".gitmodules")
(target / "visible.py").write_text("value = 1\n")
- git(target, "add", ".")
+ git(target, "add", "visible.py")
if mode == "revisions":
git(target, "commit", "-qm", "Replace source type")
inventory_path = tmp_path / "inventory.txt"
diff --git a/plugins/codex-security/tests/test_partial_coverage_projection.py b/plugins/codex-security/tests/test_partial_coverage_projection.py
index bb4cad333..783614a5a 100644
--- a/plugins/codex-security/tests/test_partial_coverage_projection.py
+++ b/plugins/codex-security/tests/test_partial_coverage_projection.py
@@ -231,12 +231,14 @@ def test_missing_deferred_projection_links_first_duplicate_surface(
workbench_db,
Namespace(scan_id=scan.scan_id, claim_token=None, cost_json=None, message="Stopped."),
)
+ coverage_path = scan.scan_dir / "coverage.json"
+ published = coverage_path.read_bytes()
recovered = workbench_api["saved_results"].recover_scan_results(
workbench_api["_WORKBENCH_DB_CONTEXT"], workbench_db, Namespace(scan_id=scan.scan_id)
)["scan"]
assert recovered["resultsRecoveryNeeded"] is False
- coverage_path = scan.scan_dir / "coverage.json"
- coverage = json.loads(coverage_path.read_text())
+ assert coverage_path.read_bytes() == published
+ coverage = json.loads(published)
pending = [item for item in coverage["deferred"] if item.get("reason") == deferred["reason"]]
assert len(pending) == 1
if parent_surfaces == "edited":
@@ -257,15 +259,17 @@ def test_missing_deferred_projection_links_first_duplicate_surface(
assert len(coverage["surfaces"]) == len(projected)
assert all(surface in coverage["surfaces"] for surface in projected)
assert result.read_bytes() == original
- published = coverage_path.read_bytes()
- workbench_api["saved_results"].recover_scan_results(
- workbench_api["_WORKBENCH_DB_CONTEXT"], workbench_db, Namespace(scan_id=scan.scan_id)
- )
- assert coverage_path.read_bytes() == published
+
+ # Replay a new checkpoint with both worker-only coverage and a conflicting parent edit.
+ if (parent_surfaces, merge_state, tagged_surface) not in {
+ ("no-parent", "buffered", False),
+ ("edited", "merged", True),
+ }:
+ return
late = write_checkpoint(
result.parent / "checkpoints",
{
- **json.loads(result.read_text()),
+ **json.loads(original),
"complete": False,
"coverage": {**scan.coverage, "openQuestions": ["Late worker checkpoint."]},
},
@@ -274,6 +278,8 @@ def test_missing_deferred_projection_links_first_duplicate_surface(
workbench_api["saved_results"].recover_scan_results(
workbench_api["_WORKBENCH_DB_CONTEXT"], workbench_db, Namespace(scan_id=scan.scan_id)
)
+ manifest = json.loads((scan.scan_dir / "scan-manifest.json").read_bytes())
+ assert late.relative_to(scan.scan_dir).as_posix() in manifest["scan"]["preservedSources"]
assert coverage_path.read_bytes() == published
assert result.read_bytes() == original
assert late.read_bytes() == original_late
diff --git a/plugins/codex-security/tests/test_plugin_contract.py b/plugins/codex-security/tests/test_plugin_contract.py
index 40c7cf208..411d89955 100644
--- a/plugins/codex-security/tests/test_plugin_contract.py
+++ b/plugins/codex-security/tests/test_plugin_contract.py
@@ -143,10 +143,3 @@ def test_every_skill_has_valid_frontmatter_and_agent_interface() -> None:
for field in ("display_name", "short_description", "default_prompt"):
assert isinstance(interface[field], str), f"{agent_path}: {field}"
assert interface[field].strip(), f"{agent_path}: {field}"
-
-
-def test_fix_finding_uses_product_language_for_workbench_state() -> None:
- skill = (PLUGIN_ROOT / "skills" / "fix-finding" / "SKILL.md").read_text(encoding="utf-8")
-
- assert "terminal workbench update" in skill
- assert "SQLite" not in skill
diff --git a/plugins/codex-security/tests/test_rank_preview.py b/plugins/codex-security/tests/test_rank_preview.py
index 61ab58b3c..c84f53d53 100644
--- a/plugins/codex-security/tests/test_rank_preview.py
+++ b/plugins/codex-security/tests/test_rank_preview.py
@@ -1,6 +1,7 @@
from __future__ import annotations
import sys
+from io import BytesIO
from pathlib import Path
from unittest.mock import patch
@@ -89,52 +90,43 @@ def test_preview_for_ignores_incomplete_utf16_tail(
assert preview_for_bytes(data, DEFAULT_PREVIEW_BYTES) == (source, False)
-def test_preview_for_does_not_fully_read_a_large_binary(tmp_path: Path) -> None:
- source = tmp_path / "payload.bin"
- with source.open("wb") as output:
- output.write(b"header\0payload")
- output.truncate(256 * 1024 * 1024)
+class BudgetedReader(BytesIO):
+ def __init__(self, data: bytes, limit: int) -> None:
+ super().__init__(data)
+ self.limit = limit
+ self.bytes_read = 0
- with patch.object(Path, "read_bytes", side_effect=MemoryError("full binary read")):
- assert preview_for(source, DEFAULT_PREVIEW_BYTES) == ("", True)
+ def read(self, size: int = -1) -> bytes:
+ assert 0 <= size <= self.limit - self.bytes_read
+ data = super().read(size)
+ self.bytes_read += len(data)
+ return data
-def test_preview_for_bounds_a_source_like_binary_after_the_initial_sample(tmp_path: Path) -> None:
- source = tmp_path / "payload.py"
- with source.open("wb") as output:
- output.write(b"header-without-a-nul" * 256)
- output.write(b"\0binary")
- output.truncate(256 * 1024 * 1024)
-
- assert preview_for(source, DEFAULT_PREVIEW_BYTES, max_read_bytes=64 * 1024) == ("", True)
+ ("filename", "nul_offset", "limit"),
+ [("payload.bin", 0, 4096), ("payload.py", 4096, 64 * 1024)],
+ ids=["initial-sample", "after-initial-sample"],
+)
+def test_preview_for_bounds_binary_reads(filename: str, nul_offset: int, limit: int) -> None:
+ reader = BudgetedReader(b"x" * nul_offset + b"\0" + b"x" * limit, limit)
+ with patch.object(Path, "open", return_value=reader):
+ assert preview_for(Path(filename), DEFAULT_PREVIEW_BYTES) == ("", True)
+ assert reader.bytes_read > nul_offset
@pytest.mark.parametrize("max_read_bytes", [None, 1024, 4096, 4097, 128 * 1024])
-def test_preview_for_bounds_large_text_reads(tmp_path: Path, max_read_bytes: int | None) -> None:
- source = tmp_path / "large.py"
- source.write_bytes(b"# source comment\n" * (128 * 1024))
+def test_preview_for_bounds_large_text_reads(max_read_bytes: int | None) -> None:
limit = 64 * 1024 if max_read_bytes is None else max_read_bytes
- bytes_read = 0
- with source.open("rb") as reader:
- read = reader.read
-
- def bounded_read(size: int = -1) -> bytes:
- nonlocal bytes_read
- assert 0 <= size <= limit - bytes_read
- data = read(size)
- bytes_read += len(data)
- return data
-
- with (
- patch.object(Path, "open", return_value=reader),
- patch.object(reader, "read", side_effect=bounded_read),
- ):
- if max_read_bytes is None:
- preview, binary = preview_for(source, 128)
- else:
- preview, binary = preview_for(source, 128, max_read_bytes=max_read_bytes)
-
- assert bytes_read == limit
+ line = b"# source comment\n"
+ reader = BudgetedReader(line * (limit // len(line) + 1), limit)
+ with patch.object(Path, "open", return_value=reader):
+ if max_read_bytes is None:
+ preview, binary = preview_for(Path("large.py"), 128)
+ else:
+ preview, binary = preview_for(Path("large.py"), 128, max_read_bytes=max_read_bytes)
+
+ assert reader.bytes_read == limit
assert not binary
assert preview
assert len(preview.encode("utf-8")) <= 128
diff --git a/plugins/codex-security/tests/test_saved_deferred_identity.py b/plugins/codex-security/tests/test_saved_deferred_identity.py
index a644b6fc7..baf160a33 100644
--- a/plugins/codex-security/tests/test_saved_deferred_identity.py
+++ b/plugins/codex-security/tests/test_saved_deferred_identity.py
@@ -2,9 +2,11 @@ from __future__ import annotations
import copy
import hashlib
+import io
import json
import os
import sys
+from contextlib import redirect_stdout
from pathlib import Path
import pytest
@@ -34,6 +36,25 @@ def saved_results():
return workbench_saved_results
+def setup_workbench(monkeypatch, workbench_api):
+ def run(state, *args, environment=None):
+ with (
+ open(os.devnull) as stdin,
+ monkeypatch.context() as patch,
+ redirect_stdout(io.StringIO()) as stdout,
+ ):
+ patch.setenv("CODEX_SECURITY_STATE_DIR", str(state))
+ for name, value in (environment or {}).items():
+ patch.setenv(name, value)
+ patch.setattr(sys, "stdin", stdin)
+ patch.setattr(sys, "argv", ["workbench_db", *args])
+ workbench_api["main"]()
+ return json.loads(stdout.getvalue())
+
+ return run
+
+
def save_worker(root: Path, module, worker_id: str, drafts: list[dict], result: dict):
output = root / worker_id
checkpoints = output / "checkpoints"
@@ -648,11 +669,16 @@ def test_generic_surface_recovery_uses_resolved_candidate_identity(
tmp_path: Path,
monkeypatch: pytest.MonkeyPatch,
saved_results,
+ setup_workbench,
candidate_identity: str,
outcome: str,
):
- state, codex_home, _, scan_dir, scan_id = deep_scan_fixture(tmp_path, workers=2)
- worker_id, result_path = accepted_standard_worker(state, codex_home, scan_dir, scan_id)
+ state, codex_home, _, scan_dir, scan_id = deep_scan_fixture(
+ tmp_path, workers=2, run=setup_workbench
+ )
+ worker_id, result_path = accepted_standard_worker(
+ state, codex_home, scan_dir, scan_id, run=setup_workbench
+ )
candidate = {
"id": "caller-review",
"reason": "Caller needs validation.",
@@ -709,7 +735,7 @@ def test_generic_surface_recovery_uses_resolved_candidate_identity(
os.utime(head, ns=(observed, observed))
if outcome == "other_worker":
other_worker_id, other_result = accepted_standard_worker(
- state, codex_home, scan_dir, scan_id, name="other-worker"
+ state, codex_home, scan_dir, scan_id, name="other-worker", run=setup_workbench
)
other_result.write_text(
json.dumps(saved_draft(scan_id, surfaces=[rejection], complete=True))
@@ -779,12 +805,15 @@ def test_split_deferred_rows_keep_distinct_ids_in_frozen_recovery(
tmp_path: Path,
monkeypatch: pytest.MonkeyPatch,
saved_results,
+ setup_workbench,
payload_field: str | None,
explicit_ids: bool,
close_first: bool,
):
- state, codex_home, _, scan_dir, scan_id = deep_scan_fixture(tmp_path)
- worker_id, result_path = accepted_standard_worker(state, codex_home, scan_dir, scan_id)
+ state, codex_home, _, scan_dir, scan_id = deep_scan_fixture(tmp_path, run=setup_workbench)
+ worker_id, result_path = accepted_standard_worker(
+ state, codex_home, scan_dir, scan_id, run=setup_workbench
+ )
combined = {
"id": "combined-review",
"reason": "Review remains.",
@@ -852,11 +881,14 @@ def test_unnamed_observation_does_not_replace_saved_context(
tmp_path: Path,
monkeypatch: pytest.MonkeyPatch,
saved_results,
+ setup_workbench,
layout: str,
observation: str,
):
- state, codex_home, _, scan_dir, scan_id = deep_scan_fixture(tmp_path)
- worker_id, result_path = accepted_standard_worker(state, codex_home, scan_dir, scan_id)
+ state, codex_home, _, scan_dir, scan_id = deep_scan_fixture(tmp_path, run=setup_workbench)
+ worker_id, result_path = accepted_standard_worker(
+ state, codex_home, scan_dir, scan_id, run=setup_workbench
+ )
broad = {
"id": "caller-review",
"reason": "Review the caller paths.",
diff --git a/plugins/codex-security/tests/test_scan_contract_examples.py b/plugins/codex-security/tests/test_scan_contract_examples.py
index 3791252b3..12ab94e26 100644
--- a/plugins/codex-security/tests/test_scan_contract_examples.py
+++ b/plugins/codex-security/tests/test_scan_contract_examples.py
@@ -24,7 +24,6 @@ def read_json(path: Path) -> dict[str, Any]:
def validate_schema_node(value: Any, schema: dict[str, Any], context: str) -> None:
- Draft202012Validator.check_schema(schema)
validator = Draft202012Validator(schema, format_checker=FormatChecker())
errors = sorted(validator.iter_errors(value), key=lambda error: list(error.absolute_path))
if not errors:
diff --git a/plugins/codex-security/tests/test_scan_finalizer_schema_and_writeups.py b/plugins/codex-security/tests/test_scan_finalizer_schema_and_writeups.py
deleted file mode 100644
index 59ed56fe2..000000000
--- a/plugins/codex-security/tests/test_scan_finalizer_schema_and_writeups.py
+++ /dev/null
@@ -1,84 +0,0 @@
-from __future__ import annotations
-
-import copy
-import os
-
-import pytest
-from workbench_test_support import load_script, windows_file_backend
-
-FINALIZER = load_script("finalize_scan_contract")
-
-
[email protected](("expected", "value"), [("integer", 1), ("number", 1.5)])
-def test_schema_numeric_types_do_not_accept_booleans(expected, value) -> None:
- schema = {"type": expected}
- FINALIZER._validate_schema_node(value, schema, "value")
- with pytest.raises(FINALIZER.ContractError, match="expected schema type"):
- FINALIZER._validate_schema_node(True, schema, "value")
-
-
[email protected](os.name == "nt", reason="POSIX symlinks and file permissions")
- "kind", ["unsafe", "symlink", "dangling-symlink", "parent-symlink", "directory", "unreadable"]
-)
-def test_missing_writeup_recovery_rejects_other_file_errors(tmp_path, kind):
- scan_dir = tmp_path / "scan"
- scan_dir.mkdir()
- report = scan_dir / "findings" / "report.md"
- report.parent.mkdir()
- outside = tmp_path / "outside.md"
- outside.write_text("# Outside the scan\n")
- report_path = "findings/report.md"
- if kind == "unsafe":
- report_path = "../outside.md"
- elif kind == "symlink":
- report.symlink_to(outside)
- elif kind == "dangling-symlink":
- report.symlink_to(report.parent / "missing.md")
- elif kind == "parent-symlink":
- report.parent.rmdir()
- report.parent.symlink_to(tmp_path, target_is_directory=True)
- elif kind == "directory":
- report.mkdir()
- else:
- if os.geteuid() == 0:
- pytest.skip("root can read files without permission bits")
- report.write_text("# Unreadable writeup\n")
- report.chmod(0)
- findings = {"findings": [{"writeup": {"reportPath": report_path}}]}
- original = copy.deepcopy(findings)
- with pytest.raises(FINALIZER.ContractError):
- FINALIZER._prepare_derived_writeups(scan_dir, findings, allow_missing=True)
- assert findings == original
-
-
[email protected]("error_code", [2, 3, 5, 22, 32, 33])
-def test_windows_writeup_recovery_only_omits_missing_files(tmp_path, monkeypatch, error_code):
- backend = windows_file_backend()
- windows_error = load_script("windows_scan_local_files").WindowsScanLocalFileError
- backend.open_read_fd.side_effect = windows_error(error_code, "Synthetic Windows file error")
- monkeypatch.setattr(FINALIZER.os, "supports_dir_fd", set())
- monkeypatch.setattr(FINALIZER, "_is_windows", lambda: True)
- monkeypatch.setattr(FINALIZER, "_windows_scan_local_files", lambda: backend)
- findings = {"findings": [{"writeup": {"reportPath": "findings/report.md"}}]}
- if error_code in {2, 3}:
- warnings = FINALIZER._prepare_derived_writeups(tmp_path, findings, allow_missing=True)
- assert findings == {"findings": [{}]}
- assert len(warnings) == 1
- else:
- with pytest.raises(FINALIZER.ContractError):
- FINALIZER._prepare_derived_writeups(tmp_path, findings, allow_missing=True)
- assert "writeup" in findings["findings"][0]
-
-
-def test_schema_references_preserve_constraints_siblings_and_cycle_errors() -> None:
- root = {"$defs": {"text": {"type": "string", "minLength": 1}}}
- reference = {"$ref": "#/$defs/text"}
- FINALIZER._validate_schema_node("yes", reference, "value", root)
- with pytest.raises(FINALIZER.ContractError, match="string is too short"):
- FINALIZER._validate_schema_node("", reference, "value", root)
- with pytest.raises(FINALIZER.ContractError, match="string does not match schema pattern"):
- FINALIZER._validate_schema_node("no", {**reference, "pattern": "^yes$"}, "value", root)
- with pytest.raises(RecursionError):
- FINALIZER._validate_schema_node("yes", {"$ref": "#"}, "value")
diff --git a/plugins/codex-security/tests/test_validate_tracking_source.py b/plugins/codex-security/tests/test_validate_tracking_source.py
index 961887bae..0c903a4df 100644
--- a/plugins/codex-security/tests/test_validate_tracking_source.py
+++ b/plugins/codex-security/tests/test_validate_tracking_source.py
@@ -4,7 +4,7 @@ import copy
import hashlib
import io
import unittest
-from contextlib import redirect_stderr, redirect_stdout
+from contextlib import redirect_stderr
from pathlib import Path
from workbench_test_support import ScanFixtureTestCase, load_script
@@ -16,8 +16,11 @@ EXAMPLE_SCAN = PLUGIN_ROOT / "examples" / "completed-scan"
VALIDATOR = load_script("validate_tracking_source")
-def test_completed_example_passes_tracking_validation() -> None:
- assert VALIDATOR.validate_source(EXAMPLE_SCAN)
+def test_tracking_cli_lists_the_checked_in_finding(capsys) -> None:
+ assert VALIDATOR.main([str(EXAMPLE_SCAN)]) == 0
+ captured = capsys.readouterr()
+ assert captured.out == "csf_852f90d6e1177502ff113d4a\n"
+ assert captured.err == ""
class ValidateTrackingSourceTest(ScanFixtureTestCase):
@@ -48,11 +51,6 @@ class ValidateTrackingSourceTest(ScanFixtureTestCase):
artifact["sha256"] = hashlib.sha256(coverage_bytes).hexdigest()
self.write_json("scan-manifest.json", manifest)
- def test_selects_the_finding_from_a_sealed_scan(self) -> None:
- findings = VALIDATOR.validate_source(self.scan_dir)
-
- self.assertEqual(findings[0]["findingId"], "csf_852f90d6e1177502ff113d4a")
-
def test_tampered_findings_are_rejected(self) -> None:
findings_path = self.scan_dir / "findings.json"
findings_path.write_text(
@@ -158,17 +156,6 @@ class ValidateTrackingSourceTest(ScanFixtureTestCase):
self.assertEqual(len(finding_ids), 25)
self.assertEqual(len(set(finding_ids)), 25)
- def test_cli_prints_only_the_selected_finding_id(self) -> None:
- stdout = io.StringIO()
- stderr = io.StringIO()
-
- with redirect_stdout(stdout), redirect_stderr(stderr):
- exit_code = VALIDATOR.main([str(self.scan_dir)])
-
- self.assertEqual(exit_code, 0)
- self.assertEqual(stdout.getvalue(), "csf_852f90d6e1177502ff113d4a\n")
- self.assertEqual(stderr.getvalue(), "")
-
if __name__ == "__main__":
unittest.main()
diff --git a/plugins/codex-security/tests/test_workbench_standard_deep_results.py b/plugins/codex-security/tests/test_workbench_standard_deep_results.py
index b286f413c..e5a62c98d 100644
--- a/plugins/codex-security/tests/test_workbench_standard_deep_results.py
+++ b/plugins/codex-security/tests/test_workbench_standard_deep_results.py
@@ -14,7 +14,6 @@ from pathlib import Path
import pytest
from workbench_test_support import (
BUDGET_COST,
- begin_deep_scan,
cancel_scan,
claim_deep_scan_dedup,
commit_deep_dedup,
@@ -1106,6 +1105,7 @@ def deep_scan_fixture(
budget: bool = False,
workers: int = 1,
include_paths: list[str] | None = None,
+ run=run_workbench,
) -> tuple[Path, Path, Path, Path, str]:
state_dir = tmp_path / "state"
codex_home = tmp_path / "codex-home"
@@ -1124,7 +1124,7 @@ def deep_scan_fixture(
if budget or include_paths is not None:
scan_dir = tmp_path / "scan"
scan_dir.mkdir(mode=0o700)
- registered = run_workbench(
+ registered = run(
state_dir,
"register-cli-scan",
"--scan-dir",
@@ -1146,12 +1146,20 @@ def deep_scan_fixture(
),
)
scan_id = str(registered["scanId"])
- begin_deep_scan(
- state_dir, "standard-worker-thread", "--scan-id", scan_id, environment=environment
+ run(
+ state_dir,
+ "begin-deep-scan",
+ "--thread-id",
+ "standard-worker-thread",
+ "--scan-id",
+ scan_id,
+ environment=environment,
)
else:
- begun = begin_deep_scan(
+ begun = run(
state_dir,
+ "begin-deep-scan",
+ "--thread-id",
"standard-worker-thread",
"--target-path",
str(target),
@@ -1176,6 +1184,7 @@ def accepted_standard_worker(
scan_id: str,
*,
name: str = "standard-worker",
+ run=run_workbench,
) -> tuple[str, Path]:
worker_id = str(uuid.uuid4())
prompt_path, artifact_dir, result_path = worker_paths(scan_dir, name)
@@ -1195,7 +1204,7 @@ def accepted_standard_worker(
"1",
)
environment = {"CODEX_HOME": str(codex_home)}
- run_workbench(state_dir, *base_args, "--status", "running", environment=environment)
+ run(state_dir, *base_args, "--status", "running", environment=environment)
result_path.write_text(
json.dumps(
{
@@ -1206,7 +1215,7 @@ def accepted_standard_worker(
}
)
)
- run_workbench(
+ run(
state_dir,
*base_args,
"--status",