Skip to content

OpenAI's Codex Security CLI and TypeScript SDK for finding, validating, and fixing security vulnerabilities. npm: https://www.npmjs.com/package/@openai/codex-security

test(plugin): share source cases across inventory and ranking (#1309)

mldangelo-oaif894f1ff64b1
diff --git a/plugins/codex-security/tests/source_cases.py b/plugins/codex-security/tests/source_cases.py
new file mode 100644
index 000000000..0a832c9a9
--- /dev/null
+++ b/plugins/codex-security/tests/source_cases.py
@@ -0,0 +1,70 @@
+"""Shared source examples for inventory and ranking integration tests.
+
+Add source-selection regressions here so every inventory and ranking mode uses
+the same examples. Expectations are independent of the production allowlist.
+"""
+
+from __future__ import annotations
+
+from typing import NamedTuple
+
+
+class SourceCase(NamedTuple):
+ path: str
+ before: str
+ after: str
+ # Sampled previews preserve the changed text; structural previews override it.
+ preview: str | None = None
+
+
+SOURCE_CASES = (
+ SourceCase(
+ "infra/main.tf",
+ 'variable "enabled" { default = false }',
+ 'variable "enabled" { default = true }',
+ ),
+ SourceCase(
+ "ios/ViewController.m",
+ 'NSString *greeting = @"hello";',
+ 'NSString *greeting = @"goodbye";',
+ ),
+ SourceCase(
+ "ios/Bridge.mm",
+ "@implementation Bridge\n@end",
+ "@implementation Bridge\n@end\n// Changed.",
+ ),
+ SourceCase(
+ "ios/ViewController.h",
+ "@interface ViewController : NSObject\n@end",
+ "@interface ViewController : NSObject\n@end\n// Changed.",
+ ),
+ *(
+ SourceCase(
+ f"include/{name}",
+ "inline int before() { return 1; }",
+ "inline int after() { return 2; }",
+ "function after",
+ )
+ for name in ("base.h", "base.hpp", "lower.hh", "lower.hxx", "upper.HH", "upper.HXX")
+ ),
+ SourceCase("contracts/Vault.sol", "uint256 public limit = 1;", "uint256 public limit = 2;"),
+ SourceCase(
+ "contracts/Vault.vy",
+ "stored: public(uint256)",
+ "stored: public(uint256)\nowner: public(address)",
+ ),
+ SourceCase("src/routes/+page.svelte", "<p>before</p>", "<p>after</p>"),
+ *(
+ SourceCase(f"src/{filename}", content, content.replace("count = 0", "count = 1"))
+ for filename, content in (
+ (
+ "Counter.svelte",
+ '<script lang="ts">\nlet count = 0;\n</script>\n<button>{count}</button>',
+ ),
+ ("routes/profile.ejs", "<% const count = 0; %>\n<p><%= count %></p>"),
+ ("profile.EJS", "<% const count = 0; %>\n<p><%= count %></p>"),
+ ("show.html.erb", "<% count = 0 %>\n<p><%= count %></p>"),
+ ("card.phtml", "<?php $count = 0; ?>\n<p><?= $count ?></p>"),
+ )
+ ),
+)
diff --git a/plugins/codex-security/tests/test_generate_in_scope_files.py b/plugins/codex-security/tests/test_generate_in_scope_files.py
index e0adceb91..c9a36e184 100644
--- a/plugins/codex-security/tests/test_generate_in_scope_files.py
+++ b/plugins/codex-security/tests/test_generate_in_scope_files.py
@@ -6,6 +6,7 @@ import sys
from pathlib import Path, PurePosixPath
import pytest
+from source_cases import SOURCE_CASES
SCRIPT = Path(__file__).resolve().parents[1] / "scripts" / "generate_in_scope_files.py"
@@ -556,75 +557,16 @@ def test_diff_inventory_only_classifies_source_bytes(tmp_path: Path, mode: str)
]
[email protected]("mode", ["revisions", "local-patch"])
-def test_diff_inventory_includes_changed_terraform(tmp_path: Path, mode: str) -> None:
- repository = make_repository(tmp_path)
- write_file(repository, "infra/main.tf", b'variable "enabled" { default = false }\n')
- git(repository, "add", ".")
- git(repository, "commit", "-qm", "base")
- base = git(repository, "rev-parse", "HEAD")
- write_file(repository, "infra/main.tf", b'variable "enabled" { default = true }\n')
- arguments = ["--diff-base", base, "--diff-mode", mode]
- if mode == "revisions":
- git(repository, "add", ".")
- git(repository, "commit", "-qm", "change")
- arguments.extend(["--diff-head", git(repository, "rev-parse", "HEAD")])
- git(repository, "checkout", "-q", base)
- output = tmp_path / "in_scope_files.txt"
-
- result = run_inventory(repository, ".", output, arguments=arguments)
-
- assert result.returncode == 0, result.stderr
- assert output.read_text(encoding="utf-8") == "infra/main.tf\n"
-
-
[email protected]("mode", ["revisions", "local-patch"])
-def test_diff_inventory_includes_changed_objective_c(tmp_path: Path, mode: str) -> None:
- repository = make_repository(tmp_path)
- sources = {
- "ios/Bridge.mm": b"@implementation Bridge\n@end\n",
- "ios/ViewController.h": b"@interface ViewController : NSObject\n@end\n",
- "ios/ViewController.m": b"@implementation ViewController\n@end\n",
- }
- for name, source in sources.items():
- write_file(repository, name, source)
- git(repository, "add", ".")
- git(repository, "commit", "-qm", "base")
- base = git(repository, "rev-parse", "HEAD")
- for name, source in sources.items():
- write_file(repository, name, source + b"// Changed.\n")
- arguments = ["--diff-base", base, "--diff-mode", mode]
- if mode == "revisions":
- git(repository, "add", ".")
- git(repository, "commit", "-qm", "change")
- arguments.extend(["--diff-head", git(repository, "rev-parse", "HEAD")])
- git(repository, "checkout", "-q", base)
- output = tmp_path / "in_scope_files.txt"
-
- result = run_inventory(repository, ".", output, arguments=arguments)
-
- assert result.returncode == 0, result.stderr
- assert output.read_text(encoding="utf-8").splitlines() == sorted(sources)
-
-
@pytest.mark.parametrize("mode", ["revisions", "staged", "unstaged"])
-def test_diff_inventory_includes_changed_cpp_headers(tmp_path: Path, mode: str) -> None:
+def test_diff_inventory_includes_source_cases(tmp_path: Path, mode: str) -> None:
repository = make_repository(tmp_path)
- names = [
- "include/base.h",
- "include/base.hpp",
- "include/lower.hh",
- "include/lower.hxx",
- "include/upper.HH",
- "include/upper.HXX",
- ]
- for name in names:
- write_file(repository, name, b"inline int answer() { return 1; }\n")
+ for case in SOURCE_CASES:
+ write_file(repository, case.path, (case.before + "\n").encode("utf-8"))
git(repository, "add", ".")
git(repository, "commit", "-qm", "base")
base = git(repository, "rev-parse", "HEAD")
- for name in names:
- write_file(repository, name, b"inline int answer() { return 2; }\n")
+ for case in SOURCE_CASES:
+ write_file(repository, case.path, (case.after + "\n").encode("utf-8"))
arguments = ["--diff-base", base, "--diff-mode", "local-patch"]
if mode in {"revisions", "staged"}:
git(repository, "add", ".")
@@ -637,79 +579,9 @@ def test_diff_inventory_includes_changed_cpp_headers(tmp_path: Path, mode: str)
result = run_inventory(repository, ".", output, arguments=arguments)
assert result.returncode == 0, result.stderr
- assert output.read_text(encoding="utf-8").splitlines() == sorted(names)
-
-
[email protected]("mode", ["revisions", "local-patch"])
-def test_diff_inventory_includes_changed_solidity(tmp_path: Path, mode: str) -> None:
- repository = make_repository(tmp_path)
- source = b"pragma solidity ^0.8.24;\ncontract Vault {}\n"
- write_file(repository, "contracts/Vault.sol", source)
- git(repository, "add", ".")
- git(repository, "commit", "-qm", "base")
- base = git(repository, "rev-parse", "HEAD")
- write_file(repository, "contracts/Vault.sol", source + b"// Changed.\n")
- arguments = ["--diff-base", base, "--diff-mode", mode]
- if mode == "revisions":
- git(repository, "add", ".")
- git(repository, "commit", "-qm", "change")
- arguments.extend(["--diff-head", git(repository, "rev-parse", "HEAD")])
- git(repository, "checkout", "-q", base)
- output = tmp_path / "in_scope_files.txt"
-
- result = run_inventory(repository, ".", output, arguments=arguments)
-
- assert result.returncode == 0, result.stderr
- assert output.read_text(encoding="utf-8") == "contracts/Vault.sol\n"
-
-
[email protected]("mode", ["revisions", "local-patch"])
-def test_diff_inventory_includes_changed_vyper(tmp_path: Path, mode: str) -> None:
- repository = make_repository(tmp_path)
- source = b"stored: public(uint256)\n"
- write_file(repository, "contracts/Vault.vy", source)
- git(repository, "add", ".")
- git(repository, "commit", "-qm", "base")
- base = git(repository, "rev-parse", "HEAD")
- write_file(repository, "contracts/Vault.vy", source + b"# Changed.\n")
- arguments = ["--diff-base", base, "--diff-mode", mode]
- if mode == "revisions":
- git(repository, "add", ".")
- git(repository, "commit", "-qm", "change")
- arguments.extend(["--diff-head", git(repository, "rev-parse", "HEAD")])
- git(repository, "checkout", "-q", base)
- output = tmp_path / "in_scope_files.txt"
-
- result = run_inventory(repository, ".", output, arguments=arguments)
-
- assert result.returncode == 0, result.stderr
- assert output.read_text(encoding="utf-8") == "contracts/Vault.vy\n"
-
-
[email protected]("mode", ["revisions", "local-patch"])
[email protected]("filename", ["+page.svelte", "profile.ejs", "show.html.erb", "card.phtml"])
-def test_diff_inventory_includes_changed_templates(
- tmp_path: Path, mode: str, filename: str
-) -> None:
- repository = make_repository(tmp_path)
- path = f"src/routes/{filename}"
- write_file(repository, path, b"<p>before</p>\n")
- git(repository, "add", ".")
- git(repository, "commit", "-qm", "base")
- base = git(repository, "rev-parse", "HEAD")
- write_file(repository, path, b"<p>after</p>\n")
- arguments = ["--diff-base", base, "--diff-mode", mode]
- if mode == "revisions":
- git(repository, "add", ".")
- git(repository, "commit", "-qm", "change")
- arguments.extend(["--diff-head", git(repository, "rev-parse", "HEAD")])
- git(repository, "checkout", "-q", base)
- output = tmp_path / "in_scope_files.txt"
-
- result = run_inventory(repository, ".", output, arguments=arguments)
-
- assert result.returncode == 0, result.stderr
- assert output.read_text(encoding="utf-8") == f"{path}\n"
+ assert output.read_text(encoding="utf-8").splitlines() == sorted(
+ case.path for case in SOURCE_CASES
+ )
def test_diff_inventory_keeps_every_javascript_module_extension(tmp_path: Path) -> None:
diff --git a/plugins/codex-security/tests/test_generate_rank_input.py b/plugins/codex-security/tests/test_generate_rank_input.py
index 004a3903d..70868900a 100644
--- a/plugins/codex-security/tests/test_generate_rank_input.py
+++ b/plugins/codex-security/tests/test_generate_rank_input.py
@@ -8,6 +8,7 @@ from pathlib import Path
from unittest.mock import patch
import pytest
+from source_cases import SOURCE_CASES
PLUGIN_ROOT = Path(__file__).resolve().parents[1]
SCRIPT = PLUGIN_ROOT / "scripts" / "generate_rank_input.py"
@@ -271,94 +272,26 @@ def test_rank_input_bounds_large_text_reads(
assert rows[0]["preview"] == "function visible()"
[email protected]("mode", ["repo", "revisions", "local-patch"])
-def test_rank_input_includes_terraform(tmp_path: Path, mode: str) -> None:
- repo = tmp_path / "repo"
- infra = repo / "infra"
- infra.mkdir(parents=True)
- initialize_repo(repo)
- source = infra / "main.tf"
- source.write_text('variable "enabled" { default = false }\n', encoding="utf-8")
- git(repo, "add", ".")
- git(repo, "commit", "-qm", "base")
- base = git(repo, "rev-parse", "HEAD")
- changed = 'variable "enabled" { default = true }'
- source.write_text(changed + "\n", encoding="utf-8")
- output = tmp_path / "rank_input.jsonl"
-
- if mode == "repo":
- arguments = ["make-repo-rank-input", "--repo", str(repo), "--scope", "infra"]
- else:
- arguments = ["make-diff-rank-input", "--repo", str(repo), "--base", base, "--mode", mode]
- if mode == "revisions":
- git(repo, "add", ".")
- git(repo, "commit", "-qm", "change")
- arguments.extend(["--head", git(repo, "rev-parse", "HEAD")])
- git(repo, "checkout", "-q", base)
-
- run_cli(*arguments, "--out", str(output))
-
- assert read_jsonl(output) == [
- {"path": "infra/main.tf", "area": "infra" if mode == "repo" else "diff", "preview": changed}
- ]
-
-
[email protected]("mode", ["repo", "revisions", "local-patch"])
-def test_rank_input_includes_objective_c(tmp_path: Path, mode: str) -> None:
- repo = tmp_path / "repo"
- ios = repo / "ios"
- ios.mkdir(parents=True)
- initialize_repo(repo)
- source = ios / "ViewController.m"
- source.write_text('NSString *greeting = @"hello";\n', encoding="utf-8")
- git(repo, "add", ".")
- git(repo, "commit", "-qm", "base")
- base = git(repo, "rev-parse", "HEAD")
- changed = 'NSString *greeting = @"goodbye";'
- source.write_text(changed + "\n", encoding="utf-8")
- output = tmp_path / "rank_input.jsonl"
-
- if mode == "repo":
- arguments = ["make-repo-rank-input", "--repo", str(repo), "--scope", "ios"]
- else:
- arguments = ["make-diff-rank-input", "--repo", str(repo), "--base", base, "--mode", mode]
- if mode == "revisions":
- git(repo, "add", ".")
- git(repo, "commit", "-qm", "change")
- arguments.extend(["--head", git(repo, "rev-parse", "HEAD")])
- git(repo, "checkout", "-q", base)
-
- run_cli(*arguments, "--out", str(output))
-
- assert read_jsonl(output) == [
- {
- "path": "ios/ViewController.m",
- "area": "ios" if mode == "repo" else "diff",
- "preview": changed,
- }
- ]
-
-
@pytest.mark.parametrize("mode", ["repo", "revisions", "staged", "unstaged"])
-def test_rank_input_includes_cpp_headers(tmp_path: Path, mode: str) -> None:
+def test_rank_input_includes_source_cases(tmp_path: Path, mode: str) -> None:
repo = tmp_path / "repo"
- include = repo / "include"
- include.mkdir(parents=True)
+ repo.mkdir()
initialize_repo(repo)
- names = ["base.h", "base.hpp", "lower.hh", "lower.hxx", "upper.HH", "upper.HXX"]
- for name in names:
- (include / name).write_text("inline int before() { return 1; }\n", encoding="utf-8")
+ for case in SOURCE_CASES:
+ source = repo / case.path
+ source.parent.mkdir(parents=True, exist_ok=True)
+ source.write_text(case.before + "\n", encoding="utf-8")
git(repo, "add", ".")
git(repo, "commit", "-qm", "base")
base = git(repo, "rev-parse", "HEAD")
- for name in names:
- (include / name).write_text("inline int after() { return 2; }\n", encoding="utf-8")
+ for case in SOURCE_CASES:
+ (repo / case.path).write_text(case.after + "\n", encoding="utf-8")
output = tmp_path / "rank_input.jsonl"
if mode in {"revisions", "staged"}:
git(repo, "add", ".")
if mode == "repo":
- arguments = ["make-repo-rank-input", "--repo", str(repo), "--scope", "include"]
+ arguments = ["make-repo-rank-input", "--repo", str(repo), "--scope", "."]
else:
diff_mode = "revisions" if mode == "revisions" else "local-patch"
arguments = [
@@ -377,133 +310,13 @@ def test_rank_input_includes_cpp_headers(tmp_path: Path, mode: str) -> None:
run_cli(*arguments, "--out", str(output))
- rows = read_jsonl(output)
- assert [row["path"] for row in rows] == [f"include/{name}" for name in sorted(names)]
- for row in rows:
- assert row["area"] == ("include" if mode == "repo" else "diff")
- assert "function after" in row["preview"]
- assert "before" not in row["preview"]
-
-
[email protected]("mode", ["repo", "revisions", "local-patch"])
-def test_rank_input_includes_solidity(tmp_path: Path, mode: str) -> None:
- repo = tmp_path / "repo"
- contracts = repo / "contracts"
- contracts.mkdir(parents=True)
- initialize_repo(repo)
- source = contracts / "Vault.sol"
- source.write_text("uint256 public limit = 1;\n", encoding="utf-8")
- git(repo, "add", ".")
- git(repo, "commit", "-qm", "base")
- base = git(repo, "rev-parse", "HEAD")
- changed = "uint256 public limit = 2;"
- source.write_text(changed + "\n", encoding="utf-8")
- output = tmp_path / "rank_input.jsonl"
-
- if mode == "repo":
- arguments = ["make-repo-rank-input", "--repo", str(repo), "--scope", "contracts"]
- else:
- arguments = ["make-diff-rank-input", "--repo", str(repo), "--base", base, "--mode", mode]
- if mode == "revisions":
- git(repo, "add", ".")
- git(repo, "commit", "-qm", "change")
- arguments.extend(["--head", git(repo, "rev-parse", "HEAD")])
- git(repo, "checkout", "-q", base)
-
- run_cli(*arguments, "--out", str(output))
-
- assert read_jsonl(output) == [
- {
- "path": "contracts/Vault.sol",
- "area": "contracts" if mode == "repo" else "diff",
- "preview": changed,
- }
- ]
-
-
[email protected]("mode", ["repo", "revisions", "local-patch"])
-def test_rank_input_includes_vyper(tmp_path: Path, mode: str) -> None:
- repo = tmp_path / "repo"
- contracts = repo / "contracts"
- contracts.mkdir(parents=True)
- initialize_repo(repo)
- source = contracts / "Vault.vy"
- source.write_text("stored: public(uint256)\n", encoding="utf-8")
- git(repo, "add", ".")
- git(repo, "commit", "-qm", "base")
- base = git(repo, "rev-parse", "HEAD")
- changed = "stored: public(uint256)\nowner: public(address)"
- source.write_text(changed + "\n", encoding="utf-8")
- output = tmp_path / "rank_input.jsonl"
-
- if mode == "repo":
- arguments = ["make-repo-rank-input", "--repo", str(repo), "--scope", "contracts"]
- else:
- arguments = ["make-diff-rank-input", "--repo", str(repo), "--base", base, "--mode", mode]
- if mode == "revisions":
- git(repo, "add", ".")
- git(repo, "commit", "-qm", "change")
- arguments.extend(["--head", git(repo, "rev-parse", "HEAD")])
- git(repo, "checkout", "-q", base)
-
- run_cli(*arguments, "--out", str(output))
-
- assert read_jsonl(output) == [
- {
- "path": "contracts/Vault.vy",
- "area": "contracts" if mode == "repo" else "diff",
- "preview": changed,
- }
- ]
-
-
[email protected]("mode", ["repo", "revisions", "local-patch"])
- ("filename", "content"),
- [
- (
- "Counter.svelte",
- '<script lang="ts">\nlet count = 0;\n</script>\n<button>{count}</button>',
- ),
- ("profile.EJS", "<% const count = 0; %>\n<p><%= count %></p>"),
- ("show.html.erb", "<% count = 0 %>\n<p><%= count %></p>"),
- ("card.phtml", "<?php $count = 0; ?>\n<p><?= $count ?></p>"),
- ],
-)
-def test_rank_input_includes_templates(
- tmp_path: Path, mode: str, filename: str, content: str
-) -> None:
- repo = tmp_path / "repo"
- components = repo / "src"
- components.mkdir(parents=True)
- initialize_repo(repo)
- source = components / filename
- source.write_text(content + "\n", encoding="utf-8")
- git(repo, "add", ".")
- git(repo, "commit", "-qm", "base")
- base = git(repo, "rev-parse", "HEAD")
- changed = content.replace("count = 0", "count = 1")
- source.write_text(changed + "\n", encoding="utf-8")
- output = tmp_path / "rank_input.jsonl"
-
- if mode == "repo":
- arguments = ["make-repo-rank-input", "--repo", str(repo), "--scope", "src"]
- else:
- arguments = ["make-diff-rank-input", "--repo", str(repo), "--base", base, "--mode", mode]
- if mode == "revisions":
- git(repo, "add", ".")
- git(repo, "commit", "-qm", "change")
- arguments.extend(["--head", git(repo, "rev-parse", "HEAD")])
- git(repo, "checkout", "-q", base)
-
- run_cli(*arguments, "--out", str(output))
-
assert read_jsonl(output) == [
{
- "path": f"src/{filename}",
- "area": "src" if mode == "repo" else "diff",
- "preview": changed,
+ "path": case.path,
+ "area": "." if mode == "repo" else "diff",
+ "preview": case.after if case.preview is None else case.preview,
}
+ for case in sorted(SOURCE_CASES, key=lambda case: case.path)
]
diff --git a/plugins/codex-security/tests/test_rank_preview.py b/plugins/codex-security/tests/test_rank_preview.py
index fb24d771a..d21577259 100644
--- a/plugins/codex-security/tests/test_rank_preview.py
+++ b/plugins/codex-security/tests/test_rank_preview.py
@@ -866,16 +866,10 @@ def test_fallback_preview_omits_marker_when_no_lines_are_skipped(tmp_path: Path)
assert "..." not in preview
- "filename",
- ["styles.css", "main.tf", "ViewController.m", "Vault.sol", "Vault.vy", "Counter.svelte"],
-)
-def test_preview_byte_budget_preserves_sampled_tail_and_valid_unicode(
- tmp_path: Path, filename: str
-) -> None:
+def test_preview_byte_budget_preserves_sampled_tail_and_valid_unicode(tmp_path: Path) -> None:
source = "\n".join(f"line_{index:02d} {'😀' * 20}" for index in range(40))
- preview = generate_preview(tmp_path, filename, source, preview_bytes=220)
+ preview = generate_preview(tmp_path, "styles.css", source, preview_bytes=220)
assert len(preview.encode("utf-8")) <= 220
assert "..." in preview