OpenAI's Codex Security CLI and TypeScript SDK for finding, validating, and fixing security vulnerabilities. npm: https://www.npmjs.com/package/@openai/codex-security
TypeScript ai-securityapplication-securityclicode-scanningcodexcodex-securitycybersecuritydevsecopsnodejsnpmopenaisdksecuritytypescriptvulnerability-scanning
fix(plugin): recognize BOM-marked UTF-16 source files (#716)
faizan-oaidc46b4f1ad92
diff --git a/plugins/codex-security/scripts/rank_preview.py b/plugins/codex-security/scripts/rank_preview.pyindex de51532bf..f4cb4cf1f 100644--- a/plugins/codex-security/scripts/rank_preview.py+++ b/plugins/codex-security/scripts/rank_preview.py@@ -11,6 +11,7 @@ from pathlib import PathDEFAULT_PREVIEW_BYTES = 1024PREVIEW_HEAD_LINES = 12PREVIEW_SAMPLE_LINES = 10+_UTF16_BOMS = (b"\xff\xfe", b"\xfe\xff")TEXT_CODE_EXTENSIONS = {".c",@@ -81,7 +82,14 @@ RUST_LIFETIME_RE = re.compile(r"'[A-Za-z_][A-Za-z0-9_]*")PHP_HEREDOC_RE = re.compile(r"<<<\s*['\"]?([A-Za-z_]\w*)['\"]?")+def _decode_source(data: bytes) -> str:+ encoding = "utf-16" if data.startswith(_UTF16_BOMS) else "utf-8"+ return data.decode(encoding, errors="ignore")++def is_binary_sample(data: bytes) -> bool:+ if data.startswith(_UTF16_BOMS):+ return "\0" in _decode_source(data)return b"\0" in data@@ -958,7 +966,7 @@ def preview_for(def preview_for_bytes(path: Path, data: bytes, preview_bytes: int) -> tuple[str, bool]:if is_binary_sample(data):return "", True- text = data.decode("utf-8", errors="ignore")+ text = _decode_source(data)outline = structural_outline(path, text)preview_lines = select_preview_lines(outline or text.splitlines())return fit_preview_lines(preview_lines, preview_bytes), Falsediff --git a/plugins/codex-security/tests/test_generate_in_scope_files.py b/plugins/codex-security/tests/test_generate_in_scope_files.pyindex dac619aec..b35055963 100644--- a/plugins/codex-security/tests/test_generate_in_scope_files.py+++ b/plugins/codex-security/tests/test_generate_in_scope_files.py@@ -439,6 +439,41 @@ def test_revision_inventory_checks_the_selected_revision_not_the_worktree(tmp_paassert output.read_text(encoding="utf-8").splitlines() == ["app/routes.py"][email protected]("mode", ["revisions", "local-patch"])+def test_diff_inventory_includes_bom_marked_utf16_text(tmp_path: Path, mode: str) -> None:+ repository = make_repository(tmp_path)+ git(repository, "add", ".")+ git(repository, "commit", "-qm", "base")+ base = git(repository, "rev-parse", "HEAD")+ source = "Write-Output 'café'\n"+ write_file(repository, "app/utf16-le.ps1", b"\xff\xfe" + source.encode("utf-16-le"))+ write_file(repository, "app/utf16-be.ps1", b"\xfe\xff" + source.encode("utf-16-be"))+ write_file(repository, "app/utf8.ps1", source.encode("utf-8"))+ write_file(repository, "app/binary.ps1", b"text\0binary")+ write_file(+ repository,+ "app/decoded-nul.ps1",+ b"\xff\xfe" + "text\0binary".encode("utf-16-le"),+ )+ write_file(repository, "tests/excluded.ps1", b"\xff\xfe" + source.encode("utf-16-le"))++ arguments = ["--diff-base", base, "--diff-mode", mode]+ if mode == "revisions":+ git(repository, "add", ".")+ git(repository, "commit", "-qm", "add encoded source")+ arguments.extend(["--diff-head", git(repository, "rev-parse", "HEAD")])+ output = tmp_path / "in_scope_files.txt"++ result = run_inventory(repository, ".", output, arguments=arguments)++ assert result.returncode == 0, result.stderr+ assert output.read_text(encoding="utf-8").splitlines() == [+ "app/utf16-be.ps1",+ "app/utf16-le.ps1",+ "app/utf8.ps1",+ ]++def test_invalid_diff_revision_preserves_previous_inventory(tmp_path: Path) -> None:repository = make_repository(tmp_path)output = tmp_path / "in_scope_files.txt"diff --git a/plugins/codex-security/tests/test_generate_rank_input.py b/plugins/codex-security/tests/test_generate_rank_input.pyindex 359c85c18..4858f9c1a 100644--- a/plugins/codex-security/tests/test_generate_rank_input.py+++ b/plugins/codex-security/tests/test_generate_rank_input.py@@ -817,6 +817,49 @@ def test_make_diff_rank_input_combines_staged_and_unstaged_patch(tmp_path: Path)assert [row["path"] for row in read_jsonl(output)] == ["src/alpha.py", "src/beta.py"][email protected]("mode", ["repo", "explicit-file", "revisions", "local-patch"])+def test_make_rank_input_decodes_bom_marked_utf16_source(tmp_path: Path, mode: str) -> None:+ repo = tmp_path / "repo"+ source_dir = repo / "src"+ source_dir.mkdir(parents=True)+ initialize_repo(repo)+ git(repo, "commit", "--allow-empty", "-qm", "base")+ base = git(repo, "rev-parse", "HEAD")+ source = "Write-Output 'café 😀'\n"+ (source_dir / "utf16-le.ps1").write_bytes(b"\xff\xfe" + source.encode("utf-16-le"))+ (source_dir / "utf16-be.ps1").write_bytes(b"\xfe\xff" + source.encode("utf-16-be"))+ (source_dir / "utf8.ps1").write_bytes(source.encode("utf-8"))+ (source_dir / "binary.ps1").write_bytes(b"text\0binary")+ (source_dir / "decoded-nul.ps1").write_bytes(b"\xff\xfe" + "text\0binary".encode("utf-16-le"))+ output = tmp_path / "rank_input.jsonl"+ expected = {+ "src/utf16-be.ps1": source.strip(),+ "src/utf16-le.ps1": source.strip(),+ "src/utf8.ps1": source.strip(),+ }++ if mode in {"repo", "explicit-file"}:+ arguments = ["make-repo-rank-input", "--repo", str(repo)]+ if mode == "explicit-file":+ expected.update({"src/binary.ps1": "", "src/decoded-nul.ps1": ""})+ scopes = tmp_path / "target-paths.json"+ scopes.write_text(json.dumps(list(expected)), encoding="utf-8")+ arguments.extend(["--scopes-file", str(scopes)])+ else:+ arguments.extend(["--scope", "src"])+ else:+ arguments = ["make-diff-rank-input", "--repo", str(repo), "--base", base, "--mode", mode]+ if mode == "revisions":+ git(repo, "add", ".")+ git(repo, "commit", "-qm", "add encoded source")+ arguments.extend(["--head", git(repo, "rev-parse", "HEAD")])+ git(repo, "checkout", "-q", base)++ run_cli(*arguments, "--out", str(output))++ assert {row["path"]: row["preview"] for row in read_jsonl(output)} == expected++def test_copy_and_select_deep_review_inputs(tmp_path: Path) -> None:rank_input = tmp_path / "rank_input.jsonl"write_jsonl(diff --git a/plugins/codex-security/tests/test_rank_preview.py b/plugins/codex-security/tests/test_rank_preview.pyindex 87c4b6e5d..c606da5ad 100644--- a/plugins/codex-security/tests/test_rank_preview.py+++ b/plugins/codex-security/tests/test_rank_preview.py@@ -8,7 +8,85 @@ import pytestPLUGIN_ROOT = Path(__file__).resolve().parents[1]sys.path.insert(0, str(PLUGIN_ROOT / "scripts"))-from rank_preview import DEFAULT_PREVIEW_BYTES, preview_for+from rank_preview import DEFAULT_PREVIEW_BYTES, preview_for, preview_for_bytes+++ params=[("utf-16-le", b"\xff\xfe"), ("utf-16-be", b"\xfe\xff")],+ ids=["utf16-le", "utf16-be"],+)+def utf16_encoding(request: pytest.FixtureRequest) -> tuple[str, bytes]:+ return request.param+++def test_preview_for_decodes_bom_marked_utf16(+ tmp_path: Path, utf16_encoding: tuple[str, bytes]+) -> None:+ encoding, bom = utf16_encoding+ source = "Write-Output 'café 😀'\n"+ path = tmp_path / "source.ps1"+ data = bom + source.encode(encoding)+ path.write_bytes(data)+ expected = (source.strip(), False)++ assert preview_for(path, DEFAULT_PREVIEW_BYTES) == expected+ assert preview_for_bytes(path, data, DEFAULT_PREVIEW_BYTES) == expected+ assert preview_for_bytes(path, source.encode("utf-8"), DEFAULT_PREVIEW_BYTES) == expected+++ "data",+ [+ b"header\0payload",+ b"\xff\xfe" + "text\0binary".encode("utf-16-le"),+ b"\xfe\xff" + "text\0binary".encode("utf-16-be"),+ "unmarked utf16".encode("utf-16-le"),+ ],+ ids=["generic-binary", "utf16-le-nul", "utf16-be-nul", "no-bom"],+)+def test_preview_for_rejects_binary_source_bytes(tmp_path: Path, data: bytes) -> None:+ path = tmp_path / "source.ps1"+ path.write_bytes(data)++ assert preview_for(path, DEFAULT_PREVIEW_BYTES) == ("", True)+ assert preview_for_bytes(path, data, DEFAULT_PREVIEW_BYTES) == ("", True)+++def test_preview_for_handles_utf16_surrogate_at_sample_boundary(+ tmp_path: Path, utf16_encoding: tuple[str, bytes]+) -> None:+ encoding, bom = utf16_encoding+ source = "a" * 2046 + "😀\nWrite-Output 'done'\n"+ path = tmp_path / "source.ps1"+ data = bom + source.encode(encoding)+ path.write_bytes(data)++ assert preview_for(path, 8192) == (source.strip(), False)+ assert preview_for_bytes(path, data, 8192) == (source.strip(), False)+++def test_preview_for_bounds_utf16_source_at_incomplete_character(+ tmp_path: Path, utf16_encoding: tuple[str, bytes]+) -> None:+ encoding, bom = utf16_encoding+ source = "a" * 32766 + "😀\nWrite-Output 'outside preview'\n"+ path = tmp_path / "source.ps1"+ path.write_bytes(bom + source.encode(encoding))++ assert preview_for(path, 128, max_read_bytes=64 * 1024) == ("a" * 128, False)+++def test_preview_for_ignores_incomplete_utf16_tail(+ tmp_path: Path, utf16_encoding: tuple[str, bytes]+) -> None:+ encoding, bom = utf16_encoding+ source = "Write-Output 'café'"+ path = tmp_path / "source.ps1"+ data = bom + source.encode(encoding) + b"\0"+ path.write_bytes(data)++ assert preview_for(path, DEFAULT_PREVIEW_BYTES) == (source, False)+ assert preview_for_bytes(path, data, DEFAULT_PREVIEW_BYTES) == (source, False)def test_preview_for_does_not_fully_read_a_large_binary(tmp_path: Path) -> None: