Skip to content

OpenAI's Codex Security CLI and TypeScript SDK for finding, validating, and fixing security vulnerabilities. npm: https://www.npmjs.com/package/@openai/codex-security

fix(plugin): recognize BOM-marked UTF-16 source files (#716)

faizan-oaidc46b4f1ad92
diff --git a/plugins/codex-security/scripts/rank_preview.py b/plugins/codex-security/scripts/rank_preview.py
index de51532bf..f4cb4cf1f 100644
--- a/plugins/codex-security/scripts/rank_preview.py
+++ b/plugins/codex-security/scripts/rank_preview.py
@@ -11,6 +11,7 @@ from pathlib import Path
DEFAULT_PREVIEW_BYTES = 1024
PREVIEW_HEAD_LINES = 12
PREVIEW_SAMPLE_LINES = 10
+_UTF16_BOMS = (b"\xff\xfe", b"\xfe\xff")
TEXT_CODE_EXTENSIONS = {
".c",
@@ -81,7 +82,14 @@ RUST_LIFETIME_RE = re.compile(r"'[A-Za-z_][A-Za-z0-9_]*")
PHP_HEREDOC_RE = re.compile(r"<<<\s*['\"]?([A-Za-z_]\w*)['\"]?")
+def _decode_source(data: bytes) -> str:
+ encoding = "utf-16" if data.startswith(_UTF16_BOMS) else "utf-8"
+ return data.decode(encoding, errors="ignore")
+
+
def is_binary_sample(data: bytes) -> bool:
+ if data.startswith(_UTF16_BOMS):
+ return "\0" in _decode_source(data)
return b"\0" in data
@@ -958,7 +966,7 @@ def preview_for(
def preview_for_bytes(path: Path, data: bytes, preview_bytes: int) -> tuple[str, bool]:
if is_binary_sample(data):
return "", True
- text = data.decode("utf-8", errors="ignore")
+ text = _decode_source(data)
outline = structural_outline(path, text)
preview_lines = select_preview_lines(outline or text.splitlines())
return fit_preview_lines(preview_lines, preview_bytes), False
diff --git a/plugins/codex-security/tests/test_generate_in_scope_files.py b/plugins/codex-security/tests/test_generate_in_scope_files.py
index dac619aec..b35055963 100644
--- a/plugins/codex-security/tests/test_generate_in_scope_files.py
+++ b/plugins/codex-security/tests/test_generate_in_scope_files.py
@@ -439,6 +439,41 @@ def test_revision_inventory_checks_the_selected_revision_not_the_worktree(tmp_pa
assert output.read_text(encoding="utf-8").splitlines() == ["app/routes.py"]
[email protected]("mode", ["revisions", "local-patch"])
+def test_diff_inventory_includes_bom_marked_utf16_text(tmp_path: Path, mode: str) -> None:
+ repository = make_repository(tmp_path)
+ git(repository, "add", ".")
+ git(repository, "commit", "-qm", "base")
+ base = git(repository, "rev-parse", "HEAD")
+ source = "Write-Output 'café'\n"
+ write_file(repository, "app/utf16-le.ps1", b"\xff\xfe" + source.encode("utf-16-le"))
+ write_file(repository, "app/utf16-be.ps1", b"\xfe\xff" + source.encode("utf-16-be"))
+ write_file(repository, "app/utf8.ps1", source.encode("utf-8"))
+ write_file(repository, "app/binary.ps1", b"text\0binary")
+ write_file(
+ repository,
+ "app/decoded-nul.ps1",
+ b"\xff\xfe" + "text\0binary".encode("utf-16-le"),
+ )
+ write_file(repository, "tests/excluded.ps1", b"\xff\xfe" + source.encode("utf-16-le"))
+
+ arguments = ["--diff-base", base, "--diff-mode", mode]
+ if mode == "revisions":
+ git(repository, "add", ".")
+ git(repository, "commit", "-qm", "add encoded source")
+ arguments.extend(["--diff-head", git(repository, "rev-parse", "HEAD")])
+ output = tmp_path / "in_scope_files.txt"
+
+ result = run_inventory(repository, ".", output, arguments=arguments)
+
+ assert result.returncode == 0, result.stderr
+ assert output.read_text(encoding="utf-8").splitlines() == [
+ "app/utf16-be.ps1",
+ "app/utf16-le.ps1",
+ "app/utf8.ps1",
+ ]
+
+
def test_invalid_diff_revision_preserves_previous_inventory(tmp_path: Path) -> None:
repository = make_repository(tmp_path)
output = tmp_path / "in_scope_files.txt"
diff --git a/plugins/codex-security/tests/test_generate_rank_input.py b/plugins/codex-security/tests/test_generate_rank_input.py
index 359c85c18..4858f9c1a 100644
--- a/plugins/codex-security/tests/test_generate_rank_input.py
+++ b/plugins/codex-security/tests/test_generate_rank_input.py
@@ -817,6 +817,49 @@ def test_make_diff_rank_input_combines_staged_and_unstaged_patch(tmp_path: Path)
assert [row["path"] for row in read_jsonl(output)] == ["src/alpha.py", "src/beta.py"]
[email protected]("mode", ["repo", "explicit-file", "revisions", "local-patch"])
+def test_make_rank_input_decodes_bom_marked_utf16_source(tmp_path: Path, mode: str) -> None:
+ repo = tmp_path / "repo"
+ source_dir = repo / "src"
+ source_dir.mkdir(parents=True)
+ initialize_repo(repo)
+ git(repo, "commit", "--allow-empty", "-qm", "base")
+ base = git(repo, "rev-parse", "HEAD")
+ source = "Write-Output 'café 😀'\n"
+ (source_dir / "utf16-le.ps1").write_bytes(b"\xff\xfe" + source.encode("utf-16-le"))
+ (source_dir / "utf16-be.ps1").write_bytes(b"\xfe\xff" + source.encode("utf-16-be"))
+ (source_dir / "utf8.ps1").write_bytes(source.encode("utf-8"))
+ (source_dir / "binary.ps1").write_bytes(b"text\0binary")
+ (source_dir / "decoded-nul.ps1").write_bytes(b"\xff\xfe" + "text\0binary".encode("utf-16-le"))
+ output = tmp_path / "rank_input.jsonl"
+ expected = {
+ "src/utf16-be.ps1": source.strip(),
+ "src/utf16-le.ps1": source.strip(),
+ "src/utf8.ps1": source.strip(),
+ }
+
+ if mode in {"repo", "explicit-file"}:
+ arguments = ["make-repo-rank-input", "--repo", str(repo)]
+ if mode == "explicit-file":
+ expected.update({"src/binary.ps1": "", "src/decoded-nul.ps1": ""})
+ scopes = tmp_path / "target-paths.json"
+ scopes.write_text(json.dumps(list(expected)), encoding="utf-8")
+ arguments.extend(["--scopes-file", str(scopes)])
+ else:
+ arguments.extend(["--scope", "src"])
+ else:
+ arguments = ["make-diff-rank-input", "--repo", str(repo), "--base", base, "--mode", mode]
+ if mode == "revisions":
+ git(repo, "add", ".")
+ git(repo, "commit", "-qm", "add encoded source")
+ arguments.extend(["--head", git(repo, "rev-parse", "HEAD")])
+ git(repo, "checkout", "-q", base)
+
+ run_cli(*arguments, "--out", str(output))
+
+ assert {row["path"]: row["preview"] for row in read_jsonl(output)} == expected
+
+
def test_copy_and_select_deep_review_inputs(tmp_path: Path) -> None:
rank_input = tmp_path / "rank_input.jsonl"
write_jsonl(
diff --git a/plugins/codex-security/tests/test_rank_preview.py b/plugins/codex-security/tests/test_rank_preview.py
index 87c4b6e5d..c606da5ad 100644
--- a/plugins/codex-security/tests/test_rank_preview.py
+++ b/plugins/codex-security/tests/test_rank_preview.py
@@ -8,7 +8,85 @@ import pytest
PLUGIN_ROOT = Path(__file__).resolve().parents[1]
sys.path.insert(0, str(PLUGIN_ROOT / "scripts"))
-from rank_preview import DEFAULT_PREVIEW_BYTES, preview_for
+from rank_preview import DEFAULT_PREVIEW_BYTES, preview_for, preview_for_bytes
+
+
+ params=[("utf-16-le", b"\xff\xfe"), ("utf-16-be", b"\xfe\xff")],
+ ids=["utf16-le", "utf16-be"],
+)
+def utf16_encoding(request: pytest.FixtureRequest) -> tuple[str, bytes]:
+ return request.param
+
+
+def test_preview_for_decodes_bom_marked_utf16(
+ tmp_path: Path, utf16_encoding: tuple[str, bytes]
+) -> None:
+ encoding, bom = utf16_encoding
+ source = "Write-Output 'café 😀'\n"
+ path = tmp_path / "source.ps1"
+ data = bom + source.encode(encoding)
+ path.write_bytes(data)
+ expected = (source.strip(), False)
+
+ assert preview_for(path, DEFAULT_PREVIEW_BYTES) == expected
+ assert preview_for_bytes(path, data, DEFAULT_PREVIEW_BYTES) == expected
+ assert preview_for_bytes(path, source.encode("utf-8"), DEFAULT_PREVIEW_BYTES) == expected
+
+
+ "data",
+ [
+ b"header\0payload",
+ b"\xff\xfe" + "text\0binary".encode("utf-16-le"),
+ b"\xfe\xff" + "text\0binary".encode("utf-16-be"),
+ "unmarked utf16".encode("utf-16-le"),
+ ],
+ ids=["generic-binary", "utf16-le-nul", "utf16-be-nul", "no-bom"],
+)
+def test_preview_for_rejects_binary_source_bytes(tmp_path: Path, data: bytes) -> None:
+ path = tmp_path / "source.ps1"
+ path.write_bytes(data)
+
+ assert preview_for(path, DEFAULT_PREVIEW_BYTES) == ("", True)
+ assert preview_for_bytes(path, data, DEFAULT_PREVIEW_BYTES) == ("", True)
+
+
+def test_preview_for_handles_utf16_surrogate_at_sample_boundary(
+ tmp_path: Path, utf16_encoding: tuple[str, bytes]
+) -> None:
+ encoding, bom = utf16_encoding
+ source = "a" * 2046 + "😀\nWrite-Output 'done'\n"
+ path = tmp_path / "source.ps1"
+ data = bom + source.encode(encoding)
+ path.write_bytes(data)
+
+ assert preview_for(path, 8192) == (source.strip(), False)
+ assert preview_for_bytes(path, data, 8192) == (source.strip(), False)
+
+
+def test_preview_for_bounds_utf16_source_at_incomplete_character(
+ tmp_path: Path, utf16_encoding: tuple[str, bytes]
+) -> None:
+ encoding, bom = utf16_encoding
+ source = "a" * 32766 + "😀\nWrite-Output 'outside preview'\n"
+ path = tmp_path / "source.ps1"
+ path.write_bytes(bom + source.encode(encoding))
+
+ assert preview_for(path, 128, max_read_bytes=64 * 1024) == ("a" * 128, False)
+
+
+def test_preview_for_ignores_incomplete_utf16_tail(
+ tmp_path: Path, utf16_encoding: tuple[str, bytes]
+) -> None:
+ encoding, bom = utf16_encoding
+ source = "Write-Output 'café'"
+ path = tmp_path / "source.ps1"
+ data = bom + source.encode(encoding) + b"\0"
+ path.write_bytes(data)
+
+ assert preview_for(path, DEFAULT_PREVIEW_BYTES) == (source, False)
+ assert preview_for_bytes(path, data, DEFAULT_PREVIEW_BYTES) == (source, False)
def test_preview_for_does_not_fully_read_a_large_binary(tmp_path: Path) -> None: