Skip to content

OpenAI's Codex Security CLI and TypeScript SDK for finding, validating, and fixing security vulnerabilities. npm: https://www.npmjs.com/package/@openai/codex-security

fix(plugin): bound source preview reads (#947)

mluck-openaiead0ee98dee8
diff --git a/plugins/codex-security/scripts/generate_rank_input.py b/plugins/codex-security/scripts/generate_rank_input.py
index 05c5b1521..e38af2c1b 100644
--- a/plugins/codex-security/scripts/generate_rank_input.py
+++ b/plugins/codex-security/scripts/generate_rank_input.py
@@ -124,7 +124,6 @@ EXCLUDED_FILENAMES = {
SHARD_INPUT_GLOB = "rank-shard-*.input.jsonl"
SHARD_OUTPUT_GLOB = "rank-shard-*.output.jsonl"
SHARD_INPUT_PATTERN = re.compile(r"^rank-shard-([0-9]{4,})\.input\.jsonl$")
-DIRECT_SCOPE_PREVIEW_READ_BYTES = 64 * 1024
RANK_POOL_PLAN_SCHEMA_VERSION = 1
RANK_POOL_STRATEGY = "round_robin"
RANK_POOL_WORKER_CAP = 6
@@ -507,11 +506,7 @@ def make_repo_rank_input(args: argparse.Namespace) -> None:
):
preview = ""
else:
- preview, is_binary = preview_for(
- path,
- args.preview_bytes,
- max_read_bytes=DIRECT_SCOPE_PREVIEW_READ_BYTES if directly_requested else None,
- )
+ preview, is_binary = preview_for(path, args.preview_bytes)
if is_binary and not directly_requested:
continue
rows_by_path.setdefault(
diff --git a/plugins/codex-security/scripts/rank_preview.py b/plugins/codex-security/scripts/rank_preview.py
index d24c7e564..2963a037e 100644
--- a/plugins/codex-security/scripts/rank_preview.py
+++ b/plugins/codex-security/scripts/rank_preview.py
@@ -9,6 +9,7 @@ import re
from pathlib import Path
DEFAULT_PREVIEW_BYTES = 1024
+DEFAULT_PREVIEW_READ_BYTES = 64 * 1024
PREVIEW_HEAD_LINES = 12
PREVIEW_SAMPLE_LINES = 10
_UTF16_BOMS = (b"\xff\xfe", b"\xfe\xff")
@@ -949,18 +950,17 @@ def structural_outline(path: Path, text: str) -> list[str]:
def preview_for(
- path: Path, preview_bytes: int, *, max_read_bytes: int | None = None
+ path: Path, preview_bytes: int, *, max_read_bytes: int = DEFAULT_PREVIEW_READ_BYTES
) -> tuple[str, bool]:
+ """Preview the first 64 KiB by default, including the binary-detection sample."""
+ if max_read_bytes <= 0:
+ raise ValueError("max_read_bytes must be positive")
try:
with path.open("rb") as source:
- sample = source.read(4096)
+ sample = source.read(min(4096, max_read_bytes))
if is_binary_sample(sample):
return "", True
- remaining = (
- source.read()
- if max_read_bytes is None
- else source.read(max(0, max_read_bytes - len(sample)))
- )
+ remaining = source.read(max_read_bytes - len(sample))
data = sample + remaining
except OSError:
return "", True
diff --git a/plugins/codex-security/tests/test_generate_rank_input.py b/plugins/codex-security/tests/test_generate_rank_input.py
index 1eeaad972..3f1abc340 100644
--- a/plugins/codex-security/tests/test_generate_rank_input.py
+++ b/plugins/codex-security/tests/test_generate_rank_input.py
@@ -2,9 +2,11 @@ from __future__ import annotations
import hashlib
import json
+import runpy
import subprocess
import sys
from pathlib import Path
+from unittest.mock import patch
import pytest
@@ -171,6 +173,60 @@ def test_make_repo_rank_input_matches_golden_and_filters_noise(tmp_path: Path) -
)
[email protected]("scope", [".", "src", "src/large.py", "explicit", "overlap", "diff"])
+def test_rank_input_bounds_large_text_reads(
+ tmp_path: Path, monkeypatch: pytest.MonkeyPatch, scope: str
+) -> None:
+ repo = tmp_path / "repo"
+ (repo / "src").mkdir(parents=True)
+ source = repo / "src" / "large.py"
+ source.write_bytes(b"def visible():\n pass\n" + b"# source comment\n" * (128 * 1024))
+ output = tmp_path / "rank_input.jsonl"
+ arguments = ["make-repo-rank-input", "--repo", str(repo), "--out", str(output)]
+ if scope in {"explicit", "overlap"}:
+ scopes = tmp_path / "scopes.json"
+ scopes.write_text(
+ json.dumps(["src/large.py"] if scope == "explicit" else ["src", "src/large.py"])
+ )
+ arguments.extend(["--scopes-file", str(scopes)])
+ elif scope == "diff":
+ initialize_repo(repo)
+ git(repo, "commit", "--allow-empty", "-qm", "initial")
+ arguments[0] = "make-diff-rank-input"
+ arguments.extend(["--mode", "local-patch", "--base", "HEAD"])
+ else:
+ arguments.extend(["--scope", scope])
+
+ original_open = Path.open
+ reads: list[int] = []
+
+ def tracked_open(path: Path, *args, **kwargs):
+ reader = original_open(path, *args, **kwargs)
+ if path == source:
+ read = reader.read
+ index = len(reads)
+ reads.append(0)
+
+ def bounded_read(size: int = -1) -> bytes:
+ assert 0 <= size <= 64 * 1024 - reads[index]
+ data = read(size)
+ reads[index] += len(data)
+ return data
+
+ reader.read = bounded_read
+ return reader
+
+ monkeypatch.setattr(sys, "argv", [str(SCRIPT), *arguments])
+ with patch.object(Path, "open", tracked_open):
+ runpy.run_path(str(SCRIPT), run_name="__main__")
+
+ assert reads and all(count == 64 * 1024 for count in reads)
+ rows = read_jsonl(output)
+ assert len(rows) == 1
+ assert rows[0]["path"] == "src/large.py"
+ assert rows[0]["preview"] == "function visible()"
+
+
@pytest.mark.parametrize("mode", ["repo", "revisions", "local-patch"])
def test_rank_input_includes_terraform(tmp_path: Path, mode: str) -> None:
repo = tmp_path / "repo"
diff --git a/plugins/codex-security/tests/test_rank_preview.py b/plugins/codex-security/tests/test_rank_preview.py
index d1fd3a01d..f1b44582e 100644
--- a/plugins/codex-security/tests/test_rank_preview.py
+++ b/plugins/codex-security/tests/test_rank_preview.py
@@ -109,6 +109,62 @@ def test_preview_for_bounds_a_source_like_binary_after_the_initial_sample(tmp_pa
assert preview_for(source, DEFAULT_PREVIEW_BYTES, max_read_bytes=64 * 1024) == ("", True)
[email protected]("max_read_bytes", [None, 1024, 4096, 4097, 128 * 1024])
+def test_preview_for_bounds_large_text_reads(tmp_path: Path, max_read_bytes: int | None) -> None:
+ source = tmp_path / "large.py"
+ source.write_bytes(b"# source comment\n" * (128 * 1024))
+ limit = 64 * 1024 if max_read_bytes is None else max_read_bytes
+ bytes_read = 0
+ with source.open("rb") as reader:
+ read = reader.read
+
+ def bounded_read(size: int = -1) -> bytes:
+ nonlocal bytes_read
+ assert 0 <= size <= limit - bytes_read
+ data = read(size)
+ bytes_read += len(data)
+ return data
+
+ with (
+ patch.object(Path, "open", return_value=reader),
+ patch.object(reader, "read", side_effect=bounded_read),
+ ):
+ if max_read_bytes is None:
+ preview, binary = preview_for(source, 128)
+ else:
+ preview, binary = preview_for(source, 128, max_read_bytes=max_read_bytes)
+
+ assert bytes_read == limit
+ assert not binary
+ assert preview
+ assert len(preview.encode("utf-8")) <= 128
+
+
[email protected]("max_read_bytes", [0, -1])
+def test_preview_for_rejects_nonpositive_read_limit(max_read_bytes: int) -> None:
+ with patch.object(Path, "open", side_effect=AssertionError("unexpected file read")):
+ with pytest.raises(ValueError, match="max_read_bytes must be positive"):
+ preview_for(Path("source.py"), 128, max_read_bytes=max_read_bytes)
+
+
+ ("filename", "prefix", "suffix", "expected"),
+ [
+ ("source.py", b"def visible():\n value = (", b"1)\n", "function visible"),
+ ("source.css", b"body { color: red; }\n\xf0\x9f", b"\x98\x80", "body { color: red; }"),
+ ("source.css", b"body { color: red; }\n", b"\0binary", "body { color: red; }"),
+ ],
+)
+def test_preview_for_uses_only_the_bounded_prefix(
+ tmp_path: Path, filename: str, prefix: bytes, suffix: bytes, expected: str
+) -> None:
+ source = tmp_path / filename
+ source.write_bytes(prefix + suffix)
+
+ for _ in range(2):
+ assert preview_for(source, 128, max_read_bytes=len(prefix)) == (expected, False)
+
+
def generate_preview(
tmp_path: Path, filename: str, source: str, *, preview_bytes: int | None = None
) -> str: