Skip to content

OpenAI's Codex Security CLI and TypeScript SDK for finding, validating, and fixing security vulnerabilities. npm: https://www.npmjs.com/package/@openai/codex-security

fix(plugin): include text files in scans regardless of extension (#1347)

mldangelo-oai7ac5a8bec74e
diff --git a/plugins/codex-security/scripts/generate_in_scope_files.py b/plugins/codex-security/scripts/generate_in_scope_files.py
index f36e98fc5..32873c8fa 100644
--- a/plugins/codex-security/scripts/generate_in_scope_files.py
+++ b/plugins/codex-security/scripts/generate_in_scope_files.py
@@ -12,7 +12,7 @@ from pathlib import Path
# Some plugin hosts launch Python with safe-path isolation enabled.
sys.path.insert(0, str(Path(__file__).resolve().parent))
-from workbench_target import git_blob_bytes, git_command
+from workbench_target import git_blob_samples, git_command
class InventoryError(ValueError):
@@ -161,6 +161,7 @@ def committed_changed_paths(repository: Path, base: str, head: str) -> list[tupl
result = git_command(
repository,
"diff",
+ "--ignore-submodules=all",
"--raw",
"-z",
"--diff-filter=ACMRD",
@@ -194,11 +195,7 @@ def generate_diff_in_scope_files(
) -> int:
"""Reuse the existing diff selection without generating previews or duplicate worklists."""
from generate_rank_input import git_changed_paths, path_is_diff_excluded
- from rank_preview import (
- TEXT_CODE_EXTENSIONS,
- is_binary_file,
- is_binary_sample,
- )
+ from rank_preview import is_binary_file
rows: list[bytes] = []
try:
@@ -211,36 +208,32 @@ def generate_diff_in_scope_files(
(path, status)
for path, status in changed
if not path_is_diff_excluded(path.relative_to(repository))
- and path.suffix.lower() in TEXT_CODE_EXTENSIONS
- ]
- revision_paths = [
- path.relative_to(repository)
- for path, status in eligible
- if mode == "revisions" and status != "D"
]
- revision_blobs = dict(
- zip(
- revision_paths,
- git_blob_bytes(
- repository,
- [f"{head}:{path.as_posix()}" for path in revision_paths],
- ),
+ revision_refs = {
+ path.relative_to(repository): (
+ f"{base if status == 'D' else head}:{path.relative_to(repository).as_posix()}"
)
+ for path, status in eligible
+ if mode == "revisions" or status == "D"
+ }
+ revision_samples = dict(
+ zip(revision_refs, git_blob_samples(repository, list(revision_refs.values())))
)
for path, status in eligible:
relative = path.relative_to(repository)
- if status != "D":
- if mode == "revisions":
- contents = revision_blobs[relative]
- if contents is None:
- raise InventoryError(
- f"could not read committed diff blob: {head}:{relative.as_posix()}"
- )
- if is_binary_sample(contents):
- continue
- elif path.is_symlink() or not path.is_file() or is_binary_file(path):
+ if mode == "revisions" or status == "D":
+ sample = revision_samples[relative]
+ if sample is None:
+ revision = base if status == "D" else head
+ raise InventoryError(
+ f"could not read committed diff blob: {revision}:{relative.as_posix()}"
+ )
+ _, is_binary = sample
+ if is_binary:
continue
+ elif path.is_symlink() or not path.is_file() or is_binary_file(path):
+ continue
relative_path = relative.as_posix()
if "\n" in relative_path or "\r" in relative_path:
raise InventoryError(
diff --git a/plugins/codex-security/scripts/generate_rank_input.py b/plugins/codex-security/scripts/generate_rank_input.py
index 92cc34887..2eaead3f0 100644
--- a/plugins/codex-security/scripts/generate_rank_input.py
+++ b/plugins/codex-security/scripts/generate_rank_input.py
@@ -8,6 +8,9 @@ This script stays deliberately model-free:
- `make-diff-rank-input` creates the deterministic diff-scoped JSONL candidate
worklist from Git changed paths. It supports committed revision diffs and
local working-tree patches.
+
+Candidate selection uses the existing path exclusions and binary detection,
+not a language-extension allowlist. Unknown formats use sampled text previews.
"""
from __future__ import annotations
@@ -24,13 +27,11 @@ sys.path.insert(0, str(Path(__file__).resolve().parent))
from generate_in_scope_files import windows_stream_component
from rank_preview import (
DEFAULT_PREVIEW_BYTES,
- DEFAULT_PREVIEW_READ_BYTES,
- TEXT_CODE_EXTENSIONS,
- is_binary_sample,
+ is_binary_file,
preview_for,
preview_for_bytes,
)
-from workbench_target import git_blob_bytes, git_command, git_directory_snapshot_paths
+from workbench_target import git_blob_samples, git_command, git_directory_snapshot_paths
EXCLUDED_DIRS = {
".cache",
@@ -290,21 +291,14 @@ def make_repo_rank_input(args: argparse.Namespace) -> None:
if explicit_scopes
else rel
)
- if not directly_requested and (
- path_is_excluded(excluded_path) or path.suffix.lower() not in TEXT_CODE_EXTENSIONS
- ):
+ if not directly_requested and path_is_excluded(excluded_path):
continue
- if (
- directly_requested
- and path.suffix.lower() not in TEXT_CODE_EXTENSIONS
- and path.name not in EXCLUDED_FILENAMES
- ):
- preview = ""
- else:
- preview, is_binary = preview_for(path, args.preview_bytes)
- if is_binary and not directly_requested:
+ preview, is_binary = preview_for(path, args.preview_bytes)
+ if is_binary or is_binary_file(path):
+ if not directly_requested:
continue
+ preview = ""
rows_by_path.setdefault(
rel.as_posix(),
{"path": rel.as_posix(), "area": area, "preview": preview},
@@ -398,6 +392,7 @@ def run_git_changed_paths(repo: Path, diff_args: list[str]) -> list[tuple[Path,
result = git_command(
repo,
"diff",
+ "--ignore-submodules=all",
"--name-status",
"-z",
"--diff-filter=ACMRD",
@@ -442,7 +437,7 @@ def git_changed_paths(repo: Path, base: str, head: str, mode: str) -> list[tuple
combined.update(
(repo / os.fsdecode(relative), "A")
for relative in untracked.stdout.split(b"\0")
- if relative
+ if relative and not relative.endswith(b"/")
)
return sorted(combined.items())
raise SystemExit(f"Unknown diff mode: {mode}")
@@ -457,21 +452,16 @@ def make_diff_rank_input(args: argparse.Namespace) -> None:
(path, status)
for path, status in git_changed_paths(repo, args.base, args.head, args.mode)
if not path_is_diff_excluded(path.relative_to(repo))
- and path.suffix.lower() in TEXT_CODE_EXTENSIONS
]
- revision_paths = [
- path.relative_to(repo)
- for path, status in changed
- if args.mode == "revisions" and status != "D"
- ]
- revision_blobs = dict(
- zip(
- revision_paths,
- git_blob_bytes(
- repo,
- [f"{args.head}:{path.as_posix()}" for path in revision_paths],
- ),
+ revision_refs = {
+ path.relative_to(repo): (
+ f"{args.base if status == 'D' else args.head}:{path.relative_to(repo).as_posix()}"
)
+ for path, status in changed
+ if args.mode == "revisions" or status == "D"
+ }
+ revision_samples = dict(
+ zip(revision_refs, git_blob_samples(repo, list(revision_refs.values())))
)
rows: list[JsonRow] = []
@@ -479,25 +469,24 @@ def make_diff_rank_input(args: argparse.Namespace) -> None:
rel = path.relative_to(repo)
preview = ""
- if status != "D" and args.mode == "revisions":
- content = revision_blobs[rel]
- if content is None:
- raise SystemExit(
- f"Unable to read committed diff blob: {args.head}:{rel.as_posix()}"
- )
- if is_binary_sample(content):
+ if args.mode == "revisions" or status == "D":
+ sample = revision_samples[rel]
+ if sample is None:
+ revision = args.base if status == "D" else args.head
+ raise SystemExit(f"Unable to read committed diff blob: {revision}:{rel.as_posix()}")
+ content, is_binary = sample
+ if is_binary:
continue
- preview, _ = preview_for_bytes(
- rel, content[:DEFAULT_PREVIEW_READ_BYTES], args.preview_bytes
- )
- elif status != "D" and not path.is_symlink() and path.is_file():
+ if status != "D":
+ preview, _ = preview_for_bytes(rel, content, args.preview_bytes)
+ elif not path.is_symlink() and path.is_file():
try:
path.resolve(strict=True).relative_to(repo)
except (OSError, ValueError):
preview = ""
else:
preview, is_binary = preview_for(path, args.preview_bytes)
- if is_binary:
+ if is_binary or is_binary_file(path):
continue
rows.append({"path": rel.as_posix(), "area": args.area, "preview": preview})
diff --git a/plugins/codex-security/scripts/rank_preview.py b/plugins/codex-security/scripts/rank_preview.py
index 3e691fd03..f88072528 100644
--- a/plugins/codex-security/scripts/rank_preview.py
+++ b/plugins/codex-security/scripts/rank_preview.py
@@ -15,74 +15,6 @@ PREVIEW_HEAD_LINES = 12
PREVIEW_SAMPLE_LINES = 10
_UTF16_BOMS = (b"\xff\xfe", b"\xfe\xff")
-TEXT_CODE_EXTENSIONS = {
- ".ascx",
- ".aspx",
- ".c",
- ".cc",
- ".cfg",
- ".cjs",
- ".clj",
- ".cpp",
- ".cs",
- ".cshtml",
- ".css",
- ".cts",
- ".cue",
- ".cxx",
- ".dart",
- ".ejs",
- ".erb",
- ".ex",
- ".exs",
- ".go",
- ".graphql",
- ".h",
- ".hh",
- ".hpp",
- ".hs",
- ".html",
- ".hxx",
- ".java",
- ".js",
- ".json",
- ".jsp",
- ".jspx",
- ".jsx",
- ".kt",
- ".kts",
- ".lua",
- ".m",
- ".mjs",
- ".mm",
- ".mts",
- ".php",
- ".phtml",
- ".proto",
- ".ps1",
- ".psd1",
- ".psm1",
- ".py",
- ".razor",
- ".rb",
- ".rs",
- ".scala",
- ".sh",
- ".sol",
- ".sql",
- ".svelte",
- ".swift",
- ".tf",
- ".toml",
- ".ts",
- ".tsx",
- ".vue",
- ".vy",
- ".xml",
- ".yaml",
- ".yml",
-}
-
JAVASCRIPT_EXTENSIONS = {".cjs", ".cts", ".js", ".jsx", ".mjs", ".mts", ".ts", ".tsx", ".vue"}
JAVA_LIKE_EXTENSIONS = {
".c",
@@ -129,11 +61,20 @@ def is_binary_sample(data: bytes) -> bool:
def is_binary_file(path: Path) -> bool:
+ """Classify the whole file in bounded chunks."""
try:
with path.open("rb") as source:
- return is_binary_sample(source.read(DEFAULT_PREVIEW_READ_BYTES))
+ sample = source.read(DEFAULT_PREVIEW_READ_BYTES)
+ if is_binary_sample(sample):
+ return True
+ bom = sample[:2] if sample.startswith(_UTF16_BOMS) else b""
+ while chunk := source.read(DEFAULT_PREVIEW_READ_BYTES):
+ # Even-sized reads preserve UTF-16 code-unit alignment and byte order.
+ if is_binary_sample(bom + chunk) if bom else b"\0" in chunk:
+ return True
except OSError:
return True
+ return False
def compact_preview_line(line: str) -> str:
diff --git a/plugins/codex-security/scripts/workbench_target.py b/plugins/codex-security/scripts/workbench_target.py
index 34a49e5ec..98b73dfb3 100644
--- a/plugins/codex-security/scripts/workbench_target.py
+++ b/plugins/codex-security/scripts/workbench_target.py
@@ -3,6 +3,7 @@
from __future__ import annotations
import argparse
+import codecs
import hashlib
import os
import shutil
@@ -11,12 +12,14 @@ import stat
import subprocess
import sys
import tempfile
+from concurrent.futures import ThreadPoolExecutor
from pathlib import Path
from typing import Any, BinaryIO
# Some plugin hosts launch Python with safe-path isolation enabled.
sys.path.insert(0, str(Path(__file__).resolve().parent))
from filesystem_identity import stored_filesystem_identity_matches
+from rank_preview import DEFAULT_PREVIEW_READ_BYTES, is_binary_sample
from workbench_constants import GIT_REPOSITORY_ENVIRONMENT
@@ -69,58 +72,74 @@ def git_bytes(
return completed.stdout if completed.returncode == 0 else None
-def git_blob_bytes(
+def git_blob_samples(
target: Path,
object_names: list[str],
*,
git_dir: Path | None = None,
work_tree: Path | None = None,
-) -> list[bytes | None]:
- """Read ordered raw blobs with one NUL-framed ``git cat-file --batch`` call."""
+) -> list[tuple[bytes, bool] | None]:
+ """Sample ordered blobs and classify all their bytes without buffering the batch."""
if not object_names:
return []
request = b"\0".join(os.fsencode(name) for name in object_names) + b"\0"
- completed = git_command(
- target,
- "cat-file",
- "--batch",
- "-Z",
- text=False,
- input_data=request,
- git_dir=git_dir,
- work_tree=work_tree,
- )
- if completed.returncode != 0:
- return [None] * len(object_names)
-
- try:
- return _decode_git_batch_blobs(completed.stdout, len(object_names))
- except ValueError:
- return [None] * len(object_names)
+ read_fd, write_fd = os.pipe()
+ with (
+ ThreadPoolExecutor(max_workers=1) as reader,
+ os.fdopen(read_fd, "rb") as output,
+ os.fdopen(write_fd, "wb") as sink,
+ ):
+
+ def read_samples() -> list[tuple[bytes, bool] | None]:
+ # Closing the reader also unblocks Git if framing is invalid.
+ with output:
+ return _read_git_batch_samples(output, len(object_names))
+
+ samples = reader.submit(read_samples)
+ try:
+ completed = git_command(
+ target,
+ "cat-file",
+ "--batch",
+ "-Z",
+ text=False,
+ input_data=request,
+ git_dir=git_dir,
+ work_tree=work_tree,
+ stdout_file=sink,
+ )
+ finally:
+ # Git may fail before producing all requested records.
+ sink.close()
+ if completed.returncode != 0:
+ return [None] * len(object_names)
+ try:
+ return samples.result()
+ except ValueError:
+ return [None] * len(object_names)
-def _decode_git_batch_blobs(output: bytes, count: int) -> list[bytes | None]:
- """Decode ordered ``cat-file --batch -Z`` records without scanning blob bytes."""
- blobs: list[bytes | None] = []
- offset = 0
+def _read_git_batch_samples(output: BinaryIO, count: int) -> list[tuple[bytes, bool] | None]:
+ """Read NUL-framed headers, then sample each length-delimited blob."""
+ samples: list[tuple[bytes, bool] | None] = []
for _ in range(count):
- header, offset = _read_nul_field(output, offset)
- size = _git_batch_blob_size(header)
+ size = _git_batch_blob_size(_read_nul_field(output))
if size is None:
- blobs.append(None)
+ samples.append(None)
continue
- blob, offset = _read_sized_nul_field(output, offset, size)
- blobs.append(blob)
- return blobs
+ samples.append(_read_git_blob_sample(output, size))
+ return samples
-def _read_nul_field(output: bytes, offset: int) -> tuple[bytes, int]:
- """Read one NUL-terminated protocol field and return the next offset."""
- end = output.find(b"\0", offset)
- if end < 0:
- raise ValueError("missing NUL terminator")
- return output[offset:end], end + 1
+def _read_nul_field(output: BinaryIO) -> bytes:
+ """Read one NUL-terminated protocol field."""
+ field = bytearray()
+ while char := output.read(1):
+ if char == b"\0":
+ return bytes(field)
+ field.extend(char)
+ raise ValueError("missing NUL terminator")
def _git_batch_blob_size(header: bytes) -> int | None:
@@ -137,16 +156,26 @@ def _git_batch_blob_size(header: bytes) -> int | None:
return size
-def _read_sized_nul_field(
- output: bytes,
- offset: int,
- size: int,
-) -> tuple[bytes, int]:
- """Read exactly ``size`` blob bytes followed by one NUL record terminator."""
- end = offset + size
- if output[end : end + 1] != b"\0":
+def _read_git_blob_sample(output: BinaryIO, size: int) -> tuple[bytes, bool]:
+ sample_size = min(size, DEFAULT_PREVIEW_READ_BYTES)
+ sample = output.read(sample_size)
+ if len(sample) != sample_size:
+ raise ValueError("truncated blob")
+ binary = is_binary_sample(sample)
+ bom = sample[:2] if sample.startswith((codecs.BOM_UTF16_LE, codecs.BOM_UTF16_BE)) else b""
+ remaining = size - sample_size
+ while remaining:
+ chunk_size = min(remaining, DEFAULT_PREVIEW_READ_BYTES)
+ chunk = output.read(chunk_size)
+ if len(chunk) != chunk_size:
+ raise ValueError("truncated blob")
+ # Even-sized reads keep UTF-16 code units aligned; preserve the original byte order.
+ if not binary:
+ binary = is_binary_sample(bom + chunk) if bom else b"\0" in chunk
+ remaining -= chunk_size
+ if output.read(1) != b"\0":
raise ValueError("missing blob terminator")
- return output[offset:end], end + 1
+ return (b"", True) if binary else (sample, False)
def _protected_repository_root(target: Path) -> Path:
diff --git a/plugins/codex-security/skills/security-diff-scan/SKILL.md b/plugins/codex-security/skills/security-diff-scan/SKILL.md
index a1d1024bc..4fee03dd1 100644
--- a/plugins/codex-security/skills/security-diff-scan/SKILL.md
+++ b/plugins/codex-security/skills/security-diff-scan/SKILL.md
@@ -34,6 +34,8 @@ For terminal scans without a `scanId`, generate the changed-file list with:
<python_command> <plugin_dir>/scripts/generate_in_scope_files.py --repo <repo_root> --scope . --diff-base <base> --diff-head <head> --diff-mode <revisions|local-patch> --out <discovery_dir>/in_scope_files.txt
```
+The terminal inventory includes changed text files regardless of extension, subject to the existing path exclusions. Deleted files are classified from the base revision. Extensions select preview formats, not scan eligibility.
+
Record candidates with `<plugin_dir>/scripts/launch_codex_security_mcp --helper normalize-candidates --input <candidate-source> --out <discovery_dir>/candidate_ledger.jsonl --repo-root <repo_root> --in-scope-files <discovery_dir>/in_scope_files.txt --allow-missing-in-scope`. Use the `.cmd` launcher on Windows. Add validation and attack-path decisions to that same file. Following `../../references/final-report.md`, assemble unsealed `scan-manifest.json`, `findings.json`, and `coverage.json` before running `finalize_scan_contract.py --scan-dir <scan_dir> --source-root <repo_root>`.
Finish only after every changed file and candidate is accounted for. Return the generated report, actual coverage gaps, and Codex review comments for confirmed findings.
diff --git a/plugins/codex-security/skills/security-scan/references/scan-artifacts-and-ledger.md b/plugins/codex-security/skills/security-scan/references/scan-artifacts-and-ledger.md
index 4680a2f16..e5493aadc 100644
--- a/plugins/codex-security/skills/security-scan/references/scan-artifacts-and-ledger.md
+++ b/plugins/codex-security/skills/security-scan/references/scan-artifacts-and-ledger.md
@@ -56,7 +56,7 @@ The parent agent must reconcile validation and attack-path subagent outputs befo
## Scoped Deep Review
- Use `deep_review_input.jsonl` as the canonical changed-file review worklist for diff scans.
-- For diff-scoped scans, generate `rank_input.jsonl` deterministically from changed source-like files with `<python_command> <plugin_dir>/scripts/generate_rank_input.py make-diff-rank-input --repo <repo_root> --base <base> --mode revisions --head <head> --out <discovery_dir>/rank_input.jsonl` for PR, commit, and branch diffs, or `<python_command> <plugin_dir>/scripts/generate_rank_input.py make-diff-rank-input --repo <repo_root> --base <base> --mode local-patch --out <discovery_dir>/rank_input.jsonl` for a local patch, then copy every row into `deep_review_input.jsonl` with `<plugin_dir>/scripts/launch_codex_security_mcp --helper copy-deep-review-input --rank-input <discovery_dir>/rank_input.jsonl --out <discovery_dir>/deep_review_input.jsonl`. On Windows, use the [PowerShell copy command](#windows-worklist-copy) below.
+- For diff-scoped scans, generate `rank_input.jsonl` deterministically from changed text files regardless of extension, subject to the existing path exclusions, with `<python_command> <plugin_dir>/scripts/generate_rank_input.py make-diff-rank-input --repo <repo_root> --base <base> --mode revisions --head <head> --out <discovery_dir>/rank_input.jsonl` for PR, commit, and branch diffs, or `<python_command> <plugin_dir>/scripts/generate_rank_input.py make-diff-rank-input --repo <repo_root> --base <base> --mode local-patch --out <discovery_dir>/rank_input.jsonl` for a local patch, then copy every row into `deep_review_input.jsonl` with `<plugin_dir>/scripts/launch_codex_security_mcp --helper copy-deep-review-input --rank-input <discovery_dir>/rank_input.jsonl --out <discovery_dir>/deep_review_input.jsonl`. On Windows, use the [PowerShell copy command](#windows-worklist-copy) below.
- Diff-scoped scans do not rank or drop changed files before deep review. Every row in diff `rank_input.jsonl` must be copied into `deep_review_input.jsonl` and receive a full-file review receipt.
- Add directly supporting files required to understand the changed security behavior only when repository evidence shows they are needed; record the add-back reason in the work ledger or per-file result.
- Deep-review every file selected into `deep_review_input.jsonl`.
diff --git a/plugins/codex-security/tests/source_cases.py b/plugins/codex-security/tests/source_cases.py
index ce2ccd585..d92a261cf 100644
--- a/plugins/codex-security/tests/source_cases.py
+++ b/plugins/codex-security/tests/source_cases.py
@@ -1,7 +1,7 @@
"""Shared source examples for inventory and ranking integration tests.
-Add source-selection regressions here so every inventory and ranking mode uses
-the same examples. Expectations are independent of the production allowlist.
+Add source-selection and preview regressions here so every inventory and ranking
+mode uses the same examples, including source with unfamiliar or no extensions.
"""
from __future__ import annotations
@@ -18,6 +18,34 @@ class SourceCase(NamedTuple):
SOURCE_CASES = (
+ SourceCase("scripts/entrypoint", "exec service --before", "exec service --after"),
+ SourceCase("src/handler.unlisted", "render(before)", "render(after)"),
+ SourceCase(
+ "src/raw.cpp",
+ "void before() {}",
+ 'void before() {}\nconst char* text = R"tag("{)tag";\nvoid after() {}',
+ "function before\nfunction after",
+ ),
+ SourceCase(
+ "src/raw.go",
+ "package sample\n\nfunc Before() {}",
+ "package sample\n\nfunc Before() {}\n\nconst Root = `C:\\`\n\nfunc After() {}",
+ "function Before\nfunction After",
+ ),
+ *(
+ SourceCase(
+ f"src/{name}.cs",
+ "class Before {}",
+ f"class Before {{}}\nclass Service {{\n string Text = {literal};\n"
+ " void Visible() {}\n}\nclass After {}",
+ "class Before\nclass Service\nmethod Service.Visible\nclass After",
+ )
+ for name, literal in (
+ ("three_quotes", '"""\n { "a": "b" }\n """'),
+ ("four_quotes", '""""\n { "a": """ }\n """"'),
+ ("five_quotes", '""""" contains """" and { """""'),
+ )
+ ),
SourceCase(
"infra/main.tf",
'variable "enabled" { default = false }',
diff --git a/plugins/codex-security/tests/test_generate_in_scope_files.py b/plugins/codex-security/tests/test_generate_in_scope_files.py
index c9a36e184..4ea5ea86a 100644
--- a/plugins/codex-security/tests/test_generate_in_scope_files.py
+++ b/plugins/codex-security/tests/test_generate_in_scope_files.py
@@ -499,10 +499,15 @@ def test_large_inventory_is_not_limited_by_a_subprocess_output_buffer(tmp_path:
assert rows == sorted(rows)
-def test_diff_inventory_keeps_changed_and_deleted_source_files(tmp_path: Path) -> None:
[email protected]("mode", ["revisions", "staged", "unstaged"])
+def test_diff_inventory_keeps_changed_and_deleted_source_files(tmp_path: Path, mode: str) -> None:
repository = make_repository(tmp_path)
+ write_file(repository, "app/entrypoint", b"exec service\n")
git(repository, "add", ".")
git(repository, "commit", "-qm", "base")
+ gitlink = git(repository, "rev-parse", "HEAD")
+ git(repository, "update-index", "--add", "--cacheinfo", f"160000,{gitlink},removed-module")
+ git(repository, "commit", "--amend", "-qm", "base")
base = git(repository, "rev-parse", "HEAD")
write_file(repository, "app/routes.py", b"changed = True\n")
@@ -510,22 +515,37 @@ def test_diff_inventory_keeps_changed_and_deleted_source_files(tmp_path: Path) -
write_file(repository, "app/binary.py", b"\x00\xff\x01")
write_file(repository, "tests/demo.py", b"excluded = True\n")
write_file(repository, ".github/workflows/ci.yml", b"name: CI\n")
+ write_file(
+ repository,
+ ".gitmodules",
+ b'[submodule "added-module"]\npath = added-module\nurl = ../module.git\n',
+ )
(repository / "app/évidence.py").unlink()
- git(repository, "add", ".")
- git(repository, "commit", "-qm", "change")
- head = git(repository, "rev-parse", "HEAD")
+ (repository / "app/entrypoint").unlink()
+ (repository / "app/binary.dat").unlink()
+ arguments = ["--diff-base", base, "--diff-mode", "local-patch"]
+ if mode in {"revisions", "staged"}:
+ git(repository, "add", ".")
+ git(repository, "update-index", "--force-remove", "removed-module")
+ git(repository, "update-index", "--add", "--cacheinfo", f"160000,{gitlink},added-module")
+ if mode == "revisions":
+ git(repository, "commit", "-qm", "change")
+ arguments = ["--diff-base", base, "--diff-head", git(repository, "rev-parse", "HEAD")]
+ git(repository, "checkout", "-q", base)
output = tmp_path / "in_scope_files.txt"
result = run_inventory(
repository,
".",
output,
- arguments=["--diff-base", base, "--diff-head", head],
+ arguments=arguments,
)
assert result.returncode == 0, result.stderr
assert output.read_text(encoding="utf-8").splitlines() == [
".github/workflows/ci.yml",
+ ".gitmodules",
+ "app/entrypoint",
"app/new handler.py",
"app/routes.py",
"app/évidence.py",
@@ -540,7 +560,8 @@ def test_diff_inventory_only_classifies_source_bytes(tmp_path: Path, mode: str)
base = git(repository, "rev-parse", "HEAD")
write_file(repository, "app/surrogate.json", b'{"\\ud800":1}')
write_file(repository, "app/integer.json", ('{"value":' + "1" * 4301 + "}").encode())
- write_file(repository, "app/binary.py", b"x" * 4096 + b"\0")
+ write_file(repository, "app/binary.py", b"x" * (64 * 1024 + 1) + b"\0")
+ write_file(repository, "app/binary.unlisted", b"x" * (64 * 1024 + 1) + b"\0")
arguments = ["--diff-base", base, "--diff-mode", mode]
if mode == "revisions":
git(repository, "add", ".")
@@ -636,6 +657,12 @@ def test_diff_inventory_combines_staged_and_unstaged_changes(tmp_path: Path) ->
index_only = write_file(repository, "app/index-only.py", b"index_only = True\n")
git(repository, "add", "app/index-only.py")
index_only.unlink()
+ nested = repository / "nested"
+ nested.mkdir()
+ git(nested, "init", "-q")
+ write_file(nested, "source.py", b"nested = True\n")
+ git(nested, "add", ".")
+ git(nested, "commit", "-qm", "nested repository")
output = tmp_path / "in_scope_files.txt"
result = run_inventory(
diff --git a/plugins/codex-security/tests/test_generate_rank_input.py b/plugins/codex-security/tests/test_generate_rank_input.py
index 70868900a..3baea5ac9 100644
--- a/plugins/codex-security/tests/test_generate_rank_input.py
+++ b/plugins/codex-security/tests/test_generate_rank_input.py
@@ -63,6 +63,7 @@ def test_make_repo_rank_input_matches_golden_and_filters_noise(tmp_path: Path) -
(repo / "src" / "zeta.py").write_text("zeta = 2", encoding="utf-8")
(repo / "src" / "alpha.py").write_text("alpha = 1", encoding="utf-8")
(repo / "src" / "binary.py").write_bytes(b"value\x00binary")
+ (repo / "src" / "binary.unlisted").write_bytes(b"x" * (64 * 1024 + 1) + b"\0binary")
(repo / "tests" / "ignored.py").write_text("ignored = True", encoding="utf-8")
(repo / "README.md").write_text("ignored", encoding="utf-8")
output = tmp_path / "rank_input.jsonl"
@@ -82,25 +83,6 @@ def test_make_repo_rank_input_matches_golden_and_filters_noise(tmp_path: Path) -
)
-def test_make_repo_rank_input_keeps_declarations_after_cpp_raw_strings(tmp_path: Path) -> None:
- repo = tmp_path / "repo"
- repo.mkdir()
- (repo / "sample.cpp").write_text(
- """void before() {}
-const char* text = R"tag("{)tag";
-void after() {}
-""",
- encoding="utf-8",
- )
- output = tmp_path / "rank_input.jsonl"
-
- run_cli("make-repo-rank-input", "--repo", str(repo), "--out", str(output))
-
- rows = read_jsonl(output)
- assert [row["path"] for row in rows] == ["sample.cpp"]
- assert rows[0]["preview"] == "function before\nfunction after"
-
-
@pytest.mark.parametrize("mode", ["repo", "revisions"])
def test_rank_input_keeps_large_generated_python(tmp_path: Path, mode: str) -> None:
repo = tmp_path / "repo"
@@ -219,7 +201,7 @@ def test_revision_rank_input_classifies_bytes_beyond_the_preview_sample(tmp_path
@pytest.mark.parametrize("scope", [".", "src", "src/large.py", "explicit", "overlap", "diff"])
-def test_rank_input_bounds_large_text_reads(
+def test_rank_input_streams_classification_and_bounds_previews(
tmp_path: Path, monkeypatch: pytest.MonkeyPatch, scope: str
) -> None:
repo = tmp_path / "repo"
@@ -253,7 +235,7 @@ def test_rank_input_bounds_large_text_reads(
reads.append(0)
def bounded_read(size: int = -1) -> bytes:
- assert 0 <= size <= 64 * 1024 - reads[index]
+ assert 0 <= size <= 64 * 1024
data = read(size)
reads[index] += len(data)
return data
@@ -265,7 +247,7 @@ def test_rank_input_bounds_large_text_reads(
with patch.object(Path, "open", tracked_open):
runpy.run_path(str(SCRIPT), run_name="__main__")
- assert reads and all(count == 64 * 1024 for count in reads)
+ assert set(reads) == {64 * 1024, source.stat().st_size}
rows = read_jsonl(output)
assert len(rows) == 1
assert rows[0]["path"] == "src/large.py"
@@ -730,7 +712,7 @@ def test_make_repo_rank_input_keeps_explicit_binary_file_without_preview(tmp_pat
repo = tmp_path / "repo"
repo.mkdir()
with (repo / "payload.bin").open("wb") as payload:
- payload.write(b"header-without-a-nul" * 256)
+ payload.write(b"header-without-a-nul" * 4096)
payload.truncate(256 * 1024 * 1024)
scopes = tmp_path / "target-paths.json"
scopes.write_text(json.dumps(["payload.bin"]), encoding="utf-8")
@@ -754,7 +736,7 @@ def test_make_repo_rank_input_bounds_explicit_source_like_binary(tmp_path: Path)
source = repo / "src"
source.mkdir(parents=True)
with (source / "payload.py").open("wb") as payload:
- payload.write(b"header-without-a-nul" * 256)
+ payload.write(b"header-without-a-nul" * 4096)
payload.write(b"\0binary")
payload.truncate(256 * 1024 * 1024)
scopes = tmp_path / "target-paths.json"
@@ -774,27 +756,47 @@ def test_make_repo_rank_input_bounds_explicit_source_like_binary(tmp_path: Path)
assert read_jsonl(output) == [{"path": "src/payload.py", "area": "src", "preview": ""}]
-def test_make_diff_rank_input_for_revision_range(tmp_path: Path) -> None:
[email protected]("mode", ["revisions", "staged", "unstaged"])
+def test_make_diff_rank_input_keeps_changed_and_deleted_text(tmp_path: Path, mode: str) -> None:
repo = tmp_path / "repo"
(repo / "src").mkdir(parents=True)
initialize_repo(repo)
(repo / "src" / "alpha.py").write_text("alpha = 1", encoding="utf-8")
deleted_guard = repo / "src" / "deleted_guard.py"
deleted_guard.write_text("guard = True", encoding="utf-8")
+ deleted_entrypoint = repo / "src" / "entrypoint"
+ deleted_entrypoint.write_text("exec service", encoding="utf-8")
+ deleted_binary = repo / "src" / "deleted.bin"
+ deleted_binary.write_bytes(b"value\x00binary")
git(repo, "add", ".")
git(repo, "commit", "-qm", "base")
+ gitlink = git(repo, "rev-parse", "HEAD")
+ git(repo, "update-index", "--add", "--cacheinfo", f"160000,{gitlink},removed-module")
+ git(repo, "commit", "--amend", "-qm", "base")
base = git(repo, "rev-parse", "HEAD")
(repo / "src" / "alpha.py").write_text("alpha = 2", encoding="utf-8")
(repo / "src" / "beta.py").write_text("beta = 1", encoding="utf-8")
+ (repo / "src" / "binary.unlisted").write_bytes(b"x" * (64 * 1024 + 1) + b"\0binary")
(repo / ".github" / "workflows").mkdir(parents=True)
(repo / ".github" / "workflows" / "ci.yml").write_text("name: CI", encoding="utf-8")
+ (repo / ".gitmodules").write_text(
+ '[submodule "added-module"]\npath = added-module\nurl = ../module.git\n',
+ encoding="utf-8",
+ )
deleted_guard.unlink()
+ deleted_entrypoint.unlink()
+ deleted_binary.unlink()
(repo / "README.md").write_text("ignored", encoding="utf-8")
- git(repo, "add", ".")
- git(repo, "commit", "-qm", "change")
- head = git(repo, "rev-parse", "HEAD")
- git(repo, "checkout", "-q", base)
+ arguments = ["--mode", "local-patch"]
+ if mode in {"revisions", "staged"}:
+ git(repo, "add", ".")
+ git(repo, "update-index", "--force-remove", "removed-module")
+ git(repo, "update-index", "--add", "--cacheinfo", f"160000,{gitlink},added-module")
+ if mode == "revisions":
+ git(repo, "commit", "-qm", "change")
+ arguments = ["--head", git(repo, "rev-parse", "HEAD")]
+ git(repo, "checkout", "-q", base)
output = tmp_path / "diff.jsonl"
run_cli(
@@ -803,8 +805,7 @@ def test_make_diff_rank_input_for_revision_range(tmp_path: Path) -> None:
str(repo),
"--base",
base,
- "--head",
- head,
+ *arguments,
"--out",
str(output),
)
@@ -812,11 +813,13 @@ def test_make_diff_rank_input_for_revision_range(tmp_path: Path) -> None:
rows = read_jsonl(output)
assert [row["path"] for row in rows] == [
".github/workflows/ci.yml",
+ ".gitmodules",
"src/alpha.py",
"src/beta.py",
"src/deleted_guard.py",
+ "src/entrypoint",
]
- assert rows[-1]["preview"] == ""
+ assert [row["preview"] for row in rows[-2:]] == ["", ""]
def test_make_diff_rank_input_uses_empty_tree_for_root_commit(tmp_path: Path) -> None:
@@ -921,6 +924,16 @@ def test_make_diff_rank_input_combines_staged_and_unstaged_patch(tmp_path: Path)
(repo / ".github" / "workflows").mkdir(parents=True)
(repo / ".github" / "workflows" / "ci.yaml").write_text("name: CI", encoding="utf-8")
git(repo, "add", "src/beta.py")
+ index_only = repo / "src" / "index-only.py"
+ index_only.write_text("index_only = True", encoding="utf-8")
+ git(repo, "add", "src/index-only.py")
+ index_only.unlink()
+ nested = repo / "nested"
+ nested.mkdir()
+ initialize_repo(nested)
+ (nested / "source.py").write_text("nested = True", encoding="utf-8")
+ git(nested, "add", ".")
+ git(nested, "commit", "-qm", "nested repository")
output = tmp_path / "patch.jsonl"
run_cli(
@@ -939,7 +952,9 @@ def test_make_diff_rank_input_combines_staged_and_unstaged_patch(tmp_path: Path)
".github/workflows/ci.yaml",
"src/alpha.py",
"src/beta.py",
+ "src/index-only.py",
]
+ assert read_jsonl(output)[-1]["preview"] == ""
@pytest.mark.parametrize("mode", ["repo", "explicit-file", "revisions", "local-patch"])
diff --git a/plugins/codex-security/tests/test_rank_preview.py b/plugins/codex-security/tests/test_rank_preview.py
index f0505d0d2..1e92237bb 100644
--- a/plugins/codex-security/tests/test_rank_preview.py
+++ b/plugins/codex-security/tests/test_rank_preview.py
@@ -787,50 +787,6 @@ TXT;
assert "method Service.visible" in preview
-def test_go_raw_string_backslash_does_not_hide_following_function(tmp_path: Path) -> None:
- source = r"""package sample
-
-func Before() {}
-
-const Root = `C:\`
-
-func After() {}
-"""
-
- preview = generate_preview(tmp_path, "sample.go", source)
-
- assert "function Before" in preview
- assert "function After" in preview
-
-
- "literal",
- [
- '"""\n { "a": "b" }\n """',
- '""""\n { "a": """ }\n """"',
- '""""" contains """" and { """""',
- ],
- ids=["three-quotes", "four-quotes", "five-quotes"],
-)
-def test_csharp_raw_string_closes_on_matching_quote_count(tmp_path: Path, literal: str) -> None:
- source = f"""class Before {{}}
-class Service {{
- string Text = {literal};
- void Visible() {{}}
-}}
-class After {{}}
-"""
-
- preview = generate_preview(tmp_path, "Service.cs", source)
-
- assert preview.splitlines() == [
- "class Before",
- "class Service",
- "method Service.Visible",
- "class After",
- ]
-
-
def test_malformed_python_uses_sampled_source_fallback(tmp_path: Path) -> None:
source = """import package
broken = (
diff --git a/plugins/codex-security/tests/test_workbench_target.py b/plugins/codex-security/tests/test_workbench_target.py
index 39297ae2e..7a2a4ac7a 100644
--- a/plugins/codex-security/tests/test_workbench_target.py
+++ b/plugins/codex-security/tests/test_workbench_target.py
@@ -51,6 +51,113 @@ def test_stale_git_binding_does_not_spawn(tmp_path: Path, monkeypatch: pytest.Mo
assert result.args[0] == "git"
+ ("encoding", "bom"),
+ [("utf-8", b""), ("utf-16-le", b"\xff\xfe"), ("utf-16-be", b"\xfe\xff")],
+)
+def test_git_blob_samples_match_full_file_classification_with_bounded_reads(
+ tmp_path: Path, monkeypatch: pytest.MonkeyPatch, encoding: str, bom: bytes
+) -> None:
+ target = tmp_path / "target"
+ initialize_git_repository(target)
+ # The UTF-16 surrogate pair crosses the first 64 KiB sample boundary.
+ text = bom + ("a" * 32766 + "😀" + "b" * 32768).encode(encoding)
+ contents = {
+ "text.bin": text + (b"\0" if bom else b""),
+ "binary-tail.bin": text + "\0".encode(encoding),
+ "binary-head.bin": b"\0" + text * 32,
+ "empty.bin": b"",
+ "after.txt": b"after\n",
+ }
+ for name, content in contents.items():
+ (target / name).write_bytes(content)
+ subprocess.run(["git", "add", "."], cwd=target, check=True)
+ subprocess.run(["git", "commit", "-qm", "source samples"], cwd=target, check=True)
+
+ git_blob_samples = WORKBENCH_TARGET["git_blob_samples"]
+ function_globals = git_blob_samples.__globals__
+ read_batch_samples = function_globals["_read_git_batch_samples"]
+ reads: list[int] = []
+
+ def require_bounded_output(output: Any, count: int):
+ assert not output.seekable()
+ read = output.read
+
+ def bounded_read(size: int = -1) -> bytes:
+ assert 0 <= size <= 64 * 1024
+ reads.append(size)
+ return read(size)
+
+ output.read = bounded_read
+ return read_batch_samples(output, count)
+
+ monkeypatch.setitem(function_globals, "_read_git_batch_samples", require_bounded_output)
+ # Both request and response exceed pipe buffers, before the multi-megabyte binary.
+ missing = ["HEAD:missing"] * 10_000
+ samples = git_blob_samples(target, [*missing, *(f"HEAD:{name}" for name in contents)])
+
+ assert samples == [
+ *([None] * len(missing)),
+ (text[: 64 * 1024], False),
+ (b"", True),
+ (b"", True),
+ (b"", False),
+ (b"after\n", False),
+ ]
+ assert max(reads) == 64 * 1024
+
+ from rank_preview import is_binary_file
+
+ assert [is_binary_file(target / name) for name in contents] == [False, True, True, False, False]
+
+
+ "output",
+ [
+ b"missing terminator",
+ b"object blob invalid\0",
+ b"object blob -1\0",
+ b"object blob 4\0abc",
+ b"object blob 4\0abcd!",
+ b"object blob invalid\0" + b"x" * (4 * 1024 * 1024),
+ b"object blob 131072\0" + b"x" * (64 * 1024 + 1),
+ ],
+)
+def test_git_blob_samples_reject_incomplete_framing(
+ tmp_path: Path, monkeypatch: pytest.MonkeyPatch, output: bytes
+) -> None:
+ git_blob_samples = WORKBENCH_TARGET["git_blob_samples"]
+
+ def git_command(*args: Any, **kwargs: Any):
+ remaining = memoryview(output)
+ try:
+ while remaining:
+ written = os.write(kwargs["stdout_file"].fileno(), remaining)
+ remaining = remaining[written:]
+ except BrokenPipeError:
+ return subprocess.CompletedProcess(args, 1)
+ return subprocess.CompletedProcess(args, 0)
+
+ monkeypatch.setitem(git_blob_samples.__globals__, "git_command", git_command)
+
+ assert git_blob_samples(tmp_path, ["HEAD:source"]) == [None]
+
+
[email protected]("output", [b"", b"object blob 4\0text\0"])
+def test_git_blob_samples_reject_git_failure(
+ tmp_path: Path, monkeypatch: pytest.MonkeyPatch, output: bytes
+) -> None:
+ git_blob_samples = WORKBENCH_TARGET["git_blob_samples"]
+
+ def git_command(*args: Any, **kwargs: Any):
+ kwargs["stdout_file"].write(output)
+ return subprocess.CompletedProcess(args, 1)
+
+ monkeypatch.setitem(git_blob_samples.__globals__, "git_command", git_command)
+
+ assert git_blob_samples(tmp_path, ["HEAD:source"]) == [None]
+
+
@pytest.mark.parametrize(
("log_encoding", "subject"),
[
diff --git a/sdk/typescript/src/custom-validation-prompt.ts b/sdk/typescript/src/custom-validation-prompt.ts
index 49d6f9610..38f45cf3f 100644
--- a/sdk/typescript/src/custom-validation-prompt.ts
+++ b/sdk/typescript/src/custom-validation-prompt.ts
@@ -13,7 +13,7 @@ const SOURCES = {
"skills/security-scan/SKILL.md":
"104e2f93fc965c34e91970f517a89e330d5b29dbb243dcea5d95d83f14d135cf",
"skills/security-diff-scan/SKILL.md":
- "f85962d46f141227d794a25253a39232bc4e1ad756509ecd3d8769b85b735136",
+ "926290b5f5cee1a30f4c8bb49682fb883fba6f47b1bc73a9a31b9561d0011c8e",
} as const;
const DISABLED_TOOLS = [
diff --git a/sdk/typescript/tests-ts/diff-rank-input.test.ts b/sdk/typescript/tests-ts/diff-rank-input.test.ts
index c0fe64a59..48ce8664d 100644
--- a/sdk/typescript/tests-ts/diff-rank-input.test.ts
+++ b/sdk/typescript/tests-ts/diff-rank-input.test.ts
@@ -95,6 +95,8 @@ test("diff previews stay inside the selected repository", () => {
"src/deleted.py",
"src/entry.py",
"src/handler.py",
+ // Git lists the untracked POSIX symlink, but not the Windows junction.
+ ...(process.platform === "win32" ? [] : ["src/nested"]),
"src/nested/linked.py",
]);
expect(rows.find((row) => row.path === "src/handler.py")?.preview).toBe(
@@ -103,6 +105,9 @@ test("diff previews stay inside the selected repository", () => {
expect(rows.find((row) => row.path === "src/nested/linked.py")?.preview).toBe(
"",
);
+ if (process.platform !== "win32") {
+ expect(rows.find((row) => row.path === "src/nested")?.preview).toBe("");
+ }
});
test("preserves Unicode Git paths and legacy-encoded commit metadata", () => {