Skip to content

OpenAI's Codex Security CLI and TypeScript SDK for finding, validating, and fixing security vulnerabilities. npm: https://www.npmjs.com/package/@openai/codex-security

fix(scans): count changed files when registering diff scans (#1916)

Michael D'Angelo63afe50e76e0
diff --git a/plugins/codex-security/mcp-app/src/helpers/inventory-git.ts b/plugins/codex-security/mcp-app/src/helpers/inventory-git.ts
index 5c9e81ddd..3e67743a5 100644
--- a/plugins/codex-security/mcp-app/src/helpers/inventory-git.ts
+++ b/plugins/codex-security/mcp-app/src/helpers/inventory-git.ts
@@ -1,5 +1,6 @@
import {
environmentValue,
+ isMissingPathError,
resolvedPathText as canonical,
} from "./helper-files";
import {
@@ -47,13 +48,16 @@ interface ToolResult {
}
const repositoryEnvironment = [
+ "GIT_ALLOW_PROTOCOL",
"GIT_ALTERNATE_OBJECT_DIRECTORIES",
"GIT_CEILING_DIRECTORIES",
"GIT_COMMON_DIR",
"GIT_DIR",
"GIT_DISCOVERY_ACROSS_FILESYSTEM",
"GIT_INDEX_FILE",
+ "GIT_LITERAL_PATHSPECS",
"GIT_NAMESPACE",
+ "GIT_NO_LAZY_FETCH",
"GIT_OBJECT_DIRECTORY",
"GIT_WORK_TREE",
];
@@ -325,10 +329,11 @@ function gitProcess(repo: string, args: string[]) {
const inherited = inheritedToolPaths();
for (const name of Object.keys(env)) {
const key = windows ? name.toUpperCase() : name;
- if (repositoryEnvironment.includes(key) || key === "GIT_LITERAL_PATHSPECS")
- delete env[name];
+ if (repositoryEnvironment.includes(key)) delete env[name];
}
env.GIT_LITERAL_PATHSPECS = "1";
+ env.GIT_NO_LAZY_FETCH = "1";
+ env.GIT_ALLOW_PROTOCOL = "";
return spawnTool(
command,
[
@@ -608,11 +613,20 @@ export async function changedPaths(
),
))
if (!path.endsWith("/")) result.set(path, { path, status: "A" });
- return [...result.values()].filter(
- ({ path, status }) =>
- !path.split("/").some((part) => sourceMetadata.has(part)) &&
- (status === "D" || regular(append(repo, path))),
- );
+ const selected: Change[] = [];
+ for (const change of result.values()) {
+ const { path, status } = change;
+ if (path.split("/").some((part) => sourceMetadata.has(part))) continue;
+ const absolute = append(repo, path);
+ try {
+ if (status === "M" && linkedParent(repo, absolute))
+ selected.push({ path, status: "D" });
+ else if (status === "D" || regular(absolute)) selected.push(change);
+ } catch (error) {
+ if (!isMissingPathError(error)) throw error;
+ }
+ }
+ return selected;
}
/** Consume each complete blob while retaining at most the preview prefix. */
diff --git a/plugins/codex-security/scripts/workbench_db.py b/plugins/codex-security/scripts/workbench_db.py
index e86164bb0..575899a18 100644
--- a/plugins/codex-security/scripts/workbench_db.py
+++ b/plugins/codex-security/scripts/workbench_db.py
@@ -114,6 +114,7 @@ from workbench_target import (
clean_worktree_content_digest,
copy_directory_excluding,
copy_git_worktree_files,
+ diff_scope_file_count,
diff_snapshot_digest,
directory_content_digest,
directory_snapshot_regular_file_count,
@@ -619,6 +620,29 @@ def scan_target_root(scan_root: str | None, target: Path) -> Path:
return target_root
+def scan_scope_file_count(
+ target: Path, paths: list[str], diff_target: dict[str, str] | None
+) -> int | None:
+ if diff_target is not None:
+ return diff_scope_file_count(target, diff_target)
+ return sum(
+ 1 if (target / path).is_file() else directory_snapshot_regular_file_count(target / path)
+ for path in paths
+ )
+
+
+def require_working_tree_unchanged(target: Path, diff_target: dict[str, str] | None) -> None:
+ if (
+ diff_target is not None
+ and diff_target["kind"] == "working_tree"
+ and (
+ git_revision(target) != diff_target["headRevision"]
+ or worktree_content_digest(target) != diff_target["contentDigest"]
+ )
+ ):
+ raise SystemExit("The selected scan target changed while the scan was starting. Try again.")
+
+
def start_scan(connection: sqlite3.Connection, args: argparse.Namespace) -> dict[str, Any]:
workspace_id = require_uuid(args.workspace_id, "workspace-id")
manages_transaction = not connection.in_transaction
@@ -657,9 +681,7 @@ def start_scan(connection: sqlite3.Connection, args: argparse.Namespace) -> dict
)
if diff_target is not None and not target_summary:
target_summary = diff_target_summary(diff_target)
- scope_file_count = directory_snapshot_regular_file_count(
- target if scope == "." else target / scope
- )
+ scope_file_count = scan_scope_file_count(target, [scope], diff_target)
target_identity = scan_target_identity(
target,
diff_target,
@@ -673,6 +695,7 @@ def start_scan(connection: sqlite3.Connection, args: argparse.Namespace) -> dict
)
target_root = scan_target_root(args.scan_root, target)
create_private_directory(target_root)
+ require_working_tree_unchanged(target, diff_target)
if manages_transaction:
connection.execute("BEGIN IMMEDIATE")
workspace = require_workspace(connection, workspace_id)
@@ -759,9 +782,7 @@ def _start_prompt_driven_scan(
target_summary = optional_text(args.target_summary, maximum=2400)
if diff_target is not None and not target_summary:
target_summary = diff_target_summary(diff_target)
- scope_file_count = directory_snapshot_regular_file_count(
- target if scope == "." else target / scope
- )
+ scope_file_count = scan_scope_file_count(target, [scope], diff_target)
diff_identity = scan_diff_identity(diff_target)
target_identity = scan_target_identity(target, diff_target)
source_scopes = capture_source_scopes(
@@ -1466,16 +1487,8 @@ def register_cli_scan(
paths or ["."],
diff_target_kind=diff_target["kind"] if diff_target is not None else None,
)
- scope_file_count = (
- directory_snapshot_regular_file_count(repository)
- if not paths
- else sum(
- 1
- if (repository / path).is_file()
- else directory_snapshot_regular_file_count(repository / path)
- for path in paths
- )
- )
+ scope_file_count = scan_scope_file_count(repository, paths or ["."], diff_target)
+ require_working_tree_unchanged(repository, diff_target)
parent_scan_id = (
require_uuid(args.parent_scan_id, "parent-scan-id")
if args.parent_scan_id is not None
diff --git a/plugins/codex-security/scripts/workbench_remediation.py b/plugins/codex-security/scripts/workbench_remediation.py
index 45b521e3f..6792bfa79 100644
--- a/plugins/codex-security/scripts/workbench_remediation.py
+++ b/plugins/codex-security/scripts/workbench_remediation.py
@@ -268,7 +268,7 @@ def require_valid_patch(target: Path, patch: bytes) -> None:
*arguments,
text=False,
input_data=data,
- pathspec_environment={"LC_ALL": "C"},
+ environment_overrides={"LC_ALL": "C", "GIT_NO_REPLACE_OBJECTS": "1"},
offline=True,
)
if result.returncode == 127:
diff --git a/plugins/codex-security/scripts/workbench_scan_start.py b/plugins/codex-security/scripts/workbench_scan_start.py
index fa6885b30..6b5a4df88 100644
--- a/plugins/codex-security/scripts/workbench_scan_start.py
+++ b/plugins/codex-security/scripts/workbench_scan_start.py
@@ -233,7 +233,7 @@ def insert_running_scan(
source_scopes: dict[str, Any],
target_root: Path,
target_summary: str | None,
- scope_file_count: int,
+ scope_file_count: int | None,
timestamp: str,
handoff_status: str = "pending",
model: str | None = None,
diff --git a/plugins/codex-security/scripts/workbench_source_scopes.py b/plugins/codex-security/scripts/workbench_source_scopes.py
index d858b10ec..88198e0c4 100644
--- a/plugins/codex-security/scripts/workbench_source_scopes.py
+++ b/plugins/codex-security/scripts/workbench_source_scopes.py
@@ -74,7 +74,13 @@ def safe_source_path(target: Path, value: str) -> Path | None:
def offline_git_bytes(repository: Path, *arguments: str) -> bytes | None:
- result = git_command(repository, *arguments, text=False, offline=True)
+ result = git_command(
+ repository,
+ *arguments,
+ text=False,
+ offline=True,
+ environment_overrides={"GIT_NO_REPLACE_OBJECTS": "1"},
+ )
return result.stdout if result.returncode == 0 else None
diff --git a/plugins/codex-security/scripts/workbench_target.py b/plugins/codex-security/scripts/workbench_target.py
index 6179a1f6c..74b98e9fe 100644
--- a/plugins/codex-security/scripts/workbench_target.py
+++ b/plugins/codex-security/scripts/workbench_target.py
@@ -3,6 +3,7 @@
from __future__ import annotations
import argparse
+import codecs
import hashlib
import os
import re
@@ -13,6 +14,7 @@ import subprocess
import sys
import tempfile
import unicodedata
+from concurrent.futures import ThreadPoolExecutor
from pathlib import Path, PurePosixPath
from typing import Any, BinaryIO, Callable
@@ -81,11 +83,119 @@ def git_bytes(
*args: str,
git_dir: Path | None = None,
work_tree: Path | None = None,
+ offline: bool = False,
) -> bytes | None:
- completed = git_command(target, *args, text=False, git_dir=git_dir, work_tree=work_tree)
+ completed = git_command(
+ target, *args, text=False, git_dir=git_dir, work_tree=work_tree, offline=offline
+ )
return completed.stdout if completed.returncode == 0 else None
+def git_blob_binary_flags(
+ target: Path,
+ object_names: list[str],
+ *,
+ git_dir: Path | None = None,
+ work_tree: Path | None = None,
+) -> list[bool | None]:
+ """Classify ordered blobs without retaining their contents."""
+ if not object_names:
+ return []
+
+ # Git before 2.38 rejects -z, so only names with line breaks need it.
+ nul_input = any("\n" in name or "\r" in name for name in object_names)
+ separator = b"\0" if nul_input else b"\n"
+ request = separator.join(os.fsencode(name) for name in object_names) + separator
+ read_fd, write_fd = os.pipe()
+ with (
+ ThreadPoolExecutor(max_workers=1) as reader,
+ os.fdopen(read_fd, "rb") as output,
+ os.fdopen(write_fd, "wb") as sink,
+ ):
+
+ def read_samples() -> list[bool | None]:
+ # Closing the reader also unblocks Git if framing is invalid.
+ with output:
+ return _read_git_batch_flags(output, object_names)
+
+ samples = reader.submit(read_samples)
+ try:
+ completed = git_command(
+ target,
+ "cat-file",
+ "--batch",
+ *(["-z"] if nul_input else []),
+ text=False,
+ input_data=request,
+ git_dir=git_dir,
+ work_tree=work_tree,
+ stdout_file=sink,
+ offline=True,
+ )
+ finally:
+ # Git may fail before producing all requested records.
+ sink.close()
+ if completed.returncode != 0:
+ return [None] * len(object_names)
+ try:
+ return samples.result()
+ except ValueError:
+ return [None] * len(object_names)
+
+
+def _read_git_batch_flags(output: BinaryIO, object_names: list[str]) -> list[bool | None]:
+ """Read the older-Git-compatible -z protocol without buffering blob bodies."""
+ samples: list[bool | None] = []
+ for name in object_names:
+ header = output.readline()
+ raw_name = os.fsencode(name)
+ # Missing lookups echo the whole request, including any embedded newlines.
+ if header and raw_name.startswith(header):
+ header += output.read(len(raw_name) - len(header)) + output.readline()
+ if header in (raw_name + b" missing\n", raw_name + b" ambiguous\n"):
+ samples.append(None)
+ continue
+ fields = header.split(b" ")
+ if len(fields) != 3 or not header.endswith(b"\n"):
+ raise ValueError("invalid batch header")
+ size = int(fields[2])
+ if size < 0:
+ raise ValueError("invalid blob size")
+ # Consume non-blob objects too, so subsequent records stay aligned.
+ sample = _stream_is_binary(output, size, drain=True)
+ if output.read(1) != b"\n":
+ raise ValueError("missing blob terminator")
+ samples.append(sample if fields[1] == b"blob" else None)
+ return samples
+
+
+def _is_binary_sample(data: bytes) -> bool:
+ if data.startswith((codecs.BOM_UTF16_LE, codecs.BOM_UTF16_BE)):
+ return "\0" in data.decode("utf-16", errors="ignore")
+ return b"\0" in data
+
+
+def _stream_is_binary(output: BinaryIO, size: int, *, drain: bool = False) -> bool:
+ read_size = 64 * 1024
+ sample_size = min(size, read_size)
+ sample = output.read(sample_size)
+ if len(sample) != sample_size:
+ raise ValueError("truncated blob")
+ binary = _is_binary_sample(sample)
+ bom = sample[:2] if sample.startswith((codecs.BOM_UTF16_LE, codecs.BOM_UTF16_BE)) else b""
+ remaining = size - sample_size
+ while remaining and (drain or not binary):
+ chunk_size = min(remaining, read_size)
+ chunk = output.read(chunk_size)
+ if len(chunk) != chunk_size:
+ raise ValueError("truncated blob")
+ # Even-sized reads keep UTF-16 code units aligned; preserve the original byte order.
+ if not binary:
+ binary = _is_binary_sample(bom + chunk) if bom else b"\0" in chunk
+ remaining -= chunk_size
+ return binary
+
+
def _protected_repository_root(target: Path) -> Path:
root = target.resolve()
if root.is_file():
@@ -162,7 +272,7 @@ def git_command(
git_dir: Path | None = None,
work_tree: Path | None = None,
stdout_file: BinaryIO | None = None,
- pathspec_environment: dict[str, str] | None = None,
+ environment_overrides: dict[str, str] | None = None,
offline: bool = False,
) -> subprocess.CompletedProcess[str] | subprocess.CompletedProcess[bytes]:
if (git_dir is None) != (work_tree is None):
@@ -171,11 +281,11 @@ def git_command(
for name in GIT_REPOSITORY_ENVIRONMENT:
environment.pop(name, None)
environment["GIT_LITERAL_PATHSPECS"] = "1"
- if pathspec_environment is not None:
- environment.update(pathspec_environment)
+ if environment_overrides is not None:
+ environment.update(environment_overrides)
if offline:
# Deny transports even on Git versions that do not honor GIT_NO_LAZY_FETCH.
- environment.update(GIT_NO_LAZY_FETCH="1", GIT_ALLOW_PROTOCOL="", GIT_NO_REPLACE_OBJECTS="1")
+ environment.update(GIT_NO_LAZY_FETCH="1", GIT_ALLOW_PROTOCOL="")
executable = trusted_git_executable(target)
# Repository-local config is untrusted; fsmonitor may name an executable hook.
command = [
@@ -228,6 +338,7 @@ def run_git_changed_paths(repo: Path, diff_args: list[str]) -> list[tuple[Path,
*diff_args,
"--",
text=False,
+ offline=True,
)
result.check_returncode()
fields = result.stdout.split(b"\0")
@@ -263,6 +374,7 @@ def git_changed_paths(repo: Path, base: str, head: str, mode: str) -> list[tuple
"--exclude-standard",
"-z",
text=False,
+ offline=True,
)
untracked.check_returncode()
combined = dict(staged)
@@ -272,15 +384,59 @@ def git_changed_paths(repo: Path, base: str, head: str, mode: str) -> list[tuple
for relative in untracked.stdout.split(b"\0")
if relative and not relative.endswith(b"/")
)
- return sorted(
- (path, status)
- for path, status in combined.items()
- if SOURCE_METADATA_DIRECTORIES.isdisjoint(path.relative_to(repo).parts)
- and (status == "D" or (not path.is_symlink() and path.is_file()))
- )
+ selected = []
+ linked_prefixes: dict[str, bool] = {}
+ for path, status in combined.items():
+ if not SOURCE_METADATA_DIRECTORIES.isdisjoint(path.relative_to(repo).parts):
+ continue
+ try:
+ if status == "M" and _directory_link_boundary(repo, path, linked_prefixes) != path:
+ selected.append((path, "D"))
+ elif status == "D" or (not path.is_symlink() and path.is_file()):
+ selected.append((path, status))
+ except (FileNotFoundError, NotADirectoryError):
+ continue
+ return sorted(selected)
raise SystemExit(f"Unknown diff mode: {mode}")
+def diff_scope_file_count(target: Path, diff_target: dict[str, str]) -> int | None:
+ base, head = diff_target["baseRevision"], diff_target["headRevision"]
+ local = diff_target["kind"] == "working_tree"
+ selected: list[str] = []
+ committed: dict[str, str] = {}
+ try:
+ changes = git_changed_paths(target, base, head, "local-patch" if local else "revisions")
+ except subprocess.CalledProcessError:
+ # A partial clone can lack a tree needed even to enumerate the changed paths.
+ return None
+ for path, status in changes:
+ name = path.relative_to(target).as_posix()
+ if not local or status == "D":
+ committed[name] = f"{base if status == 'D' else head}:{name}"
+ else:
+ try:
+ resolved = path.resolve(strict=True)
+ if not resolved.is_relative_to(target):
+ continue
+ with resolved.open("rb") as stream:
+ binary = _stream_is_binary(stream, os.fstat(stream.fileno()).st_size)
+ except (OSError, ValueError):
+ continue
+ if not binary:
+ selected.append(name)
+ for name, binary in zip(
+ committed, git_blob_binary_flags(target, list(committed.values())), strict=True
+ ):
+ if binary is None:
+ return None
+ if not binary:
+ selected.append(name)
+ if any("\n" in name or "\r" in name for name in selected):
+ return None
+ return len(selected)
+
+
def candidate_source_lines(
target: Path,
diff_target: dict[str, str],
@@ -342,7 +498,14 @@ def candidate_source_lines(
source: dict[str, Any] = {"path": name}
if raw in location_set:
if name not in counts:
- blob = git_command(target, "cat-file", "blob", f"{revision}:{name}", text=False)
+ blob = git_command(
+ target,
+ "cat-file",
+ "blob",
+ f"{revision}:{name}",
+ text=False,
+ offline=True,
+ )
blob.check_returncode()
data = blob.stdout
counts[name] = len(re.split(rb"\r\n|[\r\n]", data)) - int(
@@ -830,7 +993,7 @@ def git_directory_snapshot_paths(target: Path) -> list[Path] | None:
input_data=b"\0".join(candidates) + b"\0",
git_dir=Path(git_directory),
work_tree=repository,
- pathspec_environment={
+ environment_overrides={
"GIT_NOGLOB_PATHSPECS": "0",
},
)
@@ -1110,7 +1273,16 @@ def git_target_metadata(target: Path) -> dict[str, Any]:
branch = git_output(target, "symbolic-ref", "--quiet", "--short", "HEAD")
metadata.update({"branch": branch, "detachedHead": revision is not None and branch is None})
if revision is not None:
- subject = git_bytes(target, "show", "-s", "--no-show-signature", "--format=%s", "HEAD")
+ subject = git_bytes(
+ target,
+ "log",
+ "-1",
+ "--no-patch",
+ "--no-show-signature",
+ "--format=%s",
+ "HEAD",
+ offline=True,
+ )
metadata.update(
{
"commitSubject": (subject or b"").decode("utf-8", errors="replace").strip() or None,
diff --git a/plugins/codex-security/tests/test_git_inventory_regressions.py b/plugins/codex-security/tests/test_git_inventory_regressions.py
index 9a33642cb..20e7c9fb6 100644
--- a/plugins/codex-security/tests/test_git_inventory_regressions.py
+++ b/plugins/codex-security/tests/test_git_inventory_regressions.py
@@ -5,7 +5,7 @@ import json
import os
import sqlite3
import subprocess
-from contextlib import closing
+from contextlib import closing, nullcontext
from pathlib import Path
import pytest
@@ -43,6 +43,95 @@ def test_sha256_root_commit_uses_the_sha256_empty_tree(tmp_path: Path, workbench
assert git(target, "diff", "--name-only", *revisions) == b"README.md"
+def test_blob_batch_preserves_following_blobs_after_tree_and_commit(tmp_path: Path) -> None:
+ target = tmp_path / "target"
+ initialize_git_repository(target)
+ source = target / ("fixture.py" if os.name == "nt" else "line\nbreak.py")
+ source.write_bytes(b"print('fixture')\npayload\n")
+ git(target, "add", "--", source.name)
+ git(target, "commit", "-qm", "Add unusual fixture")
+ reader = load_script("workbench_target").git_blob_binary_flags
+ assert reader(
+ target,
+ ["HEAD^{tree}", "HEAD", f"HEAD:{source.name}", "HEAD:missing\nfile", "HEAD:README.md"],
+ ) == [
+ None,
+ None,
+ False,
+ None,
+ False,
+ ]
+
+
+def test_working_tree_count_stops_reading_a_local_binary(tmp_path: Path, monkeypatch) -> None:
+ target = tmp_path / "target"
+ head = initialize_git_repository(target)
+ binary = target / "binary.dat"
+ binary.write_bytes(b"\0" + b"x" * (1024 * 1024))
+ open_file = Path.open
+ diff = {"kind": "working_tree", "baseRevision": head, "headRevision": head}
+ with binary.open("rb") as stream:
+
+ def open_counted(path, *args, **kwargs):
+ return nullcontext(stream) if path == binary else open_file(path, *args, **kwargs)
+
+ monkeypatch.setattr(Path, "open", open_counted)
+ assert load_script("workbench_target").diff_scope_file_count(target, diff) == 0
+ assert 0 < stream.tell() < binary.stat().st_size
+
+
+def test_registration_counts_the_selected_git_replacement_view(tmp_path: Path, monkeypatch) -> None:
+ target = tmp_path / "target"
+ base = initialize_git_repository(target)
+ (target / "added.py").write_text("text\n")
+ git(target, "add", ".")
+ git(target, "commit", "-qm", "Add text file")
+ original = git(target, "rev-parse", "HEAD:added.py").decode()
+ replacement = git(target, "hash-object", "-w", "--stdin", input=b"\0binary").decode()
+ git(target, "replace", original, replacement)
+ diff = {"kind": "range", "baseRevision": base, "headRevision": "HEAD"}
+ reader = load_script("workbench_target")
+ monkeypatch.delenv("GIT_NO_REPLACE_OBJECTS", raising=False)
+ assert reader.diff_scope_file_count(target, diff) == 0
+ monkeypatch.setenv("GIT_NO_REPLACE_OBJECTS", "1")
+ assert reader.diff_scope_file_count(target, diff) == 1
+
+
+def test_working_tree_uses_base_source_below_a_replaced_directory(tmp_path: Path) -> None:
+ target = tmp_path / "target"
+ initialize_git_repository(target)
+ parent = target / "nested"
+ parent.mkdir()
+ name = "nested/source.py"
+ (target / name).write_text("base line one\nbase line two\n")
+ git(target, "add", ".")
+ git(target, "commit", "-qm", "Add base file")
+ base = git(target, "rev-parse", "HEAD").decode()
+ (target / name).write_text("changed\n")
+ git(target, "commit", "-qam", "Modify file")
+ head = git(target, "rev-parse", "HEAD").decode()
+ git(target, "update-index", "--skip-worktree", name)
+ (target / name).unlink()
+ parent.rmdir()
+ outside = tmp_path / "outside"
+ outside.mkdir()
+ (outside / "source.py").write_bytes(b"\0binary")
+ if os.name == "nt":
+ subprocess.run(["cmd", "/c", "mklink", "/J", str(parent), str(outside)], check=True)
+ else:
+ parent.symlink_to(outside, target_is_directory=True)
+
+ reader = load_script("workbench_target")
+ diff = {"kind": "working_tree", "baseRevision": base, "headRevision": head}
+ assert reader.diff_scope_file_count(target, diff) == 1
+ assert reader.candidate_source_lines(target, diff, [], [name]) == {
+ name: {"path": name, "lineCount": 2}
+ }
+
+
def test_plain_directory_inventory_ignores_nested_git_metadata(tmp_path: Path) -> None:
target = tmp_path / "sources"
target.mkdir()
diff --git a/plugins/codex-security/tests/test_workbench_completion_binding.py b/plugins/codex-security/tests/test_workbench_completion_binding.py
index 0bc9701f9..b6775ebdb 100644
--- a/plugins/codex-security/tests/test_workbench_completion_binding.py
+++ b/plugins/codex-security/tests/test_workbench_completion_binding.py
@@ -88,6 +88,140 @@ def register_cli_scan(
)
+ ("kind", "expected"), [("repository", 4), ("refs", 2), ("working_tree", 3)]
+)
+def test_cli_registration_counts_the_selected_target(
+ tmp_path: Path, kind: str, expected: int
+) -> None:
+ state_dir = tmp_path / "state"
+ target = tmp_path / "target"
+ base = initialize_git_repository(target)
+ (target / "README.md").write_text("updated fixture\n")
+ (target / "new.py").write_text("new = True\n")
+ (target / "binary.dat").write_bytes(b"\0")
+ subprocess.run(["git", "add", "."], cwd=target, check=True)
+ subprocess.run(["git", "commit", "-qm", "Update fixture"], cwd=target, check=True)
+
+ (target / "README.md").unlink()
+ (target / "new.py").write_bytes(b"\0")
+ (target / "staged.py").write_text("staged = True\n")
+ subprocess.run(["git", "add", "staged.py"], cwd=target, check=True)
+ (target / "untracked.py").write_text("untracked = True\n")
+
+ scan_target = {"kind": kind, "paths": []}
+ if kind != "repository":
+ scan_target.update(base=base if kind == "refs" else "HEAD", head="HEAD")
+ registered = register_cli_scan(state_dir, target, tmp_path / "scan", scan_target=scan_target)
+ assert registered["scopeFileCount"] == expected
+ saved = get_scan(state_dir, registered["scanId"])["scan"]
+ assert saved["progress"]["coverage"]["filesTotal"] == expected
+
+
[email protected]("missing_object", ["head_blob", "base_blob", "base_tree"])
+def test_diff_registration_keeps_unknown_total_for_missing_local_objects(
+ tmp_path: Path, missing_object: str
+) -> None:
+ target = tmp_path / "target"
+ base = initialize_git_repository(target)
+ (target / "changed.py").write_text("changed = True\n")
+
+ def git(*args: str) -> str:
+ return subprocess.check_output(["git", "-C", str(target), *args], text=True).strip()
+
+ (target / "README.md").unlink()
+ git("add", "-A")
+ git("commit", "-qm", "Add changed file")
+ oid = git(
+ "rev-parse",
+ {
+ "head_blob": "HEAD:changed.py",
+ "base_blob": f"{base}:README.md",
+ "base_tree": f"{base}^{{tree}}",
+ }[missing_object],
+ )
+ remote = tmp_path / "remote.git"
+ git("clone", "--bare", "--no-local", str(target), str(remote))
+ git("-C", str(remote), "config", "uploadpack.allowFilter", "true")
+ git("remote", "add", "origin", remote.as_uri())
+ git("config", "remote.origin.promisor", "true")
+ git("config", "remote.origin.partialCloneFilter", "blob:none")
+ missing = target / ".git" / "objects" / oid[:2] / oid[2:]
+ missing.chmod(0o600) # Git marks loose objects read-only on Windows.
+ missing.unlink()
+
+ state_dir = tmp_path / "state"
+ registered = register_cli_scan(
+ state_dir,
+ target,
+ tmp_path / "scan",
+ scan_target={"kind": "refs", "paths": [], "base": base, "head": "HEAD"},
+ )
+ assert registered["scopeFileCount"] is None
+ saved = get_scan(state_dir, registered["scanId"])["scan"]
+ assert saved["progress"]["coverage"]["filesTotal"] is None
+ assert (
+ subprocess.run(
+ ["git", "-C", str(target), "cat-file", "-e", oid],
+ env={**os.environ, "GIT_NO_LAZY_FETCH": "1", "GIT_ALLOW_PROTOCOL": ""},
+ ).returncode
+ != 0
+ )
+
+
[email protected]("entry_point", ["register-cli-scan", "start-scan"])
+def test_working_tree_change_during_count_prevents_registration(
+ tmp_path: Path, monkeypatch: pytest.MonkeyPatch, entry_point: str
+) -> None:
+ state_dir, target = tmp_path / "state", tmp_path / "target"
+ head = initialize_git_repository(target)
+ source = target / "README.md"
+ source.write_text("selected version\n")
+ monkeypatch.setenv("CODEX_SECURITY_STATE_DIR", str(state_dir))
+ db = load_script("workbench_db")
+ count = db.scan_scope_file_count
+
+ def count_then_change(*args):
+ total = count(*args)
+ source.write_text("changed during registration\n")
+ return total
+
+ monkeypatch.setattr(db, "scan_scope_file_count", count_then_change)
+ if entry_point == "register-cli-scan":
+ scan_dir = tmp_path / "scan"
+ scan_dir.mkdir(mode=0o700)
+ recipe = {
+ "repository": str(target),
+ "config": {},
+ "mode": "standard",
+ "target": {"kind": "working_tree", "base": head, "head": head, "paths": []},
+ }
+ options = [
+ "--repository",
+ str(target),
+ "--scan-dir",
+ str(scan_dir),
+ "--recipe-json",
+ json.dumps(recipe),
+ ]
+ else:
+ workspace_id = str(uuid.uuid4())
+ create_workspace(state_dir, workspace_id)
+ save_workspace(
+ state_dir, workspace_id, str(target), ".", "diff", "--diff-target-kind", "working_tree"
+ )
+ options = ["--workspace-id", workspace_id, "--scan-root", str(tmp_path / "scans")]
+ monkeypatch.setattr(sys, "argv", ["workbench_db.py", entry_point, *options])
+ args = db.parse_args(db.__doc__)
+ with db.connect() as connection:
+ start = db.register_cli_scan if entry_point == "register-cli-scan" else db.start_scan
+ with pytest.raises(SystemExit, match="changed"):
+ start(connection, args)
+ assert connection.execute("SELECT count(*) FROM scans").fetchone()[0] == 0
+
+
def _start_diff_scan_with_draft(tmp_path: Path, kind: str) -> tuple[Path, Path, dict[str, Any]]:
state_dir = tmp_path / "state"
target = tmp_path / "target"
diff --git a/plugins/codex-security/tests/test_workbench_db.py b/plugins/codex-security/tests/test_workbench_db.py
index bad47bfee..a07d015be 100644
--- a/plugins/codex-security/tests/test_workbench_db.py
+++ b/plugins/codex-security/tests/test_workbench_db.py
@@ -2440,6 +2440,7 @@ def test_workbench_starts_diff_without_presentation_label(tmp_path: Path) -> Non
state_dir = tmp_path / "state"
target = tmp_path / "target"
revision = initialize_git_repository(target)
+ (target / "checkout-only.py").write_text("untracked = True\n")
workspace_id = str(uuid.uuid4())
create_workspace(state_dir, workspace_id)
save_workspace(
@@ -2460,6 +2461,7 @@ def test_workbench_starts_diff_without_presentation_label(tmp_path: Path) -> Non
"UPDATE workspaces SET target_summary = NULL WHERE id = ?", (workspace_id,)
)
started = start_scan_command(state_dir, workspace_id, "--scan-root", str(tmp_path / "scans"))
+ assert started["results"]["progress"]["coverage"]["filesTotal"] == 1
assert started["results"]["diffTarget"]["headRevision"] == revision
assert started["results"]["targetSummary"] == f"Commit {revision[:7]}"
diff --git a/plugins/codex-security/tests/test_workbench_prompt_only_scan.py b/plugins/codex-security/tests/test_workbench_prompt_only_scan.py
index 31609da90..dabe327e3 100644
--- a/plugins/codex-security/tests/test_workbench_prompt_only_scan.py
+++ b/plugins/codex-security/tests/test_workbench_prompt_only_scan.py
@@ -477,6 +477,7 @@ def test_prompt_only_diff_scan_validates_and_persists_canonical_diff_identity(
target = tmp_path / "target"
head = initialize_git_repository(target)
(target / "README.md").write_text("changed fixture\n")
+ (target / "binary.dat").write_bytes(b"\0")
started = start_prompt_only_scan(
state_dir,
target,
@@ -485,6 +486,7 @@ def test_prompt_only_diff_scan_validates_and_persists_canonical_diff_identity(
mode="diff",
extra_args=("--diff-target-kind", "working_tree"),
)
+ assert started["scan"]["progress"]["coverage"]["filesTotal"] == 1
assert started["scan"]["diffTarget"]["kind"] == "working_tree"
assert started["scan"]["diffTarget"]["baseRevision"] == head
assert started["scan"]["diffTarget"]["headRevision"] == head
diff --git a/plugins/codex-security/tests/test_workbench_target.py b/plugins/codex-security/tests/test_workbench_target.py
index 0d5cc129f..d2f83096d 100644
--- a/plugins/codex-security/tests/test_workbench_target.py
+++ b/plugins/codex-security/tests/test_workbench_target.py
@@ -820,6 +820,140 @@ def test_stale_git_binding_does_not_spawn(tmp_path: Path, monkeypatch: pytest.Mo
assert result.args[0] == "git"
+ ("encoding", "bom"),
+ [("utf-8", b""), ("utf-16-le", b"\xff\xfe"), ("utf-16-be", b"\xfe\xff")],
+)
+def test_git_blob_binary_flags_match_full_file_classification_with_bounded_reads(
+ tmp_path: Path, monkeypatch: pytest.MonkeyPatch, encoding: str, bom: bytes
+) -> None:
+ target = tmp_path / "target"
+ initialize_git_repository(target)
+ # The UTF-16 surrogate pair crosses the first 64 KiB sample boundary.
+ text = bom + ("a" * 32766 + "😀" + "b" * 32768).encode(encoding)
+ contents = {
+ "text.bin": text + (b"\0" if bom else b""),
+ "binary-tail.bin": text + "\0".encode(encoding),
+ "binary-head.bin": b"\0" + text * 32,
+ "empty.bin": b"",
+ "after.txt": b"after\n",
+ }
+ for name, content in contents.items():
+ (target / name).write_bytes(content)
+ subprocess.run(["git", "add", "."], cwd=target, check=True)
+ subprocess.run(["git", "commit", "-qm", "source samples"], cwd=target, check=True)
+
+ git_blob_binary_flags = WORKBENCH_TARGET["git_blob_binary_flags"]
+ function_globals = git_blob_binary_flags.__globals__
+ read_batch_samples = function_globals["_read_git_batch_flags"]
+ reads: list[int] = []
+
+ def require_bounded_output(output: Any, names: list[str]):
+ assert not output.seekable()
+ read = output.read
+
+ def bounded_read(size: int = -1) -> bytes:
+ assert 0 <= size <= 64 * 1024
+ reads.append(size)
+ return read(size)
+
+ output.read = bounded_read
+ return read_batch_samples(output, names)
+
+ monkeypatch.setitem(function_globals, "_read_git_batch_flags", require_bounded_output)
+ # Both request and response exceed pipe buffers, before the multi-megabyte binary.
+ missing = ["HEAD:missing"] * 10_000
+ samples = git_blob_binary_flags(target, [*missing, *(f"HEAD:{name}" for name in contents)])
+
+ assert samples == [
+ *([None] * len(missing)),
+ False,
+ True,
+ True,
+ False,
+ False,
+ ]
+ assert max(reads) == 64 * 1024
+
+
+ "output",
+ [
+ b"missing terminator",
+ b"object blob invalid\n",
+ b"object blob -1\n",
+ b"object blob 4\nabc",
+ b"object blob 4\nabcd!",
+ b"object blob invalid\n" + b"x" * (4 * 1024 * 1024),
+ b"object blob 131072\n" + b"x" * (64 * 1024 + 1),
+ ],
+ ids=[
+ "unterminated-header",
+ "invalid-size",
+ "negative-size",
+ "short-body",
+ "invalid-terminator",
+ "large-invalid-header",
+ "truncated-large-body",
+ ],
+)
+def test_git_blob_binary_flags_reject_incomplete_framing(
+ tmp_path: Path, monkeypatch: pytest.MonkeyPatch, output: bytes
+) -> None:
+ git_blob_binary_flags = WORKBENCH_TARGET["git_blob_binary_flags"]
+
+ def git_command(*args: Any, **kwargs: Any):
+ remaining = memoryview(output)
+ try:
+ while remaining:
+ written = os.write(kwargs["stdout_file"].fileno(), remaining)
+ remaining = remaining[written:]
+ except BrokenPipeError:
+ return subprocess.CompletedProcess(args, 1)
+ return subprocess.CompletedProcess(args, 0)
+
+ monkeypatch.setitem(git_blob_binary_flags.__globals__, "git_command", git_command)
+
+ assert git_blob_binary_flags(tmp_path, ["HEAD:source"]) == [None]
+
+
[email protected]("output", [b"", b"object blob 4\ntext\n"])
+def test_git_blob_binary_flags_reject_git_failure(
+ tmp_path: Path, monkeypatch: pytest.MonkeyPatch, output: bytes
+) -> None:
+ git_blob_binary_flags = WORKBENCH_TARGET["git_blob_binary_flags"]
+
+ def git_command(*args: Any, **kwargs: Any):
+ kwargs["stdout_file"].write(output)
+ return subprocess.CompletedProcess(args, 1)
+
+ monkeypatch.setitem(git_blob_binary_flags.__globals__, "git_command", git_command)
+
+ assert git_blob_binary_flags(tmp_path, ["HEAD:source"]) == [None]
+
+
+def test_git_blob_binary_flags_work_with_git_before_2_38(
+ tmp_path: Path, monkeypatch: pytest.MonkeyPatch
+) -> None:
+ target = tmp_path / "target"
+ initialize_git_repository(target)
+ git_blob_binary_flags = WORKBENCH_TARGET["git_blob_binary_flags"]
+ git_command = git_blob_binary_flags.__globals__["git_command"]
+
+ def git_before_2_38(*args: Any, **kwargs: Any):
+ # Git before 2.38 rejects cat-file -z with a usage error.
+ if "-z" in args:
+ return subprocess.CompletedProcess(args, 129)
+ return git_command(*args, **kwargs)
+
+ monkeypatch.setitem(git_blob_binary_flags.__globals__, "git_command", git_before_2_38)
+
+ assert git_blob_binary_flags(target, ["HEAD:README.md", "HEAD:missing"]) == [
+ False,
+ None,
+ ]
+
+
@pytest.mark.parametrize(
("log_encoding", "subject"),
[
diff --git a/sdk/typescript/tests-ts/inventory-input.test.ts b/sdk/typescript/tests-ts/inventory-input.test.ts
index f7e7a7ef6..5bebcd0c3 100644
--- a/sdk/typescript/tests-ts/inventory-input.test.ts
+++ b/sdk/typescript/tests-ts/inventory-input.test.ts
@@ -961,8 +961,13 @@ for (const location of ["outside", "repository-extra", "inside"]) {
const f = fixture();
const contents = binary ? Buffer.from("\0selected\n") : "selected\n";
f.write("inside/source.py", contents);
+ f.write("inside/changed.py", Buffer.from("\0binary"));
+ f.write("src/nested/changed.py", "original\n");
f.write("safe.py", "before\n");
const base = f.commit();
+ f.write("src/nested/changed.py", "modified\n");
+ f.commit();
+ git(f.repo, "update-index", "--skip-worktree", "src/nested/changed.py");
f.write("src/nested/source.py", "staged\n");
git(f.repo, "add", "src/nested/source.py");
f.write("safe.py", "after\n");
@@ -971,6 +976,7 @@ for (const location of ["outside", "repository-extra", "inside"]) {
if (location !== "inside") {
mkdirSync(destination);
writeFileSync(join(destination, "source.py"), contents);
+ writeFileSync(join(destination, "changed.py"), Buffer.from("\0binary"));
}
rmSync(join(f.repo, "src", "nested"), { recursive: true });
symlinkSync(destination, join(f.repo, "src", "nested"), "junction");
@@ -985,7 +991,9 @@ for (const location of ["outside", "repository-extra", "inside"]) {
"--diff-mode",
"local-patch",
]),
- ).toBe(`safe.py\n${retained ? "src/nested/source.py\n" : ""}`);
+ ).toBe(
+ `safe.py\nsrc/nested/changed.py\n${retained ? "src/nested/source.py\n" : ""}`,
+ );
expect(
f.rows("make-diff-rank-input", [
"--base",
@@ -995,6 +1003,7 @@ for (const location of ["outside", "repository-extra", "inside"]) {
]),
).toEqual([
{ path: "safe.py", area: "diff", preview: "after" },
+ { path: "src/nested/changed.py", area: "diff", preview: "" },
...(location !== "inside" || !binary
? [
{