Skip to content

OpenAI's Codex Security CLI and TypeScript SDK for finding, validating, and fixing security vulnerabilities. npm: https://www.npmjs.com/package/@openai/codex-security

fix(targets): preserve scoped Git inventories and directory snapshots (#1280)

mldangelo-oaibbb64bd61dfd
diff --git a/.github/workflows/node-ci.yml b/.github/workflows/node-ci.yml
index 22df56c77..0088de432 100644
--- a/.github/workflows/node-ci.yml
+++ b/.github/workflows/node-ci.yml
@@ -404,11 +404,21 @@ jobs:
- name: Test source compatibility checker
if: matrix.os == 'ubuntu-latest' && matrix.python == '3.12'
run: node --test .github/scripts/test_check_plugin_source_compatibility.mjs
+ - name: Prepare private Windows test root
+ if: runner.os == 'Windows'
+ id: windows-temp
+ shell: pwsh
+ run: ./sdk/typescript/scripts/prepare-windows-test-root.ps1
- name: Test Python source contracts
shell: bash
env:
PYTEST_MARKERS: ${{ matrix.markers }}
- run: python -m pytest plugins/codex-security/tests -m "$PYTEST_MARKERS" -q -n 4 --dist worksteal --max-worker-restart 0 --durations=30 --junitxml=reports/python.xml
+ WINDOWS_TEST_ROOT: ${{ steps.windows-temp.outputs.path }}
+ run: |
+ if [ -n "$WINDOWS_TEST_ROOT" ]; then
+ export TEMP="$WINDOWS_TEST_ROOT" TMP="$WINDOWS_TEST_ROOT" TMPDIR="$WINDOWS_TEST_ROOT"
+ fi
+ python -m pytest plugins/codex-security/tests -m "$PYTEST_MARKERS" -q -n 4 --dist worksteal --max-worker-restart 0 --durations=30 --junitxml=reports/python.xml
- name: Require native platform coverage
if: runner.os != 'Linux'
run: python -c "import xml.etree.ElementTree as ET; assert not ET.parse('reports/python.xml').findall('.//testcase/skipped'), 'Native platform tests must run without skips'"
diff --git a/plugins/codex-security/scripts/windows_scan_local_files.py b/plugins/codex-security/scripts/windows_scan_local_files.py
index 465a40abc..b4e60490d 100644
--- a/plugins/codex-security/scripts/windows_scan_local_files.py
+++ b/plugins/codex-security/scripts/windows_scan_local_files.py
@@ -4,13 +4,16 @@ Windows does not implement Python's descriptor-relative ``dir_fd`` APIs or
``O_NOFOLLOW``. This module provides the three operations needed by the scan
finalizer without falling back to check-then-use path validation.
-The implementation opens every directory with ``FILE_FLAG_OPEN_REPARSE_POINT``
-and rejects all reparse points, including junctions. Directory handles remain
-open without ``FILE_SHARE_DELETE`` for the full operation, which prevents an
+Scan-local reads, writes, and deletes open every directory with
+``FILE_FLAG_OPEN_REPARSE_POINT`` and reject all reparse points, including junctions.
+Directory handles remain open without ``FILE_SHARE_DELETE`` for the full
+operation, which prevents an
already-validated ancestor from being renamed or replaced during a read,
write, or delete. Writes rename the exact temporary-file handle into place so
an attacker cannot substitute another file at the temporary name.
+The snapshot helper copies junction metadata without opening the link target.
+
Importing this module and comparing streams work on every platform. Filesystem
operations raise ``WindowsScanLocalFileError`` outside Windows.
"""
@@ -202,6 +205,19 @@ if os.name == "nt":
]
_WriteFile.restype = wintypes.BOOL
+ _DeviceIoControl = _kernel32.DeviceIoControl
+ _DeviceIoControl.argtypes = [
+ wintypes.HANDLE,
+ wintypes.DWORD,
+ wintypes.LPVOID,
+ wintypes.DWORD,
+ wintypes.LPVOID,
+ wintypes.DWORD,
+ ctypes.POINTER(wintypes.DWORD),
+ wintypes.LPVOID,
+ ]
+ _DeviceIoControl.restype = wintypes.BOOL
+
_FlushFileBuffers = _kernel32.FlushFileBuffers
_FlushFileBuffers.argtypes = [wintypes.HANDLE]
_FlushFileBuffers.restype = wintypes.BOOL
@@ -309,6 +325,34 @@ def _create_file(
return _OwnedHandle(int(handle))
+def copy_directory_junction(source: Path, destination: Path) -> None:
+ """Copy raw junction data to an empty directory without opening the link target."""
+ # MAXIMUM_REPARSE_DATA_BUFFER_SIZE, including the reparse data header.
+ data = ctypes.create_string_buffer(16 * 1024)
+ size = wintypes.DWORD()
+ for path, write in ((source, False), (destination, True)):
+ handle = _create_file(
+ path,
+ access=_GENERIC_WRITE if write else _FILE_READ_ATTRIBUTES,
+ share=_FILE_SHARE_READ | _FILE_SHARE_WRITE,
+ disposition=_OPEN_EXISTING,
+ flags=_FILE_FLAG_BACKUP_SEMANTICS | _FILE_FLAG_OPEN_REPARSE_POINT,
+ )
+ assert handle is not None
+ with handle:
+ if not _DeviceIoControl(
+ handle.value,
+ 0x000900A4 if write else 0x000900A8, # FSCTL_SET/GET_REPARSE_POINT.
+ data if write else None,
+ size.value if write else 0,
+ None if write else data,
+ 0 if write else len(data),
+ ctypes.byref(size),
+ None,
+ ):
+ _raise_last_error("Copy directory junction", path)
+
+
def _attributes(handle: int) -> _FileAttributeTagInfo:
info = _FileAttributeTagInfo()
if not _GetFileInformationByHandleEx(
diff --git a/plugins/codex-security/scripts/workbench_db.py b/plugins/codex-security/scripts/workbench_db.py
index e2082285f..88791d5a6 100644
--- a/plugins/codex-security/scripts/workbench_db.py
+++ b/plugins/codex-security/scripts/workbench_db.py
@@ -115,6 +115,7 @@ from workbench_target import (
require_git_worktree_head,
require_remediation_target,
require_scan_target_identity,
+ restore_directory_junctions,
scan_target_warning,
worktree_content_digest,
worktree_content_digest_for_context,
@@ -329,7 +330,11 @@ def require_diff_target(
if commit is None:
raise SystemExit(f"Commit is not available in the local checkout: {head}")
parent_line = next(
- (line for line in commit.splitlines() if line.startswith(b"parent ")),
+ (
+ line
+ for line in commit.split(b"\n\n", 1)[0].splitlines()
+ if line.startswith(b"parent ")
+ ),
None,
)
if parent_line is None:
@@ -2402,11 +2407,10 @@ def require_reviewed_patch_applied(
)
unversioned = remediation["base_revision"] == "unversioned"
excluded = (Path(scan["scan_dir"]),)
- git_dir = None
- pathspec = None
+ # Unborn repositories still need their config while metadata junctions are deferred.
+ git_dir = git_output(target, "rev-parse", "--absolute-git-dir")
if not unversioned:
- _, pathspec = git_worktree_context(target)
- git_dir = git_output(target, "rev-parse", "--absolute-git-dir")
+ repository, pathspec = git_worktree_context(target)
if git_dir is None:
raise SystemExit("Could not inspect the selected Git working tree.")
excluded += tuple(path for path, _ in git_submodule_entries(target))
@@ -2423,9 +2427,9 @@ def require_reviewed_patch_applied(
checkout_root = Path(temporary) / "checkout"
if unversioned:
checkout = checkout_root
- copy_directory_excluding(target, checkout, excluded)
+ junctions = copy_directory_excluding(target, checkout, excluded)
else:
- copy_git_worktree_files(target, checkout_root, excluded)
+ _, junctions = copy_git_worktree_files(target, checkout_root, excluded)
arguments = ["apply", "--reverse", "--whitespace=nowarn"]
if unversioned:
arguments.append("--no-index")
@@ -2443,6 +2447,11 @@ def require_reviewed_patch_applied(
raise SystemExit(
"The selected checkout does not contain the reviewed remediation patch. Apply exactly that patch before recording it as applied."
)
+ restore_directory_junctions(
+ target if unversioned else repository,
+ checkout if unversioned else checkout_root,
+ junctions,
+ )
reverted_digest = (
directory_content_digest(checkout)
if unversioned
@@ -2455,11 +2464,16 @@ def require_reviewed_patch_applied(
)
if reverted_digest != remediation["base_content_digest"] and unversioned:
checkout = Path(temporary) / "checkout-lf"
- copy_directory_excluding(target, checkout, excluded)
+ junctions = copy_directory_excluding(target, checkout, excluded)
applied_without_conversion = git_command(
- checkout, "-c", "core.autocrlf=input", *arguments, text=True
+ checkout,
+ "-c",
+ "core.autocrlf=input",
+ *arguments,
+ text=True,
)
if applied_without_conversion.returncode == 0:
+ restore_directory_junctions(target, checkout, junctions)
reverted_digest = directory_content_digest(checkout)
if reverted_digest != remediation["base_content_digest"]:
raise SystemExit(
diff --git a/plugins/codex-security/scripts/workbench_finding_workflows.py b/plugins/codex-security/scripts/workbench_finding_workflows.py
index cb425545c..6305d786f 100644
--- a/plugins/codex-security/scripts/workbench_finding_workflows.py
+++ b/plugins/codex-security/scripts/workbench_finding_workflows.py
@@ -11,7 +11,7 @@ from pathlib import Path
from typing import Any
sys.path.insert(0, str(Path(__file__).resolve().parent))
-from workbench_target import directory_content_digest, git_output, git_revision
+from workbench_target import directory_content_digest, git_bytes, git_revision
WORKFLOW_BINDINGS = {
"repositoryPath": "repository_path",
@@ -138,9 +138,9 @@ def finding_workflow(
"repository": str(target),
"revision": "unversioned" if payload.get("gitDisabled") else git_revision(target),
"refsDigest": hashlib.sha256(
- (
- "" if payload.get("gitDisabled") else (git_output(target, "show-ref") or "")
- ).encode()
+ b""
+ if payload.get("gitDisabled")
+ else (git_bytes(target, "show-ref") or b"").strip()
).hexdigest(),
"content": directory_content_digest(
target, excluded=excluded, include_ignored=True
diff --git a/plugins/codex-security/scripts/workbench_target.py b/plugins/codex-security/scripts/workbench_target.py
index 9ea0de267..b1c962ca9 100644
--- a/plugins/codex-security/scripts/workbench_target.py
+++ b/plugins/codex-security/scripts/workbench_target.py
@@ -15,14 +15,17 @@ import sys
import tempfile
from concurrent.futures import ThreadPoolExecutor
from pathlib import Path, PurePosixPath
-from typing import Any, BinaryIO
+from typing import Any, BinaryIO, Callable
# Some plugin hosts launch Python with safe-path isolation enabled.
sys.path.insert(0, str(Path(__file__).resolve().parent))
from filesystem_identity import stored_filesystem_identity_matches
from rank_preview import DEFAULT_PREVIEW_READ_BYTES, is_binary_sample
+from windows_scan_local_files import copy_directory_junction
from workbench_constants import GIT_REPOSITORY_ENVIRONMENT
+_WINDOWS = os.name == "nt"
+
def committed_diff_snapshot_digest(kind: str, base_revision: str, head_revision: str) -> str:
digest = hashlib.sha256(
@@ -95,7 +98,7 @@ def git_blob_samples(
def read_samples() -> list[tuple[bytes, bool] | None]:
# Closing the reader also unblocks Git if framing is invalid.
with output:
- return _read_git_batch_samples(output, len(object_names))
+ return _read_git_batch_samples(output, object_names)
samples = reader.submit(read_samples)
try:
@@ -103,7 +106,7 @@ def git_blob_samples(
target,
"cat-file",
"--batch",
- "-Z",
+ "-z",
text=False,
input_data=request,
git_dir=git_dir,
@@ -121,40 +124,30 @@ def git_blob_samples(
return [None] * len(object_names)
-def _read_git_batch_samples(output: BinaryIO, count: int) -> list[tuple[bytes, bool] | None]:
- """Read NUL-framed headers, then sample each length-delimited blob."""
+def _read_git_batch_samples(
+ output: BinaryIO, object_names: list[str]
+) -> list[tuple[bytes, bool] | None]:
+ """Read the older-Git-compatible -z protocol without buffering blob bodies."""
samples: list[tuple[bytes, bool] | None] = []
- for _ in range(count):
- size = _git_batch_blob_size(_read_nul_field(output))
- if size is None:
+ for name in object_names:
+ header = output.readline()
+ raw_name = os.fsencode(name)
+ # Missing lookups echo the whole request, including any embedded newlines.
+ if header and raw_name.startswith(header):
+ header += output.read(len(raw_name) - len(header)) + output.readline()
+ if header in (raw_name + b" missing\n", raw_name + b" ambiguous\n"):
samples.append(None)
continue
- samples.append(_read_git_blob_sample(output, size))
- return samples
-
-
-def _read_nul_field(output: BinaryIO) -> bytes:
- """Read one NUL-terminated protocol field."""
- field = bytearray()
- while char := output.read(1):
- if char == b"\0":
- return bytes(field)
- field.extend(char)
- raise ValueError("missing NUL terminator")
-
-
-def _git_batch_blob_size(header: bytes) -> int | None:
- """Return a blob header's byte count, or ``None`` for a non-blob record."""
- fields = header.rsplit(b" ", 2)
- if len(fields) != 3 or fields[1] != b"blob":
- return None
- try:
+ fields = header.split(b" ")
+ if len(fields) != 3 or not header.endswith(b"\n"):
+ raise ValueError("invalid batch header")
size = int(fields[2])
- except ValueError as error:
- raise ValueError("invalid blob size") from error
- if size < 0:
- raise ValueError("invalid blob size")
- return size
+ if size < 0:
+ raise ValueError("invalid blob size")
+ # Consume non-blob objects too, so subsequent records stay aligned.
+ sample = _read_git_blob_sample(output, size)
+ samples.append(sample if fields[1] == b"blob" else None)
+ return samples
def _read_git_blob_sample(output: BinaryIO, size: int) -> tuple[bytes, bool]:
@@ -174,7 +167,7 @@ def _read_git_blob_sample(output: BinaryIO, size: int) -> tuple[bytes, bool]:
if not binary:
binary = is_binary_sample(bom + chunk) if bom else b"\0" in chunk
remaining -= chunk_size
- if output.read(1) != b"\0":
+ if output.read(1) != b"\n":
raise ValueError("missing blob terminator")
return (b"", True) if binary else (sample, False)
@@ -255,6 +248,7 @@ def git_command(
git_dir: Path | None = None,
work_tree: Path | None = None,
stdout_file: BinaryIO | None = None,
+ pathspec_environment: dict[str, str] | None = None,
) -> subprocess.CompletedProcess[str] | subprocess.CompletedProcess[bytes]:
if (git_dir is None) != (work_tree is None):
raise ValueError("git_dir and work_tree must be provided together")
@@ -262,6 +256,8 @@ def git_command(
for name in GIT_REPOSITORY_ENVIRONMENT:
environment.pop(name, None)
environment["GIT_LITERAL_PATHSPECS"] = "1"
+ if pathspec_environment is not None:
+ environment.update(pathspec_environment)
executable = trusted_git_executable(target)
# Repository-local config is untrusted; fsmonitor may name an executable hook.
command = [
@@ -616,14 +612,22 @@ def clean_worktree_content_digest() -> str:
return f"codex-security-snapshot/v1:sha256:{digest.hexdigest()}"
-def _has_linked_parent(scope: Path, path: Path) -> bool:
- ancestor = scope
- for component in path.relative_to(scope).parts[:-1]:
- ancestor /= component
- metadata = ancestor.lstat()
- if stat.S_ISLNK(metadata.st_mode) or getattr(metadata, "st_reparse_tag", 0) & 0x20000000:
- return True
- return False
+def _directory_link_boundary(target: Path, path: Path, linked_prefixes: dict[str, bool]) -> Path:
+ # The index can retain descendants of a directory replaced by a link.
+ # Keep the link's identity in snapshots without following its contents.
+ prefix = target
+ for component in path.relative_to(target).parts[:-1]:
+ prefix /= component
+ key = str(prefix)
+ if key not in linked_prefixes:
+ metadata = prefix.lstat()
+ linked_prefixes[key] = bool(
+ stat.S_ISLNK(metadata.st_mode)
+ or getattr(metadata, "st_reparse_tag", 0) & 0x20000000
+ )
+ if linked_prefixes[key]:
+ return prefix
+ return path
def git_directory_snapshot_paths(target: Path) -> list[Path] | None:
@@ -634,6 +638,7 @@ def git_directory_snapshot_paths(target: Path) -> list[Path] | None:
scope = repository / pathspec
scope_depth = len(Path(pathspec).parts)
matching_prefixes: dict[str, bool] = {}
+ linked_prefixes: dict[str, bool] = {}
listing_args: list[str] = []
inventory_pathspec = pathspec
if scope_depth:
@@ -653,6 +658,7 @@ def git_directory_snapshot_paths(target: Path) -> list[Path] | None:
"--cached",
"--others",
"--exclude-standard",
+ "-t",
"-z",
"--",
inventory_pathspec,
@@ -660,8 +666,10 @@ def git_directory_snapshot_paths(target: Path) -> list[Path] | None:
if listed is None:
raise SystemExit("Could not inspect files in the selected Git working tree.")
paths: list[Path] = []
- for raw_path in (raw_path for raw_path in listed.split(b"\0") if raw_path):
- relative = Path(os.fsdecode(raw_path))
+ junctions: dict[tuple[int, int], Path] = {}
+ nested_worktrees: set[tuple[int, int]] = set()
+ for entry in (entry for entry in listed.split(b"\0") if entry):
+ relative = Path(os.fsdecode(entry[2:]))
if scope_depth:
if len(relative.parts) <= scope_depth:
continue
@@ -680,13 +688,18 @@ def git_directory_snapshot_paths(target: Path) -> list[Path] | None:
continue
path = target.joinpath(*relative.parts[scope_depth:])
try:
- # The index can retain descendants of a directory replaced by a link.
- if _has_linked_parent(target, path):
- continue
+ path = _directory_link_boundary(target, path, linked_prefixes)
metadata = path.lstat()
except (FileNotFoundError, NotADirectoryError):
# The index can retain deleted paths, including directory-to-file replacements.
continue
+ if getattr(metadata, "st_reparse_tag", 0) & 0x20000000:
+ # Case aliases share the link's identity. Prefer the stable index
+ # spelling over untracked descendants added in its external target.
+ identity = metadata.st_dev, metadata.st_ino
+ if entry[:1] != b"?" or identity not in junctions:
+ junctions[identity] = path
+ continue
paths.append(path)
if not stat.S_ISDIR(metadata.st_mode):
continue
@@ -694,47 +707,134 @@ def git_directory_snapshot_paths(target: Path) -> list[Path] | None:
if nested_repository_root is not None and Path(nested_repository_root).samefile(path):
nested_paths = git_directory_snapshot_paths(path)
if nested_paths is not None:
+ nested_worktrees.add((metadata.st_dev, metadata.st_ino))
paths.extend(nested_paths)
continue
- paths.extend(
- nested_path
- for nested_path in path.rglob("*")
- if ".git" not in nested_path.relative_to(path).parts
+ paths.extend(source_directory_snapshot_paths(path))
+
+ if _WINDOWS:
+ # Git omits empty untracked directories from its file listing. Inventory
+ # junction identities independently of whether their targets contain files.
+ git_directory = git_output(target, "rev-parse", "--absolute-git-dir")
+ if git_directory is None:
+ raise SystemExit("Could not inspect the selected Git working tree.")
+ directories = git_bytes(
+ target,
+ "--no-literal-pathspecs",
+ "ls-files",
+ "--others",
+ "--directory",
+ "--exclude-standard",
+ "-z",
+ "--",
+ # Seed only eligible descendants, never the selected directory itself.
+ ":(glob)**/*/",
+ git_dir=Path(git_directory),
+ work_tree=repository,
)
+ if directories is None:
+ raise SystemExit("Could not inspect directories in the selected Git working tree.")
+ pending = [target / os.fsdecode(path) for path in directories.split(b"\0") if path]
+ while pending:
+ children: list[Path] = []
+ for path in pending:
+ try:
+ path = _directory_link_boundary(target, path, linked_prefixes)
+ metadata = path.lstat()
+ except (FileNotFoundError, NotADirectoryError):
+ continue
+ if getattr(metadata, "st_reparse_tag", 0) & 0x20000000:
+ junctions.setdefault((metadata.st_dev, metadata.st_ino), path)
+ elif stat.S_ISLNK(metadata.st_mode):
+ paths.append(path)
+ elif stat.S_ISDIR(metadata.st_mode):
+ if (metadata.st_dev, metadata.st_ino) in nested_worktrees:
+ # Its own inventory already applied its repository exclusions.
+ continue
+ try:
+ children.extend(path.iterdir())
+ except PermissionError:
+ # Match Git's treatment of unreadable descendants.
+ continue
+ candidates: dict[bytes, Path] = {}
+ for path in children:
+ if os.path.normcase(path.name) == ".git":
+ continue
+ try:
+ metadata = path.lstat()
+ except (FileNotFoundError, NotADirectoryError, PermissionError):
+ continue
+ if not (stat.S_ISDIR(metadata.st_mode) or stat.S_ISLNK(metadata.st_mode)):
+ continue
+ relative = (Path(pathspec) / path.relative_to(target)).as_posix()
+ candidates[os.fsencode(relative)] = path
+ if not candidates:
+ break
+ # Filter a directory level at once, retaining the outer repository context.
+ ignored = git_command(
+ repository,
+ "--no-literal-pathspecs",
+ "check-ignore",
+ "-z",
+ "--stdin",
+ text=False,
+ input_data=b"\0".join(candidates) + b"\0",
+ git_dir=Path(git_directory),
+ work_tree=repository,
+ pathspec_environment={
+ "GIT_NOGLOB_PATHSPECS": "0",
+ },
+ )
+ if ignored.returncode not in (0, 1):
+ raise SystemExit("Could not inspect directories in the selected Git working tree.")
+ excluded = set(ignored.stdout.split(b"\0"))
+ pending = [path for raw, path in candidates.items() if raw not in excluded]
+ paths.extend(junctions.values())
return sorted({str(path): path for path in paths}.values(), key=str)
-def source_directory_snapshot_paths(target: Path, excluded: tuple[Path, ...] = ()) -> list[Path]:
+def source_directory_snapshot_paths(
+ target: Path,
+ excluded: tuple[Path, ...] = (),
+ *,
+ onerror: Callable[[OSError], None] | None = None,
+) -> list[Path]:
paths: list[Path] = []
- pending = [target]
- while pending:
- for path in pending.pop().iterdir():
- if path.name == ".git" or path in excluded:
+ for directory, directories, files in os.walk(
+ target, topdown=True, followlinks=False, onerror=onerror
+ ):
+ parent = Path(directory)
+ for name in directories[:]:
+ path = parent / name
+ if name == ".git" or path in excluded:
+ directories.remove(name)
continue
paths.append(path)
- metadata = path.lstat()
# Name-surrogate reparse points include Windows directory junctions.
- if (
- stat.S_ISDIR(metadata.st_mode)
- and not getattr(metadata, "st_reparse_tag", 0) & 0x20000000
- ):
- pending.append(path)
+ if getattr(path.lstat(), "st_reparse_tag", 0) & 0x20000000:
+ directories.remove(name)
+ paths.extend(
+ parent / name for name in files if name != ".git" and parent / name not in excluded
+ )
return sorted(paths)
def directory_content_digest(
target: Path, *, excluded: tuple[Path, ...] = (), include_ignored: bool = False
) -> str:
+ def raise_walk_error(error: OSError) -> None:
+ raise error
+
excluded_relative = [
path.relative_to(target) for path in excluded if path.is_relative_to(target)
]
paths = (
- source_directory_snapshot_paths(target, excluded)
+ source_directory_snapshot_paths(target, excluded, onerror=raise_walk_error)
if include_ignored
else git_directory_snapshot_paths(target)
)
if paths is None:
- paths = sorted(target.rglob("*"))
+ paths = source_directory_snapshot_paths(target)
digest = hashlib.sha256()
update_digest_field(digest, b"format", b"codex-security-directory/v1")
for path in paths:
@@ -745,17 +845,21 @@ def directory_content_digest(
metadata = path.lstat()
except OSError as exc:
raise SystemExit(f"Could not read local file: {relative_path}") from exc
+ mode = metadata.st_mode
+ link_target = (
+ os.readlink(path)
+ if stat.S_ISLNK(mode) or getattr(metadata, "st_reparse_tag", 0) & 0x20000000
+ else None
+ )
raw_path = os.fsencode(relative_path.as_posix())
update_digest_field(digest, b"path", raw_path)
- update_digest_field(digest, b"mode", str(stat.S_IMODE(metadata.st_mode)).encode())
- if stat.S_ISLNK(metadata.st_mode) or (
- include_ignored and getattr(metadata, "st_reparse_tag", 0) & 0x20000000
- ):
+ update_digest_field(digest, b"mode", str(stat.S_IMODE(mode)).encode())
+ if link_target is not None:
update_digest_field(digest, b"kind", b"symlink")
- update_digest_field(digest, b"content", os.fsencode(os.readlink(path)))
- elif stat.S_ISDIR(metadata.st_mode):
+ update_digest_field(digest, b"content", os.fsencode(link_target))
+ elif stat.S_ISDIR(mode):
update_digest_field(digest, b"kind", b"directory")
- elif stat.S_ISREG(metadata.st_mode):
+ elif stat.S_ISREG(mode):
content_digest = hashlib.sha256()
content_size = 0
try:
@@ -776,7 +880,7 @@ def directory_content_digest(
def directory_snapshot_regular_file_count(target: Path) -> int:
paths = git_directory_snapshot_paths(target)
if paths is None:
- paths = sorted(target.rglob("*"))
+ paths = source_directory_snapshot_paths(target)
count = 0
for path in paths:
try:
@@ -788,23 +892,55 @@ def directory_snapshot_regular_file_count(target: Path) -> int:
return count
-def copy_directory_excluding(source: Path, destination: Path, excluded: tuple[Path, ...]) -> None:
+def copy_directory_excluding(
+ source: Path, destination: Path, excluded: tuple[Path, ...]
+) -> list[Path]:
excluded_relative = [
path.relative_to(source) for path in excluded if path.is_relative_to(source)
]
+ junctions: list[Path] = []
+
def ignored(directory: str, names: list[str]) -> list[str]:
relative = Path(directory).relative_to(source)
- return [
+ skipped = [
path.name
for path in excluded_relative
if path.parent == relative and path.name in names
]
+ for name in names:
+ if name in skipped:
+ continue
+ path = Path(directory) / name
+ metadata = path.lstat()
+ if (
+ stat.S_ISDIR(metadata.st_mode)
+ and getattr(metadata, "st_reparse_tag", 0) & 0x20000000
+ ):
+ junctions.append(relative / name)
+ skipped.append(name)
+ return skipped
shutil.copytree(source, destination, symlinks=True, ignore=ignored)
+ # Empty placeholders retain parent directories while a patch is reversed.
+ for path in junctions:
+ (destination / path).mkdir()
+ return junctions
+
+
+def restore_directory_junctions(source: Path, destination: Path, junctions: list[Path]) -> None:
+ # Recreate links only after patch application can no longer write through them.
+ destination = destination.resolve()
+ for path in junctions:
+ placeholder = require_remediation_target(str(destination / path))
+ os.chmod(placeholder, stat.S_IMODE((source / path).lstat().st_mode))
+ copy_directory_junction(source / path, placeholder)
-def copy_git_worktree_files(source: Path, destination: Path, excluded: tuple[Path, ...]) -> Path:
+
+def copy_git_worktree_files(
+ source: Path, destination: Path, excluded: tuple[Path, ...]
+) -> tuple[Path, list[Path]]:
repository, pathspec = git_worktree_context(source)
listed = git_bytes(
repository,
@@ -829,15 +965,28 @@ def copy_git_worktree_files(source: Path, destination: Path, excluded: tuple[Pat
continue
excluded_relative.append(relative_path)
destination.mkdir()
+ junctions: set[Path] = set()
+ nested_worktrees: list[Path] = []
+ linked_prefixes: dict[str, bool] = {}
+ copied: set[Path] = set()
for raw_path in sorted(path for path in listed.split(b"\0") if path):
relative = Path(os.fsdecode(raw_path))
if any(relative.is_relative_to(path) for path in excluded_relative):
continue
source_path = repository / relative
try:
+ if _WINDOWS:
+ source_path = _directory_link_boundary(repository, source_path, linked_prefixes)
+ relative = source_path.relative_to(repository)
metadata = source_path.lstat()
except FileNotFoundError:
continue
+ if relative in copied:
+ continue
+ copied.add(relative)
+ if stat.S_ISDIR(metadata.st_mode) and getattr(metadata, "st_reparse_tag", 0) & 0x20000000:
+ junctions.add(relative)
+ continue
destination_path = destination / relative
destination_path.parent.mkdir(parents=True, exist_ok=True)
if stat.S_ISLNK(metadata.st_mode):
@@ -848,13 +997,30 @@ def copy_git_worktree_files(source: Path, destination: Path, excluded: tuple[Pat
nested_git_dir = git_output(source_path, "rev-parse", "--absolute-git-dir")
if nested_git_dir is None:
raise SystemExit(f"Could not inspect nested Git working tree: {relative}")
- copy_git_worktree_files(source_path, destination_path, excluded)
+ _, nested_junctions = copy_git_worktree_files(source_path, destination_path, excluded)
+ junctions.update(relative / path for path in nested_junctions)
+ nested_worktrees.append(relative)
(destination_path / ".git").write_text(f"gitdir: {nested_git_dir}\n", encoding="utf-8")
else:
raise SystemExit(f"Unsupported Git working-tree file type: {relative}")
+ if _WINDOWS:
+ for path in git_directory_snapshot_paths(source) or []:
+ relative = Path(pathspec) / path.relative_to(source)
+ if any(
+ relative.is_relative_to(root) for root in [*excluded_relative, *nested_worktrees]
+ ):
+ continue
+ metadata = path.lstat()
+ if (
+ stat.S_ISDIR(metadata.st_mode)
+ and getattr(metadata, "st_reparse_tag", 0) & 0x20000000
+ ):
+ junctions.add(relative)
+ for path in junctions:
+ (destination / path).mkdir(parents=True, exist_ok=True)
copied_target = destination if pathspec == "." else destination / pathspec
copied_target.mkdir(parents=True, exist_ok=True)
- return copied_target
+ return copied_target, sorted(junctions)
def git_revision(target: Path) -> str:
@@ -887,7 +1053,7 @@ def git_target_metadata(target: Path) -> dict[str, Any]:
subject = git_bytes(target, "show", "-s", "--format=%s", "HEAD")
metadata.update(
{
- "commitSubject": (subject or b"").decode("utf-8").strip() or None,
+ "commitSubject": (subject or b"").decode("utf-8", errors="replace").strip() or None,
"revision": revision,
"shortRevision": revision[:7],
}
@@ -976,9 +1142,9 @@ def scan_target_warning(scan: sqlite3.Row) -> str | None:
"Working-tree contents changed while the scan was running; "
"results were saved for the original snapshot."
)
- except (OSError, SystemExit):
+ except (OSError, SystemExit) as error:
return (
- "The scan target became unavailable while the scan was running; "
+ f"Could not verify the scan target after scanning: {error}; "
"results were saved for the original revision or snapshot."
)
return None
diff --git a/plugins/codex-security/tests/test_generate_in_scope_files.py b/plugins/codex-security/tests/test_generate_in_scope_files.py
index 7c2637467..097909ea8 100644
--- a/plugins/codex-security/tests/test_generate_in_scope_files.py
+++ b/plugins/codex-security/tests/test_generate_in_scope_files.py
@@ -1,6 +1,9 @@
from __future__ import annotations
+import errno
+import json
import os
+import runpy
import subprocess
import sys
from pathlib import Path, PurePosixPath
@@ -729,6 +732,67 @@ def test_diff_inventory_rejects_a_narrower_scope(tmp_path: Path) -> None:
assert output.read_text(encoding="utf-8") == "previous.py\n"
+ ("mode", "undo_staged_change"),
+ [("revisions", False), ("local-patch", False), ("local-patch", True)],
+)
+def test_diff_inventory_includes_type_changed_files(
+ tmp_path: Path, mode: str, undo_staged_change: bool
+) -> None:
+ repository = make_repository(tmp_path)
+ source = repository / "app" / "changed.py"
+ try:
+ source.symlink_to("routes.py")
+ except OSError:
+ pytest.skip("symlinks are unavailable")
+ git(repository, "add", ".")
+ git(repository, "commit", "-qm", "base")
+ source.unlink()
+ source.write_text("changed = True\n")
+ if undo_staged_change:
+ git(repository, "add", ".")
+ source.unlink()
+ source.symlink_to("routes.py")
+ arguments = ["--diff-base", "HEAD", "--diff-mode", mode]
+ if mode == "revisions":
+ base = git(repository, "rev-parse", "HEAD")
+ git(repository, "add", ".")
+ git(repository, "commit", "-qm", "replace symlink")
+ arguments = ["--diff-base", base, "--diff-head", "HEAD", "--diff-mode", mode]
+ output = tmp_path / "in_scope_files.txt"
+ result = run_inventory(repository, ".", output, arguments=arguments)
+ assert result.returncode == 0, result.stderr
+ assert (b"app/changed.py\n" in output.read_bytes()) is not undo_staged_change
+ rank_output = tmp_path / "rank.jsonl"
+ rank = subprocess.run(
+ [
+ sys.executable,
+ str(SCRIPT.with_name("generate_rank_input.py")),
+ "make-diff-rank-input",
+ "--repo",
+ str(repository),
+ "--base",
+ arguments[1],
+ "--head",
+ "HEAD",
+ "--mode",
+ mode,
+ "--out",
+ str(rank_output),
+ ],
+ capture_output=True,
+ text=True,
+ )
+ assert rank.returncode == 0, rank.stderr
+ assert (
+ any(
+ json.loads(line)["path"] == "app/changed.py"
+ for line in rank_output.read_text().splitlines()
+ )
+ is not undo_staged_change
+ )
+
+
@pytest.mark.skipif(os.name == "nt", reason="Windows paths cannot retain arbitrary non-UTF-8 bytes")
def test_diff_inventory_rejects_non_utf8_path_and_preserves_output(tmp_path: Path) -> None:
repository = make_repository(tmp_path)
@@ -753,3 +817,28 @@ def test_diff_inventory_rejects_non_utf8_path_and_preserves_output(tmp_path: Pat
assert result.returncode == 2
assert "cannot be encoded as UTF-8 for the file inventory" in result.stderr
assert output.read_text(encoding="utf-8") == "previous.py\n"
+
+
+def test_local_diff_inventory_skips_unreadable_files(
+ tmp_path: Path, monkeypatch: pytest.MonkeyPatch
+) -> None:
+ inventory = runpy.run_path(str(SCRIPT))
+ repository = make_repository(tmp_path)
+ git(repository, "add", ".")
+ git(repository, "commit", "-qm", "base")
+ unreadable = write_file(repository, "app/unreadable.py")
+ write_file(repository, "app/readable.py")
+ output = tmp_path / "in_scope_files.txt"
+ original_open = Path.open
+
+ def open_file(path, *args, **kwargs):
+ if path == unreadable:
+ raise PermissionError(errno.EACCES, "Synthetic unreadable file", str(path))
+ return original_open(path, *args, **kwargs)
+
+ monkeypatch.setattr(Path, "open", open_file)
+ count = inventory["generate_diff_in_scope_files"](
+ repository, "HEAD", "HEAD", "local-patch", output
+ )
+ assert count == 1
+ assert output.read_text(encoding="utf-8") == "app/readable.py\n"
diff --git a/plugins/codex-security/tests/test_git_inventory_regressions.py b/plugins/codex-security/tests/test_git_inventory_regressions.py
new file mode 100644
index 000000000..b86ed4089
--- /dev/null
+++ b/plugins/codex-security/tests/test_git_inventory_regressions.py
@@ -0,0 +1,246 @@
+from __future__ import annotations
+
+import argparse
+import hashlib
+import json
+import os
+import sqlite3
+import subprocess
+from contextlib import closing
+from pathlib import Path
+
+import pytest
+from workbench_test_support import initialize_git_repository, load_script
+
+
+def git(target: Path, *args: str, input: bytes | None = None) -> bytes:
+ return subprocess.run(
+ ["git", "-C", str(target), *args], input=input, capture_output=True, check=True
+ ).stdout.strip()
+
+
+def test_root_commit_message_does_not_supply_diff_parent(tmp_path: Path, workbench_api) -> None:
+ target = tmp_path / "target"
+ initialize_git_repository(target)
+ previous = git(target, "rev-parse", "HEAD").decode()
+ git(target, "commit", "--amend", "-qm", f"Synthetic root\n\nparent {previous}")
+ head = git(target, "rev-parse", "HEAD").decode()
+ diff = workbench_api["require_diff_target"](target, "commit", None, head, None)
+ assert diff["baseRevision"] == workbench_api["EMPTY_GIT_TREE"]
+
+
+def test_blob_batch_preserves_following_blobs_after_tree_and_commit(tmp_path: Path) -> None:
+ target = tmp_path / "target"
+ initialize_git_repository(target)
+ source = target / ("fixture.py" if os.name == "nt" else "line\nbreak.py")
+ source.write_bytes(b"print('fixture')\npayload\n")
+ git(target, "add", "--", source.name)
+ git(target, "commit", "-qm", "Add unusual fixture")
+ reader = load_script("workbench_target").git_blob_samples
+ assert reader(
+ target,
+ ["HEAD^{tree}", "HEAD", f"HEAD:{source.name}", "HEAD:missing\nfile", "HEAD:README.md"],
+ ) == [
+ None,
+ None,
+ (source.read_bytes(), False),
+ None,
+ (b"fixture\n", False),
+ ]
+
+
+def test_plain_directory_inventory_ignores_nested_git_metadata(tmp_path: Path) -> None:
+ target = tmp_path / "sources"
+ target.mkdir()
+ (target / "app.py").write_text("print('fixture')\n")
+ nested = target / "nested"
+ initialize_git_repository(nested)
+ api = load_script("workbench_target")
+ before = api.directory_content_digest(target)
+ assert api.directory_snapshot_regular_file_count(target) == 2
+ (nested / ".git" / "runtime-cache").write_text("bookkeeping\n")
+ assert api.directory_content_digest(target) == before
+ (nested / "README.md").write_text("changed source\n")
+ assert api.directory_content_digest(target) != before
+
+
+def test_git_directory_inventory_keeps_nested_bare_repository_contents(
+ tmp_path: Path, monkeypatch: pytest.MonkeyPatch
+) -> None:
+ target = tmp_path / "target"
+ initialize_git_repository(target)
+ nested = target / "nested.git"
+ git(target, "init", "--bare", "--quiet", str(nested))
+ (target / ".gitignore").write_text("nested.git/ignored-cache/\n")
+ ignored = nested / "ignored-cache"
+ ignored.mkdir()
+ (ignored / "output.txt").write_text("ignored fixture\n")
+ api = load_script("workbench_target")
+ monkeypatch.setattr(api, "_WINDOWS", True)
+
+ paths = api.git_directory_snapshot_paths(target)
+
+ assert nested / "HEAD" in paths
+ assert nested / "config" in paths
+ assert all(ignored not in path.parents for path in paths)
+ before = api.directory_content_digest(target)
+ (nested / "description").write_text("changed bare repository fixture\n")
+ assert api.directory_content_digest(target) != before
+
+
+def test_excerpt_uses_target_relative_committed_path(tmp_path: Path) -> None:
+ target = tmp_path / "target"
+ initialize_git_repository(target)
+ (target / "app.py").write_text("root version\n")
+ nested = target / "package"
+ nested.mkdir()
+ (nested / "app.py").write_text("package version\n")
+ git(target, "add", ".")
+ git(target, "commit", "-qm", "Add package")
+ scan = {
+ "target_revision": git(target, "rev-parse", "HEAD").decode(),
+ "target_snapshot_digest": None,
+ "diff_target_kind": "commit",
+ }
+ assert (
+ load_script("workbench_source_excerpt").scanned_source_text(scan, nested, "app.py")
+ == "package version\n"
+ )
+
+
+def test_working_tree_excerpt_declines_uncommitted_line_content(tmp_path: Path) -> None:
+ target = tmp_path / "target"
+ initialize_git_repository(target)
+ revision = git(target, "rev-parse", "HEAD").decode()
+ (target / "README.md").write_text("uncommitted finding line\n")
+ scan = {
+ "target_revision": revision,
+ "target_snapshot_digest": None,
+ "diff_target_kind": "working_tree",
+ "diff_content_digest": load_script("workbench_target").worktree_content_digest(target),
+ }
+ assert (
+ load_script("workbench_source_excerpt").scanned_source_text(scan, target, "README.md")
+ is None
+ )
+
+
[email protected](os.name == "nt", reason="Windows paths cannot retain arbitrary non-UTF-8 bytes")
+def test_non_utf8_git_subject_and_refs_remain_inspectable(tmp_path: Path) -> None:
+ target = tmp_path / "target"
+ initialize_git_repository(target)
+ tree = git(target, "rev-parse", "HEAD^{tree}")
+ commit = (
+ b"tree "
+ + tree
+ + b"\nauthor Fixture <[email protected]> 1700000000 +0000\ncommitter Fixture <[email protected]> 1700000000 +0000\n\ncaf\xe9\n"
+ )
+ oid = git(target, "hash-object", "-t", "commit", "-w", "--stdin", input=commit).decode()
+ git(target, "update-ref", "HEAD", oid)
+ metadata = load_script("workbench_target").git_target_metadata(target)
+ assert metadata["commitSubject"].startswith("caf")
+ api = load_script("workbench_finding_workflows")
+ connection = sqlite3.connect(":memory:")
+ try:
+ payload = {"id": "fixture", "action": "source", "repository": str(target)}
+ before = api.finding_workflow(connection, payload, "2026-01-01T00:00:00Z")["source"]
+ # Packed refs preserve raw bytes without requiring a non-UTF-8 filename.
+ (target / ".git" / "packed-refs").write_bytes(oid.encode() + b" refs/heads/caf\xe9\n")
+ after = api.finding_workflow(connection, payload, "2026-01-01T00:00:00Z")["source"]
+ assert after["refsDigest"] != before["refsDigest"]
+ finally:
+ connection.close()
+
+
+def test_dirty_submodule_warning_preserves_the_changed_target_detail(tmp_path: Path) -> None:
+ origin = tmp_path / "origin"
+ target = tmp_path / "target"
+ initialize_git_repository(origin)
+ initialize_git_repository(target)
+ git(target, "-c", "protocol.file.allow=always", "submodule", "add", str(origin), "child")
+ git(target, "commit", "-qam", "Add child")
+ api = load_script("workbench_target")
+ scan = {
+ "target_path": str(target),
+ "target_inode": target.stat().st_ino,
+ "target_revision": git(target, "rev-parse", "HEAD").decode(),
+ "target_snapshot_digest": api.worktree_content_digest(target),
+ "diff_target_kind": None,
+ "scan_dir": str(tmp_path / "artifacts"),
+ }
+ (target / "child" / "README.md").write_text("changed child source\n")
+ warning = api.scan_target_warning(scan)
+ assert "Dirty Git submodules" in warning
+ assert "child" in warning
+ assert "results were saved" in warning
+
+
+def test_disabled_git_source_snapshot_does_not_inspect_refs(
+ tmp_path: Path, monkeypatch: pytest.MonkeyPatch
+) -> None:
+ api = load_script("workbench_finding_workflows")
+ (tmp_path / "source.py").write_text("print('fixture')\n")
+
+ def unexpected_git(*args, **kwargs):
+ pytest.fail("disabled Git must not inspect revisions or refs")
+
+ monkeypatch.setattr(api, "git_revision", unexpected_git)
+ monkeypatch.setattr(api, "git_bytes", unexpected_git)
+ with closing(sqlite3.connect(":memory:")) as connection:
+ source = api.finding_workflow(
+ connection,
+ {"id": "fixture", "action": "source", "repository": str(tmp_path), "gitDisabled": True},
+ "2026-01-01T00:00:00Z",
+ )["source"]
+ assert source["revision"] == "unversioned"
+ assert source["refsDigest"] == hashlib.sha256(b"").hexdigest()
+
+
[email protected]("mode", ["revisions", "local-patch"])
[email protected]("replacement", ["symlink", "gitlink"])
+def test_diff_inventories_exclude_non_file_type_changes(
+ tmp_path: Path, mode: str, replacement: str
+) -> None:
+ target = tmp_path / "target"
+ initialize_git_repository(target)
+ source = target / "replaced.py"
+ source.write_text("old source\n")
+ git(target, "add", ".")
+ git(target, "commit", "-qm", "Add source fixture")
+ base = git(target, "rev-parse", "HEAD").decode()
+ if replacement == "symlink":
+ source.unlink()
+ try:
+ source.symlink_to("README.md")
+ except OSError:
+ pytest.skip("symlinks are unavailable")
+ else:
+ origin = tmp_path / "origin"
+ initialize_git_repository(origin)
+ git(target, "rm", "-f", source.name)
+ git(
+ target, "-c", "protocol.file.allow=always", "submodule", "add", str(origin), source.name
+ )
+ (target / "visible.py").write_text("value = 1\n")
+ git(target, "add", ".")
+ if mode == "revisions":
+ git(target, "commit", "-qm", "Replace source type")
+ inventory_path = tmp_path / "inventory.txt"
+ rank_path = tmp_path / "rank.jsonl"
+ load_script("generate_in_scope_files").generate_diff_in_scope_files(
+ target, base, "HEAD", mode, inventory_path
+ )
+ load_script("generate_rank_input").make_diff_rank_input(
+ argparse.Namespace(
+ repo=str(target),
+ base=base,
+ head="HEAD",
+ mode=mode,
+ out=str(rank_path),
+ area="fixture",
+ preview_bytes=1024,
+ )
+ )
+ expected = [".gitmodules", "visible.py"] if replacement == "gitlink" else ["visible.py"]
+ assert inventory_path.read_text().splitlines() == expected
+ assert [json.loads(line)["path"] for line in rank_path.read_text().splitlines()] == expected
diff --git a/plugins/codex-security/tests/test_windows_scan_local_files.py b/plugins/codex-security/tests/test_windows_scan_local_files.py
index 5a51c7ae1..25f4c8af3 100644
--- a/plugins/codex-security/tests/test_windows_scan_local_files.py
+++ b/plugins/codex-security/tests/test_windows_scan_local_files.py
@@ -1,6 +1,8 @@
from __future__ import annotations
import os
+import shutil
+import subprocess
from pathlib import Path
import pytest
@@ -77,3 +79,45 @@ def test_native_windows_backend_rejects_symlink_ancestor(tmp_path: Path) -> None
with pytest.raises(WINDOWS_FILES.WindowsScanLocalFileError):
WINDOWS_FILES.atomic_write(scan_dir, "exports/results.sarif", b"blocked")
assert not (external_dir / "results.sarif").exists()
+
+
[email protected](os.name != "nt", reason="requires native Win32 junction APIs")
[email protected]("dangling", [False, True], ids=["unicode", "dangling"])
+def test_native_copy_junction_retains_raw_target_and_cleanup(
+ tmp_path: Path, dangling: bool
+) -> None:
+ target = tmp_path / "Mixed-Case-日本語"
+ target.mkdir()
+ source = tmp_path / "source"
+ subprocess.run(["cmd", "/c", "mklink", "/J", str(source), str(target)], check=True)
+ if dangling:
+ target.rmdir()
+ else:
+ (target / "keep.txt").write_text("external contents\n")
+ copied_tree = tmp_path / "copy"
+ copied_tree.mkdir()
+ destination = copied_tree / "linked"
+ destination.mkdir()
+ WINDOWS_FILES.copy_directory_junction(source, destination)
+ assert os.readlink(destination) == os.readlink(source)
+ assert destination.lstat().st_reparse_tag == source.lstat().st_reparse_tag
+ shutil.rmtree(copied_tree)
+ if dangling:
+ assert not target.exists()
+ else:
+ assert (target / "keep.txt").read_text() == "external contents\n"
+
+
[email protected](os.name != "nt", reason="requires native Win32 junction APIs")
+def test_native_copy_junction_rejects_nonempty_destination(tmp_path: Path) -> None:
+ target = tmp_path / "outside"
+ target.mkdir()
+ source = tmp_path / "source"
+ subprocess.run(["cmd", "/c", "mklink", "/J", str(source), str(target)], check=True)
+ destination = tmp_path / "destination"
+ destination.mkdir()
+ (destination / "keep.txt").write_text("placeholder contents\n")
+ with pytest.raises(WINDOWS_FILES.WindowsScanLocalFileError):
+ WINDOWS_FILES.copy_directory_junction(source, destination)
+ assert (destination / "keep.txt").read_text() == "placeholder contents\n"
+ assert not (target / "keep.txt").exists()
diff --git a/plugins/codex-security/tests/test_workbench_db.py b/plugins/codex-security/tests/test_workbench_db.py
index 1ac00d1cc..bc1f47ff6 100644
--- a/plugins/codex-security/tests/test_workbench_db.py
+++ b/plugins/codex-security/tests/test_workbench_db.py
@@ -60,10 +60,6 @@ DIRECTORY_CHANGED_WARNING = (
"Directory contents changed while the scan was running; "
"results were saved for the original snapshot."
)
-TARGET_UNAVAILABLE_WARNING = (
- "The scan target became unavailable while the scan was running; "
- "results were saved for the original revision or snapshot."
-)
GIT_UNAVAILABLE_WARNING = (
"The scanned Git repository became unavailable while the scan was running; "
"results were saved for the original revision."
@@ -669,8 +665,11 @@ def test_completion_warns_when_scanned_directory_becomes_unavailable(tmp_path: P
completed = scan_command(state_dir, "complete-scan", scan_id)
assert completed["scan"]["progress"]["status"] == "complete"
- assert completed["scan"]["warnings"] == [TARGET_UNAVAILABLE_WARNING]
- assert completed["targetWarnings"] == [TARGET_UNAVAILABLE_WARNING]
+ warnings = completed["scan"]["warnings"]
+ assert len(warnings) == 1
+ assert "selected checkout is no longer accessible" in warnings[0]
+ assert "results were saved" in warnings[0]
+ assert completed["targetWarnings"] == warnings
assert completed["scan"]["findingCount"] == 1
assert completed["scan"]["remediationAvailable"] is False
assert (scan_dir / "report.md").is_file()
diff --git a/plugins/codex-security/tests/test_workbench_finding_workflows.py b/plugins/codex-security/tests/test_workbench_finding_workflows.py
index 80653e09c..251cb989a 100644
--- a/plugins/codex-security/tests/test_workbench_finding_workflows.py
+++ b/plugins/codex-security/tests/test_workbench_finding_workflows.py
@@ -1,6 +1,7 @@
from __future__ import annotations
import json
+import os
import sqlite3
from contextlib import closing
from pathlib import Path
@@ -27,13 +28,13 @@ def test_source_snapshot_excludes_private_storage_without_hiding_source(
private.mkdir()
database = private / "state.sqlite"
database.write_text("synthetic state")
- original_iterdir = Path.iterdir
+ original_scandir = os.scandir
def inspect(path):
- assert path != private, "private storage must not be traversed"
- return original_iterdir(path)
+ assert Path(path) != private, "private storage must not be traversed"
+ return original_scandir(path)
- monkeypatch.setattr(Path, "iterdir", inspect)
+ monkeypatch.setattr(os, "scandir", inspect)
def snapshot():
return workflow(
@@ -68,6 +69,47 @@ def test_source_snapshot_excludes_private_storage_without_hiding_source(
)
[email protected]("location", ["root", "descendant"])
+def test_source_snapshot_preserves_unreadable_directory_errors(
+ workbench_api, workbench_db, tmp_path, monkeypatch, location
+):
+ repository = tmp_path / "repository"
+ repository.mkdir()
+ unreadable = repository if location == "root" else repository / "unreadable"
+ unreadable.mkdir(exist_ok=True)
+ (unreadable / "contents.txt").write_text("source content")
+ private = repository / "native-state"
+ private.mkdir()
+ original_scandir = os.scandir
+ failure = PermissionError("synthetic inaccessible source directory")
+
+ def inspect(path):
+ assert Path(path) != private, "private storage must not be traversed"
+ if Path(path) == unreadable:
+ raise failure
+ return original_scandir(path)
+
+ monkeypatch.setattr(os, "scandir", inspect)
+
+ def snapshot(excluded):
+ return workflow(
+ workbench_api,
+ workbench_db,
+ "source",
+ repository=str(repository),
+ gitDisabled=True,
+ privateStatePaths=[str(path) for path in excluded],
+ )["source"]
+
+ with pytest.raises(PermissionError) as raised:
+ snapshot([private])
+ assert raised.value is failure
+ if location == "descendant":
+ assert snapshot([private, unreadable])["privateStatePaths"] == sorted(
+ [str(private), str(unreadable)]
+ )
+
+
@pytest.mark.parametrize("stage", ["scan", "publish", "dedupe"])
def test_failed_stages_resume_and_completed_results_are_immutable(
workbench_api, workbench_db, stage
diff --git a/plugins/codex-security/tests/test_workbench_target.py b/plugins/codex-security/tests/test_workbench_target.py
index a005430a6..b163f6997 100644
--- a/plugins/codex-security/tests/test_workbench_target.py
+++ b/plugins/codex-security/tests/test_workbench_target.py
@@ -1,8 +1,12 @@
from __future__ import annotations
+import difflib
+import errno
import hashlib
import os
import runpy
+import shutil
+import stat
import subprocess
from pathlib import Path
from types import SimpleNamespace
@@ -22,11 +26,645 @@ update_digest_field = cast(
)
+ params=[
+ pytest.param(False, id="emulated"),
+ pytest.param(True, id="native", marks=pytest.mark.native_windows),
+ ]
+)
+def junction_factory(request: pytest.FixtureRequest, monkeypatch: pytest.MonkeyPatch):
+ import workbench_target
+
+ native = request.param
+ if native and os.name != "nt":
+ pytest.skip("requires native Windows junctions")
+ targets: dict[Path, Path] = {}
+ modes: dict[Path, int] = {}
+ if not native:
+ monkeypatch.setattr(workbench_target, "_WINDOWS", True)
+ real_lstat = Path.lstat
+ real_readlink = os.readlink
+
+ def metadata(path: Path, *args: Any, **kwargs: Any) -> Any:
+ result = real_lstat(path, *args, **kwargs)
+ if path in targets:
+ fields = {key: getattr(result, key) for key in dir(result) if key.startswith("st_")}
+ fields["st_reparse_tag"] = 0xA0000003
+ fields["st_mode"] = modes.get(path, result.st_mode)
+ return SimpleNamespace(**fields)
+ return result
+
+ def readlink(path: Any, *args: Any, **kwargs: Any) -> Any:
+ if Path(path) in targets:
+ return str(targets[Path(path)])
+ return real_readlink(path, *args, **kwargs)
+
+ def copy_junction(source: Path, destination: Path) -> None:
+ # Emulate the filesystem result, not the native GET/SET implementation.
+ if any(destination.iterdir()):
+ raise OSError(errno.ENOTEMPTY, "junction destination is not empty")
+ mode = destination.lstat().st_mode
+ # Windows readonly directories do not impose POSIX child-write permissions.
+ os.chmod(destination, stat.S_IMODE(mode) | stat.S_IWUSR)
+ shutil.copytree(source, destination, dirs_exist_ok=True)
+ os.chmod(destination, 0o755)
+ targets[destination] = targets[source]
+ modes[destination] = mode
+
+ monkeypatch.setattr(Path, "lstat", metadata)
+ monkeypatch.setattr(os, "readlink", readlink)
+ monkeypatch.setattr(workbench_target, "copy_directory_junction", copy_junction)
+
+ def create(link: Path, target: Path) -> None:
+ if native:
+ if link.exists():
+ link.rmdir()
+ subprocess.run(["cmd", "/c", "mklink", "/J", str(link), str(target)], check=True)
+ else:
+ shutil.copytree(target, link, dirs_exist_ok=True)
+ targets[link] = target
+
+ return create
+
+
+ ("git_repository", "git_exclusion"),
+ [
+ (None, None),
+ ("untracked", None),
+ ("indexed", None),
+ ("untracked", ".gitignore"),
+ ("untracked", ".git/info/exclude"),
+ ("nested", ".git/info/exclude"),
+ ],
+ ids=[
+ "plain",
+ "git-untracked",
+ "git-indexed",
+ "gitignore",
+ "git-info-exclude",
+ "nested-info-exclude",
+ ],
+)
+ "change",
+ [
+ "unchanged",
+ "unrelated_file",
+ "junction_target",
+ "same_content_target",
+ "target_contents",
+ "target_emptied",
+ "target_populated",
+ ],
+)
+def test_reviewed_patch_preserves_junction_boundaries(
+ tmp_path: Path,
+ junction_factory: Callable[[Path, Path], None],
+ change: str,
+ git_repository: str | None,
+ git_exclusion: str | None,
+) -> None:
+ import workbench_db
+
+ source = tmp_path / "source"
+ source.mkdir()
+ if git_repository:
+ subprocess.run(["git", "init", "-q"], cwd=source, check=True)
+ junction_root = source
+ if git_repository == "nested":
+ junction_root = source / "nested"
+ initialize_git_repository(junction_root)
+ if git_exclusion is not None:
+ (junction_root / git_exclusion).write_text("dependencies/linked_directory/\n")
+ (junction_root / "dependencies").mkdir()
+ junction = junction_root / "dependencies" / "linked_directory"
+ outside = tmp_path / "outside"
+ outside.mkdir()
+ if change != "target_populated":
+ (outside / "source.txt").write_text("original contents\n")
+ junction_factory(junction, outside)
+ scan_dir = tmp_path / "scan"
+ scan_dir.mkdir(mode=0o700)
+ patch = b"diff --git a/app.txt b/app.txt\n--- a/app.txt\n+++ b/app.txt\n@@ -1 +1 @@\n-before\n+after\n"
+ (scan_dir / "reviewed.patch").write_bytes(patch)
+ scan = {
+ "target_path": str(source),
+ "target_inode": source.stat().st_ino,
+ "target_revision": "unversioned",
+ "scan_dir": str(scan_dir),
+ }
+ (source / "app.txt").write_text("before\n")
+ if git_repository == "indexed":
+ subprocess.run(["git", "add", "."], cwd=source, check=True)
+ remediation = {
+ "base_revision": "unversioned",
+ "base_content_digest": workbench_db.directory_content_digest(source),
+ "patch_digest": "sha256:" + hashlib.sha256(patch).hexdigest(),
+ }
+ (source / "app.txt").write_text("after\n")
+ if change in {"junction_target", "same_content_target"}:
+ junction_target = tmp_path / "other"
+ junction_target.mkdir()
+ (junction_target / "source.txt").write_text(
+ "original contents\n"
+ if change == "same_content_target"
+ else "different target contents\n"
+ )
+ junction_factory(junction, junction_target)
+ elif change in {"target_contents", "target_populated"}:
+ (junction / "source.txt").write_text("changed outside the snapshot boundary\n")
+ elif change == "target_emptied":
+ (junction / "source.txt").unlink()
+ elif change == "unrelated_file":
+ (source / "unrelated.txt").write_text("outside the reviewed patch\n")
+ if change == "unrelated_file" or (
+ change in {"junction_target", "same_content_target"} and git_exclusion is None
+ ):
+ with pytest.raises(SystemExit, match="changes outside the reviewed patch"):
+ workbench_db.require_reviewed_patch_applied(scan, remediation, "reviewed.patch")
+ else:
+ assert workbench_db.require_reviewed_patch_applied(scan, remediation, "reviewed.patch")
+
+
+ ("git_repository", "readonly_target"),
+ [(False, False), (True, False), (False, True)],
+ ids=["plain", "git", "readonly-empty-target"],
+)
+def test_reviewed_patch_preserves_readonly_junction(
+ tmp_path: Path,
+ junction_factory: Callable[[Path, Path], None],
+ git_repository: bool,
+ readonly_target: bool,
+) -> None:
+ source = tmp_path / "source"
+ source.mkdir()
+ if git_repository:
+ subprocess.run(["git", "init", "-q"], cwd=source, check=True)
+ outside = tmp_path / "outside"
+ outside.mkdir()
+ if readonly_target:
+ os.chmod(outside, stat.S_IREAD | stat.S_IEXEC)
+ else:
+ (outside / "file.txt").write_text("external contents\n")
+ outside_mode = outside.stat().st_mode
+ junction = source / "linked"
+ junction_factory(junction, outside)
+ os.chmod(junction, stat.S_IREAD | stat.S_IEXEC)
+ try:
+ assert not junction.lstat().st_mode & stat.S_IWRITE
+ assert outside.stat().st_mode == outside_mode
+ assert_reviewed_change(source, tmp_path, "app.txt", "before\n", "after\n")
+ assert outside.stat().st_mode == outside_mode
+ if not readonly_target:
+ assert (outside / "file.txt").read_text() == "external contents\n"
+ finally:
+ os.chmod(junction, stat.S_IREAD | stat.S_IWRITE | stat.S_IEXEC)
+
+
[email protected]("scope", [".", "component"])
[email protected]("populated", [False, True])
+def test_versioned_reviewed_patch_preserves_nested_junctions(
+ tmp_path: Path,
+ junction_factory: Callable[[Path, Path], None],
+ monkeypatch: pytest.MonkeyPatch,
+ scope: str,
+ populated: bool,
+) -> None:
+ import workbench_db
+
+ repository = tmp_path / "repository"
+ initialize_git_repository(repository)
+ target = repository / scope
+ target.mkdir(exist_ok=True)
+ (target / "app.txt").write_text("before\n")
+ subprocess.run(["git", "add", "."], cwd=repository, check=True)
+ subprocess.run(["git", "commit", "-qm", "Add target"], cwd=repository, check=True)
+ nested = target / "nested"
+ initialize_git_repository(nested)
+ outside = tmp_path / "outside"
+ outside.mkdir()
+ if populated:
+ (outside / "external.txt").write_text("outside contents\n")
+ junction = nested / "linked"
+ junction_factory(junction, outside)
+ relative_junction = junction.relative_to(repository)
+ original_command = workbench_db.git_command
+ original_restore = workbench_db.restore_directory_junctions
+ reversed_roots: list[Path] = []
+ restored: list[Path] = []
+
+ def command(path, *args, **kwargs):
+ if args[0] == "apply":
+ copied = kwargs["work_tree"]
+ placeholder = copied / relative_junction
+ assert placeholder.is_dir()
+ assert not getattr(placeholder.lstat(), "st_reparse_tag", 0)
+ assert list(placeholder.iterdir()) == []
+ result = original_command(path, *args, **kwargs)
+ if args[0] == "apply" and result.returncode == 0:
+ reversed_roots.append(copied)
+ return result
+
+ def restore(source, destination, junctions):
+ assert source == repository
+ assert reversed_roots == [destination]
+ assert relative_junction in junctions
+ original_restore(source, destination, junctions)
+ restored.append(destination)
+
+ monkeypatch.setattr(workbench_db, "git_command", command)
+ monkeypatch.setattr(workbench_db, "restore_directory_junctions", restore)
+ assert_reviewed_change(
+ target,
+ tmp_path,
+ "app.txt",
+ "before\n",
+ "after\n",
+ revision=workbench_db.git_revision(target),
+ )
+ assert len(restored) == 1
+ assert sorted(path.name for path in outside.iterdir()) == (
+ ["external.txt"] if populated else []
+ )
+ if populated:
+ assert (outside / "external.txt").read_text() == "outside contents\n"
+
+
+ "case",
+ [
+ "ordering",
+ "ignore_add",
+ "ignore_remove",
+ "git_metadata",
+ "git_metadata_crlf",
+ "git_metadata_lf",
+ "git_objects",
+ ],
+)
+def test_reviewed_patch_restores_junction_git_context(
+ tmp_path: Path,
+ junction_factory: Callable[[Path, Path], None],
+ case: str,
+ monkeypatch: pytest.MonkeyPatch,
+) -> None:
+ source = tmp_path / "source"
+ initialize_unborn_git_repository(source)
+ newline = None
+ if case in {"git_metadata_crlf", "git_metadata_lf"}:
+ global_config = tmp_path / "global.gitconfig"
+ global_config.write_text("[core]\n\tautocrlf = false\n")
+ monkeypatch.setenv("GIT_CONFIG_GLOBAL", str(global_config))
+ monkeypatch.setenv("GIT_CONFIG_NOSYSTEM", "1")
+ subprocess.run(["git", "config", "core.autocrlf", "true"], cwd=source, check=True)
+ newline = "\r\n" if case == "git_metadata_crlf" else "\n"
+ (source / "src").mkdir()
+ (source / "src" / "app.txt").write_text("stable contents\n")
+ (source / "src.py").write_text("adjacent contents\n")
+ (source / "Alpha.txt").write_text("mixed case ordering\n")
+ (source / ".gitignore").write_text("cache/\n")
+ (source / "cache").mkdir()
+ (source / "cache" / "output.txt").write_text("ignored output\n")
+ outside = tmp_path / "outside"
+ if case.startswith("git_metadata") or case == "git_objects":
+ junction = source / (".git" if case.startswith("git_metadata") else ".git/objects")
+ junction.rename(outside)
+ else:
+ outside.mkdir()
+ (outside / "file.txt").write_text("external contents\n")
+ junction = source / "linked"
+ junction_factory(junction, outside)
+ patched = ".gitignore" if case.startswith("ignore_") else "app.txt"
+ before = (
+ "# fixture\nlinked/\n"
+ if case == "ignore_remove"
+ else "# fixture\n"
+ if case == "ignore_add"
+ else "before\n"
+ )
+ after = (
+ "# fixture\nlinked/\n"
+ if case == "ignore_add"
+ else "# fixture\n"
+ if case == "ignore_remove"
+ else "after\n"
+ )
+ metadata = (outside / "config").read_bytes() if case.startswith("git_metadata") else None
+ assert_reviewed_change(source, tmp_path, patched, before, after, newline=newline)
+ assert outside.is_dir()
+ if metadata is not None:
+ assert (outside / "config").read_bytes() == metadata
+ if newline is not None:
+ assert (source / patched).read_bytes() == after.replace("\n", newline).encode()
+
+
+def test_reviewed_patch_preserves_scoped_unborn_lf_bytes(tmp_path: Path) -> None:
+ repository = tmp_path / "repository"
+ initialize_unborn_git_repository(repository)
+ attributes = tmp_path / "attributes"
+ attributes.write_text("*.txt text eol=crlf\n")
+ subprocess.run(
+ ["git", "config", "core.attributesFile", str(attributes)], cwd=repository, check=True
+ )
+ source = repository / "component"
+ source.mkdir()
+
+ assert_reviewed_change(source, tmp_path, "app.txt", "before\n", "after\n", newline="\n")
+
+ assert (source / "app.txt").read_bytes() == b"after\n"
+
+
+def assert_reviewed_change(
+ source: Path,
+ tmp_path: Path,
+ relative: str,
+ before: str,
+ after: str,
+ *,
+ newline: str | None = None,
+ revision: str = "unversioned",
+) -> None:
+ import workbench_db
+
+ patch = (
+ f"diff --git a/{relative} b/{relative}\n"
+ + "".join(
+ difflib.unified_diff(
+ before.splitlines(True),
+ after.splitlines(True),
+ fromfile=f"a/{relative}",
+ tofile=f"b/{relative}",
+ )
+ )
+ ).encode()
+ scan_dir = tmp_path / "scan"
+ scan_dir.mkdir(mode=0o700)
+ (scan_dir / "reviewed.patch").write_bytes(patch)
+ scan = {
+ "target_path": str(source),
+ "target_inode": source.stat().st_ino,
+ "target_revision": revision,
+ "scan_dir": str(scan_dir),
+ }
+ (source / relative).write_text(before, newline=newline)
+ revision, digest = workbench_db.remediation_checkout_snapshot(scan)
+ remediation = {
+ "base_revision": revision,
+ "base_content_digest": digest,
+ "patch_digest": "sha256:" + hashlib.sha256(patch).hexdigest(),
+ }
+ (source / relative).write_text(after, newline=newline)
+ assert workbench_db.require_reviewed_patch_applied(scan, remediation, "reviewed.patch")
+
+
[email protected](os.name != "nt", reason="requires native Windows path casing")
+def test_native_reviewed_patch_preserves_indexed_junction_spelling(tmp_path: Path) -> None:
+ source = tmp_path / "source"
+ initialize_unborn_git_repository(source)
+ subprocess.run(["git", "config", "core.ignorecase", "true"], cwd=source, check=True)
+ outside = tmp_path / "outside"
+ outside.mkdir()
+ (outside / "file.txt").write_text("external contents\n")
+ junction = source / "linked"
+ subprocess.run(["cmd", "/c", "mklink", "/J", str(junction), str(outside)], check=True)
+ (source / "app.txt").write_text("before\n")
+ subprocess.run(["git", "add", "."], cwd=source, check=True)
+ junction.rename(source / "Linked")
+ assert_reviewed_change(source, tmp_path, "app.txt", "before\n", "after\n")
+
+
+def test_indexed_junction_case_aliases_keep_snapshot_identity(
+ tmp_path: Path,
+ junction_factory: Callable[[Path, Path], None],
+ monkeypatch: pytest.MonkeyPatch,
+) -> None:
+ import workbench_target
+
+ source = tmp_path / "source"
+ initialize_unborn_git_repository(source)
+ subprocess.run(["git", "config", "core.ignorecase", "true"], cwd=source, check=True)
+ outside = tmp_path / "outside"
+ outside.mkdir()
+ (outside / "file.txt").write_text("external contents\n")
+ junction = source / "linked"
+ junction_factory(junction, outside)
+ subprocess.run(["git", "add", "."], cwd=source, check=True)
+ metadata = junction.lstat()
+ renamed = junction.rename(source / "Linked")
+ if not getattr(renamed.lstat(), "st_reparse_tag", 0):
+ # Supply case aliases for the emulated junction on a case-sensitive volume.
+ real_lstat = Path.lstat
+ real_readlink = os.readlink
+ monkeypatch.setattr(
+ Path,
+ "lstat",
+ lambda path: metadata if path in (junction, renamed) else real_lstat(path),
+ )
+ monkeypatch.setattr(
+ os,
+ "readlink",
+ lambda path: str(outside) if Path(path) in (junction, renamed) else real_readlink(path),
+ )
+ before = workbench_target.directory_content_digest(source)
+
+ (renamed / "new.txt").write_text("new external contents\n")
+ assert workbench_target.directory_content_digest(source) == before
+ (renamed / "file.txt").unlink()
+ (renamed / "new.txt").unlink()
+ assert workbench_target.directory_content_digest(source) == before
+
+
+def test_reverse_patch_cannot_write_through_junction(
+ tmp_path: Path, junction_factory: Callable[[Path, Path], None]
+) -> None:
+ import workbench_db
+
+ source = tmp_path / "source"
+ source.mkdir()
+ outside = tmp_path / "outside"
+ outside.mkdir()
+ (outside / "keep.txt").write_text("outside contents\n")
+ junction = source / "linked"
+ junction_factory(junction, outside)
+ scan_dir = tmp_path / "scan"
+ scan_dir.mkdir(mode=0o700)
+ patch = b"diff --git a/linked/deleted.txt b/linked/deleted.txt\ndeleted file mode 100644\n--- a/linked/deleted.txt\n+++ /dev/null\n@@ -1 +0,0 @@\n-removed\n"
+ (scan_dir / "reviewed.patch").write_bytes(patch)
+ scan = {
+ "target_path": str(source),
+ "target_inode": source.stat().st_ino,
+ "target_revision": "unversioned",
+ "scan_dir": str(scan_dir),
+ }
+ remediation = {
+ "base_revision": "unversioned",
+ "base_content_digest": "different base",
+ "patch_digest": "sha256:" + hashlib.sha256(patch).hexdigest(),
+ }
+ with pytest.raises(OSError):
+ workbench_db.require_reviewed_patch_applied(scan, remediation, "reviewed.patch")
+ assert sorted(path.name for path in outside.iterdir()) == ["keep.txt"]
+ assert (outside / "keep.txt").read_text() == "outside contents\n"
+
+
+def test_reverse_patch_cannot_redirect_junction_restoration(
+ tmp_path: Path,
+ junction_factory: Callable[[Path, Path], None],
+ monkeypatch: pytest.MonkeyPatch,
+) -> None:
+ import workbench_db
+
+ source = tmp_path / "source"
+ (source / "dep").mkdir(parents=True)
+ original_target = tmp_path / "original"
+ original_target.mkdir()
+ (original_target / "keep.txt").write_text("original contents\n")
+ junction = source / "dep" / "linked"
+ junction_factory(junction, original_target)
+ os.chmod(junction, stat.S_IREAD | stat.S_IEXEC)
+ outside = tmp_path / "outside"
+ external = outside / "linked"
+ external.mkdir(parents=True)
+ original_mode = external.stat().st_mode
+ config = tmp_path / "gitconfig"
+ config.write_text("[core]\n\tsymlinks = true\n")
+ monkeypatch.setenv("GIT_CONFIG_GLOBAL", str(config))
+ scan_dir = tmp_path / "scan"
+ scan_dir.mkdir(mode=0o700)
+ patch = f"""diff --git a/app.txt b/app.txt
+--- a/app.txt
++++ b/app.txt
+@@ -1 +1 @@
+-before
++after
+diff --git a/dep b/dep
+deleted file mode 120000
+--- a/dep
++++ /dev/null
+@@ -1 +0,0 @@
+-{outside.as_posix()}
+\\ No newline at end of file
+diff --git a/dep/linked b/dep/linked
+new file mode 160000
+index {"0" * 40}..{"1" * 40}
+--- /dev/null
++++ b/dep/linked
+@@ -0,0 +1 @@
++Subproject commit {"1" * 40}
+""".encode()
+ (scan_dir / "reviewed.patch").write_bytes(patch)
+ scan = {
+ "target_path": str(source),
+ "target_inode": source.stat().st_ino,
+ "target_revision": "unversioned",
+ "scan_dir": str(scan_dir),
+ }
+ (source / "app.txt").write_text("before\n")
+ remediation = {
+ "base_revision": "unversioned",
+ "base_content_digest": workbench_db.directory_content_digest(source),
+ "patch_digest": "sha256:" + hashlib.sha256(patch).hexdigest(),
+ }
+ (source / "app.txt").write_text("after\n")
+ try:
+ with pytest.raises(SystemExit, match="checkout path was replaced"):
+ workbench_db.require_reviewed_patch_applied(scan, remediation, "reviewed.patch")
+ assert external.lstat().st_mode == original_mode
+ assert not getattr(external.lstat(), "st_reparse_tag", 0)
+ assert list(external.iterdir()) == []
+ assert (original_target / "keep.txt").read_text() == "original contents\n"
+ finally:
+ os.chmod(junction, stat.S_IREAD | stat.S_IWRITE | stat.S_IEXEC)
+
+
def initialize_unborn_git_repository(target: Path) -> None:
target.mkdir()
subprocess.run(["git", "init", "-q"], cwd=target, check=True)
+def test_windows_inventory_batches_wide_untracked_trees(
+ tmp_path: Path, monkeypatch: pytest.MonkeyPatch
+) -> None:
+ import workbench_target
+
+ target = tmp_path / "target"
+ initialize_unborn_git_repository(target)
+ monkeypatch.setattr(workbench_target, "_WINDOWS", True)
+ run_git = workbench_target.git_command
+ calls = 0
+
+ def counted_git(*args: Any, **kwargs: Any):
+ nonlocal calls
+ calls += 1
+ return run_git(*args, **kwargs)
+
+ monkeypatch.setattr(workbench_target, "git_command", counted_git)
+ sources = []
+ for index in range(16):
+ source = target / f"component-{index}" / "src" / "app.py"
+ source.parent.mkdir(parents=True)
+ source.write_text("print('fixture')\n")
+ sources.append(source)
+ if index == 0:
+ assert workbench_target.git_directory_snapshot_paths(target) == sources
+ single_directory_calls = calls
+ calls = 0
+
+ assert set(workbench_target.git_directory_snapshot_paths(target)) == set(sources)
+ assert calls == single_directory_calls
+
+
[email protected]("change", ["unchanged", "ignored_file", "unrelated_file"])
+def test_reviewed_patch_preserves_unborn_git_inventory(tmp_path: Path, change: str) -> None:
+ import workbench_db
+
+ source = tmp_path / "source"
+ initialize_unborn_git_repository(source)
+ (source / "src").mkdir()
+ app = source / "src" / "app.txt"
+ app.write_text("before\n")
+ (source / ".gitignore").write_text("ignored-cache/\n")
+ cache = source / "ignored-cache"
+ cache.mkdir()
+ cached_file = cache / "output.txt"
+ cached_file.write_text("ignored build output\n")
+ subprocess.run(["git", "add", ".gitignore", "src/app.txt"], cwd=source, check=True)
+ scan_dir = tmp_path / "scan"
+ scan_dir.mkdir(mode=0o700)
+ patch = (
+ b"diff --git a/src/app.txt b/src/app.txt\n"
+ b"--- a/src/app.txt\n+++ b/src/app.txt\n@@ -1 +1 @@\n-before\n+after\n"
+ )
+ (scan_dir / "reviewed.patch").write_bytes(patch)
+ scan = {
+ "target_path": str(source),
+ "target_inode": source.stat().st_ino,
+ "target_revision": "unversioned",
+ "scan_dir": str(scan_dir),
+ }
+ revision, digest = workbench_db.remediation_checkout_snapshot(scan)
+ assert revision == "unversioned"
+ remediation = {
+ "base_revision": revision,
+ "base_content_digest": digest,
+ "patch_digest": "sha256:" + hashlib.sha256(patch).hexdigest(),
+ }
+ app.write_text("after\n")
+ if change == "ignored_file":
+ cached_file.write_text("updated ignored build output\n")
+ elif change == "unrelated_file":
+ (source / "unrelated.txt").write_text("outside the reviewed patch\n")
+ if change == "unrelated_file":
+ with pytest.raises(SystemExit, match="changes outside the reviewed patch"):
+ workbench_db.require_reviewed_patch_applied(scan, remediation, "reviewed.patch")
+ else:
+ assert workbench_db.require_reviewed_patch_applied(scan, remediation, "reviewed.patch")
+
+
def add_submodule_gitlink(repository: Path, revision: str, scope: str) -> None:
subprocess.run(
[
@@ -93,7 +731,7 @@ def test_git_blob_samples_match_full_file_classification_with_bounded_reads(
read_batch_samples = function_globals["_read_git_batch_samples"]
reads: list[int] = []
- def require_bounded_output(output: Any, count: int):
+ def require_bounded_output(output: Any, names: list[str]):
assert not output.seekable()
read = output.read
@@ -103,7 +741,7 @@ def test_git_blob_samples_match_full_file_classification_with_bounded_reads(
return read(size)
output.read = bounded_read
- return read_batch_samples(output, count)
+ return read_batch_samples(output, names)
monkeypatch.setitem(function_globals, "_read_git_batch_samples", require_bounded_output)
# Both request and response exceed pipe buffers, before the multi-megabyte binary.
@@ -129,12 +767,12 @@ def test_git_blob_samples_match_full_file_classification_with_bounded_reads(
"output",
[
b"missing terminator",
- b"object blob invalid\0",
- b"object blob -1\0",
- b"object blob 4\0abc",
- b"object blob 4\0abcd!",
- b"object blob invalid\0" + b"x" * (4 * 1024 * 1024),
- b"object blob 131072\0" + b"x" * (64 * 1024 + 1),
+ b"object blob invalid\n",
+ b"object blob -1\n",
+ b"object blob 4\nabc",
+ b"object blob 4\nabcd!",
+ b"object blob invalid\n" + b"x" * (4 * 1024 * 1024),
+ b"object blob 131072\n" + b"x" * (64 * 1024 + 1),
],
)
def test_git_blob_samples_reject_incomplete_framing(
@@ -157,7 +795,7 @@ def test_git_blob_samples_reject_incomplete_framing(
assert git_blob_samples(tmp_path, ["HEAD:source"]) == [None]
[email protected]("output", [b"", b"object blob 4\0text\0"])
[email protected]("output", [b"", b"object blob 4\ntext\n"])
def test_git_blob_samples_reject_git_failure(
tmp_path: Path, monkeypatch: pytest.MonkeyPatch, output: bytes
) -> None:
@@ -261,6 +899,40 @@ def test_git_output_removes_only_the_record_terminator(
assert git_output(tmp_path, "rev-parse", "--show-toplevel") == expected
[email protected]("noglob", ["0", "1"])
+def test_windows_directory_query_preserves_exclusions_with_inherited_noglob(
+ tmp_path: Path, monkeypatch: pytest.MonkeyPatch, noglob: str
+) -> None:
+ import workbench_target
+
+ repository = tmp_path / "repository"
+ initialize_git_repository(repository)
+ (repository / ".gitignore").write_text("locally-ignored/\n")
+ excludes = tmp_path / "global-excludes"
+ excludes.write_text("globally-ignored/\n")
+ for directory in ["src", "locally-ignored", "globally-ignored"]:
+ source = repository / "pkg" / directory
+ source.mkdir(parents=True)
+ (source / "app.py").write_text("source fixture\n")
+ monkeypatch.setattr(workbench_target, "_WINDOWS", True)
+ monkeypatch.setenv("GIT_NOGLOB_PATHSPECS", noglob)
+ monkeypatch.setenv("GIT_CONFIG_COUNT", "1")
+ monkeypatch.setenv("GIT_CONFIG_KEY_0", "core.excludesFile")
+ monkeypatch.setenv("GIT_CONFIG_VALUE_0", str(excludes))
+ environment = os.environ.copy()
+
+ paths = workbench_target.git_directory_snapshot_paths(repository)
+
+ assert paths is not None
+ assert {path.relative_to(repository).as_posix() for path in paths} == {
+ ".gitignore",
+ "README.md",
+ "pkg/src/app.py",
+ }
+ assert dict(os.environ) == environment
+
+
def test_directory_content_digest_uses_git_file_set(tmp_path: Path) -> None:
target = tmp_path / "target"
initialize_unborn_git_repository(target)
@@ -370,6 +1042,35 @@ def test_content_digest_expands_nested_git_repositories(
assert content_digest(selected) == original_digest
[email protected](os.name == "nt", reason="requires POSIX directory permissions")
[email protected]("inventory", ["plain", "git", "git-junctions"])
+def test_directory_inventory_skips_inaccessible_descendants(
+ tmp_path: Path, inventory: str, monkeypatch: pytest.MonkeyPatch
+) -> None:
+ target = tmp_path / "target"
+ if inventory != "plain":
+ initialize_unborn_git_repository(target)
+ else:
+ target.mkdir()
+ if inventory == "git-junctions":
+ monkeypatch.setitem(directory_content_digest.__globals__, "_WINDOWS", True)
+ source = target / "app.py"
+ source.write_text("before\n")
+ inaccessible = target / "unreadable"
+ inaccessible.mkdir()
+ (inaccessible / "private.txt").write_text("inaccessible fixture\n")
+ inaccessible.chmod(0)
+ try:
+ if os.access(inaccessible, os.R_OK | os.X_OK):
+ pytest.skip("directory permissions are not enforced for the current user")
+ assert WORKBENCH_TARGET["directory_snapshot_regular_file_count"](target) == 1
+ before = directory_content_digest(target)
+ source.write_text("after\n")
+ assert directory_content_digest(target) != before
+ finally:
+ inaccessible.chmod(0o700)
+
+
def test_directory_content_digest_skips_missing_cached_paths(tmp_path: Path) -> None:
target = tmp_path / "target"
initialize_unborn_git_repository(target)
@@ -658,7 +1359,9 @@ def test_copy_retains_alias_rooted_gitlink_exclusions(
copy = WORKBENCH_TARGET["copy_git_worktree_files"]
calls = []
- def checked_copy(source: Path, destination: Path, excluded: tuple[Path, ...]) -> Path:
+ def checked_copy(
+ source: Path, destination: Path, excluded: tuple[Path, ...]
+ ) -> tuple[Path, list[Path]]:
assert not source.samefile(submodule), "An excluded uninitialized gitlink was traversed"
calls.append(source)
return copy(source, destination, excluded)
@@ -667,8 +1370,9 @@ def test_copy_retains_alias_rooted_gitlink_exclusions(
selected_exclusions = tuple(path for path, _ in entries)
for index, excluded in enumerate({selected_exclusions, (submodule,)}):
calls.clear()
- copied = checked_copy(selected, tmp_path / f"copied-{index}", excluded)
+ copied, junctions = checked_copy(selected, tmp_path / f"copied-{index}", excluded)
assert len(calls) == 1
+ assert junctions == []
assert (copied / "fixture.py").read_text() == "synthetic = True\n"
assert not (copied / "submodule").exists()
diff --git a/sdk/typescript/src/cli.ts b/sdk/typescript/src/cli.ts
index dae47bf0b..04b3c61a2 100644
--- a/sdk/typescript/src/cli.ts
+++ b/sdk/typescript/src/cli.ts
@@ -250,6 +250,7 @@ import {
abortable,
DiffTarget,
enclosingGitWorktreeRoots,
+ UNSUPPORTED_GIT_ENVIRONMENT,
type ScanTarget,
relativePathIsOutside as isOutsidePath,
} from "./targets.js";
@@ -3726,12 +3727,17 @@ export async function main(
]
.map((path) => `'${path.replaceAll("'", `'"'"'`)}'`)
.join(" ");
- const contents = `#!/bin/sh\nset -eu\nexec ${command} scan . --working-tree --fail-on-severity ${options.failOnSeverity}\n`;
+ const invocation = `exec ${command} scan . --working-tree --fail-on-severity ${options.failOnSeverity}\n`;
+ const contents = `#!/bin/sh\nset -eu\nunset ${[...UNSUPPORTED_GIT_ENVIRONMENT].join(" ")}\n${invocation}`;
+ const previousScopedContents = `#!/bin/sh\nset -eu\nunset GIT_DIR GIT_WORK_TREE GIT_INDEX_FILE\n${invocation}`;
+ const previousContents = `#!/bin/sh\nset -eu\n${invocation}`;
const legacyContents = `#!/bin/sh\nset -eu\nexec npx --no-install codex-security scan . --working-tree --fail-on-severity ${options.failOnSeverity}\n`;
const existing = await readFile(hook, "utf8").catch(() => null);
if (
existing !== null &&
existing !== contents &&
+ existing !== previousScopedContents &&
+ existing !== previousContents &&
existing !== legacyContents
) {
throw new Error(`A pre-commit hook already exists at ${hook}.`);
@@ -3739,7 +3745,7 @@ export async function main(
if (existing === null) {
await mkdir(dirname(hook), { recursive: true });
await writeFile(hook, contents, { flag: "wx", mode: 0o755 });
- } else if (existing === legacyContents) {
+ } else if (existing !== contents) {
await writeFile(hook, contents, { flag: "w" });
}
return {
diff --git a/sdk/typescript/src/component-plan.ts b/sdk/typescript/src/component-plan.ts
index e5ff9209a..8b702b37b 100644
--- a/sdk/typescript/src/component-plan.ts
+++ b/sdk/typescript/src/component-plan.ts
@@ -1,8 +1,6 @@
-import { execFile as execFileCallback } from "node:child_process";
import { lstat, readdir } from "node:fs/promises";
import { tmpdir } from "node:os";
import { join, posix } from "node:path";
-import { promisify } from "node:util";
import { z } from "incur";
import type { CodexSecuritySurface, ScanAuthMode } from "./api.js";
import type { CodexSecurityConfig } from "./config.js";
@@ -14,13 +12,12 @@ import { CODEX_SECURITY_THREAD_SOURCES } from "./thread-source.js";
import {
nullIfMissingFile,
enclosingGitWorktreeRoot,
+ gitOutput,
normalizeRepository,
normalizeTarget,
validatedGitEnvironment,
} from "./targets.js";
-import { resolveTrustedExecutable } from "./trusted-executable.js";
-const execFile = promisify(execFileCallback);
/** @internal */
export const componentPlanSchema = z.strictObject({
components: z
@@ -270,16 +267,15 @@ async function inventoryFiles(
): Promise<string[]> {
signal?.throwIfAborted();
const utf8 = new TextDecoder("utf-8", { fatal: true, ignoreBOM: true });
- if (await enclosingGitWorktreeRoot(repository, signal)) {
+ if (
+ await enclosingGitWorktreeRoot(repository, signal, {
+ requireIfPresent: true,
+ })
+ ) {
validatedGitEnvironment();
- const git = await resolveTrustedExecutable("git", process.env, repository);
- if (git === null)
- throw new Error("Git is required to inventory this repository.");
- const { stdout } = await execFile(
- git.executable,
+ const stdout = await gitOutput(
+ repository,
[
- "-C",
- repository,
"ls-files",
"--cached",
"--others",
@@ -289,7 +285,10 @@ async function inventoryFiles(
"--",
".",
],
- { env: git.environment, signal, maxBuffer: Infinity, encoding: "buffer" },
+ signal,
+ undefined,
+ undefined,
+ "buffer",
);
const files: string[] = [];
for (const path of stdout.toString("latin1").split("\0").filter(Boolean)) {
diff --git a/sdk/typescript/src/multiscan.ts b/sdk/typescript/src/multiscan.ts
index 5f7ee310f..2d281bedb 100644
--- a/sdk/typescript/src/multiscan.ts
+++ b/sdk/typescript/src/multiscan.ts
@@ -36,7 +36,11 @@ import {
} from "./knowledge-base.js";
import { resolveScanPrompts } from "./prompt-files.js";
import { requireSecureOutputAncestry, validateOutputDir } from "./runtime.js";
-import { DiffTarget, type ScanMode } from "./targets.js";
+import {
+ DiffTarget,
+ UNSUPPORTED_GIT_ENVIRONMENT,
+ type ScanMode,
+} from "./targets.js";
import {
meetsSeverity,
type ScanPromptSettings,
@@ -1180,18 +1184,14 @@ async function checkoutRevision(
githubHost?: string,
): Promise<void> {
const environment = { ...process.env };
- const repositoryVariables = new Set([
- "GIT_DIR",
- "GIT_WORK_TREE",
- "GIT_INDEX_FILE",
- "GIT_OBJECT_DIRECTORY",
- "GIT_ALTERNATE_OBJECT_DIRECTORIES",
- ]);
for (const name of Object.keys(environment)) {
- if (repositoryVariables.has(name.toUpperCase())) delete environment[name];
+ if (UNSUPPORTED_GIT_ENVIRONMENT.has(name.toUpperCase()))
+ delete environment[name];
}
environment["GIT_TERMINAL_PROMPT"] = "0";
environment["GIT_LFS_SKIP_SMUDGE"] = "1";
+ environment["GIT_DEFAULT_HASH"] =
+ task.revision.length === 64 ? "sha256" : "sha1";
const command = await resolveTrustedExecutable(
"git",
environment,
diff --git a/sdk/typescript/src/targets.ts b/sdk/typescript/src/targets.ts
index 5ef7153b1..18bf98cd1 100644
--- a/sdk/typescript/src/targets.ts
+++ b/sdk/typescript/src/targets.ts
@@ -13,7 +13,7 @@ import {
sep,
} from "node:path";
import { promisify } from "node:util";
-import { InvalidTargetError, abortReason } from "./errors.js";
+import { InvalidTargetError, abortReason, errorMessage } from "./errors.js";
import {
resolveTrustedExecutable,
type TrustedExecutable,
@@ -24,7 +24,7 @@ import type { ScanMode } from "./scan-modes.js";
export type { ScanMode } from "./scan-modes.js";
const execFile = promisify(execFileCallback);
-const UNSUPPORTED_GIT_ENVIRONMENT = new Set([
+export const UNSUPPORTED_GIT_ENVIRONMENT: ReadonlySet<string> = new Set([
"GIT_DIR",
"GIT_WORK_TREE",
"GIT_INDEX_FILE",
@@ -190,7 +190,7 @@ export async function enclosingGitWorktreeRoot(
if (strict && error instanceof InvalidTargetError) throw error;
if (markerRoot !== null) {
throw new InvalidTargetError(
- "Could not determine the Git worktree root. Check that Git is installed and the checkout is accessible.",
+ `Could not determine the Git worktree root. Check that Git is installed and the checkout is accessible. ${errorMessage(error)}`,
{ cause: error },
);
}
@@ -261,12 +261,24 @@ export async function isGitMetadataDirectory(
"core.repositoryformatversion",
],
signal,
+ { LC_ALL: "C" },
);
return /^\d+$/u.test(version);
} catch (error) {
throwIfAborted(signal);
- // git config uses status 1 when the requested key is absent.
- if (error instanceof Error && "code" in error && error.code === 1)
+ // Application folders can share these filenames without using Git's
+ // config format. Missing or invalid declarations do not identify metadata.
+ if (
+ error instanceof Error &&
+ "code" in error &&
+ (error.code === 1 ||
+ (error.code === 128 &&
+ "stderr" in error &&
+ typeof error.stderr === "string" &&
+ /fatal: bad (?:config line|numeric config value)/u.test(
+ error.stderr,
+ )))
+ )
return false;
throw error;
}
@@ -684,7 +696,7 @@ async function requireGitRepository(
} catch (error) {
throwIfAborted(signal);
throw new InvalidTargetError(
- `Diff targets require a Git repository: ${repository}`,
+ `Diff targets require a Git repository: ${repository}. ${errorMessage(error)}`,
{
cause: error,
},
@@ -711,7 +723,10 @@ async function resolveGitRef(
);
} catch (error) {
throwIfAborted(signal);
- throw new InvalidTargetError(`unknown Git ref: ${ref}`, { cause: error });
+ throw new InvalidTargetError(
+ `unknown Git ref: ${ref}. ${errorMessage(error)}`,
+ { cause: error },
+ );
}
}
@@ -749,17 +764,35 @@ export async function gitHistoryIdentity(
return { commonDirectory, origin };
}
-async function gitOutput(
+export function gitOutput(
+ repository: string,
+ args: readonly string[],
+ signal?: AbortSignal,
+ environment?: NodeJS.ProcessEnv,
+ workingDirectory?: string,
+): Promise<string>;
+export function gitOutput(
+ repository: string,
+ args: readonly string[],
+ signal: AbortSignal | undefined,
+ environment: NodeJS.ProcessEnv | undefined,
+ workingDirectory: string | undefined,
+ encoding: "buffer",
+): Promise<Buffer>;
+export async function gitOutput(
repository: string,
args: readonly string[],
signal?: AbortSignal,
environment: NodeJS.ProcessEnv = {},
workingDirectory = repository,
-): Promise<string> {
+ encoding: "utf8" | "buffer" = "utf8",
+): Promise<string | Buffer> {
throwIfAborted(signal);
const command = await resolveTrustedExecutable(
"git",
- isolatedGitEnvironment(args[0] === "rev-parse" || args[0] === "config"),
+ isolatedGitEnvironment(
+ args[0] === "rev-parse" || args[0] === "config" || args[0] === "ls-files",
+ ),
(await gitMarkerRoot(repository, signal, "outermost")) ?? repository,
);
if (command === null)
@@ -769,13 +802,15 @@ async function gitOutput(
command.executable,
["-c", "core.fsmonitor=false", "-C", workingDirectory, ...args],
{
- encoding: "utf8",
+ encoding,
signal,
env: { ...command.environment, ...environment },
maxBuffer: Infinity,
},
);
- return stdout.replace(process.platform === "win32" ? /\r?\n$/u : /\n$/u, "");
+ return typeof stdout === "string"
+ ? stdout.replace(process.platform === "win32" ? /\r?\n$/u : /\n$/u, "")
+ : stdout;
}
export async function gitMarkerRoot(
diff --git a/sdk/typescript/tests-ts/cli.test.ts b/sdk/typescript/tests-ts/cli.test.ts
index 7acb9bda9..f14dbc2de 100644
--- a/sdk/typescript/tests-ts/cli.test.ts
+++ b/sdk/typescript/tests-ts/cli.test.ts
@@ -63,6 +63,7 @@ import {
} from "./cli-fixtures.js";
import { runCommand } from "./support/shell.js";
import { temporaryDirectory } from "./support/temporary-directories.js";
+import { runTestInSubprocess } from "./support/test-subprocess.js";
import { fail, throwing } from "./support/errors.js";
import {
createCliTest,
@@ -614,6 +615,19 @@ describe("CLI", () => {
expect(normalize(migrated.hook)).toBe(hook);
expect(migrated.failOnSeverity).toBe("medium");
expect(await readFile(hook, "utf8")).toBe(trustedHook);
+ for (const unset of [
+ "",
+ "unset GIT_DIR GIT_WORK_TREE GIT_INDEX_FILE\n",
+ ]) {
+ await writeFile(hook, trustedHook.replace(/^unset .*\n/m, unset));
+ expect(
+ await migratedHook.run(
+ ["install-hook", ".", "--fail-on-severity", "medium", "--json"],
+ deps,
+ ),
+ ).toBe(0);
+ expect(await readFile(hook, "utf8")).toBe(trustedHook);
+ }
const existingHook = captureCli(main, "stderr");
expect(await existingHook.run(["install-hook", "."], deps)).toBe(2);
@@ -693,6 +707,7 @@ describe("CLI", () => {
},
);
expect(commit.status, commit.stderr).toBeGreaterThan(0);
+ expect(commit.stderr).not.toContain("GIT_INDEX_FILE is not supported");
await expect(stat(maliciousMarker)).rejects.toMatchObject({
code: "ENOENT",
});
@@ -708,6 +723,112 @@ describe("CLI", () => {
}
});
+ (process.platform === "win32" ? test.skip : test)(
+ "installed hook clears Git repository overrides before scan execution",
+ async () => {
+ if (
+ runTestInSubprocess(
+ "./tests-ts/cli.test.ts",
+ "installed hook clears Git repository overrides before scan execution",
+ )
+ )
+ return;
+ const root = await temporaryDirectory("codex-security-hook-handoff-");
+ const repository = join(root, "repository");
+ const executable = join(root, "fixture-runtime.mjs");
+ const record = join(root, "scan-arguments.json");
+ const originalExecPath = Object.getOwnPropertyDescriptor(
+ process,
+ "execPath",
+ )!;
+ try {
+ await mkdir(repository);
+ await writeFile(
+ executable,
+ [
+ `#!${process.execPath}`,
+ `import { validatedGitEnvironment } from ${JSON.stringify(new URL("../src/targets.ts", import.meta.url).href)};`,
+ 'import { writeFileSync } from "node:fs";',
+ "validatedGitEnvironment();",
+ `writeFileSync(${JSON.stringify(record)}, JSON.stringify(process.argv.slice(3)));`,
+ 'if (process.env.FIXTURE_POLICY_FAILURE) { console.error("Synthetic severity threshold exceeded"); process.exit(1); }',
+ ].join("\n"),
+ { mode: 0o700 },
+ );
+ expect(
+ (
+ await runCommand("git", ["-C", repository, "init", "-q"], {
+ timeout: 10000,
+ })
+ ).status,
+ ).toBe(0);
+ Object.defineProperty(process, "execPath", {
+ ...originalExecPath,
+ value: executable,
+ });
+ try {
+ expect(
+ await captureCli(main, "stdout").run(
+ ["install-hook", "."],
+ dependencies({ currentDirectory: repository }),
+ ),
+ ).toBe(0);
+ } finally {
+ Object.defineProperty(process, "execPath", originalExecPath);
+ }
+ const args = [
+ "-C",
+ repository,
+ "-c",
+ "user.name=Fixture",
+ "-c",
+ "-c",
+ "commit.gpgsign=false",
+ "commit",
+ "--allow-empty",
+ "-qm",
+ "fixture",
+ ];
+ const gitDirectory = join(repository, ".git");
+ const environment = {
+ ...process.env,
+ GIT_DIR: gitDirectory,
+ GIT_WORK_TREE: repository,
+ GIT_INDEX_FILE: join(gitDirectory, "index"),
+ GIT_OBJECT_DIRECTORY: join(gitDirectory, "objects"),
+ GIT_ALTERNATE_OBJECT_DIRECTORIES: join(gitDirectory, "objects"),
+ GIT_COMMON_DIR: gitDirectory,
+ GIT_REPLACE_REF_BASE: "refs/replace/",
+ };
+ const clean = await runCommand("git", args, {
+ env: environment,
+ timeout: 10000,
+ });
+ expect(clean.status, clean.stderr).toBe(0);
+ expect(JSON.parse(await readFile(record, "utf8"))).toEqual([
+ "scan",
+ ".",
+ "--working-tree",
+ "--fail-on-severity",
+ "high",
+ ]);
+ const rejected = await runCommand("git", args, {
+ env: { ...environment, FIXTURE_POLICY_FAILURE: "1" },
+ timeout: 10_000,
+ });
+ expect(rejected.status).not.toBe(0);
+ expect(rejected.stderr).toContain(
+ "Synthetic severity threshold exceeded",
+ );
+ expect(rejected.stderr).not.toContain("is not supported");
+ } finally {
+ Object.defineProperty(process, "execPath", originalExecPath);
+ await rm(root, { recursive: true, force: true });
+ }
+ },
+ );
+
test("runs a bulk scan and keeps structured output on stdout", async () => {
const root = await temporaryDirectory("codex-security-cli-multiscan-");
const architecturePath = join(root, "architecture.md");
diff --git a/sdk/typescript/tests-ts/component-scan.test.ts b/sdk/typescript/tests-ts/component-scan.test.ts
index d51667d7f..d2d115978 100644
--- a/sdk/typescript/tests-ts/component-scan.test.ts
+++ b/sdk/typescript/tests-ts/component-scan.test.ts
@@ -2,10 +2,10 @@ import { codexWithRun } from "./support/codex.js";
import { createCliTest } from "./support/cli-run.js";
import { gitText, nodeCommand } from "./support/shell.js";
import { resolving } from "./support/promises.js";
-import { runTestInSubprocess } from "./support/test-subprocess.js";
import { execFileSync } from "node:child_process";
import {
mkdir,
+ chmod,
readFile,
readdir,
realpath,
@@ -15,7 +15,7 @@ import {
unlink,
writeFile,
} from "node:fs/promises";
-import { dirname, join, relative, sep } from "node:path";
+import { delimiter, dirname, join, relative, sep } from "node:path";
import { fileURLToPath, pathToFileURL } from "node:url";
import { afterEach, expect, spyOn, test, mock } from "bun:test";
import type { ScanOptions } from "../src/api.js";
@@ -52,6 +52,7 @@ import {
import { PLUGIN_ROOT } from "./plugin-root.js";
import { createApiTestFixtures } from "./support/temporary-directories.js";
import { readJson } from "./support/json.js";
+import { runTestInSubprocess } from "./support/test-subprocess.js";
import { rejecting, throwing } from "./support/errors.js";
const { temporaryDirectory, cleanup } =
@@ -231,19 +232,11 @@ async function scopedInventory(paths: Fixture, scope: string) {
"sys.path.insert(0, sys.argv[1])",
"import workbench_target as target",
"from generate_rank_input import make_repo_scope_input",
- "queries = []",
- "git_bytes = target.git_bytes",
- "def record_query(repository, *args, **kwargs):",
- " data = git_bytes(repository, *args, **kwargs)",
- " if 'ls-files' in args:",
- " queries.append({'pathspec': args[-1], 'count': len([path for path in (data or b'').split(b'\\0') if path])})",
- " return data",
- "target.git_bytes = record_query",
"repo, scope, scopes, output = sys.argv[2:]",
"make_repo_scope_input(Namespace(repo=repo, scopes_file=scopes, out=output))",
"rows = [json.loads(line)['path'] for line in Path(output).read_text().splitlines()]",
"count = target.directory_snapshot_regular_file_count((Path(repo) / scope).resolve())",
- "print(json.dumps({'paths': rows, 'count': count, 'queries': queries}))",
+ "print(json.dumps({'paths': rows, 'count': count}))",
].join("\n"),
join(PLUGIN_ROOT, "scripts"),
paths.repository,
@@ -256,7 +249,6 @@ async function scopedInventory(paths: Fixture, scope: string) {
return JSON.parse(stdout.trim().split("\n").at(-1)!) as {
paths: string[];
count: number;
- queries: Array<{ pathspec: string; count: number }>;
};
}
@@ -932,6 +924,124 @@ test("rejects escaped component paths and output inside the enclosing worktree",
).rejects.toThrow();
});
+test("component inventory uses the shared Git runner without fsmonitor callbacks", async () => {
+ const paths = await fixture();
+ execFileSync("git", ["-C", paths.repository, "init", "-q"]);
+ const marker = join(paths.root, "fsmonitor-called");
+ const hook = join(paths.root, "fsmonitor-hook");
+ await writeFile(
+ hook,
+ `#!/bin/sh\nprintf called > '${marker.replaceAll("'", `'"'"'`)}'\nprintf '\\0'\n`,
+ );
+ await chmod(hook, 0o700);
+ execFileSync("git", [
+ "-C",
+ paths.repository,
+ "config",
+ "core.fsmonitor",
+ hook,
+ ]);
+ const plan = await planComponents(paths.repository, {
+ codex: fakeCodex(() => ({
+ components: [{ name: "Source", paths: ["."] }],
+ })),
+ });
+ expect(plan.components).toHaveLength(1);
+ expect(await stat(marker).catch(() => null)).toBeNull();
+});
+
+test.each(["inline", "global"])(
+ "component inventory preserves inherited %s Git exclusions",
+ async (configuration) => {
+ const paths = await fixture();
+ execFileSync("git", ["-C", paths.repository, "init", "-q"]);
+ const excludes = join(paths.root, "excludes");
+ const config = join(paths.root, "gitconfig");
+ await writeFile(excludes, "ignored.ts\n");
+ await writeFile(join(paths.repository, "ignored.ts"), "export {};\n");
+ execFileSync("git", [
+ "config",
+ "--file",
+ config,
+ "core.excludesFile",
+ excludes,
+ ]);
+ const previousEnvironment = process.env;
+ process.env = {
+ ...previousEnvironment,
+ ...(configuration === "inline"
+ ? {
+ GIT_CONFIG_COUNT: "1",
+ GIT_CONFIG_KEY_0: "core.excludesFile",
+ GIT_CONFIG_VALUE_0: excludes,
+ }
+ : { GIT_CONFIG_GLOBAL: config }),
+ };
+ try {
+ const plan = await planComponents(paths.repository, {
+ codex: {
+ startThread: () => ({
+ async run(prompt) {
+ expect(prompt).not.toContain("ignored.ts");
+ return {
+ finalResponse: JSON.stringify({ components: [components[0]] }),
+ };
+ },
+ }),
+ },
+ });
+ expect(plan.components.flatMap(({ paths }) => paths)).not.toContain(
+ "ignored.ts",
+ );
+ } finally {
+ process.env = previousEnvironment;
+ }
+ },
+);
+
+test("component inventory reports broken Git configuration without walking ignored files", async () => {
+ const paths = await fixture();
+ execFileSync("git", ["-C", paths.repository, "init", "-q"]);
+ await writeFile(join(paths.repository, ".git", "config"), "[broken config\n");
+ let planned = false;
+ await expect(
+ planComponents(paths.repository, {
+ codex: fakeCodex(() => {
+ planned = true;
+ return { components: [{ name: "Source", paths: ["."] }] };
+ }),
+ }),
+ ).rejects.toThrow("bad config line");
+ expect(planned).toBe(false);
+});
+
+test("component inventory preserves non-ASCII Git failure diagnostics", async () => {
+ const paths = await fixture();
+ const name = "git-日本語-😀";
+ const metadata = join(paths.root, name);
+ execFileSync("git", [
+ "-C",
+ paths.repository,
+ "init",
+ "-q",
+ "--separate-git-dir",
+ metadata,
+ ]);
+ execFileSync("git", [
+ "--git-dir",
+ metadata,
+ "config",
+ "core.worktree",
+ paths.repository,
+ ]);
+ await writeFile(join(metadata, "index"), "broken index");
+ const response = mock(() => ({ components: [components[0]!] }));
+ await expect(
+ planComponents(paths.repository, { codex: fakeCodex(response) }),
+ ).rejects.toThrow(name);
+ expect(response).not.toHaveBeenCalled();
+});
+
test("plans from a Git inventory without tools or ignored files", async () => {
const paths = await fixture();
execFileSync("git", ["-C", paths.repository, "init", "-q"]);
@@ -1037,7 +1147,7 @@ test("plans Unicode files when directory entry types are unknown", async () => {
const root = await temporaryDirectory();
const repository = join(root, "repository");
const paths = ["name-\uFFFD.ts", "\uFEFF来源.ts", "résumé/🙂.ts"];
- for (const path of [...paths, ".git/ignored.ts"]) {
+ for (const path of [...paths, "nested/.git/ignored.ts"]) {
await mkdir(dirname(join(repository, path)), { recursive: true });
await writeFile(join(repository, path), "export {};\n");
}
@@ -1246,10 +1356,6 @@ test("keeps scoped inventories and plans aligned after a case-only Git rename",
expect(await scopedInventory(paths, source)).toEqual({
paths: ordinaryPaths,
count: 3,
- queries: [
- { pathspec: ":(icase,literal)" + source, count: 3 },
- { pathspec: ":(icase,literal)" + source, count: 3 },
- ],
});
git("switch", "-c", "case-rename");
git("mv", source, "renaming");
@@ -1268,12 +1374,7 @@ test("keeps scoped inventories and plans aligned after a case-only Git rename",
expect(entries).toContain(uppercase);
const inventory = async (scope: string) => {
- const { queries, ...selected } = await scopedInventory(paths, scope);
- const pathspec = scope === "." ? "." : ":(icase,literal)" + scope;
- expect(queries.map((query) => query.pathspec)).toEqual([
- pathspec,
- pathspec,
- ]);
+ const selected = await scopedInventory(paths, scope);
if (scope === ".") {
selected.paths = (
await Promise.all(
@@ -1310,13 +1411,7 @@ test("keeps scoped inventories and plans aligned after a case-only Git rename",
paths: [...expectedInventory.paths, scope + "/untracked.ts"].sort(),
count: 4,
};
- expect(await scopedInventory(paths, scope)).toEqual({
- ...mixedInventory,
- queries: [
- { pathspec: ":(icase,literal)" + scope, count: 4 },
- { pathspec: ":(icase,literal)" + scope, count: 4 },
- ],
- });
+ expect(await scopedInventory(paths, scope)).toEqual(mixedInventory);
const repositoryInventory = {
paths: [
".gitignore",
@@ -1383,10 +1478,10 @@ test("retains tracked Unicode aliases when scoped Git matching is incomplete", a
await writeFile(join(paths.repository, nonCased, "app.ts"), "export {};\n");
execFileSync("git", ["-C", paths.repository, "init", "-q"]);
execFileSync("git", ["-C", paths.repository, "add", "--force", "."]);
- expect((await scopedInventory(paths, nonCased)).queries).toEqual([
- { pathspec: ":(icase,literal)" + nonCased, count: 1 },
- { pathspec: ":(icase,literal)" + nonCased, count: 1 },
- ]);
+ expect(await scopedInventory(paths, nonCased)).toEqual({
+ paths: [nonCased + "/app.ts"],
+ count: 1,
+ });
await rename(
join(paths.repository, source),
join(paths.repository, "renaming"),
@@ -1422,7 +1517,6 @@ test("retains tracked Unicode aliases when scoped Git matching is incomplete", a
).sort();
expect(await identities(inventory.paths)).toEqual(await identities(selected));
expect(inventory.count).toBe(selected.length);
- expect(inventory.queries.map((query) => query.pathspec)).toEqual([".", "."]);
});
test("plans plain directories and rejects unsafe or overlapping model scopes", async () => {
@@ -1689,3 +1783,41 @@ test("CLI rejects ambiguous component selection", async () => {
expect(result.stderr).toContain("Choose exactly one");
}
});
+
+(process.platform === "win32" ? test.skip : test)(
+ "component inventory excludes checkout-local Git when planning a subdirectory",
+ async () => {
+ if (
+ runTestInSubprocess(
+ "./tests-ts/component-scan.test.ts",
+ "component inventory excludes checkout-local Git when planning a subdirectory",
+ )
+ )
+ return;
+ const paths = await fixture();
+ execFileSync("git", ["-C", paths.repository, "init", "-q"]);
+ const actualGit = Bun.which("git")!;
+ const bin = join(paths.repository, "bin");
+ const marker = join(paths.root, "local-git-called");
+ await mkdir(bin);
+ await writeFile(
+ join(bin, "git"),
+ `#!/bin/sh\nprintf called > '${marker.replaceAll("'", `'"'"'`)}'\nexec '${actualGit.replaceAll("'", `'"'"'`)}' "$@"\n`,
+ { mode: 0o700 },
+ );
+ const previous = process.env["PATH"];
+ process.env["PATH"] = bin + delimiter + previous;
+ try {
+ const plan = await planComponents(join(paths.repository, "apps", "api"), {
+ codex: fakeCodex(() => ({
+ components: [{ name: "Source", paths: ["."] }],
+ })),
+ });
+ expect(plan.components).toHaveLength(1);
+ expect(await stat(marker).catch(() => null)).toBeNull();
+ } finally {
+ if (previous === undefined) delete process.env["PATH"];
+ else process.env["PATH"] = previous;
+ }
+ },
+);
diff --git a/sdk/typescript/tests-ts/multiscan.test.ts b/sdk/typescript/tests-ts/multiscan.test.ts
index 2abfe5a5a..5597c39c2 100644
--- a/sdk/typescript/tests-ts/multiscan.test.ts
+++ b/sdk/typescript/tests-ts/multiscan.test.ts
@@ -73,6 +73,7 @@ function git(repository: string, ...args: string[]): string {
async function repository(
root: string,
name: string,
+ objectFormat = "sha1",
): Promise<{ path: string; revision: string }> {
const path = join(root, name);
await mkdir(join(path, "src"), { recursive: true });
@@ -80,7 +81,7 @@ async function repository(
join(path, "src", "app.ts"),
`export const name = "${name}";\n`,
);
- git(path, "init", "-q");
+ git(path, "init", "-q", `--object-format=${objectFormat}`);
git(path, "add", ".");
git(
path,
@@ -3040,10 +3041,20 @@ describe("multiscan", () => {
});
test("removes mixed-case repository Git variables before cloning", async () => {
+ if (
+ runTestInSubprocess(
+ "./tests-ts/multiscan.test.ts",
+ "removes mixed-case repository Git variables before cloning",
+ )
+ )
+ return;
const { paths } = await repositoryFixture("isolated");
const trace = join(paths.root, "git-events.jsonl");
const repositoryVariables = [
+ "GIT_DIR",
+ "GIT_COMMON_DIR",
+ "gIt_Replace_Ref_Base",
"Git_Dir",
"gIt_Work_Tree",
"Git_Index_File",
@@ -3062,11 +3073,19 @@ describe("multiscan", () => {
}
process.env["GIT_TRACE2_EVENT"] = trace;
process.env["GIT_TRACE2_ENV_VARS"] = repositoryVariables.join(",");
+ const inherited = repositoryVariables.map((name) => process.env[name]);
const summary = await runMultiscan(
options(paths, client(completeRunWithoutAwait)),
);
expect(summary).toMatchObject({ completed: 1, failed: 0 });
+ const resumed = await runMultiscan(
+ options(paths, client(completeRunWithoutAwait)),
+ );
+ expect(resumed).toMatchObject({ completed: 1, skipped: 1, failed: 0 });
+ expect(repositoryVariables.map((name) => process.env[name])).toEqual(
+ inherited,
+ );
const leakedVariables = parseJsonLines<{
event: string;
@@ -3462,3 +3481,37 @@ describe("multiscan", () => {
]);
});
});
+
+test.each(["sha1", "sha256"])(
+ "bulk checkout preserves %s object format despite Git defaults",
+ async (objectFormat) => {
+ const name = `bulk checkout preserves ${objectFormat} object format despite Git defaults`;
+ if (runTestInSubprocess(fileURLToPath(import.meta.url), name)) return;
+ const paths = await fixture();
+ const repo = await repository(paths.root, "source", objectFormat);
+ await writeFile(
+ paths.input,
+ `id,repository,revision\nfixture,${repo.path},${repo.revision}\n`,
+ );
+ const run = mock(async (checkout: string, scanOptions = {}) => {
+ expect(git(checkout, "rev-parse", "--show-object-format")).toBe(
+ objectFormat,
+ );
+ return completeRun(checkout, scanOptions);
+ });
+ const previous = process.env["GIT_DEFAULT_HASH"];
+ process.env["GIT_DEFAULT_HASH"] =
+ objectFormat === "sha1" ? "sha256" : "sha1";
+ try {
+ const result = await runMultiscan(options(paths, client(run)));
+ expect(result.failed).toBe(0);
+ expect(run).toHaveBeenCalledTimes(1);
+ expect(process.env["GIT_DEFAULT_HASH"]).toBe(
+ objectFormat === "sha1" ? "sha256" : "sha1",
+ );
+ } finally {
+ if (previous === undefined) delete process.env["GIT_DEFAULT_HASH"];
+ else process.env["GIT_DEFAULT_HASH"] = previous;
+ }
+ },
+);
diff --git a/sdk/typescript/tests-ts/scan-recovery.test.ts b/sdk/typescript/tests-ts/scan-recovery.test.ts
index cefabbb37..9be0ad021 100644
--- a/sdk/typescript/tests-ts/scan-recovery.test.ts
+++ b/sdk/typescript/tests-ts/scan-recovery.test.ts
@@ -420,7 +420,7 @@ describe("malformed scan artifact recovery", () => {
"sys.path.insert(0, sys.argv[1])",
"import workbench_target as target",
"source = Path(sys.argv[2])",
- "checkout = target.copy_git_worktree_files(source, Path(sys.argv[3]), ())",
+ "checkout, _ = target.copy_git_worktree_files(source, Path(sys.argv[3]), ())",
"git_dir = Path(target.git_output(source, 'rev-parse', '--absolute-git-dir'))",
"assert target.worktree_content_digest_for_context(checkout, '.', git_dir=git_dir, work_tree=checkout) == target.worktree_content_digest(source)",
].join("\n"),
diff --git a/sdk/typescript/tests-ts/targets.test.ts b/sdk/typescript/tests-ts/targets.test.ts
index 42e184d0f..5d18923c8 100644
--- a/sdk/typescript/tests-ts/targets.test.ts
+++ b/sdk/typescript/tests-ts/targets.test.ts
@@ -26,6 +26,7 @@ import {
import {
enclosingGitWorktreeRoot,
gitMarkerRoot,
+ isGitMetadataDirectory,
gitProtectionRoots,
} from "../src/targets.js";
import { createApiTestFixtures } from "./support/temporary-directories.js";
@@ -743,6 +744,35 @@ test("finds Git boundaries through directory aliases and file inputs", async ()
).toBe(repo);
});
+test("ordinary storage folders with malformed config are not Git metadata", async () => {
+ const root = await temporaryDirectory();
+ await mkdir(join(root, "objects"));
+ await mkdir(join(root, "refs"));
+ await writeFile(join(root, "config"), "ordinary application configuration\n");
+ expect(await isGitMetadataDirectory(root)).toBe(false);
+ expect(await normalizeRepository(root)).toBe(await realpath(root));
+});
+
+test("invalid diff refs retain the Git diagnostic", async () => {
+ const repo = await repository();
+ const original = spawnSync(
+ "git",
+ [
+ "-C",
+ repo,
+ "rev-parse",
+ "--verify",
+ "--end-of-options",
+ "missing-fixture-ref^{commit}",
+ ],
+ { encoding: "utf8" },
+ );
+ expect(original.status).not.toBe(0);
+ await expect(
+ normalizeTarget(repo, DiffTarget.refs({ base: "missing-fixture-ref" })),
+ ).rejects.toThrow(original.stderr.trim());
+});
+
test("executable protection retains lexical and canonical checkout roots", async () => {
const lexical = await repository();
const canonical = await repository("destination");