Skip to content

OpenAI's Codex Security CLI and TypeScript SDK for finding, validating, and fixing security vulnerabilities. npm: https://www.npmjs.com/package/@openai/codex-security

fix(plugin): stream tracked diffs while hashing (#773)

HughChawb1774f58cf40
diff --git a/plugins/codex-security/scripts/workbench_target.py b/plugins/codex-security/scripts/workbench_target.py
index 47fcde677..b380a275f 100644
--- a/plugins/codex-security/scripts/workbench_target.py
+++ b/plugins/codex-security/scripts/workbench_target.py
@@ -10,8 +10,9 @@ import sqlite3
import stat
import subprocess
import sys
+import tempfile
from pathlib import Path
-from typing import Any
+from typing import Any, BinaryIO
# Some plugin hosts launch Python with safe-path isolation enabled.
sys.path.insert(0, str(Path(__file__).resolve().parent))
@@ -127,6 +128,7 @@ def git_command(
input_data: str | bytes | None = None,
git_dir: Path | None = None,
work_tree: Path | None = None,
+ stdout_file: BinaryIO | None = None,
) -> subprocess.CompletedProcess[str] | subprocess.CompletedProcess[bytes]:
if (git_dir is None) != (work_tree is None):
raise ValueError("git_dir and work_tree must be provided together")
@@ -148,15 +150,20 @@ def git_command(
command.extend(["--git-dir", str(git_dir), "--work-tree", str(work_tree)])
full_command = [*command, *args]
try:
+ output_options = (
+ {"capture_output": True}
+ if stdout_file is None
+ else {"stdout": stdout_file, "stderr": subprocess.PIPE}
+ )
return subprocess.run(
full_command,
check=False,
- capture_output=True,
env=environment,
text=text,
encoding="utf-8" if text else None,
errors="surrogateescape" if text else None,
input=input_data,
+ **output_options,
)
except FileNotFoundError:
# Git is optional for Codebase scans. Treat an unavailable executable like
@@ -165,13 +172,45 @@ def git_command(
return subprocess.CompletedProcess(full_command, 127, empty_output, empty_output)
-def update_digest_field(digest: Any, label: bytes, value: bytes) -> None:
+def _update_digest_field_header(digest: Any, label: bytes, value_size: int) -> None:
digest.update(len(label).to_bytes(4, "big"))
digest.update(label)
- digest.update(len(value).to_bytes(8, "big"))
+ digest.update(value_size.to_bytes(8, "big"))
+
+
+def update_digest_field(digest: Any, label: bytes, value: bytes) -> None:
+ _update_digest_field_header(digest, label, len(value))
digest.update(value)
+def _update_digest_field_from_git(
+ digest: Any,
+ label: bytes,
+ target: Path,
+ *args: str,
+ git_dir: Path | None = None,
+ work_tree: Path | None = None,
+) -> bool:
+ with tempfile.TemporaryFile() as output:
+ completed = git_command(
+ target,
+ *args,
+ text=False,
+ git_dir=git_dir,
+ work_tree=work_tree,
+ stdout_file=output,
+ )
+ if completed.returncode != 0:
+ return False
+ output.seek(0, os.SEEK_END)
+ value_size = output.tell()
+ _update_digest_field_header(digest, label, value_size)
+ output.seek(0)
+ for chunk in iter(lambda: output.read(1024 * 1024), b""):
+ digest.update(chunk)
+ return True
+
+
def worktree_content_digest(target: Path) -> str:
require_clean_submodule_worktrees(target)
repository, pathspec = git_worktree_context(target)
@@ -203,7 +242,11 @@ def worktree_content_digest_for_context(
git_dir: Path | None = None,
work_tree: Path | None = None,
) -> str:
- tracked = git_bytes(
+ digest = hashlib.sha256()
+ update_digest_field(digest, b"format", b"codex-security-snapshot/v1")
+ tracked_ok = _update_digest_field_from_git(
+ digest,
+ b"tracked-diff",
repository,
"diff",
"--binary",
@@ -228,11 +271,8 @@ def worktree_content_digest_for_context(
git_dir=git_dir,
work_tree=work_tree,
)
- if tracked is None or untracked is None:
+ if not tracked_ok or untracked is None:
raise SystemExit("Could not snapshot the selected working-tree changes.")
- digest = hashlib.sha256()
- update_digest_field(digest, b"format", b"codex-security-snapshot/v1")
- update_digest_field(digest, b"tracked-diff", tracked)
for raw_path in sorted(path for path in untracked.split(b"\0") if path):
relative_path = os.fsdecode(raw_path)
path = (work_tree or repository) / relative_path
diff --git a/plugins/codex-security/tests/test_workbench_target.py b/plugins/codex-security/tests/test_workbench_target.py
index a4e37753c..734dce876 100644
--- a/plugins/codex-security/tests/test_workbench_target.py
+++ b/plugins/codex-security/tests/test_workbench_target.py
@@ -1,9 +1,10 @@
from __future__ import annotations
+import hashlib
import runpy
import subprocess
from pathlib import Path
-from typing import Callable, cast
+from typing import Any, Callable, cast
import pytest
from workbench_test_support import initialize_git_repository
@@ -13,6 +14,9 @@ WORKBENCH_TARGET = runpy.run_path(
)
directory_content_digest = cast(Callable[[Path], str], WORKBENCH_TARGET["directory_content_digest"])
worktree_content_digest = cast(Callable[[Path], str], WORKBENCH_TARGET["worktree_content_digest"])
+update_digest_field = cast(
+ Callable[[Any, bytes, bytes], None], WORKBENCH_TARGET["update_digest_field"]
+)
def initialize_unborn_git_repository(target: Path) -> None:
@@ -121,3 +125,55 @@ def test_directory_content_digest_skips_missing_cached_paths(tmp_path: Path) ->
cached_source.unlink()
assert directory_content_digest(target) == original_digest
+
+
+def test_worktree_content_digest_streams_tracked_binary_patch(
+ tmp_path: Path, monkeypatch: pytest.MonkeyPatch
+) -> None:
+ target = tmp_path / "target"
+ initialize_git_repository(target)
+ binary = target / "fixture.bin"
+ binary.write_bytes(bytes(range(256)) * 4)
+ subprocess.run(["git", "add", binary.name], cwd=target, check=True)
+ subprocess.run(["git", "commit", "-qm", "Add binary fixture"], cwd=target, check=True)
+ binary.write_bytes(bytes(reversed(range(256))) * 4)
+
+ tracked = subprocess.run(
+ [
+ "git",
+ "diff",
+ "--binary",
+ "--full-index",
+ "--no-ext-diff",
+ "--no-textconv",
+ "--ignore-submodules=none",
+ "HEAD",
+ "--",
+ ".",
+ ],
+ cwd=target,
+ check=True,
+ capture_output=True,
+ ).stdout
+ assert b"GIT binary patch" in tracked
+ expected = hashlib.sha256()
+ update_digest_field(expected, b"format", b"codex-security-snapshot/v1")
+ update_digest_field(expected, b"tracked-diff", tracked)
+
+ function_globals = cast(dict[str, Any], cast(Any, worktree_content_digest).__globals__)
+ git_command = cast(
+ Callable[..., subprocess.CompletedProcess[Any]], function_globals["git_command"]
+ )
+
+ def require_streamed_diff(
+ repository: Path, *args: str, **kwargs: object
+ ) -> subprocess.CompletedProcess[Any]:
+ if args and args[0] == "diff":
+ assert kwargs.get("stdout_file") is not None
+ return git_command(repository, *args, **kwargs)
+
+ monkeypatch.setitem(function_globals, "git_command", require_streamed_diff)
+
+ assert worktree_content_digest(target) == (
+ f"codex-security-snapshot/v1:sha256:{expected.hexdigest()}"
+ )