OpenAI's Codex Security CLI and TypeScript SDK for finding, validating, and fixing security vulnerabilities. npm: https://www.npmjs.com/package/@openai/codex-security
TypeScript ai-securityapplication-securityclicode-scanningcodexcodex-securitycybersecuritydevsecopsnodejsnpmopenaisdksecuritytypescriptvulnerability-scanning
fix(plugin): stream tracked diffs while hashing (#773)
HughChawb1774f58cf40
diff --git a/plugins/codex-security/scripts/workbench_target.py b/plugins/codex-security/scripts/workbench_target.pyindex 47fcde677..b380a275f 100644--- a/plugins/codex-security/scripts/workbench_target.py+++ b/plugins/codex-security/scripts/workbench_target.py@@ -10,8 +10,9 @@ import sqlite3import statimport subprocessimport sys+import tempfilefrom pathlib import Path-from typing import Any+from typing import Any, BinaryIO# Some plugin hosts launch Python with safe-path isolation enabled.sys.path.insert(0, str(Path(__file__).resolve().parent))@@ -127,6 +128,7 @@ def git_command(input_data: str | bytes | None = None,git_dir: Path | None = None,work_tree: Path | None = None,+ stdout_file: BinaryIO | None = None,) -> subprocess.CompletedProcess[str] | subprocess.CompletedProcess[bytes]:if (git_dir is None) != (work_tree is None):raise ValueError("git_dir and work_tree must be provided together")@@ -148,15 +150,20 @@ def git_command(command.extend(["--git-dir", str(git_dir), "--work-tree", str(work_tree)])full_command = [*command, *args]try:+ output_options = (+ {"capture_output": True}+ if stdout_file is None+ else {"stdout": stdout_file, "stderr": subprocess.PIPE}+ )return subprocess.run(full_command,check=False,- capture_output=True,env=environment,text=text,encoding="utf-8" if text else None,errors="surrogateescape" if text else None,input=input_data,+ **output_options,)except FileNotFoundError:# Git is optional for Codebase scans. Treat an unavailable executable like@@ -165,13 +172,45 @@ def git_command(return subprocess.CompletedProcess(full_command, 127, empty_output, empty_output)-def update_digest_field(digest: Any, label: bytes, value: bytes) -> None:+def _update_digest_field_header(digest: Any, label: bytes, value_size: int) -> None:digest.update(len(label).to_bytes(4, "big"))digest.update(label)- digest.update(len(value).to_bytes(8, "big"))+ digest.update(value_size.to_bytes(8, "big"))+++def update_digest_field(digest: Any, label: bytes, value: bytes) -> None:+ _update_digest_field_header(digest, label, len(value))digest.update(value)+def _update_digest_field_from_git(+ digest: Any,+ label: bytes,+ target: Path,+ *args: str,+ git_dir: Path | None = None,+ work_tree: Path | None = None,+) -> bool:+ with tempfile.TemporaryFile() as output:+ completed = git_command(+ target,+ *args,+ text=False,+ git_dir=git_dir,+ work_tree=work_tree,+ stdout_file=output,+ )+ if completed.returncode != 0:+ return False+ output.seek(0, os.SEEK_END)+ value_size = output.tell()+ _update_digest_field_header(digest, label, value_size)+ output.seek(0)+ for chunk in iter(lambda: output.read(1024 * 1024), b""):+ digest.update(chunk)+ return True++def worktree_content_digest(target: Path) -> str:require_clean_submodule_worktrees(target)repository, pathspec = git_worktree_context(target)@@ -203,7 +242,11 @@ def worktree_content_digest_for_context(git_dir: Path | None = None,work_tree: Path | None = None,) -> str:- tracked = git_bytes(+ digest = hashlib.sha256()+ update_digest_field(digest, b"format", b"codex-security-snapshot/v1")+ tracked_ok = _update_digest_field_from_git(+ digest,+ b"tracked-diff",repository,"diff","--binary",@@ -228,11 +271,8 @@ def worktree_content_digest_for_context(git_dir=git_dir,work_tree=work_tree,)- if tracked is None or untracked is None:+ if not tracked_ok or untracked is None:raise SystemExit("Could not snapshot the selected working-tree changes.")- digest = hashlib.sha256()- update_digest_field(digest, b"format", b"codex-security-snapshot/v1")- update_digest_field(digest, b"tracked-diff", tracked)for raw_path in sorted(path for path in untracked.split(b"\0") if path):relative_path = os.fsdecode(raw_path)path = (work_tree or repository) / relative_pathdiff --git a/plugins/codex-security/tests/test_workbench_target.py b/plugins/codex-security/tests/test_workbench_target.pyindex a4e37753c..734dce876 100644--- a/plugins/codex-security/tests/test_workbench_target.py+++ b/plugins/codex-security/tests/test_workbench_target.py@@ -1,9 +1,10 @@from __future__ import annotations+import hashlibimport runpyimport subprocessfrom pathlib import Path-from typing import Callable, cast+from typing import Any, Callable, castimport pytestfrom workbench_test_support import initialize_git_repository@@ -13,6 +14,9 @@ WORKBENCH_TARGET = runpy.run_path()directory_content_digest = cast(Callable[[Path], str], WORKBENCH_TARGET["directory_content_digest"])worktree_content_digest = cast(Callable[[Path], str], WORKBENCH_TARGET["worktree_content_digest"])+update_digest_field = cast(+ Callable[[Any, bytes, bytes], None], WORKBENCH_TARGET["update_digest_field"]+)def initialize_unborn_git_repository(target: Path) -> None:@@ -121,3 +125,55 @@ def test_directory_content_digest_skips_missing_cached_paths(tmp_path: Path) ->cached_source.unlink()assert directory_content_digest(target) == original_digest+++def test_worktree_content_digest_streams_tracked_binary_patch(+ tmp_path: Path, monkeypatch: pytest.MonkeyPatch+) -> None:+ target = tmp_path / "target"+ initialize_git_repository(target)+ binary = target / "fixture.bin"+ binary.write_bytes(bytes(range(256)) * 4)+ subprocess.run(["git", "add", binary.name], cwd=target, check=True)+ subprocess.run(["git", "commit", "-qm", "Add binary fixture"], cwd=target, check=True)+ binary.write_bytes(bytes(reversed(range(256))) * 4)++ tracked = subprocess.run(+ [+ "git",+ "diff",+ "--binary",+ "--full-index",+ "--no-ext-diff",+ "--no-textconv",+ "--ignore-submodules=none",+ "HEAD",+ "--",+ ".",+ ],+ cwd=target,+ check=True,+ capture_output=True,+ ).stdout+ assert b"GIT binary patch" in tracked+ expected = hashlib.sha256()+ update_digest_field(expected, b"format", b"codex-security-snapshot/v1")+ update_digest_field(expected, b"tracked-diff", tracked)++ function_globals = cast(dict[str, Any], cast(Any, worktree_content_digest).__globals__)+ git_command = cast(+ Callable[..., subprocess.CompletedProcess[Any]], function_globals["git_command"]+ )++ def require_streamed_diff(+ repository: Path, *args: str, **kwargs: object+ ) -> subprocess.CompletedProcess[Any]:+ if args and args[0] == "diff":+ assert kwargs.get("stdout_file") is not None+ return git_command(repository, *args, **kwargs)++ monkeypatch.setitem(function_globals, "git_command", require_streamed_diff)++ assert worktree_content_digest(target) == (+ f"codex-security-snapshot/v1:sha256:{expected.hexdigest()}"+ )