OpenAI's Codex Security CLI and TypeScript SDK for finding, validating, and fixing security vulnerabilities. npm: https://www.npmjs.com/package/@openai/codex-security
TypeScript ai-securityapplication-securityclicode-scanningcodexcodex-securitycybersecuritydevsecopsnodejsnpmopenaisdksecuritytypescriptvulnerability-scanning
fix(scans): count historical files for unreviewable local renames (#2101)
Michael D'Angelo14176380df0a
diff --git a/plugins/codex-security/mcp-app/src/helpers/inventory-git.ts b/plugins/codex-security/mcp-app/src/helpers/inventory-git.tsindex 94f28ebb1..6a3403a33 100644--- a/plugins/codex-security/mcp-app/src/helpers/inventory-git.ts+++ b/plugins/codex-security/mcp-app/src/helpers/inventory-git.ts@@ -595,6 +595,7 @@ export async function directoryPaths(export interface Change {path: string;status: string;+ renameSource?: string;}async function changed(repo: string, args: string[]): Promise<Change[]> {const result = await git(repo, [@@ -623,12 +624,13 @@ async function changed(repo: string, args: string[]): Promise<Change[]> {for (let index = 0; index < fields.length;) {const metadata = fields[index++]!.split(" "),status = metadata.at(-1)![0]!;+ const renameSource = status === "R" ? fields[index] : undefined;if (status === "C" || status === "R") index++;const path = fields[index++]!;if ((status === "D" ? metadata[0]!.slice(1) : metadata[1]!).startsWith("100"))- changes.push({ path, status });+ changes.push({ path, status, renameSource });}return changes;}@@ -643,26 +645,37 @@ export async function changedPaths(staged = await changed(repo, ["--cached", base]);const result = new Map(staged.map((change) => [change.path, change]));for (const change of unstaged) result.set(change.path, change);- for (const path of paths(- requireSuccess(- await git(repo, ["ls-files", "--others", "--exclude-standard", "-z"]),- ),- ))- if (!path.endsWith("/")) result.set(path, { path, status: "A" });- const selected: Change[] = [];- for (const change of result.values()) {- const { path, status } = change;- if (path.split("/").some((part) => sourceMetadata.has(part))) continue;+ function localStatus(path: string, status: string): string | undefined {+ if (path.split("/").some((part) => sourceMetadata.has(part))) return;+ if (status === "D") return status;const absolute = append(repo, path);try {- if (status === "M" && linkedParent(repo, absolute))- selected.push({ path, status: "D" });- else if (status === "D" || regular(absolute)) selected.push(change);+ if ((status === "M" || status === "R") && linkedParent(repo, absolute))+ return status === "M" ? "D" : undefined;+ return regular(absolute) ? status : undefined;} catch (error) {if (!isMissingPathError(error)) throw error;}}- return selected;++ const selected = new Map<string, string>();+ for (const { path, status, renameSource } of result.values()) {+ const chosen = localStatus(path, status);+ if (chosen) selected.set(path, chosen);+ else if (+ renameSource !== undefined &&+ !selected.has(renameSource) &&+ localStatus(renameSource, "D")+ )+ selected.set(renameSource, "D");+ }+ for (const path of paths(+ requireSuccess(+ await git(repo, ["ls-files", "--others", "--exclude-standard", "-z"]),+ ),+ ))+ if (!path.endsWith("/") && localStatus(path, "A")) selected.set(path, "A");+ return [...selected].map(([path, status]) => ({ path, status }));}/** Consume each complete blob while retaining at most the preview prefix. */diff --git a/plugins/codex-security/scripts/workbench_target.py b/plugins/codex-security/scripts/workbench_target.pyindex 55ce17a60..15cfc6791 100644--- a/plugins/codex-security/scripts/workbench_target.py+++ b/plugins/codex-security/scripts/workbench_target.py@@ -338,7 +338,7 @@ def git_command(return subprocess.CompletedProcess(full_command, 127, empty_output, empty_output)-def run_git_changed_paths(repo: Path, diff_args: list[str]) -> list[tuple[Path, str]]:+def run_git_changed_paths(repo: Path, diff_args: list[str]) -> list[tuple[str, str, str | None]]:"""Return changed regular files from the selected side of each change."""result = git_command(repo,@@ -357,25 +357,29 @@ def run_git_changed_paths(repo: Path, diff_args: list[str]) -> list[tuple[Path,if fields and not fields[-1]:fields.pop()- changed: list[tuple[Path, str]] = []+ changed: list[tuple[str, str, str | None]] = []index = 0while index < len(fields):metadata = fields[index].split()status = chr(metadata[-1][0])index += 1+ rename_source = os.fsdecode(fields[index]) if status == "R" else Noneif status in {"C", "R"}:index += 1- path = repo / os.fsdecode(fields[index])+ path = os.fsdecode(fields[index])index += 1selected_mode = metadata[0].removeprefix(b":") if status == "D" else metadata[1]if selected_mode.startswith(b"100"):- changed.append((path, status))+ changed.append((path, status, rename_source))return changeddef git_changed_paths(repo: Path, base: str, head: str, mode: str) -> list[tuple[Path, str]]:if mode == "revisions":- return run_git_changed_paths(repo, [f"{base}..{head}"])+ return [+ (repo / name, status)+ for name, status, _ in run_git_changed_paths(repo, [f"{base}..{head}"])+ ]if mode == "local-patch":unstaged = run_git_changed_paths(repo, [base])staged = run_git_changed_paths(repo, ["--cached", base])@@ -389,26 +393,37 @@ def git_changed_paths(repo: Path, base: str, head: str, mode: str) -> list[tupleoffline=True,)untracked.check_returncode()- combined = dict(staged)- combined.update(unstaged)- combined.update(- (repo / os.fsdecode(relative), "A")- for relative in untracked.stdout.split(b"\0")- if relative and not relative.endswith(b"/")- )- selected = []+ combined = {name: (status, source) for name, status, source in [*staged, *unstaged]}+ selected: dict[str, str] = {}linked_prefixes: dict[str, bool] = {}- for path, status in combined.items():- if not SOURCE_METADATA_DIRECTORIES.isdisjoint(path.relative_to(repo).parts):- continue++ def local_status(name: str, status: str) -> str | None:+ if not SOURCE_METADATA_DIRECTORIES.isdisjoint(PurePosixPath(name).parts):+ return None+ if status == "D":+ return status+ path = repo / nametry:- if status == "M" and _directory_link_boundary(repo, path, linked_prefixes) != path:- selected.append((path, "D"))- elif status == "D" or (not path.is_symlink() and path.is_file()):- selected.append((path, status))+ if (+ status in {"M", "R"}+ and _directory_link_boundary(repo, path, linked_prefixes) != path+ ):+ return "D" if status == "M" else None+ return status if stat.S_ISREG(path.lstat().st_mode) else Noneexcept (FileNotFoundError, NotADirectoryError):- continue- return sorted(selected)+ return None++ for name, (status, source) in combined.items():+ if chosen := local_status(name, status):+ selected[name] = chosen+ elif source is not None and local_status(source, "D"):+ selected.setdefault(source, "D")+ for relative in untracked.stdout.split(b"\0"):+ if relative and not relative.endswith(b"/"):+ name = os.fsdecode(relative)+ if local_status(name, "A"):+ selected[name] = "A"+ return sorted((repo / name, status) for name, status in selected.items())raise SystemExit(f"Unknown diff mode: {mode}")diff --git a/plugins/codex-security/tests/test_workbench_target.py b/plugins/codex-security/tests/test_workbench_target.pyindex e766e4af1..1c4b4dad7 100644--- a/plugins/codex-security/tests/test_workbench_target.py+++ b/plugins/codex-security/tests/test_workbench_target.py@@ -29,6 +29,48 @@ update_digest_field = cast(CASE_ALIAS = pytest.param("case", marks=pytest.mark.native_macos)[email protected]_platform+ ("destination", "missing", "binary", "selected", "count"),+ [+ ("new.py", False, False, ("new.py", "R"), 1),+ ("new.py", True, False, ("old.py", "D"), 1),+ ("new.py", True, True, ("old.py", "D"), 0),+ (".svn/new.py", False, False, ("old.py", "D"), 1),+ ],+)+def test_register_local_diff_with_renamed_source(+ tmp_path: Path,+ destination: str,+ missing: bool,+ binary: bool,+ selected: tuple[str, str],+ count: int,+) -> None:+ import workbench_db+ import workbench_target++ target = tmp_path / "repository"+ initialize_git_repository(target)+ (target / "old.py").write_bytes(b"\0binary" if binary else b"historical text\n")+ subprocess.run(["git", "add", "old.py"], cwd=target, check=True)+ subprocess.run(["git", "commit", "-qm", "add source"], cwd=target, check=True)+ subprocess.run(["git", "config", "diff.renames", "true"], cwd=target, check=True)+ (target / destination).parent.mkdir(exist_ok=True)+ subprocess.run(["git", "mv", "old.py", destination], cwd=target, check=True)+ if missing:+ subprocess.run(+ ["git", "update-index", "--skip-worktree", destination], cwd=target, check=True+ )+ (target / destination).unlink()++ assert workbench_target.git_changed_paths(target, "HEAD", "HEAD", "local-patch") == [+ (target / selected[0], selected[1])+ ]+ diff = {"kind": "working_tree", "baseRevision": "HEAD", "headRevision": "HEAD"}+ assert workbench_db.scan_scope_file_count(target, ["."], diff) == count++@pytest.fixture(params=[pytest.param(False, id="emulated"),diff --git a/sdk/typescript/tests-ts/inventory-input.test.ts b/sdk/typescript/tests-ts/inventory-input.test.tsindex 173c84cbe..89fb8b80c 100644--- a/sdk/typescript/tests-ts/inventory-input.test.ts+++ b/sdk/typescript/tests-ts/inventory-input.test.ts@@ -858,6 +858,59 @@ for (const mode of ["revisions", "local-patch"]) {});}+test("local diff retains the historical text when a rename destination cannot be reviewed", () => {+ const f = fixture();+ f.write("moved.py", "still selectable\n");+ f.write("removed.py", "destination missing\n");+ f.write("excluded.py", "destination excluded\n");+ f.write("linked.py", "destination replaced by link\n");+ f.write("binary", Buffer.from("\0historical binary"));+ const base = f.commit();+ git(f.repo, "config", "diff.renames", "true");+ mkdirSync(join(f.repo, ".svn"));+ for (const [source, destination] of [+ ["moved.py", "live.py"],+ ["removed.py", "missing.py"],+ ["excluded.py", ".svn/file.py"],+ ["linked.py", "link"],+ ["binary", "missing.bin"],+ ] as const) {+ git(f.repo, "mv", source, destination);+ }+ git(f.repo, "update-index", "--skip-worktree", "missing.py", "missing.bin");+ rmSync(join(f.repo, "missing.py"));+ rmSync(join(f.repo, "missing.bin"));+ rmSync(join(f.repo, "link"));+ const outside = join(f.root, "outside");+ mkdirSync(outside);+ symlinkSync(outside, join(f.repo, "link"), "junction");++ const expected = ["excluded.py", "linked.py", "live.py", "removed.py"];+ const ranked = f.rows("make-diff-rank-input", [+ "--base",+ base,+ "--mode",+ "local-patch",+ ]);+ expect(ranked.map(({ path }) => path)).toEqual(expected);+ expect(ranked.map(({ preview }) => preview)).toEqual([+ "",+ "",+ "still selectable",+ "",+ ]);+ expect(+ f.success("generate-in-scope-files", [+ "--scope",+ ".",+ "--diff-base",+ base,+ "--diff-mode",+ "local-patch",+ ]),+ ).toBe(expected.join("\n") + "\n");+});+for (const [mode, undoStagedChange] of [["revisions", false],["local-patch", false],