Skip to content

OpenAI's Codex Security CLI and TypeScript SDK for finding, validating, and fixing security vulnerabilities. npm: https://www.npmjs.com/package/@openai/codex-security

fix(scans): count historical files for unreviewable local renames (#2101)

Michael D'Angelo14176380df0a
diff --git a/plugins/codex-security/mcp-app/src/helpers/inventory-git.ts b/plugins/codex-security/mcp-app/src/helpers/inventory-git.ts
index 94f28ebb1..6a3403a33 100644
--- a/plugins/codex-security/mcp-app/src/helpers/inventory-git.ts
+++ b/plugins/codex-security/mcp-app/src/helpers/inventory-git.ts
@@ -595,6 +595,7 @@ export async function directoryPaths(
export interface Change {
path: string;
status: string;
+ renameSource?: string;
}
async function changed(repo: string, args: string[]): Promise<Change[]> {
const result = await git(repo, [
@@ -623,12 +624,13 @@ async function changed(repo: string, args: string[]): Promise<Change[]> {
for (let index = 0; index < fields.length;) {
const metadata = fields[index++]!.split(" "),
status = metadata.at(-1)![0]!;
+ const renameSource = status === "R" ? fields[index] : undefined;
if (status === "C" || status === "R") index++;
const path = fields[index++]!;
if (
(status === "D" ? metadata[0]!.slice(1) : metadata[1]!).startsWith("100")
)
- changes.push({ path, status });
+ changes.push({ path, status, renameSource });
}
return changes;
}
@@ -643,26 +645,37 @@ export async function changedPaths(
staged = await changed(repo, ["--cached", base]);
const result = new Map(staged.map((change) => [change.path, change]));
for (const change of unstaged) result.set(change.path, change);
- for (const path of paths(
- requireSuccess(
- await git(repo, ["ls-files", "--others", "--exclude-standard", "-z"]),
- ),
- ))
- if (!path.endsWith("/")) result.set(path, { path, status: "A" });
- const selected: Change[] = [];
- for (const change of result.values()) {
- const { path, status } = change;
- if (path.split("/").some((part) => sourceMetadata.has(part))) continue;
+ function localStatus(path: string, status: string): string | undefined {
+ if (path.split("/").some((part) => sourceMetadata.has(part))) return;
+ if (status === "D") return status;
const absolute = append(repo, path);
try {
- if (status === "M" && linkedParent(repo, absolute))
- selected.push({ path, status: "D" });
- else if (status === "D" || regular(absolute)) selected.push(change);
+ if ((status === "M" || status === "R") && linkedParent(repo, absolute))
+ return status === "M" ? "D" : undefined;
+ return regular(absolute) ? status : undefined;
} catch (error) {
if (!isMissingPathError(error)) throw error;
}
}
- return selected;
+
+ const selected = new Map<string, string>();
+ for (const { path, status, renameSource } of result.values()) {
+ const chosen = localStatus(path, status);
+ if (chosen) selected.set(path, chosen);
+ else if (
+ renameSource !== undefined &&
+ !selected.has(renameSource) &&
+ localStatus(renameSource, "D")
+ )
+ selected.set(renameSource, "D");
+ }
+ for (const path of paths(
+ requireSuccess(
+ await git(repo, ["ls-files", "--others", "--exclude-standard", "-z"]),
+ ),
+ ))
+ if (!path.endsWith("/") && localStatus(path, "A")) selected.set(path, "A");
+ return [...selected].map(([path, status]) => ({ path, status }));
}
/** Consume each complete blob while retaining at most the preview prefix. */
diff --git a/plugins/codex-security/scripts/workbench_target.py b/plugins/codex-security/scripts/workbench_target.py
index 55ce17a60..15cfc6791 100644
--- a/plugins/codex-security/scripts/workbench_target.py
+++ b/plugins/codex-security/scripts/workbench_target.py
@@ -338,7 +338,7 @@ def git_command(
return subprocess.CompletedProcess(full_command, 127, empty_output, empty_output)
-def run_git_changed_paths(repo: Path, diff_args: list[str]) -> list[tuple[Path, str]]:
+def run_git_changed_paths(repo: Path, diff_args: list[str]) -> list[tuple[str, str, str | None]]:
"""Return changed regular files from the selected side of each change."""
result = git_command(
repo,
@@ -357,25 +357,29 @@ def run_git_changed_paths(repo: Path, diff_args: list[str]) -> list[tuple[Path,
if fields and not fields[-1]:
fields.pop()
- changed: list[tuple[Path, str]] = []
+ changed: list[tuple[str, str, str | None]] = []
index = 0
while index < len(fields):
metadata = fields[index].split()
status = chr(metadata[-1][0])
index += 1
+ rename_source = os.fsdecode(fields[index]) if status == "R" else None
if status in {"C", "R"}:
index += 1
- path = repo / os.fsdecode(fields[index])
+ path = os.fsdecode(fields[index])
index += 1
selected_mode = metadata[0].removeprefix(b":") if status == "D" else metadata[1]
if selected_mode.startswith(b"100"):
- changed.append((path, status))
+ changed.append((path, status, rename_source))
return changed
def git_changed_paths(repo: Path, base: str, head: str, mode: str) -> list[tuple[Path, str]]:
if mode == "revisions":
- return run_git_changed_paths(repo, [f"{base}..{head}"])
+ return [
+ (repo / name, status)
+ for name, status, _ in run_git_changed_paths(repo, [f"{base}..{head}"])
+ ]
if mode == "local-patch":
unstaged = run_git_changed_paths(repo, [base])
staged = run_git_changed_paths(repo, ["--cached", base])
@@ -389,26 +393,37 @@ def git_changed_paths(repo: Path, base: str, head: str, mode: str) -> list[tuple
offline=True,
)
untracked.check_returncode()
- combined = dict(staged)
- combined.update(unstaged)
- combined.update(
- (repo / os.fsdecode(relative), "A")
- for relative in untracked.stdout.split(b"\0")
- if relative and not relative.endswith(b"/")
- )
- selected = []
+ combined = {name: (status, source) for name, status, source in [*staged, *unstaged]}
+ selected: dict[str, str] = {}
linked_prefixes: dict[str, bool] = {}
- for path, status in combined.items():
- if not SOURCE_METADATA_DIRECTORIES.isdisjoint(path.relative_to(repo).parts):
- continue
+
+ def local_status(name: str, status: str) -> str | None:
+ if not SOURCE_METADATA_DIRECTORIES.isdisjoint(PurePosixPath(name).parts):
+ return None
+ if status == "D":
+ return status
+ path = repo / name
try:
- if status == "M" and _directory_link_boundary(repo, path, linked_prefixes) != path:
- selected.append((path, "D"))
- elif status == "D" or (not path.is_symlink() and path.is_file()):
- selected.append((path, status))
+ if (
+ status in {"M", "R"}
+ and _directory_link_boundary(repo, path, linked_prefixes) != path
+ ):
+ return "D" if status == "M" else None
+ return status if stat.S_ISREG(path.lstat().st_mode) else None
except (FileNotFoundError, NotADirectoryError):
- continue
- return sorted(selected)
+ return None
+
+ for name, (status, source) in combined.items():
+ if chosen := local_status(name, status):
+ selected[name] = chosen
+ elif source is not None and local_status(source, "D"):
+ selected.setdefault(source, "D")
+ for relative in untracked.stdout.split(b"\0"):
+ if relative and not relative.endswith(b"/"):
+ name = os.fsdecode(relative)
+ if local_status(name, "A"):
+ selected[name] = "A"
+ return sorted((repo / name, status) for name, status in selected.items())
raise SystemExit(f"Unknown diff mode: {mode}")
diff --git a/plugins/codex-security/tests/test_workbench_target.py b/plugins/codex-security/tests/test_workbench_target.py
index e766e4af1..1c4b4dad7 100644
--- a/plugins/codex-security/tests/test_workbench_target.py
+++ b/plugins/codex-security/tests/test_workbench_target.py
@@ -29,6 +29,48 @@ update_digest_field = cast(
CASE_ALIAS = pytest.param("case", marks=pytest.mark.native_macos)
+ ("destination", "missing", "binary", "selected", "count"),
+ [
+ ("new.py", False, False, ("new.py", "R"), 1),
+ ("new.py", True, False, ("old.py", "D"), 1),
+ ("new.py", True, True, ("old.py", "D"), 0),
+ (".svn/new.py", False, False, ("old.py", "D"), 1),
+ ],
+)
+def test_register_local_diff_with_renamed_source(
+ tmp_path: Path,
+ destination: str,
+ missing: bool,
+ binary: bool,
+ selected: tuple[str, str],
+ count: int,
+) -> None:
+ import workbench_db
+ import workbench_target
+
+ target = tmp_path / "repository"
+ initialize_git_repository(target)
+ (target / "old.py").write_bytes(b"\0binary" if binary else b"historical text\n")
+ subprocess.run(["git", "add", "old.py"], cwd=target, check=True)
+ subprocess.run(["git", "commit", "-qm", "add source"], cwd=target, check=True)
+ subprocess.run(["git", "config", "diff.renames", "true"], cwd=target, check=True)
+ (target / destination).parent.mkdir(exist_ok=True)
+ subprocess.run(["git", "mv", "old.py", destination], cwd=target, check=True)
+ if missing:
+ subprocess.run(
+ ["git", "update-index", "--skip-worktree", destination], cwd=target, check=True
+ )
+ (target / destination).unlink()
+
+ assert workbench_target.git_changed_paths(target, "HEAD", "HEAD", "local-patch") == [
+ (target / selected[0], selected[1])
+ ]
+ diff = {"kind": "working_tree", "baseRevision": "HEAD", "headRevision": "HEAD"}
+ assert workbench_db.scan_scope_file_count(target, ["."], diff) == count
+
+
@pytest.fixture(
params=[
pytest.param(False, id="emulated"),
diff --git a/sdk/typescript/tests-ts/inventory-input.test.ts b/sdk/typescript/tests-ts/inventory-input.test.ts
index 173c84cbe..89fb8b80c 100644
--- a/sdk/typescript/tests-ts/inventory-input.test.ts
+++ b/sdk/typescript/tests-ts/inventory-input.test.ts
@@ -858,6 +858,59 @@ for (const mode of ["revisions", "local-patch"]) {
});
}
+test("local diff retains the historical text when a rename destination cannot be reviewed", () => {
+ const f = fixture();
+ f.write("moved.py", "still selectable\n");
+ f.write("removed.py", "destination missing\n");
+ f.write("excluded.py", "destination excluded\n");
+ f.write("linked.py", "destination replaced by link\n");
+ f.write("binary", Buffer.from("\0historical binary"));
+ const base = f.commit();
+ git(f.repo, "config", "diff.renames", "true");
+ mkdirSync(join(f.repo, ".svn"));
+ for (const [source, destination] of [
+ ["moved.py", "live.py"],
+ ["removed.py", "missing.py"],
+ ["excluded.py", ".svn/file.py"],
+ ["linked.py", "link"],
+ ["binary", "missing.bin"],
+ ] as const) {
+ git(f.repo, "mv", source, destination);
+ }
+ git(f.repo, "update-index", "--skip-worktree", "missing.py", "missing.bin");
+ rmSync(join(f.repo, "missing.py"));
+ rmSync(join(f.repo, "missing.bin"));
+ rmSync(join(f.repo, "link"));
+ const outside = join(f.root, "outside");
+ mkdirSync(outside);
+ symlinkSync(outside, join(f.repo, "link"), "junction");
+
+ const expected = ["excluded.py", "linked.py", "live.py", "removed.py"];
+ const ranked = f.rows("make-diff-rank-input", [
+ "--base",
+ base,
+ "--mode",
+ "local-patch",
+ ]);
+ expect(ranked.map(({ path }) => path)).toEqual(expected);
+ expect(ranked.map(({ preview }) => preview)).toEqual([
+ "",
+ "",
+ "still selectable",
+ "",
+ ]);
+ expect(
+ f.success("generate-in-scope-files", [
+ "--scope",
+ ".",
+ "--diff-base",
+ base,
+ "--diff-mode",
+ "local-patch",
+ ]),
+ ).toBe(expected.join("\n") + "\n");
+});
+
for (const [mode, undoStagedChange] of [
["revisions", false],
["local-patch", false],