Skip to content

OpenAI's Codex Security CLI and TypeScript SDK for finding, validating, and fixing security vulnerabilities. npm: https://www.npmjs.com/package/@openai/codex-security

perf(history): reuse repository membership checks across scans (#2249)

Michael D'Angelo525543262e41
diff --git a/plugins/codex-security/scripts/workbench_scan_history.py b/plugins/codex-security/scripts/workbench_scan_history.py
index 090e9b0a3..b0f017e1a 100644
--- a/plugins/codex-security/scripts/workbench_scan_history.py
+++ b/plugins/codex-security/scripts/workbench_scan_history.py
@@ -8,6 +8,7 @@ import os
import sqlite3
import sys
from collections.abc import Iterable, Iterator
+from functools import cache
from itertools import chain
from pathlib import Path, PurePosixPath
from typing import Any, Callable
@@ -215,7 +216,7 @@ def _same_repository(
before: sqlite3.Row,
after: sqlite3.Row,
*,
- after_identity: tuple[str | None, tuple[str, str] | None] | None = None,
+ after_git: Callable[..., str | None] | None = None,
) -> bool:
if before["target_id"] is not None and before["target_id"] == after["target_id"]:
return True
@@ -223,28 +224,23 @@ def _same_repository(
after_target = Path(after["target_path"])
if before_target.resolve() == after_target.resolve():
return True
- before_git_dir = git_output(
- before_target, "rev-parse", "--path-format=absolute", "--git-common-dir"
- )
- after_git_dir = (
- git_output(after_target, "rev-parse", "--path-format=absolute", "--git-common-dir")
- if after_identity is None
- else after_identity[0]
- )
+ common_dir = ("rev-parse", "--path-format=absolute", "--git-common-dir")
+ after_git = after_git or git_output
+ before_git_dir = git_output(before_target, *common_dir)
if (
before_git_dir is not None
- and after_git_dir is not None
+ and (after_git_dir := after_git(after_target, *common_dir)) is not None
and Path(before_git_dir).resolve() == Path(after_git_dir).resolve()
):
return True
- before_origin = _repository_origin(before_target)
- return before_origin is not None and before_origin == (
- _repository_origin(after_target) if after_identity is None else after_identity[1]
+ origin = ("remote", "get-url", "origin")
+ before_origin = _repository_origin(git_output(before_target, *origin))
+ return before_origin is not None and before_origin == _repository_origin(
+ after_git(after_target, *origin)
)
-def _repository_origin(target: Path) -> tuple[str, str] | None:
- remote = git_output(target, "remote", "get-url", "origin")
+def _repository_origin(remote: str | None) -> tuple[str, str] | None:
if remote is None:
return None
if "://" in remote:
@@ -352,16 +348,13 @@ def list_scans(
""",
(str(repository), str(repository)),
).fetchone()
- requested_identity = (
- git_output(repository, "rev-parse", "--path-format=absolute", "--git-common-dir"),
- _repository_origin(repository),
- )
+ requested_git = cache(git_output)
related_target_ids = [
target["target_id"]
for target in connection.execute(
"SELECT id AS target_id, current_path AS target_path FROM security_targets"
)
- if _same_repository(target, requested_repository, after_identity=requested_identity)
+ if _same_repository(target, requested_repository, after_git=requested_git)
]
repository_clauses = ["scans.target_path = ?"]
values.append(str(repository))
@@ -452,11 +445,15 @@ def list_unmatched_scan_pairs(
""",
(str(repository), str(repository)),
).fetchone()
- selected = [
- scan
- for scan in connection.execute("SELECT * FROM scans WHERE status = 'complete'")
- if _same_repository(scan, requested)
- ]
+ selected = []
+ requested_git = cache(git_output)
+ membership: dict[tuple[str | None, str], bool] = {}
+ for scan in connection.execute("SELECT * FROM scans WHERE status = 'complete'"):
+ target = (scan["target_id"], scan["target_path"])
+ if target not in membership:
+ membership[target] = _same_repository(scan, requested, after_git=requested_git)
+ if membership[target]:
+ selected.append(scan)
selected.sort(key=lambda scan: (timestamp_key(scan["started_at"]), scan["id"]))
diff --git a/plugins/codex-security/tests/test_workbench_native_index_queries.py b/plugins/codex-security/tests/test_workbench_native_index_queries.py
index dc9697ce2..01a780079 100644
--- a/plugins/codex-security/tests/test_workbench_native_index_queries.py
+++ b/plugins/codex-security/tests/test_workbench_native_index_queries.py
@@ -364,11 +364,12 @@ def test_scan_list_probes_requested_repository_once(
connection, targets = indexed_collections
history = workbench_api["scan_history"]
probes = []
- git_output = history.git_output
def record_git_output(target, *arguments):
probes.append((target, arguments))
- return git_output(target, *arguments)
+ if arguments[0] == "rev-parse":
+ return str(target / ".git")
+ return f"https://example.test/{target.name}.git"
monkeypatch.setattr(history, "git_output", record_git_output)
result = history.list_scans(connection, query_args(repository=str(targets[0]), limit=1))
diff --git a/plugins/codex-security/tests/test_workbench_scan_history_queries.py b/plugins/codex-security/tests/test_workbench_scan_history_queries.py
index 9e5a3c316..f0d0925c5 100644
--- a/plugins/codex-security/tests/test_workbench_scan_history_queries.py
+++ b/plugins/codex-security/tests/test_workbench_scan_history_queries.py
@@ -360,6 +360,39 @@ def test_cached_group_status_follows_coverage_uncertainty_and_confirmed_links(
assert compare() == unlinked
+def test_matching_reuses_repository_membership_for_repeated_scans(
+ workbench_api, workbench_db, tmp_path, monkeypatch
+):
+ history = workbench_api["scan_history"]
+ repository = tmp_path / "current"
+ previous_path = tmp_path / "previous"
+ add_scan(workbench_db, "current", repository, target_id="selected")
+ add_scan(workbench_db, "moved", previous_path, target_id="selected")
+ add_scan(workbench_db, "legacy-current", repository)
+ add_scan(workbench_db, "legacy-previous", previous_path)
+ add_scan(workbench_db, "another-legacy-previous", previous_path)
+ add_scan(workbench_db, "clone", tmp_path / "clone")
+ probes = []
+
+ def probe(path, *arguments):
+ probes.append((path, *arguments))
+ if arguments[0] == "remote":
+ remote = "other" if path == previous_path else "current"
+ return f"https://example.test/{remote}.git"
+ return None
+
+ monkeypatch.setattr(history, "git_output", probe)
+ selected = []
+ history.list_unmatched_scan_pairs(
+ workbench_db,
+ Namespace(repository=str(repository), force=False),
+ backfill_finding_details=lambda _, scan: selected.append(scan["id"]),
+ read_coverage=lambda _: {},
+ )
+ assert sorted(selected) == ["clone", "current", "legacy-current", "moved"]
+ assert len(probes) == len(set(probes))
+
+
def test_matching_reads_each_scan_once_and_skips_cached_links(
workbench_api, workbench_db, tmp_path
):