Skip to content

OpenAI's Codex Security CLI and TypeScript SDK for finding, validating, and fixing security vulnerabilities. npm: https://www.npmjs.com/package/@openai/codex-security

fix(index): include unscanned repository targets (#547)

Sylvester Kaczmarekdb981914b195
diff --git a/plugins/codex-security/scripts/workbench_native_indexes.py b/plugins/codex-security/scripts/workbench_native_indexes.py
index e473bbe2a..8f333c6f9 100644
--- a/plugins/codex-security/scripts/workbench_native_indexes.py
+++ b/plugins/codex-security/scripts/workbench_native_indexes.py
@@ -1,6 +1,7 @@
"""Read-only native findings and repository indexes for the Security workbench."""
import argparse
+import os
import sqlite3
import sys
from collections import Counter
@@ -204,63 +205,74 @@ def list_repositories(
args: argparse.Namespace | None = None,
) -> dict[str, Any]:
scan_count_by_target = Counter()
- latest_scan_by_target: dict[str, dict[str, Any]] = {}
- for scan in (
- scan_history.scan_summary(row)
- for row in connection.execute(
- f"""{scan_history.SCAN_SUMMARY_QUERY}
- ORDER BY julianday(upper(scans.started_at)) DESC, scans.id DESC
- """
- ).fetchall()
- ):
- scan_count_by_target[scan["targetId"]] += 1
- latest_scan_by_target.setdefault(scan["targetId"], scan)
+ latest_by_target: dict[str, tuple[sqlite3.Row, dict[str, Any] | None]] = {}
+ for row in connection.execute(
+ f"""
+ SELECT targets.id AS repository_id, targets.display_name, targets.current_path,
+ {scan_history.SCAN_SUMMARY_COLUMNS}
+ FROM security_targets AS targets
+ LEFT JOIN scans ON scans.target_id = targets.id
+ AND scans.id IN (SELECT scan_id FROM scan_progress)
+ LEFT JOIN scan_progress AS progress ON progress.scan_id = scans.id
+ ORDER BY julianday(upper(scans.started_at)) DESC, scans.id DESC, targets.id
+ """
+ ).fetchall():
+ scan = scan_history.scan_summary(row) if row["id"] is not None else None
+ target_id = row["repository_id"]
+ if scan is not None:
+ scan_count_by_target[target_id] += 1
+ latest_by_target.setdefault(target_id, (row, scan))
open_findings_by_target = Counter(
row["target_id"] for row in _indexed_findings(connection) if row["status"] == "open"
)
- targets = {row["id"]: row for row in connection.execute("SELECT * FROM security_targets")}
repositories = [
{
- "checkoutAvailable": Path(target["current_path"]).is_dir(),
"displayName": target["display_name"],
"latestScan": latest_scan,
- "openFindingsCount": open_findings_by_target.get(target_id, 0),
+ "openFindingsCount": open_findings_by_target[target_id]
+ if latest_scan is not None
+ else 0,
"scanCount": scan_count_by_target[target_id],
"targetId": target_id,
"targetPath": target["current_path"],
}
- for target_id, latest_scan in latest_scan_by_target.items()
- if (target := targets.get(target_id)) is not None
- ]
- if args is None:
- return {"repositories": repositories}
-
- query = args.query.strip().casefold() if args.query else ""
- repositories = [
- repository
- for repository in repositories
- if (args.target_id is None or repository["targetId"] == args.target_id)
- and args.status != "not_scanned"
- and (args.status != "open_findings" or repository["openFindingsCount"] > 0)
- and (
- not query
- or query in repository["displayName"].casefold()
- or query in repository["targetPath"].casefold()
- )
+ for target_id, (target, latest_scan) in latest_by_target.items()
]
- if args.limit is None and args.offset == 0:
- return {"repositories": repositories}
-
- limit = min(args.limit or FINDINGS_PAGE_MAX, FINDINGS_PAGE_MAX)
- page = repositories[args.offset : args.offset + limit]
- next_offset = args.offset + len(page)
- return {
- "repositories": page,
- "limit": limit,
- "nextOffset": next_offset if next_offset < len(repositories) else None,
- "offset": args.offset,
- }
+ pagination = {}
+ if args is not None:
+ query = args.query.strip().casefold() if args.query else ""
+ repositories = [
+ repository
+ for repository in repositories
+ if (args.target_id is None or repository["targetId"] == args.target_id)
+ and (args.status != "scanned" or repository["scanCount"] > 0)
+ and (args.status != "not_scanned" or repository["scanCount"] == 0)
+ and (args.status != "open_findings" or repository["openFindingsCount"] > 0)
+ and (
+ not query
+ or query in repository["displayName"].casefold()
+ or query in repository["targetPath"].casefold()
+ )
+ ]
+ if args.limit is not None or args.offset:
+ limit = min(args.limit or FINDINGS_PAGE_MAX, FINDINGS_PAGE_MAX)
+ end = args.offset + limit
+ pagination = {
+ "limit": limit,
+ "nextOffset": end if end < len(repositories) else None,
+ "offset": args.offset,
+ }
+ repositories = repositories[args.offset : end]
+ for repository in repositories:
+ path = os.path.expanduser(repository["targetPath"])
+ try:
+ repository["checkoutAvailable"] = Path(path).is_absolute() and os.path.isdir(
+ os.path.realpath(path)
+ )
+ except (OSError, RecursionError):
+ repository["checkoutAvailable"] = False
+ return {"repositories": repositories, **pagination}
if __name__ == "__main__":
diff --git a/plugins/codex-security/scripts/workbench_scan_history.py b/plugins/codex-security/scripts/workbench_scan_history.py
index 69aa02015..eb1c56261 100644
--- a/plugins/codex-security/scripts/workbench_scan_history.py
+++ b/plugins/codex-security/scripts/workbench_scan_history.py
@@ -265,8 +265,7 @@ def _repository_origin(target: Path) -> tuple[str, str] | None:
return (host.lower(), path) if host and path else None
-SCAN_SUMMARY_QUERY = """
-SELECT
+SCAN_SUMMARY_COLUMNS = """
scans.*,
progress.reportable_findings_count,
progress.scope_file_count,
@@ -278,6 +277,9 @@ SELECT
FROM finding_occurrences AS occurrences
WHERE occurrences.scan_id = scans.id
) AS finding_count
+"""
+SCAN_SUMMARY_QUERY = f"""
+SELECT {SCAN_SUMMARY_COLUMNS}
FROM scans
JOIN scan_progress AS progress ON progress.scan_id = scans.id
"""
diff --git a/plugins/codex-security/tests/test_workbench_native_index_queries.py b/plugins/codex-security/tests/test_workbench_native_index_queries.py
index 47b1c8027..dc9697ce2 100644
--- a/plugins/codex-security/tests/test_workbench_native_index_queries.py
+++ b/plugins/codex-security/tests/test_workbench_native_index_queries.py
@@ -5,6 +5,7 @@ import sqlite3
from argparse import Namespace
from contextlib import closing
from types import SimpleNamespace
+from unittest.mock import Mock
import pytest
from workbench_test_support import stable_target_id
@@ -379,6 +380,32 @@ def test_scan_list_probes_requested_repository_once(
]
+def test_unscanned_repositories_follow_scanned_rows_and_paginate(
+ workbench_api, indexed_collections, tmp_path, monkeypatch
+):
+ connection, _ = indexed_collections
+ indexes = workbench_api["native_indexes"]
+ query = indexes.list_repositories
+ scanned = query(connection)["repositories"]
+ unscanned = sorted(
+ workbench_api["ensure_security_target"](connection, str(tmp_path / name))
+ for name in ("needle-cedar", "needle-oak")
+ )
+ repositories = query(connection)["repositories"]
+ assert repositories[: len(scanned)] == scanned
+ assert [row["targetId"] for row in repositories[len(scanned) :]] == unscanned
+ probe = Mock(wraps=indexes.os.path.isdir)
+ monkeypatch.setattr(indexes.os.path, "isdir", probe)
+ for offset, target_id in enumerate(unscanned):
+ page = query(
+ connection, query_args(status="not_scanned", query="NeEdLe", limit=1, offset=offset)
+ )
+ assert [row["targetId"] for row in page["repositories"]] == [target_id]
+ assert page["nextOffset"] == (1 if offset == 0 else None)
+ probe.assert_called_once_with(page["repositories"][0]["targetPath"])
+ probe.reset_mock()
+
+
@pytest.mark.parametrize(
("timestamps", "order"),
[
@@ -493,6 +520,20 @@ def test_repository_index_keeps_one_scan_cohort(
indexes = workbench_api["native_indexes"]
database = tmp_path / "history.sqlite"
new_id = "00000000-0000-4000-8000-999999999999"
+ # Hold one target's first completed scan and finding for the concurrent publication.
+ with source:
+ pending = [
+ (
+ table,
+ source.execute(f"SELECT * FROM {table} WHERE {key} = ?", (SCAN_IDS[1],)).fetchone(),
+ )
+ for table, key in (
+ ("scans", "id"),
+ ("scan_progress", "scan_id"),
+ ("finding_occurrences", "scan_id"),
+ )
+ ]
+ source.execute("DELETE FROM scans WHERE id = ?", (SCAN_IDS[1],))
with closing(sqlite3.connect(database)) as reader, closing(sqlite3.connect(database)) as writer:
source.backup(reader)
reader.row_factory = sqlite3.Row
@@ -503,10 +544,14 @@ def test_repository_index_keeps_one_scan_cohort(
def execute(sql, parameters=()):
cursor = reader.execute(sql, parameters)
- if "JOIN scan_progress AS progress" not in sql:
+ if "FROM security_targets AS targets" not in sql:
return cursor
- rows = cursor.fetchall()
with writer:
+ writer.execute(
+ "UPDATE security_targets SET display_name = 'renamed' "
+ "WHERE id = (SELECT target_id FROM scans WHERE id = ?)",
+ (SCAN_IDS[0],),
+ )
writer.execute(
"INSERT INTO scans (id, workspace_id, target_id, target_path, target_revision, "
"scope, mode, scan_dir, status, phase, started_at, created_at, updated_at) "
@@ -520,7 +565,11 @@ def test_repository_index_keeps_one_scan_cohort(
"SELECT ?, updated_at FROM scan_progress WHERE scan_id = ?",
(new_id, SCAN_IDS[0]),
)
- return SimpleNamespace(fetchall=lambda: rows)
+ for table, row in pending:
+ writer.execute(
+ f"INSERT INTO {table} VALUES ({','.join('?' for _ in row)})", row
+ )
+ return cursor
proxy = SimpleNamespace(execute=execute, create_function=reader.create_function)
assert indexes.list_repositories(proxy) == before
@@ -528,7 +577,10 @@ def test_repository_index_keeps_one_scan_cohort(
reader.commit()
after = indexes.list_repositories(reader)["repositories"]
assert after[0]["latestScan"]["scanId"] == new_id
+ assert after[0]["displayName"] == "renamed"
assert after[0]["scanCount"] == 2
+ first = next(row for row in after if row["latestScan"]["scanId"] == SCAN_IDS[1])
+ assert (first["scanCount"], first["openFindingsCount"]) == (1, 1)
def test_repository_latest_keeps_sqlite_millisecond_ties(workbench_api, indexed_collections):
@@ -554,9 +606,15 @@ def test_repository_summary_excludes_scans_without_progress(workbench_api, index
target_id = stable_target_id(targets[0])
connection.execute("UPDATE scans SET target_id = ? WHERE id = ?", (target_id, SCAN_IDS[1]))
connection.execute("DELETE FROM scan_progress WHERE scan_id = ?", (SCAN_IDS[1],))
- repositories = workbench_api["native_indexes"].list_repositories(
- connection, query_args(target_id=target_id)
- )["repositories"]
+ query = workbench_api["native_indexes"].list_repositories
+ repositories = query(connection, query_args(target_id=target_id))["repositories"]
assert len(repositories) == 1
assert repositories[0]["latestScan"]["scanId"] == SCAN_IDS[0]
assert repositories[0]["scanCount"] == 1
+ connection.execute("DELETE FROM scan_progress WHERE scan_id = ?", (SCAN_IDS[0],))
+ (unscanned,) = query(connection, query_args(target_id=target_id, status="not_scanned"))[
+ "repositories"
+ ]
+ assert unscanned["targetId"] == target_id
+ assert unscanned["scanCount"] == unscanned["openFindingsCount"] == 0
+ assert unscanned["latestScan"] is None
diff --git a/plugins/codex-security/tests/test_workbench_native_indexes.py b/plugins/codex-security/tests/test_workbench_native_indexes.py
index db44cb46a..3dda22264 100644
--- a/plugins/codex-security/tests/test_workbench_native_indexes.py
+++ b/plugins/codex-security/tests/test_workbench_native_indexes.py
@@ -7,6 +7,7 @@ from pathlib import Path
import pytest
from workbench_test_support import (
create_saved_workspace,
+ create_workspace,
run_workbench,
save_workspace,
scan_command,
@@ -189,6 +190,30 @@ def test_global_findings_keep_latest_occurrence_and_stable_target_identity(tmp_p
assert second["status"] == "open"
+ ("target_path", "available"), [(".", False), ("~/repo", True), ("~/missing/../repo", True)]
+)
+def test_repository_index_probes_draft_checkout(
+ tmp_path: Path, monkeypatch, target_path: str, available: bool
+) -> None:
+ state_dir = tmp_path / "state"
+ (tmp_path / "repo").mkdir()
+ monkeypatch.setenv("HOME", str(tmp_path))
+ monkeypatch.setenv("USERPROFILE", str(tmp_path))
+ create_workspace(
+ state_dir,
+ "20000000-0000-4000-8000-000000000000",
+ "--target-path",
+ target_path,
+ "--scope",
+ "missing-directory",
+ )
+
+ (repository,) = run_workbench(state_dir, "list-repositories")["repositories"]
+ assert repository["targetPath"] == target_path
+ assert repository["checkoutAvailable"] is available
+
+
def test_repository_index_reports_latest_scan_open_findings_and_missing_checkout(
tmp_path: Path,
) -> None:
@@ -200,7 +225,30 @@ def test_repository_index_reports_latest_scan_open_findings_and_missing_checkout
second_target.mkdir()
first_target_id = stable_target_id(first_target)
second_target_id = stable_target_id(second_target)
+ create_workspace(
+ state_dir, "20000000-0000-4000-8000-000000000000", "--target-path", str(first_target)
+ )
+ (unscanned,) = run_workbench(state_dir, "list-repositories", "--status", "not_scanned")[
+ "repositories"
+ ]
+ assert unscanned == {
+ "checkoutAvailable": True,
+ "displayName": first_target.name,
+ "latestScan": None,
+ "openFindingsCount": 0,
+ "scanCount": 0,
+ "targetId": first_target_id,
+ "targetPath": str(first_target.resolve()),
+ }
+ for status in ("scanned", "open_findings"):
+ assert (
+ run_workbench(state_dir, "list-repositories", "--status", status)["repositories"] == []
+ )
older_first = complete_scan(state_dir, first_target, identity_anchor="first-finding")
+ assert (
+ run_workbench(state_dir, "list-repositories", "--status", "not_scanned")["repositories"]
+ == []
+ )
set_triage(
state_dir,
str(older_first["findings"][0]["occurrenceId"]),
diff --git a/sdk/typescript/src/cli.ts b/sdk/typescript/src/cli.ts
index 5ac6869d9..6d74e42f9 100644
--- a/sdk/typescript/src/cli.ts
+++ b/sdk/typescript/src/cli.ts
@@ -2279,7 +2279,7 @@ export async function main(
);
return presentHistory(
await history(
- ["list-repositories"],
+ ["list-repositories", "--status", "scanned"],
async (value): Promise<JsonObject> => {
const repositories = value["repositories"] as JsonObject[];
let target = repositories.find(
diff --git a/sdk/typescript/tests-ts/cli-workbench.test.ts b/sdk/typescript/tests-ts/cli-workbench.test.ts
index e14000a6f..40e99684b 100644
--- a/sdk/typescript/tests-ts/cli-workbench.test.ts
+++ b/sdk/typescript/tests-ts/cli-workbench.test.ts
@@ -80,7 +80,7 @@ describe("CLI workbench", () => {
),
).toBe(0);
expect(calls).toEqual([
- ["list-repositories"],
+ ["list-repositories", "--status", "scanned"],
...(includeMatchingAlias
? [
[
@@ -190,7 +190,7 @@ describe("CLI workbench", () => {
}
});
- test("findings list prefers an exact saved repository before directory aliases", async () => {
+ test("findings list prefers an exact scanned repository before directory aliases", async () => {
const root = await temporaryDirectory("finding-exact-repository-");
try {
const repository = join(root, "repository");
@@ -211,7 +211,7 @@ describe("CLI workbench", () => {
[repository, alias, repository],
[alias, repository, alias],
[unsavedAlias, alias, repository],
- ]) {
+ ] as const) {
for (const args of [[], [requested!]]) {
const calls: Array<readonly string[]> = [];
const stdout = captureCli(main, "stdout");
@@ -225,8 +225,16 @@ describe("CLI workbench", () => {
return args[0] === "list-repositories"
? {
repositories: [
- { targetId: "alias-target", targetPath: first! },
- { targetId: "exact-target", targetPath: exact! },
+ { targetId: "alias-target", targetPath: first },
+ { targetId: "exact-target", targetPath: exact },
+ ...(args.includes("scanned")
+ ? []
+ : [
+ {
+ targetId: "unscanned",
+ targetPath: unsavedAlias,
+ },
+ ]),
],
}
: {
@@ -361,7 +369,7 @@ describe("CLI workbench", () => {
}),
),
).toBe(0);
- expect(calls[0]).toEqual(["list-repositories"]);
+ expect(calls[0]).toEqual(["list-repositories", "--status", "scanned"]);
expect(calls[1]).toEqual([
"list-global-findings",
"--target-id",