OpenAI's Codex Security CLI and TypeScript SDK for finding, validating, and fixing security vulnerabilities. npm: https://www.npmjs.com/package/@openai/codex-security
TypeScript ai-securityapplication-securityclicode-scanningcodexcodex-securitycybersecuritydevsecopsnodejsnpmopenaisdksecuritytypescriptvulnerability-scanning
fix(index): include unscanned repository targets (#547)
Sylvester Kaczmarekdb981914b195
diff --git a/plugins/codex-security/scripts/workbench_native_indexes.py b/plugins/codex-security/scripts/workbench_native_indexes.pyindex e473bbe2a..8f333c6f9 100644--- a/plugins/codex-security/scripts/workbench_native_indexes.py+++ b/plugins/codex-security/scripts/workbench_native_indexes.py@@ -1,6 +1,7 @@"""Read-only native findings and repository indexes for the Security workbench."""import argparse+import osimport sqlite3import sysfrom collections import Counter@@ -204,63 +205,74 @@ def list_repositories(args: argparse.Namespace | None = None,) -> dict[str, Any]:scan_count_by_target = Counter()- latest_scan_by_target: dict[str, dict[str, Any]] = {}- for scan in (- scan_history.scan_summary(row)- for row in connection.execute(- f"""{scan_history.SCAN_SUMMARY_QUERY}- ORDER BY julianday(upper(scans.started_at)) DESC, scans.id DESC- """- ).fetchall()- ):- scan_count_by_target[scan["targetId"]] += 1- latest_scan_by_target.setdefault(scan["targetId"], scan)+ latest_by_target: dict[str, tuple[sqlite3.Row, dict[str, Any] | None]] = {}+ for row in connection.execute(+ f"""+ SELECT targets.id AS repository_id, targets.display_name, targets.current_path,+ {scan_history.SCAN_SUMMARY_COLUMNS}+ FROM security_targets AS targets+ LEFT JOIN scans ON scans.target_id = targets.id+ AND scans.id IN (SELECT scan_id FROM scan_progress)+ LEFT JOIN scan_progress AS progress ON progress.scan_id = scans.id+ ORDER BY julianday(upper(scans.started_at)) DESC, scans.id DESC, targets.id+ """+ ).fetchall():+ scan = scan_history.scan_summary(row) if row["id"] is not None else None+ target_id = row["repository_id"]+ if scan is not None:+ scan_count_by_target[target_id] += 1+ latest_by_target.setdefault(target_id, (row, scan))open_findings_by_target = Counter(row["target_id"] for row in _indexed_findings(connection) if row["status"] == "open")- targets = {row["id"]: row for row in connection.execute("SELECT * FROM security_targets")}repositories = [{- "checkoutAvailable": Path(target["current_path"]).is_dir(),"displayName": target["display_name"],"latestScan": latest_scan,- "openFindingsCount": open_findings_by_target.get(target_id, 0),+ "openFindingsCount": open_findings_by_target[target_id]+ if latest_scan is not None+ else 0,"scanCount": scan_count_by_target[target_id],"targetId": target_id,"targetPath": target["current_path"],}- for target_id, latest_scan in latest_scan_by_target.items()- if (target := targets.get(target_id)) is not None- ]- if args is None:- return {"repositories": repositories}-- query = args.query.strip().casefold() if args.query else ""- repositories = [- repository- for repository in repositories- if (args.target_id is None or repository["targetId"] == args.target_id)- and args.status != "not_scanned"- and (args.status != "open_findings" or repository["openFindingsCount"] > 0)- and (- not query- or query in repository["displayName"].casefold()- or query in repository["targetPath"].casefold()- )+ for target_id, (target, latest_scan) in latest_by_target.items()]- if args.limit is None and args.offset == 0:- return {"repositories": repositories}-- limit = min(args.limit or FINDINGS_PAGE_MAX, FINDINGS_PAGE_MAX)- page = repositories[args.offset : args.offset + limit]- next_offset = args.offset + len(page)- return {- "repositories": page,- "limit": limit,- "nextOffset": next_offset if next_offset < len(repositories) else None,- "offset": args.offset,- }+ pagination = {}+ if args is not None:+ query = args.query.strip().casefold() if args.query else ""+ repositories = [+ repository+ for repository in repositories+ if (args.target_id is None or repository["targetId"] == args.target_id)+ and (args.status != "scanned" or repository["scanCount"] > 0)+ and (args.status != "not_scanned" or repository["scanCount"] == 0)+ and (args.status != "open_findings" or repository["openFindingsCount"] > 0)+ and (+ not query+ or query in repository["displayName"].casefold()+ or query in repository["targetPath"].casefold()+ )+ ]+ if args.limit is not None or args.offset:+ limit = min(args.limit or FINDINGS_PAGE_MAX, FINDINGS_PAGE_MAX)+ end = args.offset + limit+ pagination = {+ "limit": limit,+ "nextOffset": end if end < len(repositories) else None,+ "offset": args.offset,+ }+ repositories = repositories[args.offset : end]+ for repository in repositories:+ path = os.path.expanduser(repository["targetPath"])+ try:+ repository["checkoutAvailable"] = Path(path).is_absolute() and os.path.isdir(+ os.path.realpath(path)+ )+ except (OSError, RecursionError):+ repository["checkoutAvailable"] = False+ return {"repositories": repositories, **pagination}if __name__ == "__main__":diff --git a/plugins/codex-security/scripts/workbench_scan_history.py b/plugins/codex-security/scripts/workbench_scan_history.pyindex 69aa02015..eb1c56261 100644--- a/plugins/codex-security/scripts/workbench_scan_history.py+++ b/plugins/codex-security/scripts/workbench_scan_history.py@@ -265,8 +265,7 @@ def _repository_origin(target: Path) -> tuple[str, str] | None:return (host.lower(), path) if host and path else None-SCAN_SUMMARY_QUERY = """-SELECT+SCAN_SUMMARY_COLUMNS = """scans.*,progress.reportable_findings_count,progress.scope_file_count,@@ -278,6 +277,9 @@ SELECTFROM finding_occurrences AS occurrencesWHERE occurrences.scan_id = scans.id) AS finding_count+"""+SCAN_SUMMARY_QUERY = f"""+SELECT {SCAN_SUMMARY_COLUMNS}FROM scansJOIN scan_progress AS progress ON progress.scan_id = scans.id"""diff --git a/plugins/codex-security/tests/test_workbench_native_index_queries.py b/plugins/codex-security/tests/test_workbench_native_index_queries.pyindex 47b1c8027..dc9697ce2 100644--- a/plugins/codex-security/tests/test_workbench_native_index_queries.py+++ b/plugins/codex-security/tests/test_workbench_native_index_queries.py@@ -5,6 +5,7 @@ import sqlite3from argparse import Namespacefrom contextlib import closingfrom types import SimpleNamespace+from unittest.mock import Mockimport pytestfrom workbench_test_support import stable_target_id@@ -379,6 +380,32 @@ def test_scan_list_probes_requested_repository_once(]+def test_unscanned_repositories_follow_scanned_rows_and_paginate(+ workbench_api, indexed_collections, tmp_path, monkeypatch+):+ connection, _ = indexed_collections+ indexes = workbench_api["native_indexes"]+ query = indexes.list_repositories+ scanned = query(connection)["repositories"]+ unscanned = sorted(+ workbench_api["ensure_security_target"](connection, str(tmp_path / name))+ for name in ("needle-cedar", "needle-oak")+ )+ repositories = query(connection)["repositories"]+ assert repositories[: len(scanned)] == scanned+ assert [row["targetId"] for row in repositories[len(scanned) :]] == unscanned+ probe = Mock(wraps=indexes.os.path.isdir)+ monkeypatch.setattr(indexes.os.path, "isdir", probe)+ for offset, target_id in enumerate(unscanned):+ page = query(+ connection, query_args(status="not_scanned", query="NeEdLe", limit=1, offset=offset)+ )+ assert [row["targetId"] for row in page["repositories"]] == [target_id]+ assert page["nextOffset"] == (1 if offset == 0 else None)+ probe.assert_called_once_with(page["repositories"][0]["targetPath"])+ probe.reset_mock()++@pytest.mark.parametrize(("timestamps", "order"),[@@ -493,6 +520,20 @@ def test_repository_index_keeps_one_scan_cohort(indexes = workbench_api["native_indexes"]database = tmp_path / "history.sqlite"new_id = "00000000-0000-4000-8000-999999999999"+ # Hold one target's first completed scan and finding for the concurrent publication.+ with source:+ pending = [+ (+ table,+ source.execute(f"SELECT * FROM {table} WHERE {key} = ?", (SCAN_IDS[1],)).fetchone(),+ )+ for table, key in (+ ("scans", "id"),+ ("scan_progress", "scan_id"),+ ("finding_occurrences", "scan_id"),+ )+ ]+ source.execute("DELETE FROM scans WHERE id = ?", (SCAN_IDS[1],))with closing(sqlite3.connect(database)) as reader, closing(sqlite3.connect(database)) as writer:source.backup(reader)reader.row_factory = sqlite3.Row@@ -503,10 +544,14 @@ def test_repository_index_keeps_one_scan_cohort(def execute(sql, parameters=()):cursor = reader.execute(sql, parameters)- if "JOIN scan_progress AS progress" not in sql:+ if "FROM security_targets AS targets" not in sql:return cursor- rows = cursor.fetchall()with writer:+ writer.execute(+ "UPDATE security_targets SET display_name = 'renamed' "+ "WHERE id = (SELECT target_id FROM scans WHERE id = ?)",+ (SCAN_IDS[0],),+ )writer.execute("INSERT INTO scans (id, workspace_id, target_id, target_path, target_revision, ""scope, mode, scan_dir, status, phase, started_at, created_at, updated_at) "@@ -520,7 +565,11 @@ def test_repository_index_keeps_one_scan_cohort("SELECT ?, updated_at FROM scan_progress WHERE scan_id = ?",(new_id, SCAN_IDS[0]),)- return SimpleNamespace(fetchall=lambda: rows)+ for table, row in pending:+ writer.execute(+ f"INSERT INTO {table} VALUES ({','.join('?' for _ in row)})", row+ )+ return cursorproxy = SimpleNamespace(execute=execute, create_function=reader.create_function)assert indexes.list_repositories(proxy) == before@@ -528,7 +577,10 @@ def test_repository_index_keeps_one_scan_cohort(reader.commit()after = indexes.list_repositories(reader)["repositories"]assert after[0]["latestScan"]["scanId"] == new_id+ assert after[0]["displayName"] == "renamed"assert after[0]["scanCount"] == 2+ first = next(row for row in after if row["latestScan"]["scanId"] == SCAN_IDS[1])+ assert (first["scanCount"], first["openFindingsCount"]) == (1, 1)def test_repository_latest_keeps_sqlite_millisecond_ties(workbench_api, indexed_collections):@@ -554,9 +606,15 @@ def test_repository_summary_excludes_scans_without_progress(workbench_api, indextarget_id = stable_target_id(targets[0])connection.execute("UPDATE scans SET target_id = ? WHERE id = ?", (target_id, SCAN_IDS[1]))connection.execute("DELETE FROM scan_progress WHERE scan_id = ?", (SCAN_IDS[1],))- repositories = workbench_api["native_indexes"].list_repositories(- connection, query_args(target_id=target_id)- )["repositories"]+ query = workbench_api["native_indexes"].list_repositories+ repositories = query(connection, query_args(target_id=target_id))["repositories"]assert len(repositories) == 1assert repositories[0]["latestScan"]["scanId"] == SCAN_IDS[0]assert repositories[0]["scanCount"] == 1+ connection.execute("DELETE FROM scan_progress WHERE scan_id = ?", (SCAN_IDS[0],))+ (unscanned,) = query(connection, query_args(target_id=target_id, status="not_scanned"))[+ "repositories"+ ]+ assert unscanned["targetId"] == target_id+ assert unscanned["scanCount"] == unscanned["openFindingsCount"] == 0+ assert unscanned["latestScan"] is Nonediff --git a/plugins/codex-security/tests/test_workbench_native_indexes.py b/plugins/codex-security/tests/test_workbench_native_indexes.pyindex db44cb46a..3dda22264 100644--- a/plugins/codex-security/tests/test_workbench_native_indexes.py+++ b/plugins/codex-security/tests/test_workbench_native_indexes.py@@ -7,6 +7,7 @@ from pathlib import Pathimport pytestfrom workbench_test_support import (create_saved_workspace,+ create_workspace,run_workbench,save_workspace,scan_command,@@ -189,6 +190,30 @@ def test_global_findings_keep_latest_occurrence_and_stable_target_identity(tmp_passert second["status"] == "open"+ ("target_path", "available"), [(".", False), ("~/repo", True), ("~/missing/../repo", True)]+)+def test_repository_index_probes_draft_checkout(+ tmp_path: Path, monkeypatch, target_path: str, available: bool+) -> None:+ state_dir = tmp_path / "state"+ (tmp_path / "repo").mkdir()+ monkeypatch.setenv("HOME", str(tmp_path))+ monkeypatch.setenv("USERPROFILE", str(tmp_path))+ create_workspace(+ state_dir,+ "20000000-0000-4000-8000-000000000000",+ "--target-path",+ target_path,+ "--scope",+ "missing-directory",+ )++ (repository,) = run_workbench(state_dir, "list-repositories")["repositories"]+ assert repository["targetPath"] == target_path+ assert repository["checkoutAvailable"] is available++def test_repository_index_reports_latest_scan_open_findings_and_missing_checkout(tmp_path: Path,) -> None:@@ -200,7 +225,30 @@ def test_repository_index_reports_latest_scan_open_findings_and_missing_checkoutsecond_target.mkdir()first_target_id = stable_target_id(first_target)second_target_id = stable_target_id(second_target)+ create_workspace(+ state_dir, "20000000-0000-4000-8000-000000000000", "--target-path", str(first_target)+ )+ (unscanned,) = run_workbench(state_dir, "list-repositories", "--status", "not_scanned")[+ "repositories"+ ]+ assert unscanned == {+ "checkoutAvailable": True,+ "displayName": first_target.name,+ "latestScan": None,+ "openFindingsCount": 0,+ "scanCount": 0,+ "targetId": first_target_id,+ "targetPath": str(first_target.resolve()),+ }+ for status in ("scanned", "open_findings"):+ assert (+ run_workbench(state_dir, "list-repositories", "--status", status)["repositories"] == []+ )older_first = complete_scan(state_dir, first_target, identity_anchor="first-finding")+ assert (+ run_workbench(state_dir, "list-repositories", "--status", "not_scanned")["repositories"]+ == []+ )set_triage(state_dir,str(older_first["findings"][0]["occurrenceId"]),diff --git a/sdk/typescript/src/cli.ts b/sdk/typescript/src/cli.tsindex 5ac6869d9..6d74e42f9 100644--- a/sdk/typescript/src/cli.ts+++ b/sdk/typescript/src/cli.ts@@ -2279,7 +2279,7 @@ export async function main();return presentHistory(await history(- ["list-repositories"],+ ["list-repositories", "--status", "scanned"],async (value): Promise<JsonObject> => {const repositories = value["repositories"] as JsonObject[];let target = repositories.find(diff --git a/sdk/typescript/tests-ts/cli-workbench.test.ts b/sdk/typescript/tests-ts/cli-workbench.test.tsindex e14000a6f..40e99684b 100644--- a/sdk/typescript/tests-ts/cli-workbench.test.ts+++ b/sdk/typescript/tests-ts/cli-workbench.test.ts@@ -80,7 +80,7 @@ describe("CLI workbench", () => {),).toBe(0);expect(calls).toEqual([- ["list-repositories"],+ ["list-repositories", "--status", "scanned"],...(includeMatchingAlias? [[@@ -190,7 +190,7 @@ describe("CLI workbench", () => {}});- test("findings list prefers an exact saved repository before directory aliases", async () => {+ test("findings list prefers an exact scanned repository before directory aliases", async () => {const root = await temporaryDirectory("finding-exact-repository-");try {const repository = join(root, "repository");@@ -211,7 +211,7 @@ describe("CLI workbench", () => {[repository, alias, repository],[alias, repository, alias],[unsavedAlias, alias, repository],- ]) {+ ] as const) {for (const args of [[], [requested!]]) {const calls: Array<readonly string[]> = [];const stdout = captureCli(main, "stdout");@@ -225,8 +225,16 @@ describe("CLI workbench", () => {return args[0] === "list-repositories"? {repositories: [- { targetId: "alias-target", targetPath: first! },- { targetId: "exact-target", targetPath: exact! },+ { targetId: "alias-target", targetPath: first },+ { targetId: "exact-target", targetPath: exact },+ ...(args.includes("scanned")+ ? []+ : [+ {+ targetId: "unscanned",+ targetPath: unsavedAlias,+ },+ ]),],}: {@@ -361,7 +369,7 @@ describe("CLI workbench", () => {}),),).toBe(0);- expect(calls[0]).toEqual(["list-repositories"]);+ expect(calls[0]).toEqual(["list-repositories", "--status", "scanned"]);expect(calls[1]).toEqual(["list-global-findings","--target-id",