Skip to content

OpenAI's Codex Security CLI and TypeScript SDK for finding, validating, and fixing security vulnerabilities. npm: https://www.npmjs.com/package/@openai/codex-security

fix(workbench): keep scan readers available during registration (#1272)

mldangelo-oai90e4577dd3ae
diff --git a/plugins/codex-security/scripts/workbench_db.py b/plugins/codex-security/scripts/workbench_db.py
index 3bdbc8eba..2b22bdaa2 100644
--- a/plugins/codex-security/scripts/workbench_db.py
+++ b/plugins/codex-security/scripts/workbench_db.py
@@ -234,7 +234,7 @@ def release_completion_file_lock(descriptor: int) -> None:
windows_file_lock.locking(descriptor, windows_file_lock.LK_UNLCK, 1)
-def connect() -> sqlite3.Connection:
+def connect(*, deferred: bool = False) -> sqlite3.Connection:
path = database_path()
create_private_directory(path.parent)
for attempt in range(SQLITE_RETRY_ATTEMPTS):
@@ -243,7 +243,8 @@ def connect() -> sqlite3.Connection:
connection.row_factory = sqlite3.Row
connection.execute("PRAGMA foreign_keys = ON")
connection.execute("PRAGMA busy_timeout = 5000")
- apply_migrations(connection)
+ # Keep writer admission retries; readers only wait when a repair needs to write.
+ apply_migrations(connection, immediate=not deferred or attempt > 0)
connection.execute("PRAGMA journal_mode = WAL")
path.chmod(0o600)
return connection
@@ -255,8 +256,10 @@ def connect() -> sqlite3.Connection:
raise AssertionError("SQLite retry loop exhausted unexpectedly.")
-def apply_migrations(connection: sqlite3.Connection) -> None:
- apply_schema_migrations(connection, MIGRATIONS, now, backfill_security_targets)
+def apply_migrations(connection: sqlite3.Connection, *, immediate: bool = False) -> None:
+ apply_schema_migrations(
+ connection, MIGRATIONS, now, backfill_security_targets, immediate=immediate
+ )
def require_target(value: str) -> Path:
@@ -2878,7 +2881,15 @@ def backfill_legacy_finding_details(connection: sqlite3.Connection, scan: sqlite
if not updates:
return
- connection.execute("BEGIN IMMEDIATE")
+ # Reads normally avoid writer admission, but legacy details still need a write.
+ for attempt in range(SQLITE_RETRY_ATTEMPTS):
+ try:
+ connection.execute("BEGIN IMMEDIATE")
+ break
+ except sqlite3.OperationalError as exc:
+ if attempt == SQLITE_RETRY_ATTEMPTS - 1 or not sqlite_busy(exc):
+ raise
+ time.sleep(0.05 * (2**attempt))
with connection:
current = require_scan(connection, scan["id"])
recorded_digest = current["seal_manifest_digest"]
@@ -3321,7 +3332,9 @@ def main() -> None:
result = publication.inspect_linear_publication(_WORKBENCH_PUBLICATION_CONTEXT, args)
print(json.dumps(result, allow_nan=False, sort_keys=True))
return
- with closing(connect()) as connection:
+ with closing(
+ connect(deferred=args.command in {"get-scan", "list-scans", "database-info"})
+ ) as connection:
remediation.require_available(connection, args, require_scan)
if args.command == "create-workspace":
result = create_workspace(connection, args)
diff --git a/plugins/codex-security/scripts/workbench_schema.py b/plugins/codex-security/scripts/workbench_schema.py
index bad12e374..cbf54bf83 100644
--- a/plugins/codex-security/scripts/workbench_schema.py
+++ b/plugins/codex-security/scripts/workbench_schema.py
@@ -944,9 +944,11 @@ def apply_migrations(
migrations: tuple[tuple[int, str, str], ...],
now: Callable[[], str],
backfill_security_targets: Callable[[sqlite3.Connection], None],
+ *,
+ immediate: bool = False,
) -> None:
connection.commit()
- connection.execute("BEGIN IMMEDIATE")
+ connection.execute("BEGIN IMMEDIATE" if immediate else "BEGIN")
with connection:
connection.execute(
"""
@@ -1123,10 +1125,7 @@ def move_pre_release_migration(
def normalize_pre_release_migrations(connection: sqlite3.Connection, timestamp: str) -> None:
normalize_mirror_lineage_migrations(connection)
- connection.execute(
- "UPDATE schema_migrations SET version = 40 WHERE version = 33 AND name = ?",
- ("index finding identity and comparison history",),
- )
+ move_pre_release_migration(connection, 33, 40, "index finding identity and comparison history")
move_pre_release_migration(connection, 25, 26, "persist scan completion warnings")
move_pre_release_migration(connection, 12, 20, "phase-specific scan progress")
diff --git a/plugins/codex-security/tests/test_workbench_prompt_only_scan.py b/plugins/codex-security/tests/test_workbench_prompt_only_scan.py
index 1541c6c5d..ea73d0a93 100644
--- a/plugins/codex-security/tests/test_workbench_prompt_only_scan.py
+++ b/plugins/codex-security/tests/test_workbench_prompt_only_scan.py
@@ -7,8 +7,10 @@ import sqlite3
import uuid
from concurrent.futures import ThreadPoolExecutor
from pathlib import Path
+from threading import Event
from unittest import mock
+import pytest
from test_workbench_db import (
SCRIPT,
create_saved_workspace,
@@ -325,3 +327,82 @@ def test_prompt_only_diff_scan_validates_and_persists_canonical_diff_identity(
assert started["scan"]["diffTarget"]["baseRevision"] == head
assert started["scan"]["diffTarget"]["headRevision"] == head
assert started["workspace"]["diffTarget"] == started["scan"]["diffTarget"]
+
+
[email protected]("change_target", [False, True])
+def test_prompt_registration_keeps_existing_scans_readable(
+ tmp_path: Path, change_target: bool
+) -> None:
+ state_dir = tmp_path / "state"
+ target = tmp_path / "target"
+ target.mkdir()
+ source = target / "fixture.py"
+ source.write_text("original\n")
+ existing = start_prompt_only_scan(state_dir, target, tmp_path / "scans")
+ scan_id = existing["scan"]["scanId"]
+ namespace = runpy.run_path(str(SCRIPT), run_name="prompt_registration_readers")
+ start = namespace["_start_prompt_driven_scan"]
+ real_identity = start.__globals__["scan_target_identity"]
+ hashing = Event()
+ resume = Event()
+ calls = 0
+
+ def pause_second_hash(*args, **kwargs):
+ nonlocal calls
+ calls += 1
+ if calls == 2:
+ hashing.set()
+ assert resume.wait(20)
+ return real_identity(*args, **kwargs)
+
+ def register():
+ connection = namespace["connect"]()
+ try:
+ return start(
+ connection,
+ argparse.Namespace(
+ thread_id="thread-prompt-only-scan",
+ target_path=str(target),
+ scope=".",
+ mode="standard",
+ diff_target_kind=None,
+ diff_base_revision=None,
+ diff_head_revision=None,
+ diff_content_digest=None,
+ user_context="Inspect authentication boundaries",
+ user_context_file=None,
+ target_summary="Prompt-only scan",
+ scan_root=str(tmp_path / "scans"),
+ model=None,
+ reasoning_effort=None,
+ ),
+ headless_standard=False,
+ )
+ finally:
+ connection.close()
+
+ def read_scans():
+ read = run_workbench(state_dir, "get-scan", "--scan-id", str(scan_id))
+ listed = run_workbench(state_dir, "list-scans")
+ assert read["scan"]["scanId"] == scan_id
+ assert any(scan["scanId"] == scan_id for scan in listed["scans"])
+
+ with (
+ mock.patch.dict(os.environ, {"CODEX_SECURITY_STATE_DIR": str(state_dir)}),
+ mock.patch.dict(start.__globals__, {"scan_target_identity": pause_second_hash}),
+ ThreadPoolExecutor(max_workers=2) as pool,
+ ):
+ registration = pool.submit(register)
+ try:
+ assert hashing.wait(10)
+ pool.submit(read_scans).result(timeout=10)
+ if change_target:
+ source.write_text("changed during registration\n")
+ finally:
+ resume.set()
+ if change_target:
+ with pytest.raises(SystemExit, match="target changed while the scan was starting"):
+ registration.result(timeout=10)
+ else:
+ assert registration.result(timeout=10)["startDisposition"] == "joined"
+ assert calls == 2
diff --git a/plugins/codex-security/tests/test_workbench_setup_and_migrations.py b/plugins/codex-security/tests/test_workbench_setup_and_migrations.py
index 4c059a54c..cf89867f9 100644
--- a/plugins/codex-security/tests/test_workbench_setup_and_migrations.py
+++ b/plugins/codex-security/tests/test_workbench_setup_and_migrations.py
@@ -11,6 +11,7 @@ import sys
import uuid
from concurrent.futures import ThreadPoolExecutor
from pathlib import Path
+from threading import Timer
from unittest import mock
import pytest
@@ -433,8 +434,21 @@ def test_workbench_defaults_to_persistent_codex_home_state(tmp_path: Path) -> No
}
-def test_workbench_serializes_concurrent_first_run_migrations(tmp_path: Path) -> None:
[email protected]("upgrade", [False, True])
+def test_workbench_serializes_concurrent_migrations(tmp_path: Path, upgrade: bool) -> None:
state_dir = tmp_path / "state"
+ if upgrade:
+ state_dir.mkdir()
+ namespace = runpy.run_path(str(SCRIPT), run_name="concurrent_migration_upgrade")
+ with sqlite3.connect(state_dir / "workbench.sqlite3") as connection:
+ connection.row_factory = sqlite3.Row
+ namespace["apply_schema_migrations"](
+ connection,
+ tuple(migration for migration in namespace["MIGRATIONS"] if migration[0] <= 32),
+ namespace["now"],
+ namespace["backfill_security_targets"],
+ )
+ connection.execute("PRAGMA journal_mode=WAL")
with ThreadPoolExecutor(max_workers=2) as executor:
results = list(executor.map(lambda _: run_workbench(state_dir, "database-info"), range(2)))
assert results == [
@@ -445,6 +459,72 @@ def test_workbench_serializes_concurrent_first_run_migrations(tmp_path: Path) ->
assert connection.execute("SELECT COUNT(*) FROM schema_migrations").fetchone() == (41,)
+def test_workbench_retries_writer_admission_and_legacy_backfill(tmp_path: Path) -> None:
+ state_dir = tmp_path / "state"
+ target = tmp_path / "target"
+ target.mkdir()
+ workspace = create_saved_workspace(state_dir, target)
+ scan_id, scan_dir = start_workspace_scan(state_dir, str(workspace["id"]), tmp_path / "scans")
+ write_completed_contract(scan_dir, scan_id, target)
+ run_workbench(state_dir, "complete-scan", "--scan-id", scan_id)
+ database = state_dir / "workbench.sqlite3"
+ with sqlite3.connect(database) as connection:
+ connection.execute(
+ "UPDATE finding_occurrences SET details_json = '{}' WHERE scan_id = ?", (scan_id,)
+ )
+
+ # Hold admission beyond one busy timeout, as a large registration hash can.
+ connection = sqlite3.connect(database, check_same_thread=False)
+ connection.execute("BEGIN IMMEDIATE")
+ release = Timer(7, connection.rollback)
+ try:
+ with ThreadPoolExecutor(max_workers=3) as pool:
+ release.start()
+ started = pool.submit(
+ run_workbench,
+ state_dir,
+ "start-prompt-only-scan",
+ "--thread-id",
+ "concurrent-thread",
+ "--target-path",
+ str(target),
+ "--scope",
+ ".",
+ "--mode",
+ "standard",
+ "--target-summary",
+ "Fixture",
+ "--scan-root",
+ str(tmp_path / "scans"),
+ )
+ updated = pool.submit(
+ run_workbench,
+ state_dir,
+ "set-scan-thread",
+ "--scan-id",
+ scan_id,
+ "--thread-id",
+ "updated-thread",
+ )
+ backfilled = pool.submit(run_workbench, state_dir, "get-scan", "--scan-id", scan_id)
+ assert started.result(timeout=15)["startDisposition"] == "created"
+ assert updated.result(timeout=15)["threadId"] == "updated-thread"
+ assert (
+ backfilled.result(timeout=15)["scan"]["findings"][0]["attackPath"]["impact"][
+ "level"
+ ]
+ == "high"
+ )
+ finally:
+ release.cancel()
+ release.join()
+ connection.close()
+ with sqlite3.connect(database) as connection:
+ assert connection.execute(
+ "SELECT details_json != '{}' FROM finding_occurrences WHERE scan_id = ?", (scan_id,)
+ ).fetchone() == (1,)
+
+
@pytest.mark.parametrize("previous_history", ["main", "comparison-preview"])
def test_comparison_indexes_upgrade_without_skipping_findings_migrations(
previous_history: str,