Skip to content

OpenAI's Codex Security CLI and TypeScript SDK for finding, validating, and fixing security vulnerabilities. npm: https://www.npmjs.com/package/@openai/codex-security

fix(plugin): retain the reviewed surface count for completed Deep scans (#2161)

Michael D'Angelo5c1bd494c6b3
diff --git a/plugins/codex-security/mcp-app/src/workbench/migrations.ts b/plugins/codex-security/mcp-app/src/workbench/migrations.ts
index 1f74e4517..b5b4a3416 100644
--- a/plugins/codex-security/mcp-app/src/workbench/migrations.ts
+++ b/plugins/codex-security/mcp-app/src/workbench/migrations.ts
@@ -225,7 +225,9 @@ export function applyMigrations(
} else if (item.version === 16) backfill = repairStableTargets(database);
else if (applied.has(item.version)) {
if (
- [2, 12, 13, 26, 28, 31, 32, 42, 47, 50, 54, 55].includes(item.version)
+ [2, 12, 13, 26, 28, 31, 32, 42, 47, 50, 54, 55, 58].includes(
+ item.version,
+ )
)
repairAdditive(database, item.version);
else if (item.version === 11) repairDeepScan(database);
diff --git a/plugins/codex-security/mcp-app/tests/test_workbench_database.ts b/plugins/codex-security/mcp-app/tests/test_workbench_database.ts
index 6104edc13..48af1bbb8 100644
--- a/plugins/codex-security/mcp-app/tests/test_workbench_database.ts
+++ b/plugins/codex-security/mcp-app/tests/test_workbench_database.ts
@@ -169,7 +169,7 @@ test("every released schema upgrades to the same current schema and remains idem
});
for (const { missingVersions, legacyCheckpoint, mainReader } of [
- { missingVersions: [57], legacyCheckpoint: 0, mainReader: false },
+ { missingVersions: [57, 58], legacyCheckpoint: 0, mainReader: false },
{ missingVersions: [], legacyCheckpoint: 47, mainReader: false },
{ missingVersions: [], legacyCheckpoint: 47, mainReader: true },
{ missingVersions: [], legacyCheckpoint: 48, mainReader: false },
@@ -293,7 +293,7 @@ for (const { missingVersions, legacyCheckpoint, mainReader } of [
reopened
.prepare("SELECT MAX(version) AS version FROM schema_migrations")
.get()?.version,
- 57,
+ migrations.at(-1)!.version,
);
for (const { version } of migrations.filter(
(item) => item.version >= 43,
@@ -314,6 +314,12 @@ for (const { missingVersions, legacyCheckpoint, mainReader } of [
.get()?.discovery_user_context_json,
legacyCheckpoint !== 47 || mainReader ? context : null,
);
+ assert.equal(
+ reopened
+ .prepare("SELECT terminal_reviewed_surface_count FROM deep_scan_runs")
+ .get()?.terminal_reviewed_surface_count,
+ null,
+ );
assertMigrationNames(
reopened,
47,
@@ -909,11 +915,17 @@ test("recorded additive migrations restore missing columns and configured error
database.exec(`INSERT INTO deep_scan_runs (scan_id, schema_version, workflow_version, status, phase,
workers, subagents, stop_after_no_new, max_discovery_runs, created_at, updated_at)
VALUES ('scan', 1, 'v1', 'running', 'discovery', 1, 0, 7, 10, 'created', 'updated')`);
- for (const version of [27, 28, 31, 32, 47, 54, 55])
+ for (const version of [27, 28, 31, 32, 47, 54, 55, 58])
database
.prepare("INSERT INTO schema_migrations VALUES (?, ?, 'original')")
.run(version, migrations[version - 1].name);
applyMigrations(database);
+ assert.equal(
+ database
+ .prepare("SELECT terminal_reviewed_surface_count FROM deep_scan_runs")
+ .get()?.terminal_reviewed_surface_count,
+ null,
+ );
assert.equal(
database.prepare("SELECT deep_scan_config_json FROM workspaces").get()
?.deep_scan_config_json,
diff --git a/plugins/codex-security/scripts/deep_scan_workbench.py b/plugins/codex-security/scripts/deep_scan_workbench.py
index 42d90fabb..e4371b7a4 100644
--- a/plugins/codex-security/scripts/deep_scan_workbench.py
+++ b/plugins/codex-security/scripts/deep_scan_workbench.py
@@ -2040,6 +2040,11 @@ def fail_from_parent_scan(
message: str | None,
timestamp: str,
) -> None:
+ # Parent failure may supersede a prepared seal even after the coordinator succeeded.
+ connection.execute(
+ "UPDATE deep_scan_runs SET terminal_reviewed_surface_count = NULL WHERE scan_id = ?",
+ (scan_id,),
+ )
connection.execute(
"""
UPDATE deep_scan_runs
@@ -2057,7 +2062,7 @@ def cancel_from_parent_scan(connection: sqlite3.Connection, scan_id: str, timest
"""
UPDATE deep_scan_runs
SET status = 'canceled', phase = 'terminal', cancel_requested = 1,
- completed_at = ?, updated_at = ?
+ terminal_reviewed_surface_count = NULL, completed_at = ?, updated_at = ?
WHERE scan_id = ? AND status IN ('running', 'succeeded')
""",
(timestamp, timestamp, scan_id),
diff --git a/plugins/codex-security/scripts/workbench_db.py b/plugins/codex-security/scripts/workbench_db.py
index 10ba234bd..44fb3b0ed 100644
--- a/plugins/codex-security/scripts/workbench_db.py
+++ b/plugins/codex-security/scripts/workbench_db.py
@@ -1268,14 +1268,24 @@ def complete_scan_locked(
)
warnings.extend(warning for warning in prepared.warnings if warning not in warnings)
add_warning()
- if warnings != json.loads(scan["completion_warnings_json"]):
- # Persist all observed warnings before output writes can fail, so
- # stopped-result publication and retries retain those diagnostics.
- with connection:
+ with connection:
+ if warnings != json.loads(scan["completion_warnings_json"]):
+ # Retain pre-publication diagnostics even if output writes fail.
connection.execute(
"UPDATE scans SET completion_warnings_json = ? WHERE id = ? AND status = 'running'",
(json.dumps(warnings), scan_id),
)
+ prepared_scan = prepared.manifest["scan"]
+ if scan["mode"] == "deep" and (
+ not prepared.was_sealed
+ or "coverageRef" in prepared_scan
+ or "surfaces" in prepared_scan.get("reviewState", {})
+ ):
+ # A summary-only seal omits surfaces. Keep the count from its first preparation.
+ connection.execute(
+ "UPDATE deep_scan_runs SET terminal_reviewed_surface_count = ? WHERE scan_id = ?",
+ (len(prepared.coverage["surfaces"]), scan_id),
+ )
wrote = True
manifest, findings, _ = _write_prepared_scan_finalization(
prepared, projection_warnings=warnings
diff --git a/plugins/codex-security/scripts/workbench_schema.py b/plugins/codex-security/scripts/workbench_schema.py
index 4876e3106..024e3c13c 100644
--- a/plugins/codex-security/scripts/workbench_schema.py
+++ b/plugins/codex-security/scripts/workbench_schema.py
@@ -75,7 +75,7 @@ def apply_migrations(
elif version == 16:
should_backfill_targets = repair_stable_targets_migration(connection)
elif version in applied:
- if version in (2, 12, 13, 26, 28, 31, 32, 42, 47, 50, 54, 55):
+ if version in (2, 12, 13, 26, 28, 31, 32, 42, 47, 50, 54, 55, 58):
repair_additive_migration(connection, version)
elif version == 11:
repair_deep_scan_migration(connection)
diff --git a/plugins/codex-security/shared/workbench-migrations.json b/plugins/codex-security/shared/workbench-migrations.json
index ac7241638..94b945de0 100644
--- a/plugins/codex-security/shared/workbench-migrations.json
+++ b/plugins/codex-security/shared/workbench-migrations.json
@@ -469,5 +469,12 @@
"statements": [
"ALTER TABLE scans ADD COLUMN retained_checkpoint_heads_json TEXT;"
]
+ },
+ {
+ "version": 58,
+ "name": "record the final Deep Scan reviewed surface count",
+ "statements": [
+ "ALTER TABLE deep_scan_runs ADD COLUMN terminal_reviewed_surface_count INTEGER CHECK (terminal_reviewed_surface_count >= 0);"
+ ]
}
]
diff --git a/plugins/codex-security/tests/test_checkpoint_schema.py b/plugins/codex-security/tests/test_checkpoint_schema.py
index 459ddefad..f0d2ced0a 100644
--- a/plugins/codex-security/tests/test_checkpoint_schema.py
+++ b/plugins/codex-security/tests/test_checkpoint_schema.py
@@ -11,7 +11,7 @@ import pytest
"omitted_versions, legacy_checkpoint, main_reader",
[
((), False, False),
- ((57,), False, False),
+ ((57, 58), False, False),
((), 47, False),
((), 47, True),
((), 48, False),
@@ -157,7 +157,7 @@ def test_frozen_checkpoint_head_migration_preserves_scan_state(
)
assert connection.execute(
"SELECT COUNT(*), MAX(version) FROM schema_migrations"
- ).fetchone()[:] == (len(migrations), 57)
+ ).fetchone()[:] == (len(migrations), migrations[-1][0])
for version in (43, 44, 45, 46, 47):
assert (
connection.execute(
@@ -168,6 +168,12 @@ def test_frozen_checkpoint_head_migration_preserves_scan_state(
assert connection.execute(
"SELECT discovery_user_context_json FROM deep_scan_runs"
).fetchone()[0] == (context if legacy_checkpoint != 47 or main_reader else None)
+ assert (
+ connection.execute(
+ "SELECT terminal_reviewed_surface_count FROM deep_scan_runs"
+ ).fetchone()[0]
+ is None
+ )
if legacy_checkpoint != 47:
assert (
connection.execute(
diff --git a/plugins/codex-security/tests/test_deep_scan_deadline_recovery.py b/plugins/codex-security/tests/test_deep_scan_deadline_recovery.py
index 7ef02408d..f20858957 100644
--- a/plugins/codex-security/tests/test_deep_scan_deadline_recovery.py
+++ b/plugins/codex-security/tests/test_deep_scan_deadline_recovery.py
@@ -71,6 +71,14 @@ def test_deadline_completion_preserves_existing_tasks_and_recovered_findings(
)
coverage = read_review_state(workbench_db, scan.scan_dir)
assert task in coverage[task_state]
+ assert coverage["surfaces"] == []
+ assert (
+ workbench_db.execute(
+ "SELECT terminal_reviewed_surface_count FROM deep_scan_runs WHERE scan_id = ?",
+ (scan.scan_id,),
+ ).fetchone()[0]
+ == 0
+ )
if task_state == "resolvedDeferred":
assert coverage[task_state] == [task]
assert any(
diff --git a/plugins/codex-security/tests/test_deep_scan_publication_replay.py b/plugins/codex-security/tests/test_deep_scan_publication_replay.py
index 16b0dd512..8c4fe52eb 100644
--- a/plugins/codex-security/tests/test_deep_scan_publication_replay.py
+++ b/plugins/codex-security/tests/test_deep_scan_publication_replay.py
@@ -133,7 +133,10 @@ def test_publication_crash_replays_selected_input_without_stale_overwrite(
)["scan"]
assert completed["progress"]["status"] == "complete"
assert completed["findingCount"] == 1
- assert dict(connection.execute("SELECT * FROM deep_scan_runs").fetchone()) == run_before
+ assert dict(connection.execute("SELECT * FROM deep_scan_runs").fetchone()) == {
+ **run_before,
+ "terminal_reviewed_surface_count": 0,
+ }
assert [
dict(row) for row in connection.execute("SELECT * FROM deep_scan_workers")
] == workers_before
diff --git a/plugins/codex-security/tests/test_deep_scan_successful_publication.py b/plugins/codex-security/tests/test_deep_scan_successful_publication.py
index e4931342c..63480d644 100644
--- a/plugins/codex-security/tests/test_deep_scan_successful_publication.py
+++ b/plugins/codex-security/tests/test_deep_scan_successful_publication.py
@@ -364,12 +364,38 @@ def test_deep_publication_does_not_require_old_worker_files(
assert result.read_text() == "{old worker output is unavailable"
+REVIEWED_SURFACE = {
+ "id": "archive",
+ "label": "Archive review",
+ "disposition": "no_issue_found",
+ "receiptRefs": [],
+}
+
+
[email protected](params=[False, True], ids=["review-state", "coverage-file"])
+def reviewed_deep_scan(publication_scan, request):
+ scan = publication_scan()
+ scan.coverage["surfaces"] = [REVIEWED_SURFACE]
+ if request.param:
+ (scan.scan_dir / "coverage.json").write_text(json.dumps(scan.coverage))
+ else:
+ manifest_path = scan.scan_dir / "scan-manifest.json"
+ draft = json.loads(manifest_path.read_text())
+ draft["scan"].update(mode="deep_repository", reviewState=scan.coverage)
+ manifest_path.write_text(json.dumps(draft))
+ (scan.scan_dir / "coverage.json").unlink()
+ return scan
+
+
def test_deep_prepare_and_complete_preserve_the_same_aggregate(
- workbench_api, workbench_db, publication_scan
+ workbench_api, workbench_db, reviewed_deep_scan
):
- scan = publication_scan()
+ scan = reviewed_deep_scan
+ count = "SELECT terminal_reviewed_surface_count FROM deep_scan_runs WHERE scan_id = ?"
+ assert workbench_db.execute(count, (scan.scan_id,)).fetchone()[0] is None
prepared = complete(workbench_api, workbench_db, scan, prepare_only=True)
assert prepared["progress"]["status"] == "running"
+ assert workbench_db.execute(count, (scan.scan_id,)).fetchone()[0] == 1
names = ("scan-manifest.json", "findings.json")
published = {name: (scan.scan_dir / name).read_bytes() for name in names}
@@ -385,10 +411,77 @@ def test_deep_prepare_and_complete_preserve_the_same_aggregate(
repeated = complete(workbench_api, workbench_db, scan)
assert repeated["progress"]["status"] == "complete"
+ assert workbench_db.execute(count, (scan.scan_id,)).fetchone()[0] == 1
assert {name: (scan.scan_dir / name).read_bytes() for name in names} == published
assert_published_aggregate(scan)
+def test_complete_prepared_deep_scan_after_count_migration(
+ workbench_api, workbench_db, reviewed_deep_scan
+):
+ scan = reviewed_deep_scan
+ complete(workbench_api, workbench_db, scan, prepare_only=True)
+ count = "SELECT terminal_reviewed_surface_count FROM deep_scan_runs WHERE scan_id = ?"
+ clear = "UPDATE deep_scan_runs SET terminal_reviewed_surface_count = NULL WHERE scan_id = ?"
+ # A seal produced before the migration has no stored count.
+ with workbench_db:
+ workbench_db.execute(clear, (scan.scan_id,))
+ complete(workbench_api, workbench_db, scan)
+ sealed = json.loads((scan.scan_dir / "scan-manifest.json").read_text())
+ expected = 1 if "coverageRef" in sealed["scan"] else None
+ assert workbench_db.execute(count, (scan.scan_id,)).fetchone()[0] == expected
+
+ with workbench_db:
+ workbench_db.execute(clear, (scan.scan_id,))
+ complete(workbench_api, workbench_db, scan)
+ assert workbench_db.execute(count, (scan.scan_id,)).fetchone()[0] is None
+
+
+ ("state", "count"),
+ [({}, None), ({"surfaces": []}, 0), ({"surfaces": [REVIEWED_SURFACE]}, 1)],
+ ids=["omitted", "empty", "reviewed"],
+)
+def test_complete_sealed_deep_scan_with_inline_review_state(
+ workbench_api, workbench_db, publication_scan, state, count
+):
+ scan = publication_scan()
+ path = scan.scan_dir / "scan-manifest.json"
+ draft = json.loads(path.read_bytes())
+ draft["scan"].update(mode="deep_repository", reviewState=state)
+ path.write_text(json.dumps(draft))
+ (scan.scan_dir / "coverage.json").unlink()
+ binding = workbench_api["workbench_completion_binding"](
+ workbench_api["require_scan"](workbench_db, scan.scan_id), workbench_api["now"](), draft
+ )
+ sealed, _, _ = workbench_api["finalize_scan"](scan.scan_dir, completion_binding=binding)
+ sealed["scan"]["reviewState"] = state
+ path.write_text(json.dumps(sealed, indent=2, sort_keys=True) + "\n")
+
+ complete(workbench_api, workbench_db, scan)
+ query = "SELECT terminal_reviewed_surface_count FROM deep_scan_runs WHERE scan_id = ?"
+ assert workbench_db.execute(query, (scan.scan_id,)).fetchone()[0] == count
+
+
[email protected]("stop", ["cancel", "fail"])
+def test_stopping_prepared_deep_scan_clears_its_surface_count(
+ workbench_api, workbench_db, reviewed_deep_scan, stop
+):
+ scan = reviewed_deep_scan
+ complete(workbench_api, workbench_db, scan, prepare_only=True)
+ count = "SELECT terminal_reviewed_surface_count FROM deep_scan_runs WHERE scan_id = ?"
+ assert workbench_db.execute(count, (scan.scan_id,)).fetchone()[0] == 1
+ if stop == "cancel":
+ workbench_api["saved_results"].cancel_scan(
+ workbench_api["_WORKBENCH_DB_CONTEXT"],
+ workbench_db,
+ Namespace(scan_id=scan.scan_id, thread_id=None),
+ )
+ else:
+ fail_scan(workbench_api, workbench_db, scan)
+ assert workbench_db.execute(count, (scan.scan_id,)).fetchone()[0] is None
+
+
@pytest.mark.parametrize("mode", ["standard", "deep"])
@pytest.mark.parametrize("after_sealing", [False, True])
def test_completion_captures_target_warnings_before_first_seal(
@@ -821,11 +914,24 @@ def test_deep_publication_write_failure_keeps_original_terminal_cause(
== "running"
)
run = workbench_db.execute(
- "SELECT status, terminal_reason FROM deep_scan_runs WHERE scan_id = ?", (scan.scan_id,)
+ "SELECT status, terminal_reason, terminal_reviewed_surface_count "
+ "FROM deep_scan_runs WHERE scan_id = ?",
+ (scan.scan_id,),
).fetchone()
- assert tuple(run) == ("succeeded", "saturated")
+ assert tuple(run) == ("succeeded", "saturated", 0)
+ scan.coverage["surfaces"] = [
+ {"id": "entry", "label": "Entry review", "disposition": "no_issue_found", "receiptRefs": []}
+ ]
+ (scan.scan_dir / "coverage.json").write_text(json.dumps(scan.coverage))
completed = complete(workbench_api, workbench_db, scan)
assert completed["progress"]["status"] == "complete"
+ assert (
+ workbench_db.execute(
+ "SELECT terminal_reviewed_surface_count FROM deep_scan_runs WHERE scan_id = ?",
+ (scan.scan_id,),
+ ).fetchone()[0]
+ == 1
+ )
assert completed["warnings"] == warnings
if missing_writeup:
scan.findings[0].pop("writeup")
diff --git a/plugins/codex-security/tests/test_workbench_deep_scan.py b/plugins/codex-security/tests/test_workbench_deep_scan.py
index 5afe63aa8..8e57b8131 100644
--- a/plugins/codex-security/tests/test_workbench_deep_scan.py
+++ b/plugins/codex-security/tests/test_workbench_deep_scan.py
@@ -96,6 +96,7 @@ def start_empty_target_scan(
def restore_v53_schema(state_dir: Path) -> None:
with sqlite3.connect(state_dir / "workbench.sqlite3") as connection:
+ connection.execute("ALTER TABLE deep_scan_runs DROP COLUMN terminal_reviewed_surface_count")
connection.execute("ALTER TABLE scans DROP COLUMN retained_checkpoint_heads_json")
connection.execute("ALTER TABLE workspaces DROP COLUMN deep_scan_config_json")
connection.execute("ALTER TABLE deep_scan_runs DROP COLUMN pending_target_project_path")
diff --git a/plugins/codex-security/tests/test_workbench_setup_and_migrations.py b/plugins/codex-security/tests/test_workbench_setup_and_migrations.py
index 349152791..8a3e7f5b5 100644
--- a/plugins/codex-security/tests/test_workbench_setup_and_migrations.py
+++ b/plugins/codex-security/tests/test_workbench_setup_and_migrations.py
@@ -100,6 +100,7 @@ EXPECTED_MIGRATIONS = [
(55, "retain undelivered headless deep scan results"),
(56, "recover unfinished legacy headless deep scans"),
(57, "freeze stopped scan checkpoint selections"),
+ (58, "record the final Deep Scan reviewed surface count"),
]
@@ -1567,7 +1568,7 @@ def test_workbench_repairs_recorded_deep_scan_migration(
assert connection.execute(
"SELECT name FROM schema_migrations WHERE version = 28"
).fetchone() == ("persist deep scan discovery time limit",)
- assert "max_time_hours" in {
+ assert {"max_time_hours", "terminal_reviewed_surface_count"} <= {
row[1] for row in connection.execute("PRAGMA table_info(deep_scan_runs)")
}
assert {