Skip to content

OpenAI's Codex Security CLI and TypeScript SDK for finding, validating, and fixing security vulnerabilities. npm: https://www.npmjs.com/package/@openai/codex-security

refactor(plugin): reuse migration repair declarations (#1225)

mldangelo-oaib462af6a0498
diff --git a/plugins/codex-security/scripts/workbench_schema.py b/plugins/codex-security/scripts/workbench_schema.py
index d3c414620..bad12e374 100644
--- a/plugins/codex-security/scripts/workbench_schema.py
+++ b/plugins/codex-security/scripts/workbench_schema.py
@@ -947,7 +947,7 @@ def apply_migrations(
) -> None:
connection.commit()
connection.execute("BEGIN IMMEDIATE")
- try:
+ with connection:
connection.execute(
"""
CREATE TABLE IF NOT EXISTS schema_migrations (
@@ -963,59 +963,15 @@ def apply_migrations(
}
should_backfill_targets = False
for version, name, sql in migrations:
- if version in applied:
- if version == 2:
- add_column_if_missing(
- connection, "workspaces", "capability_preflight_json", "TEXT"
- )
- elif version == 6:
- repair_thread_scoped_workspaces_migration(connection)
- elif version == 11:
- repair_deep_scan_migration(connection)
- elif version == 12:
- add_column_if_missing(connection, "scans", "continuation_thread_id", "TEXT")
- elif version == 13:
- add_column_if_missing(
- connection,
- "scan_progress",
- "scope_file_count",
- "INTEGER CHECK (scope_file_count >= 0)",
- )
- elif version == 16:
- should_backfill_targets = repair_stable_targets_migration(connection)
- elif version == 26:
- add_column_if_missing(
- connection,
- "scans",
- "completion_warnings_json",
- "TEXT NOT NULL DEFAULT '[]'",
- )
- elif version == 28:
- add_column_if_missing(
- connection,
- "deep_scan_runs",
- "max_time_hours",
- "REAL NOT NULL DEFAULT 96",
- )
- elif version == 31:
- add_column_if_missing(
- connection,
- "scans",
- "retained_source_digests_json",
- "TEXT",
- )
- elif version == 32:
- add_column_if_missing(
- connection,
- "deep_scan_runs",
- "publication_error_message",
- "TEXT",
- )
- continue
if version == 6:
repair_thread_scoped_workspaces_migration(connection)
elif version == 16:
should_backfill_targets = repair_stable_targets_migration(connection)
+ elif version in applied:
+ if version in (2, 12, 13, 26, 28, 31, 32):
+ repair_additive_migration(connection, version)
+ elif version == 11:
+ repair_deep_scan_migration(connection)
else:
for statement in sql_statements(sql):
connection.execute(statement)
@@ -1023,18 +979,15 @@ def apply_migrations(
migrate_finding_workflow_columns(connection)
elif version == 39:
migrate_finding_workflow_review_columns(connection)
- connection.execute(
- "INSERT INTO schema_migrations (version, name, applied_at) VALUES (?, ?, ?)",
- (version, name, now()),
- )
+ if version not in applied:
+ connection.execute(
+ "INSERT INTO schema_migrations (version, name, applied_at) VALUES (?, ?, ?)",
+ (version, name, now()),
+ )
if 27 in applied:
repair_deep_scan_failure_counter_migration(connection)
if should_backfill_targets:
backfill_security_targets(connection)
- connection.commit()
- except BaseException:
- connection.rollback()
- raise
def normalize_pre_release_execution_profile_migrations(
@@ -1121,8 +1074,7 @@ def normalize_pre_release_execution_profile_migrations(
connection.execute(
f"ALTER TABLE {table} RENAME COLUMN reasoning_effort TO legacy_reasoning_effort"
)
- add_column_if_missing(connection, "scans", "model", "TEXT")
- add_column_if_missing(connection, "scans", "reasoning_effort", "TEXT")
+ repair_additive_migration(connection, 25)
connection.execute(
"""
UPDATE scans
@@ -1183,130 +1135,34 @@ def normalize_pre_release_migrations(connection: sqlite3.Connection, timestamp:
move_pre_release_migration(connection, 13, 21, "current scan preflight state")
- delivered_claim_migration = connection.execute(
- "SELECT name FROM schema_migrations WHERE version = 18"
- ).fetchone()
- if (
- delivered_claim_migration is not None
- and delivered_claim_migration["name"] == "scan target summaries"
- ):
- connection.execute(
- "UPDATE scans SET handoff_claimed_at = NULL, handoff_claim_token = NULL "
- "WHERE handoff_status = 'delivered'"
- )
- connection.execute(
- "UPDATE schema_migrations SET name = ? WHERE version = 18 AND name = ?",
- ("clear legacy delivered handoff claims", "scan target summaries"),
- )
-
- setup_preferences_migration = connection.execute(
- "SELECT name FROM schema_migrations WHERE version = 19"
- ).fetchone()
- legacy_setup_preferences_migrations = {
- "structured scan guidance context",
- "idempotent scan lifecycle requests",
- }
- if (
- setup_preferences_migration is not None
- and setup_preferences_migration["name"] in legacy_setup_preferences_migrations
- ):
- migration_sql = next(sql for version, _, sql in MIGRATIONS if version == 19)
- for statement in sql_statements(migration_sql):
- connection.execute(statement.replace("CREATE TABLE ", "CREATE TABLE IF NOT EXISTS ", 1))
- connection.execute(
- "UPDATE schema_migrations SET name = ? WHERE version = 19 AND name = ?",
- ("persist setup workspace preference", setup_preferences_migration["name"]),
- )
-
- phase_progress_migration = connection.execute(
- "SELECT name FROM schema_migrations WHERE version = 20"
- ).fetchone()
- legacy_phase_progress_migrations = {
- "retain superseded scan lifecycle requests",
- "threat model publication receipts",
- }
- if (
- phase_progress_migration is not None
- and phase_progress_migration["name"] in legacy_phase_progress_migrations
+ for version, legacy_names in (
+ (18, {"scan target summaries"}),
+ (19, {"structured scan guidance context", "idempotent scan lifecycle requests"}),
+ (20, {"retain superseded scan lifecycle requests", "threat model publication receipts"}),
+ (21, {"scan progress projection and activity", "deep coordinator manifest receipts"}),
+ (22, {"dynamic scan execution profiles"}),
):
- add_column_if_missing(
- connection,
- "scan_progress",
- "phase_items_total",
- "INTEGER NOT NULL DEFAULT 0 CHECK (phase_items_total >= 0)",
- )
- add_column_if_missing(
- connection,
- "scan_progress",
- "phase_items_completed",
- "INTEGER NOT NULL DEFAULT 0 "
- "CHECK (phase_items_completed >= 0 AND phase_items_completed <= phase_items_total)",
- )
- add_column_if_missing(
- connection,
- "scan_progress",
- "phase_progress_unit",
- "TEXT CHECK (phase_progress_unit IS NULL OR phase_progress_unit IN ("
- "'checks', 'threat_surfaces', 'review_receipts', 'candidate_findings', "
- "'validated_findings', 'report_artifacts'))",
- )
- connection.execute(
- "UPDATE schema_migrations SET name = ? WHERE version = 20 AND name = ?",
- ("phase-specific scan progress", phase_progress_migration["name"]),
- )
-
- preflight_progress_migration = connection.execute(
- "SELECT name FROM schema_migrations WHERE version = 21"
- ).fetchone()
- legacy_preflight_progress_migrations = {
- "scan progress projection and activity",
- "deep coordinator manifest receipts",
- }
- if (
- preflight_progress_migration is not None
- and preflight_progress_migration["name"] in legacy_preflight_progress_migrations
- ):
- add_column_if_missing(
- connection,
- "scan_progress",
- "preflight_issues_json",
- "TEXT NOT NULL DEFAULT '[]'",
- )
- add_column_if_missing(
- connection,
- "scan_progress",
- "preflight_checks_total",
- "INTEGER NOT NULL DEFAULT 0 CHECK (preflight_checks_total >= 0)",
- )
- add_column_if_missing(
- connection,
- "scan_progress",
- "preflight_checks_completed",
- "INTEGER NOT NULL DEFAULT 0 CHECK (preflight_checks_completed >= 0 "
- "AND preflight_checks_completed <= preflight_checks_total)",
- )
- connection.execute(
- "UPDATE schema_migrations SET name = ? WHERE version = 21 AND name = ?",
- ("current scan preflight state", preflight_progress_migration["name"]),
- )
-
- scan_recipe_migration = connection.execute(
- "SELECT name FROM schema_migrations WHERE version = 22"
- ).fetchone()
- if (
- scan_recipe_migration is not None
- and scan_recipe_migration["name"] == "dynamic scan execution profiles"
- ):
- add_column_if_missing(connection, "scans", "recipe_json", "TEXT")
- add_column_if_missing(
- connection,
- "scans",
- "parent_scan_id",
- "TEXT REFERENCES scans(id) ON DELETE SET NULL",
- )
+ migration = connection.execute(
+ "SELECT name FROM schema_migrations WHERE version = ?", (version,)
+ ).fetchone()
+ if migration is None or migration["name"] not in legacy_names:
+ continue
+ _, name, sql = next(migration for migration in MIGRATIONS if migration[0] == version)
+ if version == 18:
+ connection.execute(
+ "UPDATE scans SET handoff_claimed_at = NULL, handoff_claim_token = NULL "
+ "WHERE handoff_status = 'delivered'"
+ )
+ elif version == 19:
+ for statement in sql_statements(sql):
+ connection.execute(
+ statement.replace("CREATE TABLE ", "CREATE TABLE IF NOT EXISTS ", 1)
+ )
+ else:
+ repair_additive_migration(connection, version)
connection.execute(
- "UPDATE schema_migrations SET name = ? WHERE version = 22 AND name = ?",
- ("replayable scan launch recipes", "dynamic scan execution profiles"),
+ "UPDATE schema_migrations SET name = ? WHERE version = ? AND name = ?",
+ (name, version, migration["name"]),
)
migration = connection.execute(
@@ -1342,7 +1198,7 @@ def normalize_pre_release_migrations(connection: sqlite3.Connection, timestamp:
"UPDATE schema_migrations SET version = ? WHERE version = ? AND name = ?",
(new_version, old_version, expected[old_version]),
)
- add_column_if_missing(connection, "workspaces", "capability_preflight_json", "TEXT")
+ repair_additive_migration(connection, 2)
add_column_if_missing(connection, "scans", "target_snapshot_digest", "TEXT")
connection.execute(
"INSERT INTO schema_migrations (version, name, applied_at) VALUES (?, ?, ?)",
@@ -1415,24 +1271,17 @@ def repair_deep_scan_migration(connection: sqlite3.Connection) -> None:
def repair_deep_scan_failure_counter_migration(connection: sqlite3.Connection) -> None:
+ threshold_column, count_column, _backfill = sql_statements(
+ next(sql for version, _, sql in MIGRATIONS if version == 27)
+ )
columns = {row["name"] for row in connection.execute("PRAGMA table_info(deep_scan_runs)")}
threshold_missing = "stop_after_consecutive_errors" not in columns
- add_column_if_missing(
- connection,
- "deep_scan_runs",
- "stop_after_consecutive_errors",
- "INTEGER NOT NULL DEFAULT 1 CHECK (stop_after_consecutive_errors >= 1)",
- )
+ add_migration_column(connection, threshold_column)
if threshold_missing:
connection.execute(
"UPDATE deep_scan_runs SET stop_after_consecutive_errors = stop_after_no_new"
)
- add_column_if_missing(
- connection,
- "deep_scan_runs",
- "consecutive_errors",
- "INTEGER NOT NULL DEFAULT 0 CHECK (consecutive_errors >= 0)",
- )
+ add_migration_column(connection, count_column)
def repair_thread_scoped_workspaces_migration(connection: sqlite3.Connection) -> None:
@@ -1461,21 +1310,8 @@ def repair_stable_targets_migration(connection: sqlite3.Connection) -> bool:
migration_sql = next(sql for version, _, sql in MIGRATIONS if version == 16)
for statement in sql_statements(migration_sql):
- if statement.startswith("ALTER TABLE workspaces"):
- add_column_if_missing(
- connection,
- "workspaces",
- "target_id",
- "TEXT REFERENCES security_targets(id)",
- )
- continue
- if statement.startswith("ALTER TABLE scans"):
- add_column_if_missing(
- connection,
- "scans",
- "target_id",
- "TEXT REFERENCES security_targets(id)",
- )
+ if statement.startswith(("ALTER TABLE workspaces", "ALTER TABLE scans")):
+ add_migration_column(connection, statement)
continue
statement = statement.replace("CREATE TABLE ", "CREATE TABLE IF NOT EXISTS ", 1)
statement = statement.replace("CREATE INDEX ", "CREATE INDEX IF NOT EXISTS ", 1)
@@ -1493,6 +1329,19 @@ def repair_stable_targets_migration(connection: sqlite3.Connection) -> bool:
return True
+def repair_additive_migration(connection: sqlite3.Connection, version: int) -> None:
+ sql = next(sql for migration_version, _, sql in MIGRATIONS if migration_version == version)
+ for statement in sql_statements(sql):
+ add_migration_column(connection, statement)
+
+
+def add_migration_column(connection: sqlite3.Connection, statement: str) -> None:
+ _, _, table, _, _, column, definition = statement.removesuffix(";").split(None, 6)
+ # Preserve the compact declarations used by historical repairs, including CHECK errors.
+ definition = " ".join(definition.split()).replace("( ", "(").replace(" )", ")")
+ add_column_if_missing(connection, table, column, definition)
+
+
def add_column_if_missing(
connection: sqlite3.Connection, table: str, column: str, definition: str
) -> None:
diff --git a/plugins/codex-security/tests/test_workbench_setup_and_migrations.py b/plugins/codex-security/tests/test_workbench_setup_and_migrations.py
index b622d7454..5cdaf00ca 100644
--- a/plugins/codex-security/tests/test_workbench_setup_and_migrations.py
+++ b/plugins/codex-security/tests/test_workbench_setup_and_migrations.py
@@ -1573,6 +1573,16 @@ def test_workbench_repairs_shadowed_phase_progress_migration(
assert connection.execute(
"SELECT name FROM schema_migrations WHERE version = 20"
).fetchone() == ("phase-specific scan progress",)
+ with pytest.raises(sqlite3.IntegrityError) as invalid_unit:
+ connection.execute(
+ "UPDATE scan_progress SET phase_progress_unit = 'invalid' WHERE scan_id = ?",
+ (scan_id,),
+ )
+ assert str(invalid_unit.value) == (
+ "CHECK constraint failed: phase_progress_unit IS NULL OR phase_progress_unit IN ("
+ "'checks', 'threat_surfaces', 'review_receipts', 'candidate_findings', "
+ "'validated_findings', 'report_artifacts')"
+ )
@pytest.mark.parametrize(