OpenAI's Codex Security CLI and TypeScript SDK for finding, validating, and fixing security vulnerabilities. npm: https://www.npmjs.com/package/@openai/codex-security
TypeScript ai-securityapplication-securityclicode-scanningcodexcodex-securitycybersecuritydevsecopsnodejsnpmopenaisdksecuritytypescriptvulnerability-scanning
fix: preserve severity assessments for each scan (#1058)
mldangelo-oai75438fccf627
diff --git a/plugins/codex-security/scripts/workbench_severity.py b/plugins/codex-security/scripts/workbench_severity.pyindex 07eecc8c0..0363c2b59 100644--- a/plugins/codex-security/scripts/workbench_severity.py+++ b/plugins/codex-security/scripts/workbench_severity.py@@ -1,4 +1,4 @@-"""Per-finding severity checkpoints and the selection requested for each scan."""+"""Reusable severity checkpoints and the assessments saved for each scan."""import argparseimport json@@ -30,12 +30,20 @@ FIELDS = {}-def assessments(connection: sqlite3.Connection, finding_ids: list[str]) -> list[dict[str, Any]]:+def assessments(+ connection: sqlite3.Connection, finding_ids: list[str], scan_id: str | None = None+) -> list[dict[str, Any]]:+ table = "finding_severity_assessments" if scan_id is None else "scan_severity_assessments"+ scope = "" if scan_id is None else "WHERE assessment.scan_id = ?"+ parameters = (+ (json.dumps(finding_ids),) if scan_id is None else (json.dumps(finding_ids), scan_id)+ )+ # Keep selected IDs first so each assessment is a primary-key lookup.rows = connection.execute(- """SELECT assessment.* FROM json_each(?) AS selected- JOIN finding_severity_assessments AS assessment ON assessment.finding_id = selected.value- ORDER BY selected.key""",- (json.dumps(finding_ids),),+ f"""SELECT assessment.* FROM json_each(?) AS selected+ CROSS JOIN {table} AS assessment ON assessment.finding_id = selected.value+ {scope} ORDER BY selected.key""",+ parameters,)return [{key: row[column] for key, column in FIELDS.items()} for row in rows]@@ -62,7 +70,20 @@ def checkpoint(payload["knowledgeBaseSha256"],),)- return {"assessments": assessments(connection, payload["findingIds"])}+ # Cache hits are not saved again by the classifier. Copy them once,+ # without replacing assessments this scan already owns.+ connection.execute(+ """INSERT INTO scan_severity_assessments+ SELECT ?, assessment.* FROM json_each(?) AS selected+ JOIN finding_severity_assessments AS assessment+ ON assessment.finding_id = selected.value+ WHERE true+ ON CONFLICT(scan_id, finding_id) DO NOTHING""",+ (payload["scanId"], json.dumps(payload["findingIds"])),+ )+ return {+ "assessments": assessments(connection, payload["findingIds"], payload["scanId"])+ }if payload["action"] != "save":raise SystemExit("Unknown severity checkpoint action.")finding = payload["finding"]@@ -76,16 +97,23 @@ def checkpoint(is None):upsert_finding(connection, finding, timestamp)- columns = ", ".join(FIELDS.values())- parameters = ", ".join("?" for _ in FIELDS)- updates = ", ".join(f"{column} = excluded.{column}" for column in FIELDS.values())- connection.execute(- f"""INSERT INTO finding_severity_assessments ({columns})- VALUES ({parameters})- ON CONFLICT(finding_id) DO UPDATE SET- {updates}""",- tuple(assessment[key] for key in FIELDS),- )+ values = {column: assessment[key] for key, column in FIELDS.items()}+ for table, key, row in (+ ("finding_severity_assessments", "finding_id", values),+ (+ "scan_severity_assessments",+ "scan_id, finding_id",+ {"scan_id": payload["scanId"], **values},+ ),+ ):+ columns = ", ".join(row)+ parameters = ", ".join("?" for _ in row)+ updates = ", ".join(f"{column} = excluded.{column}" for column in row)+ connection.execute(+ f"""INSERT INTO {table} ({columns}) VALUES ({parameters})+ ON CONFLICT({key}) DO UPDATE SET {updates}""",+ tuple(row.values()),+ )return {}@@ -108,13 +136,21 @@ def read_classification(database: Path, scan_id: str) -> dict[str, Any]:if row is None:return {}finding_ids = json.loads(row["finding_ids_json"])+ has_scan_assessments = (+ connection.execute(+ "SELECT 1 FROM sqlite_master WHERE type = 'table' AND name = 'scan_severity_assessments'"+ ).fetchone()+ is not None+ )return {"scanId": scan_id,"findingIds": finding_ids,"assessedAt": row["assessed_at"],"rubricSha256": row["rubric_sha256"],"knowledgeBaseSha256": row["knowledge_base_sha256"],- "assessments": assessments(connection, finding_ids),+ "assessments": assessments(+ connection, finding_ids, scan_id if has_scan_assessments else None+ ),}diff --git a/plugins/codex-security/shared/workbench-migrations.json b/plugins/codex-security/shared/workbench-migrations.jsonindex 0657ebfd2..a83787851 100644--- a/plugins/codex-security/shared/workbench-migrations.json+++ b/plugins/codex-security/shared/workbench-migrations.json@@ -357,5 +357,13 @@"version": 42,"name": "editable scan names","statements": ["ALTER TABLE scans ADD COLUMN name TEXT;"]+ },+ {+ "version": 43,+ "name": "preserve severity assessments per scan",+ "statements": [+ "CREATE TABLE scan_severity_assessments (\n scan_id TEXT NOT NULL REFERENCES scan_severity_classifications(scan_id) ON DELETE CASCADE,\n finding_id TEXT NOT NULL REFERENCES findings(id) ON DELETE CASCADE,\n occurrence_id TEXT,\n input_sha256 TEXT NOT NULL,\n rubric_sha256 TEXT,\n knowledge_base_sha256 TEXT,\n assessed_at TEXT NOT NULL,\n source TEXT NOT NULL CHECK (source IN ('existing-severity', 'rubric')),\n decision TEXT NOT NULL CHECK (decision IN ('assessed', 'excluded')),\n level TEXT CHECK (level IN ('critical', 'high', 'medium', 'low', 'informational')),\n rubric_label TEXT,\n rationale TEXT NOT NULL,\n confidence TEXT CHECK (confidence IN ('high', 'medium', 'low')),\n review_trigger TEXT,\n PRIMARY KEY (scan_id, finding_id),\n CHECK ((decision = 'assessed' AND level IS NOT NULL)\n OR (decision = 'excluded' AND level IS NULL AND rubric_label IS NULL))\n );",+ "INSERT INTO scan_severity_assessments\n SELECT classification.scan_id, assessment.*\n FROM scan_severity_classifications AS classification\n CROSS JOIN json_each(classification.finding_ids_json) AS selected\n CROSS JOIN finding_severity_assessments AS assessment ON assessment.finding_id = selected.value\n WHERE assessment.rubric_sha256 IS classification.rubric_sha256\n AND assessment.knowledge_base_sha256 IS classification.knowledge_base_sha256\n AND EXISTS (\n SELECT 1 FROM finding_occurrences AS occurrence\n WHERE occurrence.id = assessment.occurrence_id\n AND occurrence.scan_id = classification.scan_id\n );"+ ]}]diff --git a/plugins/codex-security/tests/test_workbench_db.py b/plugins/codex-security/tests/test_workbench_db.pyindex cd0cfacbf..f4dea43f3 100644--- a/plugins/codex-security/tests/test_workbench_db.py+++ b/plugins/codex-security/tests/test_workbench_db.py@@ -86,6 +86,7 @@ EXPECTED_TABLES = {"finding_workflow_reviews","finding_severity_assessments","scan_severity_classifications",+ "scan_severity_assessments","finding_workflows","findings","scan_artifacts",@@ -911,7 +912,7 @@ def test_workbench_persists_progress_and_indexes_completed_findings(tmp_path: Pa)}assert tables == EXPECTED_TABLES- assert connection.execute("SELECT COUNT(*) FROM schema_migrations").fetchone() == (42,)+ assert connection.execute("SELECT COUNT(*) FROM schema_migrations").fetchone() == (43,)assert connection.execute("SELECT COUNT(*) FROM findings").fetchone() == (1,)assert connection.execute("SELECT COUNT(*) FROM finding_locations").fetchone() == (1,)diff --git a/plugins/codex-security/tests/test_workbench_deep_scan.py b/plugins/codex-security/tests/test_workbench_deep_scan.pyindex 456b2bd6a..a0ff84bf4 100644--- a/plugins/codex-security/tests/test_workbench_deep_scan.py+++ b/plugins/codex-security/tests/test_workbench_deep_scan.py@@ -276,7 +276,7 @@ def test_existing_generation_safely_claims_and_reclaims_without_schema_migrationreturn claim_deep_scan_coordinator(state_dir, codex_home, scan_id)with sqlite3.connect(state_dir / "workbench.sqlite3") as connection:- assert connection.execute("SELECT MAX(version) FROM schema_migrations").fetchone() == (42,)+ schema_version = connection.execute("SELECT MAX(version) FROM schema_migrations").fetchone()assert claim()["deepScan"]["coordinatorGeneration"] == 2assert claim()["coordinatorDisposition"] == "observing"expire_deep_scan_coordinator(state_dir, scan_id)@@ -308,6 +308,11 @@ def test_existing_generation_safely_claims_and_reclaims_without_schema_migrationassert sum(result["coordinatorDisposition"] == "adopted" for result in results) == 1assert sum(result["coordinatorDisposition"] == "observing" for result in results) == 3assert {result["deepScan"]["coordinatorGeneration"] for result in results} == {3}+ with sqlite3.connect(state_dir / "workbench.sqlite3") as connection:+ assert (+ connection.execute("SELECT MAX(version) FROM schema_migrations").fetchone()+ == schema_version+ )def test_legacy_generation_with_active_worker_observes_grace_then_adopts(tmp_path: Path) -> None:diff --git a/plugins/codex-security/tests/test_workbench_setup_and_migrations.py b/plugins/codex-security/tests/test_workbench_setup_and_migrations.pyindex 8225b0822..71b4ce5f3 100644--- a/plugins/codex-security/tests/test_workbench_setup_and_migrations.py+++ b/plugins/codex-security/tests/test_workbench_setup_and_migrations.py@@ -11,6 +11,7 @@ import sysimport uuidfrom collections.abc import Callable, Iterablefrom concurrent.futures import ThreadPoolExecutor+from contextlib import closingfrom pathlib import Pathfrom threading import Timerfrom unittest import mock@@ -82,6 +83,7 @@ EXPECTED_MIGRATIONS = [(40, "index finding identity and comparison history"),(41, "checkpoint finding severity assessments"),(42, "editable scan names"),+ (43, "preserve severity assessments per scan"),]@@ -617,7 +619,9 @@ def test_workbench_serializes_concurrent_migrations(tmp_path: Path, upgrade: boo{"databasePath": str(state_dir / "workbench.sqlite3")},]with sqlite3.connect(state_dir / "workbench.sqlite3") as connection:- assert connection.execute("SELECT COUNT(*) FROM schema_migrations").fetchone() == (42,)+ assert connection.execute("SELECT COUNT(*) FROM schema_migrations").fetchone() == (+ len(EXPECTED_MIGRATIONS),+ )def test_workbench_retries_writer_admission_and_legacy_backfill(tmp_path: Path) -> None:@@ -748,6 +752,156 @@ def test_comparison_indexes_upgrade_without_skipping_findings_migrations(assert connection.execute("PRAGMA foreign_key_check").fetchall() == [][email protected]("indexed", [False, True])+def test_severity_migration_only_copies_assessments_with_matching_scan_occurrences(+ indexed: bool,+) -> None:+ connection, apply_migrations = create_historical_database(43)+ timestamp = "2026-09-01T00:00:00Z"+ with connection:+ connection.execute("PRAGMA foreign_keys = ON")+ connection.execute(+ "INSERT INTO workspaces (id, created_at, updated_at) VALUES (?, ?, ?)",+ ("workspace", timestamp, timestamp),+ )+ for scan_id in ("first-scan", "second-scan"):+ connection.execute(+ """INSERT INTO scans (+ id, workspace_id, target_path, target_revision, scope, mode, scan_dir,+ status, phase, started_at, created_at, updated_at+ ) VALUES (?, 'workspace', '/target', 'revision', '.', 'standard', ?,+ 'complete', 'reporting', ?, ?, ?)""",+ (scan_id, f"/scans/{scan_id}", timestamp, timestamp, timestamp),+ )+ connection.execute(+ """INSERT INTO scan_severity_classifications+ (scan_id, finding_ids_json, assessed_at) VALUES (?, '["finding"]', ?)""",+ (scan_id, timestamp),+ )+ connection.execute(+ """INSERT INTO findings+ (id, fingerprint, rule_id, identity_anchor, created_at, updated_at)+ VALUES ('finding', 'fingerprint', 'rule', 'anchor', ?, ?)""",+ (timestamp, timestamp),+ )+ connection.execute(+ """INSERT INTO finding_severity_assessments+ (finding_id, occurrence_id, input_sha256, assessed_at, source, decision,+ level, rationale)+ VALUES ('finding', 'second-occurrence', 'digest', ?, 'existing-severity',+ 'assessed', 'high', 'Saved severity')""",+ (timestamp,),+ )+ if indexed:+ connection.execute(+ """INSERT INTO finding_occurrences+ (id, finding_id, scan_id, title, summary, severity, confidence,+ remediation, created_at)+ VALUES ('second-occurrence', 'finding', 'second-scan', 'Title', 'Summary',+ 'high', 'high', 'Remediation', ?)""",+ (timestamp,),+ )++ apply_migrations(connection)+ apply_migrations(connection)++ migrated = connection.execute(+ "SELECT scan_id, occurrence_id FROM scan_severity_assessments"+ ).fetchall()+ assert [tuple(row) for row in migrated] == (+ [("second-scan", "second-occurrence")] if indexed else []+ )+ assert connection.execute("PRAGMA foreign_key_check").fetchall() == []++[email protected]("operation", ["migration", "read"])+def test_severity_work_scales_with_selected_findings(operation: str) -> None:+ severity = load_script("workbench_severity")+ timestamp = "2026-09-01T00:00:00Z"++ def instruction_count(scale: int) -> int:+ connection, apply_migrations = create_historical_database(43)+ with closing(connection):+ connection.execute("PRAGMA foreign_keys = ON")+ connection.execute(+ "INSERT INTO workspaces (id, created_at, updated_at) VALUES ('workspace', ?, ?)",+ (timestamp, timestamp),+ )+ scan_count, findings_per_scan = (+ (scale, 10) if operation == "migration" else (1, scale * 10)+ )+ for scan in range(scan_count):+ scan_id = f"scan-{scan}"+ finding_ids = [f"finding-{scan}-{item}" for item in range(findings_per_scan)]+ connection.execute(+ """INSERT INTO scans (+ id, workspace_id, target_path, target_revision, scope, mode, scan_dir,+ status, phase, started_at, created_at, updated_at+ ) VALUES (?, 'workspace', '/target', 'revision', '.', 'standard', ?,+ 'complete', 'reporting', ?, ?, ?)""",+ (scan_id, f"/scans/{scan_id}", timestamp, timestamp, timestamp),+ )+ connection.execute(+ """INSERT INTO scan_severity_classifications+ (scan_id, finding_ids_json, assessed_at) VALUES (?, ?, ?)""",+ (scan_id, json.dumps(finding_ids), timestamp),+ )+ for finding_id in finding_ids:+ occurrence_id = f"occurrence-{finding_id}"+ connection.execute(+ """INSERT INTO findings+ (id, fingerprint, rule_id, identity_anchor, created_at, updated_at)+ VALUES (?, ?, 'rule', 'anchor', ?, ?)""",+ (finding_id, finding_id, timestamp, timestamp),+ )+ connection.execute(+ """INSERT INTO finding_occurrences+ (id, finding_id, scan_id, title, summary, severity, confidence,+ remediation, created_at)+ VALUES (?, ?, ?, 'Title', 'Summary', 'high', 'high', 'Remediation', ?)""",+ (occurrence_id, finding_id, scan_id, timestamp),+ )+ connection.execute(+ """INSERT INTO finding_severity_assessments+ (finding_id, occurrence_id, input_sha256, assessed_at, source,+ decision, level, rationale)+ VALUES (?, ?, 'digest', ?, 'existing-severity', 'assessed',+ 'high', 'Saved severity')""",+ (finding_id, occurrence_id, timestamp),+ )+ connection.commit()+ if operation == "read":+ apply_migrations(connection)++ instructions = 0++ def progress() -> int:+ nonlocal instructions+ instructions += 100+ return 0++ connection.set_progress_handler(progress, 100)+ try:+ if operation == "migration":+ apply_migrations(connection)+ else:+ selected = list(reversed(finding_ids))+ result = severity.assessments(connection, selected, scan_id)+ assert [assessment["findingId"] for assessment in result] == selected+ finally:+ connection.set_progress_handler(None, 0)+ assert (+ connection.execute("SELECT COUNT(*) FROM scan_severity_assessments").fetchone()[0]+ == scan_count * findings_per_scan+ )+ assert connection.execute("PRAGMA foreign_key_check").fetchall() == []+ return instructions++ small, large = instruction_count(20), instruction_count(200)+ # Ten times the selected work must not become a quadratic scan of the assessment cache.+ assert large < small * 20, (small, large)++def test_workbench_backfills_repository_targets_only_during_migration() -> None:namespace = runpy.run_path(str(SCRIPT), run_name="codex_security_workbench_db")apply_migrations = namespace["apply_migrations"]@@ -1169,7 +1323,9 @@ def test_workbench_upgrades_preexisting_database(tmp_path: Path) -> None:connection.execute("ALTER TABLE scans DROP COLUMN handoff_claim_token")run_workbench(state_dir, "database-info")with sqlite3.connect(database) as connection:- assert connection.execute("SELECT MAX(version) FROM schema_migrations").fetchone() == (42,)+ assert connection.execute("SELECT MAX(version) FROM schema_migrations").fetchone() == (+ EXPECTED_MIGRATIONS[-1][0],+ )assert {row[1] for row in connection.execute("PRAGMA table_info(scans)")} >= {"handoff_claimed_at","handoff_claim_token",diff --git a/sdk/typescript/src/severity-store.ts b/sdk/typescript/src/severity-store.tsindex a5ce39395..cee256ef2 100644--- a/sdk/typescript/src/severity-store.ts+++ b/sdk/typescript/src/severity-store.ts@@ -53,6 +53,7 @@ export class SeverityStore {save: async (finding, assessment, result) => {await this.run(["severity-classification"], {action: "save",+ scanId,finding,assessment: {...assessment,diff --git a/sdk/typescript/tests-ts/classify-scan-severity.test.ts b/sdk/typescript/tests-ts/classify-scan-severity.test.tsindex e4014561b..d371ab223 100644--- a/sdk/typescript/tests-ts/classify-scan-severity.test.ts+++ b/sdk/typescript/tests-ts/classify-scan-severity.test.ts@@ -29,7 +29,7 @@ const { temporaryDirectory, cleanup } = createApiTestFixtures(const destination = { destination: "linear", teamId: "team-example" } as const;afterEach(cleanup);-async function fixture() {+async function fixture(scanId?: string) {const root = await temporaryDirectory();const scanDirectory = join(root, "scan");await copyCompletedScanFixture(scanDirectory);@@ -44,6 +44,16 @@ async function fixture() {other.identity.instance = "second-instance";setFindingIdentity(manifest.scan, other);document.findings.push(other);+ if (scanId) {+ manifest.scan.id = scanId;+ document.scanId = scanId;+ for (const finding of document.findings)+ setFindingIdentity(manifest.scan, finding);+ const coveragePath = join(scanDirectory, "coverage.json");+ const coverage = JSON.parse(await readFile(coveragePath, "utf8"));+ coverage.scanId = scanId;+ await writeFile(coveragePath, JSON.stringify(coverage));+ }await writeFile(join(scanDirectory, "findings.json"),JSON.stringify(document),@@ -221,6 +231,121 @@ test("checkpoints each finding, resumes missing work, and reprocesses only the s).toEqual(revisedRows);});+test.each(["same", "different"])(+ "preserves concurrent scan assessments with %s rubrics",+ async (rubric) => {+ const first = await fixture();+ const second = await fixture("scan_example_002");+ const environment = first.environment;+ if (rubric === "different")+ await writeFile(second.rubricPath, "Exclude administrative findings.");+ const firstModel = recordingClassifier();+ const secondModel = recordingClassifier();+ secondModel.control.excluded = true;+ const [firstResult, secondResult] = await Promise.all([+ classifyScanDirectorySeverity(first.scanDirectory, {+ environment,+ rubricPath: first.rubricPath,+ codex: firstModel.codex,+ }),+ classifyScanDirectorySeverity(second.scanDirectory, {+ environment,+ rubricPath: second.rubricPath,+ codex: secondModel.codex,+ }),+ ]);+ for (const [scan, result] of [+ [first, firstResult],+ [second, secondResult],+ ] as const) {+ const { scanId, ...classification } = result;+ expect(+ await readScanSeverityClassification(+ scan.scanDirectory,+ scanId,+ scan.findings,+ undefined,+ environment,+ ),+ ).toEqual(classification);+ }+ expect(+ (+ await prepareScanPublication(first.scanDirectory, {+ ...destination,+ environment,+ })+ ).issues.map((issue) => issue.priority),+ ).toEqual([3, 3]);+ expect(+ (+ await prepareScanPublication(second.scanDirectory, {+ ...destination,+ environment,+ })+ ).issues,+ ).toEqual([]);+ firstModel.calls.length = 0;+ expect(+ (+ await classifyScanDirectorySeverity(first.scanDirectory, {+ environment,+ rubricPath: first.rubricPath,+ codex: firstModel.codex,+ })+ ).assessments,+ ).toEqual(firstResult.assessments);+ expect(firstModel.calls).toEqual([]);+ },+);++test("migration leaves unindexed legacy assessments incomplete until reclassified", async () => {+ const first = await fixture();+ const second = await fixture("scan_example_002");+ const environment = first.environment;+ const { scanId, ...classification } = await classifyScanDirectorySeverity(+ first.scanDirectory,+ { environment },+ );+ await query(environment, "DROP TABLE scan_severity_assessments");+ await query(environment, "DELETE FROM schema_migrations WHERE version = 43");+ expect(+ await readScanSeverityClassification(+ first.scanDirectory,+ scanId,+ first.findings,+ undefined,+ environment,+ ),+ ).toEqual(classification);+ expect(+ await query(+ environment,+ "SELECT version FROM schema_migrations WHERE version = 43",+ ),+ ).toEqual([]);+ await classifyScanDirectorySeverity(second.scanDirectory, { environment });+ await expect(+ readScanSeverityClassification(+ first.scanDirectory,+ scanId,+ first.findings,+ undefined,+ environment,+ ),+ ).rejects.toThrow("incomplete");+ expect(+ (await classifyScanDirectorySeverity(first.scanDirectory, { environment }))+ .assessments,+ ).toEqual(classification.assessments);+ expect(+ await query(+ environment,+ "SELECT version FROM schema_migrations WHERE version = 43",+ ),+ ).toEqual([{ version: 43 }]);+});+test("changed rubric, context, or evidence invalidates matching checkpoints", async () => {const { environment, root, scanDirectory, rubricPath, findings } =await fixture();@@ -513,9 +638,13 @@ test("migrates existing databases without changing findings and reads older statenvironment,"SELECT * FROM findings ORDER BY id",);+ await query(environment, "DROP TABLE scan_severity_assessments");await query(environment, "DROP TABLE finding_severity_assessments");await query(environment, "DROP TABLE scan_severity_classifications");- await query(environment, "DELETE FROM schema_migrations WHERE version = 41");+ await query(+ environment,+ "DELETE FROM schema_migrations WHERE version IN (41, 43)",+ );expect((await prepareScanPublication(scanDirectory, {