OpenAI's Codex Security CLI and TypeScript SDK for finding, validating, and fixing security vulnerabilities. npm: https://www.npmjs.com/package/@openai/codex-security
TypeScript ai-securityapplication-securityclicode-scanningcodexcodex-securitycybersecuritydevsecopsnodejsnpmopenaisdksecuritytypescriptvulnerability-scanning
perf(imports): avoid redundant SQLite reads and parsing on replay (#2083)
Michael D'Angelof780c75286b0
diff --git a/plugins/codex-security/mcp-app/src/workbench/external-findings.ts b/plugins/codex-security/mcp-app/src/workbench/external-findings.tsindex 7d14f2112..78dd029bb 100644--- a/plugins/codex-security/mcp-app/src/workbench/external-findings.ts+++ b/plugins/codex-security/mcp-app/src/workbench/external-findings.ts@@ -1,6 +1,6 @@-import { createHash } from "node:crypto";+import { hash } from "node:crypto";import { isAbsolute } from "node:path";-import type { DatabaseSync } from "node:sqlite";+import type { DatabaseSync, StatementSync } from "node:sqlite";import { isDeepStrictEqual } from "node:util";import {object,@@ -48,6 +48,10 @@ function requireRepository(repository: unknown): asserts repository is string {throw new TypeError("repository must be an absolute local path.");}+function parseEvidence(json: string): StoredExternalFinding["evidence"] {+ return parseJson(json) as StoredExternalFinding["evidence"];+}+function fromRow(row: Record<string, unknown>): StoredExternalFinding {return {id: String(row.id),@@ -55,7 +59,7 @@ function fromRow(row: Record<string, unknown>): StoredExternalFinding {provider: String(row.provider),sourceKey: String(row.source_key),sourceFindingId: String(row.source_finding_id),- evidence: parseJson(String(row.evidence_json)) as Record<string, unknown>,+ evidence: parseEvidence(String(row.evidence_json)),assessment: { state: "not_assessed" },version: Number(row.version),createdAt: String(row.created_at),@@ -77,6 +81,8 @@ export function storeExternalFindings(requireSqliteText([timestamp]);if (!Array.isArray(input.findings))throw new TypeError("findings must be an array.");+ const counts = { created: 0, updated: 0, unchanged: 0 };+ if (input.findings.length === 0) return { findings: [], counts };// Validate and serialize the complete batch before taking the write lock.const sourceIds = new Set<string>();@@ -98,7 +104,7 @@ export function storeExternalFindings(finding.source_finding_id,];return {- id: `ext_${createHash("sha256").update(JSON.stringify(identity)).digest("hex")}`,+ id: `ext_${hash("sha256", JSON.stringify(identity))}`,sourceFindingId: finding.source_finding_id,evidenceJson,};@@ -108,21 +114,17 @@ export function storeExternalFindings(const stored = database.prepare("SELECT * FROM external_source_reports WHERE id = ?",);- const insert = database.prepare(- `INSERT INTO external_source_reports- (id, repository, provider, source_key, source_finding_id,- evidence_json, version, created_at, updated_at)- VALUES (?, ?, ?, ?, ?, ?, 1, ?, ?)`,- );- const update = database.prepare(- `UPDATE external_source_reports- SET evidence_json = ?, version = version + 1, updated_at = ?- WHERE id = ?`,- );- const counts = { created: 0, updated: 0, unchanged: 0 };+ let insert: StatementSync | undefined;+ let update: StatementSync | undefined;const findings = entries.map((entry) => {const previous = stored.get(entry.id);if (!previous) {+ insert ??= database.prepare(+ `INSERT INTO external_source_reports+ (id, repository, provider, source_key, source_finding_id,+ evidence_json, version, created_at, updated_at)+ VALUES (?, ?, ?, ?, ?, ?, 1, ?, ?)`,+ );insert.run(entry.id,input.repository,@@ -134,18 +136,29 @@ export function storeExternalFindings(timestamp,);counts.created++;- } else if (- isDeepStrictEqual(- parseJson(String(previous.evidence_json)),- parseJson(entry.evidenceJson),- )- ) {- counts.unchanged++;- } else {- update.run(entry.evidenceJson, timestamp, entry.id);- counts.updated++;+ return fromRow(stored.get(entry.id)!);+ }+ const finding = fromRow(previous);+ if (previous.evidence_json !== entry.evidenceJson) {+ const evidence = parseEvidence(entry.evidenceJson);+ if (!isDeepStrictEqual(finding.evidence, evidence)) {+ update ??= database.prepare(+ `UPDATE external_source_reports+ SET evidence_json = ?, version = version + 1, updated_at = ?+ WHERE id = ? RETURNING version, updated_at`,+ );+ const row = update.get(entry.evidenceJson, timestamp, entry.id)!;+ counts.updated++;+ return {+ ...finding,+ evidence,+ version: Number(row.version),+ updatedAt: String(row.updated_at),+ };+ }}- return fromRow(stored.get(entry.id)!);+ counts.unchanged++;+ return finding;});return { findings, counts };});diff --git a/plugins/codex-security/mcp-app/tests/test_external_findings.ts b/plugins/codex-security/mcp-app/tests/test_external_findings.tsindex aff718b6f..e2d397433 100644--- a/plugins/codex-security/mcp-app/tests/test_external_findings.ts+++ b/plugins/codex-security/mcp-app/tests/test_external_findings.ts@@ -1,5 +1,6 @@import assert from "node:assert/strict";import { execFileSync } from "node:child_process";+import { createHash } from "node:crypto";import { join } from "node:path";import { DatabaseSync } from "node:sqlite";import { after, test, type TestContext } from "node:test";@@ -18,8 +19,8 @@ const { applyMigrations, migrations } = (await importSource(const temporary = createTemporaryDirectories(true);after(() => temporary.cleanup());const repository = join(process.cwd(), "synthetic-repository");-const created = "2026-01-01T00:00:00.000Z";-const updated = "2026-01-02T00:00:00.000Z";+const created = "2026-01-01T00:00:00.123456Z";+const updated = "2026-01-02T00:00:00.789012Z";function open(t: TestContext) {const database = new DatabaseSync(":memory:");@@ -38,7 +39,10 @@ function batch(): External.ExternalFindingsImport {source_finding_id: "finding-one",evidence: {title: "Dependency finding",- source_data: { retained: { fields: [null, "value"] } },+ source_data: {+ retained: { fields: [null, "value"] },+ title: "Provider title",+ },commit_sha: "a".repeat(40),},},@@ -47,15 +51,57 @@ function batch(): External.ExternalFindingsImport {};}+test("storing an empty selection validates its envelope without opening a write transaction", async (t) => {+ const path = join(+ await temporary.create("external-empty-"),+ "workbench.sqlite3",+ );+ const writer = new DatabaseSync(path);+ t.after(() => writer.close());+ applyMigrations(writer);+ const database = new DatabaseSync(path);+ t.after(() => database.close());+ database.exec("PRAGMA busy_timeout = 0");+ writer.exec("BEGIN IMMEDIATE");++ const empty = { ...batch(), findings: [] };+ assert.deepEqual(storeExternalFindings(database, empty, updated), {+ findings: [],+ counts: { created: 0, updated: 0, unchanged: 0 },+ });+ assert.throws(+ () =>+ storeExternalFindings(+ database,+ { ...empty, repository: "relative" },+ updated,+ ),+ /absolute local path/u,+ );+ assert.throws(+ () => storeExternalFindings(database, empty, "\ud800"),+ /valid Unicode/u,+ );+ assert.throws(+ () => storeExternalFindings(database, batch(), updated),+ /database is locked/u,+ );+});+test("external imports preserve incomplete evidence without creating assessed findings or scans", (t) => {const database = open(t);const input = batch();const result = storeExternalFindings(database, input, created);assert.deepEqual(result.counts, { created: 2, updated: 0, unchanged: 0 });for (const [index, finding] of result.findings.entries()) {- assert.match(finding.id, /^ext_[0-9a-f]{64}$/u);+ const identity = [+ repository,+ input.provider,+ input.sourceKey,+ input.findings[index].source_finding_id,+ ];assert.deepEqual(finding, {- id: finding.id,+ id: `ext_${createHash("sha256").update(JSON.stringify(identity)).digest("hex")}`,repository,provider: input.provider,sourceKey: input.sourceKey,@@ -88,16 +134,24 @@ test("repeat imports keep identities and unchanged timestamps; changed evidencesource_finding_id: "finding-one",evidence: {commit_sha: "a".repeat(40),- source_data: { retained: { fields: [null, "value"] } },+ source_data: {+ title: "Provider title",+ retained: { fields: [null, "value"] },+ },title: "Dependency finding",},},],};- assert.deepEqual(storeExternalFindings(database, reordered, updated), {+ const replay = storeExternalFindings(database, reordered, updated);+ assert.deepEqual(replay, {findings: [first.findings[0]],counts: { created: 0, updated: 0, unchanged: 1 },});+ assert.deepEqual(+ Object.keys(replay.findings[0].evidence),+ Object.keys(input.findings[0].evidence),+ );assert.equal(database.prepare("SELECT evidence_json FROM external_source_reports WHERE id = ?")@@ -121,6 +175,59 @@ test("repeat imports keep identities and unchanged timestamps; changed evidence);});+test("a large mixed import and its replay return every stored row in input order", (t) => {+ const database = open(t);+ const findings = Array.from({ length: 512 }, (_, index) => ({+ source_finding_id: `synthetic-${index}`,+ evidence: {+ title: "Normalized title",+ source_data: {+ title: `Provider title ${index}`,+ occurrence: 9007199254740992n,+ },+ },+ }));+ const input = { ...batch(), findings };+ const first = storeExternalFindings(database, input, created);+ assert.deepEqual(first.counts, { created: 512, updated: 0, unchanged: 0 });++ const [last, ...rest] = findings.toReversed();+ const changed = structuredClone(last);+ changed.evidence.source_data.occurrence++;+ const added = { ...last, source_finding_id: "synthetic-added" };+ const mixed = { ...input, findings: [changed, ...rest, added] };+ const result = storeExternalFindings(database, mixed, updated);+ assert.deepEqual(result.counts, { created: 1, updated: 1, unchanged: 511 });+ assert.deepEqual(result.findings[0], {+ ...first.findings.at(-1),+ version: 2,+ updatedAt: updated,+ evidence: changed.evidence,+ });+ assert.notEqual(+ result.findings[0].evidence.source_data,+ changed.evidence.source_data,+ );+ assert.deepEqual(+ result.findings.slice(1, -1),+ first.findings.toReversed().slice(1),+ );+ assert.deepEqual(result.findings.at(-1), {+ ...first.findings.at(-1),+ id: result.findings.at(-1)!.id,+ sourceFindingId: added.source_finding_id,+ createdAt: updated,+ updatedAt: updated,+ });+ const changes = database.prepare("SELECT total_changes() AS writes");+ const beforeReplay = changes.get()!.writes;+ assert.deepEqual(storeExternalFindings(database, mixed, created), {+ findings: result.findings,+ counts: { created: 0, updated: 0, unchanged: 513 },+ });+ assert.equal(changes.get()!.writes, beforeReplay);+});+test("repository, provider and source namespaces keep distinct finding identities", (t) => {const database = open(t);const input = batch();@@ -168,24 +275,29 @@ test("invalid batches and failed writes preserve all previously stored evidence"assert.deepEqual(listExternalFindings(database, { repository }), before);database.exec(`CREATE TRIGGER fail_external_insert BEFORE INSERT ON external_source_reportsWHEN NEW.source_finding_id = 'fail'+ BEGIN SELECT RAISE(ABORT, 'synthetic write failure'); END;+ CREATE TRIGGER fail_external_update AFTER UPDATE ON external_source_reports+ WHEN OLD.source_finding_id = 'finding-two'BEGIN SELECT RAISE(ABORT, 'synthetic write failure'); END;`);- assert.throws(- () =>- storeExternalFindings(- database,- {- ...input,- findings: [- { ...input.findings[0], evidence: { changed: true } },- { source_finding_id: "inserted-before-failure", evidence: {} },- { source_finding_id: "fail", evidence: {} },- ],- },- updated,- ),- /synthetic write failure/u,- );- assert.deepEqual(listExternalFindings(database, { repository }), before);+ for (const source_finding_id of ["fail", "finding-two"]) {+ assert.throws(+ () =>+ storeExternalFindings(+ database,+ {+ ...input,+ findings: [+ { ...input.findings[0], evidence: { changed: true } },+ { source_finding_id: "inserted-before-failure", evidence: {} },+ { source_finding_id, evidence: { changed: true } },+ ],+ },+ updated,+ ),+ /synthetic write failure/u,+ );+ assert.deepEqual(listExternalFindings(database, { repository }), before);+ }});test("a Python database at the preceding schema upgrades in Node and remains readable in Python", async () => {