OpenAI's Codex Security CLI and TypeScript SDK for finding, validating, and fixing security vulnerabilities. npm: https://www.npmjs.com/package/@openai/codex-security
TypeScript ai-securityapplication-securityclicode-scanningcodexcodex-securitycybersecuritydevsecopsnodejsnpmopenaisdksecuritytypescriptvulnerability-scanning
fix(findings): compare stored numeric JSON exactly (#1788)
Michael D'Angeloaae0f7ab271c
diff --git a/plugins/codex-security/mcp-app/src/workbench/findings.ts b/plugins/codex-security/mcp-app/src/workbench/findings.tsindex 249e8ea64..7ef9f0010 100644--- a/plugins/codex-security/mcp-app/src/workbench/findings.ts+++ b/plugins/codex-security/mcp-app/src/workbench/findings.ts@@ -1,6 +1,6 @@import type { DatabaseSync } from "node:sqlite";import { isDeepStrictEqual } from "node:util";-import { parseJson, stringifyJson } from "../helpers/json";+import { escapeControls, parseJson, stringifyJson } from "../helpers/json";import { requireSqliteText } from "./database";import { transaction } from "./transaction";@@ -28,6 +28,63 @@ function requireFiniteNumbers(value: unknown): void {for (const item of Object.values(value)) requireFiniteNumbers(item);}+// The fresh document comes from stringifyJson after the command's finite-number+// check. Keep this comparison private: it is not equality for arbitrary JSON pairs.+function storedFindingJsonMatches(previous: string, fresh: string): boolean {+ const numberTag = Symbol("finding number");+ const outsideFreshRange = Symbol("outside fresh numeric range");+ // Finite Number output has |exponent| <= 324; BigInt output has exponent 0.+ // Normalizing either coefficient shifts its exponent by at most its document+ // length. A larger old exponent cannot match fresh, even after cancellation.+ const maxExponent = previous.length + fresh.length + 324;++ function numberToken(source: string): unknown {+ const [, sign, integer, fraction = "", exponentSign, exponentDigits = "0"] =+ /^(-?)([0-9]+)(?:\.([0-9]+))?(?:[eE]([+-]?)([0-9]+))?$/u.exec(source)!;+ const digits = integer + fraction;+ let first = 0;+ while (digits[first] === "0") first++;+ if (first === digits.length) return [numberTag, "0", 0];+ let last = digits.length;+ while (digits[last - 1] === "0") last--;++ const magnitude = Number(exponentDigits);+ if (magnitude > maxExponent) return outsideFreshRange;+ // Supported Node string lengths keep the bounded exponent and scale exact.+ const exponent = (exponentSign === "-" ? -1 : 1) * magnitude;+ const scale = exponent + digits.length - last - fraction.length;+ return [numberTag, sign + digits.slice(first, last), scale];+ }++ function comparisonView(source: string): unknown {+ try {+ return JSON.parse(+ source,+ (_key, value: unknown, context?: { source?: string }) => {+ if (typeof value !== "number") return value;+ const source = context!.source!;+ const token = numberToken(source);+ // parseJson preserves unsafe plain integer tokens as BigInt. Other+ // spellings must expose the same precision before we retain their text.+ const readback =+ !Number.isSafeInteger(value) && !/^-?[0-9]+$/u.test(source)+ ? Number.isFinite(value) && Number.isInteger(value)+ ? numberToken(String(BigInt(value)))+ : value+ : token;+ return [token, readback];+ },+ );+ } catch (error) {+ if (error instanceof SyntaxError)+ error.message = escapeControls(error.message);+ throw error;+ }+ }++ return isDeepStrictEqual(comparisonView(previous), comparisonView(fresh));+}+export function storeFindings(database: DatabaseSync,entries: readonly EmbeddedFinding[],@@ -72,7 +129,7 @@ export function storeFindings(const previous = stored.get(finding.findingId)?.details_json;if (typeof previous === "string" &&- isDeepStrictEqual(parseJson(previous), parseJson(details))+ storedFindingJsonMatches(previous, details))details = previous;const { changes } = upsert.run(diff --git a/plugins/codex-security/mcp-app/tests/test_workbench_findings.ts b/plugins/codex-security/mcp-app/tests/test_workbench_findings.tsindex 5162450cc..07da2c4d8 100644--- a/plugins/codex-security/mcp-app/tests/test_workbench_findings.ts+++ b/plugins/codex-security/mcp-app/tests/test_workbench_findings.ts@@ -45,6 +45,30 @@ function entry(id: string): Findings.EmbeddedFinding {};}+function pythonUpsert(path: string, finding: Findings.Finding | string): void {+ execFileSync(+ process.env.PYTHON ?? "python",+ [+ "-I",+ "-X",+ "utf8",+ "-c",+ `import json, sqlite3, sys+sys.path.insert(0, sys.argv[1])+from workbench_finding_index import upsert_finding+with sqlite3.connect(sys.argv[2]) as db:+ db.row_factory = sqlite3.Row+ upsert_finding(db, json.load(sys.stdin), "python")`,+ fileURLToPath(new URL("../../scripts/", import.meta.url)),+ path,+ ],+ {+ input: typeof finding === "string" ? finding : stringifyJson(finding),+ encoding: "utf8",+ },+ );+}+for (const formatting of ["spaced", "reordered", "escaped"]) {test(`unchanged service upserts retain local cache and document serialization: ${formatting}`, (t) => {const database = open(t);@@ -264,32 +288,10 @@ test("mixed writers retain unchanged Python embeddings and replace supplied Node.get()!.details_json;const embedding = () =>database.prepare("SELECT model, vector_json FROM finding_embeddings").get();- const scripts = fileURLToPath(new URL("../../scripts/", import.meta.url));- const pythonUpsert = (finding: Findings.Finding | string) =>- execFileSync(- process.env.PYTHON ?? "python",- [- "-I",- "-X",- "utf8",- "-c",- `import json, sqlite3, sys-sys.path.insert(0, sys.argv[1])-from workbench_finding_index import upsert_finding-with sqlite3.connect(sys.argv[2]) as db:- db.row_factory = sqlite3.Row- upsert_finding(db, json.load(sys.stdin), "python")`,- scripts,- path,- ],- {- input: typeof finding === "string" ? finding : stringifyJson(finding),- encoding: "utf8",- },- );const original = details();assert.match(String(original), /9007199254740993/u);pythonUpsert(+ path,stringifyJson(Object.fromEntries(Object.entries(item.finding).reverse()),).replace('"score": 10', '"score": 10.0'),@@ -297,7 +299,7 @@ with sqlite3.connect(sys.argv[2]) as db:assert.equal(details(), original);assert.ok(embedding());const changed = { ...item.finding, title: "Updated finding" };- pythonUpsert(changed);+ pythonUpsert(path, changed);assert.equal(embedding(), undefined);item.embedding = { model: "replacement-model", vector: [0, 1] };storeFindings(database, [{ ...item, finding: changed }], "node");@@ -317,12 +319,195 @@ with sqlite3.connect(sys.argv[2]) as db:[{ ...item, finding: { ...changed, extensions: { foo: before } } }],"node",);- pythonUpsert({ ...changed, extensions: { foo: after } });+ pythonUpsert(path, { ...changed, extensions: { foo: after } });assert.equal(JSON.parse(String(details())).extensions.foo, after);assert.equal(embedding(), undefined);}});+for (const [name, previousValue, value, unchanged] of [+ ["negative zero", "-0.0", 0, true],+ ["exact exponent spelling", "1e+20", 1e20, true],+ ["rounded exponent spelling", "1e+23", 1e23, true],+ ["exact BigInt readback", "1e+23", 100000000000000000000000n, false],+] as const) {+ test(`stored finding numeric equality preserves Python ${name}`, async (t) => {+ const directory = await temporary.create("workbench-numeric-finding-");+ const path = join(directory, "workbench.sqlite3");+ const database = open(t, path);+ const item = entry("finding");+ item.finding.extensions = { measurement: "numeric-placeholder" };+ pythonUpsert(+ path,+ stringifyJson(item.finding).replace(+ '"numeric-placeholder"',+ previousValue,+ ),+ );+ const previous = database+ .prepare("SELECT details_json FROM findings")+ .get()!.details_json;+ assert.ok(String(previous).includes(previousValue));+ database+ .prepare(+ "INSERT INTO local_finding_embeddings VALUES ('finding', 'local-model', '[1,0]', 'prepared-key')",+ )+ .run();+ item.finding.extensions = { measurement: value };+ storeFindings(database, [item], "native-import");+ assert.equal(+ database.prepare("SELECT details_json FROM findings").get()!.details_json,+ unchanged ? previous : stringifyJson(item.finding, 0),+ );+ const readback = listStoredFindings(database, { limit: 10, offset: 0 })+ .findings[0].extensions as { measurement: number | bigint };+ assert.equal(BigInt(readback.measurement), BigInt(value));+ assert.equal(+ database.prepare("SELECT cache_key FROM local_finding_embeddings").get()+ ?.cache_key,+ unchanged ? "prepared-key" : undefined,+ );+ if (unchanged)+ assert.deepEqual(+ findPotentialDuplicates(database, "finding", undefined, {+ finding: "prepared-key",+ }).potentialDuplicates,+ [],+ );+ assert.equal(+ database.prepare("SELECT model FROM finding_embeddings").get()!.model,+ item.embedding.model,+ );+ });+}++const numericCases: readonly [string, string, unknown, boolean][] = [+ ["fractional integer", "10.000", 10, true],+ ["negative zero exponent", "1e-0", 1, true],+ ["positive zero exponent", "1e+000", 1, true],+ ["cancelled integer scale", "10e-1", 1, true],+ ["cancelled fractional scale", "0.10e1", 1, true],+ ["shifted coefficient", "1000e-3", 1, true],+ ["negative coefficient", "-1200e-3", -1.2, true],+ ["large finite exponent", "1e308", 1e308, true],+ ["smallest positive Number", "5e-324", Number.MIN_VALUE, true],+ ["rounded subnormal neighbor", "4e-324", Number.MIN_VALUE, false],+ [+ "exact huge integer",+ "123456789012345678901234567890",+ 123456789012345678901234567890n,+ true,+ ],+ ["unsafe integer exponent", "9007199254740993e0", 9007199254740993n, false],+ ["rounded unsafe integer", "1000000000000000000001e0", 1e21, false],+ ["precise fraction", "1.00000000000000001", 1, false],+ ["positive underflow", "1e-400", 0, false],+ ["negative underflow", "-1e-400", 0, false],+ ["adjacent huge integer", "9007199254740993", 9007199254740992n, false],+ ["boolean", "true", 1, false],+ ["string", '"1"', 1, false],+ ["array", "[1]", 1, false],+ ["duplicate members", '{"value":2,"value":1.000}', { value: 1 }, true],+ ["empty member", '{"":-0.0}', { "": 0 }, true],+ [+ "nested objects and prototype member",+ '{"z":[-0.0,10e-1],"a":{"__proto__":{"value":1}}}',+ JSON.parse('{"a":{"__proto__":{"value":1}},"z":[0,1]}'),+ true,+ ],+ ["huge zero exponent", `-0.0e${"9".repeat(65_536)}`, 0, true],+ ["huge positive exponent", `1e${"9".repeat(65_536)}`, 1, false],+ ["huge negative exponent", `1e-${"9".repeat(65_536)}`, 0, false],+ ["exponent leading zeros", `1e+${"0".repeat(65_536)}1`, 10, true],+ ["long precise fraction", `1.${"0".repeat(65_536)}1`, 1, false],+ [+ "long fractional scale cancellation",+ `0.${"0".repeat(4096)}1e4097`,+ 1,+ true,+ ],+ ["long integer scale cancellation", `1${"0".repeat(4096)}e-4096`, 1, true],+];++for (const [name, previousValue, value, equal] of numericCases) {+ test(`stored finding numeric equality handles ${name}`, (t) => {+ const database = open(t);+ const item = entry("finding");+ item.finding.extensions = { measurement: value };+ storeFindings(database, [item], "initial");+ const fresh = stringifyJson(item.finding, 0);+ const previous = stringifyJson(+ {+ ...item.finding,+ extensions: { measurement: "numeric-placeholder" },+ },+ 0,+ ).replace('"numeric-placeholder"', previousValue);+ database.prepare("UPDATE findings SET details_json = ?").run(previous);+ database+ .prepare(+ "INSERT INTO local_finding_embeddings VALUES ('finding', 'local-model', '[1,0]', 'prepared-key')",+ )+ .run();+ storeFindings(database, [item], "updated");+ assert.ok(+ database.prepare("SELECT details_json FROM findings").get()!+ .details_json === (equal ? previous : fresh),+ `${name}: ${equal ? "retain the exact stored body" : "replace the changed numeric body"}`,+ );+ assert.equal(+ database.prepare("SELECT cache_key FROM local_finding_embeddings").get()+ ?.cache_key,+ equal ? "prepared-key" : undefined,+ );+ });+}++test("stored finding numeric equality preserves malformed-row errors and batch rollback", (t) => {+ const database = open(t);+ const first = entry("first");+ const corrupt = entry("corrupt");+ storeFindings(database, [first, corrupt], "initial", "original-repository");+ const malformed = '{"value":1e999999,"broken":"line\u001btail"}';+ database+ .prepare("UPDATE findings SET details_json = ? WHERE id = 'corrupt'")+ .run(malformed);+ database+ .prepare(+ "INSERT INTO local_finding_embeddings VALUES ('first', 'local-model', '[1,0]', 'prepared-key')",+ )+ .run();+ const snapshot = () =>+ [+ "findings",+ "finding_embeddings",+ "finding_repositories",+ "local_finding_embeddings",+ ].map((table) =>+ database.prepare(`SELECT * FROM ${table} ORDER BY 1`).all(),+ );+ const before = snapshot();+ let expected: unknown;+ try {+ parseJson(malformed);+ } catch (error) {+ expected = error;+ }+ assert.ok(expected instanceof SyntaxError);+ const expectedMessage = expected.message;+ first.finding.title = "Changed before malformed later entry";+ first.embedding = { model: "replacement-model", vector: [0, 1] };+ assert.throws(+ () => storeFindings(database, [first, corrupt], "later", "new-repository"),+ (error: unknown) =>+ error instanceof SyntaxError && error.message === expectedMessage,+ );+ assert.deepEqual(snapshot(), before);+ assert.deepEqual(storeFindings(database, [first], "recovered"), {+ findingIds: ["first"],+ });+});+test("imports reject lossy SQLite keys and non-finite JSON atomically", (t) => {const database = open(t);const original = entry("existing");