OpenAI's Codex Security CLI and TypeScript SDK for finding, validating, and fixing security vulnerabilities. npm: https://www.npmjs.com/package/@openai/codex-security
TypeScript ai-securityapplication-securityclicode-scanningcodexcodex-securitycybersecuritydevsecopsnodejsnpmopenaisdksecuritytypescriptvulnerability-scanning
fix: limit finding comparison input size (#638)
ianw-oai70d9ece3945f
diff --git a/sdk/typescript/README.md b/sdk/typescript/README.mdindex caaae2e60..fcf99c146 100644--- a/sdk/typescript/README.md+++ b/sdk/typescript/README.md@@ -356,10 +356,16 @@ original IDs. Uncertain matches stay separate. `summary.json` records coverageand matching status; `report.md` links to component reports. Export and publishfrom the individual scan folders, not the combined summary.+Large comparisons use bounded batches that cover every earlier/later finding+pair. Overlapping confirmed groups are joined in code. Finding text is not+truncated; pairs above Codex's input limit leave matching incomplete.+Use an empty output directory outside the project. Failed components don'tstop others, but failures, incomplete coverage, or failed matching exit with`2`. Retry failed or incomplete components with`--components-file retry-components.json` and a new output directory.+The retry report covers only those components; it does not update the original+combined report.`--max-cost` applies per component, excluding planning and matching.`--model` and `--effort` also apply to matching; `--auth` applies throughout.diff --git a/sdk/typescript/src/api.ts b/sdk/typescript/src/api.tsindex 3226c514e..f8113895a 100644--- a/sdk/typescript/src/api.ts+++ b/sdk/typescript/src/api.ts@@ -1612,6 +1612,7 @@ export class CodexSecurity {((input, comparisonOptions) =>matchScanFindingsInternal(input, comparisonOptions, {surface: this.#surface,+ allowBatching: false,})),environment,model,diff --git a/sdk/typescript/src/scan-comparison.ts b/sdk/typescript/src/scan-comparison.tsindex 9b7224e97..da63b691e 100644--- a/sdk/typescript/src/scan-comparison.ts+++ b/sdk/typescript/src/scan-comparison.ts@@ -111,6 +111,8 @@ const comparisonSchema = zexport type ScanComparisonResult = z.infer<typeof comparisonSchema>;+const CODEX_MAX_INPUT_CHARACTERS = 1024 * 1024;+export async function matchScanFindings(input: ScanComparisonInput,options: ScanComparisonOptions = {},@@ -121,30 +123,115 @@ export async function matchScanFindings(export async function matchScanFindingsInternal(input: ScanComparisonInput,options: ScanComparisonOptions = {},- runtimeOptions: { surface: CodexSecuritySurface },+ runtimeOptions: { surface: CodexSecuritySurface; allowBatching?: boolean },): Promise<ScanComparisonResult> {- const finalResponse = await runReadOnlyCodex(- comparisonPrompt(input),- z.toJSONSchema(comparisonSchema, { target: "openapi-3.0" }),- options,- {- ...runtimeOptions,- threadSource: CODEX_SECURITY_THREAD_SOURCES.scanComparison,- },- );- let response: unknown;- try {- response = JSON.parse(finalResponse);- } catch (error) {- throw new CodexSecurityError("Scan comparison returned invalid JSON.", {- cause: error,- });+ const comparisons: ScanComparisonResult[] = [];+ const allowHistoricalUncertainty =+ options.allowHistoricalUncertainty ?? false;+ const outputSchema = z.toJSONSchema(comparisonSchema, {+ target: "openapi-3.0",+ });+ for (const batch of comparisonBatches(input, runtimeOptions.allowBatching)) {+ options.signal?.throwIfAborted();+ const finalResponse = await runReadOnlyCodex(+ batch.prompt,+ outputSchema,+ options,+ {+ ...runtimeOptions,+ threadSource: CODEX_SECURITY_THREAD_SOURCES.scanComparison,+ },+ );+ let response: unknown;+ try {+ response = JSON.parse(finalResponse);+ } catch (error) {+ throw new CodexSecurityError("Scan comparison returned invalid JSON.", {+ cause: error,+ });+ }+ comparisons.push(+ validateComparison(batch.input, response, allowHistoricalUncertainty),+ );+ }+ return combineComparisons(comparisons, allowHistoricalUncertainty);+}++function* comparisonBatches(+ input: ScanComparisonInput,+ allowBatching = true,+): Generator<{ input: ScanComparisonInput; prompt: string }> {+ const prompt = comparisonPrompt(input);+ if (allowBatching && prompt.length > CODEX_MAX_INPUT_CHARACTERS / 2) {+ // Splitting one side at a time covers every before/after pair exactly once.+ const side =+ input.before.length > 1 &&+ (input.after.length < 2 ||+ JSON.stringify(input.before).length >=+ JSON.stringify(input.after).length)+ ? "before"+ : "after";+ const findings = input[side];+ if (findings.length > 1) {+ const middle = Math.ceil(findings.length / 2);+ yield* comparisonBatches({ ...input, [side]: findings.slice(0, middle) });+ yield* comparisonBatches({ ...input, [side]: findings.slice(middle) });+ return;+ }}- return validateComparison(- input,- response,- options.allowHistoricalUncertainty ?? false,+ if (prompt.length > CODEX_MAX_INPUT_CHARACTERS) {+ throw new CodexSecurityError(+ `Finding comparison exceeds Codex's ${CODEX_MAX_INPUT_CHARACTERS}-character input limit.`,+ );+ }+ yield { input, prompt };+}++function combineComparisons(+ comparisons: ScanComparisonResult[],+ allowHistoricalUncertainty: boolean,+): ScanComparisonResult {+ let matches: ScanComparisonResult["matches"] = [];+ for (const match of comparisons.flatMap(({ matches }) => matches)) {+ const before = new Set(match.beforeOccurrenceIds);+ const after = new Set(match.afterOccurrenceIds);+ const overlapping = matches.filter(+ (group) =>+ group.beforeOccurrenceIds.some((id) => before.has(id)) ||+ group.afterOccurrenceIds.some((id) => after.has(id)),+ );+ if (overlapping.length === 0) {+ matches.push(match);+ continue;+ }+ const joined = [...overlapping, match];+ const merged = {+ beforeOccurrenceIds: [+ ...new Set(joined.flatMap((group) => group.beforeOccurrenceIds)),+ ],+ afterOccurrenceIds: [+ ...new Set(joined.flatMap((group) => group.afterOccurrenceIds)),+ ],+ confidence: "high" as const,+ reason: [...new Set(joined.map((group) => group.reason))].join("\n"),+ };+ matches.splice(matches.indexOf(overlapping[0]!), 1, merged);+ matches = matches.filter((group) => !overlapping.includes(group));+ }+ const matchedBefore = new Set(+ matches.flatMap((group) => group.beforeOccurrenceIds),+ );+ const matchedAfter = new Set(+ matches.flatMap((group) => group.afterOccurrenceIds),);+ const uncertain = comparisons+ .flatMap(({ uncertain }) => uncertain)+ .filter(+ ({ beforeOccurrenceId, afterOccurrenceId }) =>+ !matchedBefore.has(beforeOccurrenceId) &&+ (allowHistoricalUncertainty || !matchedAfter.has(afterOccurrenceId)),+ );+ return { matches, uncertain };}export async function runReadOnlyCodex(diff --git a/sdk/typescript/tests-ts/scan-comparison-batches.test.ts b/sdk/typescript/tests-ts/scan-comparison-batches.test.tsnew file mode 100644index 000000000..053f7bbdd--- /dev/null+++ b/sdk/typescript/tests-ts/scan-comparison-batches.test.ts@@ -0,0 +1,247 @@+import { expect, test } from "bun:test";+import {+ matchScanFindings,+ matchScanFindingsInternal,+ type ScanComparisonInput,+ type ScanComparisonOptions,+ type ScanComparisonResult,+} from "../src/scan-comparison.js";++const noMatches: ScanComparisonResult = { matches: [], uncertain: [] };++function codex(+ compare: (input: ScanComparisonInput, prompt: string) => ScanComparisonResult,+): NonNullable<ScanComparisonOptions["codex"]> {+ return {+ startThread: () => ({+ async run(prompt) {+ const input = JSON.parse(prompt.slice(prompt.lastIndexOf("\n") + 1));+ return { finalResponse: JSON.stringify(compare(input, prompt)) };+ },+ }),+ };+}++function finding(occurrenceId: string, characters = 5000) {+ return { occurrenceId, evidence: "x".repeat(characters) };+}++function match(beforeOccurrenceIds: string[], afterOccurrenceIds: string[]) {+ return {+ beforeOccurrenceIds,+ afterOccurrenceIds,+ confidence: "high" as const,+ reason: "Same synthetic root cause.",+ };+}++test("bounds large comparisons, covers every pair, and joins cross-batch groups", async () => {+ const input: ScanComparisonInput = {+ before: Array.from({ length: 155 }, (_, index) =>+ finding(`before-${index}`),+ ),+ after: Array.from({ length: 64 }, (_, index) => finding(`after-${index}`)),+ };+ expect(JSON.stringify(input).length).toBeGreaterThan(1048576);+ const pairs: string[] = [];+ let calls = 0;+ const result = await matchScanFindings(input, {+ codex: codex((batch, prompt) => {+ calls++;+ expect(prompt.length).toBeLessThanOrEqual(512 * 1024);+ expect(input.before).toEqual(expect.arrayContaining(batch.before));+ expect(input.after).toEqual(expect.arrayContaining(batch.after));+ for (const before of batch.before) {+ for (const after of batch.after) {+ pairs.push(`${before.occurrenceId}:${after.occurrenceId}`);+ }+ }+ return {+ matches: [+ match(+ batch.before.map(({ occurrenceId }) => occurrenceId),+ batch.after.map(({ occurrenceId }) => occurrenceId),+ ),+ ],+ uncertain: [],+ };+ }),+ });+ expect(calls).toBeGreaterThan(1);+ expect(pairs).toHaveLength(input.before.length * input.after.length);+ expect(new Set(pairs).size).toBe(pairs.length);+ expect(result.matches).toHaveLength(1);+ expect(new Set(result.matches[0]!.beforeOccurrenceIds)).toEqual(+ new Set(input.before.map(({ occurrenceId }) => occurrenceId)),+ );+ expect(new Set(result.matches[0]!.afterOccurrenceIds)).toEqual(+ new Set(input.after.map(({ occurrenceId }) => occurrenceId)),+ );+ expect(result.uncertain).toEqual([]);+});++test("joins connected groups and keeps unrelated findings separate", async () => {+ const targets: Record<string, string[]> = {+ one: ["first"],+ two: ["second"],+ bridge: ["first", "second"],+ independent: ["independent"],+ unmatched: [],+ };+ const result = await matchScanFindings(+ {+ before: Object.keys(targets).map((id) => finding(id, 300000)),+ after: ["first", "second", "independent", "new"].map((id) => finding(id)),+ },+ {+ codex: codex(({ before, after }) => {+ const earlier = before[0]!.occurrenceId;+ const later = after+ .map(({ occurrenceId }) => occurrenceId)+ .filter((id) => targets[earlier]!.includes(id));+ return {+ matches: later.length ? [match([earlier], later)] : [],+ uncertain: [],+ };+ }),+ },+ );+ expect(result.matches).toHaveLength(2);+ expect(new Set(result.matches[0]!.beforeOccurrenceIds)).toEqual(+ new Set(["one", "two", "bridge"]),+ );+ expect(new Set(result.matches[0]!.afterOccurrenceIds)).toEqual(+ new Set(["first", "second"]),+ );+ expect(result.matches[1]).toEqual(match(["independent"], ["independent"]));+ expect(result.uncertain).toEqual([]);+});++test.each([false, true])(+ "preserves cross-history uncertainty across batches (allowed: %p)",+ async (allowHistoricalUncertainty) => {+ const uncertain = (+ beforeOccurrenceId: string,+ afterOccurrenceId: string,+ ) => ({+ beforeOccurrenceId,+ afterOccurrenceId,+ reason: "Needs more evidence.",+ });+ const result = await matchScanFindings(+ {+ before: [finding("a", 300000), finding("b", 300000)],+ after: [finding("x", 300000), finding("y", 300000)],+ },+ {+ allowHistoricalUncertainty,+ codex: codex(({ before, after }) => {+ const left = before[0]!.occurrenceId;+ const right = after[0]!.occurrenceId;+ return left === "a" && right === "y"+ ? { matches: [match([left], [right])], uncertain: [] }+ : { matches: [], uncertain: [uncertain(left, right)] };+ }),+ },+ );+ expect(result.matches).toEqual([match(["a"], ["y"])]);+ expect(result.uncertain).toEqual([+ uncertain("b", "x"),+ ...(allowHistoricalUncertainty ? [uncertain("b", "y")] : []),+ ]);+ },+);++test("validates references against the current batch, not the whole input", async () => {+ await expect(+ matchScanFindings(+ {+ before: [finding("first", 300000), finding("other-batch", 300000)],+ after: [finding("after")],+ },+ {+ codex: codex(() => ({+ matches: [match(["other-batch"], ["after"])],+ uncertain: [],+ })),+ },+ ),+ ).rejects.toThrow("unknown before occurrence");+});++test("uses Codex's full allowance for individual pairs without truncation", async () => {+ const input = {+ before: [finding("before", 600000)],+ after: [finding("after")],+ };+ const calls: ScanComparisonInput[] = [];+ const options = {+ codex: codex((batch) => {+ calls.push(batch);+ return noMatches;+ }),+ };+ expect(await matchScanFindings(input, options)).toEqual(noMatches);+ expect(calls).toEqual([input]);+ const oversized = {+ before: [finding("before", 1048576)],+ after: [finding("after")],+ };+ await expect(matchScanFindings(oversized, options)).rejects.toThrow(+ "input limit",+ );+ expect(calls).toHaveLength(1);+ expect(oversized.before[0]!.evidence).toHaveLength(1048576);+});++test.each([200000, 300000])(+ "keeps automatic post-scan matching to at most one call (%p characters per finding)",+ async (characters) => {+ const input = {+ before: Array.from({ length: 4 }, (_, index) =>+ finding(`before-${index}`, characters),+ ),+ after: [finding("after")],+ };+ const calls: ScanComparisonInput[] = [];+ const comparison = matchScanFindingsInternal(+ input,+ {+ codex: codex((batch) => {+ calls.push(batch);+ return noMatches;+ }),+ },+ { surface: "sdk", allowBatching: false },+ );+ if (characters === 300000) {+ await expect(comparison).rejects.toThrow("input limit");+ expect(calls).toEqual([]);+ } else {+ expect(await comparison).toEqual(noMatches);+ expect(calls).toEqual([input]);+ }+ },+);++test("stops between batches when canceled", async () => {+ const controller = new AbortController();+ let calls = 0;+ await expect(+ matchScanFindings(+ {+ before: [finding("first", 300000), finding("second", 300000)],+ after: [finding("after")],+ },+ {+ signal: controller.signal,+ codex: codex(() => {+ calls++;+ controller.abort();+ return noMatches;+ }),+ },+ ),+ ).rejects.toThrow();+ expect(calls).toBe(1);+});diff --git a/sdk/typescript/tests-ts/scan-comparison.test.ts b/sdk/typescript/tests-ts/scan-comparison.test.tsindex 5417ff9a4..991f664dd 100644--- a/sdk/typescript/tests-ts/scan-comparison.test.ts+++ b/sdk/typescript/tests-ts/scan-comparison.test.ts@@ -406,7 +406,7 @@ describe("semantic scan comparison", () => {},);- test("compares all findings with one restricted structured-output turn", async () => {+ test("compares small inputs with one restricted structured-output turn", async () => {const input: ScanComparisonInput = {before: [finding("before-1"), finding("before-2")],after: [finding("after-1"), finding("after-2"), finding("after-3")],