Skip to content

OpenAI's Codex Security CLI and TypeScript SDK for finding, validating, and fixing security vulnerabilities. npm: https://www.npmjs.com/package/@openai/codex-security

fix: limit finding comparison input size (#638)

ianw-oai70d9ece3945f
diff --git a/sdk/typescript/README.md b/sdk/typescript/README.md
index caaae2e60..fcf99c146 100644
--- a/sdk/typescript/README.md
+++ b/sdk/typescript/README.md
@@ -356,10 +356,16 @@ original IDs. Uncertain matches stay separate. `summary.json` records coverage
and matching status; `report.md` links to component reports. Export and publish
from the individual scan folders, not the combined summary.
+Large comparisons use bounded batches that cover every earlier/later finding
+pair. Overlapping confirmed groups are joined in code. Finding text is not
+truncated; pairs above Codex's input limit leave matching incomplete.
+
Use an empty output directory outside the project. Failed components don't
stop others, but failures, incomplete coverage, or failed matching exit with
`2`. Retry failed or incomplete components with
`--components-file retry-components.json` and a new output directory.
+The retry report covers only those components; it does not update the original
+combined report.
`--max-cost` applies per component, excluding planning and matching.
`--model` and `--effort` also apply to matching; `--auth` applies throughout.
diff --git a/sdk/typescript/src/api.ts b/sdk/typescript/src/api.ts
index 3226c514e..f8113895a 100644
--- a/sdk/typescript/src/api.ts
+++ b/sdk/typescript/src/api.ts
@@ -1612,6 +1612,7 @@ export class CodexSecurity {
((input, comparisonOptions) =>
matchScanFindingsInternal(input, comparisonOptions, {
surface: this.#surface,
+ allowBatching: false,
})),
environment,
model,
diff --git a/sdk/typescript/src/scan-comparison.ts b/sdk/typescript/src/scan-comparison.ts
index 9b7224e97..da63b691e 100644
--- a/sdk/typescript/src/scan-comparison.ts
+++ b/sdk/typescript/src/scan-comparison.ts
@@ -111,6 +111,8 @@ const comparisonSchema = z
export type ScanComparisonResult = z.infer<typeof comparisonSchema>;
+const CODEX_MAX_INPUT_CHARACTERS = 1024 * 1024;
+
export async function matchScanFindings(
input: ScanComparisonInput,
options: ScanComparisonOptions = {},
@@ -121,30 +123,115 @@ export async function matchScanFindings(
export async function matchScanFindingsInternal(
input: ScanComparisonInput,
options: ScanComparisonOptions = {},
- runtimeOptions: { surface: CodexSecuritySurface },
+ runtimeOptions: { surface: CodexSecuritySurface; allowBatching?: boolean },
): Promise<ScanComparisonResult> {
- const finalResponse = await runReadOnlyCodex(
- comparisonPrompt(input),
- z.toJSONSchema(comparisonSchema, { target: "openapi-3.0" }),
- options,
- {
- ...runtimeOptions,
- threadSource: CODEX_SECURITY_THREAD_SOURCES.scanComparison,
- },
- );
- let response: unknown;
- try {
- response = JSON.parse(finalResponse);
- } catch (error) {
- throw new CodexSecurityError("Scan comparison returned invalid JSON.", {
- cause: error,
- });
+ const comparisons: ScanComparisonResult[] = [];
+ const allowHistoricalUncertainty =
+ options.allowHistoricalUncertainty ?? false;
+ const outputSchema = z.toJSONSchema(comparisonSchema, {
+ target: "openapi-3.0",
+ });
+ for (const batch of comparisonBatches(input, runtimeOptions.allowBatching)) {
+ options.signal?.throwIfAborted();
+ const finalResponse = await runReadOnlyCodex(
+ batch.prompt,
+ outputSchema,
+ options,
+ {
+ ...runtimeOptions,
+ threadSource: CODEX_SECURITY_THREAD_SOURCES.scanComparison,
+ },
+ );
+ let response: unknown;
+ try {
+ response = JSON.parse(finalResponse);
+ } catch (error) {
+ throw new CodexSecurityError("Scan comparison returned invalid JSON.", {
+ cause: error,
+ });
+ }
+ comparisons.push(
+ validateComparison(batch.input, response, allowHistoricalUncertainty),
+ );
+ }
+ return combineComparisons(comparisons, allowHistoricalUncertainty);
+}
+
+function* comparisonBatches(
+ input: ScanComparisonInput,
+ allowBatching = true,
+): Generator<{ input: ScanComparisonInput; prompt: string }> {
+ const prompt = comparisonPrompt(input);
+ if (allowBatching && prompt.length > CODEX_MAX_INPUT_CHARACTERS / 2) {
+ // Splitting one side at a time covers every before/after pair exactly once.
+ const side =
+ input.before.length > 1 &&
+ (input.after.length < 2 ||
+ JSON.stringify(input.before).length >=
+ JSON.stringify(input.after).length)
+ ? "before"
+ : "after";
+ const findings = input[side];
+ if (findings.length > 1) {
+ const middle = Math.ceil(findings.length / 2);
+ yield* comparisonBatches({ ...input, [side]: findings.slice(0, middle) });
+ yield* comparisonBatches({ ...input, [side]: findings.slice(middle) });
+ return;
+ }
}
- return validateComparison(
- input,
- response,
- options.allowHistoricalUncertainty ?? false,
+ if (prompt.length > CODEX_MAX_INPUT_CHARACTERS) {
+ throw new CodexSecurityError(
+ `Finding comparison exceeds Codex's ${CODEX_MAX_INPUT_CHARACTERS}-character input limit.`,
+ );
+ }
+ yield { input, prompt };
+}
+
+function combineComparisons(
+ comparisons: ScanComparisonResult[],
+ allowHistoricalUncertainty: boolean,
+): ScanComparisonResult {
+ let matches: ScanComparisonResult["matches"] = [];
+ for (const match of comparisons.flatMap(({ matches }) => matches)) {
+ const before = new Set(match.beforeOccurrenceIds);
+ const after = new Set(match.afterOccurrenceIds);
+ const overlapping = matches.filter(
+ (group) =>
+ group.beforeOccurrenceIds.some((id) => before.has(id)) ||
+ group.afterOccurrenceIds.some((id) => after.has(id)),
+ );
+ if (overlapping.length === 0) {
+ matches.push(match);
+ continue;
+ }
+ const joined = [...overlapping, match];
+ const merged = {
+ beforeOccurrenceIds: [
+ ...new Set(joined.flatMap((group) => group.beforeOccurrenceIds)),
+ ],
+ afterOccurrenceIds: [
+ ...new Set(joined.flatMap((group) => group.afterOccurrenceIds)),
+ ],
+ confidence: "high" as const,
+ reason: [...new Set(joined.map((group) => group.reason))].join("\n"),
+ };
+ matches.splice(matches.indexOf(overlapping[0]!), 1, merged);
+ matches = matches.filter((group) => !overlapping.includes(group));
+ }
+ const matchedBefore = new Set(
+ matches.flatMap((group) => group.beforeOccurrenceIds),
+ );
+ const matchedAfter = new Set(
+ matches.flatMap((group) => group.afterOccurrenceIds),
);
+ const uncertain = comparisons
+ .flatMap(({ uncertain }) => uncertain)
+ .filter(
+ ({ beforeOccurrenceId, afterOccurrenceId }) =>
+ !matchedBefore.has(beforeOccurrenceId) &&
+ (allowHistoricalUncertainty || !matchedAfter.has(afterOccurrenceId)),
+ );
+ return { matches, uncertain };
}
export async function runReadOnlyCodex(
diff --git a/sdk/typescript/tests-ts/scan-comparison-batches.test.ts b/sdk/typescript/tests-ts/scan-comparison-batches.test.ts
new file mode 100644
index 000000000..053f7bbdd
--- /dev/null
+++ b/sdk/typescript/tests-ts/scan-comparison-batches.test.ts
@@ -0,0 +1,247 @@
+import { expect, test } from "bun:test";
+import {
+ matchScanFindings,
+ matchScanFindingsInternal,
+ type ScanComparisonInput,
+ type ScanComparisonOptions,
+ type ScanComparisonResult,
+} from "../src/scan-comparison.js";
+
+const noMatches: ScanComparisonResult = { matches: [], uncertain: [] };
+
+function codex(
+ compare: (input: ScanComparisonInput, prompt: string) => ScanComparisonResult,
+): NonNullable<ScanComparisonOptions["codex"]> {
+ return {
+ startThread: () => ({
+ async run(prompt) {
+ const input = JSON.parse(prompt.slice(prompt.lastIndexOf("\n") + 1));
+ return { finalResponse: JSON.stringify(compare(input, prompt)) };
+ },
+ }),
+ };
+}
+
+function finding(occurrenceId: string, characters = 5000) {
+ return { occurrenceId, evidence: "x".repeat(characters) };
+}
+
+function match(beforeOccurrenceIds: string[], afterOccurrenceIds: string[]) {
+ return {
+ beforeOccurrenceIds,
+ afterOccurrenceIds,
+ confidence: "high" as const,
+ reason: "Same synthetic root cause.",
+ };
+}
+
+test("bounds large comparisons, covers every pair, and joins cross-batch groups", async () => {
+ const input: ScanComparisonInput = {
+ before: Array.from({ length: 155 }, (_, index) =>
+ finding(`before-${index}`),
+ ),
+ after: Array.from({ length: 64 }, (_, index) => finding(`after-${index}`)),
+ };
+ expect(JSON.stringify(input).length).toBeGreaterThan(1048576);
+ const pairs: string[] = [];
+ let calls = 0;
+ const result = await matchScanFindings(input, {
+ codex: codex((batch, prompt) => {
+ calls++;
+ expect(prompt.length).toBeLessThanOrEqual(512 * 1024);
+ expect(input.before).toEqual(expect.arrayContaining(batch.before));
+ expect(input.after).toEqual(expect.arrayContaining(batch.after));
+ for (const before of batch.before) {
+ for (const after of batch.after) {
+ pairs.push(`${before.occurrenceId}:${after.occurrenceId}`);
+ }
+ }
+ return {
+ matches: [
+ match(
+ batch.before.map(({ occurrenceId }) => occurrenceId),
+ batch.after.map(({ occurrenceId }) => occurrenceId),
+ ),
+ ],
+ uncertain: [],
+ };
+ }),
+ });
+ expect(calls).toBeGreaterThan(1);
+ expect(pairs).toHaveLength(input.before.length * input.after.length);
+ expect(new Set(pairs).size).toBe(pairs.length);
+ expect(result.matches).toHaveLength(1);
+ expect(new Set(result.matches[0]!.beforeOccurrenceIds)).toEqual(
+ new Set(input.before.map(({ occurrenceId }) => occurrenceId)),
+ );
+ expect(new Set(result.matches[0]!.afterOccurrenceIds)).toEqual(
+ new Set(input.after.map(({ occurrenceId }) => occurrenceId)),
+ );
+ expect(result.uncertain).toEqual([]);
+});
+
+test("joins connected groups and keeps unrelated findings separate", async () => {
+ const targets: Record<string, string[]> = {
+ one: ["first"],
+ two: ["second"],
+ bridge: ["first", "second"],
+ independent: ["independent"],
+ unmatched: [],
+ };
+ const result = await matchScanFindings(
+ {
+ before: Object.keys(targets).map((id) => finding(id, 300000)),
+ after: ["first", "second", "independent", "new"].map((id) => finding(id)),
+ },
+ {
+ codex: codex(({ before, after }) => {
+ const earlier = before[0]!.occurrenceId;
+ const later = after
+ .map(({ occurrenceId }) => occurrenceId)
+ .filter((id) => targets[earlier]!.includes(id));
+ return {
+ matches: later.length ? [match([earlier], later)] : [],
+ uncertain: [],
+ };
+ }),
+ },
+ );
+ expect(result.matches).toHaveLength(2);
+ expect(new Set(result.matches[0]!.beforeOccurrenceIds)).toEqual(
+ new Set(["one", "two", "bridge"]),
+ );
+ expect(new Set(result.matches[0]!.afterOccurrenceIds)).toEqual(
+ new Set(["first", "second"]),
+ );
+ expect(result.matches[1]).toEqual(match(["independent"], ["independent"]));
+ expect(result.uncertain).toEqual([]);
+});
+
+test.each([false, true])(
+ "preserves cross-history uncertainty across batches (allowed: %p)",
+ async (allowHistoricalUncertainty) => {
+ const uncertain = (
+ beforeOccurrenceId: string,
+ afterOccurrenceId: string,
+ ) => ({
+ beforeOccurrenceId,
+ afterOccurrenceId,
+ reason: "Needs more evidence.",
+ });
+ const result = await matchScanFindings(
+ {
+ before: [finding("a", 300000), finding("b", 300000)],
+ after: [finding("x", 300000), finding("y", 300000)],
+ },
+ {
+ allowHistoricalUncertainty,
+ codex: codex(({ before, after }) => {
+ const left = before[0]!.occurrenceId;
+ const right = after[0]!.occurrenceId;
+ return left === "a" && right === "y"
+ ? { matches: [match([left], [right])], uncertain: [] }
+ : { matches: [], uncertain: [uncertain(left, right)] };
+ }),
+ },
+ );
+ expect(result.matches).toEqual([match(["a"], ["y"])]);
+ expect(result.uncertain).toEqual([
+ uncertain("b", "x"),
+ ...(allowHistoricalUncertainty ? [uncertain("b", "y")] : []),
+ ]);
+ },
+);
+
+test("validates references against the current batch, not the whole input", async () => {
+ await expect(
+ matchScanFindings(
+ {
+ before: [finding("first", 300000), finding("other-batch", 300000)],
+ after: [finding("after")],
+ },
+ {
+ codex: codex(() => ({
+ matches: [match(["other-batch"], ["after"])],
+ uncertain: [],
+ })),
+ },
+ ),
+ ).rejects.toThrow("unknown before occurrence");
+});
+
+test("uses Codex's full allowance for individual pairs without truncation", async () => {
+ const input = {
+ before: [finding("before", 600000)],
+ after: [finding("after")],
+ };
+ const calls: ScanComparisonInput[] = [];
+ const options = {
+ codex: codex((batch) => {
+ calls.push(batch);
+ return noMatches;
+ }),
+ };
+ expect(await matchScanFindings(input, options)).toEqual(noMatches);
+ expect(calls).toEqual([input]);
+ const oversized = {
+ before: [finding("before", 1048576)],
+ after: [finding("after")],
+ };
+ await expect(matchScanFindings(oversized, options)).rejects.toThrow(
+ "input limit",
+ );
+ expect(calls).toHaveLength(1);
+ expect(oversized.before[0]!.evidence).toHaveLength(1048576);
+});
+
+test.each([200000, 300000])(
+ "keeps automatic post-scan matching to at most one call (%p characters per finding)",
+ async (characters) => {
+ const input = {
+ before: Array.from({ length: 4 }, (_, index) =>
+ finding(`before-${index}`, characters),
+ ),
+ after: [finding("after")],
+ };
+ const calls: ScanComparisonInput[] = [];
+ const comparison = matchScanFindingsInternal(
+ input,
+ {
+ codex: codex((batch) => {
+ calls.push(batch);
+ return noMatches;
+ }),
+ },
+ { surface: "sdk", allowBatching: false },
+ );
+ if (characters === 300000) {
+ await expect(comparison).rejects.toThrow("input limit");
+ expect(calls).toEqual([]);
+ } else {
+ expect(await comparison).toEqual(noMatches);
+ expect(calls).toEqual([input]);
+ }
+ },
+);
+
+test("stops between batches when canceled", async () => {
+ const controller = new AbortController();
+ let calls = 0;
+ await expect(
+ matchScanFindings(
+ {
+ before: [finding("first", 300000), finding("second", 300000)],
+ after: [finding("after")],
+ },
+ {
+ signal: controller.signal,
+ codex: codex(() => {
+ calls++;
+ controller.abort();
+ return noMatches;
+ }),
+ },
+ ),
+ ).rejects.toThrow();
+ expect(calls).toBe(1);
+});
diff --git a/sdk/typescript/tests-ts/scan-comparison.test.ts b/sdk/typescript/tests-ts/scan-comparison.test.ts
index 5417ff9a4..991f664dd 100644
--- a/sdk/typescript/tests-ts/scan-comparison.test.ts
+++ b/sdk/typescript/tests-ts/scan-comparison.test.ts
@@ -406,7 +406,7 @@ describe("semantic scan comparison", () => {
},
);
- test("compares all findings with one restricted structured-output turn", async () => {
+ test("compares small inputs with one restricted structured-output turn", async () => {
const input: ScanComparisonInput = {
before: [finding("before-1"), finding("before-2")],
after: [finding("after-1"), finding("after-2"), finding("after-3")],