OpenAI's Codex Security CLI and TypeScript SDK for finding, validating, and fixing security vulnerabilities. npm: https://www.npmjs.com/package/@openai/codex-security
TypeScript ai-securityapplication-securityclicode-scanningcodexcodex-securitycybersecuritydevsecopsnodejsnpmopenaisdksecuritytypescriptvulnerability-scanning
fix(findings): guide cross-mode matching with saved evidence (#832)
Ivan Caceres6d356635eb7e
diff --git a/plugins/codex-security/scripts/workbench_scan_history.py b/plugins/codex-security/scripts/workbench_scan_history.pyindex 2c1de0a5e..f6719df2e 100644--- a/plugins/codex-security/scripts/workbench_scan_history.py+++ b/plugins/codex-security/scripts/workbench_scan_history.py@@ -450,6 +450,61 @@ def list_scans(return result+_FindingSites = dict[tuple[str, str | None, PurePosixPath, int, int], dict[str, Any] | None]+# Match the first location shown by compactFinding in finding-catalogue.ts.+_COMPARISON_LOCATION_RANK = {"root_control": 0, "expected_control": 1, "concrete_implementation": 1}+++def _unique_finding_sites(findings: list[dict[str, Any]]) -> _FindingSites:+ sites = {}+ for finding in findings:+ if not finding.get("locations"):+ return {}+ location = min(+ finding["locations"],+ key=lambda site: _COMPARISON_LOCATION_RANK.get(site.get("role"), 2),+ )+ key = (+ finding["ruleId"],+ finding["identity"].get("instance"),+ PurePosixPath(location["path"]),+ location["startLine"],+ location.get("endLine", location["startLine"]),+ )+ sites[key] = None if key in sites else finding+ return sites+++def _cross_mode_context(+ before: sqlite3.Row,+ after: sqlite3.Row,+ findings: dict[str, list[dict[str, Any]]],+ sites: dict[str, _FindingSites],+) -> dict[str, Any]:+ # Different repository subdirectories can share both the Git revision and clean digest.+ if (+ {before["mode"], after["mode"]} != {"standard", "deep"}+ or before["target_id"] is None+ or before["target_id"] != after["target_id"]+ or before["target_revision"] != after["target_revision"]+ or before["target_snapshot_digest"] is None+ or before["target_snapshot_digest"] != after["target_snapshot_digest"]+ or not findings[before["id"]]+ or not findings[after["id"]]+ ):+ return {}+ for scan_id in (before["id"], after["id"]):+ if scan_id not in sites:+ sites[scan_id] = _unique_finding_sites(findings[scan_id])+ later = sites[after["id"]]+ occurrences = []+ for site, old in sites[before["id"]].items():+ new = later.get(site)+ if old and new and old["identity"]["anchor"] != new["identity"]["anchor"]:+ occurrences.extend((old["occurrenceId"], new["occurrenceId"]))+ return {"crossModeOccurrenceIds": occurrences} if occurrences else {}++def list_unmatched_scan_pairs(connection: sqlite3.Connection,args: argparse.Namespace,@@ -492,6 +547,7 @@ def list_unmatched_scan_pairs(batches = []skipped = 0matching_findings: dict[str, list[dict[str, Any]]] = {}+ sites: dict[str, _FindingSites] = {}known_links: list[sqlite3.Row] | None = Nonefor index, (selected_index, after) in enumerate(available):previous = [@@ -524,6 +580,7 @@ def list_unmatched_scan_pairs("afterScanId": after["id"],"beforeScans": [{+ **_cross_mode_context(before, after, matching_findings, sites),"findings": matching_findings[before["id"]],"scanId": before["id"],}@@ -772,10 +829,15 @@ def compare_scans(],known_scan_ids,)+ inputs = {+ before["id"]: [_matching_input(row) for row in before_findings.values()],+ after["id"]: [_matching_input(row) for row in after_findings.values()],+ }result["matchingCached"] = cached is not Noneresult["matchingInputs"] = {- "before": [_matching_input(row) for row in before_findings.values()],- "after": [_matching_input(row) for row in after_findings.values()],+ "before": inputs[before["id"]],+ "after": inputs[after["id"]],+ **_cross_mode_context(before, after, inputs, {}),**({"knownFindingGroups": known_groups} if known_groups else {}),}return resultdiff --git a/plugins/codex-security/tests/test_workbench_scan_history.py b/plugins/codex-security/tests/test_workbench_scan_history.pyindex 9966e6ddf..4afe3f4d3 100644--- a/plugins/codex-security/tests/test_workbench_scan_history.py+++ b/plugins/codex-security/tests/test_workbench_scan_history.py@@ -184,6 +184,8 @@ def create_cli_scan(extra_anchors: tuple[str, ...] = (),finding: bool = True,identity_anchor: str = "archive-entry-write-without-containment",+ identity_instance: str | None = None,+ locations: list[dict[str, Any]] | None = None,mode: str = "standard",parent_scan_id: str | None = None,paths: list[str] | None = None,@@ -242,9 +244,13 @@ def create_cli_scan(target_revision=target_revision,snapshot_digest=snapshot_digest,)- if not finding or extra_anchors:+ if not finding or extra_anchors or identity_instance is not None or locations is not None:findings_path = scan_dir / "findings.json"findings = json.loads(findings_path.read_text())+ if locations is not None:+ findings["findings"][0]["locations"] = locations+ if identity_instance is not None:+ findings["findings"][0]["identity"]["instance"] = identity_instanceif not finding:findings["findings"] = []else:@@ -1092,6 +1098,127 @@ def test_scan_comparison_requires_saved_matches_and_remains_read_only(tmp_path:assert connection.execute("SELECT * FROM finding_occurrences").fetchall() == occurrences+def test_scan_matching_selects_cross_mode_evidence_for_compatible_instances(tmp_path: Path) -> None:+ state_dir, repository = empty_history_paths(tmp_path)+ root = tmp_path / "results"+ before = {+ instance: create_cli_scan(state_dir, root, repository, identity_instance=instance)+ for instance in (None, "entry")+ }+ after = {+ instance: create_cli_scan(+ state_dir,+ root,+ repository,+ mode="deep",+ identity_anchor="unchecked-extraction",+ identity_instance=instance,+ )+ for instance in (None, "entry", "other")+ }+ for old_instance, old in before.items():+ for new_instance, new in after.items():+ inputs = compare_scan_pair(state_dir, old, new, "--include-matching-inputs")[+ "matchingInputs"+ ]+ if old_instance == new_instance:+ assert inputs["crossModeOccurrenceIds"] == [+ inputs[side][0]["occurrenceId"] for side in ("before", "after")+ ]+ else:+ assert "crossModeOccurrenceIds" not in inputs+++def test_cross_mode_matching_uses_the_first_displayed_control(tmp_path: Path) -> None:+ state_dir, repository = empty_history_paths(tmp_path)+ root = tmp_path / "results"+ witness = {"role": "source", "path": "request.py", "startLine": 2}+ primary = {"role": "expected_control", "path": "extract.py", "startLine": 41}+ secondary = {"role": "concrete_implementation", "path": "preview.py", "startLine": 66}+ before = create_cli_scan(state_dir, root, repository, locations=[witness, primary, secondary])+ for locations, same_control in [+ ([{**witness, "path": "upload.py"}, {**primary, "role": "concrete_implementation"}], True),+ ([witness, secondary, primary], False),+ ]:+ after = create_cli_scan(+ state_dir,+ root,+ repository,+ mode="deep",+ identity_anchor="unchecked-write",+ locations=locations,+ )+ inputs = compare_scan_pair(state_dir, before, after, "--include-matching-inputs")[+ "matchingInputs"+ ]+ if same_control:+ assert inputs["crossModeOccurrenceIds"] == [+ inputs[side][0]["occurrenceId"] for side in ("before", "after")+ ]+ else:+ assert "crossModeOccurrenceIds" not in inputs+++def test_matching_plan_keeps_cross_mode_context_with_each_scan_pair(tmp_path: Path) -> None:+ state_dir, repository = empty_history_paths(tmp_path)+ root = tmp_path / "results"+ before = create_cli_scan(state_dir, root, repository)+ after = create_cli_scan(+ state_dir, root, repository, mode="deep", identity_anchor="unchecked-write"+ )+ # In mixed history, only Standard vs Deep pairs receive this context.+ later = create_cli_scan(+ state_dir, root, repository, mode="deep", identity_anchor="untrusted-output"+ )+ plan = run_workbench(state_dir, "list-unmatched-scan-pairs", "--repository", str(repository))+ first, second = plan["batches"]+ old = first["beforeScans"][0]+ assert old["crossModeOccurrenceIds"] == [+ old["findings"][0]["occurrenceId"],+ first["afterFindings"][0]["occurrenceId"],+ ]+ assert second["afterScanId"] == later["scanId"]+ prior, same_mode = second["beforeScans"]+ assert prior["crossModeOccurrenceIds"] == [+ old["findings"][0]["occurrenceId"],+ second["afterFindings"][0]["occurrenceId"],+ ]+ assert "crossModeOccurrenceIds" not in same_mode+ compared = save_scan_matches(state_dir, before, after)+ assert sorted(item["status"] for item in compared["findings"]) == ["new", "not_observed"]+++def test_scan_comparison_keeps_changed_content_and_ambiguous_locations_for_semantic_matching(+ tmp_path: Path,+) -> None:+ state_dir = tmp_path / "state"+ repository = tmp_path / "repository"+ repository.mkdir()+ (repository / "extract.py").write_text("original\n")+ root = tmp_path / "results"+ original = create_cli_scan(state_dir, root, repository)+ (repository / "extract.py").write_text("changed\n")+ changed = create_cli_scan(+ state_dir, root, repository, mode="deep", identity_anchor="renamed-archive-entry-write"+ )+ compared = compare_scan_pair(state_dir, original, changed, "--include-matching-inputs")+ assert "crossModeOccurrenceIds" not in compared["matchingInputs"]+ assert compared["summary"]["new"] == 1+ assert compared["summary"]["not_observed"] == 1++ siblings = create_cli_scan(+ state_dir,+ root,+ repository,+ identity_anchor="sibling-archive-entry-write",+ extra_anchors=("second-sibling-archive-entry-write",),+ )+ ambiguous = compare_scan_pair(state_dir, changed, siblings, "--include-matching-inputs")+ assert "crossModeOccurrenceIds" not in ambiguous["matchingInputs"]+ assert ambiguous["summary"]["new"] == 2+ assert ambiguous["summary"]["not_observed"] == 1++def test_list_unmatched_scan_pairs_groups_pending_pairs_and_skips_saved_results(tmp_path: Path,) -> None:diff --git a/sdk/typescript/docs/cli.md b/sdk/typescript/docs/cli.mdindex f7ae7402b..8bf0d386d 100644--- a/sdk/typescript/docs/cli.md+++ b/sdk/typescript/docs/cli.md@@ -2156,6 +2156,13 @@ One comparison ID is compared with the latest completed scan.codex-security scans match --all --force --model gpt-6.1-sol --effort high```+When a saved Standard scan and a Deep scan have the same recorded target,+revision, and snapshot digest, matching flags findings that have different+anchors but share a unique rule, first displayed source range, and optional+instance. Matching preserves the case of saved paths. Codex can request the+saved evidence to distinguish separate causes at that site. Automatic matching+under a cost limit keeps its single comparison call.+Only high-confidence root-cause matches are grouped. Uncertain and relatedfindings remain separate. Rebuilding retains stable identities, triage, andsealed artifacts; interruption preserves completed comparisons. Model/effortdiff --git a/sdk/typescript/src/cli.ts b/sdk/typescript/src/cli.tsindex a4976798f..f9aa8bec8 100644--- a/sdk/typescript/src/cli.ts+++ b/sdk/typescript/src/cli.ts@@ -301,6 +301,7 @@ import {} from "./runtime.js";import {comparisonFindingGroups,+ selectedComparisonContext,comparisonForScan,matchScanFindingsInternal,unionFindingGroups,@@ -8083,6 +8084,7 @@ async function matchAllScans(const input: ScanComparisonInput = {before,after: afterFindings,+ ...selectedComparisonContext(beforeScans),...(knownGroups.length === 0 ? {} : { knownFindingGroups: knownGroups }),};const matching =diff --git a/sdk/typescript/src/scan-comparison.ts b/sdk/typescript/src/scan-comparison.tsindex bb5856203..4b64d5d2b 100644--- a/sdk/typescript/src/scan-comparison.ts+++ b/sdk/typescript/src/scan-comparison.ts@@ -87,13 +87,27 @@ export interface ScanComparisonInput {knownFindingGroups?: readonly (readonly string[])[];}+interface SavedComparisonContext {+ crossModeOccurrenceIds?: readonly string[];+}+export interface ScanMatchingBatch {afterScanId: string;afterFindings: readonly Finding[];- beforeScans: { scanId: string; findings: readonly Finding[] }[];+ beforeScans: ({+ scanId: string;+ findings: readonly Finding[];+ } & SavedComparisonContext)[];knownFindingGroups?: readonly (readonly string[])[];}+export function selectedComparisonContext(+ scans: ScanMatchingBatch["beforeScans"],+): SavedComparisonContext {+ const ids = scans.flatMap((scan) => scan.crossModeOccurrenceIds ?? []);+ return ids.length ? { crossModeOccurrenceIds: ids } : {};+}+export interface ScanComparisonProgress {phase: "catalogue" | "evidence" | "complete";beforeFindings: number;@@ -253,7 +267,7 @@ export async function matchScanFindings(}export async function matchScanFindingsInternal(- input: ScanComparisonInput,+ input: ScanComparisonInput & SavedComparisonContext,options: ScanComparisonOptions = {},runtimeOptions: { surface: CodexSecuritySurface; singleTurn?: boolean },): Promise<ScanComparisonResult> {@@ -276,6 +290,18 @@ export async function matchScanFindingsInternal(before: [...catalogue.values()].map(({ card }) => card),after: input.after.map(compactFinding),};+ const crossModeSites =+ !runtimeOptions.singleTurn && Boolean(input.crossModeOccurrenceIds?.length);+ if (crossModeSites) {+ const candidates = new Set(input.crossModeOccurrenceIds);+ for (const { card, occurrences } of catalogue.values()) {+ if (occurrences.some((finding) => candidates.has(finding.occurrenceId)))+ card["crossModeSite"] = true;+ }+ for (const card of initialCatalogue.after) {+ if (candidates.has(card.occurrenceId)) card["crossModeSite"] = true;+ }+ }// Cost-limited scans retain the existing one-call post-scan allowance.if (runtimeOptions.singleTurn &&@@ -286,7 +312,7 @@ export async function matchScanFindingsInternal(}const pages = runtimeOptions.singleTurn? [initialCatalogue]- : cataloguePages(initialCatalogue);+ : cataloguePages(initialCatalogue, crossModeSites);const omittedEvidence = {before: new Set<string>(),after: new Set<string>(),@@ -330,7 +356,7 @@ export async function matchScanFindingsInternal(cyberAccessProgram: options.cyberAccessProgram,signal: options.signal,};- let prompt = comparisonPrompt(pages[0]!, 0, pages.length);+ let prompt = comparisonPrompt(pages[0]!, 0, pages.length, crossModeSites);progress("catalogue", 1);try {for (;;) {@@ -394,7 +420,12 @@ export async function matchScanFindingsInternal("Scan comparison repeated a request without making progress.",);}- prompt = comparisonPrompt(page, request.page, pages.length);+ prompt = comparisonPrompt(+ page,+ request.page,+ pages.length,+ crossModeSites,+ );progress("catalogue", request.page + 1);} else {request.beforeOccurrenceIds = [@@ -814,6 +845,7 @@ export async function matchCompletedScan(const input: ScanComparisonInput = {before: beforeScans.flatMap(({ findings }) => findings),after: batch.afterFindings,+ ...selectedComparisonContext(beforeScans),...(batch.knownFindingGroups === undefined? {}: { knownFindingGroups: batch.knownFindingGroups }),@@ -993,6 +1025,7 @@ function comparisonPrompt(input: CataloguePage,page: number,pages: number,+ crossModeSites = false,): string {return ["Compare every finding from one or more earlier scans against a later scan of the same repository.",@@ -1000,6 +1033,11 @@ function comparisonPrompt("Different routes reaching the same vulnerable helper share one root cause. Group findings when either scan split or combined that issue.","When several earlier scans contain the same issue, include every earlier occurrence in one group with the matching later occurrences.","Keep distinct independently vulnerable controls or instances separate.",+ ...(crossModeSites+ ? [+ "Cards marked crossModeSite include an occurrence with a cross-mode candidate sharing its rule and site despite different anchors. Request full evidence if the control or fix is unclear.",+ ]+ : []),"The earlier findings form a catalogue of known issues. Each top-level before occurrenceId represents that issue. Its earlierDescriptions contain fields that differ from the current card. Return the top-level IDs; the host expands the saved historical occurrences.","Judge the defective control, failed security invariant, trust boundary, and smallest root-cause correction. Similar titles, CWE labels, or broad hardening advice do not establish a duplicate.","Return only high-confidence matches; put plausible uncertain pairs in uncertain. Use related for findings that are meaningfully related but have distinct root causes. Each occurrenceId may appear in only one confirmed group.",@@ -1010,16 +1048,20 @@ function comparisonPrompt(].join("\n");}-function cataloguePages(input: CataloguePage): CataloguePage[] {+function cataloguePages(+ input: CataloguePage,+ crossModeSites: boolean,+): CataloguePage[] {if (- characterCount(comparisonPrompt(input, 0, 1)) <= MAX_CODEX_INPUT_CHARACTERS+ characterCount(comparisonPrompt(input, 0, 1, crossModeSites)) <=+ MAX_CODEX_INPUT_CHARACTERS) {return [input];}const maximumPages = input.before.length + input.after.length;const empty = (): CataloguePage => ({ before: [], after: [] });const overhead = characterCount(- comparisonPrompt(empty(), maximumPages, maximumPages),+ comparisonPrompt(empty(), maximumPages, maximumPages, crossModeSites),);const pages: CataloguePage[] = [];let page = empty();@@ -1029,7 +1071,11 @@ function cataloguePages(input: CataloguePage): CataloguePage[] {let card = original;let length = characterCount(JSON.stringify(card));if (overhead + length > MAX_CODEX_INPUT_CHARACTERS) {- card = { occurrenceId: original.occurrenceId, detailsOmitted: true };+ card = {+ occurrenceId: original.occurrenceId,+ detailsOmitted: true,+ ...(original["crossModeSite"] === true && { crossModeSite: true }),+ };length = characterCount(JSON.stringify(card));if (overhead + length > MAX_CODEX_INPUT_CHARACTERS) {throw new CodexSecurityError(diff --git a/sdk/typescript/tests-ts/finding-catalogue.test.ts b/sdk/typescript/tests-ts/finding-catalogue.test.tsindex deb27fd68..d254b1a1d 100644--- a/sdk/typescript/tests-ts/finding-catalogue.test.ts+++ b/sdk/typescript/tests-ts/finding-catalogue.test.ts@@ -645,13 +645,27 @@ describe("finding catalogue", () => {const identity = (prefix: string) => `${prefix}🙂${"x".repeat(360_000)}`;const beforeIds = [identity("before-a"), identity("before-b")];const afterIds = [identity("after")];+ const crossModeOccurrenceIds = [beforeIds[0]!, afterIds[0]!];const rootCause = "🙂".repeat(700_000);const selected = new Set<string>();const selections = new Set<string>();const observed = conversation((prompt) => {expect(characters(prompt)).toBeLessThanOrEqual(1 << 20);const payload = data<CatalogueData | EvidenceData>(prompt);- if (!("content" in payload)) return empty;+ if (!("content" in payload)) {+ for (const card of [+ ...payload.findings.before,+ ...payload.findings.after,+ ]) {+ expect(card["detailsOmitted"]).toBe(true);+ expect(card["crossModeSite"]).toBe(+ crossModeOccurrenceIds.includes(card.occurrenceId)+ ? true+ : undefined,+ );+ }+ return empty;+ }for (const identity of [...payload.beforeOccurrenceIds,@@ -669,12 +683,16 @@ describe("finding catalogue", () => {});expect(- await matchScanFindings(+ await matchScanFindingsInternal({- before: beforeIds.map((id) => finding(id, { rootCause })),+ before: beforeIds.map((id) =>+ finding(id, { rootCause, crossModeSite: true }),+ ),+ crossModeOccurrenceIds,after: afterIds.map((id) => finding(id, { rootCause })),},{ codex: observed.codex },+ { surface: "cli" },),).toEqual(empty);expect(selected).toEqual(new Set([...beforeIds, ...afterIds]));diff --git a/sdk/typescript/tests-ts/scan-comparison.test.ts b/sdk/typescript/tests-ts/scan-comparison.test.tsindex 0cc0b97a7..e1e7f0c36 100644--- a/sdk/typescript/tests-ts/scan-comparison.test.ts+++ b/sdk/typescript/tests-ts/scan-comparison.test.ts@@ -74,14 +74,16 @@ function finding(occurrenceId: string): ScanComparisonInput["before"][number] {function fakeCodex(response: unknown) {const calls: {prompt?: string;+ turns: number;threadOptions?: ThreadOptions;turnOptions?: TurnOptions;- } = {};+ } = { turns: 0 };const codex: NonNullable<ScanComparisonOptions["codex"]> = {startThread(options) {calls.threadOptions = options;return {async run(prompt, turnOptions) {+ calls.turns++;calls.prompt = prompt;calls.turnOptions = turnOptions;return { finalResponse: modelResponseText(response) };@@ -128,6 +130,43 @@ function captureProfileClient() {}describe("semantic scan comparison", () => {+ test.each([false, true])(+ "suggests full evidence for selected cross-mode sites (single turn %s)",+ async (singleTurn) => {+ const { codex, calls } = fakeCodex({ matches: [], uncertain: [] });+ await matchScanFindingsInternal(+ {+ before: [+ { occurrenceId: "older", findingId: "same" },+ { occurrenceId: "representative", findingId: "same" },+ { occurrenceId: "unrelated" },+ ],+ after: [+ {+ occurrenceId: "current",+ rootCause: { code: "synthetic source excerpt" },+ },+ ],+ crossModeOccurrenceIds: ["older", "current"],+ },+ { codex },+ { surface: "cli", singleTurn },+ );+ const { before, after } = JSON.parse(+ calls.prompt!.split("\n").at(-1)!,+ ).findings;+ expect(before[0].occurrenceId).toBe("representative");+ expect(Boolean(before[0].crossModeSite)).toBe(!singleTurn);+ expect(before[1].crossModeSite).toBeUndefined();+ expect(Boolean(after[0].crossModeSite)).toBe(!singleTurn);+ expect(calls.prompt!.includes("Cards marked crossModeSite")).toBe(+ !singleTurn,+ );+ expect(calls.prompt).not.toContain("synthetic source excerpt");+ expect(calls.turns).toBe(1);+ },+ );+test.each([["sdk", "matching"],["cli", "matching"],@@ -2181,7 +2220,14 @@ describe("semantic scan comparison", () => {afterScanId: "current",afterFindings: [after],beforeScans: [- { scanId: "unselected", findings: [unselected] },+ {+ scanId: "unselected",+ findings: [unselected],+ crossModeOccurrenceIds: [+ unselected.occurrenceId,+ after.occurrenceId,+ ],+ },{ scanId: "first", findings: [firstShared, firstOther] },{ scanId: "latest", findings: [latestShared] },],