Skip to content

OpenAI's Codex Security CLI and TypeScript SDK for finding, validating, and fixing security vulnerabilities. npm: https://www.npmjs.com/package/@openai/codex-security

Automatically match scan comparisons (#114)

ianw-oai630810bacbb8
diff --git a/README.md b/README.md
index 58953319b..293ae1ebf 100644
--- a/README.md
+++ b/README.md
@@ -46,6 +46,11 @@ Scan history is stored in the Codex Security workbench state directory. If that
directory cannot be written, set `CODEX_SECURITY_STATE_DIR` to a writable
directory outside the repository.
+`scans compare BEFORE_SCAN_ID AFTER_SCAN_ID` automatically matches findings by
+root cause, reuses saved matches, and identifies new, persisting, reopened,
+resolved, or unknown findings. Missing findings remain unknown when coverage is
+incomplete or their original location was not reviewed.
+
## TypeScript SDK
```ts
diff --git a/sdk/typescript/README.md b/sdk/typescript/README.md
index d36e5d82b..9e1f4a512 100644
--- a/sdk/typescript/README.md
+++ b/sdk/typescript/README.md
@@ -441,10 +441,10 @@ cause; `scans match --all` matches all completed scans of the current repository
including other worktrees and clones. Saved matches appear in `scans show` and
are reused unless `--force` is passed. Scans without sealed artifacts are skipped.
-`scans compare BEFORE_SCAN_ID AFTER_SCAN_ID` reads saved matches and reports
-findings as new, persisting, reopened, resolved, or unknown. Missing findings
-are not treated as resolved when the later scan is incomplete or does not cover
-their original scope.
+`scans compare BEFORE_SCAN_ID AFTER_SCAN_ID` automatically matches findings by
+root cause, reuses saved matches, and reports findings as new, persisting,
+reopened, resolved, or unknown. Missing findings are not treated as resolved when
+the later scan is incomplete or does not cover their original scope.
The CLI uses [Incur](https://github.com/wevm/incur) for agent-friendly discovery
and structured output. Inspect the command manifest with `--llms`, inspect a
diff --git a/sdk/typescript/src/cli.ts b/sdk/typescript/src/cli.ts
index 924023849..12c3cd7d0 100644
--- a/sdk/typescript/src/cli.ts
+++ b/sdk/typescript/src/cli.ts
@@ -600,16 +600,48 @@ export async function main(
let renderedHistory: string | undefined;
const history = async (
args: readonly string[],
- select: (value: JsonObject) => JsonObject = (value) => value,
+ select: (value: JsonObject) => JsonObject | Promise<JsonObject> = (value) =>
+ value,
): Promise<JsonObject | undefined> => {
try {
- return select(await dependencies.runWorkbench(args));
+ return await select(await dependencies.runWorkbench(args));
} catch (error) {
errorOutput.write(`codex-security: ${cliErrorMessage(error)}\n`);
exitCode = 2;
return undefined;
}
};
+ const matchScanPair = async (
+ beforeId: string,
+ afterId: string,
+ force = false,
+ ): Promise<JsonObject | undefined> =>
+ history(
+ [
+ "compare-scans",
+ "--before-scan-id",
+ beforeId,
+ "--after-scan-id",
+ afterId,
+ "--include-matching-inputs",
+ ],
+ async ({ matchingCached, matchingInputs, ...comparison }) => {
+ if (matchingCached && !force) return comparison;
+ return await dependencies.runWorkbench([
+ "save-scan-comparison",
+ "--before-scan-id",
+ beforeId,
+ "--after-scan-id",
+ afterId,
+ "--matches-json",
+ JSON.stringify(
+ await dependencies.matchFindings(
+ matchingInputs as JsonObject & ScanComparisonInput,
+ ),
+ ),
+ ]);
+ },
+ );
const presentHistory = (
result: JsonObject | undefined,
command: HistoryCommand,
@@ -830,34 +862,8 @@ export async function main(
format,
);
}
- const comparison = await history([
- "compare-scans",
- "--before-scan-id",
- args.beforeId!,
- "--after-scan-id",
- args.afterId!,
- "--include-matching-inputs",
- ]);
- if (comparison === undefined) return undefined;
- const { matchingCached, matchingInputs, ...visibleComparison } =
- comparison;
- if (matchingCached && !options.force) {
- return presentHistory(visibleComparison, "compare", format);
- }
-
- const matching = await dependencies.matchFindings(
- matchingInputs as JsonObject & ScanComparisonInput,
- );
return presentHistory(
- await history([
- "save-scan-comparison",
- "--before-scan-id",
- args.beforeId!,
- "--after-scan-id",
- args.afterId!,
- "--matches-json",
- JSON.stringify(matching),
- ]),
+ await matchScanPair(args.beforeId!, args.afterId!, options.force),
"compare",
format,
);
@@ -869,7 +875,8 @@ export async function main(
},
})
.command("compare", {
- description: "Compare findings and coverage using saved matches.",
+ description: "Match and compare findings and coverage between scans.",
+ destructive: true,
mcp: false,
args: z.object({
beforeId: z.string().min(1).describe("Earlier saved scan identifier."),
@@ -878,14 +885,7 @@ export async function main(
output: z.record(z.string(), z.unknown()).optional(),
async run({ args, format }) {
return presentHistory(
- await history([
- "compare-scans",
- "--before-scan-id",
- args.beforeId,
- "--after-scan-id",
- args.afterId,
- "--require-matches",
- ]),
+ await matchScanPair(args.beforeId, args.afterId),
"compare",
format,
);
diff --git a/sdk/typescript/tests-ts/cli-workbench.test.ts b/sdk/typescript/tests-ts/cli-workbench.test.ts
index 3995d78a6..90156a43a 100644
--- a/sdk/typescript/tests-ts/cli-workbench.test.ts
+++ b/sdk/typescript/tests-ts/cli-workbench.test.ts
@@ -121,10 +121,12 @@ describe("CLI workbench", () => {
"before",
"--after-scan-id",
"after",
- "--require-matches",
+ "--include-matching-inputs",
],
{
comparable: true,
+ matchingCached: true,
+ matchingInputs: { before: [], after: [] },
summary: { persisting: 1, resolved: 1 },
},
{ comparable: true, summary: { persisting: 1, resolved: 1 } },
@@ -169,7 +171,7 @@ describe("CLI workbench", () => {
}
});
- test("matches findings and saves the result", async () => {
+ test("matches findings before matching or comparing scans", async () => {
const before = [{ occurrenceId: "before" }];
const after = [{ occurrenceId: "after" }];
const matching = {
@@ -183,34 +185,64 @@ describe("CLI workbench", () => {
],
uncertain: [],
};
- const calls: Array<readonly string[]> = [];
- const stdout = capture();
+
+ for (const command of ["match", "compare"]) {
+ const calls: Array<readonly string[]> = [];
+ const stdout = capture();
+
+ expect(
+ await main(
+ ["scans", command, "before", "after", "--json"],
+ stdout.stream,
+ capture().stream,
+ dependencies({
+ onWorkbench: (args): JsonObject => {
+ calls.push(args);
+ return args[0] === "compare-scans"
+ ? { matchingCached: false, matchingInputs: { before, after } }
+ : { summary: { persisting: 1 } };
+ },
+ onMatch: async (input) => {
+ expect(input).toEqual({ before, after });
+ return matching;
+ },
+ }),
+ ),
+ ).toBe(0);
+ expect(calls.map((args) => args[0])).toEqual([
+ "compare-scans",
+ "save-scan-comparison",
+ ]);
+ expect(JSON.parse(calls[1]![6]!)).toEqual(matching);
+ expect(JSON.parse(stdout.text())).toEqual({ summary: { persisting: 1 } });
+ }
+ });
+
+ test("reports automatic matching failures without saving a comparison", async () => {
+ const calls: string[] = [];
+ const stderr = capture();
expect(
await main(
- ["scans", "match", "before", "after", "--json"],
- stdout.stream,
+ ["scans", "compare", "before", "after"],
capture().stream,
+ stderr.stream,
dependencies({
- onWorkbench: (args): JsonObject => {
- calls.push(args);
- return args[0] === "compare-scans"
- ? { matchingCached: false, matchingInputs: { before, after } }
- : { summary: { persisting: 1 } };
+ onWorkbench: (args) => {
+ calls.push(args[0]!);
+ return {
+ matchingCached: false,
+ matchingInputs: { before: [], after: [] },
+ };
},
- onMatch: async (input) => {
- expect(input).toEqual({ before, after });
- return matching;
+ onMatch: async () => {
+ throw new Error("Root-cause matching failed.");
},
}),
),
- ).toBe(0);
- expect(calls.map((args) => args[0])).toEqual([
- "compare-scans",
- "save-scan-comparison",
- ]);
- expect(JSON.parse(calls[1]![6]!)).toEqual(matching);
- expect(JSON.parse(stdout.text())).toEqual({ summary: { persisting: 1 } });
+ ).toBe(2);
+ expect(stderr.text()).toContain("Root-cause matching failed.");
+ expect(calls).toEqual(["compare-scans"]);
});
test("matches all scans once per later scan", async () => {