Skip to content

OpenAI's Codex Security CLI and TypeScript SDK for finding, validating, and fixing security vulnerabilities. npm: https://www.npmjs.com/package/@openai/codex-security

fix(cli): preserve complete evidence for saved finding remediation (#1608)

Michael D'Angelo11e33e0d9352
diff --git a/sdk/typescript/docs/cli.md b/sdk/typescript/docs/cli.md
index 0299c96f9..26a43ed65 100644
--- a/sdk/typescript/docs/cli.md
+++ b/sdk/typescript/docs/cli.md
@@ -1771,6 +1771,8 @@ Both candidate IDs remain in the output, including the unresolved claim.
`--scan latest` uses the current repository's latest completed scan. Patching
supports JSON for saved findings, literal text, and files. Each finding gets a
saved Codex desktop task. `--model` and `--effort` control the model.
+Saved-scan patching and fix verification use the complete sealed finding evidence
+and exclude findings that are currently closed in triage.
`--validation-prompt-file PATH` supplies setup, authorized targets, expected
results, and cleanup for dynamic validation. It works with saved findings,
diff --git a/sdk/typescript/src/cli.ts b/sdk/typescript/src/cli.ts
index 6683d8542..d1aa7dc83 100644
--- a/sdk/typescript/src/cli.ts
+++ b/sdk/typescript/src/cli.ts
@@ -1186,6 +1186,7 @@ interface CliDependencies {
linearClient?: LinearClientFactory;
importGitHubAlerts?: typeof importGitHubCodeScanningAlerts;
importScan?: typeof importScan;
+ readSavedFindings?: typeof readSavedFindings;
runWorkbench(
args: readonly string[],
input?: string,
@@ -6835,13 +6836,64 @@ async function selectSavedFindings(
"The requested finding does not belong to the selected scan.",
);
}
+ const fullFindings =
+ selected.length === 0
+ ? []
+ : await (dependencies.readSavedFindings ?? readSavedFindings)(
+ scan.scanId,
+ dependencies.runWorkbench,
+ );
+ const byOccurrence = new Map(
+ fullFindings.map((finding) => [finding.occurrenceId, finding]),
+ );
return {
repository: scan.targetPath,
scanId: scan.scanId,
- findings: selected,
+ findings: selected.map((finding) => {
+ const full = byOccurrence.get(finding.occurrenceId);
+ if (full === undefined || full.findingId !== finding.findingId) {
+ throw new CodexSecurityError(
+ `Finding ${finding.occurrenceId} is missing from the saved scan artifacts.`,
+ );
+ }
+ const hydrated = { ...finding, ...full };
+ // Sealed artifacts supply evidence; the workbench owns live and derived state.
+ for (const key of [
+ "createdAt",
+ "triage",
+ "remediationState",
+ "matches",
+ "knownSince",
+ "knownScanIds",
+ "related",
+ "sourceExcerpt",
+ "artifactPaths",
+ ]) {
+ if (Object.hasOwn(finding, key)) hydrated[key] = finding[key];
+ else delete hydrated[key];
+ }
+ return hydrated;
+ }),
};
}
+async function readSavedFindings(
+ scanId: string,
+ workbench: CliDependencies["runWorkbench"],
+): Promise<readonly Finding[]> {
+ const directory = await resolveSavedArtifactDirectory(
+ scanId,
+ "findings",
+ "json",
+ workbench,
+ );
+ const { findings } = await loadContract(directory, {
+ pluginRoot: await bundledPluginRoot(),
+ expectedScanId: scanId,
+ });
+ return findings.findings;
+}
+
function patchExitCode(patches: readonly FindingPatch[]): number {
if (patches.some(({ status }) => status === "failed")) return 2;
return patches.some(({ status }) => status === "blocked") ? 1 : 0;
diff --git a/sdk/typescript/tests-ts/cli-fixtures.ts b/sdk/typescript/tests-ts/cli-fixtures.ts
index 7e8dffcbc..088d3c0e6 100644
--- a/sdk/typescript/tests-ts/cli-fixtures.ts
+++ b/sdk/typescript/tests-ts/cli-fixtures.ts
@@ -219,6 +219,7 @@ export function dependencies(
input?: string,
signal?: AbortSignal,
) => JsonObject | Promise<JsonObject>;
+ readSavedFindings?: MainDependencies["readSavedFindings"];
onMatch?: MainDependencies["matchFindings"];
onUpdateCheck?: (signal: AbortSignal) => Promise<UpdateNotice | undefined>;
currentDirectory?: string;
@@ -295,6 +296,8 @@ export function dependencies(
: { importGitHubAlerts: options.importGitHubAlerts }),
runWorkbench: async (args, input, signal) =>
(await options.onWorkbench?.(args, input, signal)) ?? { scans: [] },
+ readSavedFindings:
+ options.readSavedFindings ?? (async () => result.findings.findings),
matchFindings: async (input, comparisonOptions) =>
(await options.onMatch?.(input, comparisonOptions)) ?? {
matches: [],
diff --git a/sdk/typescript/tests-ts/cli-patch-results.test.ts b/sdk/typescript/tests-ts/cli-patch-results.test.ts
index 03a41fabe..8b46995f0 100644
--- a/sdk/typescript/tests-ts/cli-patch-results.test.ts
+++ b/sdk/typescript/tests-ts/cli-patch-results.test.ts
@@ -5,6 +5,7 @@ import { join } from "node:path";
import { promisify } from "node:util";
import { describe, expect, test } from "bun:test";
import { main, runCodexSkillCommand } from "../src/cli.js";
+import type { Finding } from "../src/models.js";
import { dependencies } from "./cli-fixtures.js";
import { createCliTest } from "./support/cli-run.js";
@@ -199,19 +200,20 @@ lines.on("line", (line) => {
test("rejects a verified saved-finding result when no files change", async () => {
await using fixture = await repositoryFixture();
+ const finding = {
+ occurrenceId: "occ_1",
+ findingId: "csf_1",
+ title: "Synthetic finding",
+ severity: { level: "high" },
+ locations: [{ path: "app.ts", startLine: 1 }],
+ };
const outcome = await fixture.patch(["--scan", "scan-1"], {
+ readSavedFindings: async () => [finding as Finding],
onWorkbench: () => ({
scan: {
scanId: "scan-1",
targetPath: fixture.directory,
- findings: [
- {
- occurrenceId: "occ_1",
- title: "Synthetic finding",
- severity: { level: "high" },
- locations: [{ path: "app.ts", startLine: 1 }],
- },
- ],
+ findings: [finding],
},
}),
onCodex: (_args, output) => {
diff --git a/sdk/typescript/tests-ts/cli-patch.test.ts b/sdk/typescript/tests-ts/cli-patch.test.ts
index 8159999c6..8dc1aaa88 100644
--- a/sdk/typescript/tests-ts/cli-patch.test.ts
+++ b/sdk/typescript/tests-ts/cli-patch.test.ts
@@ -517,6 +517,7 @@ describe("scan and patch workflow", () => {
],
{
currentDirectory: directory,
+ result,
onWorkbench: () => savedScan(result, "scan-1", repository),
onCodex: async (args, output) => {
calls++;
@@ -594,6 +595,7 @@ describe("scan and patch workflow", () => {
],
{
currentDirectory: directory,
+ result,
onWorkbench: () => savedScan(result, "scan-1", repository),
onCodex: (args, output) => {
started = true;
@@ -722,6 +724,7 @@ describe("scan and patch workflow", () => {
"--json",
],
{
+ result,
onWorkbench: () => savedScan(result),
onRepositoryCommand: (command, args) => {
repositoryCommands.push({ command, args });
@@ -1517,6 +1520,7 @@ describe("scan and patch workflow", () => {
const fixtures: Parameters<typeof dependencies>[0] = {
currentDirectory: repository,
+ result,
onWorkbench: () => savedScan(result, "scan-1", repository),
onCodex: async (args, output) => {
modelCalls += 1;
@@ -2049,6 +2053,7 @@ describe("scan and patch workflow", () => {
const outcome = await runWorkflow(
["patch", "--scan", "scan-1", "--severity", "high", "--json"],
{
+ result,
onWorkbench: (args): JsonObject => {
expect(args).toEqual(["get-scan", "--scan-id", "scan-1"]);
return savedScan(result);
@@ -2146,6 +2151,7 @@ describe("scan and patch workflow", () => {
["patch", "--scan", "scan-1", "--create-pr", "--json"],
{
environment,
+ result,
onWorkbench: () => savedScan(result),
onRepositoryCommand: (command, args, target) => {
expect(target).toBe(SAVED_REPOSITORY);
@@ -2376,6 +2382,7 @@ describe("scan and patch workflow", () => {
"--json",
],
{
+ result,
onWorkbench: () => savedScan(result),
onRepositoryCommand: (command, args) => {
if (command === "git") {
@@ -2435,6 +2442,7 @@ describe("scan and patch workflow", () => {
const outcome = await runWorkflow(
["patch", "--scan", "scan-1", "--create-pr", "--json"],
{
+ result,
onWorkbench: () => savedScan(result),
onRepositoryCommand: (command, args) => {
if (command === "git") {
@@ -2506,6 +2514,7 @@ describe("scan and patch workflow", () => {
const calls: Array<readonly string[]> = [];
let patched: Finding[] = [];
const outcome = await runWorkflow(["patch", "occ_1"], {
+ result,
onWorkbench: (args): JsonObject => {
calls.push(args);
if (args[0] === "list-global-findings") {
@@ -2535,6 +2544,7 @@ describe("scan and patch workflow", () => {
const calls: Array<readonly string[]> = [];
const outcome = await runWorkflow(["patch", "--scan", "latest"], {
currentDirectory: SAVED_REPOSITORY,
+ result,
onWorkbench: (args): JsonObject => {
calls.push(args);
if (args[0] === "list-scans") {
@@ -2555,6 +2565,7 @@ describe("scan and patch workflow", () => {
const patched: Finding[] = [];
const calls: Array<readonly string[]> = [];
const outcome = await runWorkflow(["patch", "--scan", "scan-1"], {
+ result,
onWorkbench: (args): JsonObject => {
calls.push(args);
if (args[0] === "get-scan") {
diff --git a/sdk/typescript/tests-ts/cli-saved-finding-evidence.test.ts b/sdk/typescript/tests-ts/cli-saved-finding-evidence.test.ts
new file mode 100644
index 000000000..8a9b2180f
--- /dev/null
+++ b/sdk/typescript/tests-ts/cli-saved-finding-evidence.test.ts
@@ -0,0 +1,246 @@
+import { mkdir, readFile, rm, writeFile } from "node:fs/promises";
+import { dirname, join } from "node:path";
+import { expect, test } from "bun:test";
+import { exportEnvironment, main } from "../src/cli.js";
+import type { Finding, FindingsDocument, ScanManifest } from "../src/models.js";
+import { resolvePluginPython } from "../src/runtime.js";
+import { dependencies } from "./cli-fixtures.js";
+import { copyCompletedScanFixture } from "./plugin-root.js";
+import { createCliTest } from "./support/cli-run.js";
+import { setFindingIdentity } from "./support/finding-identity.js";
+import { temporaryDirectory } from "./support/temporary-directories.js";
+import {
+ scanRegistrationArguments,
+ workbenchCommand,
+} from "./support/workbench-command.js";
+
+async function savedEvidence() {
+ const root = await temporaryDirectory("saved-finding-evidence-", true);
+ const repository = join(root, "repository");
+ const scanDir = join(root, "scan");
+ const environment = {
+ ...exportEnvironment(),
+ CODEX_SECURITY_STATE_DIR: join(root, "state"),
+ };
+ const workbench = workbenchCommand(
+ await resolvePluginPython(),
+ () => environment,
+ );
+ try {
+ await mkdir(repository);
+ await mkdir(scanDir, { mode: 0o700 });
+ const registered = await workbench(
+ scanRegistrationArguments(repository, scanDir),
+ );
+ const scanId = registered["scanId"] as string;
+ await copyCompletedScanFixture(scanDir);
+ const manifest = JSON.parse(
+ await readFile(join(scanDir, "scan-manifest.json"), "utf8"),
+ ) as ScanManifest;
+ manifest.scan.id = scanId;
+ manifest.scan.target.kind = "directory_snapshot";
+ manifest.scan.target.targetId = registered["targetId"] as string;
+ Reflect.deleteProperty(manifest.scan, "sealedAt");
+ Reflect.deleteProperty(manifest.scan, "artifacts");
+ const document = JSON.parse(
+ await readFile(join(scanDir, "findings.json"), "utf8"),
+ ) as FindingsDocument;
+ document.scanId = scanId;
+ const finding = document.findings[0]!;
+ const slug = "synthetic".repeat(28);
+ const reportPath = `findings/${slug}/${slug}.md`;
+ finding.codeEvidence = Array.from({ length: 9 }, (_, index) => ({
+ id: `evidence-${index}`,
+ label: `Synthetic evidence ${index}`,
+ path: "src/extract.py",
+ startLine: index + 1,
+ code: `synthetic_source_${index}();\n`.repeat(200),
+ explanation: "Synthetic source-to-sink evidence.",
+ }));
+ finding.validation = {
+ summary: "Synthetic validation evidence.",
+ counterEvidence: ["A mitigation remains unverified. ".repeat(300)],
+ };
+ finding.writeup = { reportPath };
+ Object.assign(finding, {
+ createdAt: "2000-01-01T00:00:00Z",
+ remediationState: { status: "fixed" },
+ matches: [{ occurrenceId: "stale-occurrence" }],
+ knownSince: "2000-01-01T00:00:00Z",
+ knownScanIds: ["stale-scan"],
+ related: [{ occurrenceId: "stale-related-occurrence" }],
+ sourceExcerpt: "Stale source excerpt.",
+ artifactPaths: ["stale-report.md"],
+ });
+ const closed = structuredClone(finding);
+ closed.identity.anchor = "closed-synthetic-finding";
+ delete closed.writeup;
+ document.findings.push(closed);
+ document.findings.forEach((item) =>
+ setFindingIdentity(manifest.scan, item),
+ );
+ await mkdir(dirname(join(scanDir, reportPath)), { recursive: true });
+ await writeFile(join(scanDir, reportPath), "# Synthetic finding report\n");
+ const coverage = JSON.parse(
+ await readFile(join(scanDir, "coverage.json"), "utf8"),
+ );
+ coverage.scanId = scanId;
+ await Promise.all([
+ writeFile(join(scanDir, "scan-manifest.json"), JSON.stringify(manifest)),
+ writeFile(join(scanDir, "findings.json"), JSON.stringify(document)),
+ writeFile(join(scanDir, "coverage.json"), JSON.stringify(coverage)),
+ ]);
+ await workbench(["complete-scan", "--scan-id", scanId]);
+ const canonical = JSON.parse(
+ await readFile(join(scanDir, "findings.json"), "utf8"),
+ ) as FindingsDocument;
+ expect(canonical.findings).toHaveLength(2);
+ await workbench([
+ "set-finding-triage",
+ "--occurrence-id",
+ canonical.findings[1]!.occurrenceId,
+ "--status",
+ "closed",
+ "--close-reason",
+ "false_positive",
+ "--note",
+ "Synthetic closed finding retained for selection coverage.",
+ ]);
+ return {
+ root,
+ repository,
+ scanDir,
+ scanId,
+ workbench,
+ canonical,
+ async [Symbol.asyncDispose]() {
+ await rm(root, { recursive: true, force: true });
+ },
+ };
+ } catch (error) {
+ await rm(root, { recursive: true, force: true });
+ throw error;
+ }
+}
+
+test("saved patch and verification tasks receive complete sealed evidence and live workbench state", async () => {
+ await using fixture = await savedEvidence();
+ const full = fixture.canonical.findings[0]!;
+ expect(full.codeEvidence).toHaveLength(9);
+ const preview = await fixture.workbench([
+ "get-scan",
+ "--scan-id",
+ fixture.scanId,
+ ]);
+ const previewFinding = (preview["scan"] as { findings: Finding[] })
+ .findings[0]!;
+ expect(previewFinding.codeEvidence).toHaveLength(4);
+ expect(previewFinding.writeup?.reportPath).not.toBe(full.writeup!.reportPath);
+
+ for (const command of ["patch", "verify-fix"]) {
+ const { runCli, stderr } = createCliTest(main);
+ const received: Finding[] = [];
+ const current = dependencies({
+ onWorkbench: fixture.workbench,
+ onRepositoryCommand: (_command, args) =>
+ args.includes("--name-only") ? "src/extract.py\0" : "",
+ onCodex: (_args, output) => {
+ const findings = JSON.parse(
+ output!.appServer!.prompt.split("\n").at(-1)!,
+ ) as Finding[];
+ received.push(...findings);
+ output!.stdout.write(
+ JSON.stringify(
+ command === "patch"
+ ? {
+ patches: findings.map(({ occurrenceId }) => ({
+ occurrenceId,
+ status: "verified",
+ files: ["src/extract.py"],
+ verification: "Synthetic regression passed.",
+ })),
+ }
+ : {
+ results: findings.map(({ occurrenceId }) => ({
+ id: occurrenceId,
+ status: "fixed",
+ evidence: "Synthetic regression passed.",
+ })),
+ },
+ ),
+ );
+ return 0;
+ },
+ });
+ delete current.readSavedFindings;
+ expect(
+ await runCli([command, "--scan", fixture.scanId, "--json"], current),
+ stderr.text(),
+ ).toBe(0);
+ expect(received).toHaveLength(1);
+ expect(received[0]!.codeEvidence).toHaveLength(9);
+ expect(received[0]).toMatchObject({
+ occurrenceId: full.occurrenceId,
+ findingId: full.findingId,
+ codeEvidence: full.codeEvidence,
+ validation: full.validation,
+ writeup: full.writeup,
+ triage: { status: "open" },
+ });
+ for (const key of [
+ "createdAt",
+ "remediationState",
+ "matches",
+ "knownSince",
+ "knownScanIds",
+ "related",
+ "sourceExcerpt",
+ "artifactPaths",
+ ]) {
+ expect(received[0]![key]).toEqual(previewFinding[key]);
+ }
+ }
+});
+
+test.each(["missing findings", "changed manifest", "wrong scan"])(
+ "saved patch tasks stop before Codex when canonical evidence has %s",
+ async (failure) => {
+ await using fixture = await savedEvidence();
+ const { runCli, stderr } = createCliTest(main);
+ let calls = 0;
+ const current = dependencies({
+ onWorkbench: async (args) => {
+ if (failure === "wrong scan" && args[0] === "get-scan") {
+ const result = await fixture.workbench(args);
+ (result["scan"] as { scanId: string }).scanId = "another-scan";
+ return result;
+ }
+ if (failure === "wrong scan" && args[0] === "export-findings") {
+ return { scan: { scanDir: fixture.scanDir } };
+ }
+ return fixture.workbench(args);
+ },
+ onCodex: () => {
+ calls++;
+ return 0;
+ },
+ });
+ delete current.readSavedFindings;
+ if (failure === "missing findings")
+ await rm(join(fixture.scanDir, "findings.json"));
+ if (failure === "changed manifest")
+ await writeFile(join(fixture.scanDir, "scan-manifest.json"), "{}\n");
+ expect(
+ await runCli(["patch", "--scan", fixture.scanId], current),
+ stderr.text(),
+ ).toBe(2);
+ expect(calls).toBe(0);
+ expect(stderr.text()).toContain(
+ failure === "missing findings"
+ ? "findings.json"
+ : failure === "changed manifest"
+ ? "manifest changed"
+ : "do not match selected scan",
+ );
+ },
+);
diff --git a/sdk/typescript/tests-ts/cli-verify-fix.test.ts b/sdk/typescript/tests-ts/cli-verify-fix.test.ts
index 96696e3a0..ce6fb763d 100644
--- a/sdk/typescript/tests-ts/cli-verify-fix.test.ts
+++ b/sdk/typescript/tests-ts/cli-verify-fix.test.ts
@@ -334,6 +334,7 @@ describe("read-only finding verification", () => {
"--json",
],
dependencies({
+ result,
onWorkbench: (args) => {
expect(args).toEqual(["get-scan", "--scan-id", "scan-1"]);
return {