OpenAI's Codex Security CLI and TypeScript SDK for finding, validating, and fixing security vulnerabilities. npm: https://www.npmjs.com/package/@openai/codex-security
TypeScript ai-securityapplication-securityclicode-scanningcodexcodex-securitycybersecuritydevsecopsnodejsnpmopenaisdksecuritytypescriptvulnerability-scanning
fix(cli): preserve complete evidence for saved finding remediation (#1608)
Michael D'Angelo11e33e0d9352
diff --git a/sdk/typescript/docs/cli.md b/sdk/typescript/docs/cli.mdindex 0299c96f9..26a43ed65 100644--- a/sdk/typescript/docs/cli.md+++ b/sdk/typescript/docs/cli.md@@ -1771,6 +1771,8 @@ Both candidate IDs remain in the output, including the unresolved claim.`--scan latest` uses the current repository's latest completed scan. Patchingsupports JSON for saved findings, literal text, and files. Each finding gets asaved Codex desktop task. `--model` and `--effort` control the model.+Saved-scan patching and fix verification use the complete sealed finding evidence+and exclude findings that are currently closed in triage.`--validation-prompt-file PATH` supplies setup, authorized targets, expectedresults, and cleanup for dynamic validation. It works with saved findings,diff --git a/sdk/typescript/src/cli.ts b/sdk/typescript/src/cli.tsindex 6683d8542..d1aa7dc83 100644--- a/sdk/typescript/src/cli.ts+++ b/sdk/typescript/src/cli.ts@@ -1186,6 +1186,7 @@ interface CliDependencies {linearClient?: LinearClientFactory;importGitHubAlerts?: typeof importGitHubCodeScanningAlerts;importScan?: typeof importScan;+ readSavedFindings?: typeof readSavedFindings;runWorkbench(args: readonly string[],input?: string,@@ -6835,13 +6836,64 @@ async function selectSavedFindings("The requested finding does not belong to the selected scan.",);}+ const fullFindings =+ selected.length === 0+ ? []+ : await (dependencies.readSavedFindings ?? readSavedFindings)(+ scan.scanId,+ dependencies.runWorkbench,+ );+ const byOccurrence = new Map(+ fullFindings.map((finding) => [finding.occurrenceId, finding]),+ );return {repository: scan.targetPath,scanId: scan.scanId,- findings: selected,+ findings: selected.map((finding) => {+ const full = byOccurrence.get(finding.occurrenceId);+ if (full === undefined || full.findingId !== finding.findingId) {+ throw new CodexSecurityError(+ `Finding ${finding.occurrenceId} is missing from the saved scan artifacts.`,+ );+ }+ const hydrated = { ...finding, ...full };+ // Sealed artifacts supply evidence; the workbench owns live and derived state.+ for (const key of [+ "createdAt",+ "triage",+ "remediationState",+ "matches",+ "knownSince",+ "knownScanIds",+ "related",+ "sourceExcerpt",+ "artifactPaths",+ ]) {+ if (Object.hasOwn(finding, key)) hydrated[key] = finding[key];+ else delete hydrated[key];+ }+ return hydrated;+ }),};}+async function readSavedFindings(+ scanId: string,+ workbench: CliDependencies["runWorkbench"],+): Promise<readonly Finding[]> {+ const directory = await resolveSavedArtifactDirectory(+ scanId,+ "findings",+ "json",+ workbench,+ );+ const { findings } = await loadContract(directory, {+ pluginRoot: await bundledPluginRoot(),+ expectedScanId: scanId,+ });+ return findings.findings;+}+function patchExitCode(patches: readonly FindingPatch[]): number {if (patches.some(({ status }) => status === "failed")) return 2;return patches.some(({ status }) => status === "blocked") ? 1 : 0;diff --git a/sdk/typescript/tests-ts/cli-fixtures.ts b/sdk/typescript/tests-ts/cli-fixtures.tsindex 7e8dffcbc..088d3c0e6 100644--- a/sdk/typescript/tests-ts/cli-fixtures.ts+++ b/sdk/typescript/tests-ts/cli-fixtures.ts@@ -219,6 +219,7 @@ export function dependencies(input?: string,signal?: AbortSignal,) => JsonObject | Promise<JsonObject>;+ readSavedFindings?: MainDependencies["readSavedFindings"];onMatch?: MainDependencies["matchFindings"];onUpdateCheck?: (signal: AbortSignal) => Promise<UpdateNotice | undefined>;currentDirectory?: string;@@ -295,6 +296,8 @@ export function dependencies(: { importGitHubAlerts: options.importGitHubAlerts }),runWorkbench: async (args, input, signal) =>(await options.onWorkbench?.(args, input, signal)) ?? { scans: [] },+ readSavedFindings:+ options.readSavedFindings ?? (async () => result.findings.findings),matchFindings: async (input, comparisonOptions) =>(await options.onMatch?.(input, comparisonOptions)) ?? {matches: [],diff --git a/sdk/typescript/tests-ts/cli-patch-results.test.ts b/sdk/typescript/tests-ts/cli-patch-results.test.tsindex 03a41fabe..8b46995f0 100644--- a/sdk/typescript/tests-ts/cli-patch-results.test.ts+++ b/sdk/typescript/tests-ts/cli-patch-results.test.ts@@ -5,6 +5,7 @@ import { join } from "node:path";import { promisify } from "node:util";import { describe, expect, test } from "bun:test";import { main, runCodexSkillCommand } from "../src/cli.js";+import type { Finding } from "../src/models.js";import { dependencies } from "./cli-fixtures.js";import { createCliTest } from "./support/cli-run.js";@@ -199,19 +200,20 @@ lines.on("line", (line) => {test("rejects a verified saved-finding result when no files change", async () => {await using fixture = await repositoryFixture();+ const finding = {+ occurrenceId: "occ_1",+ findingId: "csf_1",+ title: "Synthetic finding",+ severity: { level: "high" },+ locations: [{ path: "app.ts", startLine: 1 }],+ };const outcome = await fixture.patch(["--scan", "scan-1"], {+ readSavedFindings: async () => [finding as Finding],onWorkbench: () => ({scan: {scanId: "scan-1",targetPath: fixture.directory,- findings: [- {- occurrenceId: "occ_1",- title: "Synthetic finding",- severity: { level: "high" },- locations: [{ path: "app.ts", startLine: 1 }],- },- ],+ findings: [finding],},}),onCodex: (_args, output) => {diff --git a/sdk/typescript/tests-ts/cli-patch.test.ts b/sdk/typescript/tests-ts/cli-patch.test.tsindex 8159999c6..8dc1aaa88 100644--- a/sdk/typescript/tests-ts/cli-patch.test.ts+++ b/sdk/typescript/tests-ts/cli-patch.test.ts@@ -517,6 +517,7 @@ describe("scan and patch workflow", () => {],{currentDirectory: directory,+ result,onWorkbench: () => savedScan(result, "scan-1", repository),onCodex: async (args, output) => {calls++;@@ -594,6 +595,7 @@ describe("scan and patch workflow", () => {],{currentDirectory: directory,+ result,onWorkbench: () => savedScan(result, "scan-1", repository),onCodex: (args, output) => {started = true;@@ -722,6 +724,7 @@ describe("scan and patch workflow", () => {"--json",],{+ result,onWorkbench: () => savedScan(result),onRepositoryCommand: (command, args) => {repositoryCommands.push({ command, args });@@ -1517,6 +1520,7 @@ describe("scan and patch workflow", () => {const fixtures: Parameters<typeof dependencies>[0] = {currentDirectory: repository,+ result,onWorkbench: () => savedScan(result, "scan-1", repository),onCodex: async (args, output) => {modelCalls += 1;@@ -2049,6 +2053,7 @@ describe("scan and patch workflow", () => {const outcome = await runWorkflow(["patch", "--scan", "scan-1", "--severity", "high", "--json"],{+ result,onWorkbench: (args): JsonObject => {expect(args).toEqual(["get-scan", "--scan-id", "scan-1"]);return savedScan(result);@@ -2146,6 +2151,7 @@ describe("scan and patch workflow", () => {["patch", "--scan", "scan-1", "--create-pr", "--json"],{environment,+ result,onWorkbench: () => savedScan(result),onRepositoryCommand: (command, args, target) => {expect(target).toBe(SAVED_REPOSITORY);@@ -2376,6 +2382,7 @@ describe("scan and patch workflow", () => {"--json",],{+ result,onWorkbench: () => savedScan(result),onRepositoryCommand: (command, args) => {if (command === "git") {@@ -2435,6 +2442,7 @@ describe("scan and patch workflow", () => {const outcome = await runWorkflow(["patch", "--scan", "scan-1", "--create-pr", "--json"],{+ result,onWorkbench: () => savedScan(result),onRepositoryCommand: (command, args) => {if (command === "git") {@@ -2506,6 +2514,7 @@ describe("scan and patch workflow", () => {const calls: Array<readonly string[]> = [];let patched: Finding[] = [];const outcome = await runWorkflow(["patch", "occ_1"], {+ result,onWorkbench: (args): JsonObject => {calls.push(args);if (args[0] === "list-global-findings") {@@ -2535,6 +2544,7 @@ describe("scan and patch workflow", () => {const calls: Array<readonly string[]> = [];const outcome = await runWorkflow(["patch", "--scan", "latest"], {currentDirectory: SAVED_REPOSITORY,+ result,onWorkbench: (args): JsonObject => {calls.push(args);if (args[0] === "list-scans") {@@ -2555,6 +2565,7 @@ describe("scan and patch workflow", () => {const patched: Finding[] = [];const calls: Array<readonly string[]> = [];const outcome = await runWorkflow(["patch", "--scan", "scan-1"], {+ result,onWorkbench: (args): JsonObject => {calls.push(args);if (args[0] === "get-scan") {diff --git a/sdk/typescript/tests-ts/cli-saved-finding-evidence.test.ts b/sdk/typescript/tests-ts/cli-saved-finding-evidence.test.tsnew file mode 100644index 000000000..8a9b2180f--- /dev/null+++ b/sdk/typescript/tests-ts/cli-saved-finding-evidence.test.ts@@ -0,0 +1,246 @@+import { mkdir, readFile, rm, writeFile } from "node:fs/promises";+import { dirname, join } from "node:path";+import { expect, test } from "bun:test";+import { exportEnvironment, main } from "../src/cli.js";+import type { Finding, FindingsDocument, ScanManifest } from "../src/models.js";+import { resolvePluginPython } from "../src/runtime.js";+import { dependencies } from "./cli-fixtures.js";+import { copyCompletedScanFixture } from "./plugin-root.js";+import { createCliTest } from "./support/cli-run.js";+import { setFindingIdentity } from "./support/finding-identity.js";+import { temporaryDirectory } from "./support/temporary-directories.js";+import {+ scanRegistrationArguments,+ workbenchCommand,+} from "./support/workbench-command.js";++async function savedEvidence() {+ const root = await temporaryDirectory("saved-finding-evidence-", true);+ const repository = join(root, "repository");+ const scanDir = join(root, "scan");+ const environment = {+ ...exportEnvironment(),+ CODEX_SECURITY_STATE_DIR: join(root, "state"),+ };+ const workbench = workbenchCommand(+ await resolvePluginPython(),+ () => environment,+ );+ try {+ await mkdir(repository);+ await mkdir(scanDir, { mode: 0o700 });+ const registered = await workbench(+ scanRegistrationArguments(repository, scanDir),+ );+ const scanId = registered["scanId"] as string;+ await copyCompletedScanFixture(scanDir);+ const manifest = JSON.parse(+ await readFile(join(scanDir, "scan-manifest.json"), "utf8"),+ ) as ScanManifest;+ manifest.scan.id = scanId;+ manifest.scan.target.kind = "directory_snapshot";+ manifest.scan.target.targetId = registered["targetId"] as string;+ Reflect.deleteProperty(manifest.scan, "sealedAt");+ Reflect.deleteProperty(manifest.scan, "artifacts");+ const document = JSON.parse(+ await readFile(join(scanDir, "findings.json"), "utf8"),+ ) as FindingsDocument;+ document.scanId = scanId;+ const finding = document.findings[0]!;+ const slug = "synthetic".repeat(28);+ const reportPath = `findings/${slug}/${slug}.md`;+ finding.codeEvidence = Array.from({ length: 9 }, (_, index) => ({+ id: `evidence-${index}`,+ label: `Synthetic evidence ${index}`,+ path: "src/extract.py",+ startLine: index + 1,+ code: `synthetic_source_${index}();\n`.repeat(200),+ explanation: "Synthetic source-to-sink evidence.",+ }));+ finding.validation = {+ summary: "Synthetic validation evidence.",+ counterEvidence: ["A mitigation remains unverified. ".repeat(300)],+ };+ finding.writeup = { reportPath };+ Object.assign(finding, {+ createdAt: "2000-01-01T00:00:00Z",+ remediationState: { status: "fixed" },+ matches: [{ occurrenceId: "stale-occurrence" }],+ knownSince: "2000-01-01T00:00:00Z",+ knownScanIds: ["stale-scan"],+ related: [{ occurrenceId: "stale-related-occurrence" }],+ sourceExcerpt: "Stale source excerpt.",+ artifactPaths: ["stale-report.md"],+ });+ const closed = structuredClone(finding);+ closed.identity.anchor = "closed-synthetic-finding";+ delete closed.writeup;+ document.findings.push(closed);+ document.findings.forEach((item) =>+ setFindingIdentity(manifest.scan, item),+ );+ await mkdir(dirname(join(scanDir, reportPath)), { recursive: true });+ await writeFile(join(scanDir, reportPath), "# Synthetic finding report\n");+ const coverage = JSON.parse(+ await readFile(join(scanDir, "coverage.json"), "utf8"),+ );+ coverage.scanId = scanId;+ await Promise.all([+ writeFile(join(scanDir, "scan-manifest.json"), JSON.stringify(manifest)),+ writeFile(join(scanDir, "findings.json"), JSON.stringify(document)),+ writeFile(join(scanDir, "coverage.json"), JSON.stringify(coverage)),+ ]);+ await workbench(["complete-scan", "--scan-id", scanId]);+ const canonical = JSON.parse(+ await readFile(join(scanDir, "findings.json"), "utf8"),+ ) as FindingsDocument;+ expect(canonical.findings).toHaveLength(2);+ await workbench([+ "set-finding-triage",+ "--occurrence-id",+ canonical.findings[1]!.occurrenceId,+ "--status",+ "closed",+ "--close-reason",+ "false_positive",+ "--note",+ "Synthetic closed finding retained for selection coverage.",+ ]);+ return {+ root,+ repository,+ scanDir,+ scanId,+ workbench,+ canonical,+ async [Symbol.asyncDispose]() {+ await rm(root, { recursive: true, force: true });+ },+ };+ } catch (error) {+ await rm(root, { recursive: true, force: true });+ throw error;+ }+}++test("saved patch and verification tasks receive complete sealed evidence and live workbench state", async () => {+ await using fixture = await savedEvidence();+ const full = fixture.canonical.findings[0]!;+ expect(full.codeEvidence).toHaveLength(9);+ const preview = await fixture.workbench([+ "get-scan",+ "--scan-id",+ fixture.scanId,+ ]);+ const previewFinding = (preview["scan"] as { findings: Finding[] })+ .findings[0]!;+ expect(previewFinding.codeEvidence).toHaveLength(4);+ expect(previewFinding.writeup?.reportPath).not.toBe(full.writeup!.reportPath);++ for (const command of ["patch", "verify-fix"]) {+ const { runCli, stderr } = createCliTest(main);+ const received: Finding[] = [];+ const current = dependencies({+ onWorkbench: fixture.workbench,+ onRepositoryCommand: (_command, args) =>+ args.includes("--name-only") ? "src/extract.py\0" : "",+ onCodex: (_args, output) => {+ const findings = JSON.parse(+ output!.appServer!.prompt.split("\n").at(-1)!,+ ) as Finding[];+ received.push(...findings);+ output!.stdout.write(+ JSON.stringify(+ command === "patch"+ ? {+ patches: findings.map(({ occurrenceId }) => ({+ occurrenceId,+ status: "verified",+ files: ["src/extract.py"],+ verification: "Synthetic regression passed.",+ })),+ }+ : {+ results: findings.map(({ occurrenceId }) => ({+ id: occurrenceId,+ status: "fixed",+ evidence: "Synthetic regression passed.",+ })),+ },+ ),+ );+ return 0;+ },+ });+ delete current.readSavedFindings;+ expect(+ await runCli([command, "--scan", fixture.scanId, "--json"], current),+ stderr.text(),+ ).toBe(0);+ expect(received).toHaveLength(1);+ expect(received[0]!.codeEvidence).toHaveLength(9);+ expect(received[0]).toMatchObject({+ occurrenceId: full.occurrenceId,+ findingId: full.findingId,+ codeEvidence: full.codeEvidence,+ validation: full.validation,+ writeup: full.writeup,+ triage: { status: "open" },+ });+ for (const key of [+ "createdAt",+ "remediationState",+ "matches",+ "knownSince",+ "knownScanIds",+ "related",+ "sourceExcerpt",+ "artifactPaths",+ ]) {+ expect(received[0]![key]).toEqual(previewFinding[key]);+ }+ }+});++test.each(["missing findings", "changed manifest", "wrong scan"])(+ "saved patch tasks stop before Codex when canonical evidence has %s",+ async (failure) => {+ await using fixture = await savedEvidence();+ const { runCli, stderr } = createCliTest(main);+ let calls = 0;+ const current = dependencies({+ onWorkbench: async (args) => {+ if (failure === "wrong scan" && args[0] === "get-scan") {+ const result = await fixture.workbench(args);+ (result["scan"] as { scanId: string }).scanId = "another-scan";+ return result;+ }+ if (failure === "wrong scan" && args[0] === "export-findings") {+ return { scan: { scanDir: fixture.scanDir } };+ }+ return fixture.workbench(args);+ },+ onCodex: () => {+ calls++;+ return 0;+ },+ });+ delete current.readSavedFindings;+ if (failure === "missing findings")+ await rm(join(fixture.scanDir, "findings.json"));+ if (failure === "changed manifest")+ await writeFile(join(fixture.scanDir, "scan-manifest.json"), "{}\n");+ expect(+ await runCli(["patch", "--scan", fixture.scanId], current),+ stderr.text(),+ ).toBe(2);+ expect(calls).toBe(0);+ expect(stderr.text()).toContain(+ failure === "missing findings"+ ? "findings.json"+ : failure === "changed manifest"+ ? "manifest changed"+ : "do not match selected scan",+ );+ },+);diff --git a/sdk/typescript/tests-ts/cli-verify-fix.test.ts b/sdk/typescript/tests-ts/cli-verify-fix.test.tsindex 96696e3a0..ce6fb763d 100644--- a/sdk/typescript/tests-ts/cli-verify-fix.test.ts+++ b/sdk/typescript/tests-ts/cli-verify-fix.test.ts@@ -334,6 +334,7 @@ describe("read-only finding verification", () => {"--json",],dependencies({+ result,onWorkbench: (args) => {expect(args).toEqual(["get-scan", "--scan-id", "scan-1"]);return {