OpenAI's Codex Security CLI and TypeScript SDK for finding, validating, and fixing security vulnerabilities. npm: https://www.npmjs.com/package/@openai/codex-security
TypeScript ai-securityapplication-securityclicode-scanningcodexcodex-securitycybersecuritydevsecopsnodejsnpmopenaisdksecuritytypescriptvulnerability-scanning
fix: stop retrying sealed incomplete bulk scans (#286)
Kyle Brownf17222f5821e
diff --git a/sdk/typescript/src/cli.ts b/sdk/typescript/src/cli.tsindex 2ba944810..a97868fb7 100644--- a/sdk/typescript/src/cli.ts+++ b/sdk/typescript/src/cli.ts@@ -1499,13 +1499,16 @@ export async function main(},createSecurity: dependencies.createSecurity,signal: controller.signal,- onProgress: ({ repository, status, attempt, error }) => {+ onProgress: ({ repository, status, attempt, error, warning }) => {+ const detail = error ?? warning;errorOutput.write(- `codex-security: ${repository} ${status} (attempt ${attempt})${error === undefined ? "" : `: ${redactedErrorMessage(error)}`}\n`,+ `codex-security: ${repository} ${status} (attempt ${attempt})${detail === undefined ? "" : `: ${redactedErrorMessage(detail)}`}\n`,);},});- exitCode = interruptedExitCode() ?? (result.failed > 0 ? 2 : 0);+ exitCode =+ interruptedExitCode() ??+ (result.failed > 0 || result.incomplete > 0 ? 2 : 0);return { ...result };} catch (error) {exitCode =diff --git a/sdk/typescript/src/multiscan.ts b/sdk/typescript/src/multiscan.tsindex c0ae08aff..9704014c2 100644--- a/sdk/typescript/src/multiscan.ts+++ b/sdk/typescript/src/multiscan.ts@@ -20,6 +20,7 @@ import type { CodexSecurity } from "./api.js";import type { CodexSecurityConfig } from "./config.js";import type { ScanCost } from "./cost.js";import { redactedErrorMessage } from "./errors.js";+import type { CoverageDocument } from "./models.js";import type { ScanMode } from "./targets.js";import { resolveTrustedExecutable } from "./trusted-executable.js";@@ -43,11 +44,13 @@ interface MultiscanTask {}interface MultiscanReceipt extends MultiscanTask {- status: "completed" | "failed";+ status: "completed" | "completed_with_incomplete_coverage" | "failed";attempt: number;outputDir: string;+ coverage?: CoverageDocument["completeness"];cost?: ScanCost;error?: string;+ warning?: string;}export interface MultiscanOptions {@@ -67,15 +70,21 @@ export interface MultiscanOptions {signal?: AbortSignal;onProgress?(event: {repository: string;- status: "started" | "completed" | "failed";+ status:+ | "started"+ | "completed"+ | "completed_with_incomplete_coverage"+ | "failed";attempt: number;error?: string;+ warning?: string;}): void;}export interface MultiscanResult {total: number;completed: number;+ incomplete: number;failed: number;skipped: number;resultsPath: string;@@ -118,24 +127,44 @@ async function runCampaign(const receipts = await readReceipts(ledger);const pending: MultiscanTask[] = [];let completed = 0;+ let incomplete = 0;for (const task of tasks) {const receipt = receipts.get(task.id.toLowerCase());if (- receipt?.status === "completed" &&+ receipt !== undefined &&receipt.outputDir ===join(output, "artifacts", task.id, `attempt-${receipt.attempt}`) &&(await hasArtifacts(receipt.outputDir))) {- completed += 1;- } else {- pending.push(task);+ if (receipt.status === "completed") {+ completed += 1;+ continue;+ }+ const coverage =+ receipt.status === "completed_with_incomplete_coverage"+ ? receipt.coverage ?? "unknown"+ : await legacyIncompleteCoverage(receipt);+ if (coverage !== undefined) {+ incomplete += 1;+ options.onProgress?.({+ repository: task.id,+ status: "completed_with_incomplete_coverage",+ attempt: receipt.attempt,+ warning:+ receipt.warning ??+ `Scan coverage is ${coverage}; results may be incomplete.`,+ });+ continue;+ }}+ pending.push(task);}- const skipped = completed;+ const skipped = completed + incomplete;if (pending.length === 0) {return {total: tasks.length,completed,+ incomplete,failed: 0,skipped,resultsPath: ledger,@@ -165,6 +194,8 @@ async function runCampaign(const progress = { repository: task.id, attempt };options.onProgress?.({ ...progress, status: "started" });let failure: string | undefined;+ let warning: string | undefined;+ let coverage: CoverageDocument["completeness"] | undefined;let cost: Readonly<ScanCost> | null = null;try {await mkdir(dirname(scanDir), { recursive: true, mode: 0o700 });@@ -204,8 +235,14 @@ async function runCampaign(...(options.signal === undefined ? {} : { signal: options.signal }),});cost = result.cost;- if (result.coverage.completeness !== "complete") {- throw new Error("Multiscan repository coverage is incomplete.");+ coverage = result.coverage.completeness;+ if (coverage !== "complete") {+ if (!(await hasArtifacts(scanDir))) {+ throw new Error(+ "Multiscan scan output is missing required artifacts.",+ );+ }+ warning = `Scan coverage is ${coverage}; results may be incomplete.`;}} catch (error) {if (options.signal?.aborted === true) options.signal.throwIfAborted();@@ -213,7 +250,12 @@ async function runCampaign(} finally {await rm(checkout, { recursive: true, force: true });}- const status = failure === undefined ? "completed" : "failed";+ const status =+ failure !== undefined+ ? "failed"+ : warning === undefined+ ? "completed"+ : "completed_with_incomplete_coverage";await appendReceipt(ledger,`${JSON.stringify({@@ -221,17 +263,21 @@ async function runCampaign(status,attempt,outputDir: scanDir,+ ...(coverage === undefined ? {} : { coverage }),...(cost === null ? {} : { cost }),...(failure === undefined ? {} : { error: failure }),+ ...(warning === undefined ? {} : { warning }),})}\n`,);options.onProgress?.({...progress,status,...(failure === undefined ? {} : { error: failure }),+ ...(warning === undefined ? {} : { warning }),});if (failure === undefined) {- completed += 1;+ if (warning === undefined) completed += 1;+ else incomplete += 1;break;}if (retry === options.maxAttempts - 1) failed += 1;@@ -256,6 +302,7 @@ async function runCampaign(return {total: tasks.length,completed,+ incomplete,failed,skipped,resultsPath: ledger,@@ -508,6 +555,28 @@ async function hasArtifacts(path: string): Promise<boolean> {}}+async function legacyIncompleteCoverage(+ receipt: MultiscanReceipt,+): Promise<Exclude<CoverageDocument["completeness"], "complete"> | undefined> {+ if (+ receipt.status !== "failed" ||+ receipt.error !== "Multiscan repository coverage is incomplete."+ ) {+ return undefined;+ }+ try {+ const coverage = JSON.parse(+ await readFile(join(receipt.outputDir, "coverage.json"), "utf8"),+ ) as { completeness?: unknown };+ return coverage.completeness === "partial" ||+ coverage.completeness === "unknown"+ ? coverage.completeness+ : undefined;+ } catch {+ return undefined;+ }+}+function parseInventory(source: string,directory: string,diff --git a/sdk/typescript/tests-ts/multiscan.test.ts b/sdk/typescript/tests-ts/multiscan.test.tsindex 21cb3cb6d..3ef41a0b2 100644--- a/sdk/typescript/tests-ts/multiscan.test.ts+++ b/sdk/typescript/tests-ts/multiscan.test.ts@@ -17,9 +17,11 @@ import * as filesystem from "node:fs/promises";import { hostname, tmpdir } from "node:os";import { join } from "node:path";import { afterEach, describe, expect, spyOn, test } from "bun:test";+import { main } from "../src/cli.js";import type { ScanResult } from "../src/result.js";import { buildGitHubCredentialArgs, runMultiscan } from "../src/multiscan.js";import { resolveTrustedExecutable } from "../src/trusted-executable.js";+import { capture, dependencies, fakeResult } from "./cli-fixtures.js";type MultiscanOptions = Parameters<typeof runMultiscan>[0];type SecurityClient = ReturnType<MultiscanOptions["createSecurity"]>;@@ -82,7 +84,7 @@ async function repository(async function completedScan(outputDir: string,- completeness: "complete" | "partial" = "complete",+ completeness: "complete" | "partial" | "unknown" = "complete",): Promise<ScanResult> {await mkdir(outputDir, { recursive: true });await Promise.all(@@ -251,8 +253,383 @@ describe("multiscan", () => {),);+ expect(summary).toMatchObject({ completed: 1, incomplete: 0, failed: 0 });expect(await results(summary.resultsPath)).toMatchObject([- { id: "priced", status: "completed", cost },+ { id: "priced", status: "completed", coverage: "complete", cost },+ ]);+ });++ test.each(["partial", "unknown"] as const)(+ "retains sealed %s coverage without retries or multiplied costs",+ async (completeness) => {+ const paths = await fixture();+ const source = await repository(paths.root, completeness);+ await writeFile(+ paths.input,+ `id,repository,revision\nsealed,${source.path},${source.revision}\n`,+ );+ const cost = {+ model: "gpt-5.6-sol",+ inputTokens: 1_250,+ cachedInputTokens: 200,+ cacheWriteInputTokens: 0,+ outputTokens: 30,+ estimatedUsd: 12.5,+ };+ const progress: Parameters<+ NonNullable<MultiscanOptions["onProgress"]>+ >[0][] = [];+ let attempts = 0;+ const security = client(async (_repository, scanOptions = {}) => {+ attempts += 1;+ return Object.assign(+ await completedScan(scanOptions.outputDir!, completeness),+ { cost },+ );+ });++ const summary = await runMultiscan(+ options(paths, security, {+ maxAttempts: 3,+ onProgress: (event) => progress.push(event),+ }),+ );++ expect(attempts).toBe(1);+ expect(summary).toMatchObject({+ total: 1,+ completed: 0,+ incomplete: 1,+ failed: 0,+ skipped: 0,+ });+ const outputDir = join(paths.output, "artifacts", "sealed", "attempt-1");+ const warning = `Scan coverage is ${completeness}; results may be incomplete.`;+ const receipts = await results(summary.resultsPath);+ expect(receipts).toMatchObject([+ {+ id: "sealed",+ status: "completed_with_incomplete_coverage",+ attempt: 1,+ outputDir,+ coverage: completeness,+ cost,+ warning,+ },+ ]);+ expect(+ receipts.reduce(+ (total, receipt) =>+ total + (receipt["cost"] as typeof cost).estimatedUsd,+ 0,+ ),+ ).toBe(cost.estimatedUsd);+ await Promise.all(+ [+ "scan-manifest.json",+ "findings.json",+ "coverage.json",+ "report.md",+ ].map((name) => access(join(outputDir, name))),+ );+ expect(progress).toMatchObject([+ { repository: "sealed", status: "started", attempt: 1 },+ {+ repository: "sealed",+ status: "completed_with_incomplete_coverage",+ attempt: 1,+ warning,+ },+ ]);++ const resumed = await runMultiscan(+ options(paths, security, { maxAttempts: 3 }),+ );+ expect(resumed).toMatchObject({+ completed: 0,+ incomplete: 1,+ failed: 0,+ skipped: 1,+ });+ expect(attempts).toBe(1);+ expect(await results(resumed.resultsPath)).toHaveLength(1);+ },+ );++ test.each(["partial", "unknown"] as const)(+ "resumes legacy sealed %s coverage without rerunning or duplicating cost",+ async (completeness) => {+ const paths = await fixture();+ const source = await repository(paths.root, `legacy-${completeness}`);+ await writeFile(+ paths.input,+ `id,repository,revision\nlegacy,${source.path},${source.revision}\n`,+ );+ const outputDir = join(paths.output, "artifacts", "legacy", "attempt-1");+ await completedScan(outputDir, completeness);+ await writeFile(+ join(outputDir, "coverage.json"),+ `${JSON.stringify({ completeness })}\n`,+ );+ const cost = {+ model: "gpt-5.6-sol",+ inputTokens: 1_250,+ cachedInputTokens: 200,+ cacheWriteInputTokens: 0,+ outputTokens: 30,+ estimatedUsd: 231.73,+ };+ const receipt = {+ id: "legacy",+ repository: source.path,+ revision: source.revision,+ mode: "standard",+ status: "failed",+ attempt: 1,+ outputDir,+ cost,+ error: "Multiscan repository coverage is incomplete.",+ };+ await writeFile(+ join(paths.output, "results.jsonl"),+ `${JSON.stringify(receipt)}\n`,+ );+ const progress: Parameters<+ NonNullable<MultiscanOptions["onProgress"]>+ >[0][] = [];+ let attempts = 0;+ const security = client(async (_repository, scanOptions = {}) => {+ attempts += 1;+ return await completedScan(scanOptions.outputDir!);+ });++ const summary = await runMultiscan(+ options(paths, security, {+ maxAttempts: 3,+ onProgress: (event) => progress.push(event),+ }),+ );++ expect(summary).toMatchObject({+ total: 1,+ completed: 0,+ incomplete: 1,+ failed: 0,+ skipped: 1,+ });+ expect(attempts).toBe(0);+ expect(progress).toEqual([+ {+ repository: "legacy",+ status: "completed_with_incomplete_coverage",+ attempt: 1,+ warning: `Scan coverage is ${completeness}; results may be incomplete.`,+ },+ ]);+ expect(await results(summary.resultsPath)).toEqual([receipt]);++ await runMultiscan(options(paths, security, { maxAttempts: 3 }));+ expect(attempts).toBe(0);+ expect(await results(summary.resultsPath)).toEqual([receipt]);+ },+ );++ test.each([+ ["operational failures", "partial", "Worker exited unexpectedly.", false],+ [+ "complete coverage",+ "complete",+ "Multiscan repository coverage is incomplete.",+ false,+ ],+ [+ "malformed coverage",+ "malformed",+ "Multiscan repository coverage is incomplete.",+ false,+ ],+ [+ "missing artifacts",+ "partial",+ "Multiscan repository coverage is incomplete.",+ true,+ ],+ ] as const)(+ "continues retrying legacy %s",+ async (_scenario, completeness, error, missingArtifact) => {+ const paths = await fixture();+ const source = await repository(paths.root, "legacy-retry");+ await writeFile(+ paths.input,+ `id,repository,revision\nlegacy,${source.path},${source.revision}\n`,+ );+ const outputDir = join(paths.output, "artifacts", "legacy", "attempt-1");+ await completedScan(outputDir);+ await writeFile(+ join(outputDir, "coverage.json"),+ completeness === "malformed"+ ? "{\n"+ : `${JSON.stringify({ completeness })}\n`,+ );+ if (missingArtifact) await rm(join(outputDir, "report.md"));+ await writeFile(+ join(paths.output, "results.jsonl"),+ `${JSON.stringify({+ id: "legacy",+ repository: source.path,+ revision: source.revision,+ mode: "standard",+ status: "failed",+ attempt: 1,+ outputDir,+ error,+ })}\n`,+ );+ let attempts = 0;++ const summary = await runMultiscan(+ options(+ paths,+ client(async (_repository, scanOptions = {}) => {+ attempts += 1;+ return await completedScan(scanOptions.outputDir!);+ }),+ ),+ );++ expect(summary).toMatchObject({+ completed: 1,+ incomplete: 0,+ failed: 0,+ skipped: 0,+ });+ expect(attempts).toBe(1);+ expect(await results(summary.resultsPath)).toMatchObject([+ { status: "failed", attempt: 1, error },+ { status: "completed", attempt: 2, coverage: "complete" },+ ]);+ },+ );++ test.each(["partial", "unknown"] as const)(+ "keeps sealed %s-coverage CLI runs fail-closed without retrying",+ async (completeness) => {+ const paths = await fixture();+ const source = await repository(paths.root, "sample");+ await writeFile(+ paths.input,+ `id,repository,revision\nsample,${source.path},${source.revision}\n`,+ );+ const outputDir = join(paths.output, "artifacts", "sample", "attempt-1");+ await completedScan(outputDir, completeness);+ const stdout = capture();+ const stderr = capture();+ let attempts = 0;+ const arguments_ = [+ "bulk-scan",+ "repositories.csv",+ "--output-dir",+ "results",+ "--max-attempts",+ "3",+ "--json",+ ];+ const clientDependencies = dependencies({+ currentDirectory: paths.root,+ result: fakeResult([], completeness),+ onRun: () => {+ attempts += 1;+ },+ });++ expect(+ await main(+ arguments_,+ stdout.stream,+ stderr.stream,+ clientDependencies,+ ),+ ).toBe(2);+ expect(attempts).toBe(1);+ expect(JSON.parse(stdout.text())).toMatchObject({+ total: 1,+ completed: 0,+ incomplete: 1,+ failed: 0,+ skipped: 0,+ });+ const warning = `Scan coverage is ${completeness}; results may be incomplete.`;+ expect(stderr.text()).toContain(+ "sample completed_with_incomplete_coverage (attempt 1)",+ );+ expect(stderr.text()).toContain(warning);+ expect(stderr.text()).not.toContain("attempt 2");+ expect(await results(join(paths.output, "results.jsonl"))).toMatchObject([+ {+ status: "completed_with_incomplete_coverage",+ coverage: completeness,+ outputDir,+ },+ ]);++ const resumedOutput = capture();+ const resumedError = capture();+ expect(+ await main(+ arguments_,+ resumedOutput.stream,+ resumedError.stream,+ clientDependencies,+ ),+ ).toBe(2);+ expect(JSON.parse(resumedOutput.text())).toMatchObject({+ completed: 0,+ incomplete: 1,+ failed: 0,+ skipped: 1,+ });+ expect(resumedError.text()).toContain(warning);+ expect(attempts).toBe(1);+ },+ );++ test("retries incomplete scans that are missing required artifacts", async () => {+ const paths = await fixture();+ const source = await repository(paths.root, "missing-artifact");+ await writeFile(+ paths.input,+ `id,repository,revision\nmissing,${source.path},${source.revision}\n`,+ );++ let attempts = 0;+ const summary = await runMultiscan(+ options(+ paths,+ client(async (_repository, scanOptions = {}) => {+ attempts += 1;+ const result = await completedScan(+ scanOptions.outputDir!,+ attempts === 1 ? "partial" : "complete",+ );+ if (attempts === 1) {+ await rm(join(scanOptions.outputDir!, "report.md"));+ }+ return result;+ }),+ ),+ );++ expect(attempts).toBe(2);+ expect(summary).toMatchObject({ completed: 1, incomplete: 0, failed: 0 });+ expect(await results(summary.resultsPath)).toMatchObject([+ {+ id: "missing",+ status: "failed",+ attempt: 1,+ coverage: "partial",+ error: "Multiscan scan output is missing required artifacts.",+ },+ { id: "missing", status: "completed", attempt: 2, coverage: "complete" },]);});@@ -1109,7 +1486,7 @@ describe("multiscan", () => {expect(scans).toBe(0);});- test("treats incomplete coverage as a failure and still finishes other repositories", async () => {+ test("records incomplete coverage separately and still finishes other repositories", async () => {const paths = await fixture();const incomplete = await repository(paths.root, "incomplete");const complete = await repository(paths.root, "complete");@@ -1136,14 +1513,24 @@ describe("multiscan", () => {: "complete",),),- { maxAttempts: 1 },+ { maxAttempts: 3 },),);- expect(summary).toMatchObject({ total: 2, completed: 1, failed: 1 });+ expect(summary).toMatchObject({+ total: 2,+ completed: 1,+ incomplete: 1,+ failed: 0,+ });expect(await results(summary.resultsPath)).toMatchObject([- { id: "incomplete", status: "failed", attempt: 1 },- { id: "complete", status: "completed", attempt: 1 },+ {+ id: "incomplete",+ status: "completed_with_incomplete_coverage",+ attempt: 1,+ coverage: "partial",+ },+ { id: "complete", status: "completed", attempt: 1, coverage: "complete" },]);});});