Skip to content

OpenAI's Codex Security CLI and TypeScript SDK for finding, validating, and fixing security vulnerabilities. npm: https://www.npmjs.com/package/@openai/codex-security

fix: stop retrying sealed incomplete bulk scans (#286)

Kyle Brownf17222f5821e
diff --git a/sdk/typescript/src/cli.ts b/sdk/typescript/src/cli.ts
index 2ba944810..a97868fb7 100644
--- a/sdk/typescript/src/cli.ts
+++ b/sdk/typescript/src/cli.ts
@@ -1499,13 +1499,16 @@ export async function main(
},
createSecurity: dependencies.createSecurity,
signal: controller.signal,
- onProgress: ({ repository, status, attempt, error }) => {
+ onProgress: ({ repository, status, attempt, error, warning }) => {
+ const detail = error ?? warning;
errorOutput.write(
- `codex-security: ${repository} ${status} (attempt ${attempt})${error === undefined ? "" : `: ${redactedErrorMessage(error)}`}\n`,
+ `codex-security: ${repository} ${status} (attempt ${attempt})${detail === undefined ? "" : `: ${redactedErrorMessage(detail)}`}\n`,
);
},
});
- exitCode = interruptedExitCode() ?? (result.failed > 0 ? 2 : 0);
+ exitCode =
+ interruptedExitCode() ??
+ (result.failed > 0 || result.incomplete > 0 ? 2 : 0);
return { ...result };
} catch (error) {
exitCode =
diff --git a/sdk/typescript/src/multiscan.ts b/sdk/typescript/src/multiscan.ts
index c0ae08aff..9704014c2 100644
--- a/sdk/typescript/src/multiscan.ts
+++ b/sdk/typescript/src/multiscan.ts
@@ -20,6 +20,7 @@ import type { CodexSecurity } from "./api.js";
import type { CodexSecurityConfig } from "./config.js";
import type { ScanCost } from "./cost.js";
import { redactedErrorMessage } from "./errors.js";
+import type { CoverageDocument } from "./models.js";
import type { ScanMode } from "./targets.js";
import { resolveTrustedExecutable } from "./trusted-executable.js";
@@ -43,11 +44,13 @@ interface MultiscanTask {
}
interface MultiscanReceipt extends MultiscanTask {
- status: "completed" | "failed";
+ status: "completed" | "completed_with_incomplete_coverage" | "failed";
attempt: number;
outputDir: string;
+ coverage?: CoverageDocument["completeness"];
cost?: ScanCost;
error?: string;
+ warning?: string;
}
export interface MultiscanOptions {
@@ -67,15 +70,21 @@ export interface MultiscanOptions {
signal?: AbortSignal;
onProgress?(event: {
repository: string;
- status: "started" | "completed" | "failed";
+ status:
+ | "started"
+ | "completed"
+ | "completed_with_incomplete_coverage"
+ | "failed";
attempt: number;
error?: string;
+ warning?: string;
}): void;
}
export interface MultiscanResult {
total: number;
completed: number;
+ incomplete: number;
failed: number;
skipped: number;
resultsPath: string;
@@ -118,24 +127,44 @@ async function runCampaign(
const receipts = await readReceipts(ledger);
const pending: MultiscanTask[] = [];
let completed = 0;
+ let incomplete = 0;
for (const task of tasks) {
const receipt = receipts.get(task.id.toLowerCase());
if (
- receipt?.status === "completed" &&
+ receipt !== undefined &&
receipt.outputDir ===
join(output, "artifacts", task.id, `attempt-${receipt.attempt}`) &&
(await hasArtifacts(receipt.outputDir))
) {
- completed += 1;
- } else {
- pending.push(task);
+ if (receipt.status === "completed") {
+ completed += 1;
+ continue;
+ }
+ const coverage =
+ receipt.status === "completed_with_incomplete_coverage"
+ ? receipt.coverage ?? "unknown"
+ : await legacyIncompleteCoverage(receipt);
+ if (coverage !== undefined) {
+ incomplete += 1;
+ options.onProgress?.({
+ repository: task.id,
+ status: "completed_with_incomplete_coverage",
+ attempt: receipt.attempt,
+ warning:
+ receipt.warning ??
+ `Scan coverage is ${coverage}; results may be incomplete.`,
+ });
+ continue;
+ }
}
+ pending.push(task);
}
- const skipped = completed;
+ const skipped = completed + incomplete;
if (pending.length === 0) {
return {
total: tasks.length,
completed,
+ incomplete,
failed: 0,
skipped,
resultsPath: ledger,
@@ -165,6 +194,8 @@ async function runCampaign(
const progress = { repository: task.id, attempt };
options.onProgress?.({ ...progress, status: "started" });
let failure: string | undefined;
+ let warning: string | undefined;
+ let coverage: CoverageDocument["completeness"] | undefined;
let cost: Readonly<ScanCost> | null = null;
try {
await mkdir(dirname(scanDir), { recursive: true, mode: 0o700 });
@@ -204,8 +235,14 @@ async function runCampaign(
...(options.signal === undefined ? {} : { signal: options.signal }),
});
cost = result.cost;
- if (result.coverage.completeness !== "complete") {
- throw new Error("Multiscan repository coverage is incomplete.");
+ coverage = result.coverage.completeness;
+ if (coverage !== "complete") {
+ if (!(await hasArtifacts(scanDir))) {
+ throw new Error(
+ "Multiscan scan output is missing required artifacts.",
+ );
+ }
+ warning = `Scan coverage is ${coverage}; results may be incomplete.`;
}
} catch (error) {
if (options.signal?.aborted === true) options.signal.throwIfAborted();
@@ -213,7 +250,12 @@ async function runCampaign(
} finally {
await rm(checkout, { recursive: true, force: true });
}
- const status = failure === undefined ? "completed" : "failed";
+ const status =
+ failure !== undefined
+ ? "failed"
+ : warning === undefined
+ ? "completed"
+ : "completed_with_incomplete_coverage";
await appendReceipt(
ledger,
`${JSON.stringify({
@@ -221,17 +263,21 @@ async function runCampaign(
status,
attempt,
outputDir: scanDir,
+ ...(coverage === undefined ? {} : { coverage }),
...(cost === null ? {} : { cost }),
...(failure === undefined ? {} : { error: failure }),
+ ...(warning === undefined ? {} : { warning }),
})}\n`,
);
options.onProgress?.({
...progress,
status,
...(failure === undefined ? {} : { error: failure }),
+ ...(warning === undefined ? {} : { warning }),
});
if (failure === undefined) {
- completed += 1;
+ if (warning === undefined) completed += 1;
+ else incomplete += 1;
break;
}
if (retry === options.maxAttempts - 1) failed += 1;
@@ -256,6 +302,7 @@ async function runCampaign(
return {
total: tasks.length,
completed,
+ incomplete,
failed,
skipped,
resultsPath: ledger,
@@ -508,6 +555,28 @@ async function hasArtifacts(path: string): Promise<boolean> {
}
}
+async function legacyIncompleteCoverage(
+ receipt: MultiscanReceipt,
+): Promise<Exclude<CoverageDocument["completeness"], "complete"> | undefined> {
+ if (
+ receipt.status !== "failed" ||
+ receipt.error !== "Multiscan repository coverage is incomplete."
+ ) {
+ return undefined;
+ }
+ try {
+ const coverage = JSON.parse(
+ await readFile(join(receipt.outputDir, "coverage.json"), "utf8"),
+ ) as { completeness?: unknown };
+ return coverage.completeness === "partial" ||
+ coverage.completeness === "unknown"
+ ? coverage.completeness
+ : undefined;
+ } catch {
+ return undefined;
+ }
+}
+
function parseInventory(
source: string,
directory: string,
diff --git a/sdk/typescript/tests-ts/multiscan.test.ts b/sdk/typescript/tests-ts/multiscan.test.ts
index 21cb3cb6d..3ef41a0b2 100644
--- a/sdk/typescript/tests-ts/multiscan.test.ts
+++ b/sdk/typescript/tests-ts/multiscan.test.ts
@@ -17,9 +17,11 @@ import * as filesystem from "node:fs/promises";
import { hostname, tmpdir } from "node:os";
import { join } from "node:path";
import { afterEach, describe, expect, spyOn, test } from "bun:test";
+import { main } from "../src/cli.js";
import type { ScanResult } from "../src/result.js";
import { buildGitHubCredentialArgs, runMultiscan } from "../src/multiscan.js";
import { resolveTrustedExecutable } from "../src/trusted-executable.js";
+import { capture, dependencies, fakeResult } from "./cli-fixtures.js";
type MultiscanOptions = Parameters<typeof runMultiscan>[0];
type SecurityClient = ReturnType<MultiscanOptions["createSecurity"]>;
@@ -82,7 +84,7 @@ async function repository(
async function completedScan(
outputDir: string,
- completeness: "complete" | "partial" = "complete",
+ completeness: "complete" | "partial" | "unknown" = "complete",
): Promise<ScanResult> {
await mkdir(outputDir, { recursive: true });
await Promise.all(
@@ -251,8 +253,383 @@ describe("multiscan", () => {
),
);
+ expect(summary).toMatchObject({ completed: 1, incomplete: 0, failed: 0 });
expect(await results(summary.resultsPath)).toMatchObject([
- { id: "priced", status: "completed", cost },
+ { id: "priced", status: "completed", coverage: "complete", cost },
+ ]);
+ });
+
+ test.each(["partial", "unknown"] as const)(
+ "retains sealed %s coverage without retries or multiplied costs",
+ async (completeness) => {
+ const paths = await fixture();
+ const source = await repository(paths.root, completeness);
+ await writeFile(
+ paths.input,
+ `id,repository,revision\nsealed,${source.path},${source.revision}\n`,
+ );
+ const cost = {
+ model: "gpt-5.6-sol",
+ inputTokens: 1_250,
+ cachedInputTokens: 200,
+ cacheWriteInputTokens: 0,
+ outputTokens: 30,
+ estimatedUsd: 12.5,
+ };
+ const progress: Parameters<
+ NonNullable<MultiscanOptions["onProgress"]>
+ >[0][] = [];
+ let attempts = 0;
+ const security = client(async (_repository, scanOptions = {}) => {
+ attempts += 1;
+ return Object.assign(
+ await completedScan(scanOptions.outputDir!, completeness),
+ { cost },
+ );
+ });
+
+ const summary = await runMultiscan(
+ options(paths, security, {
+ maxAttempts: 3,
+ onProgress: (event) => progress.push(event),
+ }),
+ );
+
+ expect(attempts).toBe(1);
+ expect(summary).toMatchObject({
+ total: 1,
+ completed: 0,
+ incomplete: 1,
+ failed: 0,
+ skipped: 0,
+ });
+ const outputDir = join(paths.output, "artifacts", "sealed", "attempt-1");
+ const warning = `Scan coverage is ${completeness}; results may be incomplete.`;
+ const receipts = await results(summary.resultsPath);
+ expect(receipts).toMatchObject([
+ {
+ id: "sealed",
+ status: "completed_with_incomplete_coverage",
+ attempt: 1,
+ outputDir,
+ coverage: completeness,
+ cost,
+ warning,
+ },
+ ]);
+ expect(
+ receipts.reduce(
+ (total, receipt) =>
+ total + (receipt["cost"] as typeof cost).estimatedUsd,
+ 0,
+ ),
+ ).toBe(cost.estimatedUsd);
+ await Promise.all(
+ [
+ "scan-manifest.json",
+ "findings.json",
+ "coverage.json",
+ "report.md",
+ ].map((name) => access(join(outputDir, name))),
+ );
+ expect(progress).toMatchObject([
+ { repository: "sealed", status: "started", attempt: 1 },
+ {
+ repository: "sealed",
+ status: "completed_with_incomplete_coverage",
+ attempt: 1,
+ warning,
+ },
+ ]);
+
+ const resumed = await runMultiscan(
+ options(paths, security, { maxAttempts: 3 }),
+ );
+ expect(resumed).toMatchObject({
+ completed: 0,
+ incomplete: 1,
+ failed: 0,
+ skipped: 1,
+ });
+ expect(attempts).toBe(1);
+ expect(await results(resumed.resultsPath)).toHaveLength(1);
+ },
+ );
+
+ test.each(["partial", "unknown"] as const)(
+ "resumes legacy sealed %s coverage without rerunning or duplicating cost",
+ async (completeness) => {
+ const paths = await fixture();
+ const source = await repository(paths.root, `legacy-${completeness}`);
+ await writeFile(
+ paths.input,
+ `id,repository,revision\nlegacy,${source.path},${source.revision}\n`,
+ );
+ const outputDir = join(paths.output, "artifacts", "legacy", "attempt-1");
+ await completedScan(outputDir, completeness);
+ await writeFile(
+ join(outputDir, "coverage.json"),
+ `${JSON.stringify({ completeness })}\n`,
+ );
+ const cost = {
+ model: "gpt-5.6-sol",
+ inputTokens: 1_250,
+ cachedInputTokens: 200,
+ cacheWriteInputTokens: 0,
+ outputTokens: 30,
+ estimatedUsd: 231.73,
+ };
+ const receipt = {
+ id: "legacy",
+ repository: source.path,
+ revision: source.revision,
+ mode: "standard",
+ status: "failed",
+ attempt: 1,
+ outputDir,
+ cost,
+ error: "Multiscan repository coverage is incomplete.",
+ };
+ await writeFile(
+ join(paths.output, "results.jsonl"),
+ `${JSON.stringify(receipt)}\n`,
+ );
+ const progress: Parameters<
+ NonNullable<MultiscanOptions["onProgress"]>
+ >[0][] = [];
+ let attempts = 0;
+ const security = client(async (_repository, scanOptions = {}) => {
+ attempts += 1;
+ return await completedScan(scanOptions.outputDir!);
+ });
+
+ const summary = await runMultiscan(
+ options(paths, security, {
+ maxAttempts: 3,
+ onProgress: (event) => progress.push(event),
+ }),
+ );
+
+ expect(summary).toMatchObject({
+ total: 1,
+ completed: 0,
+ incomplete: 1,
+ failed: 0,
+ skipped: 1,
+ });
+ expect(attempts).toBe(0);
+ expect(progress).toEqual([
+ {
+ repository: "legacy",
+ status: "completed_with_incomplete_coverage",
+ attempt: 1,
+ warning: `Scan coverage is ${completeness}; results may be incomplete.`,
+ },
+ ]);
+ expect(await results(summary.resultsPath)).toEqual([receipt]);
+
+ await runMultiscan(options(paths, security, { maxAttempts: 3 }));
+ expect(attempts).toBe(0);
+ expect(await results(summary.resultsPath)).toEqual([receipt]);
+ },
+ );
+
+ test.each([
+ ["operational failures", "partial", "Worker exited unexpectedly.", false],
+ [
+ "complete coverage",
+ "complete",
+ "Multiscan repository coverage is incomplete.",
+ false,
+ ],
+ [
+ "malformed coverage",
+ "malformed",
+ "Multiscan repository coverage is incomplete.",
+ false,
+ ],
+ [
+ "missing artifacts",
+ "partial",
+ "Multiscan repository coverage is incomplete.",
+ true,
+ ],
+ ] as const)(
+ "continues retrying legacy %s",
+ async (_scenario, completeness, error, missingArtifact) => {
+ const paths = await fixture();
+ const source = await repository(paths.root, "legacy-retry");
+ await writeFile(
+ paths.input,
+ `id,repository,revision\nlegacy,${source.path},${source.revision}\n`,
+ );
+ const outputDir = join(paths.output, "artifacts", "legacy", "attempt-1");
+ await completedScan(outputDir);
+ await writeFile(
+ join(outputDir, "coverage.json"),
+ completeness === "malformed"
+ ? "{\n"
+ : `${JSON.stringify({ completeness })}\n`,
+ );
+ if (missingArtifact) await rm(join(outputDir, "report.md"));
+ await writeFile(
+ join(paths.output, "results.jsonl"),
+ `${JSON.stringify({
+ id: "legacy",
+ repository: source.path,
+ revision: source.revision,
+ mode: "standard",
+ status: "failed",
+ attempt: 1,
+ outputDir,
+ error,
+ })}\n`,
+ );
+ let attempts = 0;
+
+ const summary = await runMultiscan(
+ options(
+ paths,
+ client(async (_repository, scanOptions = {}) => {
+ attempts += 1;
+ return await completedScan(scanOptions.outputDir!);
+ }),
+ ),
+ );
+
+ expect(summary).toMatchObject({
+ completed: 1,
+ incomplete: 0,
+ failed: 0,
+ skipped: 0,
+ });
+ expect(attempts).toBe(1);
+ expect(await results(summary.resultsPath)).toMatchObject([
+ { status: "failed", attempt: 1, error },
+ { status: "completed", attempt: 2, coverage: "complete" },
+ ]);
+ },
+ );
+
+ test.each(["partial", "unknown"] as const)(
+ "keeps sealed %s-coverage CLI runs fail-closed without retrying",
+ async (completeness) => {
+ const paths = await fixture();
+ const source = await repository(paths.root, "sample");
+ await writeFile(
+ paths.input,
+ `id,repository,revision\nsample,${source.path},${source.revision}\n`,
+ );
+ const outputDir = join(paths.output, "artifacts", "sample", "attempt-1");
+ await completedScan(outputDir, completeness);
+ const stdout = capture();
+ const stderr = capture();
+ let attempts = 0;
+ const arguments_ = [
+ "bulk-scan",
+ "repositories.csv",
+ "--output-dir",
+ "results",
+ "--max-attempts",
+ "3",
+ "--json",
+ ];
+ const clientDependencies = dependencies({
+ currentDirectory: paths.root,
+ result: fakeResult([], completeness),
+ onRun: () => {
+ attempts += 1;
+ },
+ });
+
+ expect(
+ await main(
+ arguments_,
+ stdout.stream,
+ stderr.stream,
+ clientDependencies,
+ ),
+ ).toBe(2);
+ expect(attempts).toBe(1);
+ expect(JSON.parse(stdout.text())).toMatchObject({
+ total: 1,
+ completed: 0,
+ incomplete: 1,
+ failed: 0,
+ skipped: 0,
+ });
+ const warning = `Scan coverage is ${completeness}; results may be incomplete.`;
+ expect(stderr.text()).toContain(
+ "sample completed_with_incomplete_coverage (attempt 1)",
+ );
+ expect(stderr.text()).toContain(warning);
+ expect(stderr.text()).not.toContain("attempt 2");
+ expect(await results(join(paths.output, "results.jsonl"))).toMatchObject([
+ {
+ status: "completed_with_incomplete_coverage",
+ coverage: completeness,
+ outputDir,
+ },
+ ]);
+
+ const resumedOutput = capture();
+ const resumedError = capture();
+ expect(
+ await main(
+ arguments_,
+ resumedOutput.stream,
+ resumedError.stream,
+ clientDependencies,
+ ),
+ ).toBe(2);
+ expect(JSON.parse(resumedOutput.text())).toMatchObject({
+ completed: 0,
+ incomplete: 1,
+ failed: 0,
+ skipped: 1,
+ });
+ expect(resumedError.text()).toContain(warning);
+ expect(attempts).toBe(1);
+ },
+ );
+
+ test("retries incomplete scans that are missing required artifacts", async () => {
+ const paths = await fixture();
+ const source = await repository(paths.root, "missing-artifact");
+ await writeFile(
+ paths.input,
+ `id,repository,revision\nmissing,${source.path},${source.revision}\n`,
+ );
+
+ let attempts = 0;
+ const summary = await runMultiscan(
+ options(
+ paths,
+ client(async (_repository, scanOptions = {}) => {
+ attempts += 1;
+ const result = await completedScan(
+ scanOptions.outputDir!,
+ attempts === 1 ? "partial" : "complete",
+ );
+ if (attempts === 1) {
+ await rm(join(scanOptions.outputDir!, "report.md"));
+ }
+ return result;
+ }),
+ ),
+ );
+
+ expect(attempts).toBe(2);
+ expect(summary).toMatchObject({ completed: 1, incomplete: 0, failed: 0 });
+ expect(await results(summary.resultsPath)).toMatchObject([
+ {
+ id: "missing",
+ status: "failed",
+ attempt: 1,
+ coverage: "partial",
+ error: "Multiscan scan output is missing required artifacts.",
+ },
+ { id: "missing", status: "completed", attempt: 2, coverage: "complete" },
]);
});
@@ -1109,7 +1486,7 @@ describe("multiscan", () => {
expect(scans).toBe(0);
});
- test("treats incomplete coverage as a failure and still finishes other repositories", async () => {
+ test("records incomplete coverage separately and still finishes other repositories", async () => {
const paths = await fixture();
const incomplete = await repository(paths.root, "incomplete");
const complete = await repository(paths.root, "complete");
@@ -1136,14 +1513,24 @@ describe("multiscan", () => {
: "complete",
),
),
- { maxAttempts: 1 },
+ { maxAttempts: 3 },
),
);
- expect(summary).toMatchObject({ total: 2, completed: 1, failed: 1 });
+ expect(summary).toMatchObject({
+ total: 2,
+ completed: 1,
+ incomplete: 1,
+ failed: 0,
+ });
expect(await results(summary.resultsPath)).toMatchObject([
- { id: "incomplete", status: "failed", attempt: 1 },
- { id: "complete", status: "completed", attempt: 1 },
+ {
+ id: "incomplete",
+ status: "completed_with_incomplete_coverage",
+ attempt: 1,
+ coverage: "partial",
+ },
+ { id: "complete", status: "completed", attempt: 1, coverage: "complete" },
]);
});
});