Skip to content

OpenAI's Codex Security CLI and TypeScript SDK for finding, validating, and fixing security vulnerabilities. npm: https://www.npmjs.com/package/@openai/codex-security

fix(bulk-scan): preserve outcomes when checkout cleanup fails (#1433)

Bharath Vemulacdce5d6040df
diff --git a/sdk/typescript/src/multiscan.ts b/sdk/typescript/src/multiscan.ts
index 82dd3d5f3..fb55890ad 100644
--- a/sdk/typescript/src/multiscan.ts
+++ b/sdk/typescript/src/multiscan.ts
@@ -73,6 +73,7 @@ interface MultiscanReceipt extends MultiscanTask {
warnings?: string[];
policyFailed?: boolean;
knowledgeBaseFailure?: true;
+ cleanupPending?: "checkout" | "recovery-checkout";
}
interface MultiscanHistory {
@@ -299,6 +300,31 @@ async function runCampaign(
receipt.outputDir === selectedArtifactOutput) &&
(await hasArtifacts(artifactOutput))
) {
+ if (
+ receipt.status === "completed" &&
+ Number.isSafeInteger(receipt.attempt) &&
+ receipt.attempt > 0
+ ) {
+ try {
+ if (receipt.cleanupPending === "checkout") {
+ await rm(join(output, "checkouts", task.id), {
+ recursive: true,
+ force: true,
+ });
+ } else if (receipt.cleanupPending === "recovery-checkout") {
+ const recoveryRoot = join(output, "recovery-checkouts");
+ await lstat(recoveryRoot);
+ await ensureOutputDirectory(recoveryRoot);
+ const taskRoot = join(recoveryRoot, task.id);
+ await lstat(taskRoot);
+ await ensureOutputDirectory(taskRoot);
+ await rm(join(taskRoot, `attempt-${receipt.attempt}`), {
+ recursive: true,
+ force: true,
+ });
+ }
+ } catch {}
+ }
if (receipt.status !== "failed" && receipt.warnings?.length) {
warnings.push({ repository: task.id, warnings: receipt.warnings });
for (const warning of receipt.warnings) {
@@ -370,6 +396,7 @@ async function runCampaign(
let protectedRoot = join(output, "checkouts", task.id);
let attemptedResume = false;
let failure: string | undefined;
+ let cleanupFailure: string | undefined;
const runWarnings: string[] = [];
let attemptPolicyFailed: boolean | undefined;
let cost: Readonly<ScanCost> | null = null;
@@ -524,9 +551,14 @@ async function runCampaign(
: errorMessage(error);
} finally {
if (options.recoverScan === undefined && checkout !== undefined) {
- await rm(checkout, { recursive: true, force: true });
+ cleanupFailure = await removeCheckout(checkout);
}
}
+ const reportedErrors = [failure, cleanupFailure].filter(
+ (message): message is string => message !== undefined,
+ );
+ let reportedError =
+ reportedErrors.length === 0 ? undefined : reportedErrors.join("; ");
const status = failure !== undefined ? "failed" : "completed";
if (threatModelPath === undefined)
threatModelPath = await readThreatModelPath(scanDir, {
@@ -534,33 +566,36 @@ async function runCampaign(
protectedRoot,
signal: options.signal,
});
- await appendReceipt(
- ledger,
- `${JSON.stringify({
- ...task,
- status,
- attempt,
- outputDir: scanDir,
- ...(threatModelPath === null ? {} : { threatModelPath }),
- ...(cost === null ? {} : { cost }),
- ...(failure === undefined ? {} : { error: failure }),
- ...(knowledgeBaseFailure ? { knowledgeBaseFailure: true } : {}),
- ...(runWarnings.length === 0 ? {} : { warnings: runWarnings }),
- policyFailed: attemptPolicyFailed,
- })}\n`,
- );
- if (
- options.recoverScan !== undefined &&
- failure === undefined &&
- checkout !== undefined
- ) {
- await rm(checkout, { recursive: true, force: true });
+ const recoveryCheckout =
+ options.recoverScan !== undefined && failure === undefined
+ ? checkout
+ : undefined;
+ const receipt: MultiscanReceipt = {
+ ...task,
+ status,
+ attempt,
+ outputDir: scanDir,
+ ...(threatModelPath === null ? {} : { threatModelPath }),
+ ...(cost === null ? {} : { cost }),
+ ...(reportedError === undefined ? {} : { error: reportedError }),
+ ...(recoveryCheckout !== undefined
+ ? { cleanupPending: "recovery-checkout" }
+ : failure === undefined && cleanupFailure !== undefined
+ ? { cleanupPending: "checkout" }
+ : {}),
+ ...(knowledgeBaseFailure ? { knowledgeBaseFailure: true } : {}),
+ ...(runWarnings.length === 0 ? {} : { warnings: runWarnings }),
+ policyFailed: attemptPolicyFailed,
+ };
+ await appendReceipt(ledger, `${JSON.stringify(receipt)}\n`);
+ if (recoveryCheckout !== undefined) {
+ reportedError = await removeCheckout(recoveryCheckout);
}
notifyProgress(options, {
repository: task.id,
attempt,
status,
- ...(failure === undefined ? {} : { error: failure }),
+ ...(reportedError === undefined ? {} : { error: reportedError }),
});
if (failure === undefined) {
policyFailed ||= attemptPolicyFailed === true;
@@ -603,6 +638,14 @@ function notifyProgress(
notify(() => options.onProgress?.(event));
}
+async function removeCheckout(path: string): Promise<string | undefined> {
+ return rm(path, { recursive: true, force: true }).then(
+ () => undefined,
+ (error: unknown) =>
+ `Multiscan checkout cleanup failed: ${errorMessage(error)}`,
+ );
+}
+
async function ensureOutputDirectory(path: string): Promise<string> {
const metadata = await lstat(path, { bigint: true }).catch(
undefinedIfMissingFile,
diff --git a/sdk/typescript/tests-ts/multiscan.test.ts b/sdk/typescript/tests-ts/multiscan.test.ts
index ecf6f9f0c..5dd20698a 100644
--- a/sdk/typescript/tests-ts/multiscan.test.ts
+++ b/sdk/typescript/tests-ts/multiscan.test.ts
@@ -20,7 +20,7 @@ import {
} from "node:fs/promises";
import * as filesystem from "node:fs/promises";
import { hostname } from "node:os";
-import { join } from "node:path";
+import { dirname, join } from "node:path";
import { fileURLToPath } from "node:url";
import { afterEach, describe, expect, spyOn, test, mock } from "bun:test";
import { main } from "../src/cli.js";
@@ -551,12 +551,11 @@ describe("multiscan", () => {
);
return completedScan(scan.outputDir!);
});
- const result = await runMultiscan(
- options(paths, client(runs), {
- maxAttempts: 1,
- recoverScan: resumed,
- }),
- );
+ const configured = options(paths, client(runs), {
+ maxAttempts: 1,
+ recoverScan: resumed,
+ });
+ const result = await runMultiscan(configured);
expect(result).toMatchObject({
completed: 2,
failed: 0,
@@ -579,6 +578,17 @@ describe("multiscan", () => {
attempt: 6,
status: "completed",
});
+ expect(await runMultiscan(configured)).toMatchObject({
+ completed: 2,
+ failed: 0,
+ skipped: 2,
+ });
+ expect(runs).toHaveBeenCalledTimes(1);
+ expect(resumed).toHaveBeenCalledTimes(1);
+ expect((await lstat(checkout)).ino).toBe(inode);
+ expect(await readFile(join(checkout, "source"), "utf8")).toBe(
+ "keep checkout",
+ );
});
test.each([false, true])(
@@ -737,6 +747,22 @@ describe("multiscan", () => {
status: failure ? "failed" : "completed",
});
expect(await readFile(join(dir, "checkpoint"), "utf8")).toBe("keep");
+ if (!failure) {
+ expect(
+ await runMultiscan(
+ options(paths, client(runs), {
+ maxAttempts: 3,
+ config: { pythonPath: PYTHON },
+ recoverScan,
+ }),
+ ),
+ ).toMatchObject({ completed: 1, failed: 0, skipped: 1 });
+ expect(recoverScan).toHaveBeenCalledTimes(1);
+ expect(runs).not.toHaveBeenCalled();
+ expect(await readFile(join(checkout, "source"), "utf8")).toBe(
+ "keep checkout",
+ );
+ }
},
);
@@ -1522,6 +1548,355 @@ describe("multiscan", () => {
]);
});
+ test.each(["completed", "failed"] as const)(
+ "records a %s scan outcome when checkout cleanup fails",
+ async (outcome) => {
+ const { paths } = await repositoryFixture(
+ `cleanup-${outcome}`,
+ "stubborn",
+ );
+ const checkout = join(paths.output, "checkouts", "stubborn");
+ const originalRm = filesystem.rm;
+ let scanned = false;
+ let rejectedCleanup = false;
+ const removeSpy = spyOn(filesystem, "rm").mockImplementation(
+ async (...args: Parameters<typeof originalRm>) => {
+ if (scanned && !rejectedCleanup && String(args[0]) === checkout) {
+ rejectedCleanup = true;
+ throw Object.assign(
+ new Error(`EACCES: permission denied, rm '${checkout}'`),
+ { code: "EACCES" },
+ );
+ }
+ return await Reflect.apply(originalRm, filesystem, args);
+ },
+ );
+ const run = mock<SecurityClient["run"]>(
+ async (_repository, scanOptions = {}) => {
+ if (outcome === "failed") {
+ scanned = true;
+ throw new Error("ORIGINAL_SCAN_FAILURE");
+ }
+ const result = await completedScan(scanOptions.outputDir!);
+ scanned = true;
+ return result;
+ },
+ );
+ const configured = options(paths, client(run), { maxAttempts: 1 });
+
+ try {
+ const summary = await runMultiscan(configured);
+ expect(summary).toMatchObject(
+ outcome === "completed"
+ ? { completed: 1, failed: 0 }
+ : { completed: 0, failed: 1 },
+ );
+ const receipts = await results(summary.resultsPath);
+ expect(receipts).toHaveLength(1);
+ expect(receipts[0]).toMatchObject({ status: outcome });
+ expect(receipts[0]?.["cleanupPending"]).toBe(
+ outcome === "completed" ? "checkout" : undefined,
+ );
+ const error = String(receipts[0]?.["error"]);
+ expect(error).toContain("Multiscan checkout cleanup failed");
+ if (outcome === "failed") {
+ expect(error).toContain("ORIGINAL_SCAN_FAILURE");
+ } else {
+ expect((await lstat(checkout)).isDirectory()).toBe(true);
+ expect(await runMultiscan(configured)).toMatchObject({ skipped: 1 });
+ expect(run).toHaveBeenCalledTimes(1);
+ await expect(lstat(checkout)).rejects.toMatchObject({
+ code: "ENOENT",
+ });
+ }
+ } finally {
+ removeSpy.mockRestore();
+ }
+ },
+ );
+
+ test("keeps recovery receipts readable and continues after checkout cleanup fails", async () => {
+ const { paths, source } = await repositoryFixture(
+ "recovery-cleanup",
+ "stubborn",
+ );
+ await appendFile(paths.input, `next,${source.path},${source.revision}\n`);
+ await runMultiscan(
+ options(paths, client(rejecting("Interrupted scan")), { maxAttempts: 1 }),
+ );
+ const checkout = join(
+ paths.output,
+ "recovery-checkouts",
+ "stubborn",
+ "attempt-2",
+ );
+ const originalRm = filesystem.rm;
+ const originalOpen = filesystem.open;
+ let diagnosticWrites = 0;
+ const openSpy = spyOn(filesystem, "open").mockImplementation(
+ async (...args: Parameters<typeof originalOpen>) => {
+ const file = await Reflect.apply(originalOpen, filesystem, args);
+ if (String(args[0]) === join(paths.output, "results.jsonl")) {
+ const write = file.writeFile.bind(file);
+ file.writeFile = async (data, options) => {
+ const text = String(data);
+ if (
+ JSON.parse(text).error?.includes(
+ "Multiscan checkout cleanup failed",
+ )
+ ) {
+ diagnosticWrites += 1;
+ await write(text.slice(0, 24), options);
+ throw new Error("EIO: synthetic partial diagnostic write");
+ }
+ return await write(data, options);
+ };
+ }
+ return file;
+ },
+ );
+ let scanned = false;
+ let rejectedCleanup = false;
+ const removeSpy = spyOn(filesystem, "rm").mockImplementation(
+ async (...args: Parameters<typeof originalRm>) => {
+ if (scanned && !rejectedCleanup && String(args[0]) === checkout) {
+ rejectedCleanup = true;
+ throw Object.assign(
+ new Error(`EBUSY: resource busy, rm '${checkout}'`),
+ { code: "EBUSY" },
+ );
+ }
+ return await Reflect.apply(originalRm, filesystem, args);
+ },
+ );
+ const run = mock<SecurityClient["run"]>(
+ async (_repository, scanOptions = {}) => {
+ const result = await completedScan(scanOptions.outputDir!);
+ scanned = true;
+ return result;
+ },
+ );
+ const progress: Array<
+ Parameters<NonNullable<MultiscanOptions["onProgress"]>>[0]
+ > = [];
+ const configured = options(paths, client(run), {
+ maxAttempts: 1,
+ recoverScan: async () => undefined,
+ onProgress: (event) => progress.push(event),
+ });
+
+ try {
+ const summary = await runMultiscan(configured);
+ expect(summary).toMatchObject({ completed: 2, failed: 0, skipped: 0 });
+ expect(
+ progress.find(
+ (event) =>
+ event.repository === "stubborn" && event.status === "completed",
+ ),
+ ).toMatchObject({
+ repository: "stubborn",
+ attempt: 2,
+ status: "completed",
+ error: expect.stringContaining("Multiscan checkout cleanup failed"),
+ });
+ const receipts = await results(summary.resultsPath);
+ const recovered = receipts.find(
+ (receipt) =>
+ receipt["id"] === "stubborn" && receipt["status"] === "completed",
+ );
+ expect(recovered).toMatchObject({
+ status: "completed",
+ attempt: 2,
+ cleanupPending: "recovery-checkout",
+ });
+ expect(recovered).not.toHaveProperty("error");
+ expect(receipts.at(-1)).toMatchObject({
+ id: "next",
+ status: "completed",
+ });
+ expect(diagnosticWrites).toBe(0);
+ expect((await lstat(checkout)).isDirectory()).toBe(true);
+
+ expect(await runMultiscan(options(paths, client(run)))).toMatchObject({
+ completed: 2,
+ failed: 0,
+ skipped: 2,
+ });
+ expect(run).toHaveBeenCalledTimes(2);
+ await expect(lstat(checkout)).rejects.toMatchObject({ code: "ENOENT" });
+ } finally {
+ removeSpy.mockRestore();
+ openSpy.mockRestore();
+ }
+ });
+
+ test("reuses recovered completion saved before checkout removal", async () => {
+ const { paths } = await repositoryFixture("cleanup-interruption", "done");
+ await runMultiscan(
+ options(paths, client(rejecting("Interrupted scan")), { maxAttempts: 1 }),
+ );
+ const checkout = join(
+ paths.output,
+ "recovery-checkouts",
+ "done",
+ "attempt-2",
+ );
+ const ledger = join(paths.output, "results.jsonl");
+ const originalRm = filesystem.rm;
+ let persistedBeforeRemoval: string | undefined;
+ const removeSpy = spyOn(filesystem, "rm").mockImplementation(
+ async (...args: Parameters<typeof originalRm>) => {
+ if (String(args[0]) === checkout) {
+ persistedBeforeRemoval = await readFile(ledger, "utf8");
+ }
+ return await Reflect.apply(originalRm, filesystem, args);
+ },
+ );
+ const run = mock(completeRun);
+ try {
+ await runMultiscan(
+ options(paths, client(run), { recoverScan: async () => undefined }),
+ );
+ } finally {
+ removeSpy.mockRestore();
+ }
+
+ expect(persistedBeforeRemoval).toBeDefined();
+ await writeFile(ledger, persistedBeforeRemoval!);
+ expect((await results(ledger)).at(-1)).toMatchObject({
+ status: "completed",
+ attempt: 2,
+ cleanupPending: "recovery-checkout",
+ });
+ expect(await runMultiscan(options(paths, client(run)))).toMatchObject({
+ completed: 1,
+ failed: 0,
+ skipped: 1,
+ });
+ expect(run).toHaveBeenCalledTimes(1);
+ });
+
+ test("retries failed scans and continues later tasks when checkout cleanup fails", async () => {
+ const { paths, source } = await repositoryFixture("cleanup-retry", "retry");
+ await appendFile(paths.input, `next,${source.path},${source.revision}\n`);
+ const checkout = join(paths.output, "checkouts", "retry");
+ const originalRm = filesystem.rm;
+ let attempted = false;
+ let rejectedCleanup = false;
+ const removeSpy = spyOn(filesystem, "rm").mockImplementation(
+ async (...args: Parameters<typeof originalRm>) => {
+ if (attempted && !rejectedCleanup && String(args[0]) === checkout) {
+ rejectedCleanup = true;
+ throw new Error("EACCES: synthetic cleanup failure");
+ }
+ return await Reflect.apply(originalRm, filesystem, args);
+ },
+ );
+ const run = mock<SecurityClient["run"]>(
+ async (_repository, scanOptions = {}) => {
+ if (!attempted) {
+ attempted = true;
+ throw new Error("ORIGINAL_SCAN_FAILURE");
+ }
+ return await completedScan(scanOptions.outputDir!);
+ },
+ );
+
+ try {
+ const summary = await runMultiscan(options(paths, client(run)));
+
+ expect(summary).toMatchObject({ total: 2, completed: 2, failed: 0 });
+ expect(run).toHaveBeenCalledTimes(3);
+ expect(await results(summary.resultsPath)).toMatchObject([
+ {
+ id: "retry",
+ status: "failed",
+ attempt: 1,
+ error:
+ "ORIGINAL_SCAN_FAILURE; Multiscan checkout cleanup failed: EACCES: synthetic cleanup failure",
+ },
+ { id: "retry", status: "completed", attempt: 2 },
+ { id: "next", status: "completed", attempt: 1 },
+ ]);
+ } finally {
+ removeSpy.mockRestore();
+ }
+ });
+
+ test.each(["root", "task"] as const)(
+ "preserves completed scans and external files when the recovery checkout %s is linked",
+ async (parent) => {
+ const { paths } = await repositoryFixture("linked-cleanup", "done");
+ const scans = mock(completeRun);
+ const configured = options(paths, client(scans));
+ await runMultiscan(configured);
+ const ledger = join(paths.output, "results.jsonl");
+ const [receipt] = await results(ledger);
+ await writeFile(
+ ledger,
+ `${JSON.stringify({ ...receipt, cleanupPending: "recovery-checkout" })}\n`,
+ );
+ const recoveryRoot = join(paths.output, "recovery-checkouts");
+ const external = await temporaryDirectory();
+ const preserved = join(
+ external,
+ ...(parent === "root" ? ["done"] : []),
+ "attempt-1",
+ "keep.txt",
+ );
+ await mkdir(dirname(preserved), { recursive: true });
+ await writeFile(preserved, "preserved\n");
+ if (parent === "task") await mkdir(recoveryRoot, { mode: 0o700 });
+ await symlink(
+ external,
+ parent === "root" ? recoveryRoot : join(recoveryRoot, "done"),
+ process.platform === "win32" ? "junction" : "dir",
+ );
+
+ expect(await runMultiscan(configured)).toMatchObject({
+ completed: 1,
+ failed: 0,
+ skipped: 1,
+ });
+ expect(scans).toHaveBeenCalledTimes(1);
+ expect(await readFile(preserved, "utf8")).toBe("preserved\n");
+ },
+ );
+
+ test("preserves sibling artifacts when a cleanup receipt has a traversal attempt", async () => {
+ const { paths } = await repositoryFixture("cleanup-traversal", "done");
+ const scans = mock(completeRun);
+ const configured = options(paths, client(scans));
+ await runMultiscan(configured);
+ const external = join(paths.root, "other-scan");
+ await completedScan(external);
+ const preserved = join(external, "keep.txt");
+ await writeFile(preserved, "preserved\n");
+ await mkdir(join(paths.output, "recovery-checkouts", "done"), {
+ recursive: true,
+ mode: 0o700,
+ });
+ const ledger = join(paths.output, "results.jsonl");
+ const [receipt] = await results(ledger);
+ await writeFile(
+ ledger,
+ `${JSON.stringify({
+ ...receipt,
+ attempt: "1/../../../../other-scan",
+ outputDir: external,
+ cleanupPending: "recovery-checkout",
+ })}\n`,
+ );
+
+ expect(await runMultiscan(configured)).toMatchObject({
+ completed: 1,
+ failed: 0,
+ skipped: 1,
+ });
+ expect(scans).toHaveBeenCalledTimes(1);
+ expect(await readFile(preserved, "utf8")).toBe("preserved\n");
+ });
+
test("limits simultaneous checkouts to the requested worker count", async () => {
const paths = await fixture();
const knowledgeBasePaths = [