Skip to content

OpenAI's Codex Security CLI and TypeScript SDK for finding, validating, and fixing security vulnerabilities. npm: https://www.npmjs.com/package/@openai/codex-security

fix(bulk-scan): preserve occupied and replaced checkouts (#2438)

Michael D'Angelocb619eb7bdf1
diff --git a/sdk/typescript/docs/cli.md b/sdk/typescript/docs/cli.md
index 3d2b6d2a8..41ee08c52 100644
--- a/sdk/typescript/docs/cli.md
+++ b/sdk/typescript/docs/cli.md
@@ -1345,6 +1345,12 @@ Concurrency defaults to four repositories. `--max-attempts` defaults to one
attempt per pending repository per invocation. Repeating the command continues
the campaign, skips completed results, and starts pending attempts. Occupied
attempt directories stop that repository and suggest `--recover`.
+If an ordinary attempt finds its checkout path already occupied, it stops before
+fetching. Use `--recover` to scan in a fresh checkout, or inspect and move the
+existing checkout before an ordinary retry.
+If checkout cleanup previously failed, the next invocation removes it only when
+its marker still identifies the same directory and scan attempt. Older or
+replacement checkouts without a matching marker are preserved.
Completed results are skipped only when their saved contract and seal are valid
and the manifest records a completed scan. Damaged or unsuccessful results get a
new attempt, including with `--recover`, while the old artifacts remain available.
diff --git a/sdk/typescript/src/multiscan.ts b/sdk/typescript/src/multiscan.ts
index 80813884c..31aa94e28 100644
--- a/sdk/typescript/src/multiscan.ts
+++ b/sdk/typescript/src/multiscan.ts
@@ -31,7 +31,7 @@ import { promisify } from "node:util";
import Papa from "papaparse";
import type { CodexSecurity, ScanOptions } from "./api.js";
import type { CodexSecurityConfig, JsonObject } from "./config.js";
-import { ContractSchemaError, loadContract } from "./contract.js";
+import { ContractSchemaError, loadContract, readScanFile } from "./contract.js";
import type { ScanCost, ScanUsageMeasurement } from "./cost.js";
import { deepScanOptions } from "./deep-config.js";
import { readThreatModelPath } from "./artifact-export.js";
@@ -68,6 +68,7 @@ import type { ScanResult } from "./result.js";
import { resolveTrustedExecutable } from "./trusted-executable.js";
const execFile = promisify(execFileCallback);
+const CHECKOUT_MARKER = ".git/codex-security-campaign-attempt";
const REQUIRED_ARTIFACTS = ["scan-manifest.json", "findings.json", "report.md"];
const LOCK_LEASE_MS = 30_000;
const LOCK_HEARTBEAT_MS = 5_000;
@@ -92,6 +93,7 @@ interface MultiscanReceipt extends MultiscanTask {
warnings?: string[];
policyFailed?: boolean;
knowledgeBaseFailure?: true;
+ checkoutOccupied?: true;
cleanupPending?: "checkout" | "recovery-checkout";
}
@@ -377,10 +379,11 @@ async function runCampaign(
if (receipt.status === "completed") {
try {
if (receipt.cleanupPending === "checkout") {
- await rm(join(output, "checkouts", task.id), {
- recursive: true,
- force: true,
- });
+ await removeCheckout(
+ join(output, "checkouts", task.id),
+ artifactOutput,
+ options.signal,
+ );
} else if (receipt.cleanupPending === "recovery-checkout") {
const recoveryRoot = join(output, "recovery-checkouts");
await lstat(recoveryRoot);
@@ -388,10 +391,11 @@ async function runCampaign(
const taskRoot = join(recoveryRoot, task.id);
await lstat(taskRoot);
await ensureOutputDirectory(taskRoot);
- await rm(join(taskRoot, `attempt-${receipt.attempt}`), {
- recursive: true,
- force: true,
- });
+ await removeCheckout(
+ join(taskRoot, `attempt-${receipt.attempt}`),
+ artifactOutput,
+ options.signal,
+ );
}
} catch {}
}
@@ -488,6 +492,7 @@ async function runCampaign(
let exhaustedBudget = false;
let requiresRecovery = false;
let knowledgeBaseFailure = false;
+ let checkoutOccupied = false;
try {
await ensureOutputDirectory(artifactRoot);
let result:
@@ -566,11 +571,27 @@ async function runCampaign(
await mkdir(scanDir, { mode: 0o700 });
await mkdir(checkout, { mode: 0o700 });
} else {
- checkout = join(output, "checkouts", task.id);
- await rm(checkout, { recursive: true, force: true });
- await mkdir(checkout, { mode: 0o700 });
+ const destination = join(output, "checkouts", task.id);
+ try {
+ await mkdir(destination, { mode: 0o700 });
+ checkout = destination;
+ } catch (error) {
+ if ((error as NodeJS.ErrnoException).code !== "EEXIST")
+ throw error;
+ checkoutOccupied = true;
+ throw new Error(
+ `Bulk checkout is already occupied: ${destination}. Retry with --recover to use a fresh checkout, or inspect and move the retained checkout before an ordinary retry.`,
+ );
+ }
}
protectedRoot = checkout;
+ // Claim the fresh checkout before Git or the scanner can fail.
+ const marker = await checkoutMarker(checkout, scanDir);
+ await mkdir(join(checkout, ".git"), { mode: 0o700 });
+ await writeFile(join(checkout, CHECKOUT_MARKER), marker, {
+ flag: "wx",
+ mode: 0o600,
+ });
await checkoutRevision(
task,
checkout,
@@ -655,7 +676,7 @@ async function runCampaign(
: errorMessage(error);
} finally {
if (options.recoverScan === undefined && checkout !== undefined) {
- cleanupFailure = await removeCheckout(checkout);
+ cleanupFailure = await removeCheckout(checkout, scanDir);
}
}
const reportedErrors = [failure, cleanupFailure].filter(
@@ -692,6 +713,7 @@ async function runCampaign(
? { cleanupPending: "checkout" }
: {}),
...(knowledgeBaseFailure ? { knowledgeBaseFailure: true } : {}),
+ ...(checkoutOccupied ? { checkoutOccupied: true } : {}),
...(runWarnings.length === 0 ? {} : { warnings: runWarnings }),
policyFailed: attemptPolicyFailed,
};
@@ -702,7 +724,7 @@ async function runCampaign(
await receiptTail;
options.signal?.throwIfAborted();
if (recoveryCheckout !== undefined) {
- reportedError = await removeCheckout(recoveryCheckout);
+ reportedError = await removeCheckout(recoveryCheckout, scanDir);
}
notifyProgress(options, {
repository: task.id,
@@ -718,7 +740,12 @@ async function runCampaign(
completed += 1;
break;
}
- if (exhaustedBudget || attemptedResume || requiresRecovery) {
+ if (
+ exhaustedBudget ||
+ attemptedResume ||
+ requiresRecovery ||
+ checkoutOccupied
+ ) {
failed += 1;
break;
}
@@ -751,12 +778,48 @@ function notifyProgress(
notify(() => options.onProgress?.(event));
}
-async function removeCheckout(path: string): Promise<string | undefined> {
- return rm(path, { recursive: true, force: true }).then(
- () => undefined,
- (error: unknown) =>
- `Multiscan checkout cleanup failed: ${errorMessage(error)}`,
- );
+// A completed attempt can be revisited after its checkout path was reused.
+async function checkoutMarker(
+ checkout: string,
+ scanDir: string,
+): Promise<string> {
+ const directory = await lstat(checkout, { bigint: true });
+ if (!directory.isDirectory())
+ throw new Error("The campaign checkout is no longer a directory.");
+ return JSON.stringify([
+ scanDir,
+ String(directory.dev),
+ String(directory.ino),
+ ]);
+}
+
+async function removeCheckout(
+ checkout: string,
+ scanDir: string,
+ signal?: AbortSignal,
+): Promise<string | undefined> {
+ try {
+ const marker = await lstat(join(checkout, CHECKOUT_MARKER)).catch(
+ undefinedIfMissingFile,
+ );
+ if (!marker?.isFile()) return undefined;
+ const expected = Buffer.from(await checkoutMarker(checkout, scanDir));
+ if (
+ marker.size === expected.length &&
+ (
+ await readScanFile(
+ checkout,
+ CHECKOUT_MARKER,
+ "campaign checkout marker",
+ signal,
+ )
+ ).equals(expected)
+ ) {
+ await rm(checkout, { recursive: true, force: true });
+ }
+ } catch (error) {
+ return `Multiscan checkout cleanup failed: ${errorMessage(error)}`;
+ }
}
async function ensureOutputDirectory(
@@ -1139,8 +1202,11 @@ async function readReceipts(
const previous = receipts.get(id);
receipts.set(id, {
maxAttempt: Math.max(previous?.maxAttempt ?? 0, receipt.attempt),
- // Extraction failures never start or modify a scan; keep its last receipt.
- scan: receipt.knowledgeBaseFailure ? previous?.scan : receipt,
+ // Pre-scan failures reserve an attempt but leave the last scan recoverable.
+ scan:
+ receipt.knowledgeBaseFailure || receipt.checkoutOccupied
+ ? previous?.scan
+ : receipt,
});
}
if (contents.at(-1) !== 10) {
diff --git a/sdk/typescript/tests-ts/multiscan-checkout.test.ts b/sdk/typescript/tests-ts/multiscan-checkout.test.ts
index 89701e71a..244bf9787 100644
--- a/sdk/typescript/tests-ts/multiscan-checkout.test.ts
+++ b/sdk/typescript/tests-ts/multiscan-checkout.test.ts
@@ -433,7 +433,36 @@ describe("multiscan", () => {
expect(run).toHaveBeenCalledTimes(1);
});
- test("retries failed scans and continues later tasks when checkout cleanup fails", async () => {
+ test("preserves an existing checkout directory on the first campaign attempt", async () => {
+ const { paths } = await repositoryFixture("occupied-first", "demo");
+ const checkout = join(paths.output, "checkouts", "demo");
+ await mkdir(checkout, { recursive: true });
+ const note = join(checkout, "local-notes.txt");
+ await writeFile(note, "Local work.\n");
+ const run = mock<SecurityClient["run"]>(completeRun);
+ const configured = options(paths, client(run));
+
+ expect(await runMultiscan(configured)).toMatchObject({
+ completed: 0,
+ failed: 1,
+ });
+ expect(run).not.toHaveBeenCalled();
+ expect(await readFile(note, "utf8")).toBe("Local work.\n");
+ const ledger = join(paths.output, "results.jsonl");
+ expect(await results(ledger)).toMatchObject([
+ { attempt: 1, status: "failed", checkoutOccupied: true },
+ ]);
+
+ const recovered = await runMultiscan({
+ ...configured,
+ recoverScan: async () => undefined,
+ });
+ expect(recovered).toMatchObject({ completed: 1, failed: 0 });
+ expect(run).toHaveBeenCalledTimes(1);
+ expect(await readFile(note, "utf8")).toBe("Local work.\n");
+ });
+
+ test("continues later tasks when a failed scan leaves its checkout occupied", async () => {
const { paths, source } = await repositoryFixture("cleanup-retry", "retry");
await appendFile(paths.input, `next,${source.path},${source.revision}\n`);
const checkout = join(paths.output, "checkouts", "retry");
@@ -462,8 +491,8 @@ describe("multiscan", () => {
try {
const summary = await runMultiscan(options(paths, client(run)));
- expect(summary).toMatchObject({ total: 2, completed: 2, failed: 0 });
- expect(run).toHaveBeenCalledTimes(3);
+ expect(summary).toMatchObject({ total: 2, completed: 1, failed: 1 });
+ expect(run).toHaveBeenCalledTimes(2);
expect(await results(summary.resultsPath)).toMatchObject([
{
id: "retry",
@@ -472,7 +501,12 @@ describe("multiscan", () => {
error:
"ORIGINAL_SCAN_FAILURE; Multiscan checkout cleanup failed: EACCES: synthetic cleanup failure",
},
- { id: "retry", status: "completed", attempt: 2 },
+ {
+ id: "retry",
+ status: "failed",
+ attempt: 2,
+ error: expect.stringContaining("move the retained checkout"),
+ },
{ id: "next", status: "completed", attempt: 1 },
]);
} finally {
@@ -615,8 +649,8 @@ describe("multiscan", () => {
expect(await results(summary.resultsPath)).toHaveLength(3);
});
- test("rejects another supervisor and recovers a crashed owner's checkout", async () => {
- const { paths } = await repositoryFixture("exclusive");
+ test("rejects another supervisor and preserves a crashed owner's checkout", async () => {
+ const { paths, source } = await repositoryFixture("exclusive");
const running = Promise.withResolvers<void>();
const finish = Promise.withResolvers<void>();
const security = client(async (_repository, scanOptions = {}) => {
@@ -656,11 +690,29 @@ describe("multiscan", () => {
JSON.stringify({ pid: 999_999_999 }),
);
const checkout = join(paths.output, "checkouts", "exclusive");
- await mkdir(checkout);
+ git(paths.root, "clone", "--quiet", source.path, checkout);
+ const marker = join(checkout, "retained.txt");
+ await writeFile(marker, "Preserved crashed-owner checkout data.\n");
+ const identity = await lstat(checkout);
const recovered = await runMultiscan(options(paths, security));
- expect(recovered).toMatchObject({ completed: 1, failed: 0, skipped: 0 });
- expect(await readdir(join(paths.output, "checkouts"))).toEqual([]);
+ expect(recovered).toMatchObject({ completed: 0, failed: 1, skipped: 0 });
+ const receipts = await results(recovered.resultsPath);
+ expect(receipts).toHaveLength(2);
+ expect(receipts[0]).toEqual(receipt);
+ expect(receipts[1]).toMatchObject({
+ attempt: 2,
+ status: "failed",
+ error: expect.stringContaining("move the retained checkout"),
+ });
+ expect(await readdir(join(paths.output, "checkouts"))).toEqual([
+ "exclusive",
+ ]);
+ const retained = await lstat(checkout);
+ expect([retained.dev, retained.ino]).toEqual([identity.dev, identity.ino]);
+ expect(await readFile(marker, "utf8")).toBe(
+ "Preserved crashed-owner checkout data.\n",
+ );
await expect(access(lock)).rejects.toThrow();
});
diff --git a/sdk/typescript/tests-ts/multiscan-recovery.test.ts b/sdk/typescript/tests-ts/multiscan-recovery.test.ts
index 526bcea5d..cca5beedd 100644
--- a/sdk/typescript/tests-ts/multiscan-recovery.test.ts
+++ b/sdk/typescript/tests-ts/multiscan-recovery.test.ts
@@ -7,6 +7,7 @@ import {
mkdir,
readFile,
readdir,
+ rename,
rm,
writeFile,
} from "node:fs/promises";
@@ -62,6 +63,100 @@ function receiptCost(onSerialized: () => void) {
}
describe("multiscan", () => {
+ test.each(["ordinary", "recovery"] as const)(
+ "preserves a replacement checkout when a completed %s attempt is skipped",
+ async (kind) => {
+ const { paths } = await repositoryFixture("replaced");
+ const run = mock(completeRun);
+ const configured = options(paths, client(run), { maxAttempts: 1 });
+ let checkout: string;
+ if (kind === "recovery") {
+ checkout = join(
+ paths.output,
+ "recovery-checkouts",
+ "replaced",
+ "attempt-2",
+ );
+ await runMultiscan(
+ options(paths, client(rejecting("Stopped")), { maxAttempts: 1 }),
+ );
+ expect(
+ await runMultiscan({
+ ...configured,
+ recoverScan: async () => undefined,
+ }),
+ ).toMatchObject({ completed: 1 });
+ await expect(lstat(checkout)).rejects.toMatchObject({ code: "ENOENT" });
+ } else {
+ checkout = join(paths.output, "checkouts", "replaced");
+ const originalRm = filesystem.rm;
+ const removeSpy = spyOn(filesystem, "rm").mockImplementation(
+ async (...args) => {
+ if (run.mock.calls.length > 0 && String(args[0]) === checkout)
+ throw new Error("Checkout is busy");
+ return originalRm(...args);
+ },
+ );
+ try {
+ expect(await runMultiscan(configured)).toMatchObject({
+ completed: 1,
+ });
+ } finally {
+ removeSpy.mockRestore();
+ }
+ await rename(checkout, join(paths.root, "old-checkout"));
+ }
+ await mkdir(checkout);
+ git(checkout, "init", "-q");
+ const note = join(checkout, "uncommitted.txt");
+ await writeFile(note, "Independent checkout.\n");
+ const ledger = join(paths.output, "results.jsonl");
+ const before = await readFile(ledger);
+ expect(await runMultiscan(configured)).toMatchObject({
+ completed: 1,
+ skipped: 1,
+ });
+ expect(run).toHaveBeenCalledTimes(1);
+ expect(await readFile(ledger)).toEqual(before);
+ expect(await readFile(note, "utf8")).toBe("Independent checkout.\n");
+ expect((await lstat(join(checkout, ".git"))).isDirectory()).toBe(true);
+ },
+ );
+
+ test.each(["ordinary", "recovery"] as const)(
+ "preserves a checkout replaced while its %s scan is running",
+ async (kind) => {
+ const { paths } = await repositoryFixture("replaced");
+ const recovery = kind === "recovery";
+ if (recovery) {
+ await runMultiscan(
+ options(paths, client(rejecting("Stopped")), { maxAttempts: 1 }),
+ );
+ }
+ const run = mock<SecurityClient["run"]>(async (checkout, scan = {}) => {
+ const result = await completedScan(scan.outputDir!);
+ await rename(checkout, join(paths.root, "old-checkout"));
+ await mkdir(checkout);
+ git(checkout, "init", "-q");
+ await writeFile(join(checkout, "note.txt"), "New work.\n");
+ return result;
+ });
+ const summary = await runMultiscan(
+ options(paths, client(run), {
+ maxAttempts: 1,
+ recoverScan: recovery ? async () => undefined : undefined,
+ }),
+ );
+ expect(summary).toMatchObject({ completed: 1, failed: 0 });
+ expect(run).toHaveBeenCalledTimes(1);
+ const checkout = run.mock.calls[0]![0];
+ expect(await readFile(join(checkout, "note.txt"), "utf8")).toBe(
+ "New work.\n",
+ );
+ expect((await lstat(join(checkout, ".git"))).isDirectory()).toBe(true);
+ },
+ );
+
test("serializes receipt writes through close and drains workers before unlocking", async () => {
const { paths, source } = await repositoryFixture("receipt-order", "first");
await appendFile(paths.input, `second,${source.path},${source.revision}\n`);