OpenAI's Codex Security CLI and TypeScript SDK for finding, validating, and fixing security vulnerabilities. npm: https://www.npmjs.com/package/@openai/codex-security
TypeScript ai-securityapplication-securityclicode-scanningcodexcodex-securitycybersecuritydevsecopsnodejsnpmopenaisdksecuritytypescriptvulnerability-scanning
fix(bulk-scan): preserve occupied and replaced checkouts (#2438)
Michael D'Angelocb619eb7bdf1
diff --git a/sdk/typescript/docs/cli.md b/sdk/typescript/docs/cli.mdindex 3d2b6d2a8..41ee08c52 100644--- a/sdk/typescript/docs/cli.md+++ b/sdk/typescript/docs/cli.md@@ -1345,6 +1345,12 @@ Concurrency defaults to four repositories. `--max-attempts` defaults to oneattempt per pending repository per invocation. Repeating the command continuesthe campaign, skips completed results, and starts pending attempts. Occupiedattempt directories stop that repository and suggest `--recover`.+If an ordinary attempt finds its checkout path already occupied, it stops before+fetching. Use `--recover` to scan in a fresh checkout, or inspect and move the+existing checkout before an ordinary retry.+If checkout cleanup previously failed, the next invocation removes it only when+its marker still identifies the same directory and scan attempt. Older or+replacement checkouts without a matching marker are preserved.Completed results are skipped only when their saved contract and seal are validand the manifest records a completed scan. Damaged or unsuccessful results get anew attempt, including with `--recover`, while the old artifacts remain available.diff --git a/sdk/typescript/src/multiscan.ts b/sdk/typescript/src/multiscan.tsindex 80813884c..31aa94e28 100644--- a/sdk/typescript/src/multiscan.ts+++ b/sdk/typescript/src/multiscan.ts@@ -31,7 +31,7 @@ import { promisify } from "node:util";import Papa from "papaparse";import type { CodexSecurity, ScanOptions } from "./api.js";import type { CodexSecurityConfig, JsonObject } from "./config.js";-import { ContractSchemaError, loadContract } from "./contract.js";+import { ContractSchemaError, loadContract, readScanFile } from "./contract.js";import type { ScanCost, ScanUsageMeasurement } from "./cost.js";import { deepScanOptions } from "./deep-config.js";import { readThreatModelPath } from "./artifact-export.js";@@ -68,6 +68,7 @@ import type { ScanResult } from "./result.js";import { resolveTrustedExecutable } from "./trusted-executable.js";const execFile = promisify(execFileCallback);+const CHECKOUT_MARKER = ".git/codex-security-campaign-attempt";const REQUIRED_ARTIFACTS = ["scan-manifest.json", "findings.json", "report.md"];const LOCK_LEASE_MS = 30_000;const LOCK_HEARTBEAT_MS = 5_000;@@ -92,6 +93,7 @@ interface MultiscanReceipt extends MultiscanTask {warnings?: string[];policyFailed?: boolean;knowledgeBaseFailure?: true;+ checkoutOccupied?: true;cleanupPending?: "checkout" | "recovery-checkout";}@@ -377,10 +379,11 @@ async function runCampaign(if (receipt.status === "completed") {try {if (receipt.cleanupPending === "checkout") {- await rm(join(output, "checkouts", task.id), {- recursive: true,- force: true,- });+ await removeCheckout(+ join(output, "checkouts", task.id),+ artifactOutput,+ options.signal,+ );} else if (receipt.cleanupPending === "recovery-checkout") {const recoveryRoot = join(output, "recovery-checkouts");await lstat(recoveryRoot);@@ -388,10 +391,11 @@ async function runCampaign(const taskRoot = join(recoveryRoot, task.id);await lstat(taskRoot);await ensureOutputDirectory(taskRoot);- await rm(join(taskRoot, `attempt-${receipt.attempt}`), {- recursive: true,- force: true,- });+ await removeCheckout(+ join(taskRoot, `attempt-${receipt.attempt}`),+ artifactOutput,+ options.signal,+ );}} catch {}}@@ -488,6 +492,7 @@ async function runCampaign(let exhaustedBudget = false;let requiresRecovery = false;let knowledgeBaseFailure = false;+ let checkoutOccupied = false;try {await ensureOutputDirectory(artifactRoot);let result:@@ -566,11 +571,27 @@ async function runCampaign(await mkdir(scanDir, { mode: 0o700 });await mkdir(checkout, { mode: 0o700 });} else {- checkout = join(output, "checkouts", task.id);- await rm(checkout, { recursive: true, force: true });- await mkdir(checkout, { mode: 0o700 });+ const destination = join(output, "checkouts", task.id);+ try {+ await mkdir(destination, { mode: 0o700 });+ checkout = destination;+ } catch (error) {+ if ((error as NodeJS.ErrnoException).code !== "EEXIST")+ throw error;+ checkoutOccupied = true;+ throw new Error(+ `Bulk checkout is already occupied: ${destination}. Retry with --recover to use a fresh checkout, or inspect and move the retained checkout before an ordinary retry.`,+ );+ }}protectedRoot = checkout;+ // Claim the fresh checkout before Git or the scanner can fail.+ const marker = await checkoutMarker(checkout, scanDir);+ await mkdir(join(checkout, ".git"), { mode: 0o700 });+ await writeFile(join(checkout, CHECKOUT_MARKER), marker, {+ flag: "wx",+ mode: 0o600,+ });await checkoutRevision(task,checkout,@@ -655,7 +676,7 @@ async function runCampaign(: errorMessage(error);} finally {if (options.recoverScan === undefined && checkout !== undefined) {- cleanupFailure = await removeCheckout(checkout);+ cleanupFailure = await removeCheckout(checkout, scanDir);}}const reportedErrors = [failure, cleanupFailure].filter(@@ -692,6 +713,7 @@ async function runCampaign(? { cleanupPending: "checkout" }: {}),...(knowledgeBaseFailure ? { knowledgeBaseFailure: true } : {}),+ ...(checkoutOccupied ? { checkoutOccupied: true } : {}),...(runWarnings.length === 0 ? {} : { warnings: runWarnings }),policyFailed: attemptPolicyFailed,};@@ -702,7 +724,7 @@ async function runCampaign(await receiptTail;options.signal?.throwIfAborted();if (recoveryCheckout !== undefined) {- reportedError = await removeCheckout(recoveryCheckout);+ reportedError = await removeCheckout(recoveryCheckout, scanDir);}notifyProgress(options, {repository: task.id,@@ -718,7 +740,12 @@ async function runCampaign(completed += 1;break;}- if (exhaustedBudget || attemptedResume || requiresRecovery) {+ if (+ exhaustedBudget ||+ attemptedResume ||+ requiresRecovery ||+ checkoutOccupied+ ) {failed += 1;break;}@@ -751,12 +778,48 @@ function notifyProgress(notify(() => options.onProgress?.(event));}-async function removeCheckout(path: string): Promise<string | undefined> {- return rm(path, { recursive: true, force: true }).then(- () => undefined,- (error: unknown) =>- `Multiscan checkout cleanup failed: ${errorMessage(error)}`,- );+// A completed attempt can be revisited after its checkout path was reused.+async function checkoutMarker(+ checkout: string,+ scanDir: string,+): Promise<string> {+ const directory = await lstat(checkout, { bigint: true });+ if (!directory.isDirectory())+ throw new Error("The campaign checkout is no longer a directory.");+ return JSON.stringify([+ scanDir,+ String(directory.dev),+ String(directory.ino),+ ]);+}++async function removeCheckout(+ checkout: string,+ scanDir: string,+ signal?: AbortSignal,+): Promise<string | undefined> {+ try {+ const marker = await lstat(join(checkout, CHECKOUT_MARKER)).catch(+ undefinedIfMissingFile,+ );+ if (!marker?.isFile()) return undefined;+ const expected = Buffer.from(await checkoutMarker(checkout, scanDir));+ if (+ marker.size === expected.length &&+ (+ await readScanFile(+ checkout,+ CHECKOUT_MARKER,+ "campaign checkout marker",+ signal,+ )+ ).equals(expected)+ ) {+ await rm(checkout, { recursive: true, force: true });+ }+ } catch (error) {+ return `Multiscan checkout cleanup failed: ${errorMessage(error)}`;+ }}async function ensureOutputDirectory(@@ -1139,8 +1202,11 @@ async function readReceipts(const previous = receipts.get(id);receipts.set(id, {maxAttempt: Math.max(previous?.maxAttempt ?? 0, receipt.attempt),- // Extraction failures never start or modify a scan; keep its last receipt.- scan: receipt.knowledgeBaseFailure ? previous?.scan : receipt,+ // Pre-scan failures reserve an attempt but leave the last scan recoverable.+ scan:+ receipt.knowledgeBaseFailure || receipt.checkoutOccupied+ ? previous?.scan+ : receipt,});}if (contents.at(-1) !== 10) {diff --git a/sdk/typescript/tests-ts/multiscan-checkout.test.ts b/sdk/typescript/tests-ts/multiscan-checkout.test.tsindex 89701e71a..244bf9787 100644--- a/sdk/typescript/tests-ts/multiscan-checkout.test.ts+++ b/sdk/typescript/tests-ts/multiscan-checkout.test.ts@@ -433,7 +433,36 @@ describe("multiscan", () => {expect(run).toHaveBeenCalledTimes(1);});- test("retries failed scans and continues later tasks when checkout cleanup fails", async () => {+ test("preserves an existing checkout directory on the first campaign attempt", async () => {+ const { paths } = await repositoryFixture("occupied-first", "demo");+ const checkout = join(paths.output, "checkouts", "demo");+ await mkdir(checkout, { recursive: true });+ const note = join(checkout, "local-notes.txt");+ await writeFile(note, "Local work.\n");+ const run = mock<SecurityClient["run"]>(completeRun);+ const configured = options(paths, client(run));++ expect(await runMultiscan(configured)).toMatchObject({+ completed: 0,+ failed: 1,+ });+ expect(run).not.toHaveBeenCalled();+ expect(await readFile(note, "utf8")).toBe("Local work.\n");+ const ledger = join(paths.output, "results.jsonl");+ expect(await results(ledger)).toMatchObject([+ { attempt: 1, status: "failed", checkoutOccupied: true },+ ]);++ const recovered = await runMultiscan({+ ...configured,+ recoverScan: async () => undefined,+ });+ expect(recovered).toMatchObject({ completed: 1, failed: 0 });+ expect(run).toHaveBeenCalledTimes(1);+ expect(await readFile(note, "utf8")).toBe("Local work.\n");+ });++ test("continues later tasks when a failed scan leaves its checkout occupied", async () => {const { paths, source } = await repositoryFixture("cleanup-retry", "retry");await appendFile(paths.input, `next,${source.path},${source.revision}\n`);const checkout = join(paths.output, "checkouts", "retry");@@ -462,8 +491,8 @@ describe("multiscan", () => {try {const summary = await runMultiscan(options(paths, client(run)));- expect(summary).toMatchObject({ total: 2, completed: 2, failed: 0 });- expect(run).toHaveBeenCalledTimes(3);+ expect(summary).toMatchObject({ total: 2, completed: 1, failed: 1 });+ expect(run).toHaveBeenCalledTimes(2);expect(await results(summary.resultsPath)).toMatchObject([{id: "retry",@@ -472,7 +501,12 @@ describe("multiscan", () => {error:"ORIGINAL_SCAN_FAILURE; Multiscan checkout cleanup failed: EACCES: synthetic cleanup failure",},- { id: "retry", status: "completed", attempt: 2 },+ {+ id: "retry",+ status: "failed",+ attempt: 2,+ error: expect.stringContaining("move the retained checkout"),+ },{ id: "next", status: "completed", attempt: 1 },]);} finally {@@ -615,8 +649,8 @@ describe("multiscan", () => {expect(await results(summary.resultsPath)).toHaveLength(3);});- test("rejects another supervisor and recovers a crashed owner's checkout", async () => {- const { paths } = await repositoryFixture("exclusive");+ test("rejects another supervisor and preserves a crashed owner's checkout", async () => {+ const { paths, source } = await repositoryFixture("exclusive");const running = Promise.withResolvers<void>();const finish = Promise.withResolvers<void>();const security = client(async (_repository, scanOptions = {}) => {@@ -656,11 +690,29 @@ describe("multiscan", () => {JSON.stringify({ pid: 999_999_999 }),);const checkout = join(paths.output, "checkouts", "exclusive");- await mkdir(checkout);+ git(paths.root, "clone", "--quiet", source.path, checkout);+ const marker = join(checkout, "retained.txt");+ await writeFile(marker, "Preserved crashed-owner checkout data.\n");+ const identity = await lstat(checkout);const recovered = await runMultiscan(options(paths, security));- expect(recovered).toMatchObject({ completed: 1, failed: 0, skipped: 0 });- expect(await readdir(join(paths.output, "checkouts"))).toEqual([]);+ expect(recovered).toMatchObject({ completed: 0, failed: 1, skipped: 0 });+ const receipts = await results(recovered.resultsPath);+ expect(receipts).toHaveLength(2);+ expect(receipts[0]).toEqual(receipt);+ expect(receipts[1]).toMatchObject({+ attempt: 2,+ status: "failed",+ error: expect.stringContaining("move the retained checkout"),+ });+ expect(await readdir(join(paths.output, "checkouts"))).toEqual([+ "exclusive",+ ]);+ const retained = await lstat(checkout);+ expect([retained.dev, retained.ino]).toEqual([identity.dev, identity.ino]);+ expect(await readFile(marker, "utf8")).toBe(+ "Preserved crashed-owner checkout data.\n",+ );await expect(access(lock)).rejects.toThrow();});diff --git a/sdk/typescript/tests-ts/multiscan-recovery.test.ts b/sdk/typescript/tests-ts/multiscan-recovery.test.tsindex 526bcea5d..cca5beedd 100644--- a/sdk/typescript/tests-ts/multiscan-recovery.test.ts+++ b/sdk/typescript/tests-ts/multiscan-recovery.test.ts@@ -7,6 +7,7 @@ import {mkdir,readFile,readdir,+ rename,rm,writeFile,} from "node:fs/promises";@@ -62,6 +63,100 @@ function receiptCost(onSerialized: () => void) {}describe("multiscan", () => {+ test.each(["ordinary", "recovery"] as const)(+ "preserves a replacement checkout when a completed %s attempt is skipped",+ async (kind) => {+ const { paths } = await repositoryFixture("replaced");+ const run = mock(completeRun);+ const configured = options(paths, client(run), { maxAttempts: 1 });+ let checkout: string;+ if (kind === "recovery") {+ checkout = join(+ paths.output,+ "recovery-checkouts",+ "replaced",+ "attempt-2",+ );+ await runMultiscan(+ options(paths, client(rejecting("Stopped")), { maxAttempts: 1 }),+ );+ expect(+ await runMultiscan({+ ...configured,+ recoverScan: async () => undefined,+ }),+ ).toMatchObject({ completed: 1 });+ await expect(lstat(checkout)).rejects.toMatchObject({ code: "ENOENT" });+ } else {+ checkout = join(paths.output, "checkouts", "replaced");+ const originalRm = filesystem.rm;+ const removeSpy = spyOn(filesystem, "rm").mockImplementation(+ async (...args) => {+ if (run.mock.calls.length > 0 && String(args[0]) === checkout)+ throw new Error("Checkout is busy");+ return originalRm(...args);+ },+ );+ try {+ expect(await runMultiscan(configured)).toMatchObject({+ completed: 1,+ });+ } finally {+ removeSpy.mockRestore();+ }+ await rename(checkout, join(paths.root, "old-checkout"));+ }+ await mkdir(checkout);+ git(checkout, "init", "-q");+ const note = join(checkout, "uncommitted.txt");+ await writeFile(note, "Independent checkout.\n");+ const ledger = join(paths.output, "results.jsonl");+ const before = await readFile(ledger);+ expect(await runMultiscan(configured)).toMatchObject({+ completed: 1,+ skipped: 1,+ });+ expect(run).toHaveBeenCalledTimes(1);+ expect(await readFile(ledger)).toEqual(before);+ expect(await readFile(note, "utf8")).toBe("Independent checkout.\n");+ expect((await lstat(join(checkout, ".git"))).isDirectory()).toBe(true);+ },+ );++ test.each(["ordinary", "recovery"] as const)(+ "preserves a checkout replaced while its %s scan is running",+ async (kind) => {+ const { paths } = await repositoryFixture("replaced");+ const recovery = kind === "recovery";+ if (recovery) {+ await runMultiscan(+ options(paths, client(rejecting("Stopped")), { maxAttempts: 1 }),+ );+ }+ const run = mock<SecurityClient["run"]>(async (checkout, scan = {}) => {+ const result = await completedScan(scan.outputDir!);+ await rename(checkout, join(paths.root, "old-checkout"));+ await mkdir(checkout);+ git(checkout, "init", "-q");+ await writeFile(join(checkout, "note.txt"), "New work.\n");+ return result;+ });+ const summary = await runMultiscan(+ options(paths, client(run), {+ maxAttempts: 1,+ recoverScan: recovery ? async () => undefined : undefined,+ }),+ );+ expect(summary).toMatchObject({ completed: 1, failed: 0 });+ expect(run).toHaveBeenCalledTimes(1);+ const checkout = run.mock.calls[0]![0];+ expect(await readFile(join(checkout, "note.txt"), "utf8")).toBe(+ "New work.\n",+ );+ expect((await lstat(join(checkout, ".git"))).isDirectory()).toBe(true);+ },+ );+test("serializes receipt writes through close and drains workers before unlocking", async () => {const { paths, source } = await repositoryFixture("receipt-order", "first");await appendFile(paths.input, `second,${source.path},${source.revision}\n`);