Skip to content

OpenAI's Codex Security CLI and TypeScript SDK for finding, validating, and fixing security vulnerabilities. npm: https://www.npmjs.com/package/@openai/codex-security

fix(deep-scan): recover failed reducer workers (#342)

Dane Schneiderac3b71f94e2c
diff --git a/sdk/typescript/_bundled_plugin/mcp/server.mjs.br.part-000 b/sdk/typescript/_bundled_plugin/mcp/server.mjs.br.part-000
index 4bbb9370d..0792e6c88 100644
Binary files a/sdk/typescript/_bundled_plugin/mcp/server.mjs.br.part-000 and b/sdk/typescript/_bundled_plugin/mcp/server.mjs.br.part-000 differ
diff --git a/sdk/typescript/_bundled_plugin/mcp/server.mjs.br.part-001 b/sdk/typescript/_bundled_plugin/mcp/server.mjs.br.part-001
index 5a7967da2..040a561c6 100644
Binary files a/sdk/typescript/_bundled_plugin/mcp/server.mjs.br.part-001 and b/sdk/typescript/_bundled_plugin/mcp/server.mjs.br.part-001 differ
diff --git a/sdk/typescript/_bundled_plugin/scripts/deep_scan_workbench.py b/sdk/typescript/_bundled_plugin/scripts/deep_scan_workbench.py
index f601bb497..641eb4dff 100644
--- a/sdk/typescript/_bundled_plugin/scripts/deep_scan_workbench.py
+++ b/sdk/typescript/_bundled_plugin/scripts/deep_scan_workbench.py
@@ -1274,6 +1274,28 @@ def upsert_deep_scan_worker(
""",
(timestamp, scan_id),
)
+ elif args.kind == "dedup" and args.status == "failed" and existing["status"] == "running":
+ connection.execute(
+ """
+ UPDATE deep_scan_workers
+ SET merge_state = 'buffered', updated_at = ?
+ WHERE scan_id = ? AND kind = 'discovery' AND status = 'succeeded'
+ AND merge_state = 'merging'
+ AND id IN (
+ SELECT discovery_worker_id FROM deep_scan_dedup_inputs
+ WHERE scan_id = ? AND dedup_worker_id = ?
+ )
+ """,
+ (timestamp, scan_id, scan_id, worker_id),
+ )
+ connection.execute(
+ """
+ UPDATE deep_scan_runs
+ SET phase = 'discovery', updated_at = ?
+ WHERE scan_id = ?
+ """,
+ (timestamp, scan_id),
+ )
completed_at = (
timestamp
if args.status in {"succeeded", "failed", "canceled"}
diff --git a/sdk/typescript/tests-ts/deep-scan-reducer-recovery.test.ts b/sdk/typescript/tests-ts/deep-scan-reducer-recovery.test.ts
new file mode 100644
index 000000000..1577bf94c
--- /dev/null
+++ b/sdk/typescript/tests-ts/deep-scan-reducer-recovery.test.ts
@@ -0,0 +1,231 @@
+import { spawnSync } from "node:child_process";
+import { mkdirSync, mkdtempSync, rmSync } from "node:fs";
+import { readFile } from "node:fs/promises";
+import { tmpdir } from "node:os";
+import { join } from "node:path";
+import { brotliDecompressSync } from "node:zlib";
+import { expect, test } from "bun:test";
+import { PLUGIN_ROOT } from "./plugin-root.js";
+
+const bundledRuntime = Promise.all(
+ ["000", "001"].map((part) =>
+ readFile(join(PLUGIN_ROOT, "mcp", `server.mjs.br.part-${part}`)),
+ ),
+).then((parts) => brotliDecompressSync(Buffer.concat(parts)).toString("utf8"));
+
+function bundledFunction(runtime: string, name: string): string {
+ const source = new RegExp(
+ `function ${name}\\([^\\n]*\\) \\{[\\s\\S]*?\\n\\}`,
+ "u",
+ ).exec(runtime)?.[0];
+ if (!source) throw new Error(`Missing bundled runtime function: ${name}.`);
+ return source;
+}
+
+test("keeps every advertised Deep worker tool within Codex's name limit", async () => {
+ const runtime = await bundledRuntime;
+ const method = / compactArtifactServer\(request\) \{[\s\S]*?\n \}/u.exec(
+ runtime,
+ )?.[0];
+ expect(method).toBeDefined();
+ const pathImport = /\(0, (import_node_path\d+)\.join\)/u.exec(method!)?.[1];
+ expect(pathImport).toBeDefined();
+ const compactArtifactServer = new Function(
+ pathImport!,
+ `return ({${method}}).compactArtifactServer;`,
+ )({ join }) as (
+ this: { modelSettings: { artifactContext: Record<string, string> } },
+ request: Record<string, unknown>,
+ ) => Record<string, { args: string[]; env: NodeJS.ProcessEnv }>;
+
+ const node = Bun.which("node");
+ expect(node).not.toBeNull();
+ const root = mkdtempSync(join(tmpdir(), "codex-security-deep-tools-"));
+ const repoRoot = join(root, "repository");
+ const scanRoot = join(root, "scan");
+ mkdirSync(repoRoot);
+ mkdirSync(scanRoot);
+
+ try {
+ for (const layout of ["worker", "reducer"] as const) {
+ const artifactRoot = join(scanRoot, layout);
+ mkdirSync(artifactRoot);
+ const servers = compactArtifactServer.call(
+ {
+ modelSettings: {
+ artifactContext: {
+ pluginRoot: PLUGIN_ROOT,
+ scanRoot,
+ repoRoot,
+ scanId: "test-scan",
+ },
+ },
+ },
+ {
+ kind: layout === "reducer" ? "dedup" : "discovery",
+ artifactContext: {
+ root: artifactRoot,
+ layout,
+ ...(layout === "reducer"
+ ? { deepReducer: { scanRoot, claimedWorkers: [] } }
+ : {}),
+ },
+ },
+ );
+ expect(Object.keys(servers)).toEqual(["cs_artifacts"]);
+ const server = servers["cs_artifacts"]!;
+ const result = spawnSync(node!, server.args, {
+ encoding: "utf8",
+ env: { ...process.env, ...server.env },
+ input: [
+ '{"jsonrpc":"2.0","id":1,"method":"initialize","params":{"protocolVersion":"2025-11-25","capabilities":{},"clientInfo":{"name":"codex-security-test","version":"1.0.0"}}}',
+ '{"jsonrpc":"2.0","method":"notifications/initialized","params":{}}',
+ '{"jsonrpc":"2.0","id":2,"method":"tools/list","params":{}}',
+ "",
+ ].join("\n"),
+ timeout: 30_000,
+ });
+ expect(result.status, result.stderr).toBe(0);
+ const response = result.stdout
+ .trim()
+ .split("\n")
+ .map((line) => JSON.parse(line) as { id?: number; result?: unknown })
+ .find((message) => message.id === 2)?.result as
+ | { tools: Array<{ name: string }> }
+ | undefined;
+ expect(response).toBeDefined();
+ expect(response!.tools.length).toBeGreaterThan(0);
+ for (const tool of response!.tools) {
+ expect(`mcp__cs_artifacts__${tool.name}`.length).toBeLessThanOrEqual(
+ 64,
+ );
+ }
+ if (layout === "reducer") {
+ expect(response!.tools.map((tool) => tool.name)).toContain(
+ "record_codex_security_deep_reduction",
+ );
+ }
+ }
+ } finally {
+ rmSync(root, { recursive: true, force: true });
+ }
+});
+
+test("classifies owned worker tool failures without exposing their contents", async () => {
+ const runtime = await bundledRuntime;
+ const diagnosticSource = bundledFunction(runtime, "appendSafeItemDiagnostic");
+ const recordHelper = /\b(isRecord\d*)\(item\)/u.exec(diagnosticSource)?.[1];
+ expect(recordHelper).toBeDefined();
+ const appendDiagnostic = new Function(
+ [
+ bundledFunction(runtime, recordHelper!),
+ bundledFunction(runtime, "isSandboxNamespaceExhaustion"),
+ bundledFunction(runtime, "appendUniqueDiagnostic"),
+ diagnosticSource,
+ "return appendSafeItemDiagnostic;",
+ ].join("\n"),
+ )() as (
+ diagnostics: Array<{ code: string; message: string }>,
+ event: Record<string, unknown>,
+ ) => void;
+
+ const secret = "synthetic-secret-never-log";
+ for (const fixture of [
+ {
+ server: "cs_artifacts",
+ tool: "record_codex_security_deep_reduction",
+ result: { isError: true, content: [{ text: secret }] },
+ error: null,
+ reason: "returned an error",
+ },
+ {
+ server: "cs_artifacts",
+ tool: "additional_codex_security_worker_tool",
+ result: null,
+ error: { message: secret },
+ reason: "transport failed",
+ },
+ {
+ server: "codex_security_artifacts",
+ tool: "record_codex_security_discovery_candidates",
+ result: null,
+ error: null,
+ reason: "failed",
+ },
+ ]) {
+ const diagnostics: Array<{ code: string; message: string }> = [];
+ appendDiagnostic(diagnostics, {
+ type: "mcp_tool_call",
+ status: "failed",
+ arguments: { token: secret },
+ ...fixture,
+ });
+ expect(diagnostics).toEqual([
+ {
+ code: "artifact_tool_failed",
+ message: `Codex worker artifact tool ${fixture.tool} ${fixture.reason}.`,
+ },
+ ]);
+ expect(JSON.stringify(diagnostics)).not.toContain(secret);
+ }
+
+ const unrelatedDiagnostics: Array<{ code: string; message: string }> = [];
+ appendDiagnostic(unrelatedDiagnostics, {
+ type: "mcp_tool_call",
+ status: "failed",
+ server: "unrelated_server",
+ tool: "record_codex_security_deep_reduction",
+ result: { isError: true },
+ error: null,
+ });
+ expect(unrelatedDiagnostics).toEqual([]);
+});
+
+test("resumes only when the exact reducer result is missing", async () => {
+ const runtime = await bundledRuntime;
+ const source = bundledFunction(runtime, "isMissingReducerResult");
+ const pathImport = /\(0, (import_node_path\d+)\.join\)/u.exec(source)?.[1];
+ expect(pathImport).toBeDefined();
+ const isMissingReducerResult = new Function(
+ pathImport!,
+ `${source}\nreturn isMissingReducerResult;`,
+ )({ join }) as (error: Error, artifactDirectory: string) => boolean;
+ const artifactDirectory = join(tmpdir(), "codex-security-reducer-artifacts");
+ const missingResult = Object.assign(new Error("result missing"), {
+ code: "ENOENT",
+ path: join(artifactDirectory, "result.json"),
+ });
+ const diagnosedMissingResult = Object.assign(
+ new Error("artifact tool failed", { cause: missingResult }),
+ { code: "artifact_tool_failed" },
+ );
+
+ expect(isMissingReducerResult(missingResult, artifactDirectory)).toBe(true);
+ expect(
+ isMissingReducerResult(diagnosedMissingResult, artifactDirectory),
+ ).toBe(true);
+ expect(
+ isMissingReducerResult(
+ Object.assign(new Error("different artifact missing"), {
+ code: "ENOENT",
+ path: join(artifactDirectory, "candidates.jsonl"),
+ }),
+ artifactDirectory,
+ ),
+ ).toBe(false);
+ expect(
+ isMissingReducerResult(
+ Object.assign(new Error("result cannot be read"), {
+ code: "EACCES",
+ path: join(artifactDirectory, "result.json"),
+ }),
+ artifactDirectory,
+ ),
+ ).toBe(false);
+
+ const continuation = new Function(
+ `${bundledFunction(runtime, "reducerCompletionContinuation")}\nreturn reducerCompletionContinuation;`,
+ )() as (attempt: number) => string;
+ expect(continuation(1)).toContain("record_codex_security_deep_reduction");
+ expect(continuation(1)).toMatch(/retry.*until it succeeds/iu);
+});
diff --git a/sdk/typescript/tests-ts/deep-scan-workbench.test.ts b/sdk/typescript/tests-ts/deep-scan-workbench.test.ts
index bf94c471b..56c399c25 100644
--- a/sdk/typescript/tests-ts/deep-scan-workbench.test.ts
+++ b/sdk/typescript/tests-ts/deep-scan-workbench.test.ts
@@ -1,4 +1,11 @@
-import { mkdir, mkdtemp, realpath, rm, writeFile } from "node:fs/promises";
+import {
+ mkdir,
+ mkdtemp,
+ readFile,
+ realpath,
+ rm,
+ writeFile,
+} from "node:fs/promises";
import { tmpdir } from "node:os";
import { join } from "node:path";
import { afterEach, describe, expect, test } from "bun:test";
@@ -345,4 +352,274 @@ describe("deep scan workbench ownership", () => {
expect(staleProgress["status"]).not.toBe(0);
expect(staleProgress["stderr"]).toContain("newer generation");
});
+
+ test("requeues a failed reducer's inputs and completes without losing findings", async () => {
+ const root = await realpath(
+ await mkdtemp(join(tmpdir(), "codex-security-deep-reducer-recovery-")),
+ );
+ temporaryDirectories.push(root);
+ const repository = join(root, "repository");
+ const stateDir = join(root, "state");
+ const codexHome = join(root, "codex-home");
+ const configDir = join(codexHome, "codex-security");
+ await mkdir(repository);
+ await mkdir(configDir, { recursive: true });
+ await writeFile(join(repository, "source.py"), "# source fixture\n");
+ await writeFile(
+ join(configDir, "config.toml"),
+ "[deep_scan]\nworkers = 3\nmax_discovery_runs = 5\n",
+ );
+
+ const python = Bun.which("python3") ?? Bun.which("python");
+ expect(python).not.toBeNull();
+ const command = (args: string[]): Record<string, unknown> => {
+ const result = Bun.spawnSync(
+ [
+ python!,
+ "-I",
+ "-B",
+ join(PLUGIN_ROOT, "scripts", "workbench_db.py"),
+ ...args,
+ ],
+ {
+ env: {
+ ...process.env,
+ CODEX_SECURITY_STATE_DIR: stateDir,
+ CODEX_HOME: codexHome,
+ },
+ stdout: "pipe",
+ stderr: "pipe",
+ },
+ );
+ expect(result.exitCode, new TextDecoder().decode(result.stderr)).toBe(0);
+ return JSON.parse(new TextDecoder().decode(result.stdout)) as Record<
+ string,
+ unknown
+ >;
+ };
+ const state = (result: Record<string, unknown>) =>
+ result["deepScan"] as Record<string, unknown>;
+ const initial = state(
+ command([
+ "begin-deep-scan",
+ "--thread-id",
+ "thread-deep-scan",
+ "--target-path",
+ repository,
+ "--scope",
+ ".",
+ "--scan-root",
+ join(root, "scans"),
+ "--available-parallelism",
+ "4",
+ ]),
+ );
+ const scanId = initial["scanId"] as string;
+ const scanDir = initial["scanDir"] as string;
+ const discoveryDir = join(scanDir, "artifacts", "02_discovery");
+ const ledgerPath = join(discoveryDir, "candidate_ledger.jsonl");
+ const existingFinding = '{"candidate":"previously committed"}\n';
+ await mkdir(discoveryDir, { recursive: true });
+ await writeFile(join(discoveryDir, "in_scope_files.txt"), "source.py\n");
+ await writeFile(ledgerPath, existingFinding);
+
+ const workerPaths = async (workerId: string) => {
+ const artifactDir = join(
+ scanDir,
+ "artifacts",
+ "deep_discovery",
+ workerId,
+ );
+ const promptPath = join(artifactDir, "prompt.md");
+ const resultPath = join(artifactDir, "result.json");
+ await mkdir(artifactDir, { recursive: true });
+ await writeFile(promptPath, "Review the source.\n");
+ return { artifactDir, promptPath, resultPath };
+ };
+ const discoveryIds = [
+ "10000000-0000-4000-8000-000000000001",
+ "10000000-0000-4000-8000-000000000002",
+ "10000000-0000-4000-8000-000000000003",
+ "10000000-0000-4000-8000-000000000004",
+ "10000000-0000-4000-8000-000000000005",
+ ];
+ for (const workerId of discoveryIds) {
+ const paths = await workerPaths(workerId);
+ const args = [
+ "upsert-deep-scan-worker",
+ "--scan-id",
+ scanId,
+ "--worker-id",
+ workerId,
+ "--kind",
+ "discovery",
+ "--prompt-path",
+ paths.promptPath,
+ "--artifact-dir",
+ paths.artifactDir,
+ "--attempt",
+ "1",
+ ];
+ command([...args, "--status", "running"]);
+ await writeFile(paths.resultPath, "{}\n");
+ command([
+ ...args,
+ "--status",
+ "succeeded",
+ "--result-manifest-path",
+ paths.resultPath,
+ ]);
+ }
+
+ const startReducer = async (workerId: string, inputIds: string[]) => {
+ const paths = await workerPaths(workerId);
+ command([
+ "claim-deep-scan-dedup",
+ "--scan-id",
+ scanId,
+ "--worker-id",
+ workerId,
+ "--prompt-path",
+ paths.promptPath,
+ "--artifact-dir",
+ paths.artifactDir,
+ ...inputIds.flatMap((inputId) => ["--input-worker-id", inputId]),
+ ]);
+ const args = [
+ "upsert-deep-scan-worker",
+ "--scan-id",
+ scanId,
+ "--worker-id",
+ workerId,
+ "--kind",
+ "dedup",
+ "--prompt-path",
+ paths.promptPath,
+ "--artifact-dir",
+ paths.artifactDir,
+ "--attempt",
+ "1",
+ ];
+ command([...args, "--status", "running"]);
+ return { args, ...paths };
+ };
+ const commitReducer = async (
+ workerId: string,
+ resultPath: string,
+ newFindingsCount: number,
+ ) => {
+ await writeFile(resultPath, "{}\n");
+ return state(
+ command([
+ "commit-deep-scan-dedup",
+ "--scan-id",
+ scanId,
+ "--worker-id",
+ workerId,
+ "--result-manifest-path",
+ resultPath,
+ "--new-findings-count",
+ String(newFindingsCount),
+ ]),
+ );
+ };
+ const previousReducerId = "20000000-0000-4000-8000-000000000001";
+ const previous = await startReducer(
+ previousReducerId,
+ discoveryIds.slice(0, 2),
+ );
+ await commitReducer(previousReducerId, previous.resultPath, 1);
+
+ const failedReducerId = "20000000-0000-4000-8000-000000000002";
+ const claimedInputs = discoveryIds.slice(2, 4);
+ const failedReducer = await startReducer(failedReducerId, claimedInputs);
+ const failureArgs = [
+ ...failedReducer.args,
+ "--status",
+ "failed",
+ "--error-message",
+ "reducer exhausted its attempts",
+ ];
+ const failed = state(command(failureArgs));
+ const workersById = (result: Record<string, unknown>) =>
+ new Map(
+ (result["workers"] as Record<string, unknown>[]).map((worker) => [
+ worker["id"] as string,
+ worker,
+ ]),
+ );
+ const failedWorkers = workersById(failed);
+ expect(failed).toMatchObject({
+ status: "running",
+ phase: "discovery",
+ dispatchedCount: 5,
+ consecutiveErrors: 0,
+ });
+ for (const workerId of discoveryIds.slice(0, 2)) {
+ expect(failedWorkers.get(workerId)).toMatchObject({
+ status: "succeeded",
+ mergeState: "merged",
+ });
+ }
+ for (const workerId of discoveryIds.slice(2)) {
+ expect(failedWorkers.get(workerId)).toMatchObject({
+ status: "succeeded",
+ mergeState: "buffered",
+ });
+ }
+ expect(failedWorkers.get(failedReducerId)).toMatchObject({
+ status: "failed",
+ error: "reducer exhausted its attempts",
+ });
+ expect(await readFile(ledgerPath, "utf8")).toBe(existingFinding);
+
+ const replacementReducerId = "20000000-0000-4000-8000-000000000003";
+ const replacementInputs = discoveryIds.slice(2);
+ const replacement = await startReducer(
+ replacementReducerId,
+ replacementInputs,
+ );
+ const replayed = state(command(failureArgs));
+ expect(replayed["phase"]).toBe("reducing");
+ for (const workerId of replacementInputs) {
+ expect(workersById(replayed).get(workerId)?.["mergeState"]).toBe(
+ "merging",
+ );
+ }
+
+ const committed = await commitReducer(
+ replacementReducerId,
+ replacement.resultPath,
+ 0,
+ );
+ for (const workerId of discoveryIds) {
+ expect(workersById(committed).get(workerId)?.["mergeState"]).toBe(
+ "merged",
+ );
+ }
+ expect(workersById(committed).get(failedReducerId)?.["status"]).toBe(
+ "failed",
+ );
+ expect(await readFile(ledgerPath, "utf8")).toBe(existingFinding);
+
+ const manifestPath = join(scanDir, "coordinator-manifest.json");
+ await writeFile(manifestPath, "{}\n");
+ const completed = state(
+ command([
+ "finish-deep-scan",
+ "--scan-id",
+ scanId,
+ "--terminal-reason",
+ "capped",
+ "--manifest-path",
+ manifestPath,
+ ]),
+ );
+ expect(completed).toMatchObject({
+ status: "succeeded",
+ terminalReason: "capped",
+ consecutiveErrors: 0,
+ });
+ expect(await readFile(ledgerPath, "utf8")).toBe(existingFinding);
+ });
});