OpenAI's Codex Security CLI and TypeScript SDK for finding, validating, and fixing security vulnerabilities. npm: https://www.npmjs.com/package/@openai/codex-security
TypeScript ai-securityapplication-securityclicode-scanningcodexcodex-securitycybersecuritydevsecopsnodejsnpmopenaisdksecuritytypescriptvulnerability-scanning
fix(deep-scan): page reducer inputs by serialized size (#1025)
Dane Schneider203d299e49d5
diff --git a/plugins/codex-security/mcp-app/evals/README.md b/plugins/codex-security/mcp-app/evals/README.mdnew file mode 100644index 000000000..b8ac619b4--- /dev/null+++ b/plugins/codex-security/mcp-app/evals/README.md@@ -0,0 +1,25 @@+# Deep reducer paging eval++This eval runs one reducer against synthetic persisted worker findings and a previous aggregate. It uses the pinned Codex CLI, its external code-mode host, and the production artifact tool handlers. It does not launch a repository scan.++The fixture has a 32 MiB individual semantic field containing quotes, backslashes, newlines, and Unicode. Its old duplicated MCP response is about 114 MiB. The first input read deliberately returns that old response; all subsequent calls use the production paged implementation. The IPC error comes from the real transport, not a mocked error message.++The grader requires a smaller byte budget on the retry with the same cursor and reference, byte-bounded subsequent pages, one successful result submission, every source accounted for exactly once, complete original payloads matching their SHA-256 hashes, and preservation of the previous finding identity and synthesized history.++Install the MCP app's locked dependencies using the repository's normal setup, then run from the repository root:++```sh+node plugins/codex-security/mcp-app/evals/deep-reducer-paging.mjs deterministic+```++The deterministic mode supplies scripted model responses through a local HTTP provider. It uses a private temporary Codex home and no model credentials or external model requests. The script reads every input page, fetches original and prior canonical details by reference, and records the reduction through the real tools. This mode also runs in the normal MCP test suite as `test_deep_reducer_paging_eval.mjs`.++To evaluate a model's recovery decisions using the production reducer prompt:++```sh+node plugins/codex-security/mcp-app/evals/deep-reducer-paging.mjs model+```++The model run uses the caller's normal Codex credentials and configuration and consumes model usage. An optional final argument selects a model; otherwise Codex uses its configured model. The fault injection and result grader are the same. This mode is opt-in and does not run in CI. Unlike the deterministic run, it does not supply the recovery code to the model. The SDK does not expose every code-mode error event, so the model-mode grade relies on the observed oversized response, changed request, and saved result; the deterministic mode additionally asserts the exact transport diagnostic.++Each command prints the path to an ignored directory under `mcp-app/reports/` containing `report.json`, the compact tool-call trace, and the generated artifacts. The fixture is generated at runtime; no large payload files are checked in.diff --git a/plugins/codex-security/mcp-app/evals/controlled-code-mode.mjs b/plugins/codex-security/mcp-app/evals/controlled-code-mode.mjsnew file mode 100644index 000000000..6ab985280--- /dev/null+++ b/plugins/codex-security/mcp-app/evals/controlled-code-mode.mjs@@ -0,0 +1,194 @@+import { spawn } from "node:child_process";+import { mkdir, mkdtemp, rm } from "node:fs/promises";+import { createServer } from "node:http";+import { createRequire } from "node:module";+import { tmpdir } from "node:os";+import path from "node:path";++function responseEvents(id, item) {+ return [+ { type: "response.created", response: { id } },+ { type: "response.output_item.done", item },+ {+ type: "response.completed",+ response: {+ id,+ usage: {+ input_tokens: 0,+ input_tokens_details: null,+ output_tokens: 0,+ output_tokens_details: null,+ total_tokens: 0,+ },+ },+ },+ ]+ .map((event) => `event: ${event.type}\ndata: ${JSON.stringify(event)}\n\n`)+ .join("");+}++/** Execute one scripted cell through the real CLI, external code host, and MCP. */+export async function runControlledCodeMode({+ code,+ mcpServers,+ workingDirectory,+ prompt = "Run the deterministic transport eval.",+}) {+ // Resolve through the SDK so the launcher and native host use its pinned version.+ const sdkRequire = createRequire(import.meta.resolve("@openai/codex-sdk"));+ const launcher = path.join(+ path.dirname(sdkRequire.resolve("@openai/codex/package.json")),+ "bin",+ "codex.js",+ );+ const fixtureRoot = await mkdtemp(+ path.join(tmpdir(), "codex-controlled-ipc-"),+ );+ const codexHome = path.join(fixtureRoot, "home");+ const toolOutputs = new Map();+ const serverErrors = [];+ let responseCount = 0;+ let pendingCallId = "controlled-exec";+ let child;+ let closed;+ let stderr = "";+ const server = createServer(async (request, response) => {+ try {+ // The CLI also sends analytics here; those must not advance the model script.+ if (request.url !== "/v1/responses") {+ request.resume();+ response.writeHead(200, { "content-type": "application/json" });+ response.end("{}");+ return;+ }+ const chunks = [];+ for await (const chunk of request) chunks.push(chunk);+ const body = JSON.parse(Buffer.concat(chunks).toString("utf8"));+ for (const item of body.input ?? []) {+ if (+ item.type === "custom_tool_call_output" ||+ item.type === "function_call_output"+ ) {+ toolOutputs.set(item.call_id, item);+ }+ }+ responseCount += 1;+ const id = `controlled-response-${responseCount}`;+ const output = toolOutputs.get(pendingCallId)?.output;+ const status = typeof output === "string" ? output : output?.[0]?.text;+ const runningCell = status?.match(/^Script running with cell ID (\d+)\n/);+ if (runningCell) pendingCallId = `controlled-wait-${responseCount}`;+ const item =+ responseCount === 1+ ? {+ type: "custom_tool_call",+ call_id: pendingCallId,+ name: "exec",+ input: code,+ }+ : runningCell+ ? {+ type: "function_call",+ call_id: pendingCallId,+ name: "wait",+ arguments: JSON.stringify({ cell_id: runningCell[1] }),+ }+ : {+ type: "message",+ id: `controlled-message-${responseCount}`,+ role: "assistant",+ content: [{ type: "output_text", text: "Eval complete." }],+ };+ response.writeHead(200, { "content-type": "text/event-stream" });+ response.end(responseEvents(id, item));+ } catch (error) {+ serverErrors.push(String(error));+ response.writeHead(500);+ response.end("Controlled provider failed.");+ }+ });+ try {+ await mkdir(codexHome, { mode: 0o700 });+ await new Promise((resolve, reject) => {+ server.once("error", reject);+ server.listen(0, "127.0.0.1", resolve);+ });+ const port = server.address().port;+ const config = [+ 'model_provider="controlled"',+ `model_providers.controlled={name="controlled",base_url="http://127.0.0.1:${port}/v1",wire_api="responses",requires_openai_auth=false,supports_websockets=false}`,+ "features.code_mode.enabled=true",+ "features.code_mode_host.enabled=true",+ "features.code_mode_host.disable_in_process_fallback=true",+ ];+ for (const [name, entry] of Object.entries(mcpServers)) {+ const prefix = `mcp_servers.${name}`;+ config.push(`${prefix}.command=${JSON.stringify(entry.command)}`);+ config.push(`${prefix}.args=${JSON.stringify(entry.args ?? [])}`);+ if (entry.env) {+ config.push(+ `${prefix}.env={${Object.entries(entry.env)+ .map(+ ([key, value]) =>+ `${JSON.stringify(key)}=${JSON.stringify(value)}`,+ )+ .join(",")}}`,+ );+ }+ }+ // Pass only process-launch essentials; never inherit the caller's model auth.+ const env = { CODEX_HOME: codexHome, CODEX_SQLITE_HOME: codexHome };+ for (const key of [+ "PATH",+ "SystemRoot",+ "SystemDrive",+ "ComSpec",+ "PATHEXT",+ "TEMP",+ "TMP",+ "TMPDIR",+ ]) {+ if (process.env[key] !== undefined) env[key] = process.env[key];+ }+ const args = [+ launcher,+ "exec",+ "--skip-git-repo-check",+ "--experimental-json",+ "--sandbox",+ "read-only",+ "-m",+ "gpt-5.1-codex",+ ...config.flatMap((entry) => ["-c", entry]),+ prompt,+ ];+ child = spawn(process.execPath, args, {+ cwd: workingDirectory ?? fixtureRoot,+ env,+ // JSONL stdout can duplicate the deliberately oversized MCP response.+ stdio: ["ignore", "ignore", "pipe"],+ });+ closed = new Promise((resolve) => child.once("close", resolve));+ child.stderr.setEncoding("utf8").on("data", (chunk) => {+ stderr += chunk;+ });+ const outcome = await new Promise((resolve, reject) => {+ child.once("error", reject);+ child.once("close", (code, signal) => resolve({ code, signal }));+ });+ return {+ outcome,+ toolOutputs: [...toolOutputs.values()],+ stderr,+ serverErrors,+ };+ } finally {+ if (child && child.exitCode === null && child.signalCode === null) {+ child.kill();+ }+ if (closed) await closed;+ server.closeAllConnections();+ await new Promise((resolve) => server.close(resolve));+ await rm(fixtureRoot, { recursive: true, force: true });+ }+}diff --git a/plugins/codex-security/mcp-app/evals/deep-reducer-paging-fixture.mjs b/plugins/codex-security/mcp-app/evals/deep-reducer-paging-fixture.mjsnew file mode 100644index 000000000..293f18349--- /dev/null+++ b/plugins/codex-security/mcp-app/evals/deep-reducer-paging-fixture.mjs@@ -0,0 +1,320 @@+import assert from "node:assert/strict";+import { createHash } from "node:crypto";+import { mkdir, readFile, realpath, writeFile } from "node:fs/promises";+import path from "node:path";++// This is the existing code-mode transport ceiling, used only to size the eval.+const IPC_FRAME_LIMIT_BYTES = 64 * 1024 * 1024;+const SCAN_ID = "7fc17317-9594-49e0-b06a-d72fd7e14bba";++/** Generate one reducer assignment; no repository scan or external service runs. */+export async function createReducerPagingFixture(root) {+ await mkdir(root, { recursive: true });+ const fixtureRoot = await realpath(root);+ const scanRoot = path.join(fixtureRoot, "scan");+ const deepRoot = path.join(scanRoot, "artifacts", "deep_discovery");+ const workerRoot = path.join(deepRoot, "workers", "discovery-0001", "output");+ const previousRoot = path.join(deepRoot, "dedup", "dedup-0001", "output");+ const outputRoot = path.join(deepRoot, "dedup", "dedup-0002", "output");+ await Promise.all(+ [workerRoot, previousRoot, outputRoot].map((directory) =>+ mkdir(directory, { recursive: true }),+ ),+ );++ const freshHtml = finding("reflected-html", {+ title: "Request text reaches an HTML response without encoding",+ summary: paddedSummary(IPC_FRAME_LIMIT_BYTES / 2),+ remediation: "HTML-encode request text at the response rendering boundary.",+ });+ const freshArchive = finding("archive-path", {+ title: "Archive entry paths escape the extraction directory",+ summary:+ "An independently reachable archive importer joins an untrusted entry path to the extraction directory without checking containment. This is unrelated to HTML rendering or database queries.",+ remediation:+ "Resolve each archive entry against the extraction directory and enforce containment before writing it.",+ });+ const previousOriginal = finding("query-injection", {+ title: "Account lookup interpolates request text into a database query",+ summary:+ "The account lookup builds a query by concatenating request-controlled text. Parameter binding is absent on this path.",+ remediation: "Use parameter binding for account lookup query values.",+ });+ const synthesizedHistory = finding("query-injection", {+ ...previousOriginal,+ summary:+ "Earlier synthesis also established that the transactional account lookup calls the same unsafe query builder. Both paths need the shared parameter-binding fix.",+ });+ const previousSourceId = "previous-worker:0";+ const previousCanonical = {+ ...previousOriginal,+ summary:+ "Account lookup and transactional lookup share a query builder that interpolates request text. A shared parameter-binding fix covers both reachable paths.",+ provenance: {+ ...previousOriginal.provenance,+ sourceFindingIds: [previousSourceId],+ sourceFindings: [{ id: previousSourceId, finding: previousOriginal }],+ previousFindings: [synthesizedHistory],+ },+ };+ const workerId = "current-worker";+ const currentFindings = [freshHtml, freshArchive];+ const expectedSources = new Map([+ ...currentFindings.map((value, index) => [`${workerId}:${index}`, value]),+ [previousSourceId, previousOriginal],+ ]);+ const sourceSha256ById = Object.fromEntries(+ [...expectedSources].map(([id, value]) => [id, canonicalSha256(value)]),+ );+ const result = {+ scanId: SCAN_ID,+ findings: currentFindings,+ scope: {+ summary: "Three independent synthetic request-processing issues.",+ },+ };+ const previous = {+ scanId: SCAN_ID,+ findings: [previousCanonical],+ threatModel: {+ summary:+ "Untrusted request inputs cross distinct output, filesystem, and query boundaries.",+ },+ };+ const workerPath = path.join(workerRoot, "result.json");+ const previousPath = path.join(previousRoot, "result.json");+ await writeFile(+ workerPath,+ JSON.stringify({+ ...result,+ coverage: {+ completeness: "complete",+ surfaces: [],+ explicitExclusions: [],+ deferred: [],+ },+ }),+ );+ await writeFile(previousPath, JSON.stringify(previous));++ const logicalInputs = {+ discoveries: [+ {+ workerId,+ result: {+ ...result,+ findings: currentFindings.map((value, index) => ({+ ...value,+ provenance: {+ ...value.provenance,+ sourceFindingIds: [`${workerId}:${index}`],+ },+ })),+ },+ },+ ],+ previous,+ };+ const { bytes, stringEscapes } = jsonMetrics(logicalInputs);+ // The old response included JSON.stringify(inputs) as text and inputs again+ // as structuredContent. Count its exact bytes without allocating both copies.+ const envelopeBytes =+ Buffer.byteLength(+ JSON.stringify({+ content: [{ type: "text", text: "" }],+ structuredContent: null,+ }),+ ) - Buffer.byteLength("null");+ const legacyResponseBytes = envelopeBytes + bytes * 2 + stringEscapes;+ assert.ok(legacyResponseBytes > IPC_FRAME_LIMIT_BYTES);+ return {+ context: {+ root: outputRoot,+ repoRoot: fixtureRoot,+ scanId: SCAN_ID,+ layout: "reducer",+ deepReducer: {+ scanRoot,+ claimedWorkers: [{ id: workerId, resultPath: workerPath }],+ previousReducerResultPath: previousPath,+ },+ },+ resultPath: path.join(outputRoot, "result.json"),+ expected: {+ sourceSha256ById,+ sourceIdentityById: Object.fromEntries(+ [...expectedSources].map(([id, value]) => [id, value.identity]),+ ),+ previousSourceId,+ previousIdentity: previousOriginal.identity,+ historySha256: canonicalSha256(synthesizedHistory),+ },+ measurements: {+ ipcFrameLimitBytes: IPC_FRAME_LIMIT_BYTES,+ largeSemanticFieldBytes: Buffer.byteLength(freshHtml.summary),+ logicalInputBytes: bytes,+ legacyResponseBytes,+ },+ };+}++/** Fail on lost, duplicated, substituted, or de-identified reducer evidence. */+export async function gradeReducerPagingResult(fixture) {+ const result = JSON.parse(await readFile(fixture.resultPath, "utf8"));+ assert.equal(result.scanId, fixture.context.scanId);+ assert.notEqual(result.complete, false);+ assert.ok(Array.isArray(result.findings));+ const expectedIds = Object.keys(fixture.expected.sourceSha256ById).sort();+ assert.equal(+ result.findings.length,+ expectedIds.length,+ "the three independent issues must remain separate output findings",+ );+ const refs = [];+ const retainedIds = [];+ for (const finding of result.findings) {+ const provenance = finding.provenance;+ assert.ok(Array.isArray(provenance?.sourceFindingIds));+ assert.equal(+ provenance.sourceFindingIds.length,+ 1,+ "each independent output finding must reference only its own source",+ );+ const [sourceId] = provenance.sourceFindingIds;+ assert.deepEqual(+ finding.identity,+ fixture.expected.sourceIdentityById[sourceId],+ `output identity must match its source ${sourceId}`,+ );+ assert.ok(Array.isArray(provenance?.sourceFindings));+ refs.push(...provenance.sourceFindingIds);+ assert.deepEqual(+ provenance.sourceFindings.map((source) => source.id).sort(),+ [...provenance.sourceFindingIds].sort(),+ "each output finding must retain exactly its attributed originals",+ );+ for (const source of provenance.sourceFindings) {+ assert.equal(+ canonicalSha256(source.finding),+ fixture.expected.sourceSha256ById[source.id],+ `original evidence changed for ${source.id}`,+ );+ retainedIds.push(source.id);+ }+ }+ assert.deepEqual(+ refs.sort(),+ expectedIds,+ "each assigned reference must occur exactly once",+ );+ assert.deepEqual(+ retainedIds.sort(),+ expectedIds,+ "each original must be retained exactly once",+ );+ const previous = result.findings.find((finding) =>+ finding.provenance.sourceFindingIds.includes(+ fixture.expected.previousSourceId,+ ),+ );+ assert.deepEqual(previous.identity, fixture.expected.previousIdentity);+ assert.ok(+ previous.provenance.previousFindings?.some(+ (history) => canonicalSha256(history) === fixture.expected.historySha256,+ ),+ "the prior synthesized history must survive losslessly",+ );+ return {+ findingCount: result.findings.length,+ accountedSourceCount: refs.length,+ preservedOriginalCount: retainedIds.length,+ previousIdentityPreserved: true,+ synthesizedHistoryPreserved: true,+ };+}++function finding(id, extra = {}) {+ return {+ ruleId: "synthetic." + id,+ identity: { anchor: id },+ title: "Synthetic finding " + id,+ summary: "An untrusted input reaches a sensitive operation.",+ severity: { level: "high" },+ confidence: { level: "high", rationale: "Synthetic validated fixture." },+ taxonomy: {+ category: id,+ cwe: [+ {+ "reflected-html": "CWE-79",+ "archive-path": "CWE-22",+ "query-injection": "CWE-89",+ }[id],+ ],+ },+ locations: [{ path: `src/${id}.ts`, startLine: 1, endLine: 2 }],+ remediation: "Apply the issue-specific boundary check.",+ provenance: { source: "local_plugin" },+ ...extra,+ };+}++function paddedSummary(bytes) {+ const beginning =+ "BEGIN ISSUE: Request text reaches an HTML response without encoding. The following repeated text is synthetic padding to exercise byte-aware paging, not additional evidence.\n";+ const ending =+ "\nEND ISSUE: The remedy is HTML encoding at the response boundary. Archive extraction and query interpolation are distinct issues requiring independent fixes.";+ const available =+ bytes - Buffer.byteLength(beginning) - Buffer.byteLength(ending);+ const unit = '"\\\n😀';+ const repeats = Math.floor(available / Buffer.byteLength(unit));+ return (+ beginning ++ unit.repeat(repeats) ++ "x".repeat(available % Buffer.byteLength(unit)) ++ ending+ );+}++function canonicalSha256(value) {+ const hash = createHash("sha256");+ visitJson(value, (chunk) => hash.update(chunk), true);+ return hash.digest("hex");+}++function jsonMetrics(value) {+ let bytes = 0;+ let stringEscapes = 0;+ visitJson(value, (chunk) => {+ bytes += Buffer.byteLength(chunk);+ for (let index = 0; index < chunk.length; index += 1) {+ if (chunk[index] === '"' || chunk[index] === "\\") stringEscapes += 1;+ }+ });+ return { bytes, stringEscapes };+}++// Hash and measure one scalar at a time instead of retaining another complete+// copy of the oversized document. Fixture values contain only JSON types.+function visitJson(value, emit, sorted = false) {+ if (Array.isArray(value)) {+ emit("[");+ value.forEach((item, index) => {+ if (index > 0) emit(",");+ visitJson(item, emit, sorted);+ });+ emit("]");+ } else if (value !== null && typeof value === "object") {+ emit("{");+ const keys = Object.keys(value);+ if (sorted) keys.sort();+ keys.forEach((key, index) => {+ if (index > 0) emit(",");+ emit(JSON.stringify(key));+ emit(":");+ visitJson(value[key], emit, sorted);+ });+ emit("}");+ } else {+ emit(JSON.stringify(value));+ }+}diff --git a/plugins/codex-security/mcp-app/evals/deep-reducer-paging-server.mjs b/plugins/codex-security/mcp-app/evals/deep-reducer-paging-server.mjsnew file mode 100644index 000000000..6703ab1a4--- /dev/null+++ b/plugins/codex-security/mcp-app/evals/deep-reducer-paging-server.mjs@@ -0,0 +1,75 @@+import { appendFile, readFile } from "node:fs/promises";+import { McpServer } from "@modelcontextprotocol/sdk/server/mcp.js";+import { StdioServerTransport } from "@modelcontextprotocol/sdk/server/stdio.js";+import { getCodexSecurityDeepReducerInputs } from "../src/artifact-deep-reducer.ts";+import { registerCompactWorkerArtifactTools } from "../src/server/compact-artifact-tools.ts";++async function main() {+ const { context, tracePath } = JSON.parse(+ await readFile(process.argv[2], "utf8"),+ );+ const server = new McpServer({+ name: "deep-reducer-paging-eval",+ version: "1",+ });+ let injected = false;+ let sequence = 0;++ // Exercise the production tool registration and handlers. Only the first read+ // returns the old duplicated result, forcing a real code-mode transport error.+ registerCompactWorkerArtifactTools(+ {+ registerTool(name, definition, handler) {+ return server.registerTool(name, definition, async (input, extra) => {+ const id = ++sequence;+ const read = name === "get_codex_security_deep_reducer_inputs";+ await trace({+ event: "request",+ id,+ tool: name,+ ...(read ? { input } : { findingCount: input.findings?.length }),+ });+ try {+ let result;+ const inject = read && !injected;+ if (inject) {+ injected = true;+ const full = await getCodexSecurityDeepReducerInputs(context);+ result = {+ content: [{ type: "text", text: JSON.stringify(full) }],+ structuredContent: full,+ };+ } else {+ result = await handler(input, extra);+ }+ await trace({+ event: "response",+ id,+ injected: inject,+ bytes: Buffer.byteLength(JSON.stringify(result)),+ ...(read && !inject+ ? { nextCursor: JSON.parse(result.content[0].text).nextCursor }+ : {}),+ });+ return result;+ } catch (error) {+ await trace({ event: "error", id, message: error.message });+ throw error;+ }+ });+ },+ },+ context,+ );++ await server.connect(new StdioServerTransport());++ async function trace(event) {+ await appendFile(tracePath, JSON.stringify(event) + "\n");+ }+}++main().catch((error) => {+ console.error(error);+ process.exitCode = 1;+});diff --git a/plugins/codex-security/mcp-app/evals/deep-reducer-paging.mjs b/plugins/codex-security/mcp-app/evals/deep-reducer-paging.mjsnew file mode 100644index 000000000..505ace7c1--- /dev/null+++ b/plugins/codex-security/mcp-app/evals/deep-reducer-paging.mjs@@ -0,0 +1,268 @@+import assert from "node:assert/strict";+import { mkdir, mkdtemp, readFile, writeFile } from "node:fs/promises";+import path from "node:path";+import { fileURLToPath, pathToFileURL } from "node:url";+import { Codex } from "@openai/codex-sdk";+import { build } from "esbuild";+import { runControlledCodeMode } from "./controlled-code-mode.mjs";+import {+ createReducerPagingFixture,+ gradeReducerPagingResult,+} from "./deep-reducer-paging-fixture.mjs";++const evalDirectory = path.dirname(fileURLToPath(import.meta.url));++/** Run one reducer through real code-mode IPC and the production artifact tools. */+export async function runReducerPagingEval({+ root,+ mode = "deterministic",+ model,+}) {+ assert.ok(mode === "deterministic" || mode === "model");+ const fixture = await createReducerPagingFixture(root);+ const tracePath = path.join(root, "tool-trace.jsonl");+ const serverConfigPath = path.join(root, "server-config.json");+ const serverPath = path.join(root, "artifact-server.cjs");+ const promptModulePath = path.join(root, "prompt.mjs");+ await writeFile(tracePath, "");+ await writeFile(+ serverConfigPath,+ JSON.stringify({ context: fixture.context, tracePath }),+ );+ await Promise.all([+ build({+ entryPoints: [path.join(evalDirectory, "deep-reducer-paging-server.mjs")],+ outfile: serverPath,+ bundle: true,+ platform: "node",+ format: "cjs",+ loader: { ".md": "text" },+ }),+ build({+ entryPoints: [path.join(evalDirectory, "../src/deep-scan/templates.ts")],+ outfile: promptModulePath,+ bundle: true,+ platform: "node",+ format: "esm",+ loader: { ".md": "text" },+ }),+ ]);+ const { renderDedupPrompt } = await import(+ pathToFileURL(promptModulePath).href+ );+ const prompt = renderDedupPrompt({+ reducerLabel: "paging-eval",+ discoveries: fixture.context.deepReducer.claimedWorkers,+ });+ const mcpServers = {+ cs_artifacts: {+ command: process.execPath,+ args: [serverPath, serverConfigPath],+ required: true,+ },+ };+ let transport;+ let usage;+ if (mode === "deterministic") {+ transport = await runControlledCodeMode({+ code: controlledReducerCode,+ mcpServers,+ workingDirectory: fixture.context.repoRoot,+ prompt,+ });+ assert.equal(transport.outcome.code, 0, transport.stderr);+ assert.deepEqual(transport.serverErrors, []);+ const output = JSON.stringify(transport.toolOutputs);+ assert.match(+ output,+ /code-mode delegate response exceeds the IPC frame limit: code-mode IPC frame length [0-9]+ exceeds 67108864 bytes/,+ "the real transport must reject the legacy payload",+ );+ assert.match(output, /reductionRecorded/);+ } else {+ const codex = new Codex({+ config: {+ mcp_servers: mcpServers,+ features: {+ code_mode: { enabled: true },+ code_mode_host: { enabled: true, disable_in_process_fallback: true },+ },+ },+ });+ const thread = codex.startThread({+ ...(model ? { model } : {}),+ workingDirectory: fixture.context.repoRoot,+ skipGitRepoCheck: true,+ approvalPolicy: "never",+ sandboxMode: "read-only",+ });+ const result = await thread.run(prompt);+ usage = result.usage;+ await writeFile(+ path.join(root, "model-response.txt"),+ result.finalResponse,+ );+ }+ const trace = (await readFile(tracePath, "utf8"))+ .trim()+ .split("\n")+ .map((line) => JSON.parse(line));+ const report = {+ mode,+ ...(model ? { model } : {}),+ ...fixture.measurements,+ ...gradeReducerPagingTrace(trace, fixture.measurements.ipcFrameLimitBytes),+ ...(transport ? { realIpcErrorObserved: true } : {}),+ ...(await gradeReducerPagingResult(fixture)),+ ...(usage ? { usage } : {}),+ };+ await writeFile(+ path.join(root, "report.json"),+ JSON.stringify(report, null, 2),+ );+ return report;+}++/** Grade the captured I/O independently of model wording or another model run. */+export function gradeReducerPagingTrace(trace, ipcFrameLimitBytes) {+ const reads = trace.filter(+ (event) =>+ event.event === "request" &&+ event.tool === "get_codex_security_deep_reducer_inputs",+ );+ const readsById = new Map(reads.map((event) => [event.id, event]));+ const injected = trace.filter((event) => event.injected);+ assert.equal(injected.length, 1);+ assert.ok(injected[0].bytes > ipcFrameLimitBytes);+ const failed = reads.find((event) => event.id === injected[0].id);+ const retried = reads.find((event) => event.id > failed.id);+ assert.ok(retried, "the reducer must recover from the oversized response");+ assert.ok(+ retried.input.maxBytes < failed.input.maxBytes,+ "recovery must change the failed request's byte budget",+ );+ assert.equal(retried.input.cursor ?? "0", failed.input.cursor ?? "0");+ assert.equal(retried.input.findingRef, failed.input.findingRef);+ const pages = trace.filter(+ (event) =>+ event.event === "response" && !event.injected && readsById.has(event.id),+ );+ for (const page of pages) {+ const request = readsById.get(page.id);+ assert.ok(page.bytes <= request.input.maxBytes);+ }+ assert.ok(pages.some((page) => page.nextCursor !== undefined));+ const submissions = trace.filter(+ (event) =>+ event.event === "request" &&+ event.tool === "record_codex_security_deep_reduction",+ );+ const submissionIds = new Set(submissions.map((event) => event.id));+ const recorded = trace.filter(+ (event) => event.event === "response" && submissionIds.has(event.id),+ );+ assert.equal(+ recorded.length,+ 1,+ "the reducer must successfully record its result exactly once",+ );+ const beforeRecording = trace.slice(+ 0,+ trace.findIndex(+ (event) => event.event === "request" && event.id === recorded[0].id,+ ),+ );+ const pageEdges = new Map();+ for (const page of beforeRecording) {+ const request = readsById.get(page.id);+ if (+ page.event !== "response" ||+ page.injected ||+ !request ||+ request.input.findingRef !== undefined+ )+ continue;+ const cursor = request.input.cursor ?? "0";+ const edges = pageEdges.get(cursor) ?? new Set();+ edges.add(page.nextCursor);+ pageEdges.set(cursor, edges);+ }+ const reached = new Set(["0"]);+ let inputsComplete = false;+ for (const cursor of reached) {+ for (const next of pageEdges.get(cursor) ?? []) {+ if (next === undefined) inputsComplete = true;+ else reached.add(next);+ }+ }+ assert.equal(+ inputsComplete,+ true,+ "read every assigned-input page before recording",+ );+ return {+ actualOversizedResponseBytes: injected[0].bytes,+ firstBudget: failed.input.maxBytes,+ recoveryBudget: retried.input.maxBytes,+ successfulPages: pages.length,+ assignedInputsFullyRead: inputsComplete,+ referenceReads: reads.filter((event) => event.input.findingRef).length,+ };+}++// The controlled provider drives the real tools. It does not simulate their+// results, the frame limit, or recording. The model mode uses only the shipped+// reducer prompt and must discover and carry out recovery itself.+const controlledReducerCode = `// @exec: {"yield_time_ms": 60000}+let maxBytes = 1024 * 1024;+async function readDocument(findingRef) {+ let cursor;+ const fragments = [];+ do {+ const input = {maxBytes, ...(cursor === undefined ? {} : {cursor}),+ ...(findingRef === undefined ? {} : {findingRef})};+ let response;+ try {+ response = await tools.mcp__cs_artifacts__get_codex_security_deep_reducer_inputs(input);+ } catch (error) {+ text(String(error));+ maxBytes = Math.floor(maxBytes / 2);+ response = await tools.mcp__cs_artifacts__get_codex_security_deep_reducer_inputs({...input, maxBytes});+ }+ if (response.isError) throw new Error(response.content[0].text);+ const page = JSON.parse(response.content[0].text);+ fragments.push(page.json);+ cursor = page.nextCursor;+ } while (cursor !== undefined);+ return JSON.parse(fragments.join(''));+}+const inputs = await readDocument();+const original = await readDocument('source:' + inputs.previous.findings[0].provenance.sourceFindingIds[0]);+const prior = await readDocument('previous:0');+if (original.identity.anchor !== prior.identity.anchor) throw new Error('Wrong reference resolved');+const findings = [...inputs.discoveries.flatMap(entry => entry.result.findings), ...inputs.previous.findings];+for (const finding of findings) {+ // The large summary explicitly marks repeated transport padding, not evidence.+ if (finding.summary.startsWith('BEGIN ISSUE:')) {+ finding.summary = finding.summary.split('\\n', 1)[0] + '\\n' + finding.summary.slice(finding.summary.lastIndexOf('END ISSUE:'));+ }+}+const result = await tools.mcp__cs_artifacts__record_codex_security_deep_reduction({+ scanId: inputs.previous.scanId, findings+});+if (result.isError) throw new Error(result.content[0].text);+text({reductionRecorded: true});+`;++if (process.argv[1] === fileURLToPath(import.meta.url)) {+ const reports = path.join(evalDirectory, "../reports");+ await mkdir(reports, { recursive: true });+ const root = await mkdtemp(path.join(reports, "deep-reducer-paging-"));+ console.log(`Eval artifacts: ${root}`);+ const report = await runReducerPagingEval({+ root,+ mode: process.argv[2] ?? "deterministic",+ model: process.argv[3],+ });+ console.log(JSON.stringify(report, null, 2));+}diff --git a/plugins/codex-security/mcp-app/src/artifact-deep-reducer-pages.ts b/plugins/codex-security/mcp-app/src/artifact-deep-reducer-pages.tsnew file mode 100644index 000000000..ac787f86f--- /dev/null+++ b/plugins/codex-security/mcp-app/src/artifact-deep-reducer-pages.ts@@ -0,0 +1,188 @@+import type { ArtifactContext } from "./artifact-context.js";+import {+ deepReducerInputsInputSchema,+ getCodexSecurityDeepReducerInputs,+} from "./artifact-deep-reducer.js";+import type {+ DeepReductionInput,+ DeepReductionSources,+} from "./deep-scan/artifact-validation.js";++export interface DeepReducerPageInput {+ cursor?: string;+ maxBytes: number;+ findingRef?: string;+}++/** Concatenate every json fragment for one document before parsing it. */+export interface DeepReducerPage {+ json: string;+ nextCursor?: string;+}++type Finding = DeepReductionInput["findings"][number];++interface ReducerDocuments {+ inputs: string;+ findings: Map<string, Finding>;+ serializedFindings: Map<string, string>;+}++// Each artifact server binds one immutable reducer assignment. Cache that+// snapshot so small pages do not repeatedly read and serialize all inputs.+const documents = new WeakMap<ArtifactContext, Promise<ReducerDocuments>>();++/** The single payload representation used by both the tool and byte accounting. */+export function deepReducerPageResponse(page: DeepReducerPage): {+ content: [{ type: "text"; text: string }];+} {+ return { content: [{ type: "text", text: JSON.stringify(page) }] };+}++export async function getCodexSecurityDeepReducerInputsPage(+ context: ArtifactContext,+ value: DeepReducerPageInput,+): Promise<DeepReducerPage> {+ const input = deepReducerInputsInputSchema.parse(value);+ let pending = documents.get(context);+ if (!pending) {+ pending = getCodexSecurityDeepReducerInputs(context)+ .then(createDocuments)+ .catch((error: unknown) => {+ documents.delete(context);+ throw error;+ });+ documents.set(context, pending);+ }+ const snapshot = await pending;+ let json = snapshot.inputs;+ if (input.findingRef !== undefined) {+ const finding = snapshot.findings.get(input.findingRef);+ if (!finding)+ throw new Error(+ "The requested findingRef is not assigned to this reducer.",+ );+ const cached = snapshot.serializedFindings.get(input.findingRef);+ json = cached ?? JSON.stringify(finding);+ if (cached === undefined)+ snapshot.serializedFindings.set(input.findingRef, json);+ }+ return pageDocument(json, input);+}++function createDocuments(sources: DeepReductionSources): ReducerDocuments {+ const findings = new Map<string, Finding>();+ const discoveries = sources.discoveries.map((discovery) => ({+ workerId: discovery.workerId,+ result: {+ ...discovery.result,+ findings: discovery.result.findings.map((finding, index) => {+ findings.set(`source:${discovery.workerId}:${index}`, finding);+ return projectFinding(finding);+ }),+ },+ }));+ const previous = sources.previous+ ? {+ ...sources.previous,+ findings: sources.previous.findings.map((finding, index) => {+ findings.set(`previous:${index}`, finding);+ const originals = (finding.provenance as Record<string, unknown>)+ .sourceFindings as+ Array<{ id: string; finding: Finding }> | undefined;+ // Match retainSourceFindings: old aggregates without retained+ // originals contribute one source named previous:<index>.+ const refs = originals?.length+ ? originals.map((original) => {+ findings.set(`source:${original.id}`, original.finding);+ return original.id;+ })+ : [`previous:${index}`];+ if (!originals?.length)+ findings.set(`source:previous:${index}`, finding);+ return projectFinding(finding, refs);+ }),+ }+ : null;+ return {+ inputs: JSON.stringify({ discoveries, previous }),+ findings,+ serializedFindings: new Map(),+ };+}++function projectFinding(finding: Finding, refs?: string[]): Finding {+ const {+ sourceFindings: _sources,+ previousFindings: _previous,+ originalCandidates: _candidates,+ ...provenance+ } = finding.provenance as Record<string, unknown>;+ return {+ ...finding,+ provenance: {+ ...provenance,+ ...(refs === undefined ? {} : { sourceFindingIds: refs }),+ },+ };+}++function pageDocument(+ document: string,+ input: DeepReducerPageInput,+): DeepReducerPage {+ const start = Number(input.cursor ?? "0");+ if (!Number.isSafeInteger(start) || start > document.length)+ throw new Error(+ "The reducer input cursor is outside the selected document.",+ );+ if (unicodeBoundary(document, start) !== start)+ throw new Error("The reducer input cursor splits a Unicode character.");++ const pageAt = (end: number): DeepReducerPage => ({+ json: document.slice(start, end),+ ...(end < document.length ? { nextCursor: String(end) } : {}),+ });+ const fits = (page: DeepReducerPage): boolean =>+ Buffer.byteLength(JSON.stringify(deepReducerPageResponse(page)), "utf8") <=+ input.maxBytes;++ // A UTF-16 code unit cannot occupy less than one byte in this JSON response.+ // Bound the search by the requested budget, not the entire remaining document.+ const maximumEnd = unicodeBoundary(+ document,+ start + Math.min(document.length - start, input.maxBytes),+ );+ const complete = pageAt(maximumEnd);+ if (fits(complete)) return complete;++ let low = start;+ let high = maximumEnd;+ let acceptedEnd = start;+ while (low <= high) {+ const middle = low + Math.floor((high - low) / 2);+ const end = unicodeBoundary(document, middle);+ if (fits(pageAt(end))) {+ acceptedEnd = end;+ low = middle + 1;+ } else {+ high = middle - 1;+ }+ }+ if (acceptedEnd === start)+ throw new Error(+ "The reducer input maxBytes budget cannot fit a response with the next character. Increase maxBytes and retry the same cursor.",+ );+ return pageAt(acceptedEnd);+}++function unicodeBoundary(value: string, offset: number): number {+ const before = value.charCodeAt(offset - 1);+ const after = value.charCodeAt(offset);+ return before >= 0xd800 &&+ before <= 0xdbff &&+ after >= 0xdc00 &&+ after <= 0xdfff+ ? offset - 1+ : offset;+}diff --git a/plugins/codex-security/mcp-app/src/artifact-deep-reducer.ts b/plugins/codex-security/mcp-app/src/artifact-deep-reducer.tsindex 7e1fa8f35..72b859295 100644--- a/plugins/codex-security/mcp-app/src/artifact-deep-reducer.ts+++ b/plugins/codex-security/mcp-app/src/artifact-deep-reducer.ts@@ -7,6 +7,7 @@ import type {ArtifactContext,DeepReducerContext,} from "./artifact-context.js";+import type { DeepReducerPageInput } from "./artifact-deep-reducer-pages.js";import {parsePersistedScanDraft,saveScanDraftCheckpoint,@@ -39,7 +40,7 @@ export const deepReducerInputsInputSchema = loadArtifactZodSchema(schemaDocuments,reducerSchema.$id,"reducerInputs",-) as ZodType<Record<string, never>>;+) as ZodType<DeepReducerPageInput>;export const deepReductionInputSchema = loadArtifactZodSchema(schemaDocuments,diff --git a/plugins/codex-security/mcp-app/src/deep-scan/executor.ts b/plugins/codex-security/mcp-app/src/deep-scan/executor.tsindex 12115cac8..286b2d36c 100644--- a/plugins/codex-security/mcp-app/src/deep-scan/executor.ts+++ b/plugins/codex-security/mcp-app/src/deep-scan/executor.ts@@ -195,6 +195,7 @@ export class CodexSdkWorkerExecutor implements CodexWorkerExecutor {}// Codex exec currently emits retry-in-progress notifications as error events.lastStreamError = event.message;+ appendCodeModeFrameDiagnostic(diagnostics, event.message);}}if (!turnCompleted) {@@ -370,10 +371,16 @@ function appendSafeItemDiagnostic(diagnostics: CodexWorkerDiagnostic[],item: unknown,): void {+ if (!isRecord(item) || typeof item.type !== "string") return;+ if (item.type === "error") {+ appendCodeModeFrameDiagnostic(diagnostics, item.message);+ return;+ }+ if (item.status !== "failed") return;if (- !isRecord(item) ||- item.status !== "failed" ||- typeof item.type !== "string"+ item.type === "mcp_tool_call" &&+ isRecord(item.error) &&+ appendCodeModeFrameDiagnostic(diagnostics, item.error.message))return;if (item.type === "command_execution") {@@ -401,6 +408,16 @@ function appendSafeItemDiagnostic(item.server === "codex_security_artifacts") &&typeof item.tool === "string") {+ if (isRecord(item.result) && Array.isArray(item.result.content)) {+ for (const content of item.result.content) {+ if (+ isRecord(content) &&+ content.type === "text" &&+ appendCodeModeFrameDiagnostic(diagnostics, content.text)+ )+ return;+ }+ }const reason = isRecord(item.result)? "returned an error": isRecord(item.error)@@ -413,6 +430,30 @@ function appendSafeItemDiagnostic(}}+function appendCodeModeFrameDiagnostic(+ diagnostics: CodexWorkerDiagnostic[],+ message: unknown,+): boolean {+ // Codex exposes this transport error as text, without a structured code.+ // Preserve only its complete numeric template, never surrounding tool output.+ if (typeof message !== "string") return false;+ const match =+ /^code-mode delegate response exceeds the IPC frame limit: code-mode IPC frame length [0-9]+ exceeds [0-9]+ bytes$/u.exec(+ message,+ );+ if (match?.[0] !== message) return false;+ const diagnostic: CodexWorkerDiagnostic = {+ code: "artifact_tool_failed",+ message,+ };+ const index = diagnostics.findIndex(+ (existing) => existing.code === diagnostic.code,+ );+ if (index === -1) diagnostics.push(diagnostic);+ else diagnostics[index] = diagnostic;+ return true;+}+function isSandboxNamespaceExhaustion(output: string): boolean {return /bwrap:\s*Creating new namespace failed:.*(?:ENOSPC|max_[a-z_]*_namespaces exceeded|Resource temporarily unavailable)/is.test(output,diff --git a/plugins/codex-security/mcp-app/src/deep-scan/worker-runner.ts b/plugins/codex-security/mcp-app/src/deep-scan/worker-runner.tsindex fd442c5f7..8d9fb8a3b 100644--- a/plugins/codex-security/mcp-app/src/deep-scan/worker-runner.ts+++ b/plugins/codex-security/mcp-app/src/deep-scan/worker-runner.ts@@ -864,7 +864,7 @@ function withWorkerDiagnostics();const diagnostic = namespaceFailure ?? diagnostics[0];const combined = new Error(- `${diagnostic.message} Deterministic artifact validation also reported: ${normalized.message}`,+ `${diagnostics.map((item) => item.message).join(" ")} Deterministic artifact validation also reported: ${normalized.message}`,{ cause: normalized },);Object.defineProperty(combined, "code", {@@ -901,12 +901,21 @@ function standardScanCompletionContinuation(attempt: number): string {function reducerCompletionContinuation(attempt: number): string {return [`Continue the existing Deep Scan reducer after attempt ${attempt} ended without its required result.`,+ ...reducerInputRecoveryInstructions(),"Submit the aggregate with record_codex_security_deep_reduction({ scanId, findings, threatModel?, scope? }).","If the tool rejects the arguments, use its error to correct them and retry the call until it succeeds.","Do not end your turn, write the result directly, or call the tool again after it succeeds.",].join("\n");}+function reducerInputRecoveryInstructions(): string[] {+ return [+ "If input retrieval exceeded a response or IPC size limit, call get_codex_security_deep_reducer_inputs({ cursor?, maxBytes, findingRef? }) with a smaller maxBytes budget (for example, halve the failed request's budget), preserving its cursor and findingRef to retry the same page.",+ "Do not repeat an oversized request unchanged. If inputs have not been read yet, paginate all assigned findings and the previous aggregate, assembling each json fragment in code and following nextCursor until absent; inspect assembled data selectively and fetch details by findingRef.",+ "Preserve pages already read and account for every finding before submitting the aggregate.",+ ];+}+function transientExecutionContinuation(kind: DeepScanWorkerKind,attempt: number,@@ -922,6 +931,7 @@ function transientExecutionContinuation(return [`Continue the existing Deep Scan ${kind} worker objective after transient Codex execution failure on attempt ${attempt - 1}.`,"Resume from the current worker-local artifacts and conversation context.",+ ...(kind === "dedup" ? reducerInputRecoveryInstructions() : []),"Do not restart or discard completed work. Finish every artifact required by the original worker contract,","settle all nested work, and return only after the original objective is complete.",].join("\n");diff --git a/plugins/codex-security/mcp-app/src/server/compact-artifact-tools.ts b/plugins/codex-security/mcp-app/src/server/compact-artifact-tools.tsindex d40a83af5..2efdb936f 100644--- a/plugins/codex-security/mcp-app/src/server/compact-artifact-tools.ts+++ b/plugins/codex-security/mcp-app/src/server/compact-artifact-tools.ts@@ -28,9 +28,12 @@ import {import {deepReducerInputsInputSchema,deepReductionInputSchema,- getCodexSecurityDeepReducerInputs,recordCodexSecurityDeepReduction,} from "../artifact-deep-reducer.js";+import {+ deepReducerPageResponse,+ getCodexSecurityDeepReducerInputsPage,+} from "../artifact-deep-reducer-pages.js";import {completedScanInputSchema,getCodexSecurityCompletedScan,@@ -344,14 +347,23 @@ export function registerCompactWorkerArtifactTools();}- registerCompactTool(server, {- name: "get_codex_security_deep_reducer_inputs",- title: "Get Codex Security Deep Reducer Inputs",- description: "Read the assigned findings, context, and previous aggregate.",- inputSchema: deepReducerInputsInputSchema,- readOnly: true,- handler: async () => getCodexSecurityDeepReducerInputs(context),- });+ server.registerTool(+ "get_codex_security_deep_reducer_inputs",+ {+ title: "Get Codex Security Deep Reducer Inputs",+ description:+ "Read a byte-budgeted JSON fragment of the assigned findings and previous aggregate. " ++ "Continue with nextCursor; concatenate json fragments before parsing. " ++ "Use findingRef source:<sourceFindingId> or previous:N to read full retained provenance.",+ inputSchema: deepReducerInputsInputSchema,+ annotations: readingAnnotations,+ _meta: modelOnlyMeta,+ },+ async (input) =>+ deepReducerPageResponse(+ await getCodexSecurityDeepReducerInputsPage(context, input),+ ),+ );registerCompactTool(server, {name: "record_codex_security_deep_reduction",diff --git a/plugins/codex-security/mcp-app/templates/deep-scan/dedup.md b/plugins/codex-security/mcp-app/templates/deep-scan/dedup.mdindex 5f6271a29..e544fdca0 100644--- a/plugins/codex-security/mcp-app/templates/deep-scan/dedup.md+++ b/plugins/codex-security/mcp-app/templates/deep-scan/dedup.md@@ -6,7 +6,13 @@ Use this exact reducer configuration:{{DEDUP_CONTEXT_JSON}}```-Call `get_codex_security_deep_reducer_inputs({})` to read the findings and context from each assigned, already-validated Standard scan and the previous aggregate, if any.+Read the assigned, already-validated Standard findings and the previous aggregate with `get_codex_security_deep_reducer_inputs({ maxBytes, cursor? })`. Choose a `maxBytes` budget for a small response you can work with, well below the code-mode IPC frame limit. The budget counts serialized response bytes, including JSON escaping, rather than findings. Both new findings and the previous aggregate are paged; even one large finding can span pages.++Each tool response has one text content block containing `{ json, nextCursor? }`. Parse that block to obtain the page. Its `json` is a fragment, not necessarily a complete JSON value. Follow `nextCursor` until absent, concatenate the fragments inside code mode, and only then parse the assembled document as `{ discoveries, previous }`. Inspect the findings and context selectively; do not print or return the entire assembled document from code mode. Do not concatenate pages from different `findingRef` requests.++The input view omits the retained `provenance.sourceFindings`, `previousFindings`, and `originalCandidates` bodies. The host preserves those bodies when saving the aggregate. Fetch full details when needed with the same paged tool and `findingRef`: use `source:<sourceFindingId>` for an original source, or `previous:N` for the previous aggregate's zero-based finding index, including its synthesized history. Never copy those retained bodies into your submission; submit the semantic finding and its source refs.++If a response exceeds the IPC frame limit, retry the failed page with a smaller `maxBytes` (halve the budget used for that request), keeping its cursor and findingRef. Do not retry an unchanged oversized request. Preserve already-read fragments and continue from the failed page. If the budget is too small to hold even one character and the page envelope, increase it as directed by the tool's diagnostic.Read every finding and the previous aggregate. Merge only the same actionable root issue using remediation-subsumption: fixing the retained finding must also fix every absorbed finding. Preserve distinct reachable vulnerable instances, proof tuples, useful evidence, uncertainty, locations, provenance, severity, validation, attack paths, and remediation.diff --git a/plugins/codex-security/mcp-app/tests/test_artifact_deep_reducer.mjs b/plugins/codex-security/mcp-app/tests/test_artifact_deep_reducer.mjsindex cf228c509..6e5976d4d 100644--- a/plugins/codex-security/mcp-app/tests/test_artifact_deep_reducer.mjs+++ b/plugins/codex-security/mcp-app/tests/test_artifact_deep_reducer.mjs@@ -34,7 +34,11 @@ const {const scanId = "7fc17317-9594-49e0-b06a-d72fd7e14bba";const validReduction = reduction([]);-assert.equal(deepReducerInputsInputSchema.safeParse({}).success, true);+assert.equal(+ deepReducerInputsInputSchema.safeParse({ maxBytes: 4096 }).success,+ true,+);+assert.equal(deepReducerInputsInputSchema.safeParse({}).success, false);assert.equal(deepReducerInputsInputSchema.safeParse({ path: "/tmp" }).success,false,diff --git a/plugins/codex-security/mcp-app/tests/test_artifact_deep_reducer_pages.mjs b/plugins/codex-security/mcp-app/tests/test_artifact_deep_reducer_pages.mjsnew file mode 100644index 000000000..22ecbb4d9--- /dev/null+++ b/plugins/codex-security/mcp-app/tests/test_artifact_deep_reducer_pages.mjs@@ -0,0 +1,323 @@+import assert from "node:assert/strict";+import {+ mkdir,+ mkdtemp,+ readFile,+ realpath,+ rm,+ writeFile,+} from "node:fs/promises";+import { tmpdir } from "node:os";+import path from "node:path";+import { build } from "esbuild";++const bundle = await build({+ bundle: true,+ stdin: {+ contents: `+ export * from "./artifact-deep-reducer-pages.ts";+ export * from "./artifact-deep-reducer.ts";+ `,+ resolveDir: new URL("../src/", import.meta.url).pathname,+ },+ format: "esm",+ platform: "node",+ write: false,+});+const {+ deepReducerInputsInputSchema,+ deepReducerPageResponse,+ getCodexSecurityDeepReducerInputs,+ getCodexSecurityDeepReducerInputsPage,+ recordCodexSecurityDeepReduction,+} = await import(+ "data:text/javascript;base64," ++ Buffer.from(bundle.outputFiles[0].contents).toString("base64")+);++for (const maxBytes of [0, -1, 1.5, Infinity, Number.MAX_SAFE_INTEGER + 1]) {+ assert.equal(+ deepReducerInputsInputSchema.safeParse({ maxBytes }).success,+ false,+ );+}+assert.equal(+ deepReducerInputsInputSchema.safeParse({ maxBytes: Number.MAX_SAFE_INTEGER })+ .success,+ true,+);+assert.equal(+ deepReducerInputsInputSchema.safeParse({ maxBytes: 256, cursor: "-1" })+ .success,+ false,+);++const scanId = "7fc17317-9594-49e0-b06a-d72fd7e14bba";+const root = await realpath(+ await mkdtemp(path.join(tmpdir(), "reducer-pages-")),+);+try {+ const scanRoot = path.join(root, "scan");+ const workerRoot = path.join(+ scanRoot,+ "artifacts",+ "deep_discovery",+ "workers",+ "one",+ "output",+ );+ const previousRoot = path.join(+ scanRoot,+ "artifacts",+ "deep_discovery",+ "dedup",+ "previous",+ "output",+ );+ const outputRoot = path.join(+ scanRoot,+ "artifacts",+ "deep_discovery",+ "dedup",+ "current",+ "output",+ );+ await Promise.all(+ [workerRoot, previousRoot, outputRoot].map((directory) =>+ mkdir(directory, { recursive: true }),+ ),+ );+ const largeText =+ 'Quoted "text" with \\ newline\n and Unicode 😀é\u0000'.repeat(2048);+ const fresh = finding("fresh", {+ summary: largeText,+ provenance: {+ source: "local_plugin",+ previousFindings: [{ summary: "retained worker history" }],+ originalCandidates: [{ summary: "retained candidate details" }],+ },+ });+ const original = finding("old", { summary: "original evidence" });+ const canonical = finding("old", {+ summary: "current synthesized evidence",+ provenance: {+ source: "local_plugin",+ sourceFindingIds: ["original-worker:0"],+ sourceFindings: [{ id: "original-worker:0", finding: original }],+ previousFindings: [{ summary: largeText }],+ },+ });+ const legacyOriginal = finding("legacy-source", {+ summary: "legacy original differs from current aggregate zero",+ });+ const legacyCanonical = finding("legacy-source", {+ provenance: {+ source: "local_plugin",+ sourceFindings: [{ id: "previous:0", finding: legacyOriginal }],+ },+ });+ const noRefs = finding("legacy-without-refs");+ const worker = {+ scanId,+ findings: [fresh],+ scope: { summary: largeText },+ coverage: {+ completeness: "complete",+ surfaces: [],+ explicitExclusions: [],+ deferred: [],+ },+ };+ const previous = {+ scanId,+ findings: [canonical, legacyCanonical, noRefs],+ threatModel: { summary: largeText },+ };+ const workerPath = path.join(workerRoot, "result.json");+ const previousPath = path.join(previousRoot, "result.json");+ await Promise.all([+ writeFile(workerPath, JSON.stringify(worker)),+ writeFile(previousPath, JSON.stringify(previous)),+ ]);+ const context = {+ root: outputRoot,+ repoRoot: root,+ scanId,+ layout: "reducer",+ deepReducer: {+ scanRoot,+ claimedWorkers: [{ id: "worker-one", resultPath: workerPath }],+ previousReducerResultPath: previousPath,+ },+ };++ const full = await getCodexSecurityDeepReducerInputs(context);+ const projected = await readDocument(context);+ assert.equal(projected.discoveries[0].result.findings[0].summary, largeText);+ assert.deepEqual(projected.discoveries[0].result.scope, worker.scope);+ assert.deepEqual(projected.previous.threatModel, previous.threatModel);+ assert.deepEqual(+ projected.previous.findings.map(+ (entry) => entry.provenance.sourceFindingIds,+ ),+ [["original-worker:0"], ["previous:0"], ["previous:2"]],+ );+ for (const entry of [+ ...projected.discoveries[0].result.findings,+ ...projected.previous.findings,+ ]) {+ for (const field of [+ "sourceFindings",+ "previousFindings",+ "originalCandidates",+ ])+ assert.equal(field in entry.provenance, false);+ }+ assert.deepEqual(+ await readDocument(context, { findingRef: "source:worker-one:0" }),+ full.discoveries[0].result.findings[0],+ );+ assert.deepEqual(+ await readDocument(context, { findingRef: "source:original-worker:0" }),+ original,+ );+ assert.deepEqual(+ await readDocument(context, { findingRef: "previous:0" }),+ canonical,+ );+ assert.deepEqual(+ await readDocument(context, { findingRef: "source:previous:0" }),+ legacyOriginal,+ );+ assert.deepEqual(+ await readDocument(context, { findingRef: "source:previous:2" }),+ noRefs,+ );++ // Changing files cannot change the already-bound page stream or its references.+ await writeFile(workerPath, "unreadable replacement");+ assert.deepEqual(await readDocument(context), projected);+ assert.deepEqual(+ await readDocument(context, { findingRef: "source:worker-one:0" }),+ full.discoveries[0].result.findings[0],+ );+ const recoveredContext = { ...context };+ await assert.rejects(+ getCodexSecurityDeepReducerInputsPage(recoveredContext, { maxBytes: 4096 }),+ );+ await writeFile(workerPath, JSON.stringify(worker));+ assert.deepEqual(+ await readDocument(recoveredContext),+ projected,+ "a failed input read must not poison later retries",+ );++ const firstPage = await getCodexSecurityDeepReducerInputsPage(context, {+ maxBytes: 4096,+ });+ const smaller = await getCodexSecurityDeepReducerInputsPage(context, {+ maxBytes: 1024,+ });+ assert.ok(smaller.json.length < firstPage.json.length);+ assert.ok(firstPage.json.startsWith(smaller.json));+ assert.deepEqual(await readDocument(context, { maxBytes: 1024 }), projected);+ await assert.rejects(+ getCodexSecurityDeepReducerInputsPage(context, { maxBytes: 1 }),+ /Increase maxBytes and retry the same cursor/,+ );+ await assert.rejects(+ getCodexSecurityDeepReducerInputsPage(context, {+ maxBytes: 4096,+ cursor: String(Number.MAX_SAFE_INTEGER),+ }),+ /cursor is outside/,+ );+ await assert.rejects(+ getCodexSecurityDeepReducerInputsPage(context, {+ maxBytes: 4096,+ findingRef: "source:unassigned:0",+ }),+ /not assigned/,+ );++ // Submitting only the projection still preserves complete originals and history.+ await recordCodexSecurityDeepReduction(context, {+ scanId,+ findings: [+ ...projected.discoveries[0].result.findings,+ ...projected.previous.findings,+ ],+ scope: projected.discoveries[0].result.scope,+ threatModel: projected.previous.threatModel,+ });+ const saved = JSON.parse(+ await readFile(path.join(outputRoot, "result.json"), "utf8"),+ );+ const allSources = saved.findings.flatMap(+ (entry) => entry.provenance.sourceFindings,+ );+ assert.deepEqual(allSources.map((source) => source.id).sort(), [+ "original-worker:0",+ "previous:0",+ "previous:2",+ "worker-one:0",+ ]);+ assert.deepEqual(+ allSources.find((source) => source.id === "worker-one:0").finding,+ fresh,+ );+ assert.deepEqual(+ allSources.find((source) => source.id === "original-worker:0").finding,+ original,+ );+ assert.deepEqual(+ saved.findings.find((entry) => entry.identity.anchor === "old").provenance+ .previousFindings,+ canonical.provenance.previousFindings,+ );+ console.log("deep reducer paging tests passed");+} finally {+ await rm(root, { recursive: true, force: true });+}++async function readDocument(context, options = {}) {+ const maxBytes = options.maxBytes ?? 4096;+ let cursor;+ const fragments = [];+ do {+ const page = await getCodexSecurityDeepReducerInputsPage(context, {+ ...options,+ maxBytes,+ ...(cursor === undefined ? {} : { cursor }),+ });+ const response = deepReducerPageResponse(page);+ assert.deepEqual(Object.keys(response), ["content"]);+ assert.ok(Buffer.byteLength(JSON.stringify(response), "utf8") <= maxBytes);+ assert.deepEqual(JSON.parse(response.content[0].text), page);+ assert.equal(page.json.isWellFormed(), true);+ if (page.nextCursor !== undefined)+ assert.ok(Number(page.nextCursor) > Number(cursor ?? "0"));+ fragments.push(page.json);+ cursor = page.nextCursor;+ } while (cursor !== undefined);+ return JSON.parse(fragments.join(""));+}++function finding(id, extra = {}) {+ return {+ ruleId: "cross-site-scripting." + id,+ identity: { anchor: id },+ title: "Unsafe request output " + id,+ summary: "A request-controlled value reaches an HTML response.",+ severity: { level: "high" },+ confidence: {+ level: "high",+ rationale: "The source establishes reachability.",+ },+ taxonomy: { category: "cross-site-scripting", cwe: ["CWE-79"] },+ locations: [{ path: `src/${id}.ts`, startLine: 1, endLine: 2 }],+ remediation: "Encode request-controlled values before emitting HTML.",+ provenance: { source: "local_plugin" },+ ...extra,+ };+}diff --git a/plugins/codex-security/mcp-app/tests/test_compact_artifact_server.mjs b/plugins/codex-security/mcp-app/tests/test_compact_artifact_server.mjsindex fa1864201..50b738044 100644--- a/plugins/codex-security/mcp-app/tests/test_compact_artifact_server.mjs+++ b/plugins/codex-security/mcp-app/tests/test_compact_artifact_server.mjs@@ -1,7 +1,14 @@import assert from "node:assert/strict";import { execFileSync } from "node:child_process";import { createHash, randomUUID } from "node:crypto";-import { mkdtemp, mkdir, readFile, rm, writeFile } from "node:fs/promises";+import {+ mkdtemp,+ mkdir,+ readFile,+ realpath,+ rm,+ writeFile,+} from "node:fs/promises";import { tmpdir } from "node:os";import path from "node:path";import { fileURLToPath, pathToFileURL } from "node:url";@@ -1438,27 +1445,97 @@ async function testDiscoveryWorkerToolList(bundle) {async function testReducerWorkerToolList(bundle) {const repoRoot = path.join(temporaryRoot, "reducer-repository");- const scanRoot = path.join(temporaryRoot, "reducer-scan");+ const scanRoot = await realpath(+ await mkdtemp(path.join(temporaryRoot, "reducer-scan-")),+ );+ const scanId = randomUUID();+ const workerId = "worker-paging-fixture";+ const workerRoot = path.join(+ scanRoot,+ "artifacts",+ "deep_discovery",+ "workers",+ "discovery-0001",+ "output",+ );+ const previousRoot = path.join(+ scanRoot,+ "artifacts",+ "deep_discovery",+ "dedup",+ "dedup-0001",+ "output",+ );const artifactRoot = path.join(scanRoot,"artifacts","deep_discovery","dedup",+ "dedup-0002","output",);await Promise.all([mkdir(repoRoot, { recursive: true }),+ mkdir(workerRoot, { recursive: true }),+ mkdir(previousRoot, { recursive: true }),mkdir(artifactRoot, { recursive: true }),]);+ const sourceOriginal = reducerPagingFinding("original-source");+ const workerFinding = {+ ...reducerPagingFinding("fresh-source"),+ summary: 'A large finding with quoted "evidence" and Unicode 🧭. '.repeat(+ 120,+ ),+ provenance: {+ source: "local_plugin",+ sourceFindings: [{ id: "earlier-worker:0", finding: sourceOriginal }],+ previousFindings: [sourceOriginal],+ originalCandidates: [{ summary: "retained source candidate" }],+ },+ };+ const previousOriginal = reducerPagingFinding("previous-original");+ const previousFinding = {+ ...reducerPagingFinding("previous-aggregate"),+ provenance: {+ source: "local_plugin",+ sourceFindingIds: ["previous-worker:0"],+ sourceFindings: [{ id: "previous-worker:0", finding: previousOriginal }],+ previousFindings: [previousOriginal],+ originalCandidates: [{ summary: "retained previous candidate" }],+ },+ };+ const workerResultPath = path.join(workerRoot, "result.json");+ const previousReducerResultPath = path.join(previousRoot, "result.json");+ await Promise.all([+ writeFile(+ workerResultPath,+ JSON.stringify({+ scanId,+ findings: [workerFinding],+ coverage: {+ completeness: "complete",+ surfaces: [],+ explicitExclusions: [],+ deferred: [],+ },+ }),+ ),+ writeFile(+ previousReducerResultPath,+ JSON.stringify({ scanId, findings: [previousFinding] }),+ ),+ ]);const client = await startClient(bundle, {CODEX_SECURITY_ARTIFACT_ROOT: artifactRoot,CODEX_SECURITY_REPO_ROOT: repoRoot,CODEX_SECURITY_ARTIFACT_LAYOUT: "reducer",+ CODEX_SECURITY_SCAN_ID: scanId,CODEX_SECURITY_PLUGIN_ROOT: pluginRoot,CODEX_SECURITY_REDUCER_CONTEXT_JSON: JSON.stringify({scanRoot,- claimedWorkers: [],+ claimedWorkers: [{ id: workerId, resultPath: workerResultPath }],+ previousReducerResultPath,}),});try {@@ -1501,6 +1578,14 @@ async function testReducerWorkerToolList(bundle) {false,"The reducer must not be asked to submit coverage.",);+ } else {+ assert.deepEqual(tool.inputSchema.required, ["maxBytes"]);+ assert.equal(tool.inputSchema.additionalProperties, false);+ assert.deepEqual(Object.keys(tool.inputSchema.properties).sort(), [+ "cursor",+ "findingRef",+ "maxBytes",+ ]);}for (const forbidden of ["path",@@ -1516,11 +1601,123 @@ async function testReducerWorkerToolList(bundle) {);}}++ const maxBytes = 768;+ async function readPagedInputs(findingRef) {+ const fragments = [];+ const cursors = new Set();+ let cursor;+ do {+ const response = await client.callTool({+ name: "get_codex_security_deep_reducer_inputs",+ arguments: {+ maxBytes,+ ...(cursor === undefined ? {} : { cursor }),+ ...(findingRef === undefined ? {} : { findingRef }),+ },+ });+ assert.notEqual(response.isError, true, JSON.stringify(response));+ assert.equal(Object.hasOwn(response, "structuredContent"), false);+ assert.equal(response.content.length, 1);+ assert.equal(response.content[0].type, "text");+ assert.ok(+ Buffer.byteLength(JSON.stringify(response), "utf8") <= maxBytes,+ "The full encoded MCP result must fit the requested byte budget.",+ );+ const page = JSON.parse(response.content[0].text);+ assert.equal(typeof page.json, "string");+ assert.ok(page.json.length > 0, "Every page must make progress.");+ fragments.push(page.json);+ cursor = page.nextCursor;+ if (cursor !== undefined) {+ assert.equal(typeof cursor, "string");+ assert.equal(+ cursors.has(cursor),+ false,+ "Paging must not repeat a cursor.",+ );+ cursors.add(cursor);+ }+ } while (cursor !== undefined);+ return {+ value: JSON.parse(fragments.join("")),+ pageCount: fragments.length,+ };+ }++ const inputs = await readPagedInputs();+ assert.ok(inputs.pageCount > 1, "A single large finding must span pages.");+ assert.equal(inputs.value.discoveries.length, 1);+ assert.equal(inputs.value.discoveries[0].workerId, workerId);+ assert.equal(inputs.value.discoveries[0].result.scanId, scanId);+ assert.equal(inputs.value.previous.scanId, scanId);+ const fresh = inputs.value.discoveries[0].result.findings[0];+ const previous = inputs.value.previous.findings[0];+ assert.equal(fresh.summary, workerFinding.summary);+ assert.equal(previous.title, previousFinding.title);+ assert.deepEqual(fresh.provenance.sourceFindingIds, [`${workerId}:0`]);+ assert.deepEqual(previous.provenance.sourceFindingIds, [+ "previous-worker:0",+ ]);+ for (const finding of [fresh, previous]) {+ for (const body of [+ "sourceFindings",+ "previousFindings",+ "originalCandidates",+ ]) {+ assert.equal(+ Object.hasOwn(finding.provenance, body),+ false,+ `The slim reducer input must omit ${body} bodies.`,+ );+ }+ }+ const fullSource = await readPagedInputs(+ `source:${fresh.provenance.sourceFindingIds[0]}`,+ );+ assert.ok(fullSource.pageCount > 1);+ assert.deepEqual(fullSource.value, {+ ...workerFinding,+ provenance: {+ ...workerFinding.provenance,+ sourceFindingIds: [`${workerId}:0`],+ },+ });+ assert.deepEqual(+ (await readPagedInputs("previous:0")).value,+ previousFinding,+ );+ assert.deepEqual(+ (+ await readPagedInputs(+ `source:${previous.provenance.sourceFindingIds[0]}`,+ )+ ).value,+ previousOriginal,+ );} finally {await client.close();}}+function reducerPagingFinding(id) {+ return {+ ruleId: `cross-site-scripting.${id}`,+ identity: { anchor: id },+ title: `Unsafe request output ${id}`,+ summary: "A request-controlled value reaches an HTML response.",+ severity: { level: "high" },+ confidence: {+ level: "high",+ rationale: "The source establishes reachability.",+ },+ taxonomy: { category: "cross-site-scripting", cwe: ["CWE-79"] },+ locations: [{ path: "src/fixture.ts", startLine: 1, endLine: 2 }],+ remediation: "Encode request-controlled values before emitting HTML.",+ provenance: { source: "local_plugin" },+ };+}+async function bundleEntrypoint(entrypoint, outfile) {await build({bundle: true,diff --git a/plugins/codex-security/mcp-app/tests/test_deep_reducer_paging_eval.mjs b/plugins/codex-security/mcp-app/tests/test_deep_reducer_paging_eval.mjsnew file mode 100644index 000000000..e97926009--- /dev/null+++ b/plugins/codex-security/mcp-app/tests/test_deep_reducer_paging_eval.mjs@@ -0,0 +1,71 @@+import assert from "node:assert/strict";+import { mkdtemp, readFile, rm, writeFile } from "node:fs/promises";+import { tmpdir } from "node:os";+import path from "node:path";+import test from "node:test";+import {+ gradeReducerPagingTrace,+ runReducerPagingEval,+} from "../evals/deep-reducer-paging.mjs";+import {+ createReducerPagingFixture,+ gradeReducerPagingResult,+} from "../evals/deep-reducer-paging-fixture.mjs";++test("a reducer recovers from the real IPC frame limit and records all sources", async (t) => {+ const root = await mkdtemp(path.join(tmpdir(), "deep-reducer-ipc-eval-"));+ t.after(() => rm(root, { recursive: true, force: true }));+ const report = await runReducerPagingEval({ root });+ assert.equal(report.realIpcErrorObserved, true);+ assert.ok(report.actualOversizedResponseBytes > report.ipcFrameLimitBytes);+ assert.ok(report.recoveryBudget < report.firstBudget);+ assert.ok(report.successfulPages > 1);+ assert.equal(report.referenceReads, 2);+ assert.equal(report.accountedSourceCount, 3);+ assert.equal(report.preservedOriginalCount, 3);+ assert.equal(report.previousIdentityPreserved, true);+ assert.equal(report.synthesizedHistoryPreserved, true);++ const trace = (await readFile(path.join(root, "tool-trace.jsonl"), "utf8"))+ .trim()+ .split("\n")+ .map((line) => JSON.parse(line));+ const rootPages = trace.filter(+ (event) =>+ event.event === "request" &&+ event.tool === "get_codex_security_deep_reducer_inputs" &&+ event.input.findingRef === undefined,+ );+ const missingPage = rootPages[2].id;+ assert.throws(+ () =>+ gradeReducerPagingTrace(+ trace.filter(+ (event) => !(event.event === "response" && event.id === missingPage),+ ),+ report.ipcFrameLimitBytes,+ ),+ /read every assigned-input page/,+ );++ // Keeping every original is insufficient if distinct issues are collapsed.+ const fixture = await createReducerPagingFixture(root);+ const result = JSON.parse(await readFile(fixture.resultPath, "utf8"));+ const previous = result.findings.find((finding) =>+ finding.provenance.sourceFindingIds.includes(+ fixture.expected.previousSourceId,+ ),+ );+ previous.provenance.sourceFindingIds = result.findings.flatMap(+ (finding) => finding.provenance.sourceFindingIds,+ );+ previous.provenance.sourceFindings = result.findings.flatMap(+ (finding) => finding.provenance.sourceFindings,+ );+ result.findings = [previous];+ await writeFile(fixture.resultPath, JSON.stringify(result));+ await assert.rejects(+ gradeReducerPagingResult(fixture),+ /distinct|independent/,+ );+});diff --git a/plugins/codex-security/mcp-app/tests/test_deep_scan_coordinator.mjs b/plugins/codex-security/mcp-app/tests/test_deep_scan_coordinator.mjsindex 28596eccf..924c8d74f 100644--- a/plugins/codex-security/mcp-app/tests/test_deep_scan_coordinator.mjs+++ b/plugins/codex-security/mcp-app/tests/test_deep_scan_coordinator.mjs@@ -1812,7 +1812,9 @@ async function testInvalidReducerResultRetriesFromSnapshot(}}-async function testMissingReducerResultResumesExistingThread() {+async function testMissingReducerResultResumesExistingThread(+ diagnosticMessage = "Codex worker artifact tool record_codex_security_deep_reduction failed.",+) {const fixture = await fixtureRun({workers: 1,subagents: 0,@@ -1826,8 +1828,7 @@ async function testMissingReducerResultResumesExistingThread() {dedupDiagnostics: [{code: "artifact_tool_failed",- message:- "Codex worker artifact tool record_codex_security_deep_reduction failed.",+ message: diagnosticMessage,},],});@@ -1852,6 +1853,15 @@ async function testMissingReducerResultResumesExistingThread() {assert.equal(store.dedupClaims.length, 1);assert.equal(executor.dedupCalls, 2);assert.deepEqual(sleeps, [1]);+ assert.ok(+ store.workerUpdates.some(+ (update) =>+ update.kind === "dedup" &&+ update.error?.startsWith(diagnosticMessage) &&+ update.error.includes("result.json"),+ ),+ "the persisted missing-result error must retain the tool failure reason",+ );assert.deepEqual(executor.dedupResumeThreadIds, [undefined,executor.dedupThreadIds[0],@@ -1869,6 +1879,18 @@ async function testMissingReducerResultResumesExistingThread() {executor.dedupContinuationPrompts[1] ?? "",/retry the call until it succeeds/,);+ assert.match(+ executor.dedupContinuationPrompts[1] ?? "",+ /smaller maxBytes budget.*halve the failed request's budget/,+ );+ assert.match(+ executor.dedupContinuationPrompts[1] ?? "",+ /preserving its cursor and findingRef to retry the same page/,+ );+ assert.match(+ executor.dedupContinuationPrompts[1] ?? "",+ /paginate all assigned findings and the previous aggregate/,+ );assert.equal(new Set(executor.dedupPromptPaths).size, 1);await assert.rejects(realpath(@@ -1887,6 +1909,53 @@ async function testMissingReducerResultResumesExistingThread() {);}+async function testMissingReducerResultRetainsSizeDiagnosticAfterOtherFailures() {+ const sizeMessage =+ "code-mode delegate response exceeds the IPC frame limit: code-mode IPC frame length 76008279 exceeds 67108864 bytes";+ for (const earlierDiagnostic of [+ { code: "file_change_failed", message: "Codex worker file change failed." },+ {+ code: "sandbox_namespace_exhausted",+ message: "Codex worker sandbox namespace creation failed (bwrap ENOSPC).",+ },+ ]) {+ const fixture = await fixtureRun({+ workers: 1,+ subagents: 0,+ stopAfterNoNew: 1,+ maxDiscoveryRuns: 1,+ });+ const store = new FakeStore(fixture.run);+ const executor = new FakeExecutor({+ missingDedupResultsByLabel: { "dedup-0001": 1 },+ dedupDiagnostics: [+ earlierDiagnostic,+ { code: "artifact_tool_failed", message: sizeMessage },+ ],+ });+ const coordinator = new DeepScanCoordinator({+ run: fixture.run,+ store,+ executor,+ pluginRoot: fixture.pluginRoot,+ retryDelaysMs: [1],+ clock: immediateClock,+ });+ coordinator.start();+ const terminal = await coordinator.wait(undefined, 5_000);+ assert.equal(terminal?.status, "succeeded", terminal?.error);+ const failure = store.workerUpdates.find(+ (update) => update.kind === "dedup" && update.error,+ );+ assert.ok(failure.error.includes(earlierDiagnostic.message));+ assert.ok(failure.error.includes(sizeMessage));+ assert.ok(failure.error.includes("result.json"));+ const retryPrompt = await readFile(executor.dedupPromptPaths[1], "utf8");+ assert.ok(retryPrompt.includes(earlierDiagnostic.message));+ assert.ok(retryPrompt.includes(sizeMessage));+ }+}+async function testExhaustedReducerIsReplacedAtDiscoveryLimit() {const fixture = await fixtureRun({workers: 2,@@ -4629,6 +4698,10 @@ try {await testInvalidReducerResultRetriesFromSnapshot();await testInvalidReducerResultRetriesFromSnapshot(true);await testMissingReducerResultResumesExistingThread();+ await testMissingReducerResultResumesExistingThread(+ "code-mode delegate response exceeds the IPC frame limit: code-mode IPC frame length 76008279 exceeds 67108864 bytes",+ );+ await testMissingReducerResultRetainsSizeDiagnosticAfterOtherFailures();await testExhaustedReducerIsReplacedAtDiscoveryLimit();await testPolicyRefusedReducerPreservesInputsAndCommittedAggregate();await testNonRetryableReducerAbortsScanWithoutRetry();diff --git a/plugins/codex-security/mcp-app/tests/test_deep_scan_executor.mjs b/plugins/codex-security/mcp-app/tests/test_deep_scan_executor.mjsindex 70c0356d6..8c1a757db 100644--- a/plugins/codex-security/mcp-app/tests/test_deep_scan_executor.mjs+++ b/plugins/codex-security/mcp-app/tests/test_deep_scan_executor.mjs@@ -26,7 +26,7 @@ const bundle = await build({},stdin: {// Test the environment snapshot without adding a production export.- contents: `${await readFile(executorSource, "utf8")}\nexport { snapshotWorkerEnvironment };`,+ contents: `${await readFile(executorSource, "utf8")}\nexport { snapshotWorkerEnvironment, appendSafeItemDiagnostic };`,loader: "ts",resolveDir: path.dirname(fileURLToPath(executorSource)),sourcefile: fileURLToPath(executorSource),@@ -35,10 +35,14 @@ const bundle = await build({platform: "node",write: false,});-const { CodexSdkWorkerExecutor, resolveCodexPath, snapshotWorkerEnvironment } =- await import(- `data:text/javascript;base64,${Buffer.from(bundle.outputFiles[0].contents).toString("base64")}`- );+const {+ CodexSdkWorkerExecutor,+ resolveCodexPath,+ snapshotWorkerEnvironment,+ appendSafeItemDiagnostic,+} = await import(+ `data:text/javascript;base64,${Buffer.from(bundle.outputFiles[0].contents).toString("base64")}`+);const errorsBundle = await build({bundle: true,entryPoints: [@@ -77,6 +81,8 @@ const emptyWorkerPermissionProfile = {filesystem: { ":root": "read" },network: { enabled: false },};+const ipcFrameError =+ "code-mode delegate response exceeds the IPC frame limit: code-mode IPC frame length 76008279 exceeds 67108864 bytes";const deniedWorkerPermissionProfile = {extends: ":read-only",filesystem: {@@ -90,6 +96,7 @@ const deniedWorkerPermissionProfile = {};try {+ testCodeModeFrameDiagnosticBoundaries();await testOpenAiCredentialsReachWorker();await testWorkerReasoningSummaries();if (process.platform !== "win32") {@@ -106,6 +113,7 @@ try {await testRetryNotificationDoesNotInterruptTurn();await testSandboxNamespaceDiagnosticIsSanitized();await testOwnedArtifactToolFailureDiagnosticIsSanitized();+ await testCodeModeFrameDiagnosticSurvivesSuccessfulTurn();await testStreamTerminationWithoutTerminalEventFails();await testCompletedWorkerSettlesWithoutWaitingForProcessExit();await testAbortPropagation();@@ -1499,6 +1507,107 @@ async function testOwnedArtifactToolFailureDiagnosticIsSanitized() {}}+function testCodeModeFrameDiagnosticBoundaries() {+ const failedArtifactTool = {+ type: "mcp_tool_call",+ server: "cs_artifacts",+ tool: "get_codex_security_deep_reducer_inputs",+ status: "failed",+ };+ const resultWithText = (text) => ({ content: [{ type: "text", text }] });+ for (const item of [+ { type: "error", message: ipcFrameError },+ { ...failedArtifactTool, error: { message: ipcFrameError } },+ { ...failedArtifactTool, result: resultWithText(ipcFrameError) },+ ]) {+ const diagnostics = [];+ appendSafeItemDiagnostic(diagnostics, failedArtifactTool);+ appendSafeItemDiagnostic(diagnostics, item);+ appendSafeItemDiagnostic(diagnostics, failedArtifactTool);+ assert.deepEqual(diagnostics, [+ { code: "artifact_tool_failed", message: ipcFrameError },+ ]);+ }+ for (const message of [+ `private source: ${ipcFrameError}`,+ `${ipcFrameError} private output`,+ `${ipcFrameError}\n`,+ JSON.stringify({ error: ipcFrameError }),+ ipcFrameError.replace("76008279", "unknown"),+ "private path /customer/repo: IPC frame limit exceeded",+ ]) {+ const diagnostics = [];+ appendSafeItemDiagnostic(diagnostics, { type: "error", message });+ appendSafeItemDiagnostic(diagnostics, {+ ...failedArtifactTool,+ result: resultWithText(message),+ });+ assert.deepEqual(diagnostics, [+ {+ code: "artifact_tool_failed",+ message:+ "Codex worker artifact tool get_codex_security_deep_reducer_inputs returned an error.",+ },+ ]);+ }+ for (const item of [+ {+ ...failedArtifactTool,+ server: "foreign_server",+ result: resultWithText(ipcFrameError),+ },+ {+ type: "command_execution",+ status: "failed",+ aggregated_output: ipcFrameError,+ },+ { type: "agent_message", text: ipcFrameError },+ { type: "unknown", status: "failed", error: { message: ipcFrameError } },+ ]) {+ const diagnostics = [];+ appendSafeItemDiagnostic(diagnostics, item);+ assert.deepEqual(diagnostics, []);+ }+}++async function testCodeModeFrameDiagnosticSurvivesSuccessfulTurn() {+ const fixture = await fakeCodexFixture();+ const previousPath = process.env.CODEX_CLI_PATH;+ process.env.CODEX_CLI_PATH = fixture.executablePath;+ try {+ const promptPath = path.join(fixture.root, "prompt.md");+ const workingDirectory = path.join(fixture.root, "artifacts");+ await mkdir(workingDirectory);+ for (const event of [+ { type: "error", message: ipcFrameError },+ {+ type: "item.completed",+ item: { id: "error-1", type: "error", message: ipcFrameError },+ },+ ]) {+ await writeFile(+ promptPath,+ `IPC_DIAGNOSTIC_EVENT\n${JSON.stringify(event)}\n`,+ );+ const result = await new CodexSdkWorkerExecutor({+ parentSandbox: trustedParentSandbox,+ }).run({+ kind: "dedup",+ promptPath,+ workingDirectory,+ subagents: 0,+ signal: new AbortController().signal,+ });+ assert.equal(result.finalResponse, "fixture final response");+ assert.deepEqual(result.diagnostics, [+ { code: "artifact_tool_failed", message: ipcFrameError },+ ]);+ }+ } finally {+ restoreEnv("CODEX_CLI_PATH", previousPath);+ }+}+async function testStreamTerminationWithoutTerminalEventFails() {const fixture = await fakeCodexFixture();const previousPath = process.env.CODEX_CLI_PATH;@@ -2059,6 +2168,7 @@ async function fakeCodexFixture("const resumeIndex = process.argv.indexOf('resume');","const threadId = resumeIndex === -1 ? 'fixture-thread-id' : process.argv[resumeIndex + 1];","console.log(JSON.stringify({ type: 'thread.started', thread_id: threadId }));",+ "if (stdin.includes('IPC_DIAGNOSTIC_EVENT')) console.log(stdin.split('\\n')[1]);","if (stdin.includes('MALFORMED_COMMAND_EVENT')) {"," const output = JSON.parse(stdin.split('\\n')[1]);"," const event = { type: 'item.completed', item: { id: 'fixture-command', type: 'command_execution', command: 'cat example.ts', aggregated_output: output, exit_code: 0, status: 'completed' } };",diff --git a/plugins/codex-security/schemas/tools/deep-reducer.schema.json b/plugins/codex-security/schemas/tools/deep-reducer.schema.jsonindex 69726ccdc..e459e9762 100644--- a/plugins/codex-security/schemas/tools/deep-reducer.schema.json+++ b/plugins/codex-security/schemas/tools/deep-reducer.schema.json@@ -6,6 +6,24 @@"$defs": {"reducerInputs": {"type": "object",+ "properties": {+ "cursor": {+ "type": "string",+ "pattern": "^(0|[1-9][0-9]*)$",+ "description": "Continue the same document from its returned nextCursor."+ },+ "maxBytes": {+ "type": "integer",+ "minimum": 1,+ "maximum": 9007199254740991,+ "description": "Caller-selected byte budget for the serialized MCP response. Choose below the transport and output budgets; reduce it if transport reports an oversized response."+ },+ "findingRef": {+ "type": "string",+ "description": "Omit to page assigned findings and the previous aggregate without retained provenance bodies. Use source:<sourceFindingId> to fetch a complete original, or previous:<zero-based index> to fetch a complete previous aggregate finding, including its retained history."+ }+ },+ "required": ["maxBytes"],"additionalProperties": false},"reductionInput": {@@ -31,10 +49,7 @@"$ref": "codex-security://schemas/tools/scan-draft.schema.json#/$defs/scanDraftInput/properties/findings"}},- "required": [- "scanId",- "findings"- ],+ "required": ["scanId", "findings"],"additionalProperties": false}}diff --git a/sdk/typescript/tests-ts/deep-scan-reducer-recovery.test.ts b/sdk/typescript/tests-ts/deep-scan-reducer-recovery.test.tsindex 71bf5f893..25daf49a3 100644--- a/sdk/typescript/tests-ts/deep-scan-reducer-recovery.test.ts+++ b/sdk/typescript/tests-ts/deep-scan-reducer-recovery.test.ts@@ -137,6 +137,7 @@ test("classifies owned worker tool failures without exposing their contents", asbundledFunction(runtime, recordHelper!),bundledFunction(runtime, "isSandboxNamespaceExhaustion"),bundledFunction(runtime, "appendUniqueDiagnostic"),+ bundledFunction(runtime, "appendCodeModeFrameDiagnostic"),diagnosticSource,"return appendSafeItemDiagnostic;",].join("\n"),@@ -267,7 +268,11 @@ test("resumes only when the exact Standard worker or reducer result is missing",expect(standardContinuation(1)).toMatch(/retry.*until it succeeds/iu);const continuation = new Function(- `${bundledFunction(runtime, "reducerCompletionContinuation")}\nreturn reducerCompletionContinuation;`,+ [+ bundledFunction(runtime, "reducerInputRecoveryInstructions"),+ bundledFunction(runtime, "reducerCompletionContinuation"),+ "return reducerCompletionContinuation;",+ ].join("\n"),)() as (attempt: number) => string;expect(continuation(1)).toContain("record_codex_security_deep_reduction");expect(continuation(1)).toMatch(/retry.*until it succeeds/iu);