Skip to content

OpenAI's Codex Security CLI and TypeScript SDK for finding, validating, and fixing security vulnerabilities. npm: https://www.npmjs.com/package/@openai/codex-security

fix(sdk): keep bulk-scan knowledge and settings stable on resume (#1059)

mldangelo-oai00a3f1cd05a0
diff --git a/plugins/codex-security/mcp-app/tests/test_deep_scan_executor.ts b/plugins/codex-security/mcp-app/tests/test_deep_scan_executor.ts
index a3b5957d2..4ca2074d7 100644
--- a/plugins/codex-security/mcp-app/tests/test_deep_scan_executor.ts
+++ b/plugins/codex-security/mcp-app/tests/test_deep_scan_executor.ts
@@ -1273,6 +1273,7 @@ async function testWorkerRuntimeSettings() {
"CODEX_SECURITY_CONFIG_PATH",
"CODEX_SECURITY_PLUGIN_ROOT",
"CODEX_SECURITY_DEEP_SCAN_CONFIG_PATH",
+ "CODEX_SECURITY_KNOWLEDGE_BASE",
"OPENAI_API_KEY",
"CODEX_API_KEY",
"SYNTHETIC_GATEWAY_KEY",
@@ -1423,6 +1424,10 @@ env_key = "SYNTHETIC_GATEWAY_KEY"`,
},
};
return {
+ knowledgePath: path.join(fixture.root, `knowledge-${index}`),
+ knowledgeDocuments: {
+ "1-architecture.md.txt": `Synthetic architecture for scan ${index}.`,
+ },
path: entryPath,
deepPath,
nativeProfile,
@@ -1481,6 +1486,13 @@ env_key = "SYNTHETIC_GATEWAY_KEY"`,
await Promise.all(
workerConfigurations.map((entry) =>
Promise.all([
+ mkdir(entry.knowledgePath).then(() =>
+ Promise.all(
+ Object.entries(entry.knowledgeDocuments).map(([name, text]) =>
+ writeFile(path.join(entry.knowledgePath, name), text),
+ ),
+ ),
+ ),
writeFile(entry.path, stringifyToml(entry.configuration)),
writeFile(
entry.deepPath,
@@ -1589,6 +1601,8 @@ env_key = "SYNTHETIC_GATEWAY_KEY"`,
// Each concurrent launch snapshots its own scan environment.
process.env.CODEX_SECURITY_CONFIG_PATH =
workerConfigurations[index].path;
+ process.env.CODEX_SECURITY_KNOWLEDGE_BASE =
+ workerConfigurations[index].knowledgePath;
process.env.XDG_CACHE_HOME = path.join(
fixture.root,
`cache-${index} `,
@@ -1667,7 +1681,19 @@ env_key = "SYNTHETIC_GATEWAY_KEY"`,
workerConfigurations[index].permissionProfile,
},
);
+ assert.equal(
+ workerLaunch.environment!.CODEX_SECURITY_KNOWLEDGE_BASE,
+ workerConfigurations[index].knowledgePath,
+ );
const invocation = await readJson(workerLaunch.markerPath);
+ assert.equal(
+ invocation.knowledgePath,
+ workerConfigurations[index].knowledgePath,
+ );
+ assert.deepEqual(
+ invocation.knowledgeDocuments,
+ workerConfigurations[index].knowledgeDocuments,
+ );
assert.equal(invocation.codexHome, await realpath(codexHome));
assert.equal(invocation.providerKey, providerKeys[index]);
assert.equal(invocation.providerHeader, providerHeaders[index]);
@@ -2780,7 +2806,7 @@ async function fakeCodexFixture(
await writeFile(
scriptPath,
`#!/usr/bin/env node
-import { readFileSync, writeFileSync } from "node:fs";
+import { readFileSync, readdirSync, writeFileSync } from "node:fs";
import { spawnSync } from "node:child_process";
import { join } from "node:path";
const preflightProfile = process.env.FAKE_CODEX_PREFLIGHT_PROFILE ? JSON.parse(process.env.FAKE_CODEX_PREFLIGHT_PROFILE) : ${JSON.stringify(preflightProfile)};
@@ -2835,7 +2861,9 @@ const runtimeEnvironment = Object.fromEntries(['PATH', 'HOME', 'PYTHON', 'PYTHON
const pythonProbe = stdin.includes('CAPTURE_SYNTHETIC_PYTHON') ? spawnSync(process.env.PYTHON, ['-I', '-c', 'import json,os,sys; print(json.dumps([sys.prefix,os.environ.get("LD_LIBRARY_PATH")]))'], { encoding: 'utf8' }) : undefined;
if (pythonProbe && pythonProbe.status !== 0) throw new Error(pythonProbe.stderr || String(pythonProbe.error));
const pythonRuntime = pythonProbe ? JSON.parse(pythonProbe.stdout) : undefined;
-writeFileSync(process.env.FAKE_CODEX_MARKER, JSON.stringify({ argv: process.argv.slice(2), stdin, cwd: process.cwd(), codexHome: process.env.CODEX_HOME, cacheDirectory: process.env.XDG_CACHE_HOME, gitEnvironment: Object.fromEntries(['PATH', 'CODEX_SECURITY_GIT', 'GIT_SSH_COMMAND', 'GIT_CONFIG_GLOBAL'].map(name => [name, process.env[name]])), configPath: process.env.CODEX_SECURITY_CONFIG_PATH, deepConfigPath: process.env.CODEX_SECURITY_DEEP_SCAN_CONFIG_PATH, python: process.env.PYTHON, pythonPrefix: pythonRuntime?.[0], pythonLibraryPath: pythonRuntime?.[1], runtimeEnvironment, providerKey: process.env.SYNTHETIC_GATEWAY_KEY, providerHeader: process.env.SYNTHETIC_HEADER_VALUE, originator: process.env.CODEX_INTERNAL_ORIGINATOR_OVERRIDE, ...(profileContents === undefined ? {} : { profileContents }), ...(stdin.includes('COMPLETE_THEN_HANG') ? { pid: process.pid } : {}), ...(openaiAuthentication ? { openaiAuthentication } : {}), ...(bedrockAuthentication ? { bedrockAuthentication } : {}) }));
+const knowledgePath = stdin.includes('synthetic worker configuration fixture') ? process.env.CODEX_SECURITY_KNOWLEDGE_BASE : undefined;
+const knowledgeDocuments = knowledgePath === undefined ? undefined : Object.fromEntries(readdirSync(knowledgePath).map(name => [name, readFileSync(join(knowledgePath, name), 'utf8')]));
+writeFileSync(process.env.FAKE_CODEX_MARKER, JSON.stringify({ argv: process.argv.slice(2), stdin, cwd: process.cwd(), knowledgePath, knowledgeDocuments, codexHome: process.env.CODEX_HOME, cacheDirectory: process.env.XDG_CACHE_HOME, gitEnvironment: Object.fromEntries(['PATH', 'CODEX_SECURITY_GIT', 'GIT_SSH_COMMAND', 'GIT_CONFIG_GLOBAL'].map(name => [name, process.env[name]])), configPath: process.env.CODEX_SECURITY_CONFIG_PATH, deepConfigPath: process.env.CODEX_SECURITY_DEEP_SCAN_CONFIG_PATH, python: process.env.PYTHON, pythonPrefix: pythonRuntime?.[0], pythonLibraryPath: pythonRuntime?.[1], runtimeEnvironment, providerKey: process.env.SYNTHETIC_GATEWAY_KEY, providerHeader: process.env.SYNTHETIC_HEADER_VALUE, originator: process.env.CODEX_INTERNAL_ORIGINATOR_OVERRIDE, ...(profileContents === undefined ? {} : { profileContents }), ...(stdin.includes('COMPLETE_THEN_HANG') ? { pid: process.pid } : {}), ...(openaiAuthentication ? { openaiAuthentication } : {}), ...(bedrockAuthentication ? { bedrockAuthentication } : {}) }));
if (stdin.includes('COMPLETE_THEN_HANG')) process.on('SIGTERM', () => { if (!stdin.includes('IGNORE_TERMINATION')) setTimeout(() => process.exit(0), 100); });
if (stdin.includes('THREAD_START_CONFIG_ERROR')) { console.error('Error: thread/start: thread/start failed: agents.max_threads cannot be set when features.multi_agent_v2 is enabled (code -32600)'); process.exit(1); }
if (stdin.includes('CONFIG_ERROR')) { console.error('failed to load configuration: invalid value'); process.exit(2); }
diff --git a/sdk/typescript/docs/cli.md b/sdk/typescript/docs/cli.md
index afe62db69..d0b8b5a6c 100644
--- a/sdk/typescript/docs/cli.md
+++ b/sdk/typescript/docs/cli.md
@@ -666,7 +666,11 @@ Concurrency defaults to four repositories. `--max-attempts` defaults to one
attempt per pending repository per invocation. Repeating the command continues
the campaign, skips completed results, and starts pending attempts. Occupied
attempt directories stop that repository and suggest `--recover`.
-Changed project configuration requires a new output directory.
+Changes to project configuration, extracted knowledge-base text, staged document
+filenames, direct Codex overrides, or explicit `--plugin-path`/`--python` selections
+require a new output directory. Version 1 manifests also require a new directory
+because their original knowledge inputs and direct overrides cannot be verified.
+Worker and retry counts can change when resuming.
### Recovering failed or interrupted bulk scans
diff --git a/sdk/typescript/src/api.ts b/sdk/typescript/src/api.ts
index a2954e565..85f1f5ce3 100644
--- a/sdk/typescript/src/api.ts
+++ b/sdk/typescript/src/api.ts
@@ -135,6 +135,7 @@ import {
import {
prepareKnowledgeBase,
type PreparedKnowledgeBase,
+ type KnowledgeBaseSnapshot,
} from "./knowledge-base.js";
import { FindingWorkflow, workflowDigest } from "./finding-workflow.js";
import {
@@ -294,6 +295,8 @@ const DEEP_SCAN_CONFIG_PATH_ENVIRONMENT =
"CODEX_SECURITY_DEEP_SCAN_CONFIG_PATH";
export interface ScanOptions extends ScanSettings {
+ /** @internal Reuse the knowledge inputs bound to a bulk campaign manifest. */
+ knowledgeBaseSnapshot?: KnowledgeBaseSnapshot;
/** @internal Resume a CLI Deep Scan with its saved launch recipe. */
resumeScanId?: string;
/** Save synthetic Standard scan results without calling Codex or a model. */
@@ -790,7 +793,7 @@ export class CodexSecurity {
);
if (options.knowledgeBasePaths?.length) {
const knowledgeBase = await prepareKnowledgeBase(
- options.knowledgeBasePaths,
+ options.knowledgeBaseSnapshot ?? options.knowledgeBasePaths,
options.signal,
);
await knowledgeBase.cleanup();
@@ -1283,7 +1286,7 @@ export class CodexSecurity {
}
if (options.knowledgeBasePaths?.length) {
knowledgeBase = await prepareKnowledgeBase(
- options.knowledgeBasePaths,
+ options.knowledgeBaseSnapshot ?? options.knowledgeBasePaths,
signal,
);
}
@@ -1315,12 +1318,11 @@ export class CodexSecurity {
modelProvider,
),
};
- for (const source of [repo, ...(knowledgeBase?.sources ?? [])]) {
- git = await inspectTrustedExecutable(
- "git",
- git.environment,
- (await gitMarkerRoot(source, signal, "outermost")) ?? source,
- );
+ for (const root of [
+ (await gitMarkerRoot(repo, signal, "outermost")) ?? repo,
+ ...(knowledgeBase?.protectedRoots ?? []),
+ ]) {
+ git = await inspectTrustedExecutable("git", git.environment, root);
}
checkOpen();
const scanOutputRoot =
@@ -3198,7 +3200,7 @@ export class CodexSecurity {
});
if (options.knowledgeBasePaths?.length) {
const knowledgeBase = await prepareKnowledgeBase(
- options.knowledgeBasePaths,
+ options.knowledgeBaseSnapshot ?? options.knowledgeBasePaths,
signal,
);
await knowledgeBase.cleanup();
diff --git a/sdk/typescript/src/classify-severity.ts b/sdk/typescript/src/classify-severity.ts
index 2761df62f..85b49dbb7 100644
--- a/sdk/typescript/src/classify-severity.ts
+++ b/sdk/typescript/src/classify-severity.ts
@@ -1,10 +1,8 @@
import { findingEntry } from "./value.js";
-import { readFile, readdir } from "node:fs/promises";
-import { join } from "node:path";
import { z } from "incur";
import { CodexSecurityError } from "./errors.js";
import { workflowDigest } from "./finding-workflow.js";
-import { prepareKnowledgeBase } from "./knowledge-base.js";
+import { readKnowledgeBaseDocuments } from "./knowledge-base.js";
import type { Finding, SeverityLevel } from "./models.js";
import {
runReadOnlyCodex,
@@ -235,23 +233,11 @@ async function readDocuments(
paths: readonly string[],
signal?: AbortSignal,
): Promise<string[]> {
- const prepared = await prepareKnowledgeBase(paths, signal);
- try {
- const files = (await readdir(prepared.path)).sort();
- const contents = await Promise.all(
- files.map((file) =>
- readFile(join(prepared.path, file), { encoding: "utf8", signal }),
- ),
- );
- if (contents.every((text) => !text.trim())) {
- throw new CodexSecurityError(
- "Classification documents must not be empty.",
- );
- }
- return contents;
- } finally {
- await prepared.cleanup();
+ const contents = await readKnowledgeBaseDocuments(paths, signal);
+ if (contents.every((text) => !text.trim())) {
+ throw new CodexSecurityError("Classification documents must not be empty.");
}
+ return contents;
}
/** @internal Check parsed assessments against the actual finding evidence. */
diff --git a/sdk/typescript/src/cli.ts b/sdk/typescript/src/cli.ts
index de2772ed2..dadb903f6 100644
--- a/sdk/typescript/src/cli.ts
+++ b/sdk/typescript/src/cli.ts
@@ -4594,6 +4594,7 @@ export async function main(
try {
return await security.run(recipe.repository!, {
...pickScanSettings(recipe),
+ knowledgeBaseSnapshot: prompts.knowledgeBaseSnapshot,
resumeScanId: scan.scanId,
outputDir: scanDir,
safetyIdentifier: recipe.safetyIdentifier,
diff --git a/sdk/typescript/src/knowledge-base.ts b/sdk/typescript/src/knowledge-base.ts
index a8eb691a4..83534f112 100644
--- a/sdk/typescript/src/knowledge-base.ts
+++ b/sdk/typescript/src/knowledge-base.ts
@@ -13,7 +13,11 @@ import { tmpdir } from "node:os";
import { basename, extname, join, resolve } from "node:path";
import { unzipSync } from "fflate";
import { expandHome } from "./runtime.js";
-import { isGitMetadataDirectory, nullIfMissingFile } from "./targets.js";
+import {
+ gitMarkerRoot,
+ isGitMetadataDirectory,
+ nullIfMissingFile,
+} from "./targets.js";
const DOCUMENT_EXTENSIONS = new Set([
".md",
@@ -26,14 +30,21 @@ const DOCUMENT_EXTENSIONS = new Set([
export interface PreparedKnowledgeBase {
path: string;
sources: string[];
+ protectedRoots: string[];
cleanup(): Promise<void>;
}
-export async function prepareKnowledgeBase(
+export interface KnowledgeBaseSnapshot {
+ readonly sources: readonly string[];
+ readonly protectedRoots?: readonly string[];
+ readonly documents: Readonly<Record<string, string>>;
+}
+
+/** @internal Extract once so campaign identity and workers use identical inputs. */
+export async function readKnowledgeBaseSnapshot(
paths: readonly string[],
signal?: AbortSignal,
- directory?: string,
-): Promise<PreparedKnowledgeBase> {
+): Promise<KnowledgeBaseSnapshot> {
const sources = new Set<string>();
const documents = new Set<string>();
@@ -54,7 +65,7 @@ export async function prepareKnowledgeBase(
const source = await realpath(path);
const selected = metadata.isDirectory()
- ? await discover(source, signal)
+ ? (await discover(source, signal)).sort()
: [source];
if (selected.length === 0) {
throw new Error(
@@ -67,55 +78,101 @@ export async function prepareKnowledgeBase(
sources.add(source);
}
+ const extracted: Record<string, string> = {};
+ let index = 0;
+ for (const document of documents) {
+ signal?.throwIfAborted();
+ const metadata = await lstat(document);
+ if (process.platform !== "win32" && (metadata.mode & 0o444) === 0) {
+ throw new Error(`Knowledge base document is not readable: ${document}`);
+ }
+ const bytes = await readFile(document, {
+ flag: constants.O_RDONLY | (constants.O_NOFOLLOW ?? 0),
+ signal,
+ });
+ const extension = extname(document).toLowerCase();
+ const text =
+ extension === ".pdf"
+ ? await extractPdf(document, bytes)
+ : extension === ".docx"
+ ? extractDocx(document, bytes)
+ : decodeText(document, bytes);
+ if ((extension === ".pdf" || extension === ".docx") && !text.trim()) {
+ throw new Error(
+ `Knowledge base document contains no extractable text: ${document}`,
+ );
+ }
+ const name = `${index}-${basename(document)}.txt`;
+ // The prefix and suffix can exceed the filesystem's 255-byte name limit.
+ const filename = Buffer.byteLength(name) > 255 ? `${index}.txt` : name;
+ extracted[filename] = text;
+ index++;
+ }
+ return {
+ sources: [...sources],
+ protectedRoots: await Promise.all(
+ [...sources].map(
+ async (source) =>
+ (await gitMarkerRoot(source, signal, "outermost")) ?? source,
+ ),
+ ),
+ documents: extracted,
+ };
+}
+
+export async function prepareKnowledgeBase(
+ input: readonly string[] | KnowledgeBaseSnapshot,
+ signal?: AbortSignal,
+ directory?: string,
+): Promise<PreparedKnowledgeBase> {
+ const snapshot =
+ "documents" in input
+ ? input
+ : await readKnowledgeBaseSnapshot(input, signal);
+ const protectedRoots =
+ snapshot.protectedRoots === undefined
+ ? await Promise.all(
+ snapshot.sources.map(
+ async (source) =>
+ (await gitMarkerRoot(source, signal, "outermost")) ?? source,
+ ),
+ )
+ : [...snapshot.protectedRoots];
const path = await mkdtemp(
join(directory ?? tmpdir(), "codex-security-knowledge-"),
);
try {
- let index = 0;
- for (const document of documents) {
+ for (const [filename, text] of Object.entries(snapshot.documents)) {
signal?.throwIfAborted();
- const metadata = await lstat(document);
- if (process.platform !== "win32" && (metadata.mode & 0o444) === 0) {
- throw new Error(`Knowledge base document is not readable: ${document}`);
- }
- const bytes = await readFile(document, {
- flag: constants.O_RDONLY | (constants.O_NOFOLLOW ?? 0),
- signal,
- });
- const extension = extname(document).toLowerCase();
- const text =
- extension === ".pdf"
- ? await extractPdf(document, bytes)
- : extension === ".docx"
- ? extractDocx(document, bytes)
- : decodeText(document, bytes);
- if ((extension === ".pdf" || extension === ".docx") && !text.trim()) {
- throw new Error(
- `Knowledge base document contains no extractable text: ${document}`,
- );
- }
- const name = `${index}-${basename(document)}.txt`;
- // The prefix and suffix can exceed the filesystem's 255-byte name limit.
- const filename = Buffer.byteLength(name) > 255 ? `${index}.txt` : name;
await writeFile(join(path, filename), text, {
encoding: "utf8",
mode: 0o600,
signal,
});
- index++;
}
} catch (error) {
await rm(path, { recursive: true, force: true });
throw error;
}
-
return {
path,
- sources: [...sources],
+ sources: [...snapshot.sources],
+ protectedRoots,
cleanup: () => rm(path, { recursive: true, force: true }),
};
}
+/** @internal Read the same extracted document text used by scans. */
+export async function readKnowledgeBaseDocuments(
+ paths: readonly string[],
+ signal?: AbortSignal,
+): Promise<string[]> {
+ const { documents } = await readKnowledgeBaseSnapshot(paths, signal);
+ return Object.keys(documents)
+ .sort()
+ .map((name) => documents[name]!);
+}
+
async function discover(
directory: string,
signal?: AbortSignal,
diff --git a/sdk/typescript/src/multiscan.ts b/sdk/typescript/src/multiscan.ts
index 215281f03..fe2c94cf7 100644
--- a/sdk/typescript/src/multiscan.ts
+++ b/sdk/typescript/src/multiscan.ts
@@ -29,6 +29,10 @@ import {
ScanCostLimitExceededError,
} from "./errors.js";
import type { CoverageDocument } from "./models.js";
+import {
+ readKnowledgeBaseSnapshot,
+ type KnowledgeBaseSnapshot,
+} from "./knowledge-base.js";
import { resolveScanPrompts } from "./prompt-files.js";
import { requireSecureOutputAncestry, validateOutputDir } from "./runtime.js";
import { DiffTarget, type ScanMode } from "./targets.js";
@@ -73,8 +77,22 @@ interface MultiscanReceipt extends MultiscanTask {
warning?: string;
warnings?: string[];
policyFailed?: boolean;
+ knowledgeBaseFailure?: true;
+}
+
+interface MultiscanHistory {
+ maxAttempt: number;
+ scan?: MultiscanReceipt;
}
+type MultiscanKnowledge = Partial<
+ Record<
+ ScanMode,
+ | { snapshot: KnowledgeBaseSnapshot; failure?: never }
+ | { snapshot?: never; failure: { error: unknown } }
+ >
+>;
+
export interface MultiscanOptions extends ScanPromptSettings {
inputPath: string;
outputDir: string;
@@ -85,7 +103,9 @@ export interface MultiscanOptions extends ScanPromptSettings {
maxAttempts: number;
recoverScan?(
scanDir: string,
- prompts: ScanPromptSettings,
+ prompts: ScanPromptSettings & {
+ knowledgeBaseSnapshot?: KnowledgeBaseSnapshot;
+ },
): Promise<Pick<ScanResult, "coverage" | "cost" | "findings"> | undefined>;
maxCostUsd?: number;
// Prompts are shared across modes and prepared from the top-level options.
@@ -214,7 +234,29 @@ async function runCampaign(
const ledger = join(output, "results.jsonl");
await ensureOutputDirectory(join(output, "checkouts"));
await ensureOutputDirectory(join(output, "artifacts"));
- await ensureManifest(join(output, "manifest.json"), tasks, options);
+ const knowledgeByMode: MultiscanKnowledge = {};
+ const sharedKnowledge = options.knowledgeBasePaths?.length
+ ? readKnowledgeBaseSnapshot(options.knowledgeBasePaths, options.signal)
+ : undefined;
+ for (const mode of new Set(tasks.map((task) => task.mode))) {
+ try {
+ const paths = options.scanOptionsByMode?.[mode]?.knowledgeBasePaths;
+ const snapshot = await (sharedKnowledge ??
+ (paths?.length
+ ? readKnowledgeBaseSnapshot(paths, options.signal)
+ : undefined));
+ if (snapshot !== undefined) knowledgeByMode[mode] = { snapshot };
+ } catch (error) {
+ options.signal?.throwIfAborted();
+ knowledgeByMode[mode] = { failure: { error } };
+ }
+ }
+ await ensureManifest(
+ join(output, "manifest.json"),
+ tasks,
+ options,
+ knowledgeByMode,
+ );
const receipts = await readReceipts(
ledger,
options.recoverScan !== undefined,
@@ -229,9 +271,13 @@ async function runCampaign(
);
let untouched = 0;
for (const task of tasks) {
- const receipt = receipts.get(task.id.toLowerCase());
- if (receipt === undefined) {
- if (options.recoverScan !== undefined) {
+ const history = receipts.get(task.id.toLowerCase());
+ const receipt = history?.scan;
+ if (
+ receipt === undefined ||
+ knowledgeByMode[task.mode]?.failure !== undefined
+ ) {
+ if (history === undefined && options.recoverScan !== undefined) {
const attempts = await readdir(
join(output, "artifacts", task.id),
).catch((error: NodeJS.ErrnoException) => {
@@ -323,18 +369,26 @@ async function runCampaign(
options.signal?.throwIfAborted();
const task = pending[next++];
if (task === undefined) return;
- let attempt = receipts.get(task.id.toLowerCase())?.attempt ?? 0;
+ const knowledge = knowledgeByMode[task.mode];
+ const history = receipts.get(task.id.toLowerCase());
+ let maxAttempt = history?.maxAttempt ?? 0;
+ let recoveryAttempt = history?.scan?.attempt ?? 0;
const artifactRoot = join(output, "artifacts", task.id);
if (options.recoverScan !== undefined) {
await ensureOutputDirectory(artifactRoot);
for (const name of await readdir(artifactRoot)) {
- const prior = Number(/^attempt-([1-9][0-9]*)$/u.exec(name)?.[1]);
- if (Number.isSafeInteger(prior)) attempt = Math.max(attempt, prior);
+ const match = /^attempt-([1-9][0-9]*)$/u.exec(name);
+ if (match && Number.isSafeInteger(Number(match[1]))) {
+ maxAttempt = Math.max(maxAttempt, Number(match[1]));
+ recoveryAttempt = Math.max(recoveryAttempt, Number(match[1]));
+ }
}
}
for (let retry = 0; retry < options.maxAttempts; retry += 1) {
options.signal?.throwIfAborted();
- if (options.recoverScan === undefined) attempt += 1;
+ let attempt = maxAttempt;
+ if (options.recoverScan === undefined) attempt = ++maxAttempt;
+ else if (retry === 0) attempt = recoveryAttempt;
let scanDir = join(artifactRoot, `attempt-${attempt}`);
let checkout: string | undefined;
let protectedRoot = join(output, "checkouts", task.id);
@@ -348,6 +402,7 @@ async function runCampaign(
let threatModelPath: string | null | undefined;
let exhaustedBudget = false;
let requiresRecovery = false;
+ let knowledgeBaseFailure = false;
try {
await ensureOutputDirectory(artifactRoot);
let result:
@@ -377,19 +432,32 @@ async function runCampaign(
attempt,
status: "started",
});
- result = await options.recoverScan(scanDir, options);
+ if (knowledge?.failure !== undefined) {
+ knowledgeBaseFailure = true;
+ throw knowledge.failure.error;
+ }
+ result = await options.recoverScan(scanDir, {
+ ...options,
+ knowledgeBaseSnapshot: knowledge?.snapshot,
+ });
attemptedResume = result !== undefined;
}
}
const scanSettings = options.scanOptionsByMode?.[task.mode];
if (result === undefined) {
- if (options.recoverScan !== undefined) attempt += 1;
+ if (options.recoverScan !== undefined) attempt = ++maxAttempt;
scanDir = join(artifactRoot, `attempt-${attempt}`);
notifyProgress(options, {
repository: task.id,
attempt,
status: "started",
});
+ if (options.recoverScan === undefined)
+ await validateOutputDir(scanDir);
+ if (knowledge?.failure !== undefined) {
+ knowledgeBaseFailure = true;
+ throw knowledge.failure.error;
+ }
if (options.recoverScan !== undefined) {
const checkoutRoot = await ensureOutputDirectory(
join(output, "recovery-checkouts"),
@@ -402,7 +470,6 @@ async function runCampaign(
await mkdir(scanDir, { mode: 0o700 });
await mkdir(checkout, { mode: 0o700 });
} else {
- await validateOutputDir(scanDir);
checkout = join(output, "checkouts", task.id);
await rm(checkout, { recursive: true, force: true });
await mkdir(checkout, { mode: 0o700 });
@@ -430,6 +497,7 @@ async function runCampaign(
.join("\n\n");
result = await security.run(checkout, {
...scanSettings,
+ knowledgeBaseSnapshot: knowledge?.snapshot,
...(task.scope === undefined ? {} : { target: [task.scope] }),
...(options.knowledgeBasePaths?.length
? { knowledgeBasePaths: options.knowledgeBasePaths }
@@ -515,6 +583,7 @@ async function runCampaign(
...(coverage === undefined ? {} : { coverage }),
...(cost === null ? {} : { cost }),
...(failure === undefined ? {} : { error: failure }),
+ ...(knowledgeBaseFailure ? { knowledgeBaseFailure: true } : {}),
...(warning === undefined ? {} : { warning }),
...(runWarnings.length === 0 ? {} : { warnings: runWarnings }),
...(attemptPolicyFailed === undefined
@@ -846,35 +915,89 @@ async function ensureManifest(
| "scanOptionsByMode"
| "config"
>,
+ knowledgeByMode: MultiscanKnowledge,
): Promise<void> {
- const expected = `${JSON.stringify(
- {
- version: 1,
- tasks,
- scanPrompt: options.scanPrompt,
- validationPrompt: options.validationPrompt,
- postScanPrompt: options.postScanPrompt,
- maxCostUsd: options.maxCostUsd,
- ...(options.scanOptionsByMode === undefined
- ? {}
- : {
- configurationDigest: workflowDigest({
- scanOptions: options.scanOptionsByMode,
- codex: options.config.codexOverrides,
- }),
+ const knowledgeDigests: Partial<Record<ScanMode, string | null>> =
+ Object.fromEntries(
+ Object.entries(knowledgeByMode).map(([mode, knowledge]) => [
+ mode,
+ knowledge.snapshot === undefined
+ ? null
+ : workflowDigest(knowledge.snapshot.documents),
+ ]),
+ );
+ const manifest = {
+ version: 2,
+ tasks,
+ scanPrompt: options.scanPrompt,
+ validationPrompt: options.validationPrompt,
+ postScanPrompt: options.postScanPrompt,
+ maxCostUsd: options.maxCostUsd,
+ // Failed modes cannot start scans; bind their inputs only after repair.
+ ...(Object.keys(knowledgeDigests).length > 0
+ ? { knowledgeBaseDigests: knowledgeDigests }
+ : {}),
+ ...(options.scanOptionsByMode === undefined &&
+ Object.keys(options.config.codexOverrides ?? {}).length === 0 &&
+ options.config.pluginPath === undefined &&
+ options.config.pythonPath === undefined
+ ? {}
+ : {
+ configurationDigest: workflowDigest({
+ scanOptions: options.scanOptionsByMode,
+ codex: options.config.codexOverrides,
+ pluginPath: options.config.pluginPath,
+ pythonPath: options.config.pythonPath,
}),
- },
- null,
- 2,
- )}\n`;
+ }),
+ };
+ const expected = `${JSON.stringify(manifest, null, 2)}\n`;
try {
await writeFile(path, expected, { flag: "wx", mode: 0o600 });
} catch (error) {
if ((error as NodeJS.ErrnoException).code !== "EEXIST") throw error;
- if ((await readFile(path, "utf8")) !== expected) {
+ const existing = await readFile(path, "utf8");
+ if (existing === expected) return;
+ const { knowledgeBaseDigests: savedDigests = {}, ...savedInputs } =
+ JSON.parse(existing) as Record<string, unknown> & {
+ knowledgeBaseDigests?: Partial<Record<ScanMode, string | null>>;
+ };
+ const { knowledgeBaseDigests: _, ...expectedInputs } = manifest;
+ const mismatch = (): never => {
throw new Error(
"Multiscan manifest does not match existing output directory.",
);
+ };
+ if (JSON.stringify(savedInputs) !== JSON.stringify(expectedInputs))
+ mismatch();
+ const boundDigests = { ...savedDigests };
+ for (const mode of Object.keys({
+ ...savedDigests,
+ ...knowledgeDigests,
+ }) as ScanMode[]) {
+ const saved = savedDigests[mode];
+ const current = knowledgeDigests[mode];
+ if (
+ saved === undefined ||
+ current === undefined ||
+ (saved !== null && current !== null && saved !== current)
+ ) {
+ mismatch();
+ }
+ boundDigests[mode] = saved ?? current;
+ }
+ // Preserve bound modes when their inputs are temporarily unavailable.
+ if (Object.keys(boundDigests).length > 0)
+ manifest.knowledgeBaseDigests = boundDigests;
+ const bound = `${JSON.stringify(manifest, null, 2)}\n`;
+ if (bound !== existing) {
+ const temporary = `${path}.${randomUUID()}.tmp`;
+ try {
+ await writeFile(temporary, bound, { flag: "wx", mode: 0o600 });
+ await rename(temporary, path);
+ } finally {
+ await rm(temporary, { force: true });
+ }
}
}
}
@@ -882,7 +1005,7 @@ async function ensureManifest(
async function readReceipts(
path: string,
preserveInterrupted = false,
-): Promise<Map<string, MultiscanReceipt>> {
+): Promise<Map<string, MultiscanHistory>> {
let contents: string;
try {
contents = await readFile(path, "utf8");
@@ -904,12 +1027,18 @@ async function readReceipts(
Buffer.byteLength(contents) - Buffer.byteLength(partial),
);
}
- return new Map(
- lines.filter(Boolean).map((line): [string, MultiscanReceipt] => {
- const receipt = JSON.parse(line) as MultiscanReceipt;
- return [receipt.id.toLowerCase(), receipt];
- }),
- );
+ const receipts = new Map<string, MultiscanHistory>();
+ for (const line of lines.filter(Boolean)) {
+ const receipt = JSON.parse(line) as MultiscanReceipt;
+ const id = receipt.id.toLowerCase();
+ const previous = receipts.get(id);
+ receipts.set(id, {
+ maxAttempt: Math.max(previous?.maxAttempt ?? 0, receipt.attempt),
+ // Extraction failures never start or modify a scan; keep its last receipt.
+ scan: receipt.knowledgeBaseFailure ? previous?.scan : receipt,
+ });
+ }
+ return receipts;
}
async function hasArtifacts(path: string): Promise<boolean> {
diff --git a/sdk/typescript/tests-ts/cli.test.ts b/sdk/typescript/tests-ts/cli.test.ts
index 47363b5d3..ec4bcbd0f 100644
--- a/sdk/typescript/tests-ts/cli.test.ts
+++ b/sdk/typescript/tests-ts/cli.test.ts
@@ -653,12 +653,17 @@ describe("CLI", () => {
test("runs a bulk scan and keeps structured output on stdout", async () => {
const root = await temporaryDirectory("codex-security-cli-multiscan-");
- const architecturePath = resolve(root, "/shared/architecture.pdf");
- const threatModelsPath = resolve(root, "/shared/threat-models");
+ const architecturePath = join(root, "architecture.md");
+ const threatModelsPath = join(root, "threat-models");
try {
await multiscanInventory(root);
+ await writeFile(architecturePath, "Synthetic architecture context.");
+ await mkdir(threatModelsPath);
+ await writeFile(
+ join(threatModelsPath, "model.md"),
+ "Synthetic threat model.",
+ );
const { stdout, stderr, runCli } = createCliTest(main);
-
let config: CodexSecurityConfig | undefined;
let scanOptions: unknown;
expect(
diff --git a/sdk/typescript/tests-ts/knowledge-base.test.ts b/sdk/typescript/tests-ts/knowledge-base.test.ts
index 68de78b6b..4435fb0bc 100644
--- a/sdk/typescript/tests-ts/knowledge-base.test.ts
+++ b/sdk/typescript/tests-ts/knowledge-base.test.ts
@@ -249,11 +249,15 @@ describe("scan knowledge bases", () => {
await writeFile(join(root, "ignored.bin"), new Uint8Array([0, 1, 2]));
await writeFile(join(root, "invalid-utf8.bin"), new Uint8Array([0xff]));
- const knowledgeBase = await prepareKnowledgeBase([root, scope, scope]);
+ const knowledgeBase = await prepareKnowledgeBase([scope, root, scope]);
temporaryDirectories.track(knowledgeBase.path);
- expect(knowledgeBase.sources).toEqual([root, scope]);
- expect((await readdir(knowledgeBase.path)).length).toBe(3);
+ expect(knowledgeBase.sources).toEqual([scope, root]);
+ expect((await readdir(knowledgeBase.path)).sort()).toEqual([
+ "0-scope.md.txt",
+ "1-deployment.MARKDOWN.txt",
+ "2-notes.txt.txt",
+ ]);
const documents = await extractedDocuments(knowledgeBase.path);
expect(documents).toContain("Ignore local debug endpoints.");
expect(documents).toContain("Public API gateway.");
@@ -400,22 +404,24 @@ describe("scan knowledge bases", () => {
const controller = new AbortController();
const reason = new Error("Knowledge-base preparation canceled.");
- let checks = 0;
- const signalSpy = spyOn(controller.signal, "throwIfAborted");
- signalSpy.mockImplementation(() => {
- if (++checks === 4) controller.abort(reason);
- if (controller.signal.aborted) throw controller.signal.reason;
- });
- const temporarySpy = spyOn(os, "tmpdir").mockImplementation(() => staging);
+ const originalWriteFile = filesystem.writeFile;
+ let staged = false;
+ const writeSpy = spyOn(filesystem, "writeFile").mockImplementation(
+ async (...args) => {
+ await Reflect.apply(originalWriteFile, filesystem, args);
+ staged = true;
+ controller.abort(reason);
+ },
+ );
try {
await expect(
- prepareKnowledgeBase([first, second], controller.signal),
+ prepareKnowledgeBase([first, second], controller.signal, staging),
).rejects.toBe(reason);
+ expect(staged).toBe(true);
expect(await readdir(staging)).toEqual([]);
} finally {
- signalSpy.mockRestore();
- temporarySpy.mockRestore();
+ writeSpy.mockRestore();
}
});
diff --git a/sdk/typescript/tests-ts/multiscan.test.ts b/sdk/typescript/tests-ts/multiscan.test.ts
index 0c79cac2c..2abfe5a5a 100644
--- a/sdk/typescript/tests-ts/multiscan.test.ts
+++ b/sdk/typescript/tests-ts/multiscan.test.ts
@@ -32,6 +32,9 @@ import { buildGitHubCredentialArgs, runMultiscan } from "../src/multiscan.js";
import { resolveTrustedExecutable } from "../src/trusted-executable.js";
import { DiffTarget } from "../src/targets.js";
import { prepareOutputDir } from "../src/runtime.js";
+import { workflowDigest } from "../src/finding-workflow.js";
+import { TestClient, mockWorkbench } from "./support/api-client.js";
+import { preparedRuntime } from "./support/api-events.js";
import * as runtime from "../src/runtime.js";
import { capture, dependencies, fakeResult } from "./cli-fixtures.js";
import { runTestInSubprocess } from "./support/test-subprocess.js";
@@ -335,58 +338,101 @@ describe("multiscan", () => {
);
});
- test("occupied bulk attempts preserve the original checkout and recommend bulk recovery", async () => {
- const { paths } = await repositoryFixture("occupied", "repo");
- const scanDir = join(paths.output, "artifacts", "repo", "attempt-1");
- const checkout = join(paths.output, "checkouts", "repo");
- await mkdir(scanDir, { recursive: true, mode: 0o700 });
- await mkdir(checkout, { recursive: true });
- await writeFile(join(scanDir, "checkpoint"), "keep");
- await writeFile(join(checkout, "source"), "keep checkout");
- const error = capture();
- const output = capture();
- const deps = dependencies();
- const code = await main(
- [
+ test.each([false, true])(
+ "occupied bulk attempts preserve the checkout with missing knowledge=%p",
+ async (missingKnowledge) => {
+ const { paths } = await repositoryFixture("occupied", "repo");
+ const document = join(paths.root, "context.md");
+ await writeFile(document, "Original context.");
+ const scanDir = join(paths.output, "artifacts", "repo", "attempt-1");
+ const checkout = join(paths.output, "checkouts", "repo");
+ const arguments_ = [
"bulk-scan",
paths.input,
"--output-dir",
paths.output,
- "--max-attempts",
- "3",
+ "--knowledge-base",
+ document,
"--json",
- ],
- output.stream,
- error.stream,
- {
+ ];
+ const deps = dependencies();
+ let initializing = true;
+ let scans = 0;
+ const configured: typeof deps = {
...deps,
createSecurity: (config) => ({
...deps.createSecurity(config),
- run: async (_repo, scan = {}) => {
+ run: async (repo, scan = {}) => {
+ if (initializing) throw new Error("Synthetic interrupted setup");
+ scans++;
+ expect(repo).not.toBe(checkout);
await prepareOutputDir(scan.outputDir, "repo");
return completedScan(scan.outputDir!);
},
}),
- },
- );
- expect(code).toBe(2);
- expect(error.text()).toContain("--recover");
- expect(error.text()).not.toContain("--archive-existing");
- const receipts = await results(JSON.parse(output.text()).resultsPath);
- expect(receipts).toHaveLength(1);
- expect(receipts[0]).toMatchObject({
- status: "failed",
- attempt: 1,
- error: expect.stringContaining("--recover"),
- });
- expect(await readFile(join(checkout, "source"), "utf8")).toBe(
- "keep checkout",
- );
- expect(await readFile(join(scanDir, "checkpoint"), "utf8")).toBe("keep");
- await expect(prepareOutputDir(scanDir, "repo")).rejects.toThrow(
- "--archive-existing",
- );
- });
+ };
+ await main(arguments_, capture().stream, capture().stream, configured);
+ initializing = false;
+ await rm(join(paths.output, "results.jsonl"));
+ await mkdir(scanDir, { recursive: true, mode: 0o700 });
+ await mkdir(checkout, { recursive: true });
+ await writeFile(join(scanDir, "checkpoint"), "keep");
+ await writeFile(join(checkout, "source"), "keep checkout");
+ if (missingKnowledge) await rm(document);
+ const error = capture();
+ const output = capture();
+ const code = await main(
+ [...arguments_, "--max-attempts", "3"],
+ output.stream,
+ error.stream,
+ configured,
+ );
+ expect(code).toBe(2);
+ expect(error.text()).toContain("--recover");
+ expect(error.text()).not.toContain("--archive-existing");
+ expect(scans).toBe(0);
+ const receipts = await results(JSON.parse(output.text()).resultsPath);
+ expect(receipts).toHaveLength(1);
+ expect(receipts[0]).toMatchObject({
+ status: "failed",
+ attempt: 1,
+ error: expect.stringContaining("--recover"),
+ });
+ expect(receipts[0]!["knowledgeBaseFailure"]).toBeUndefined();
+ expect(await readFile(join(checkout, "source"), "utf8")).toBe(
+ "keep checkout",
+ );
+ expect(await readFile(join(scanDir, "checkpoint"), "utf8")).toBe("keep");
+ await expect(prepareOutputDir(scanDir, "repo")).rejects.toThrow(
+ "--archive-existing",
+ );
+ await writeFile(document, "Original context.");
+ const recovered = capture();
+ expect(
+ await main(
+ [...arguments_, "--recover"],
+ recovered.stream,
+ capture().stream,
+ configured,
+ ),
+ ).toBe(0);
+ expect(JSON.parse(recovered.text())).toMatchObject({
+ completed: 1,
+ failed: 0,
+ });
+ expect(scans).toBe(1);
+ expect(
+ (await results(JSON.parse(recovered.text()).resultsPath)).at(-1),
+ ).toMatchObject({
+ status: "completed",
+ attempt: 2,
+ });
+ expect(await readFile(join(checkout, "source"), "utf8")).toBe(
+ "keep checkout",
+ );
+ expect(await readFile(join(scanDir, "checkpoint"), "utf8")).toBe("keep");
+ },
+ );
test("CLI escapes bulk failure controls while preserving the saved receipt", async () => {
const { paths } = await repositoryFixture("failure", "repo");
@@ -439,7 +485,7 @@ describe("multiscan", () => {
await mkdir(paths.output);
await writeFile(
join(paths.output, "manifest.json"),
- JSON.stringify({ version: 1, tasks }, null, 2) + "\n",
+ JSON.stringify({ version: 2, tasks }, null, 2) + "\n",
);
const tail = '{"id":"failed","status":';
const original =
@@ -538,6 +584,98 @@ describe("multiscan", () => {
});
});
+ test.each([false, true])(
+ "recovery records the original attempt and preserves it after resume failure=%p",
+ async (failure) => {
+ const paths = await fixture();
+ const source = await repository(paths.root, "retained");
+ await writeFile(
+ paths.input,
+ `id,repository,revision\nretained,${source.path},${source.revision}\n`,
+ );
+ const document = join(paths.root, "context.md");
+ await writeFile(document, "Synthetic context.");
+ const knowledgeBasePaths = [document];
+ await runMultiscan(
+ options(
+ paths,
+ client(async () => {
+ throw new Error("Stopped");
+ }),
+ { maxAttempts: 1, knowledgeBasePaths },
+ ),
+ );
+ const dir = join(paths.output, "artifacts", "retained", "attempt-1");
+ await mkdir(dir, { recursive: true });
+ await writeFile(join(dir, "checkpoint"), "keep");
+ let resumed = 0;
+ let runs = 0;
+ const configured = options(
+ paths,
+ client(async (_repo, scan = {}) => {
+ runs++;
+ return completedScan(scan.outputDir!);
+ }),
+ {
+ maxAttempts: 3,
+ knowledgeBasePaths,
+ recoverScan: async (scanDir) => {
+ resumed++;
+ expect(scanDir).toBe(dir);
+ if (failure) throw new Error("Resume transport failed");
+ return completedScan(scanDir);
+ },
+ },
+ );
+ await rm(document);
+ expect(
+ await runMultiscan({
+ ...configured,
+ recoverScan: undefined,
+ maxAttempts: 2,
+ }),
+ ).toMatchObject({ completed: 0, failed: 1 });
+ expect(
+ (await results(join(paths.output, "results.jsonl"))).at(-1),
+ ).toMatchObject({ status: "failed", attempt: 3 });
+ expect(await runMultiscan(configured)).toMatchObject({
+ completed: 0,
+ failed: 1,
+ });
+ expect(resumed).toBe(0);
+ expect(
+ (await results(join(paths.output, "results.jsonl"))).at(-1),
+ ).toMatchObject({
+ status: "failed",
+ attempt: 1,
+ });
+ await writeFile(document, "Synthetic context.");
+ const summary = await runMultiscan(configured);
+ expect(runs).toBe(0);
+ expect(resumed).toBe(1);
+ expect(summary.failed).toBe(failure ? 1 : 0);
+ expect(summary.completed).toBe(failure ? 0 : 1);
+ expect((await results(summary.resultsPath)).at(-1)).toMatchObject({
+ attempt: 1,
+ outputDir: dir,
+ status: failure ? "failed" : "completed",
+ });
+ expect(await readFile(join(dir, "checkpoint"), "utf8")).toBe("keep");
+ if (failure) {
+ const retried = await runMultiscan({
+ ...configured,
+ recoverScan: undefined,
+ maxAttempts: 1,
+ });
+ expect(retried).toMatchObject({ completed: 1, failed: 0 });
+ expect((await results(retried.resultsPath)).at(-1)).toMatchObject({
+ status: "completed",
+ attempt: 4,
+ });
+ }
+ },
+ );
+
test.each([
[false, "checkouts"],
[true, "checkouts"],
@@ -548,7 +686,10 @@ describe("multiscan", () => {
async (failure, layout) => {
const { paths } = await repositoryFixture("retained");
await runMultiscan(
- options(paths, client(rejecting("Stopped")), { maxAttempts: 1 }),
+ options(paths, client(rejecting("Stopped")), {
+ maxAttempts: 1,
+ config: { pythonPath: PYTHON },
+ }),
);
const dir = join(paths.output, "artifacts", "retained", "attempt-1");
await mkdir(dir, { recursive: true });
@@ -950,6 +1091,9 @@ describe("multiscan", () => {
"retains sealed %s coverage without retries or multiplied costs",
async (completeness) => {
const { paths } = await repositoryFixture(completeness, "sealed");
+ const document = join(paths.root, "context.md");
+ await writeFile(document, "Original context.");
+ const knowledgeBasePaths = [document];
const cost = {
model: "gpt-5.6-sol",
inputTokens: 1_250,
@@ -973,6 +1117,7 @@ describe("multiscan", () => {
const summary = await runMultiscan(
options(paths, security, {
maxAttempts: 3,
+ knowledgeBasePaths,
onProgress: (event) => progress.push(event),
}),
);
@@ -1027,6 +1172,7 @@ describe("multiscan", () => {
const resumed = await runMultiscan(
options(paths, security, {
maxAttempts: 3,
+ knowledgeBasePaths,
onProgress: throwing("Optional progress observer failed."),
}),
);
@@ -1038,6 +1184,25 @@ describe("multiscan", () => {
});
expect(attempts).toBe(1);
expect(await results(resumed.resultsPath)).toHaveLength(1);
+ await rm(document);
+ expect(
+ await runMultiscan(options(paths, security, { knowledgeBasePaths })),
+ ).toMatchObject({ completed: 0, incomplete: 0, failed: 1 });
+ await writeFile(document, "Original context.");
+ expect(
+ await runMultiscan(options(paths, security, { knowledgeBasePaths })),
+ ).toMatchObject({ incomplete: 1, failed: 0, skipped: 1 });
+ expect(attempts).toBe(1);
+ const afterRepair = await results(resumed.resultsPath);
+ expect(afterRepair).toHaveLength(3);
+ expect(
+ afterRepair.reduce(
+ (total, receipt) =>
+ total +
+ ((receipt["cost"] as typeof cost | undefined)?.estimatedUsd ?? 0),
+ 0,
+ ),
+ ).toBe(cost.estimatedUsd);
},
);
@@ -1387,8 +1552,11 @@ describe("multiscan", () => {
const paths = await fixture();
const knowledgeBasePaths = [
join(paths.root, "architecture.md"),
- "shared/threat-model.md",
+ join(paths.root, "threat-model.md"),
];
+ await Promise.all(
+ knowledgeBasePaths.map((path) => writeFile(path, "Synthetic context.")),
+ );
const sources = await Promise.all(
["one", "two", "three"].map((name) => repository(paths.root, name)),
);
@@ -1835,7 +2003,9 @@ describe("multiscan", () => {
test("retries a failed attempt and records both durable receipts", async () => {
const { paths } = await repositoryFixture("retry");
const failure = "temporary failure: token=SYNTHETIC_MULTISCAN_TOKEN";
- const knowledgeBasePaths = ["architecture.md"];
+ const document = join(paths.root, "architecture.md");
+ await writeFile(document, "Synthetic architecture.");
+ const knowledgeBasePaths = [document];
let attempts = 0;
const summary = await runMultiscan(
@@ -1885,6 +2055,7 @@ describe("multiscan", () => {
{ scanPrompt: "Review different boundaries." },
{ postScanPrompt: "Draft confirmed fixes." },
{ maxCostUsd: 12.5 },
+ { config: { codexOverrides: { model: "synthetic-model" } } },
]) {
await expect(
runMultiscan(options(paths, security, prompts)),
@@ -1908,6 +2079,872 @@ describe("multiscan", () => {
expect(security.run.mock.calls.length).toBe(2);
});
+ test("knowledge failures do not hide a later real scan failure behind an older completed result", async () => {
+ const paths = await fixture();
+ const source = await repository(paths.root, "later-failure");
+ await writeFile(
+ paths.input,
+ `id,repository,revision\nrepo,${source.path},${source.revision}\n`,
+ );
+ const document = join(paths.root, "context.md");
+ await writeFile(document, "Original context.");
+ let calls = 0;
+ const configured = options(
+ paths,
+ client(async (_repo, scan = {}) => {
+ calls++;
+ if (calls === 2) {
+ await mkdir(scan.outputDir!, { recursive: true, mode: 0o700 });
+ await writeFile(join(scan.outputDir!, "checkpoint"), "keep");
+ throw new Error("Actual scan failed");
+ }
+ return completedScan(scan.outputDir!);
+ }),
+ { knowledgeBasePaths: [document], maxAttempts: 1 },
+ );
+ await runMultiscan(configured);
+ const oldReport = join(
+ paths.output,
+ "artifacts",
+ "repo",
+ "attempt-1",
+ "report.md",
+ );
+ await rm(oldReport);
+ expect(await runMultiscan(configured)).toMatchObject({ failed: 1 });
+ await writeFile(oldReport, "{}\n");
+ await rm(document);
+ expect(await runMultiscan(configured)).toMatchObject({ failed: 1 });
+ await writeFile(document, "Original context.");
+ const repaired = await runMultiscan(configured);
+ expect(repaired).toMatchObject({ completed: 1, failed: 0, skipped: 0 });
+ expect(calls).toBe(3);
+ expect((await results(repaired.resultsPath)).at(-1)).toMatchObject({
+ status: "completed",
+ attempt: 4,
+ });
+ });
+
+ test.each(["partial write", "rename"])(
+ "preserves the campaign manifest after a knowledge repair %s failure",
+ async (failure) => {
+ const testName = `preserves the campaign manifest after a knowledge repair ${failure} failure`;
+ if (runTestInSubprocess(import.meta.path, testName)) return;
+ const paths = await fixture();
+ const source = await repository(paths.root, "manifest-repair");
+ await writeFile(
+ paths.input,
+ `id,repository,revision,mode\nfailed,${source.path},${source.revision},deep\ngood,${source.path},${source.revision},standard\n`,
+ );
+ const document = join(paths.root, "missing.md");
+ const scans: string[] = [];
+ const configured = options(
+ paths,
+ client(async (_repo, scan = {}) => {
+ scans.push(scan.mode!);
+ return completedScan(scan.outputDir!);
+ }),
+ {
+ maxAttempts: 1,
+ scanOptionsByMode: { deep: { knowledgeBasePaths: [document] } },
+ },
+ );
+ const initial = await runMultiscan(configured);
+ expect(initial).toMatchObject({ completed: 1, failed: 1 });
+ const manifestPath = join(paths.output, "manifest.json");
+ const originalManifest = await readFile(manifestPath, "utf8");
+ const originalReceipts = await readFile(initial.resultsPath, "utf8");
+ await writeFile(document, "Repaired context.");
+ const originalWrite = filesystem.writeFile;
+ const originalRename = filesystem.rename;
+ const write = spyOn(filesystem, "writeFile").mockImplementation(
+ async (file, data, opts) => {
+ if (
+ failure === "partial write" &&
+ ((String(file) === manifestPath &&
+ (typeof opts !== "object" || opts?.flag !== "wx")) ||
+ String(file).startsWith(`${manifestPath}.`))
+ ) {
+ await originalWrite(file, "{", opts);
+ throw Object.assign(new Error("Synthetic manifest I/O failure"), {
+ code: "EIO",
+ });
+ }
+ return originalWrite(file, data, opts);
+ },
+ );
+ const move = spyOn(filesystem, "rename").mockImplementation(
+ async (from, to) => {
+ if (failure === "rename" && String(to) === manifestPath)
+ throw Object.assign(new Error("Synthetic manifest I/O failure"), {
+ code: "EIO",
+ });
+ return originalRename(from, to);
+ },
+ );
+ try {
+ await expect(runMultiscan(configured)).rejects.toThrow(
+ "Synthetic manifest I/O failure",
+ );
+ } finally {
+ write.mockRestore();
+ move.mockRestore();
+ }
+ expect(await readFile(manifestPath, "utf8")).toBe(originalManifest);
+ expect(await readFile(initial.resultsPath, "utf8")).toBe(
+ originalReceipts,
+ );
+ expect(scans).toEqual(["standard"]);
+ expect(await readdir(paths.output)).not.toContainEqual(
+ expect.stringMatching(/^manifest\.json\..*\.tmp$/),
+ );
+ expect(await runMultiscan(configured)).toMatchObject({
+ completed: 2,
+ failed: 0,
+ skipped: 1,
+ });
+ expect(scans).toEqual(["standard", "deep"]);
+ expect(await runMultiscan(configured)).toMatchObject({ skipped: 2 });
+ },
+ );
+
+ test("rejects legacy campaigns when unrecorded inputs are omitted", async () => {
+ const paths = await fixture();
+ const source = await repository(paths.root, "legacy-inputs");
+ await writeFile(
+ paths.input,
+ `id,repository,revision\nrepo,${source.path},${source.revision}\n`,
+ );
+ const document = join(paths.root, "architecture.md");
+ await writeFile(document, "Original context.");
+ let calls = 0;
+ const security = client(async (_repository, scanOptions = {}) => {
+ calls++;
+ return completedScan(scanOptions.outputDir!);
+ });
+ const initial = options(paths, security, {
+ knowledgeBasePaths: [document],
+ config: { codexOverrides: { model: "synthetic-model" } },
+ });
+ await runMultiscan(initial);
+ expect(await runMultiscan(initial)).toMatchObject({ skipped: 1 });
+
+ const manifestPath = join(paths.output, "manifest.json");
+ const { tasks } = JSON.parse(await readFile(manifestPath, "utf8"));
+ await writeFile(
+ manifestPath,
+ JSON.stringify({ version: 1, tasks }, null, 2) + "\n",
+ );
+ await expect(runMultiscan(options(paths, security))).rejects.toThrow(
+ "manifest does not match",
+ );
+ expect(calls).toBe(1);
+ });
+
+ test.each([false, true])(
+ "campaign resume binds extracted knowledge from mode settings=%p",
+ async (perMode) => {
+ const paths = await fixture();
+ const source = await repository(paths.root, "knowledge");
+ await writeFile(
+ paths.input,
+ `id,repository,revision\nknowledge,${source.path},${source.revision}\n`,
+ );
+ const knowledge = join(paths.root, "knowledge-base");
+ const nested = join(knowledge, "nested");
+ await mkdir(nested, { recursive: true });
+ const document = join(knowledge, "architecture.md");
+ const deployment = join(nested, "deployment.md");
+ const priorities = join(nested, "priorities.md");
+ await writeFile(document, "Original context.");
+ await writeFile(deployment, "Deployment context.");
+ await writeFile(priorities, "Review priorities.");
+ let calls = 0;
+ const security = client(async (_repository, scanOptions = {}) => {
+ calls++;
+ return completedScan(scanOptions.outputDir!);
+ });
+ const run = (
+ knowledgeBasePaths: string[],
+ overrides: Partial<MultiscanOptions> = {},
+ ) =>
+ runMultiscan(
+ options(paths, security, {
+ ...(perMode
+ ? { scanOptionsByMode: { standard: { knowledgeBasePaths } } }
+ : { knowledgeBasePaths }),
+ ...overrides,
+ }),
+ );
+ const recover = { recoverScan: async () => undefined };
+ const manifestPath = join(paths.output, "manifest.json");
+ let reverseListing = false;
+ const originalReaddir = filesystem.readdir;
+ const listingSpy = spyOn(filesystem, "readdir").mockImplementation(
+ async (...args) => {
+ const entries = await Reflect.apply(
+ originalReaddir,
+ filesystem,
+ args,
+ );
+ if (args[0] === knowledge || args[0] === nested) {
+ entries.sort((left: { name: string }, right: { name: string }) =>
+ left.name.localeCompare(right.name),
+ );
+ if (reverseListing) entries.reverse();
+ }
+ return entries;
+ },
+ );
+ try {
+ let attempts = 0;
+ for (const failure of [
+ "missing",
+ ...(process.platform === "win32" ? [] : ["unreadable"]),
+ "invalid",
+ ]) {
+ const moved = `${knowledge}-moved`;
+ if (failure === "missing") await rename(knowledge, moved);
+ else if (failure === "unreadable") await chmod(document, 0);
+ else await writeFile(document, Buffer.from([0xff]));
+ try {
+ const failed = await run([knowledge], attempts > 0 ? recover : {});
+ attempts += 2;
+ expect(failed).toMatchObject({
+ completed: 0,
+ failed: 1,
+ skipped: 0,
+ });
+ const receipts = await results(failed.resultsPath);
+ expect(receipts).toHaveLength(attempts);
+ expect(receipts.at(-1)).toMatchObject({
+ id: "knowledge",
+ status: "failed",
+ attempt: attempts,
+ });
+ expect(calls).toBe(0);
+ expect(
+ JSON.parse(await readFile(manifestPath, "utf8"))
+ .knowledgeBaseDigests,
+ ).toEqual({ standard: null });
+ await expect(run([])).rejects.toThrow("manifest does not match");
+ expect(calls).toBe(0);
+ } finally {
+ if (failure === "missing") await rename(moved, knowledge);
+ else if (failure === "unreadable") await chmod(document, 0o600);
+ else await writeFile(document, "Original context.");
+ }
+ }
+ await run([knowledge], perMode ? recover : {});
+ reverseListing = true;
+ expect(await run([knowledge])).toMatchObject({ skipped: 1 });
+ } finally {
+ listingSpy.mockRestore();
+ }
+ const boundManifest = await readFile(manifestPath, "utf8");
+ const completedReceipt = (
+ await results(join(paths.output, "results.jsonl"))
+ ).at(-1)!;
+ await writeFile(document, Buffer.from([0xff]));
+ await expect(
+ run([knowledge], { scanPrompt: "Changed review scope." }),
+ ).rejects.toThrow("manifest does not match");
+ expect(await run([knowledge], perMode ? recover : {})).toMatchObject({
+ completed: 0,
+ failed: 1,
+ skipped: 0,
+ });
+ expect(calls).toBe(1);
+ expect(await readFile(manifestPath, "utf8")).toBe(boundManifest);
+ expect(
+ await readFile(
+ join(completedReceipt["outputDir"] as string, "report.md"),
+ "utf8",
+ ),
+ ).toBe("{}\n");
+ await writeFile(document, "Original context.");
+ expect(await run([knowledge], perMode ? recover : {})).toMatchObject({
+ completed: 1,
+ failed: 0,
+ skipped: 1,
+ });
+ expect(
+ await run(
+ perMode ? [knowledge] : [document, deployment, priorities, document],
+ ),
+ ).toMatchObject({ skipped: 1 });
+ if (!perMode) {
+ await expect(run([deployment, document, priorities])).rejects.toThrow(
+ "manifest does not match",
+ );
+ }
+ await writeFile(document, "Revised context.");
+ await expect(run([knowledge])).rejects.toThrow("manifest does not match");
+ await writeFile(document, "Original context.");
+ const additional = join(knowledge, "constraints.txt");
+ await writeFile(additional, "Additional context.");
+ await expect(run([knowledge])).rejects.toThrow("manifest does not match");
+ await rm(additional);
+ expect(await run([knowledge])).toMatchObject({ skipped: 1 });
+ const manifest = JSON.parse(await readFile(manifestPath, "utf8"));
+ delete manifest.knowledgeBaseDigests;
+ await writeFile(manifestPath, JSON.stringify(manifest, null, 2) + "\n");
+ await expect(run([knowledge])).rejects.toThrow("manifest does not match");
+ expect(calls).toBe(1);
+ },
+ );
+
+ test.each([
+ ["standard", true],
+ ["deep", true],
+ ["deep", false],
+ ] as const)(
+ "knowledge failure in %s preserves other modes with knowledge=%p",
+ async (failedMode, goodKnowledge) => {
+ const paths = await fixture();
+ const source = await repository(paths.root, "mixed-knowledge");
+ const goodMode = failedMode === "standard" ? "deep" : "standard";
+ await writeFile(
+ paths.input,
+ `id,repository,revision,mode\nfailed,${source.path},${source.revision},${failedMode}\ngood,${source.path},${source.revision},${goodMode}\n`,
+ );
+ const missing = join(paths.root, "missing.md");
+ const good = join(paths.root, "good.md");
+ await writeFile(good, "Original context.");
+ const scans: string[] = [];
+ const configured = options(
+ paths,
+ client(async (_repo, scan = {}) => {
+ scans.push(scan.mode!);
+ return completedScan(scan.outputDir!);
+ }),
+ {
+ maxAttempts: 1,
+ scanOptionsByMode: {
+ [failedMode]: { knowledgeBasePaths: [missing] },
+ ...(goodKnowledge
+ ? { [goodMode]: { knowledgeBasePaths: [good] } }
+ : {}),
+ },
+ },
+ );
+ const initial = await runMultiscan(configured);
+ expect(initial).toMatchObject({ completed: 1, failed: 1, skipped: 0 });
+ expect(scans).toEqual([goodMode]);
+ expect(await results(initial.resultsPath)).toMatchObject([
+ { id: "failed", status: "failed", attempt: 1 },
+ { id: "good", status: "completed", attempt: 1 },
+ ]);
+ const manifestPath = join(paths.output, "manifest.json");
+ const partialManifest = await readFile(manifestPath, "utf8");
+ expect(JSON.parse(partialManifest).knowledgeBaseDigests).toEqual({
+ [failedMode]: null,
+ ...(goodKnowledge
+ ? {
+ [goodMode]: workflowDigest({
+ "0-good.md.txt": "Original context.",
+ }),
+ }
+ : {}),
+ });
+ expect(await runMultiscan(configured)).toMatchObject({
+ completed: 1,
+ failed: 1,
+ skipped: 1,
+ });
+ expect(scans).toEqual([goodMode]);
+ for (const repaired of [false, true]) {
+ if (repaired) await writeFile(missing, "Repaired context.");
+ if (goodKnowledge) {
+ await writeFile(good, "Changed context.");
+ await expect(runMultiscan(configured)).rejects.toThrow(
+ "manifest does not match",
+ );
+ }
+ expect(await readFile(manifestPath, "utf8")).toBe(partialManifest);
+ }
+ await writeFile(good, "Original context.");
+ expect(
+ await runMultiscan({
+ ...configured,
+ recoverScan: async () => {
+ throw new Error("An extraction failure has no scan to recover.");
+ },
+ }),
+ ).toMatchObject({ completed: 2, failed: 0, skipped: 1 });
+ expect(scans).toEqual([goodMode, failedMode]);
+ expect((await results(initial.resultsPath)).at(-1)).toMatchObject({
+ id: "failed",
+ status: "completed",
+ attempt: 3,
+ });
+ if (!goodKnowledge) return;
+ const boundManifest = await readFile(manifestPath, "utf8");
+ await rm(good);
+ expect(await runMultiscan(configured)).toMatchObject({
+ completed: 1,
+ failed: 1,
+ skipped: 1,
+ });
+ expect(await readFile(manifestPath, "utf8")).toBe(boundManifest);
+ await writeFile(good, "Changed context.");
+ await expect(runMultiscan(configured)).rejects.toThrow(
+ "manifest does not match",
+ );
+ expect(scans).toEqual([goodMode, failedMode]);
+ },
+ );
+
+ test("a mode-specific knowledge failure preserves other interrupted scans for recovery", async () => {
+ const paths = await fixture();
+ const source = await repository(paths.root, "mixed-recovery");
+ await writeFile(
+ paths.input,
+ `id,repository,revision,mode\nstandard,${source.path},${source.revision},standard\ndeep,${source.path},${source.revision},deep\n`,
+ );
+ const document = join(paths.root, "standard.md");
+ await writeFile(document, "Original context.");
+ let runs = 0;
+ const configured = options(
+ paths,
+ client(async (_repo, scan = {}) => {
+ runs++;
+ await mkdir(scan.outputDir!, { recursive: true, mode: 0o700 });
+ await writeFile(join(scan.outputDir!, "checkpoint"), "keep");
+ throw new Error("Interrupted scan");
+ }),
+ {
+ maxAttempts: 1,
+ scanOptionsByMode: { standard: { knowledgeBasePaths: [document] } },
+ },
+ );
+ await runMultiscan(configured);
+ const resumed: string[] = [];
+ const recover = {
+ ...configured,
+ recoverScan: async (dir: string) => {
+ resumed.push(dir);
+ return completedScan(dir);
+ },
+ };
+ const standard = join(paths.output, "artifacts", "standard", "attempt-1");
+ const deep = join(paths.output, "artifacts", "deep", "attempt-1");
+ await rm(document);
+ expect(await runMultiscan(recover)).toMatchObject({
+ completed: 1,
+ failed: 1,
+ skipped: 0,
+ });
+ expect(resumed).toEqual([deep]);
+ expect(
+ (await results(join(paths.output, "results.jsonl"))).slice(-2),
+ ).toMatchObject([
+ { id: "standard", status: "failed", attempt: 1, outputDir: standard },
+ { id: "deep", status: "completed", attempt: 1, outputDir: deep },
+ ]);
+ await writeFile(document, "Original context.");
+ expect(await runMultiscan(recover)).toMatchObject({
+ completed: 2,
+ failed: 0,
+ skipped: 1,
+ });
+ expect(resumed).toEqual([deep, standard]);
+ expect(runs).toBe(2);
+ expect(await readFile(join(standard, "checkpoint"), "utf8")).toBe("keep");
+ expect(await readFile(join(deep, "checkpoint"), "utf8")).toBe("keep");
+ });
+
+ test("shared knowledge failures affect all modes before mode-specific inputs", async () => {
+ const paths = await fixture();
+ const source = await repository(paths.root, "shared-knowledge");
+ await writeFile(
+ paths.input,
+ `id,repository,revision,mode\nstandard,${source.path},${source.revision},standard\ndeep,${source.path},${source.revision},deep\n`,
+ );
+ const shared = join(paths.root, "shared.md");
+ const perMode = join(paths.root, "per-mode.md");
+ await writeFile(perMode, "Mode-specific context.");
+ const scans: string[] = [];
+ const configured = options(
+ paths,
+ client(async (_repo, scan = {}) => {
+ scans.push(scan.mode!);
+ expect(scan.knowledgeBaseSnapshot?.documents).toEqual({
+ "0-shared.md.txt": "Shared context.",
+ });
+ return completedScan(scan.outputDir!);
+ }),
+ {
+ maxAttempts: 1,
+ knowledgeBasePaths: [shared],
+ scanOptionsByMode: {
+ standard: { knowledgeBasePaths: [perMode] },
+ deep: { knowledgeBasePaths: [perMode] },
+ },
+ },
+ );
+ expect(await runMultiscan(configured)).toMatchObject({
+ completed: 0,
+ failed: 2,
+ });
+ expect(scans).toEqual([]);
+ await writeFile(shared, "Shared context.");
+ await rm(perMode);
+ expect(await runMultiscan(configured)).toMatchObject({
+ completed: 2,
+ failed: 0,
+ });
+ expect(scans).toEqual(["standard", "deep"]);
+ });
+
+ test.each([false, true])(
+ "campaign workers stage the fingerprinted snapshot with mode settings=%p",
+ async (perMode) => {
+ const paths = await fixture();
+ const source = await repository(paths.root, "snapshot");
+ const modes = ["standard", "deep", "standard", "deep"] as const;
+ await writeFile(
+ paths.input,
+ "id,repository,revision,mode\n" +
+ modes
+ .map(
+ (mode, index) =>
+ `repo-${index},${source.path},${source.revision},${mode}`,
+ )
+ .join("\n") +
+ "\n",
+ );
+ const standard = join(paths.root, "architecture.md");
+ const deep = join(paths.root, "threat-model.txt");
+ await writeFile(standard, "Original architecture.");
+ await writeFile(deep, "Original threat model.");
+ const homes = [join(paths.root, "home-1"), join(paths.root, "home-2")];
+ await Promise.all(homes.map((home) => mkdir(home)));
+ let created = 0;
+ let calls = 0;
+ const staged: Record<string, string>[] = [];
+ const recipes: Record<string, unknown>[] = [];
+ const summary = await runMultiscan(
+ options(
+ paths,
+ client(async () => {
+ throw new Error("Unexpected client");
+ }),
+ {
+ workers: 2,
+ maxAttempts: 1,
+ ...(perMode ? {} : { knowledgeBasePaths: [standard] }),
+ scanOptionsByMode: {
+ standard: { knowledgeBasePaths: [standard] },
+ deep: { knowledgeBasePaths: [deep] },
+ },
+ createSecurity: (config) => {
+ const home = homes[created++]!;
+ return new TestClient(config, {
+ environment: {},
+ prepareRuntime: async () => preparedRuntime(home),
+ resolvePluginPython: async () => "/managed/python",
+ repositoryRevision: async () => source.revision,
+ runWorkbench: async (_options, args, input) => {
+ if (args[0] === "register-cli-scan")
+ recipes.push(JSON.parse(input!).recipe);
+ return mockWorkbench(args, input);
+ },
+ createCodex: (codex) => ({
+ startThread: () => ({
+ id: null,
+ async runStreamed() {
+ const first = calls++ === 0;
+ if (first) {
+ await writeFile(
+ standard,
+ "Changed after the manifest was saved.",
+ );
+ await rm(deep);
+ }
+ const directory =
+ codex.env!["CODEX_SECURITY_KNOWLEDGE_BASE"]!;
+ staged.push(
+ Object.fromEntries(
+ await Promise.all(
+ (await readdir(directory)).map(async (name) => [
+ name,
+ await readFile(join(directory, name), "utf8"),
+ ]),
+ ),
+ ),
+ );
+ throw new Error(
+ "Synthetic model stop after reading staged knowledge.",
+ );
+ },
+ }),
+ }),
+ });
+ },
+ },
+ ),
+ );
+ expect(created).toBe(2);
+ expect(calls).toBe(4);
+ expect(summary).toMatchObject({ total: 4, failed: 4 });
+ expect(
+ (await results(summary.resultsPath)).every((row) =>
+ String(row["error"]).includes("Synthetic model stop"),
+ ),
+ ).toBe(true);
+ const standardDocuments = {
+ "0-architecture.md.txt": "Original architecture.",
+ };
+ const deepDocuments = perMode
+ ? { "0-threat-model.txt.txt": "Original threat model." }
+ : standardDocuments;
+ expect(
+ staged.filter(
+ (documents) =>
+ JSON.stringify(documents) === JSON.stringify(standardDocuments),
+ ),
+ ).toHaveLength(perMode ? 2 : 4);
+ if (perMode)
+ expect(
+ staged.filter(
+ (documents) =>
+ JSON.stringify(documents) === JSON.stringify(deepDocuments),
+ ),
+ ).toHaveLength(2);
+ for (const recipe of recipes) {
+ expect(recipe["knowledgeBasePaths"]).toEqual([
+ perMode && recipe["mode"] === "deep" ? deep : standard,
+ ]);
+ expect(recipe["knowledgeBaseSnapshot"]).toBeUndefined();
+ }
+ const manifestText = await readFile(
+ join(paths.output, "manifest.json"),
+ "utf8",
+ );
+ expect(JSON.parse(manifestText).knowledgeBaseDigests).toEqual({
+ standard: workflowDigest(standardDocuments),
+ deep: workflowDigest(deepDocuments),
+ });
+ expect(manifestText).not.toContain("Original architecture.");
+ expect(manifestText).not.toContain("Original threat model.");
+ },
+ );
+
+ test("renaming a top-level knowledge document invalidates saved results", async () => {
+ const paths = await fixture();
+ const source = await repository(paths.root, "renamed-knowledge");
+ await writeFile(
+ paths.input,
+ `id,repository,revision\nrepo,${source.path},${source.revision}\n`,
+ );
+ const document = join(paths.root, "architecture.md");
+ const renamed = join(paths.root, "deployment.md");
+ await writeFile(document, "Unchanged text.");
+ const security = client(async (_repository, scan = {}) =>
+ completedScan(scan.outputDir!),
+ );
+ await runMultiscan(
+ options(paths, security, { knowledgeBasePaths: [document] }),
+ );
+ await rename(document, renamed);
+ await expect(
+ runMultiscan(options(paths, security, { knowledgeBasePaths: [renamed] })),
+ ).rejects.toThrow("manifest does not match");
+ });
+
+ test.each([
+ ["pluginPath", false],
+ ["pluginPath", true],
+ ["pythonPath", false],
+ ["pythonPath", true],
+ ] as const)(
+ "campaign binds explicit %s before client creation and recovery=%p",
+ async (field, recovery) => {
+ const paths = await fixture();
+ const source = await repository(paths.root, "runtime-selection");
+ await writeFile(
+ paths.input,
+ `id,repository,revision\ndone,${source.path},${source.revision}\npending,${source.path},${source.revision}\n`,
+ );
+ const pending = join(paths.output, "artifacts", "pending", "attempt-1");
+ const run = mock(
+ async (
+ _repository: string,
+ scan: Parameters<SecurityClient["run"]>[1] = {},
+ ) => {
+ if (scan.outputDir === pending) {
+ await mkdir(pending, { recursive: true });
+ await writeFile(
+ join(pending, "checkpoint"),
+ "Preserve this attempt.",
+ );
+ throw new Error("Synthetic interruption before completion.");
+ }
+ return completedScan(scan.outputDir!);
+ },
+ );
+ const security = client(run);
+ const createSecurity = mock(
+ (_config: MultiscanOptions["config"]) => security,
+ );
+ const recoverScan = mock(async (scanDir: string) =>
+ completedScan(scanDir),
+ );
+ const config: MultiscanOptions["config"] = {
+ [field]: join(paths.root, "selected-runtime"),
+ };
+ const configured = options(paths, security, {
+ config,
+ createSecurity,
+ maxAttempts: 1,
+ });
+ expect(await runMultiscan(configured)).toMatchObject({
+ completed: 1,
+ failed: 1,
+ });
+ const manifest = await readFile(
+ join(paths.output, "manifest.json"),
+ "utf8",
+ );
+ const receipts = await readFile(
+ join(paths.output, "results.jsonl"),
+ "utf8",
+ );
+ const continuation = {
+ ...configured,
+ workers: 3,
+ maxAttempts: 4,
+ ...(recovery ? { recoverScan } : {}),
+ };
+ for (const changed of [
+ { [field]: join(paths.root, "other-runtime") },
+ {},
+ ]) {
+ await expect(
+ runMultiscan({ ...continuation, config: changed }),
+ ).rejects.toThrow("manifest does not match");
+ expect(createSecurity).toHaveBeenCalledTimes(1);
+ expect(run).toHaveBeenCalledTimes(2);
+ expect(recoverScan).not.toHaveBeenCalled();
+ expect(
+ await readFile(join(paths.output, "manifest.json"), "utf8"),
+ ).toBe(manifest);
+ expect(
+ await readFile(join(paths.output, "results.jsonl"), "utf8"),
+ ).toBe(receipts);
+ }
+ expect(await runMultiscan(continuation)).toMatchObject({
+ completed: 2,
+ failed: 0,
+ skipped: 1,
+ });
+ expect(createSecurity).toHaveBeenLastCalledWith(config);
+ expect(recoverScan).toHaveBeenCalledTimes(recovery ? 1 : 0);
+ expect(run).toHaveBeenCalledTimes(recovery ? 2 : 3);
+ expect(await readFile(join(pending, "checkpoint"), "utf8")).toBe(
+ "Preserve this attempt.",
+ );
+ expect(await readFile(join(paths.output, "manifest.json"), "utf8")).toBe(
+ manifest,
+ );
+ },
+ );
+
+ test("campaign preserves absent runtime selections and rejects adding one", async () => {
+ const { paths } = await repositoryFixture("default-runtime");
+ const security = client(completeRun);
+ const createSecurity = mock(() => security);
+ const configured = options(paths, security, { createSecurity });
+ await runMultiscan(configured);
+ const manifest = await readFile(
+ join(paths.output, "manifest.json"),
+ "utf8",
+ );
+ expect(JSON.parse(manifest).configurationDigest).toBeUndefined();
+ for (const field of ["pluginPath", "pythonPath"] as const) {
+ await expect(
+ runMultiscan({
+ ...configured,
+ config: { [field]: join(paths.root, "selected-runtime") },
+ }),
+ ).rejects.toThrow("manifest does not match");
+ }
+ expect(
+ await runMultiscan({
+ ...configured,
+ workers: 3,
+ maxAttempts: 4,
+ config: { pluginPath: undefined, pythonPath: undefined },
+ }),
+ ).toMatchObject({ skipped: 1 });
+ expect(createSecurity).toHaveBeenCalledTimes(1);
+ expect(await readFile(join(paths.output, "manifest.json"), "utf8")).toBe(
+ manifest,
+ );
+ });
+
+ test.each([false, true])(
+ "campaign resume binds overrides with configured modes=%p",
+ async (configuredModes) => {
+ const paths = await fixture();
+ const source = await repository(paths.root, "configuration");
+ await writeFile(
+ paths.input,
+ `id,repository,revision\nconfiguration,${source.path},${source.revision}\n`,
+ );
+ let calls = 0;
+ const security = client(async (_repository, scanOptions = {}) => {
+ calls++;
+ return completedScan(scanOptions.outputDir!);
+ });
+ const initial = options(paths, security, {
+ ...(configuredModes
+ ? { scanOptionsByMode: { standard: { target: ["src"] } } }
+ : {}),
+ config: {
+ codexOverrides: {
+ model: "synthetic-model",
+ features: { example: true },
+ },
+ },
+ });
+ await runMultiscan(initial);
+ expect(
+ await runMultiscan({
+ ...initial,
+ workers: 3,
+ maxAttempts: 4,
+ config: {
+ codexOverrides: {
+ features: { example: true },
+ model: "synthetic-model",
+ },
+ },
+ }),
+ ).toMatchObject({ skipped: 1 });
+ await expect(
+ runMultiscan({
+ ...initial,
+ config: {
+ codexOverrides: {
+ model: "changed-model",
+ features: { example: true },
+ },
+ },
+ }),
+ ).rejects.toThrow("manifest does not match");
+ const manifestPath = join(paths.output, "manifest.json");
+ const manifest = JSON.parse(await readFile(manifestPath, "utf8"));
+ delete manifest.configurationDigest;
+ await writeFile(manifestPath, JSON.stringify(manifest, null, 2) + "\n");
+ await expect(runMultiscan(initial)).rejects.toThrow(
+ "manifest does not match",
+ );
+ expect(calls).toBe(1);
+ },
+ );
+
test.skipIf(process.platform !== "win32")(
"resumes campaigns across Windows repository path aliases",
async () => {
diff --git a/sdk/typescript/tests-ts/scan-resume.test.ts b/sdk/typescript/tests-ts/scan-resume.test.ts
index 0661d058f..7e5b2a232 100644
--- a/sdk/typescript/tests-ts/scan-resume.test.ts
+++ b/sdk/typescript/tests-ts/scan-resume.test.ts
@@ -8,6 +8,7 @@ import {
cp,
mkdir,
readFile,
+ readdir,
rename,
rm,
writeFile,
@@ -18,6 +19,8 @@ import { parse as parseToml } from "smol-toml";
import { main } from "../src/cli.js";
import type { ScanOptions } from "../src/api.js";
import { runWorkbench } from "../src/runtime.js";
+import { readKnowledgeBaseSnapshot } from "../src/knowledge-base.js";
+import { workflowDigest } from "../src/finding-workflow.js";
import { capture, dependencies } from "./cli-fixtures.js";
import { runPython } from "./support/python-probe.js";
import { copyCompletedScanFixture, PLUGIN_ROOT } from "./plugin-root.js";
@@ -34,7 +37,11 @@ async function interruptedScan(
bulk = false,
settings: Pick<
ScanOptions,
- "safetyIdentifier" | "postScanPrompt" | "auth" | "cyberAccessProgram"
+ | "safetyIdentifier"
+ | "postScanPrompt"
+ | "auth"
+ | "knowledgeBasePaths"
+ | "cyberAccessProgram"
> = {},
resolvedDeep = false,
modelProvider?: string,
@@ -80,7 +87,27 @@ async function interruptedScan(
);
await writeFile(
join(root, "manifest.json"),
- JSON.stringify({ version: 1, tasks: [task] }, null, 2) + "\n",
+ JSON.stringify(
+ {
+ version: 2,
+ tasks: [task],
+ ...(settings.knowledgeBasePaths?.length
+ ? {
+ knowledgeBaseDigests: {
+ [mode]: workflowDigest(
+ (
+ await readKnowledgeBaseSnapshot(
+ settings.knowledgeBasePaths,
+ )
+ ).documents,
+ ),
+ },
+ }
+ : {}),
+ },
+ null,
+ 2,
+ ) + "\n",
);
await writeFile(
join(root, "results.jsonl"),
@@ -995,6 +1022,70 @@ test.each([
},
);
+test("bulk Deep resume stages campaign knowledge after its source is removed", async () => {
+ const documentRoot = await temporaryDirectory();
+ const document = join(documentRoot, "architecture.md");
+ await writeFile(document, "Original architecture.");
+ const f = await interruptedScan("deep", true, {
+ knowledgeBasePaths: [document],
+ });
+ const stdout = capture();
+ const stderr = capture();
+ let resumed = false;
+ const code = await main(
+ [
+ "bulk-scan",
+ f.input,
+ "--output-dir",
+ f.root,
+ "--recover",
+ "--knowledge-base",
+ document,
+ "--json",
+ ],
+ stdout.stream,
+ stderr.stream,
+ {
+ ...resumeDependencies(f, (codex) => ({
+ startThread() {
+ throw new Error("Expected original session");
+ },
+ resumeThread(threadId) {
+ expect(threadId).toBe(f.threadId);
+ return {
+ id: threadId,
+ async runStreamed() {
+ const directory = codex.env!["CODEX_SECURITY_KNOWLEDGE_BASE"]!;
+ expect(await readdir(directory)).toEqual([
+ "0-architecture.md.txt",
+ ]);
+ expect(
+ await readFile(
+ join(directory, "0-architecture.md.txt"),
+ "utf8",
+ ),
+ ).toBe("Original architecture.");
+ resumed = true;
+ await finishDiscovery(f);
+ return { events: completedEvents(threadId) };
+ },
+ };
+ },
+ })),
+ runWorkbench: async (args, input) => {
+ if (args[0] === "get-cli-scan-resume") await rm(document);
+ return f.command(args, input);
+ },
+ },
+ );
+ expect(resumed, stderr.text()).toBe(true);
+ expect(code, stderr.text()).toBe(2);
+ expect(JSON.parse(stdout.text())).toMatchObject({ incomplete: 1, failed: 0 });
+ expect(
+ (await f.command(["get-scan-recipe", "--scan-id", f.scanId]))["recipe"],
+ ).toMatchObject({ knowledgeBasePaths: [document] });
+});
+
test("missing session logs do not create another session or fail the original scan", async () => {
const f = await interruptedScan();
await rm(f.sessionPath);