Skip to content

OpenAI's Codex Security CLI and TypeScript SDK for finding, validating, and fixing security vulnerabilities. npm: https://www.npmjs.com/package/@openai/codex-security

fix(scans): preserve saved results when scans stop or restart (#1269)

mldangelo-oai587a3577b694
diff --git a/plugins/codex-security/mcp-app/server.ts b/plugins/codex-security/mcp-app/server.ts
index 4c27108ac..4234343da 100644
--- a/plugins/codex-security/mcp-app/server.ts
+++ b/plugins/codex-security/mcp-app/server.ts
@@ -1243,9 +1243,18 @@ export function createCodexSecurityServer(): McpServer {
workspace = await runWorkbench(args);
},
);
- if (!canceledLocally) workspace = await runWorkbench(args);
- if (workspace === undefined) {
- throw new Error(`Canceling scan ${scanId} did not return its workspace.`);
+ if (workspace === undefined && canceledLocally) {
+ workspace = (await runWorkbench(["get-scan", "--scan-id", scanId]))[
+ "workspace"
+ ] as JsonObject;
+ }
+ const scan = workspace?.["results"];
+ const progress = isJsonObject(scan) ? scan["progress"] : undefined;
+ if (
+ workspace === undefined ||
+ (isJsonObject(progress) && progress["status"] === "running")
+ ) {
+ workspace = await runWorkbench(args);
}
return workspaceResult(workspace as unknown as WorkspaceState);
};
@@ -2497,6 +2506,34 @@ async function executeWorkbench(
workbenchArgs.splice(userContextIndex, 2, "--user-context-stdin");
}
const workbenchInput = input ?? userContext;
+ const timeout = [
+ "begin-deep-scan",
+ "cancel-scan",
+ "fail-scan",
+ "claim-deep-scan-dedup",
+ "commit-deep-scan-dedup",
+ "complete-scan",
+ "export-findings",
+ "finish-deep-scan",
+ "get-scan",
+ "get-deep-scan",
+ "get-workspace",
+ "inspect-setup",
+ "list-findings",
+ "preserve-scan-results",
+ "recover-scan-results",
+ "request-finding-remediation",
+ "request-finding-remediation-action",
+ "save-workspace",
+ "set-finding-triage",
+ "set-finding-remediation",
+ "start-headless-standard-scan",
+ "start-prompt-only-scan",
+ "start-scan",
+ "upsert-deep-scan-worker",
+ ].includes(args[0] ?? "")
+ ? 300_000
+ : 30_000;
const execution = execFileAsync(
pythonCommand,
[workbenchScriptPath(), ...workbenchArgs],
@@ -2509,32 +2546,7 @@ async function executeWorkbench(
encoding: "utf8" as const,
// Artifact bytes are base64-encoded here; retain the existing file-size behavior.
maxBuffer: args[0] === "read-artifact" ? Infinity : 4 * 1024 * 1024,
- timeout: [
- "begin-deep-scan",
- "claim-deep-scan-dedup",
- "commit-deep-scan-dedup",
- "complete-scan",
- "export-findings",
- "finish-deep-scan",
- "get-scan",
- "get-deep-scan",
- "get-workspace",
- "inspect-setup",
- "list-findings",
- "preserve-scan-results",
- "recover-scan-results",
- "request-finding-remediation",
- "request-finding-remediation-action",
- "save-workspace",
- "set-finding-triage",
- "set-finding-remediation",
- "start-headless-standard-scan",
- "start-prompt-only-scan",
- "start-scan",
- "upsert-deep-scan-worker",
- ].includes(args[0] ?? "")
- ? 300_000
- : 30_000,
+ timeout,
},
);
if (workbenchInput !== undefined) {
@@ -2543,7 +2555,21 @@ async function executeWorkbench(
});
execution.child.stdin!.end(workbenchInput);
}
- const { stdout } = await execution;
+ const { stdout } = await execution.catch((error: unknown) => {
+ if (
+ error instanceof Error &&
+ "killed" in error &&
+ error.killed === true &&
+ "signal" in error &&
+ error.signal === "SIGTERM"
+ ) {
+ throw new Error(
+ `Codex Security workbench ${args[0]} timed out after ${timeout / 1000} seconds: ${error.message}`,
+ { cause: error },
+ );
+ }
+ throw error;
+ });
const result = JSON.parse(stdout) as unknown;
if (!isJsonObject(result)) {
throw new Error("Codex Security workbench helper returned invalid JSON.");
diff --git a/plugins/codex-security/mcp-app/src/artifact-io.ts b/plugins/codex-security/mcp-app/src/artifact-io.ts
index d8e19a61d..4b5930db9 100644
--- a/plugins/codex-security/mcp-app/src/artifact-io.ts
+++ b/plugins/codex-security/mcp-app/src/artifact-io.ts
@@ -1,7 +1,7 @@
import { canonicalDirectory } from "./artifact-context.js";
import { isRecord } from "./record.js";
import { randomUUID } from "node:crypto";
-import { constants as fsConstants, promises as fs } from "node:fs";
+import { promises as fs } from "node:fs";
import { dirname, isAbsolute, join, sep } from "node:path";
export interface DeepReducerWorkerContext {
@@ -249,19 +249,17 @@ export async function replaceArtifactText(
path: string,
content: string,
): Promise<void> {
- await withArtifactLock(path, async () => {
- const temporary = join(dirname(path), "." + randomUUID() + ".tmp");
- try {
- await fs.writeFile(temporary, content, {
- encoding: "utf8",
- mode: 0o600,
- flag: "wx",
- });
- await fs.rename(temporary, path);
- } finally {
- await fs.rm(temporary, { force: true });
- }
- });
+ const temporary = join(dirname(path), "." + randomUUID() + ".tmp");
+ try {
+ await fs.writeFile(temporary, content, {
+ encoding: "utf8",
+ mode: 0o600,
+ flag: "wx",
+ });
+ await fs.rename(temporary, path);
+ } finally {
+ await fs.rm(temporary, { force: true });
+ }
}
export async function replaceArtifactJson(
@@ -281,39 +279,6 @@ export async function replaceArtifactJsonl(
await replaceArtifactText(path, content);
}
-async function withArtifactLock(
- path: string,
- action: () => Promise<void>,
-): Promise<void> {
- const lockPath = path + ".lock";
- let lock: fs.FileHandle | undefined;
- for (let attempt = 0; attempt < 500; attempt += 1) {
- try {
- lock = await fs.open(
- lockPath,
- fsConstants.O_WRONLY |
- fsConstants.O_CREAT |
- fsConstants.O_EXCL |
- fsConstants.O_NOFOLLOW,
- 0o600,
- );
- break;
- } catch (error) {
- if (!isNodeError(error) || error.code !== "EEXIST") throw error;
- await new Promise<void>((done) => setTimeout(done, 20));
- }
- }
- if (!lock) {
- throw new Error("Timed out waiting for the artifact write lock.");
- }
- try {
- await action();
- } finally {
- await lock.close();
- await fs.rm(lockPath, { force: true });
- }
-}
-
function validateArtifactComponents(
components: readonly string[],
label: string,
diff --git a/plugins/codex-security/mcp-app/src/deep-scan/coordinator.ts b/plugins/codex-security/mcp-app/src/deep-scan/coordinator.ts
index 11d5bb6d4..a5fe4b449 100644
--- a/plugins/codex-security/mcp-app/src/deep-scan/coordinator.ts
+++ b/plugins/codex-security/mcp-app/src/deep-scan/coordinator.ts
@@ -1,7 +1,7 @@
/// <reference lib="es2023.array" />
import { randomUUID } from "node:crypto";
import { promises as fs } from "node:fs";
-import { join } from "node:path";
+import { basename, dirname, join } from "node:path";
import { setTimeout as delay } from "node:timers/promises";
import {
createDeepScanArtifacts,
@@ -71,7 +71,10 @@ export interface CoordinatorOptions {
handoffClaimToken?: string;
threadId?: string;
heartbeatIntervalMs?: number;
- observeReplacement?: (run: DeepScanRunState) => Promise<DeepScanRunState>;
+ observeReplacement?: (
+ run: DeepScanRunState,
+ signal: AbortSignal,
+ ) => Promise<DeepScanRunState>;
onComplete?: (draft: ScanDraftInput, signal: AbortSignal) => Promise<void>;
onStopped?: (run: DeepScanRunState) => Promise<void>;
}
@@ -83,6 +86,7 @@ export class DeepScanCoordinator {
private readonly abortController = new AbortController();
private readonly discoveryAbortController = new AbortController();
private readonly publicationAbortController = new AbortController();
+ private readonly observationAbortController = new AbortController();
private readonly clock: DeepScanClock;
private readonly log: DeepScanLogger;
private readonly artifacts: DeepScanArtifacts;
@@ -102,7 +106,7 @@ export class DeepScanCoordinator {
private cancellationPersistence?: {
promise: Promise<void>;
resolve: () => void;
- reject: (error: unknown) => void;
+ failure?: { error: unknown };
};
private started = false;
private terminal = false;
@@ -230,6 +234,7 @@ export class DeepScanCoordinator {
});
this.abortController.abort(reason);
this.publicationAbortController.abort(reason);
+ this.observationAbortController.abort(reason);
// The cancel operation returns promptly. Terminal waiters remain attached
// until worker cleanup and stopped-result publication settle.
}
@@ -238,24 +243,25 @@ export class DeepScanCoordinator {
reason: string,
persistCancellation: () => Promise<void>,
): Promise<DeepScanRunState> {
- if (this.terminal) return await this.settled();
+ if (this.terminal || this.state.status !== "running")
+ return await this.settled();
if (!this.cancellationPersistence) {
let resolve!: () => void;
- let reject!: (error: unknown) => void;
- const promise = new Promise<void>((resolvePromise, rejectPromise) => {
+ const promise = new Promise<void>((resolvePromise) => {
resolve = resolvePromise;
- reject = rejectPromise;
});
- this.cancellationPersistence = { promise, resolve, reject };
+ this.cancellationPersistence = { promise, resolve };
}
this.cancel(reason);
await this.cancellationReady;
try {
await persistCancellation();
- this.cancellationPersistence.resolve();
} catch (error) {
- this.cancellationPersistence.reject(error);
- throw error;
+ this.cancellationPersistence.failure = { error };
+ } finally {
+ // Cleanup still inspects durable state and preserves results when the
+ // process lost a committed response, then reports the persistence failure.
+ this.cancellationPersistence.resolve();
}
return await this.settled();
}
@@ -320,14 +326,14 @@ export class DeepScanCoordinator {
this.publicationAbortController.signal,
);
if (this.canceled || this.externallyFailed) return;
- this.state = await this.finishWithReplay(schedulerResult);
+ const completed = await this.finishWithReplay(schedulerResult);
if (this.canceled || this.externallyFailed) return;
+ this.state = completed;
this.log({
event: "coordinator_terminal",
scanId: this.state.scanId,
reason: schedulerResult.reason,
});
- if (this.stopLocally()) this.resolveTerminal(cloneState(this.state));
} catch (error) {
if (this.canceled || this.externallyFailed) {
return;
@@ -351,13 +357,17 @@ export class DeepScanCoordinator {
}
this.abortController.abort(message);
await this.settleSchedulerWork();
+ if (this.canceled || this.externallyFailed) return;
try {
- this.state = await this.options.store.fail(
+ const failed = await this.options.store.fail(
this.state.scanId,
persistedMessage,
"failed",
);
+ if (this.canceled || this.externallyFailed) return;
+ this.state = failed;
} catch (persistError) {
+ if (this.canceled || this.externallyFailed) return;
this.state = {
...this.state,
status: "failed",
@@ -378,6 +388,7 @@ export class DeepScanCoordinator {
// A worker can finish an atomic draft write while cancellation is being
// persisted. Publish saved output only after every local writer settles.
await this.settleSchedulerWork();
+ await this.ownershipCheck;
this.resolveCancellationReady();
await this.cancellationPersistence?.promise;
if (this.options.onStopped && this.options.threadId) {
@@ -448,6 +459,9 @@ export class DeepScanCoordinator {
scanId: this.state.scanId,
});
}
+ if (this.cancellationPersistence?.failure) {
+ throw this.cancellationPersistence.failure.error;
+ }
if (this.canceled) this.state = { ...this.state, status: "canceled" };
if (this.stopLocally()) this.resolveTerminal(cloneState(this.state));
}
@@ -501,6 +515,7 @@ export class DeepScanCoordinator {
private scheduleHeartbeat(): void {
if (
this.terminal ||
+ this.externallyFailed ||
!this.options.threadId ||
!this.state.coordinatorGeneration
) {
@@ -559,9 +574,10 @@ export class DeepScanCoordinator {
scanId: this.state.scanId,
reason: errorKind(readError),
});
- if (!leaseLossConfirmed) return false;
+ if (!leaseLossConfirmed) return this.externallyFailed;
current = this.state;
}
+ if (this.externallyFailed || this.terminal) return this.externallyFailed;
const replacementConfirmed =
leaseLossConfirmed ||
(current.status === "running" &&
@@ -571,11 +587,17 @@ export class DeepScanCoordinator {
if (current.status === "running" && !replacementConfirmed) return false;
this.externallyFailed = true;
+ this.state = current;
+ clearTimeout(this.heartbeatTimeout);
+ this.heartbeatTimeout = undefined;
this.abortController.abort("deep_scan_coordinator_lease_lost");
this.publicationAbortController.abort("deep_scan_coordinator_lease_lost");
if (replacementConfirmed && this.options.observeReplacement) {
try {
- this.state = await this.options.observeReplacement(current);
+ this.state = await this.options.observeReplacement(
+ current,
+ this.observationAbortController.signal,
+ );
} catch (observeError) {
this.log({
event: "coordinator_replacement_observation_failed",
@@ -591,7 +613,6 @@ export class DeepScanCoordinator {
} else {
this.state = current;
}
- if (this.stopLocally()) this.resolveTerminal(cloneState(this.state));
return true;
}
@@ -624,18 +645,33 @@ export class DeepScanCoordinator {
let reducer: Promise<DedupOutcome> | undefined;
let previousReducerResultPath = recoveredReducers.resultPath;
let dispatched = this.state.dispatchedCount;
- let workerSequence = Math.max(
- dispatched,
+ const [discoveryLabels, reducerLabels] = await Promise.all([
+ fs.readdir(this.artifacts.workersRoot),
+ fs.readdir(this.artifacts.dedupRoot),
+ ]);
+ let workerSequence = [
+ BigInt(dispatched),
+ ...discoveryLabels.map((label) =>
+ workerLabelSequence(label, "discovery"),
+ ),
...(this.state.persistedWorkers ?? [])
.filter((worker) => worker.kind === "discovery")
- .map((worker) => workerLabelSequence(worker, "discovery")),
- );
- let reducerSequence = Math.max(
- 0,
+ .map((worker) =>
+ workerLabelSequence(
+ basename(dirname(worker.promptPath)),
+ "discovery",
+ ),
+ ),
+ ].reduce((maximum, value) => (value > maximum ? value : maximum), 0n);
+ let reducerSequence = [
+ 0n,
+ ...reducerLabels.map((label) => workerLabelSequence(label, "dedup")),
...(this.state.persistedWorkers ?? [])
.filter((worker) => worker.kind === "dedup")
- .map((worker) => workerLabelSequence(worker, "dedup")),
- );
+ .map((worker) =>
+ workerLabelSequence(basename(dirname(worker.promptPath)), "dedup"),
+ ),
+ ].reduce((maximum, value) => (value > maximum ? value : maximum), 0n);
let stopReason: DeepScanTerminalReason | undefined;
let lastReplaceableFailure:
Extract<DiscoveryOutcome, { status: "failed" }> | undefined;
@@ -729,7 +765,7 @@ export class DeepScanCoordinator {
if (result.status === "rejected") return result.reason;
const outcome = result.value;
if ("status" in outcome) return outcome.error;
- this.state = outcome.run;
+ if (!this.canceled && !this.externallyFailed) this.state = outcome.run;
return undefined;
};
@@ -757,7 +793,7 @@ export class DeepScanCoordinator {
dispatched < config.maxDiscoveryRuns
) {
dispatched += 1;
- workerSequence += 1;
+ workerSequence += 1n;
const workerLabel = `discovery-${String(workerSequence).padStart(4, "0")}`;
const workerId = randomUUID();
const workerPromise = this.trackSchedulerWork(
@@ -779,7 +815,7 @@ export class DeepScanCoordinator {
) {
const consumed = [...buffer].sort(compareCompletionSequence);
buffer = [];
- reducerSequence += 1;
+ reducerSequence += 1n;
reducer = this.trackSchedulerWork(
this.workers.runReducer({
id: randomUUID(),
@@ -892,7 +928,7 @@ export class DeepScanCoordinator {
throw thresholdError;
}
reducerFailures = 0;
- this.state = outcome.run;
+ if (!this.canceled && !this.externallyFailed) this.state = outcome.run;
previousReducerResultPath = outcome.resultPath;
latestResult = outcome.result;
if (
@@ -963,12 +999,21 @@ export class DeepScanCoordinator {
.filter(
(worker) => worker.kind === "dedup" && worker.status === "succeeded",
)
- .sort(
- (left, right) =>
- workerLabelSequence(left, "dedup") -
- workerLabelSequence(right, "dedup") ||
- left.id.localeCompare(right.id),
- );
+ .sort((left, right) => {
+ const leftSequence = workerLabelSequence(
+ basename(dirname(left.promptPath)),
+ "dedup",
+ );
+ const rightSequence = workerLabelSequence(
+ basename(dirname(right.promptPath)),
+ "dedup",
+ );
+ return leftSequence < rightSequence
+ ? -1
+ : leftSequence > rightSequence
+ ? 1
+ : left.id.localeCompare(right.id);
+ });
for (const worker of completedReducers) {
if (!worker.resultManifestPath) {
throw new Error(
@@ -1108,11 +1153,11 @@ function compareCompletionSequence(
}
function workerLabelSequence(
- worker: PersistedDeepScanWorker,
+ label: string,
kind: "discovery" | "dedup",
-): number {
- const match = worker.promptPath.match(new RegExp(`${kind}-(\\d+)`));
- return match ? Number(match[1]) : 0;
+): bigint {
+ const match = label.match(new RegExp(`^${kind}-(\\d+)$`));
+ return match ? BigInt(match[1]!) : 0n;
}
function cloneState(state: DeepScanRunState): DeepScanRunState {
diff --git a/plugins/codex-security/mcp-app/src/deep-scan/registry.ts b/plugins/codex-security/mcp-app/src/deep-scan/registry.ts
index 6b957cef1..c28d3f6c0 100644
--- a/plugins/codex-security/mcp-app/src/deep-scan/registry.ts
+++ b/plugins/codex-security/mcp-app/src/deep-scan/registry.ts
@@ -15,6 +15,7 @@ export {
/** Owns the live coordinators in this MCP server process. */
export class DeepScanCoordinatorRegistry {
private readonly coordinators = new Map<string, DeepScanCoordinator>();
+ private readonly observers = new Set<DeepScanCoordinator>();
get(scanId: string): DeepScanCoordinator | undefined {
return this.coordinators.get(scanId);
@@ -26,15 +27,20 @@ export class DeepScanCoordinatorRegistry {
let coordinator!: DeepScanCoordinator;
coordinator = new DeepScanCoordinator({
...options,
- observeReplacement: async (run) => {
+ observeReplacement: async (run, signal) => {
if (this.coordinators.get(run.scanId) === coordinator) {
this.coordinators.delete(run.scanId);
}
- return await new DeepScanRemoteCoordinator({
- run,
- registry: this,
- options,
- }).wait(undefined);
+ this.observers.add(coordinator);
+ try {
+ return await new DeepScanRemoteCoordinator({
+ run,
+ registry: this,
+ options,
+ }).wait(signal);
+ } finally {
+ this.observers.delete(coordinator);
+ }
},
});
this.coordinators.set(options.run.scanId, coordinator);
@@ -62,6 +68,7 @@ export class DeepScanCoordinatorRegistry {
shutdown(reason: string): void {
for (const coordinator of this.coordinators.values())
coordinator.cancel(reason);
+ for (const coordinator of this.observers) coordinator.cancel(reason);
}
}
diff --git a/plugins/codex-security/mcp-app/tests/deep_scan_lifecycle_cases.ts b/plugins/codex-security/mcp-app/tests/deep_scan_lifecycle_cases.ts
new file mode 100644
index 000000000..7f3371119
--- /dev/null
+++ b/plugins/codex-security/mcp-app/tests/deep_scan_lifecycle_cases.ts
@@ -0,0 +1,571 @@
+import assert from "node:assert/strict";
+import { randomUUID } from "node:crypto";
+import { mkdir, readFile, rename, writeFile } from "node:fs/promises";
+import path from "node:path";
+import type { DeepScanRunState } from "../src/deep-scan/types.js";
+
+type LifecycleFixtures = Pick<
+ typeof import("./deep_scan_coordinator_fixture.ts"),
+ | "fixtureRun"
+ | "FakeStore"
+ | "FakeExecutor"
+ | "createCoordinator"
+ | "DeepScanCoordinatorRegistry"
+ | "immediateClock"
+>;
+
+export async function testDeepScanLifecycle({
+ fixtureRun,
+ FakeStore,
+ FakeExecutor,
+ createCoordinator,
+ DeepScanCoordinatorRegistry,
+ immediateClock,
+}: LifecycleFixtures) {
+ const config = {
+ workers: 1,
+ subagents: 0,
+ stopAfterNoNew: 1,
+ maxDiscoveryRuns: 1,
+ };
+ const errors = [];
+ for (const test of [
+ canceledPublicationWaitsForHeartbeat,
+ delayedReducerDoesNotReplaceStoppedState,
+ replacementWaitsForTerminalResult,
+ concurrentOwnershipReadsKeepReplacementResult,
+ lateFailurePersistenceKeepsStoppedState,
+ shutdownStopsReplacementObservation,
+ failedCancellationStillPreservesResults,
+ lateCancellationKeepsPersistedFailure,
+ terminalDiscoveryWaitsForCleanup,
+ orphanWorkerDirectoriesAreNotReused,
+ ancestorNamesDoNotChooseWorkerSequence,
+ ]) {
+ try {
+ await test();
+ } catch (error) {
+ errors.push(new Error(test.name, { cause: error }));
+ }
+ }
+ if (errors.length)
+ throw new AggregateError(errors, "Deep Scan lifecycle regressions");
+
+ async function canceledPublicationWaitsForHeartbeat() {
+ const fixture = await fixtureRun(config);
+ fixture.run.coordinatorGeneration = 1;
+ const store = new FakeStore(fixture.run);
+ const executor = new FakeExecutor({ blockDiscoveryAfterCalls: 0 });
+ const publishing = Promise.withResolvers<void>();
+ const release = Promise.withResolvers<void>();
+ const coordinator = createCoordinator(fixture, store, executor, {
+ threadId: "fixture-owner",
+ heartbeatIntervalMs: 60_000,
+ onStopped: async () => {
+ publishing.resolve();
+ await release.promise;
+ },
+ });
+ coordinator.start();
+ await executor.discoveryStarted.promise;
+ store.run.status = "canceled";
+ coordinator.cancel("fixture cancellation");
+ await publishing.promise;
+ try {
+ await coordinator.renewHeartbeat();
+ assert.equal(
+ await coordinator.wait(undefined, 0),
+ undefined,
+ "heartbeat must not release waiters while saved results are being published",
+ );
+ } finally {
+ release.resolve();
+ await coordinator.settled();
+ }
+ }
+
+ async function delayedReducerDoesNotReplaceStoppedState() {
+ for (const status of ["canceled", "failed"] as const) {
+ const fixture = await fixtureRun(config);
+ fixture.run.coordinatorGeneration = 1;
+ const store = new FakeStore(fixture.run);
+ const committed = Promise.withResolvers<void>();
+ const release = Promise.withResolvers<void>();
+ const commit = store.commitDedup.bind(store);
+ store.commitDedup = async (input) => {
+ const response = await commit(input);
+ committed.resolve();
+ await release.promise;
+ return response;
+ };
+ const coordinator = createCoordinator(
+ fixture,
+ store,
+ new FakeExecutor(),
+ {
+ threadId: "fixture-owner",
+ heartbeatIntervalMs: 60_000,
+ },
+ );
+ coordinator.start();
+ await committed.promise;
+ store.run.status = status;
+ await coordinator.renewHeartbeat();
+ release.resolve();
+ assert.equal((await coordinator.settled()).status, status);
+ }
+ }
+
+ async function replacementWaitsForTerminalResult() {
+ const fixture = await fixtureRun(config);
+ fixture.run.coordinatorGeneration = 1;
+ const store = new FakeStore(fixture.run);
+ const executor = new FakeExecutor({ blockDiscoveryAfterCalls: 0 });
+ const observing = Promise.withResolvers<void>();
+ const replacement = Promise.withResolvers<DeepScanRunState>();
+ const coordinator = createCoordinator(fixture, store, executor, {
+ threadId: "fixture-owner",
+ heartbeatIntervalMs: 60_000,
+ observeReplacement: async () => {
+ observing.resolve();
+ return await replacement.promise;
+ },
+ });
+ coordinator.start();
+ await executor.discoveryStarted.promise;
+ store.run.coordinatorGeneration = 2;
+ const heartbeat = coordinator.renewHeartbeat();
+ await observing.promise;
+ try {
+ assert.equal(
+ await coordinator.wait(undefined, 0),
+ undefined,
+ "the original caller must keep waiting for the replacement coordinator",
+ );
+ } finally {
+ replacement.resolve({
+ ...store.run,
+ status: "succeeded",
+ terminalReason: "capped",
+ });
+ await heartbeat;
+ }
+ assert.equal((await coordinator.settled()).status, "succeeded");
+ }
+
+ async function concurrentOwnershipReadsKeepReplacementResult() {
+ for (const [readFails, leaseLossConfirmed] of [
+ [false, false],
+ [true, false],
+ [true, true],
+ ]) {
+ const fixture = await fixtureRun(config);
+ fixture.run.coordinatorGeneration = 2;
+ const store = new FakeStore(fixture.run);
+ const preparing = Promise.withResolvers<void>();
+ const failProgress = Promise.withResolvers<void>();
+ const heartbeatReadStarted = Promise.withResolvers<void>();
+ const failureReadStarted = Promise.withResolvers<void>();
+ const heartbeatRead = Promise.withResolvers<DeepScanRunState>();
+ const failureRead = Promise.withResolvers<DeepScanRunState>();
+ const observing = Promise.withResolvers<void>();
+ const replacement = Promise.withResolvers<DeepScanRunState>();
+ const releaseStaleFailure = Promise.withResolvers<void>();
+ let reads = 0;
+ let failureCalls = 0;
+ let observations = 0;
+ store.updateProgress = async () => {
+ preparing.resolve();
+ await failProgress.promise;
+ throw new Error(
+ leaseLossConfirmed
+ ? "Deep Scan coordinator lease belongs to a newer generation."
+ : "fixture progress persistence failure",
+ );
+ };
+ store.get = async () => {
+ reads += 1;
+ if (reads === 1) {
+ heartbeatReadStarted.resolve();
+ return await heartbeatRead.promise;
+ }
+ failureReadStarted.resolve();
+ return await failureRead.promise;
+ };
+ store.fail = async () => {
+ failureCalls += 1;
+ await releaseStaleFailure.promise;
+ throw new Error(
+ "Deep Scan coordinator lease belongs to a newer generation.",
+ );
+ };
+ const coordinator = createCoordinator(
+ fixture,
+ store,
+ new FakeExecutor(),
+ {
+ threadId: "fixture-owner",
+ heartbeatIntervalMs: 60_000,
+ observeReplacement: async () => {
+ observations += 1;
+ observing.resolve();
+ return await replacement.promise;
+ },
+ },
+ );
+ coordinator.start();
+ await preparing.promise;
+ const heartbeat = coordinator.renewHeartbeat();
+ await heartbeatReadStarted.promise;
+ failProgress.resolve();
+ await failureReadStarted.promise;
+ const newer = { ...store.run, coordinatorGeneration: 3 };
+ store.run = newer;
+ heartbeatRead.resolve(structuredClone(newer));
+ await observing.promise;
+ if (readFails) failureRead.reject(new Error("database is locked"));
+ else failureRead.resolve(structuredClone(newer));
+ replacement.resolve({
+ ...newer,
+ status: "succeeded",
+ terminalReason: "capped",
+ });
+ await heartbeat;
+ // A stale failure response must arrive only after replacement observation.
+ releaseStaleFailure.resolve();
+ const terminal = await coordinator.settled();
+ assert.equal(terminal.status, "succeeded");
+ assert.equal(terminal.coordinatorGeneration, 3);
+ assert.equal(
+ failureCalls,
+ 0,
+ "a replaced coordinator must not fail the scan",
+ );
+ assert.equal(observations, 1, "concurrent reads must share one observer");
+ }
+ }
+
+ async function lateFailurePersistenceKeepsStoppedState() {
+ for (const status of ["succeeded", "canceled", "failed"] as const) {
+ for (const rejectFailure of [false, true]) {
+ const fixture = await fixtureRun(config);
+ fixture.run.coordinatorGeneration = 2;
+ const store = new FakeStore(fixture.run);
+ store.failProgressAt = 1;
+ const failing = Promise.withResolvers<void>();
+ const release = Promise.withResolvers<void>();
+ store.fail = async (_scanId, message) => {
+ const stale = {
+ ...store.run,
+ status: "failed" as const,
+ error: message,
+ };
+ failing.resolve();
+ await release.promise;
+ if (rejectFailure) throw new Error("fixture stale failure rejected");
+ return stale;
+ };
+ const coordinator = createCoordinator(
+ fixture,
+ store,
+ new FakeExecutor(),
+ {
+ threadId: "fixture-owner",
+ heartbeatIntervalMs: 60_000,
+ observeReplacement: async (run) => ({
+ ...run,
+ status: "succeeded",
+ terminalReason: "capped",
+ }),
+ },
+ );
+ coordinator.start();
+ await failing.promise;
+ try {
+ if (status === "succeeded") {
+ store.run.coordinatorGeneration = 3;
+ await coordinator.renewHeartbeat();
+ } else if (status === "canceled") {
+ store.run.status = "canceled";
+ coordinator.cancel("fixture cancellation");
+ } else {
+ store.run.status = "failed";
+ coordinator.failExternallyPersisted("fixture external failure");
+ }
+ } finally {
+ release.resolve();
+ }
+ const terminal = await coordinator.settled();
+ assert.equal(terminal.status, status);
+ assert.equal(
+ terminal.coordinatorGeneration,
+ status === "succeeded" ? 3 : 2,
+ );
+ assert.equal(
+ terminal.error,
+ status === "failed" ? "fixture external failure" : undefined,
+ "stale failure persistence must not replace the stopped result",
+ );
+ }
+ }
+ }
+
+ async function shutdownStopsReplacementObservation() {
+ const fixture = await fixtureRun(config);
+ fixture.run.coordinatorGeneration = 1;
+ const store = new FakeStore(fixture.run);
+ const executor = new FakeExecutor({ blockDiscoveryAfterCalls: 0 });
+ const registry = new DeepScanCoordinatorRegistry();
+ const coordinator = registry.start({
+ run: fixture.run,
+ store,
+ executor,
+ pluginRoot: fixture.pluginRoot,
+ clock: immediateClock,
+ threadId: "fixture-owner",
+ heartbeatIntervalMs: 60_000,
+ });
+ await executor.discoveryStarted.promise;
+ store.run.coordinatorGeneration = 2;
+ const heartbeat = coordinator.renewHeartbeat();
+ await new Promise(setImmediate);
+ registry.shutdown("fixture transport closed");
+ const terminal = await coordinator.wait(undefined, 100);
+ // Release the old implementation's detached observer even on failure.
+ store.run.status = "succeeded";
+ await heartbeat;
+ assert.equal(terminal?.status, "canceled");
+ }
+
+ async function failedCancellationStillPreservesResults() {
+ const fixture = await fixtureRun(config);
+ const store = new FakeStore(fixture.run);
+ const executor = new FakeExecutor({ blockDiscoveryAfterCalls: 0 });
+ const publishing = Promise.withResolvers<void>();
+ const release = Promise.withResolvers<void>();
+ let publications = 0;
+ let cancellationSettled = false;
+ const coordinator = createCoordinator(fixture, store, executor, {
+ threadId: "fixture-owner",
+ onStopped: async () => {
+ publishing.resolve();
+ await release.promise;
+ publications += 1;
+ },
+ });
+ coordinator.start();
+ const terminal = coordinator.settled().catch((error: Error) => error);
+ await executor.discoveryStarted.promise;
+ const cancellation = coordinator
+ .cancelAfterPersistence("fixture cancellation", async () => {
+ store.run.status = "canceled";
+ throw new Error("fixture cancellation response lost");
+ })
+ .catch((error: Error) => {
+ cancellationSettled = true;
+ return error;
+ });
+ await publishing.promise;
+ await new Promise<void>((resolve) => setImmediate(resolve));
+ try {
+ assert.equal(
+ cancellationSettled,
+ false,
+ "lost response must wait for saved result publication",
+ );
+ } finally {
+ release.resolve();
+ await terminal;
+ }
+ assert.match((await cancellation).message, /response lost/);
+ const result = await terminal;
+ assert.equal(
+ publications,
+ 1,
+ "cancellation response failure must not skip saved results",
+ );
+ assert.match(result.message, /response lost/);
+ }
+
+ async function lateCancellationKeepsPersistedFailure() {
+ const fixture = await fixtureRun(config);
+ const store = new FakeStore(fixture.run);
+ const executor = new FakeExecutor({ blockDiscoveryAfterCalls: 0 });
+ const publishing = Promise.withResolvers<void>();
+ const release = Promise.withResolvers<void>();
+ let cancellations = 0;
+ const coordinator = createCoordinator(fixture, store, executor, {
+ threadId: "fixture-owner",
+ onStopped: async () => {
+ publishing.resolve();
+ await release.promise;
+ },
+ });
+ coordinator.start();
+ await executor.discoveryStarted.promise;
+ store.run.status = "failed";
+ coordinator.failExternallyPersisted("fixture worker failure");
+ await publishing.promise;
+ const cancellation = coordinator.cancelAfterPersistence(
+ "late cancellation",
+ async () => {
+ cancellations += 1;
+ },
+ );
+ release.resolve();
+ const result = await cancellation;
+ assert.equal(
+ cancellations,
+ 0,
+ "a failed scan cannot be canceled while publication settles",
+ );
+ assert.equal(result.status, "failed");
+ }
+
+ async function terminalDiscoveryWaitsForCleanup() {
+ for (const status of ["succeeded", "failed"] as const) {
+ const fixture = await fixtureRun(config);
+ const store = new FakeStore(fixture.run);
+ store.failFinish = status === "failed";
+ store.rejectFailurePersistence = status === "failed";
+ let coordinator: ReturnType<typeof createCoordinator>;
+ const finalRead = Promise.withResolvers<void>();
+ const release = Promise.withResolvers<void>();
+ const originalGet = store.get.bind(store);
+ store.get = async (...args) => {
+ if (coordinator?.snapshot().status === status) {
+ finalRead.resolve();
+ await release.promise;
+ }
+ return await originalGet(...args);
+ };
+ const registry = new DeepScanCoordinatorRegistry();
+ coordinator = registry.start({
+ run: fixture.run,
+ store,
+ executor: new FakeExecutor(),
+ pluginRoot: fixture.pluginRoot,
+ clock: immediateClock,
+ threadId: "fixture-owner",
+ onStopped: async () => {},
+ });
+ await finalRead.promise;
+ let persisted = false;
+ let resolved = false;
+ const cancellation = registry
+ .cancelAndWait(fixture.run.scanId, "cancel parent", async () => {
+ persisted = true;
+ })
+ .then((handled: boolean) => {
+ resolved = true;
+ return handled;
+ });
+ await Promise.resolve();
+ assert.equal(
+ resolved,
+ false,
+ "wait for all coordinator cleanup before durable parent cancellation",
+ );
+ release.resolve();
+ assert.equal(
+ await cancellation,
+ true,
+ "local cleanup completed; the server checks durable parent state",
+ );
+ assert.equal(
+ persisted,
+ false,
+ "the server performs durable cancellation after coordinator cleanup",
+ );
+ assert.equal(
+ store.run.status,
+ status === "failed" ? "running" : "succeeded",
+ );
+ }
+ }
+
+ async function orphanWorkerDirectoriesAreNotReused() {
+ for (const suffix of ["0001", "9007199254740992"]) {
+ const fixture = await fixtureRun(config);
+ const savedPrompts: string[] = [];
+ for (const [directory, kind] of [
+ ["workers", "discovery"],
+ ["dedup", "dedup"],
+ ]) {
+ const root = path.join(
+ fixture.run.scanDir,
+ "artifacts",
+ "deep_discovery",
+ directory,
+ `${kind}-${suffix}`,
+ );
+ await mkdir(root, { recursive: true });
+ const prompt = path.join(root, "prompt.md");
+ savedPrompts.push(prompt);
+ await writeFile(prompt, "interrupted before worker registration\n");
+ }
+ const store = new FakeStore(fixture.run);
+ const executor = new FakeExecutor();
+ const coordinator = createCoordinator(fixture, store, executor);
+ coordinator.start();
+ const terminal = await coordinator.wait(undefined, 5_000);
+ assert.equal(terminal?.status, "succeeded", terminal?.error);
+ const next = String(BigInt(suffix) + 1n).padStart(4, "0");
+ for (const kind of ["discovery", "dedup"]) {
+ assert.ok(
+ [...store.workers.values()].some((worker) =>
+ worker.promptPath.includes(`${kind}-${next}`),
+ ),
+ );
+ }
+ for (const prompt of savedPrompts) {
+ assert.equal(
+ await readFile(prompt, "utf8"),
+ "interrupted before worker registration\n",
+ );
+ }
+ }
+ }
+
+ async function ancestorNamesDoNotChooseWorkerSequence() {
+ const fixture = await fixtureRun({ ...config, maxDiscoveryRuns: 3 });
+ const scanDir = path.join(
+ path.dirname(fixture.run.scanDir),
+ "service-discovery-2",
+ );
+ await mkdir(fixture.run.scanDir, { recursive: true });
+ await rename(fixture.run.scanDir, scanDir);
+ fixture.run.scanDir = scanDir;
+ const root = path.join(
+ scanDir,
+ "artifacts",
+ "deep_discovery",
+ "workers",
+ "discovery-0003",
+ );
+ await mkdir(root, { recursive: true });
+ await writeFile(path.join(root, "prompt.md"), "previous worker\n");
+ fixture.run.dispatchedCount = 2;
+ fixture.run.persistedWorkers = [
+ {
+ id: randomUUID(),
+ kind: "discovery",
+ status: "canceled",
+ attempt: 0,
+ promptPath: path.join(root, "prompt.md"),
+ artifactDir: path.join(root, "output"),
+ mergeState: "none",
+ },
+ ];
+ const store = new FakeStore(fixture.run);
+ const coordinator = createCoordinator(fixture, store, new FakeExecutor());
+ coordinator.start();
+ const terminal = await coordinator.wait(undefined, 5_000);
+ assert.equal(terminal?.status, "succeeded", terminal?.error);
+ assert.ok(
+ [...store.workers.values()].some((worker) =>
+ worker.promptPath.includes("discovery-0004"),
+ ),
+ );
+ }
+}
diff --git a/plugins/codex-security/mcp-app/tests/test_artifact_foundation.ts b/plugins/codex-security/mcp-app/tests/test_artifact_foundation.ts
index 73bcdb882..0927d4eb4 100644
--- a/plugins/codex-security/mcp-app/tests/test_artifact_foundation.ts
+++ b/plugins/codex-security/mcp-app/tests/test_artifact_foundation.ts
@@ -1,5 +1,7 @@
import { readJson } from "./support/json.ts";
import assert from "node:assert/strict";
+import { spawn } from "node:child_process";
+import { once } from "node:events";
import {
mkdir,
mkdtemp,
@@ -57,6 +59,7 @@ try {
await testScanContext();
await testWorkerStandardLayout();
await testSafeJsonAndJsonl();
+ await testAtomicReplacementAfterInterruptedWriter();
await testAtomicReplacement();
await testBoundedPagination();
await testUnsafeArtifacts();
@@ -329,6 +332,29 @@ async function testWorkerStandardLayout() {
assert.equal(reducer.deepReducer.claimedWorkers[0].id, "worker-1");
}
+async function testAtomicReplacementAfterInterruptedWriter() {
+ const destination = await io.artifactDestination(
+ context,
+ ["interrupted.json"],
+ "interrupted artifact",
+ );
+ const child = spawn(process.execPath, [
+ "-e",
+ 'require("node:fs").openSync(process.argv[1], "wx", 0o600); process.stdout.write("locked"); setInterval(() => {}, 1000);',
+ destination + ".lock",
+ ]);
+ await once(child.stdout, "data");
+ child.kill("SIGKILL");
+ await once(child, "close");
+ await io.replaceArtifactText(destination, "recovered\n");
+ assert.equal(await readFile(destination, "utf8"), "recovered\n");
+ const values = ["a", "b", "c"].map((value) => value.repeat(32_000));
+ await Promise.all(
+ values.map((value) => io.replaceArtifactText(destination, value)),
+ );
+ assert.ok(values.includes(await readFile(destination, "utf8")));
+}
+
async function testSafeJsonAndJsonl() {
const destination = await io.artifactDestination(
context,
diff --git a/plugins/codex-security/mcp-app/tests/test_deep_scan_cancel_response.ts b/plugins/codex-security/mcp-app/tests/test_deep_scan_cancel_response.ts
new file mode 100644
index 000000000..d5dde3b96
--- /dev/null
+++ b/plugins/codex-security/mcp-app/tests/test_deep_scan_cancel_response.ts
@@ -0,0 +1,205 @@
+import assert from "node:assert/strict";
+import { mkdir, readFile, rm, writeFile } from "node:fs/promises";
+import { execFileSync } from "node:child_process";
+import { join } from "node:path";
+import { temporaryDirectory } from "./support/temporary-directories.ts";
+import { fileURLToPath } from "node:url";
+import { importModule } from "./import-module.ts";
+
+interface FixtureScanResults {
+ progress: { status: string };
+}
+
+interface FixtureWorkspace {
+ setup: { submitted: boolean };
+ results: FixtureScanResults;
+}
+
+interface WorkbenchScanRecord {
+ workspace: FixtureWorkspace;
+ scan: FixtureScanResults;
+}
+
+interface CancelResponseFixture {
+ registry: {
+ cancelAndWait(
+ scanId: string,
+ reason: string,
+ persist: () => Promise<void>,
+ ): Promise<boolean>;
+ shutdown(): void;
+ };
+ workbench(args: string[]): Promise<FixtureWorkspace | WorkbenchScanRecord>;
+}
+
+declare global {
+ var cancelResponseFixture: CancelResponseFixture | undefined;
+}
+
+interface CancelResponseModule {
+ createCodexSecurityServer(): {
+ _registeredTools: {
+ cancel_codex_security_scan_from_app: {
+ handler(args: { scanId: string }): Promise<{
+ structuredContent: { workspace: FixtureWorkspace };
+ }>;
+ };
+ };
+ close(): Promise<void>;
+ };
+}
+
+const applicationRoot = fileURLToPath(new URL("../", import.meta.url));
+const source = await readFile(new URL("../server.ts", import.meta.url), "utf8");
+const calls: string[][] = [];
+let workspace: FixtureWorkspace;
+let fixtureRoot: string | undefined;
+let joining = false;
+let persisted: PromiseWithResolvers<void>;
+let release: PromiseWithResolvers<void>;
+const fixture: CancelResponseFixture = {
+ registry: {
+ async cancelAndWait(_scanId, _reason, persist) {
+ if (!joining) {
+ joining = true;
+ await persist();
+ persisted.resolve();
+ }
+ await release.promise;
+ return true;
+ },
+ shutdown() {},
+ },
+ async workbench(args) {
+ calls.push(args);
+ if (args[0] === "cancel-scan")
+ workspace.results.progress.status = "canceled";
+ return args[0] === "get-scan"
+ ? { workspace, scan: workspace.results }
+ : workspace;
+ },
+};
+globalThis.cancelResponseFixture = fixture;
+try {
+ const { createCodexSecurityServer } = (await importModule({
+ stdin: {
+ contents: source
+ .replace(
+ "const deepScanCoordinators = new DeepScanCoordinatorRegistry();",
+ "const deepScanCoordinators = globalThis.cancelResponseFixture.registry;",
+ )
+ .replace(
+ "async function runWorkbench(",
+ "const runWorkbench = (...args) => globalThis.cancelResponseFixture.workbench(...args);\nasync function unusedRunWorkbench(",
+ ),
+ loader: "ts",
+ resolveDir: applicationRoot,
+ },
+ define: {
+ __dirname: JSON.stringify(applicationRoot),
+ "import.meta.url": JSON.stringify(
+ new URL("../server.ts", import.meta.url).href,
+ ),
+ },
+ loader: { ".md": "text" },
+ })) as CancelResponseModule;
+ const server = createCodexSecurityServer();
+ const cancel =
+ server._registeredTools.cancel_codex_security_scan_from_app.handler;
+ workspace = {
+ setup: { submitted: true },
+ results: { progress: { status: "canceled" } },
+ };
+ persisted = Promise.withResolvers<void>();
+ release = Promise.withResolvers<void>();
+ const first = cancel({ scanId: "fixture-scan" });
+ await persisted.promise;
+ const second = cancel({ scanId: "fixture-scan" });
+ release.resolve();
+ for (const result of await Promise.all([first, second])) {
+ assert.deepEqual(result.structuredContent.workspace, workspace);
+ }
+ assert.deepEqual(
+ calls.map(([command]) => command),
+ ["cancel-scan", "get-scan"],
+ );
+ workspace = {
+ setup: { submitted: true },
+ results: { progress: { status: "failed" } },
+ };
+ assert.deepEqual(
+ (await cancel({ scanId: "fixture-scan" })).structuredContent.workspace,
+ workspace,
+ );
+ assert.equal(
+ calls.at(-1)![0],
+ "get-scan",
+ "late cancellation must not overwrite a saved failure",
+ );
+ fixtureRoot = await temporaryDirectory("cancel-saved-parent-", true);
+ const target = join(fixtureRoot, "repository");
+ const output = join(fixtureRoot, "output");
+ await mkdir(target);
+ await mkdir(output, { mode: 0o700 });
+ await writeFile(join(target, "fixture.py"), "value = 1\n");
+ const workbenchPath = fileURLToPath(
+ new URL("../../scripts/workbench_db.py", import.meta.url),
+ );
+ const run = (args: string[]): WorkbenchScanRecord & { scanId: string } =>
+ JSON.parse(
+ execFileSync(
+ process.env.PYTHON || "python3",
+ ["-I", "-B", workbenchPath, ...args],
+ {
+ encoding: "utf8",
+ env: {
+ ...process.env,
+ CODEX_SECURITY_STATE_DIR: join(fixtureRoot!, "state"),
+ },
+ },
+ ),
+ );
+ const registered = run([
+ "register-cli-scan",
+ "--repository",
+ target,
+ "--scan-dir",
+ output,
+ "--recipe-json",
+ JSON.stringify({
+ repository: target,
+ target: { kind: "repository", paths: [] },
+ mode: "standard",
+ config: {},
+ }),
+ ]);
+ assert.equal(
+ run(["get-scan", "--scan-id", registered.scanId]).workspace.results.progress
+ .status,
+ "running",
+ );
+ // Local completion can leave the durable parent running after discovery or a rejected failure write.
+ fixture.registry.cancelAndWait = async () => true;
+ fixture.workbench = async (args) => {
+ calls.push(args);
+ return run(args);
+ };
+ const parentCanceled = await cancel({ scanId: registered.scanId });
+ assert.equal(
+ parentCanceled.structuredContent.workspace.results.progress.status,
+ "canceled",
+ );
+ assert.equal(
+ run(["get-scan", "--scan-id", registered.scanId]).scan.progress.status,
+ "canceled",
+ );
+ assert.equal(
+ calls.at(-1)![0],
+ "cancel-scan",
+ "completed discovery still permits cancellation of its running parent",
+ );
+ await server.close();
+} finally {
+ delete globalThis.cancelResponseFixture;
+ if (fixtureRoot) await rm(fixtureRoot, { recursive: true, force: true });
+}
diff --git a/plugins/codex-security/mcp-app/tests/test_deep_scan_coordinator.ts b/plugins/codex-security/mcp-app/tests/test_deep_scan_coordinator.ts
index 08d8553aa..55ec4b9cb 100644
--- a/plugins/codex-security/mcp-app/tests/test_deep_scan_coordinator.ts
+++ b/plugins/codex-security/mcp-app/tests/test_deep_scan_coordinator.ts
@@ -18,6 +18,7 @@ import {
writeFile,
} from "node:fs/promises";
import path from "node:path";
+import { testDeepScanLifecycle } from "./deep_scan_lifecycle_cases.ts";
import {
DeepScanCoordinatorRegistry,
DeepScanNonRetryableError,
@@ -3708,6 +3709,14 @@ async function testNonRetryableReducerAbortsScanWithoutRetry(
}
try {
+ await testDeepScanLifecycle({
+ fixtureRun,
+ FakeStore,
+ FakeExecutor,
+ createCoordinator,
+ DeepScanCoordinatorRegistry,
+ immediateClock,
+ });
await testCappedQueueAndSerialDedup();
await testStandardWorkersReceiveExistingFalsePositiveFeedback();
await testDiscoveryWorkersKeepOneContextAfterPersistedUpdate();
diff --git a/plugins/codex-security/mcp-app/tests/test_mcp_app_smoke.ts b/plugins/codex-security/mcp-app/tests/test_mcp_app_smoke.ts
index f2ae81693..d98947b24 100644
--- a/plugins/codex-security/mcp-app/tests/test_mcp_app_smoke.ts
+++ b/plugins/codex-security/mcp-app/tests/test_mcp_app_smoke.ts
@@ -103,11 +103,6 @@ assert.deepEqual(
"The MCP process and SDK workers must inherit Codex, external-provider, and Bedrock authentication, AWS credential-chain settings, runtime paths, and enterprise proxy/certificate configuration.",
);
const serverSource = await readFile(path.join(mcpAppRoot, "server.ts"), "utf8");
-assert.match(
- serverSource,
- /timeout:\s*\[[^\]]*"start-prompt-only-scan"[^\]]*\]\.includes\(args\[0\] \?\? ""\)\s*\?\s*300_000\s*:\s*30_000/,
- "Prompt-only scan startup must use the same five-minute timeout as other scan starts.",
-);
const authenticatedArtifactClaimSource = serverSource.match(
/if \(\s*handoffClaimToken\s*&&\s*threadId[\s\S]*?authenticatedArtifactClaims\.set\(scanId,[\s\S]*?\n\s*\}/,
)?.[0];
diff --git a/plugins/codex-security/mcp-app/tests/test_workbench_process.ts b/plugins/codex-security/mcp-app/tests/test_workbench_process.ts
new file mode 100644
index 000000000..7d70734cd
--- /dev/null
+++ b/plugins/codex-security/mcp-app/tests/test_workbench_process.ts
@@ -0,0 +1,99 @@
+import assert from "node:assert/strict";
+import type { ExecFileOptionsWithStringEncoding } from "node:child_process";
+
+type WorkbenchProcessFixture = (
+ python: string,
+ args: string[],
+ options: ExecFileOptionsWithStringEncoding,
+) => Promise<{ stdout: string; stderr: string }>;
+
+declare global {
+ var workbenchProcessFixture: WorkbenchProcessFixture | undefined;
+}
+
+interface WorkbenchModule {
+ executeWorkbench(
+ python: string,
+ args: string[],
+ stateDir?: string,
+ input?: string | Buffer,
+ ): Promise<Record<string, unknown>>;
+}
+import { readFile } from "node:fs/promises";
+import { fileURLToPath } from "node:url";
+import { importModule } from "./import-module.ts";
+
+const applicationRoot = fileURLToPath(new URL("../", import.meta.url));
+const source = await readFile(new URL("../server.ts", import.meta.url), "utf8");
+const invocations: {
+ args: string[];
+ options: ExecFileOptionsWithStringEncoding;
+}[] = [];
+let failure: Error | undefined;
+globalThis.workbenchProcessFixture = (_python, args, options) => {
+ invocations.push({ args, options });
+ return failure
+ ? Promise.reject(failure)
+ : Promise.resolve({ stdout: "{}", stderr: "" });
+};
+try {
+ const { executeWorkbench } = (await importModule({
+ stdin: {
+ contents:
+ source.replace(
+ "const execFileAsync = promisify(execFile);",
+ "const execFileAsync = globalThis.workbenchProcessFixture;",
+ ) + "\nexport { executeWorkbench };",
+ loader: "ts",
+ resolveDir: applicationRoot,
+ },
+ define: {
+ __dirname: JSON.stringify(applicationRoot),
+ "import.meta.url": JSON.stringify(
+ new URL("../server.ts", import.meta.url).href,
+ ),
+ },
+ loader: { ".md": "text" },
+ })) as WorkbenchModule;
+ for (const command of [
+ "cancel-scan",
+ "fail-scan",
+ "preserve-scan-results",
+ "complete-scan",
+ "start-prompt-only-scan",
+ ]) {
+ await executeWorkbench("fixture-python", [command, "--scan-id", "fixture"]);
+ assert.equal(
+ invocations.at(-1)!.options.timeout,
+ 300_000,
+ `${command} must allow its saved-result publication to finish`,
+ );
+ }
+ await executeWorkbench("fixture-python", [
+ "inspect-target",
+ "--target-path",
+ "/fixture",
+ ]);
+ assert.equal(invocations.at(-1)!.options.timeout, 30_000);
+ failure = Object.assign(
+ new Error("Command failed: fixture-python\nfixture diagnostic"),
+ {
+ killed: true,
+ signal: "SIGTERM",
+ stderr: "fixture diagnostic",
+ stdout: "",
+ },
+ );
+ await assert.rejects(
+ executeWorkbench("fixture-python", ["cancel-scan"]),
+ (error: unknown) => {
+ assert.ok(error instanceof Error);
+ assert.match(error.message, /cancel-scan.*timed out.*300/);
+ assert.match(error.message, /fixture diagnostic/);
+ assert.equal(error.cause, failure);
+ return true;
+ },
+ );
+} finally {
+ delete globalThis.workbenchProcessFixture;
+}
diff --git a/plugins/codex-security/scripts/workbench_cli.py b/plugins/codex-security/scripts/workbench_cli.py
index 1d01c0d28..51f2603c0 100644
--- a/plugins/codex-security/scripts/workbench_cli.py
+++ b/plugins/codex-security/scripts/workbench_cli.py
@@ -161,7 +161,11 @@ def parse_args(description: str) -> argparse.Namespace:
recipe.add_argument("--recipe-json-stdin", action="store_true")
recipe.add_argument("--registration-json-stdin", action="store_true")
register_cli_scan.add_argument("--parent-scan-id")
- register_cli_scan.add_argument("--archive-existing", action="store_true")
+ register_cli_scan.add_argument(
+ "--archive-existing",
+ action="store_true",
+ help="Archive output in the registration transaction. Supports cancellable archival preparation.",
+ )
register_cli_scan.add_argument("--archived-scan-dir")
set_scan_thread = subparsers.add_parser("set-scan-thread")
diff --git a/plugins/codex-security/scripts/workbench_db.py b/plugins/codex-security/scripts/workbench_db.py
index 5968b470d..1300905ed 100644
--- a/plugins/codex-security/scripts/workbench_db.py
+++ b/plugins/codex-security/scripts/workbench_db.py
@@ -16,6 +16,7 @@ import sys
import tempfile
import time
import uuid
+from collections.abc import Callable
from contextlib import closing, contextmanager, nullcontext
from datetime import datetime, timedelta, timezone
from pathlib import Path, PurePosixPath
@@ -926,126 +927,148 @@ def _start_prompt_driven_scan(
target_identity = scan_target_identity(target, diff_target)
target_root = scan_target_root(args.scan_root, target)
- connection.execute("BEGIN IMMEDIATE")
- with connection:
- current_target = require_remediation_target(target_path)
- current_diff_target = (
- require_diff_target(
- current_target,
- args.diff_target_kind,
- args.diff_base_revision,
- args.diff_head_revision,
- args.diff_content_digest,
- )
- if args.mode == "diff"
- else None
- )
- if (
- scan_target_identity(current_target, current_diff_target) != target_identity
- or scan_diff_identity(current_diff_target) != diff_identity
- ):
- raise SystemExit(
- "The selected scan target changed while the scan was starting. Try again."
+ busy_timeout = int(connection.execute("PRAGMA busy_timeout").fetchone()[0])
+ for attempt in range(SQLITE_RETRY_ATTEMPTS):
+ try:
+ # Wait before hashing, then release the writer slot while reading the target.
+ connection.execute("BEGIN IMMEDIATE")
+ connection.rollback()
+ current_target = require_remediation_target(target_path)
+ current_diff_target = (
+ require_diff_target(
+ current_target,
+ args.diff_target_kind,
+ args.diff_base_revision,
+ args.diff_head_revision,
+ args.diff_content_digest,
+ )
+ if args.mode == "diff"
+ else None
)
- existing = connection.execute(
- """
- SELECT scans.* FROM scans
- JOIN workspaces ON workspaces.active_scan_id = scans.id
- WHERE workspaces.thread_id = ? AND workspaces.target_path = ?
- AND workspaces.default_scope = ? AND workspaces.default_mode = ?
- AND workspaces.user_context IS ? AND workspaces.target_summary IS ?
- AND workspaces.diff_target_kind IS ? AND workspaces.diff_base_revision IS ?
- AND workspaces.diff_head_revision IS ? AND workspaces.diff_content_digest IS ?
- AND workspaces.submitted = 1 AND scans.target_revision = ?
- AND scans.target_snapshot_digest IS ? AND scans.target_device = ?
- AND scans.target_inode = ? AND scans.status = 'running'
- AND scans.handoff_status = 'delivered'
- AND (
- (? = 0 AND scans.handoff_claim_token IS NULL)
- OR (
- ? = 1 AND scans.handoff_claim_token IS NOT NULL
- AND scans.continuation_thread_id = ?
- )
+ if (
+ scan_target_identity(current_target, current_diff_target) != target_identity
+ or scan_diff_identity(current_diff_target) != diff_identity
+ ):
+ raise SystemExit(
+ "The selected scan target changed while the scan was starting. Try again."
)
- ORDER BY scans.updated_at DESC, scans.started_at DESC, scans.id LIMIT 1
- """,
- (
- thread_id,
- target_path,
- scope,
- args.mode,
- user_context,
- target_summary,
- *diff_identity,
- *target_identity,
- int(headless_standard),
- int(headless_standard),
- thread_id,
- ),
- ).fetchone()
- if existing is not None:
- connection.commit()
- return {
- **scan_context(connection, existing["id"]),
- "startDisposition": "joined",
- }
- create_private_directory(target_root)
- workspace_id = str(uuid.uuid4())
- scan_id = str(uuid.uuid4())
- timestamp = now()
- target_id = ensure_security_target(connection, target_path)
- connection.execute(
- """
- INSERT INTO workspaces (
- id, thread_id, target_id, target_path, target_title, target_summary, default_scope,
- default_mode, user_context, diff_target_kind, diff_base_revision,
- diff_head_revision, diff_content_digest, submitted, created_at, updated_at
- ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, 1, ?, ?)
- """,
- (
- workspace_id,
- thread_id,
- target_id,
- target_path,
- target.name,
- target_summary,
- scope,
- args.mode,
- user_context,
- *diff_identity,
- timestamp,
- timestamp,
- ),
- )
- workspace = require_workspace(connection, workspace_id)
- insert_running_scan(
- connection,
- scan_id=scan_id,
- workspace=workspace,
- target=target,
- scope=scope,
- diff_target=diff_target,
- target_identity=target_identity,
- target_root=target_root,
- target_summary=target_summary,
- scope_file_count=scope_file_count,
- timestamp=timestamp,
- handoff_status="delivered",
- model=args.model,
- reasoning_effort=args.reasoning_effort,
- )
- if headless_standard:
- claimed = connection.execute(
+ # If another writer arrived during hashing, retry without waiting on a stale hash.
+ connection.execute("PRAGMA busy_timeout = 0")
+ try:
+ connection.execute("BEGIN IMMEDIATE")
+ finally:
+ connection.execute(f"PRAGMA busy_timeout = {busy_timeout}")
+ existing = connection.execute(
"""
- UPDATE scans
- SET handoff_claim_token = ?, continuation_thread_id = ?
- WHERE id = ? AND status = 'running' AND handoff_status = 'delivered'
- AND handoff_claim_token IS NULL AND continuation_thread_id IS NULL
+ SELECT scans.* FROM scans
+ JOIN workspaces ON workspaces.active_scan_id = scans.id
+ WHERE workspaces.thread_id = ? AND workspaces.target_path = ?
+ AND workspaces.default_scope = ? AND workspaces.default_mode = ?
+ AND workspaces.user_context IS ? AND workspaces.target_summary IS ?
+ AND workspaces.diff_target_kind IS ? AND workspaces.diff_base_revision IS ?
+ AND workspaces.diff_head_revision IS ? AND workspaces.diff_content_digest IS ?
+ AND workspaces.submitted = 1 AND scans.target_revision = ?
+ AND scans.target_snapshot_digest IS ? AND scans.target_device = ?
+ AND scans.target_inode = ? AND scans.status = 'running'
+ AND scans.handoff_status = 'delivered'
+ AND (
+ (? = 0 AND scans.handoff_claim_token IS NULL)
+ OR (
+ ? = 1 AND scans.handoff_claim_token IS NOT NULL
+ AND scans.continuation_thread_id = ?
+ )
+ )
+ ORDER BY scans.updated_at DESC, scans.started_at DESC, scans.id LIMIT 1
""",
- (str(uuid.uuid4()), thread_id, scan_id),
+ (
+ thread_id,
+ target_path,
+ scope,
+ args.mode,
+ user_context,
+ target_summary,
+ *diff_identity,
+ *target_identity,
+ int(headless_standard),
+ int(headless_standard),
+ thread_id,
+ ),
+ ).fetchone()
+ if existing is not None:
+ connection.commit()
+ return {
+ **scan_context(connection, existing["id"]),
+ "startDisposition": "joined",
+ }
+ create_private_directory(target_root)
+ workspace_id = str(uuid.uuid4())
+ scan_id = str(uuid.uuid4())
+ timestamp = now()
+ target_id = ensure_security_target(connection, target_path)
+ connection.execute(
+ """
+ INSERT INTO workspaces (
+ id, thread_id, target_id, target_path, target_title, target_summary, default_scope,
+ default_mode, user_context, diff_target_kind, diff_base_revision,
+ diff_head_revision, diff_content_digest, submitted, created_at, updated_at
+ ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, 1, ?, ?)
+ """,
+ (
+ workspace_id,
+ thread_id,
+ target_id,
+ target_path,
+ target.name,
+ target_summary,
+ scope,
+ args.mode,
+ user_context,
+ *diff_identity,
+ timestamp,
+ timestamp,
+ ),
)
- if claimed.rowcount != 1:
- raise SystemExit("Codex Security headless scan ownership could not be recorded.")
+ workspace = require_workspace(connection, workspace_id)
+ insert_running_scan(
+ connection,
+ scan_id=scan_id,
+ workspace=workspace,
+ target=target,
+ scope=scope,
+ diff_target=diff_target,
+ target_identity=target_identity,
+ target_root=target_root,
+ target_summary=target_summary,
+ scope_file_count=scope_file_count,
+ timestamp=timestamp,
+ handoff_status="delivered",
+ model=args.model,
+ reasoning_effort=args.reasoning_effort,
+ )
+ if headless_standard:
+ claimed = connection.execute(
+ """
+ UPDATE scans
+ SET handoff_claim_token = ?, continuation_thread_id = ?
+ WHERE id = ? AND status = 'running' AND handoff_status = 'delivered'
+ AND handoff_claim_token IS NULL AND continuation_thread_id IS NULL
+ """,
+ (str(uuid.uuid4()), thread_id, scan_id),
+ )
+ if claimed.rowcount != 1:
+ raise SystemExit(
+ "Codex Security headless scan ownership could not be recorded."
+ )
+ connection.commit()
+ break
+ except sqlite3.OperationalError as exc:
+ connection.rollback()
+ if attempt == SQLITE_RETRY_ATTEMPTS - 1 or not sqlite_busy(exc):
+ raise
+ time.sleep(0.05 * (2**attempt))
+ except BaseException:
+ connection.rollback()
+ raise
return {**scan_context(connection, scan_id), "startDisposition": "created"}
@@ -1598,13 +1621,22 @@ def complete_scan_locked(
return context
-def register_cli_scan(connection: sqlite3.Connection, args: argparse.Namespace) -> dict[str, Any]:
+def register_cli_scan(
+ connection: sqlite3.Connection,
+ args: argparse.Namespace,
+ *,
+ before_archive: Callable[[], None] | None = None,
+) -> dict[str, Any]:
repository = require_target(args.repository)
require_scannable_target(repository)
scan_dir = require_canonical_scan_directory(Path(args.scan_dir).expanduser())
- if scan_dir == repository or repository in scan_dir.parents:
- raise SystemExit("The scan artifact directory must be outside the selected target.")
- if next(scan_dir.iterdir(), None) is not None:
+ if scan_dir == repository or repository in scan_dir.parents or scan_dir in repository.parents:
+ raise SystemExit(
+ "The scan artifact directory must be outside and not contain the selected target."
+ )
+ if (not args.archive_existing or args.archived_scan_dir is not None) and next(
+ scan_dir.iterdir(), None
+ ) is not None:
raise SystemExit("The scan artifact directory must be empty before the scan starts.")
user_context = None
@@ -1656,63 +1688,77 @@ def register_cli_scan(connection: sqlite3.Connection, args: argparse.Namespace)
workspace_id = str(uuid.uuid4())
connection.execute("BEGIN IMMEDIATE")
- with connection:
- archive_scan(connection, args, scan_dir, timestamp, require_canonical_scan_directory)
- target_id = ensure_security_target(connection, str(repository))
- if parent_scan_id is not None:
- parent = require_scan(connection, parent_scan_id)
- if parent["target_id"] != target_id:
- raise SystemExit("A rerun must belong to the same repository as its parent scan.")
-
- connection.execute(
- """
- INSERT INTO workspaces (
- id, target_id, target_path, target_title, default_scope, default_mode,
- diff_target_kind, diff_base_revision, diff_head_revision,
- diff_content_digest, submitted, created_at, updated_at
- ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, 1, ?, ?)
- """,
- (
- workspace_id,
- target_id,
- str(repository),
- repository.name,
- scope,
- mode,
- *scan_diff_identity(diff_target),
- timestamp,
- timestamp,
- ),
- )
- workspace = require_workspace(connection, workspace_id)
- insert_running_scan(
+ try:
+ with archive_scan(
connection,
- scan_id=scan_id,
- workspace=workspace,
- target=repository,
- scope=scope,
- diff_target=diff_target,
- target_identity=target_identity,
- target_root=scan_dir.parent,
- target_summary=None,
- scope_file_count=scope_file_count,
- timestamp=timestamp,
- handoff_status="delivered",
- scan_dir=scan_dir,
- )
- connection.execute(
- "UPDATE scans SET recipe_json = ?, parent_scan_id = ?, user_context = ? WHERE id = ?",
- (
- json.dumps(recipe, allow_nan=False, separators=(",", ":"), sort_keys=True),
- parent_scan_id,
- user_context,
- scan_id,
- ),
- )
- if workflow_id is not None:
- register_workflow_scan(connection, workflow_id, scan_id, str(scan_dir), timestamp)
+ args,
+ scan_dir,
+ timestamp,
+ require_canonical_scan_directory,
+ before_archive=before_archive,
+ ) as archived_scan_dir:
+ target_id = ensure_security_target(connection, str(repository))
+ if parent_scan_id is not None:
+ parent = require_scan(connection, parent_scan_id)
+ if parent["target_id"] != target_id:
+ raise SystemExit(
+ "A rerun must belong to the same repository as its parent scan."
+ )
+
+ connection.execute(
+ """
+ INSERT INTO workspaces (
+ id, target_id, target_path, target_title, default_scope, default_mode,
+ diff_target_kind, diff_base_revision, diff_head_revision,
+ diff_content_digest, submitted, created_at, updated_at
+ ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, 1, ?, ?)
+ """,
+ (
+ workspace_id,
+ target_id,
+ str(repository),
+ repository.name,
+ scope,
+ mode,
+ *scan_diff_identity(diff_target),
+ timestamp,
+ timestamp,
+ ),
+ )
+ workspace = require_workspace(connection, workspace_id)
+ insert_running_scan(
+ connection,
+ scan_id=scan_id,
+ workspace=workspace,
+ target=repository,
+ scope=scope,
+ diff_target=diff_target,
+ target_identity=target_identity,
+ target_root=scan_dir.parent,
+ target_summary=None,
+ scope_file_count=scope_file_count,
+ timestamp=timestamp,
+ handoff_status="delivered",
+ scan_dir=scan_dir,
+ )
+ connection.execute(
+ "UPDATE scans SET recipe_json = ?, parent_scan_id = ?, user_context = ? WHERE id = ?",
+ (
+ json.dumps(recipe, allow_nan=False, separators=(",", ":"), sort_keys=True),
+ parent_scan_id,
+ user_context,
+ scan_id,
+ ),
+ )
+ if workflow_id is not None:
+ register_workflow_scan(connection, workflow_id, scan_id, str(scan_dir), timestamp)
+ connection.commit()
+ except BaseException:
+ connection.rollback()
+ raise
scan = require_scan(connection, scan_id)
return {
+ **({"archivedScanDir": str(archived_scan_dir)} if archived_scan_dir is not None else {}),
"contract": scan_contract(scan),
"scanDir": str(scan_dir),
"scanId": scan_id,
@@ -3231,7 +3277,7 @@ def read_json_object(path: Path) -> dict[str, Any]:
_WORKBENCH_PUBLICATION_CONTEXT = _WORKBENCH_DB_CONTEXT = SimpleNamespace(**globals())
-def main() -> None:
+def main(*, before_archive: Callable[[], None] | None = None) -> None:
# Workbench callers send UTF-8 even when Windows uses a legacy code page.
sys.stdin.reconfigure(encoding="utf-8")
args = parse_args(__doc__)
@@ -3326,7 +3372,7 @@ def main() -> None:
read_coverage=coverage_for_comparison,
)
elif args.command == "register-cli-scan":
- result = register_cli_scan(connection, args)
+ result = register_cli_scan(connection, args, before_archive=before_archive)
elif args.command == "set-scan-thread":
result = set_scan_thread(connection, args)
elif args.command == "set-scan-cost-limit":
diff --git a/plugins/codex-security/scripts/workbench_scan_start.py b/plugins/codex-security/scripts/workbench_scan_start.py
index e4aa20191..06f5509f2 100644
--- a/plugins/codex-security/scripts/workbench_scan_start.py
+++ b/plugins/codex-security/scripts/workbench_scan_start.py
@@ -9,7 +9,8 @@ import re
import sqlite3
import sys
import tempfile
-from collections.abc import Callable
+from collections.abc import Callable, Iterator
+from contextlib import contextmanager
from datetime import datetime, timezone
from pathlib import Path
@@ -17,6 +18,7 @@ from pathlib import Path
sys.path.insert(0, str(Path(__file__).resolve().parent))
from filesystem_identity import serialize_filesystem_identity
from finalize_scan_contract import write_scan_local_bytes
+from workbench.storage import state_dir
from workbench_feedback import get_scan_feedback
from workbench_target import (
directory_content_digest,
@@ -84,13 +86,16 @@ def stored_diff_target(row: sqlite3.Row) -> dict[str, str] | None:
return target
+@contextmanager
def archive_scan(
connection: sqlite3.Connection,
args: argparse.Namespace,
scan_dir: Path,
timestamp: str,
canonical_directory: Callable[[Path], Path],
-) -> None:
+ *,
+ before_archive: Callable[[], None] | None = None,
+) -> Iterator[Path | None]:
archived_scan_dir = (
canonical_directory(Path(args.archived_scan_dir).expanduser())
if args.archived_scan_dir is not None
@@ -106,44 +111,89 @@ def archive_scan(
previous_scan = connection.execute(
"SELECT id, status FROM scans WHERE scan_dir = ?", (str(scan_dir),)
).fetchone()
- if previous_scan is None:
- return
- if not args.archive_existing:
- raise SystemExit(
- "The scan artifact directory belongs to an existing scan. "
- "Use --archive-existing to preserve that scan and start a new one."
- )
- if previous_scan["status"] == "running":
- raise SystemExit("Cannot archive the output of a running scan.")
- artifacts = connection.execute(
- "SELECT kind, path FROM scan_artifacts WHERE scan_id = ?",
- (previous_scan["id"],),
- ).fetchall()
- if archived_scan_dir is None:
- if artifacts:
+ if previous_scan is not None:
+ if not args.archive_existing:
raise SystemExit(
- "The archived scan directory is required to preserve existing scan artifacts."
+ "The scan artifact directory belongs to an existing scan. "
+ "Use --archive-existing to preserve that scan and start a new one."
)
- archived_scan_dir = Path(
- tempfile.mkdtemp(prefix=f"{scan_dir.name}.previous-", dir=scan_dir.parent)
- ).resolve()
- connection.execute(
- "UPDATE scans SET scan_dir = ?, updated_at = ? WHERE id = ?",
- (str(archived_scan_dir), timestamp, previous_scan["id"]),
- )
- for artifact in artifacts:
- try:
- relative_path = Path(artifact["path"]).relative_to(scan_dir)
- except ValueError:
- continue
+ if previous_scan["status"] == "running":
+ raise SystemExit("Cannot archive the output of a running scan.")
+ has_contents = next(scan_dir.iterdir(), None) is not None
+ if previous_scan is None and not (args.archive_existing and has_contents):
+ yield archived_scan_dir
+ return
+ artifacts = (
connection.execute(
- "UPDATE scan_artifacts SET path = ? WHERE scan_id = ? AND kind = ?",
- (
- str(archived_scan_dir / relative_path),
- previous_scan["id"],
- artifact["kind"],
- ),
- )
+ "SELECT kind, path FROM scan_artifacts WHERE scan_id = ?", (previous_scan["id"],)
+ ).fetchall()
+ if previous_scan is not None
+ else []
+ )
+ moved = False
+ try:
+ if archived_scan_dir is None:
+ protected_directories = [
+ state_dir(),
+ *(
+ Path(row[2]).resolve().parent
+ for row in connection.execute("PRAGMA database_list")
+ if row[2]
+ ),
+ ]
+ if any(path == scan_dir or scan_dir in path.parents for path in protected_directories):
+ raise SystemExit(
+ "Cannot archive output containing the workbench state or active database."
+ )
+ if artifacts and not has_contents:
+ raise SystemExit(
+ "The archived scan directory is required to preserve existing scan artifacts."
+ )
+ if before_archive is not None:
+ before_archive()
+ archived_scan_dir = Path(
+ tempfile.mkdtemp(prefix=f"{scan_dir.name}.previous-", dir=scan_dir.parent)
+ ).resolve()
+ archived_scan_dir.rmdir()
+ scan_dir.rename(archived_scan_dir)
+ moved = True
+ scan_dir.mkdir(mode=0o700)
+ scan_dir.chmod(0o700)
+ if previous_scan is not None:
+ connection.execute(
+ "UPDATE scans SET scan_dir = ?, updated_at = ? WHERE id = ?",
+ (str(archived_scan_dir), timestamp, previous_scan["id"]),
+ )
+ for artifact in artifacts:
+ try:
+ relative_path = Path(artifact["path"]).relative_to(scan_dir)
+ except ValueError:
+ continue
+ connection.execute(
+ "UPDATE scan_artifacts SET path = ? WHERE scan_id = ? AND kind = ?",
+ (str(archived_scan_dir / relative_path), previous_scan["id"], artifact["kind"]),
+ )
+ # The caller commits registration before leaving this context. Older SDKs
+ # supplied an already-moved directory; only restore moves we own.
+ yield archived_scan_dir
+ except BaseException:
+ if moved:
+ if not connection.in_transaction:
+ current_owner = connection.execute(
+ "SELECT id FROM scans WHERE scan_dir = ?", (str(scan_dir),)
+ ).fetchone()
+ if current_owner is not None and (
+ previous_scan is None or current_owner["id"] != previous_scan["id"]
+ ):
+ # A post-commit interrupt must not undo the saved archive location.
+ raise
+ try:
+ scan_dir.rmdir()
+ except FileNotFoundError:
+ # Registration may fail before recreating the empty output directory.
+ pass
+ archived_scan_dir.rename(scan_dir)
+ raise
def insert_running_scan(
diff --git a/plugins/codex-security/tests/test_workbench_archive.py b/plugins/codex-security/tests/test_workbench_archive.py
new file mode 100644
index 000000000..4953f1881
--- /dev/null
+++ b/plugins/codex-security/tests/test_workbench_archive.py
@@ -0,0 +1,252 @@
+from __future__ import annotations
+
+import argparse
+import json
+import runpy
+import signal
+import sqlite3
+import subprocess
+import uuid
+from concurrent.futures import ThreadPoolExecutor
+from pathlib import Path
+
+import pytest
+from workbench_test_support import SCRIPT, run_workbench
+
+
+def recipe(target: Path) -> str:
+ return json.dumps(
+ {
+ "repository": str(target),
+ "target": {"kind": "repository", "paths": []},
+ "mode": "standard",
+ "config": {},
+ }
+ )
+
+
+def register(state: Path, target: Path, output: Path, *args: str) -> dict:
+ return run_workbench(
+ state,
+ "register-cli-scan",
+ "--repository",
+ str(target),
+ "--scan-dir",
+ str(output),
+ "--recipe-json",
+ recipe(target),
+ *args,
+ )
+
+
+def previous_scan(tmp_path):
+ state, target, output = (tmp_path / name for name in ("state", "target", "scan"))
+ target.mkdir()
+ (target / "fixture.py").write_text("value = 1\n")
+ output.mkdir(mode=0o700)
+ previous = register(state, target, output)
+ (output / "report.md").write_text("previous scan\n")
+ with sqlite3.connect(state / "workbench.sqlite3") as connection:
+ connection.execute(
+ "INSERT INTO scan_artifacts VALUES (?, 'markdownReport', ?, 'before')",
+ (previous["scanId"], str(output / "report.md")),
+ )
+ return state, target, output, previous["scanId"]
+
+
+def mark_stopped(state, scan_id):
+ with sqlite3.connect(state / "workbench.sqlite3") as connection:
+ connection.execute("UPDATE scans SET status = 'failed' WHERE id = ?", (scan_id,))
+
+
+def stored_paths(state, scan_id):
+ with sqlite3.connect(state / "workbench.sqlite3") as connection:
+ return (
+ connection.execute("SELECT scan_dir FROM scans WHERE id = ?", (scan_id,)).fetchone()[0],
+ connection.execute(
+ "SELECT path FROM scan_artifacts WHERE scan_id = ?", (scan_id,)
+ ).fetchone()[0],
+ )
+
+
+def test_concurrent_registration_keeps_running_scan_output(previous_scan):
+ state, target, output, scan_id = previous_scan
+
+ def attempt(_):
+ with pytest.raises(subprocess.CalledProcessError) as error:
+ register(state, target, output, "--archive-existing")
+ assert "Cannot archive the output of a running scan" in error.value.stderr
+
+ with ThreadPoolExecutor(max_workers=2) as pool:
+ list(pool.map(attempt, range(2)))
+ assert (output / "report.md").read_text() == "previous scan\n"
+ assert list(output.parent.glob("scan.previous-*")) == []
+ assert stored_paths(state, scan_id) == (str(output), str(output / "report.md"))
+
+
+def test_only_one_concurrent_registration_archives_previous_output(previous_scan):
+ state, target, output, scan_id = previous_scan
+ mark_stopped(state, scan_id)
+
+ def attempt(_):
+ try:
+ return register(state, target, output, "--archive-existing")
+ except subprocess.CalledProcessError as error:
+ assert "Cannot archive the output of a running scan" in error.stderr
+ return None
+
+ with ThreadPoolExecutor(max_workers=2) as pool:
+ results = [result for result in pool.map(attempt, range(2)) if result is not None]
+ assert len(results) == 1
+ archive = Path(results[0]["archivedScanDir"])
+ assert (archive / "report.md").read_text() == "previous scan\n"
+ assert list(output.iterdir()) == []
+ assert stored_paths(state, scan_id) == (str(archive), str(archive / "report.md"))
+
+
+def test_registration_rejection_restores_files_and_database(previous_scan):
+ state, target, output, scan_id = previous_scan
+ mark_stopped(state, scan_id)
+ with pytest.raises(subprocess.CalledProcessError):
+ register(state, target, output, "--archive-existing", "--parent-scan-id", str(uuid.uuid4()))
+ assert (output / "report.md").read_text() == "previous scan\n"
+ assert list(output.parent.glob("scan.previous-*")) == []
+ assert stored_paths(state, scan_id) == (str(output), str(output / "report.md"))
+
+
+def test_commit_failure_restores_archived_output(previous_scan, monkeypatch):
+ state, target, output, scan_id = previous_scan
+ mark_stopped(state, scan_id)
+ monkeypatch.setenv("CODEX_SECURITY_STATE_DIR", str(state))
+ module = runpy.run_path(str(SCRIPT))
+ connection = module["connect"]()
+
+ class FailedCommit:
+ def __getattr__(self, name):
+ return getattr(connection, name)
+
+ def commit(self):
+ raise sqlite3.OperationalError("fixture commit failed")
+
+ args = argparse.Namespace(
+ repository=str(target),
+ scan_dir=str(output),
+ archive_existing=True,
+ archived_scan_dir=None,
+ recipe_json=recipe(target),
+ recipe_json_stdin=False,
+ registration_json_stdin=False,
+ parent_scan_id=None,
+ )
+ try:
+ with pytest.raises(sqlite3.OperationalError, match="fixture commit failed"):
+ module["register_cli_scan"](FailedCommit(), args)
+ finally:
+ connection.close()
+ assert (output / "report.md").read_text() == "previous scan\n"
+ assert list(output.parent.glob("scan.previous-*")) == []
+ assert stored_paths(state, scan_id) == (str(output), str(output / "report.md"))
+
+
+def test_legacy_caller_can_supply_already_archived_output(previous_scan):
+ state, target, output, scan_id = previous_scan
+ mark_stopped(state, scan_id)
+ archive = output.with_name("scan.previous-fixture")
+ output.rename(archive)
+ output.mkdir(mode=0o700)
+ registered = register(
+ state, target, output, "--archive-existing", "--archived-scan-dir", str(archive)
+ )
+ assert registered["archivedScanDir"] == str(archive)
+ assert (archive / "report.md").read_text() == "previous scan\n"
+ assert stored_paths(state, scan_id) == (str(archive), str(archive / "report.md"))
+
+
+def test_registration_does_not_archive_an_ancestor_of_its_repository(tmp_path):
+ state = tmp_path / "state"
+ output = tmp_path / "output"
+ target = output / "repository"
+ target.mkdir(parents=True)
+ output.chmod(0o700)
+ source = target / "fixture.py"
+ source.write_text("preserved source\n")
+ with pytest.raises(subprocess.CalledProcessError) as error:
+ register(state, target, output, "--archive-existing")
+ assert "selected target" in error.value.stderr
+ assert source.read_text() == "preserved source\n"
+ assert list(tmp_path.glob("output.previous-*")) == []
+ with sqlite3.connect(state / "workbench.sqlite3") as connection:
+ assert connection.execute("SELECT COUNT(*) FROM scans").fetchone()[0] == 0
+
+
[email protected]("committed", [False, True])
+def test_interrupted_commit_keeps_files_at_their_saved_paths(previous_scan, monkeypatch, committed):
+ state, target, output, scan_id = previous_scan
+ mark_stopped(state, scan_id)
+ monkeypatch.setenv("CODEX_SECURITY_STATE_DIR", str(state))
+ module = runpy.run_path(str(SCRIPT))
+ connection = module["connect"]()
+
+ class InterruptedCommit:
+ def __getattr__(self, name):
+ return getattr(connection, name)
+
+ def commit(self):
+ if committed:
+ connection.commit()
+ else:
+ # SQLite can end a failed transaction itself before reporting its error.
+ connection.rollback()
+ signal.raise_signal(signal.SIGINT)
+
+ args = argparse.Namespace(
+ repository=str(target),
+ scan_dir=str(output),
+ archive_existing=True,
+ archived_scan_dir=None,
+ recipe_json=recipe(target),
+ recipe_json_stdin=False,
+ registration_json_stdin=False,
+ parent_scan_id=None,
+ )
+ try:
+ with pytest.raises(KeyboardInterrupt):
+ module["register_cli_scan"](InterruptedCommit(), args)
+ finally:
+ connection.close()
+ saved_directory, saved_report = stored_paths(state, scan_id)
+ assert Path(saved_report).read_text() == "previous scan\n"
+ assert (Path(saved_directory) / "report.md").read_text() == "previous scan\n"
+ with sqlite3.connect(state / "workbench.sqlite3") as connection:
+ assert connection.execute("SELECT COUNT(*) FROM scans").fetchone()[0] == 1 + committed
+ assert list(output.iterdir()) == ([] if committed else [output / "report.md"])
+
+
[email protected]("layout", ["same", "nested", "alias"])
+def test_registration_does_not_archive_its_active_workbench_state(tmp_path, layout):
+ target = tmp_path / "target"
+ target.mkdir()
+ (target / "fixture.py").write_text("preserved source\n")
+ output = tmp_path / "output"
+ output.mkdir(mode=0o700)
+ state = output if layout == "same" else output / "state"
+ state.mkdir(mode=0o700, exist_ok=True)
+ if layout == "alias":
+ alias = tmp_path / "state-alias"
+ try:
+ alias.symlink_to(state, target_is_directory=True)
+ except OSError:
+ pytest.skip("Directory symlinks are unavailable")
+ state = alias
+ saved = tmp_path / "saved"
+ saved.mkdir(mode=0o700)
+ previous = register(state, target, saved)
+ (saved / "report.md").write_text("saved report\n")
+ with pytest.raises(subprocess.CalledProcessError) as error:
+ register(state, target, output, "--archive-existing")
+ assert "workbench state" in error.value.stderr
+ assert list(tmp_path.glob("output.previous-*")) == []
+ assert (saved / "report.md").read_text() == "saved report\n"
+ with sqlite3.connect(state / "workbench.sqlite3") as connection:
+ assert connection.execute("SELECT id FROM scans").fetchall() == [(previous["scanId"],)]
diff --git a/plugins/codex-security/tests/test_workbench_prompt_only_scan.py b/plugins/codex-security/tests/test_workbench_prompt_only_scan.py
index 2ec1e131a..630aa105e 100644
--- a/plugins/codex-security/tests/test_workbench_prompt_only_scan.py
+++ b/plugins/codex-security/tests/test_workbench_prompt_only_scan.py
@@ -8,7 +8,7 @@ import uuid
from concurrent.futures import ThreadPoolExecutor
from contextlib import closing
from pathlib import Path
-from threading import Event
+from threading import Barrier, Event, local
from unittest import mock
import pytest
@@ -216,6 +216,248 @@ def test_prompt_only_standard_phase_uses_latest_persisted_scan_context(
assert next_phase["scan"]["userContext"] == updated_context
[email protected]("headless_standard", [False, True])
[email protected]("target_changes", [False, True])
[email protected]("existing_scan", [False, True])
+def test_prompt_scan_revalidates_target_after_concurrent_database_write(
+ tmp_path: Path, target_changes: bool, existing_scan: bool, headless_standard: bool
+) -> None:
+ state_dir = tmp_path / "state"
+ target = tmp_path / "target"
+ target.mkdir()
+ source = target / "fixture.py"
+ source.write_text("original\n")
+ saved = create_saved_workspace(state_dir, target)
+ namespace = runpy.run_path(str(SCRIPT), run_name="prompt_scan_lock_test")
+ start = namespace["_start_prompt_driven_scan"]
+ globals_ = start.__globals__
+ identity = globals_["scan_target_identity"]
+ calls = 0
+ args = argparse.Namespace(
+ thread_id="thread-fixture",
+ target_path=str(target),
+ scope=".",
+ mode="standard",
+ diff_target_kind=None,
+ diff_base_revision=None,
+ diff_head_revision=None,
+ diff_content_digest=None,
+ user_context=None,
+ user_context_stdin=False,
+ target_summary=None,
+ scan_root=str(tmp_path / "scans"),
+ model=None,
+ reasoning_effort=None,
+ )
+
+ def inspect_identity(*args, **kwargs):
+ nonlocal calls
+ calls += 1
+ # Unrelated progress commits during every hash must not prevent startup.
+ with sqlite3.connect(state_dir / "workbench.sqlite3", timeout=0) as other:
+ other.execute("BEGIN IMMEDIATE")
+ other.execute(
+ "UPDATE workspaces SET target_summary = ? WHERE id = ?",
+ (f"Concurrent setup update {calls}", saved["id"]),
+ )
+ if calls == 2:
+ if existing_scan:
+ other.execute("UPDATE scans SET status = 'failed'")
+ if target_changes:
+ source.write_text("changed during hashing\n")
+ return identity(*args, **kwargs)
+
+ with (
+ mock.patch.dict(os.environ, {"CODEX_SECURITY_STATE_DIR": str(state_dir)}),
+ ):
+ connection = globals_["connect"]()
+ connection.execute("PRAGMA busy_timeout = 1375")
+ if existing_scan:
+ start(connection, args, headless_standard=headless_standard)
+ try:
+ with mock.patch.dict(globals_, {"scan_target_identity": inspect_identity}):
+ if target_changes:
+ with pytest.raises(SystemExit, match="target changed"):
+ start(connection, args, headless_standard=headless_standard)
+ else:
+ assert start(connection, args, headless_standard=headless_standard)[
+ "startDisposition"
+ ] == ("created")
+ count = connection.execute("SELECT COUNT(*) FROM scans").fetchone()[0]
+ assert count == int(existing_scan) + (0 if target_changes else 1)
+ assert not connection.in_transaction
+ assert connection.execute("PRAGMA busy_timeout").fetchone()[0] == 1375
+ finally:
+ connection.close()
+ assert calls == 2
+
+
[email protected]("target_changes", [False, True])
[email protected]("writer_arrival", ["before_hash", "during_hash"])
+def test_prompt_scan_revalidates_target_after_database_lock_wait(
+ tmp_path: Path, target_changes: bool, writer_arrival: str
+) -> None:
+ state_dir = tmp_path / "state"
+ target = tmp_path / "target"
+ target.mkdir()
+ source = target / "fixture.py"
+ source.write_text("original\n")
+ namespace = runpy.run_path(str(SCRIPT), run_name="prompt_scan_lock_test")
+ start = namespace["_start_prompt_driven_scan"]
+ globals_ = start.__globals__
+ identity = globals_["scan_target_identity"]
+ calls = 0
+ args = argparse.Namespace(
+ thread_id="thread-fixture",
+ target_path=str(target),
+ scope=".",
+ mode="standard",
+ diff_target_kind=None,
+ diff_base_revision=None,
+ diff_head_revision=None,
+ diff_content_digest=None,
+ user_context=None,
+ user_context_stdin=False,
+ target_summary=None,
+ scan_root=str(tmp_path / "scans"),
+ model=None,
+ reasoning_effort=None,
+ )
+
+ acquiring = Event()
+
+ def inspect_identity(*args, **kwargs):
+ nonlocal calls
+ calls += 1
+ if calls == 1:
+ # Initial hashing does not need the database writer lock.
+ with sqlite3.connect(state_dir / "workbench.sqlite3", timeout=0) as other:
+ other.execute("BEGIN IMMEDIATE")
+ result = identity(*args, **kwargs)
+ if calls == 2 and writer_arrival == "during_hash":
+ blocker.execute("BEGIN IMMEDIATE")
+ if target_changes:
+ source.write_text("changed before writer admission\n")
+ return result
+
+ with mock.patch.dict(os.environ, {"CODEX_SECURITY_STATE_DIR": str(state_dir)}):
+ connection = globals_["connect"]()
+ blocker = sqlite3.connect(state_dir / "workbench.sqlite3", check_same_thread=False)
+
+ class ContendedConnection:
+ def __getattr__(self, name):
+ return getattr(connection, name)
+
+ def execute(self, sql, *parameters):
+ if (
+ writer_arrival == "before_hash"
+ and sql == "BEGIN IMMEDIATE"
+ and not acquiring.is_set()
+ ):
+ blocker.execute("BEGIN IMMEDIATE")
+ acquiring.set()
+ try:
+ return connection.execute(sql, *parameters)
+ except sqlite3.OperationalError:
+ if writer_arrival == "during_hash" and blocker.in_transaction:
+ assert connection.execute("PRAGMA busy_timeout").fetchone()[0] == 0
+ blocker.commit()
+ raise
+
+ def release_writer():
+ assert acquiring.wait(timeout=10)
+ if target_changes:
+ source.write_text("changed during lock acquisition\n")
+ blocker.commit()
+
+ try:
+ with (
+ ThreadPoolExecutor(max_workers=1) as pool,
+ mock.patch.dict(globals_, {"scan_target_identity": inspect_identity}),
+ ):
+ released = pool.submit(release_writer) if writer_arrival == "before_hash" else None
+ if target_changes:
+ with pytest.raises(SystemExit, match="target changed"):
+ start(ContendedConnection(), args, headless_standard=False)
+ else:
+ assert (
+ start(ContendedConnection(), args, headless_standard=False)[
+ "startDisposition"
+ ]
+ == "created"
+ )
+ if released is not None:
+ released.result()
+ count = connection.execute("SELECT COUNT(*) FROM scans").fetchone()[0]
+ assert count == (0 if target_changes else 1)
+ assert not connection.in_transaction
+ assert connection.execute("PRAGMA busy_timeout").fetchone()[0] == 5000
+ finally:
+ acquiring.set()
+ blocker.close()
+ connection.close()
+ assert calls == (2 if writer_arrival == "before_hash" else 3)
+
+
[email protected]("headless_standard", [False, True])
+def test_concurrent_prompt_scan_starts_join_the_winner(
+ tmp_path: Path, headless_standard: bool
+) -> None:
+ state_dir = tmp_path / "state"
+ target = tmp_path / "target"
+ target.mkdir()
+ (target / "fixture.py").write_text("original\n")
+ create_saved_workspace(state_dir, target)
+ namespace = runpy.run_path(str(SCRIPT), run_name="prompt_scan_concurrent_test")
+ start = namespace["_start_prompt_driven_scan"]
+ globals_ = start.__globals__
+ identity = globals_["scan_target_identity"]
+ ready = Barrier(2)
+ caller = local()
+ args = argparse.Namespace(
+ thread_id="thread-fixture",
+ target_path=str(target),
+ scope=".",
+ mode="standard",
+ diff_target_kind=None,
+ diff_base_revision=None,
+ diff_head_revision=None,
+ diff_content_digest=None,
+ user_context=None,
+ user_context_stdin=False,
+ target_summary=None,
+ scan_root=str(tmp_path / "scans"),
+ model=None,
+ reasoning_effort=None,
+ )
+
+ def synchronized_identity(*args, **kwargs):
+ caller.calls = getattr(caller, "calls", 0) + 1
+ result = identity(*args, **kwargs)
+ if caller.calls == 2:
+ ready.wait(timeout=10)
+ return result
+
+ def begin():
+ connection = globals_["connect"]()
+ try:
+ return start(connection, args, headless_standard=headless_standard)
+ finally:
+ connection.close()
+
+ with (
+ mock.patch.dict(os.environ, {"CODEX_SECURITY_STATE_DIR": str(state_dir)}),
+ mock.patch.dict(globals_, {"scan_target_identity": synchronized_identity}),
+ ThreadPoolExecutor(max_workers=2) as pool,
+ ):
+ futures = [pool.submit(begin) for _ in range(2)]
+ results = [future.result(timeout=20) for future in futures]
+ assert sorted(result["startDisposition"] for result in results) == ["created", "joined"]
+ assert results[0]["scan"]["scanId"] == results[1]["scan"]["scanId"]
+ with sqlite3.connect(state_dir / "workbench.sqlite3") as connection:
+ assert connection.execute("SELECT COUNT(*) FROM scans").fetchone()[0] == 1
+
+
def test_setup_scan_reuses_checked_target_metadata(tmp_path: Path) -> None:
state_dir = tmp_path / "state"
target = tmp_path / "target"
diff --git a/sdk/typescript/src/api.ts b/sdk/typescript/src/api.ts
index 988e2ca30..86a0088e7 100644
--- a/sdk/typescript/src/api.ts
+++ b/sdk/typescript/src/api.ts
@@ -201,6 +201,7 @@ import {
planOutputArchive,
prepareScanArtifactRestorer,
prepareOutputDir,
+ prepareScanRegistrationOutput,
preparePersistentOutputRoot,
probeCodexSandbox,
requireModelSafeOutputDir,
@@ -1208,7 +1209,6 @@ export class CodexSecurity {
let latestCost: Readonly<ScanCost> | null = null;
let notifiedLimit: number | undefined;
let scanDir = "";
- let archivedScanDir: string | null = null;
let targetPathsFile: string | null = null;
let knowledgeBase: PreparedKnowledgeBase | null = null;
let costTracker: ScanCostTracker | null = null;
@@ -1341,17 +1341,21 @@ export class CodexSecurity {
scanDir =
options.resumeScanId !== undefined
? requestedOutput!
- : await (this.#dependencies.prepareOutputDir ?? prepareOutputDir)(
+ : await (
+ this.#dependencies.prepareOutputDir ??
+ prepareScanRegistrationOutput
+ )(
requestedOutput ?? undefined,
basename(repo),
scanOutputRoot,
(path) => requireOutputOutsideRepository(protectedRoot, path),
options.archiveExisting,
- archiveObserver(options, (path) => (archivedScanDir = path)),
);
requireOutputOutsideRepository(protectedRoot, scanDir);
requireModelSafeOutputDir(scanDir);
- notifyObserver(options, "onOutputDirReady")(scanDir);
+ if (!options.archiveExisting) {
+ notifyObserver(options, "onOutputDirReady")(scanDir);
+ }
checkOpen();
const workerProviderEnvironment = {
@@ -1625,6 +1629,7 @@ export class CodexSecurity {
signal,
failureMessage: "Could not save the Codex Security scan",
};
+ checkOpen();
const registration =
options.resumeScanId !== undefined
? await workbench(workbenchOptions, [
@@ -1644,9 +1649,6 @@ export class CodexSecurity {
...(options.archiveExisting === true
? ["--archive-existing"]
: []),
- ...(archivedScanDir === null
- ? []
- : ["--archived-scan-dir", archivedScanDir]),
...(options.parentScanId === undefined
? []
: ["--parent-scan-id", options.parentScanId]),
@@ -1759,6 +1761,16 @@ export class CodexSecurity {
});
}
activeScan = { id: scanId, options: workbenchOptions };
+ if (typeof registration["archivedScanDir"] === "string") {
+ notifyObserver(
+ options,
+ "onOutputArchived",
+ )(registration["archivedScanDir"]);
+ }
+ if (options.archiveExisting) {
+ notifyObserver(options, "onOutputDirReady")(scanDir);
+ }
+ throwIfAborted(signal, scanDir);
if (mode === "deep" && options.onDeepProgress !== undefined) {
let progressWarningReported = false;
deepProgressTracker = new DeepScanProgressTracker({
@@ -3188,17 +3200,17 @@ export class CodexSecurity {
basename(local.repository),
)
: undefined;
- let archivedScanDir: string | undefined;
- scanDir = await prepareOutputDir(
+ scanDir = await prepareScanRegistrationOutput(
local.outputDir ?? undefined,
basename(local.repository),
outputRoot,
(path) => requireOutputOutsideRepository(local.protectedRoot, path),
options.archiveExisting,
- archiveObserver(options, (path) => (archivedScanDir = path)),
);
requireModelSafeOutputDir(scanDir);
- notifyObserver(options, "onOutputDirReady")(scanDir);
+ if (!options.archiveExisting) {
+ notifyObserver(options, "onOutputDirReady")(scanDir);
+ }
const revision = await repositoryRevision(local.repository, signal);
const { model } = scanModelConfiguration({
...DEFAULT_CODEX_CONFIG,
@@ -3214,6 +3226,7 @@ export class CodexSecurity {
signal,
failureMessage: "Could not save the mock scan",
};
+ throwIfAborted(signal, scanDir);
const registration = await workbench(
workbenchOptions,
[
@@ -3224,9 +3237,6 @@ export class CodexSecurity {
scanDir,
"--registration-json-stdin",
...(options.archiveExisting ? ["--archive-existing"] : []),
- ...(archivedScanDir === undefined
- ? []
- : ["--archived-scan-dir", archivedScanDir]),
...(options.parentScanId === undefined
? []
: ["--parent-scan-id", options.parentScanId]),
@@ -3266,6 +3276,16 @@ export class CodexSecurity {
);
}
activeScan = { id: scanId, options: workbenchOptions };
+ if (typeof registration["archivedScanDir"] === "string") {
+ notifyObserver(
+ options,
+ "onOutputArchived",
+ )(registration["archivedScanDir"]);
+ }
+ if (options.archiveExisting) {
+ notifyObserver(options, "onOutputDirReady")(scanDir);
+ }
+ throwIfAborted(signal, scanDir);
notifyObserver(options, "onScanStarted")();
await writeMockScanDraft(
scanDir,
@@ -4391,12 +4411,6 @@ export function selectedScanEnvironment(
);
}
-const archiveObserver =
- (options: ScanOptions, save: (path: string) => void) => (path: string) => {
- save(path);
- notifyObserver(options, "onOutputArchived")(path);
- };
-
function notifyObserver<Name extends ScanObserverName>(
options: Pick<ScanOptions & SecurityPolicyOptions, Name | "onObserverError">,
observerName: Name,
diff --git a/sdk/typescript/src/import-scan.ts b/sdk/typescript/src/import-scan.ts
index 48d1856a6..c1b8e2cf2 100644
--- a/sdk/typescript/src/import-scan.ts
+++ b/sdk/typescript/src/import-scan.ts
@@ -28,7 +28,7 @@ import {
codexSecurityStateDirectory,
expandHome,
pluginMetadata,
- prepareOutputDir,
+ prepareScanRegistrationOutput,
preparePersistentOutputRoot,
requireOutputOutsideRepository,
requirePrivateOutputDirectory,
@@ -186,16 +186,12 @@ export async function importScan(
options.outputDir === undefined
? await preparePersistentOutputRoot(stateDirectory, "scans", "imports")
: undefined;
- let archivedScanDir: string | undefined;
- scanDir = await prepareOutputDir(
+ scanDir = await prepareScanRegistrationOutput(
options.outputDir,
"import",
outputRoot,
(path) => requireOutputOutsideRepository(repository, path),
options.archiveExisting,
- (path) => {
- archivedScanDir = path;
- },
);
const workbenchOptions: WorkbenchCommandOptions = {
python,
@@ -204,6 +200,7 @@ export async function importScan(
signal,
failureMessage: "Could not save the imported scan",
};
+ signal?.throwIfAborted();
const registration = await workbench(
workbenchOptions,
[
@@ -214,9 +211,6 @@ export async function importScan(
scanDir,
"--registration-json-stdin",
...(options.archiveExisting ? ["--archive-existing"] : []),
- ...(archivedScanDir === undefined
- ? []
- : ["--archived-scan-dir", archivedScanDir]),
...(options.parentScanId === undefined
? []
: ["--parent-scan-id", options.parentScanId]),
@@ -244,6 +238,7 @@ export async function importScan(
);
}
activeScan = { id: scanId, options: workbenchOptions };
+ signal?.throwIfAborted();
const bound = bindImportedFindings(
findings,
options.format,
diff --git a/sdk/typescript/src/runtime.ts b/sdk/typescript/src/runtime.ts
index 48abb6ee2..32e7f3217 100644
--- a/sdk/typescript/src/runtime.ts
+++ b/sdk/typescript/src/runtime.ts
@@ -1549,6 +1549,22 @@ export async function preparePersistentOutputRoot(
return root;
}
+const ARCHIVE_READY = "codex-security-archive-ready\n";
+const ARCHIVE_REGISTRATION_PROGRAM = String.raw`
+import io, json, runpy, sys
+control = sys.stdin
+payload = json.loads(control.readline())
+sys.stdin = io.TextIOWrapper(io.BytesIO(payload.encode("utf-8")), encoding="utf-8")
+sys.argv = sys.argv[1:]
+workbench = runpy.run_path(sys.argv[0])
+def before_archive():
+ sys.stdout.buffer.write(b"codex-security-archive-ready\n")
+ sys.stdout.buffer.flush()
+ if control.readline() != "\n":
+ raise SystemExit("Scan registration canceled before archival.")
+workbench["main"](before_archive=before_archive)
+`;
+
const workbenchComparisonSupport = new Map<
string,
{ stdin: boolean; related: boolean }
@@ -1560,9 +1576,11 @@ export async function runWorkbench(
input?: string,
): Promise<JsonObject> {
const script = join(options.pluginRoot, "scripts", "workbench_db.py");
+ let signal = options.signal;
const run = async (
arguments_: readonly string[],
input?: string,
+ archiveHandshake = false,
): Promise<string> => {
const native = [
"database-info",
@@ -1599,7 +1617,15 @@ export async function runWorkbench(
{ command },
native
? [join(options.pluginRoot, "mcp", "helpers.mjs"), ...arguments_]
- : ["-I", "-X", "utf8", "-B", script, ...arguments_],
+ : [
+ "-I",
+ "-X",
+ "utf8",
+ "-B",
+ ...(archiveHandshake ? ["-c", ARCHIVE_REGISTRATION_PROGRAM] : []),
+ script,
+ ...arguments_,
+ ],
pluginHelperEnvironment(node?.environment ?? options.environment),
// The SDK owns configuration normalization; the helper receives its resolved location.
native
@@ -1612,7 +1638,8 @@ export async function runWorkbench(
},
)
: input,
- options.signal,
+ signal,
+ archiveHandshake,
);
if (!result.success) {
throw new Error(
@@ -1624,8 +1651,69 @@ export async function runWorkbench(
return result.stdout;
};
let stdout: string;
+ const savedScanIdentities = async (scanDir: string): Promise<string> => {
+ const result: unknown = JSON.parse(
+ await run(["list-scans", "--scan-root", scanDir]),
+ );
+ if (!isRecord(result) || !Array.isArray(result["scans"])) {
+ throw new Error(
+ "The workbench returned an invalid scan history response.",
+ );
+ }
+ return JSON.stringify(
+ result["scans"]
+ .map((scan: unknown) => {
+ if (
+ !isRecord(scan) ||
+ typeof scan["scanId"] !== "string" ||
+ typeof scan["scanDir"] !== "string"
+ ) {
+ throw new Error(
+ "The workbench returned an invalid scan history entry.",
+ );
+ }
+ return JSON.stringify([scan["scanId"], scan["scanDir"]]);
+ })
+ .sort(),
+ );
+ };
+ let legacyArchive:
+ { scanDir: string; archiveDir: string; savedScans: string } | undefined;
try {
const arguments_ = [...args];
+ let archiveHandshake = false;
+ if (
+ arguments_[0] === "register-cli-scan" &&
+ arguments_.includes("--archive-existing") &&
+ !arguments_.includes("--archived-scan-dir")
+ ) {
+ const help = (await run(["register-cli-scan", "--help"])).replace(
+ /\s+/gu,
+ " ",
+ );
+ if (help.includes("Supports cancellable archival preparation.")) {
+ archiveHandshake = signal !== undefined;
+ } else if (
+ help.includes("Archive output in the registration transaction.")
+ ) {
+ // Earlier transactional helpers cannot acknowledge the safe cancellation
+ // boundary. Keep their move and database commit together.
+ signal?.throwIfAborted();
+ signal = undefined;
+ } else {
+ const scanDir = arguments_[arguments_.indexOf("--scan-dir") + 1]!;
+ const archiveDir = await planOutputArchive(scanDir);
+ if (archiveDir !== null) {
+ const savedScans = await savedScanIdentities(scanDir);
+ signal?.throwIfAborted();
+ await rename(scanDir, archiveDir);
+ legacyArchive = { scanDir, archiveDir, savedScans };
+ await mkdir(scanDir, { mode: 0o700 });
+ if ((process.umask() & 0o700) !== 0) await chmod(scanDir, 0o700);
+ arguments_.push("--archived-scan-dir", archiveDir);
+ }
+ }
+ }
const matchesStdinIndex = arguments_.indexOf("--matches-json-stdin");
if (
arguments_[0] === "save-scan-comparison" &&
@@ -1655,8 +1743,33 @@ export async function runWorkbench(
input = undefined;
}
}
- stdout = await run(arguments_, input);
+ stdout = await run(arguments_, input, archiveHandshake);
} catch (error) {
+ if (legacyArchive !== undefined) {
+ // Cancellation can stop legacy registration; recovery must still settle.
+ signal = undefined;
+ try {
+ // A helper can commit registration and lose its response. Restore only
+ // when saved identities prove that registration did not change them.
+ if (
+ (await savedScanIdentities(legacyArchive.scanDir)) ===
+ legacyArchive.savedScans
+ ) {
+ await rmdir(legacyArchive.scanDir).catch(
+ (error: NodeJS.ErrnoException) => {
+ if (error.code !== "ENOENT") throw error;
+ },
+ );
+ await rename(legacyArchive.archiveDir, legacyArchive.scanDir);
+ }
+ } catch (restoreError) {
+ throw new AggregateError(
+ [error, restoreError],
+ `Scan registration failed: ${processErrorDetail(error)}. Previous output remains at ${legacyArchive.archiveDir}; restoration could not be verified or completed: ${processErrorDetail(restoreError)}`,
+ { cause: error },
+ );
+ }
+ }
if (options.signal?.aborted) throw error;
const detail = processErrorDetail(error);
const databaseFailure =
@@ -1690,6 +1803,8 @@ export async function runWorkbench(
"The Codex Security workbench returned an invalid response.",
);
}
+ if (legacyArchive !== undefined)
+ result["archivedScanDir"] = legacyArchive.archiveDir;
return result as JsonObject;
}
@@ -1932,12 +2047,50 @@ export async function prepareOutputDir(
validateLocation?: (path: string) => void,
archiveExisting = false,
onOutputArchived?: (archiveDir: string) => void,
+): Promise<string> {
+ return prepareOutputDirectory(
+ outputDirectory,
+ repositoryName,
+ temporaryRoot,
+ validateLocation,
+ archiveExisting ? "archive" : "reject",
+ onOutputArchived,
+ );
+}
+
+/** Leave archival to the scan registration transaction. */
+export async function prepareScanRegistrationOutput(
+ outputDirectory: string | undefined,
+ repositoryName: string,
+ temporaryRoot: string = tmpdir(),
+ validateLocation?: (path: string) => void,
+ archiveExisting = false,
+): Promise<string> {
+ return prepareOutputDirectory(
+ outputDirectory,
+ repositoryName,
+ temporaryRoot,
+ validateLocation,
+ archiveExisting ? "preserve" : "reject",
+ );
+}
+
+async function prepareOutputDirectory(
+ outputDirectory: string | undefined,
+ repositoryName: string,
+ temporaryRoot: string,
+ validateLocation: ((path: string) => void) | undefined,
+ existingOutput: "reject" | "archive" | "preserve",
+ onOutputArchived?: (archiveDir: string) => void,
): Promise<string> {
if (outputDirectory === undefined) {
requireModelSafeOutputDir(temporaryRoot);
requireModelSafeOutputDir(await realpath(temporaryRoot));
}
- const path = await validateOutputDir(outputDirectory, archiveExisting);
+ const path = await validateOutputDir(
+ outputDirectory,
+ existingOutput !== "reject",
+ );
validateLocation?.(path ?? (await realpath(temporaryRoot)));
if (path === null) {
const created = await mkdtemp(
@@ -1954,7 +2107,7 @@ export async function prepareOutputDir(
let createdRoot: string | undefined;
try {
let existing = await lstat(path).catch(nullIfMissingFileError);
- if (existing !== null && archiveExisting) {
+ if (existing !== null && existingOutput === "archive") {
const archiveDir = await planOutputArchive(path);
if (archiveDir !== null) {
await rename(path, archiveDir);
@@ -1966,7 +2119,11 @@ export async function prepareOutputDir(
createdRoot = await mkdir(path, { recursive: true, mode: 0o700 });
if ((process.umask() & 0o700) !== 0) await chmod(path, 0o700);
}
- return await validatePreparedOutputDir(path, validateLocation);
+ return await validatePreparedOutputDir(
+ path,
+ validateLocation,
+ existingOutput === "preserve",
+ );
} catch (error) {
if (createdRoot !== undefined) {
await removeEmptyDirectories(path, createdRoot);
@@ -1984,6 +2141,7 @@ export async function prepareOutputDir(
export async function validatePreparedOutputDir(
path: string,
validateLocation?: (path: string) => void,
+ allowNonempty = false,
): Promise<string> {
const metadata = await lstat(path);
if (!metadata.isDirectory()) {
@@ -1993,7 +2151,7 @@ export async function validatePreparedOutputDir(
requireModelSafeOutputDir(canonical);
validateLocation?.(canonical);
const entries = await readdir(canonical);
- if (entries.length !== 0) {
+ if (!allowNonempty && entries.length !== 0) {
throw new OutputDirectoryError(
`Scan output directory must be empty: ${path}`,
);
@@ -3052,7 +3210,11 @@ export async function runCodexCommand(
environment: ProcessEnvironment,
input?: string | Uint8Array,
signal?: AbortSignal,
+ archiveHandshake = false,
): Promise<CodexCommandResult> {
+ const cancellation = archiveHandshake ? new AbortController() : undefined;
+ const abort = () => cancellation?.abort(signal?.reason);
+ let awaitingArchive = archiveHandshake;
const child = spawn(
executablePathForSpawn(command.command),
[...(command.args ?? []), ...args],
@@ -3060,7 +3222,7 @@ export async function runCodexCommand(
env: environment,
stdio: ["pipe", "pipe", "pipe"],
windowsHide: true,
- signal,
+ signal: cancellation?.signal ?? signal,
},
);
let stdout = "";
@@ -3070,6 +3232,15 @@ export async function runCodexCommand(
child.stderr.setEncoding("utf8");
child.stdout.on("data", (chunk: string) => {
stdout += chunk;
+ if (awaitingArchive && stdout.startsWith(ARCHIVE_READY)) {
+ awaitingArchive = false;
+ stdout = stdout.slice(ARCHIVE_READY.length);
+ signal?.removeEventListener("abort", abort);
+ // The helper cannot rename until this acknowledgement. Detach and check
+ // cancellation before allowing the filesystem/database transaction.
+ if (signal?.aborted) abort();
+ else child.stdin.end("\n");
+ }
});
child.stderr.on("data", (chunk: string) => {
stderr += chunk;
@@ -3092,8 +3263,17 @@ export async function runCodexCommand(
: reject(processError),
);
});
- child.stdin.end(input);
- return await completion;
+ if (archiveHandshake) {
+ signal?.addEventListener("abort", abort, { once: true });
+ if (signal?.aborted) abort();
+ }
+ try {
+ if (archiveHandshake) child.stdin.write(`${JSON.stringify(input ?? "")}\n`);
+ else child.stdin.end(input);
+ return await completion;
+ } finally {
+ if (archiveHandshake) signal?.removeEventListener("abort", abort);
+ }
}
export async function probeCodexSandbox(
diff --git a/sdk/typescript/tests-ts/api.test.ts b/sdk/typescript/tests-ts/api.test.ts
index 027df380b..bb25db78f 100644
--- a/sdk/typescript/tests-ts/api.test.ts
+++ b/sdk/typescript/tests-ts/api.test.ts
@@ -1,4 +1,5 @@
import { once } from "node:events";
+import { createServer, type Socket } from "node:net";
import {
appendFile,
chmod,
@@ -86,6 +87,7 @@ import {
import { createApiTestFixtures } from "./support/temporary-directories.js";
import { runTestInSubprocess } from "./support/test-subprocess.js";
import { writeSession as writeUsageSession } from "./support/usage-rollout.js";
+import { importScan } from "../src/import-scan.js";
import { FindingWorkflow } from "../src/finding-workflow.js";
import { DEFAULT_DEEP_SCAN_SETTINGS } from "../src/deep-scan-defaults.js";
import { createProviderProfile } from "../src/provider-profile.js";
@@ -1994,59 +1996,377 @@ describe("CodexSecurity orchestration", () => {
await client.close();
});
- test("archives existing output before starting a fresh scan", async () => {
- const root = await temporaryDirectory();
- const repository = join(root, "repository");
- const codexHome = join(root, "codex-home");
- const output = join(root, "scan");
- await mkdir(repository);
- await mkdir(codexHome);
- await mkdir(output, { mode: 0o700 });
- await writeFile(join(output, "previous.txt"), "previous scan\n");
- let archived: string | undefined;
- let registration: readonly string[] | undefined;
- const observerErrors: Array<[ScanObserverName, string]> = [];
- const client = TestClient.withDependencies({
- prepareRuntime: async () => preparedRuntime(codexHome),
- resolvePluginPython: async () => "/managed/python",
- repositoryRevision: async () => null,
- runWorkbench: async (
- _options: unknown,
- args: readonly string[],
- input?: string,
- ): Promise<JsonObject> => {
- if (args[0] !== "register-cli-scan") return mockWorkbench(args, input);
- registration = args;
- return mockScanRegistration(args, input);
- },
- createCodex: codexFactory(scanDidNotStart),
- });
+ test.each(["revision", "registration", "cancellation", "mock"] as const)(
+ "keeps existing output when %s prevents scan registration",
+ async (failurePoint) => {
+ const root = await temporaryDirectory();
+ const repository = join(root, "repository");
+ const codexHome = join(root, "codex-home");
+ const output = join(root, "scan");
+ await mkdir(repository);
+ await mkdir(codexHome);
+ await mkdir(output, { mode: 0o700 });
+ await writeFile(join(output, "previous.txt"), "previous scan\n");
+ const cancellation = new AbortController();
+ const client = new TestClient(
+ { pluginPath: PLUGIN_ROOT },
+ {
+ prepareRuntime: async () => preparedRuntime(codexHome),
+ resolvePluginPython: async () => "/managed/python",
+ repositoryRevision: async () => {
+ if (failurePoint === "revision")
+ throw new Error("fixture revision rejected");
+ if (failurePoint === "cancellation") cancellation.abort();
+ return null;
+ },
+ runWorkbench: async () => {
+ throw new Error("fixture registration rejected");
+ },
+ createCodex: codexFactory(scanDidNotStart),
+ },
+ );
+ try {
+ await expect(
+ client.run(repository, {
+ outputDir: output,
+ archiveExisting: true,
+ signal: cancellation.signal,
+ mock: failurePoint === "mock",
+ }),
+ ).rejects.toThrow();
+ expect(await readFile(join(output, "previous.txt"), "utf8")).toBe(
+ "previous scan\n",
+ );
+ expect(
+ (await readdir(root)).filter((name) =>
+ name.startsWith("scan.previous-"),
+ ),
+ ).toEqual([]);
+ } finally {
+ await client.close();
+ }
+ },
+ );
- await expect(
- client.run(repository, {
+ test.each([false, true])(
+ "archives accepted output before starting, cancellation=%s",
+ async (cancelRegistration) => {
+ const root = await temporaryDirectory();
+ const repository = join(root, "repository");
+ const codexHome = join(root, "codex-home");
+ const output = join(root, "scan");
+ await mkdir(repository);
+ await mkdir(codexHome);
+ await mkdir(output, { mode: 0o700 });
+ await writeFile(join(output, "previous.txt"), "previous scan\n");
+ const cancellation = new AbortController();
+ const notifications: string[] = [];
+ let archived: string | undefined;
+ let registration: readonly string[] | undefined;
+ const observerErrors: Array<[ScanObserverName, string]> = [];
+ const client = new TestClient(
+ {},
+ {
+ environment: { CODEX_SECURITY_STATE_DIR: join(root, "state") },
+ prepareRuntime: async () => preparedRuntime(codexHome),
+ resolvePluginPython: async () => "/managed/python",
+ repositoryRevision: async () => null,
+ runWorkbench: async (
+ _options: Parameters<typeof runWorkbench>[0],
+ args: readonly string[],
+ input?: string,
+ ): Promise<JsonObject> => {
+ if (args[0] === "register-cli-scan") registration = args;
+ const result = await runWorkbench(
+ {
+ ..._options,
+ python: pythonExecutable()!,
+ pluginRoot: PLUGIN_ROOT,
+ },
+ args,
+ input,
+ );
+ if (args[0] === "register-cli-scan" && cancelRegistration) {
+ expect(_options.signal).toBeDefined();
+ cancellation.abort();
+ }
+ return result;
+ },
+ createCodex: codexFactory(scanDidNotStart),
+ },
+ );
+
+ await expect(
+ client.run(repository, {
+ outputDir: output,
+ archiveExisting: true,
+ signal: cancellation.signal,
+ onOutputDirReady: () => {
+ notifications.push("ready");
+ },
+ onOutputArchived: (archiveDir) => {
+ notifications.push("archived");
+ archived = archiveDir;
+ throw new Error("archive observer exploded");
+ },
+ onObserverError: collectObserverErrors(observerErrors),
+ }),
+ ).rejects.toThrow(
+ cancelRegistration ? "interrupted" : "scan did not start",
+ );
+ expect(notifications).toEqual(["archived", "ready"]);
+ expect(observerErrors).toEqual([
+ ["onOutputArchived", "archive observer exploded"],
+ ]);
+ expect(archived?.startsWith(`${output}.previous-`)).toBe(true);
+ expect(registration).toContain("--archive-existing");
+ expect(registration).not.toContain("--archived-scan-dir");
+ expect(await readFile(join(archived!, "previous.txt"), "utf8")).toBe(
+ "previous scan\n",
+ );
+ await expect(stat(output)).resolves.toBeDefined();
+ await client.close();
+ },
+ );
+
+ test.each(
+ (["preparation", "empty", "legacy", "commit", "rollback"] as const).flatMap(
+ (boundary) =>
+ (["real", "mock", "import"] as const).flatMap((mode) =>
+ (mode === "import"
+ ? (["signal"] as const)
+ : (["signal", "close"] as const)
+ ).map((cancel) => ({ boundary, cancel, mode })),
+ ),
+ ),
+ )(
+ "cancels archived registration at $boundary via $cancel, mode=$mode",
+ async ({ boundary, cancel, mode }) => {
+ const { root, repository, codexHome } = await runtimeDirectories();
+ const output = join(root, "scan");
+ await mkdir(output, { mode: 0o700 });
+ const python = pythonExecutable()!;
+ const environment = {
+ PATH: process.env["PATH"],
+ SystemRoot: process.env["SystemRoot"],
+ TEMP: process.env["TEMP"],
+ TMP: process.env["TMP"],
+ CODEX_SECURITY_STATE_DIR: join(root, "state"),
+ };
+ const workbenchOptions = { python, pluginRoot: PLUGIN_ROOT, environment };
+ const databaseInfo = await runWorkbench(workbenchOptions, [
+ "database-info",
+ ]);
+ expect(databaseInfo["databasePath"]).toBe(
+ await realpath(
+ join(environment.CODEX_SECURITY_STATE_DIR, "workbench.sqlite3"),
+ ),
+ );
+ let previousId: string | undefined;
+ if (boundary !== "empty") {
+ const previous = await runWorkbench(workbenchOptions, [
+ "register-cli-scan",
+ "--repository",
+ repository,
+ "--scan-dir",
+ output,
+ "--recipe-json",
+ JSON.stringify({
+ repository,
+ target: { kind: "repository", paths: [] },
+ mode: "standard",
+ config: {},
+ }),
+ ]);
+ previousId = previous["scanId"] as string;
+ await runWorkbench(workbenchOptions, [
+ "fail-scan",
+ "--scan-id",
+ previousId,
+ "--message",
+ "synthetic stopped scan",
+ ]);
+ await writeFile(join(output, "previous.txt"), "previous scan\n");
+ }
+ const connections = new Set<Socket>();
+ let cleaningUp = false;
+ const server = createServer((connection) => {
+ connections.add(connection);
+ connection.once("close", () => connections.delete(connection));
+ if (cleaningUp) connection.destroy();
+ });
+ server.listen(0, "127.0.0.1");
+ await once(server, "listening");
+ const address = server.address();
+ if (address === null || typeof address === "string")
+ throw new Error("Missing fixture port");
+ const pluginRoot = join(root, "controlled-plugin");
+ await cp(PLUGIN_ROOT, pluginRoot, { recursive: true });
+ await writeFile(
+ join(pluginRoot, "scripts", "workbench_db.py"),
+ [
+ "import io, runpy, socket, sys",
+ "from pathlib import Path",
+ `workbench = runpy.run_path(${JSON.stringify(join(PLUGIN_ROOT, "scripts", "workbench_db.py"))})`,
+ "namespace = workbench['main'].__globals__",
+ "def pause():",
+ ` with socket.create_connection(('127.0.0.1', ${address.port})) as connection:`,
+ " connection.sendall(b'ready')",
+ " if connection.recv(1) == b'p':",
+ " connection.sendall(b'alive')",
+ " connection.recv(1)",
+ "def main(**kwargs):",
+ " sys.stdout.reconfigure(newline='\\r\\n')",
+ ...(boundary === "legacy"
+ ? [
+ " if sys.argv[1:3] == ['register-cli-scan', '--help']:",
+ " print('--archive-existing --archived-scan-dir')",
+ " return",
+ ]
+ : []),
+ " if sys.argv[1:2] == ['register-cli-scan'] and '--help' not in sys.argv:",
+ " payload = sys.stdin.buffer.read()",
+ ` Path(${JSON.stringify(join(root, "registration-input.json"))}).write_bytes(payload)`,
+ ' sys.stdin = io.TextIOWrapper(io.BytesIO(payload), encoding="utf-8")',
+ ` original = namespace[${JSON.stringify(boundary === "commit" || boundary === "rollback" ? "insert_running_scan" : "directory_snapshot_regular_file_count")}]`,
+ " def paused(*args, **options):",
+ " pause()",
+ ...(boundary === "rollback"
+ ? [
+ " raise RuntimeError('synthetic registration failure after rename')",
+ ]
+ : [" return original(*args, **options)"]),
+ ` namespace[${JSON.stringify(boundary === "commit" || boundary === "rollback" ? "insert_running_scan" : "directory_snapshot_regular_file_count")}] = paused`,
+ " workbench['main'](**kwargs)",
+ "if __name__ == '__main__': main()",
+ ].join("\n"),
+ );
+ let submitted: string | undefined;
+ const registration: typeof runWorkbench = async (
+ options,
+ args,
+ input,
+ ) => {
+ if (args[0] === "register-cli-scan") {
+ submitted = `\n${JSON.stringify(JSON.parse(input!), null, 2)}\r\n`;
+ input = submitted;
+ }
+ return runWorkbench(options, args, input);
+ };
+ const controller = new AbortController();
+ const client = new TestClient(
+ { pluginPath: pluginRoot },
+ {
+ environment,
+ prepareRuntime: async () => {
+ const runtime = preparedRuntime(codexHome);
+ runtime.plugin.pluginRoot = pluginRoot;
+ return runtime;
+ },
+ resolvePluginPython: async () => python,
+ repositoryRevision: async () => null,
+ runWorkbench: registration,
+ createCodex: codexFactory(scanDidNotStart),
+ },
+ );
+ // The deadline only bounds a broken fixture; cancellation is observed by
+ // the paused child closing its socket, without releasing its work gate.
+ const guard = new AbortController();
+ // Use a refed timer: Bun can suspend AbortSignal.timeout at this gate.
+ const guardTimer = setTimeout(() => guard.abort(), 10_000);
+ const deadline = guard.signal;
+ const connected = once(server, "connection", { signal: deadline });
+ const options = {
outputDir: output,
archiveExisting: true,
- onOutputArchived: (archiveDir) => {
- archived = archiveDir;
- throw new Error("archive observer exploded");
- },
- onObserverError: collectObserverErrors(observerErrors),
- }),
- ).rejects.toThrow("scan did not start");
- expect(observerErrors).toEqual([
- ["onOutputArchived", "archive observer exploded"],
- ]);
- expect(archived?.startsWith(`${output}.previous-`)).toBe(true);
- expect(registration).toContain("--archive-existing");
- expect(
- registration?.[registration.indexOf("--archived-scan-dir") + 1],
- ).toBe(archived);
- expect(await readFile(join(archived!, "previous.txt"), "utf8")).toBe(
- "previous scan\n",
- );
- await expect(stat(output)).resolves.toBeDefined();
- await client.close();
- });
+ signal: controller.signal,
+ };
+ const operation = (
+ mode === "import"
+ ? importScan(
+ {
+ ...options,
+ sourcePath: join(EXAMPLE, "findings.json"),
+ format: "json",
+ config: { pluginPath: pluginRoot },
+ },
+ {
+ environment,
+ resolvePluginPython: async () => python,
+ runWorkbench: registration,
+ },
+ )
+ : client.run(repository, {
+ ...options,
+ mock: mode === "mock",
+ scanPrompt: "Review café boundaries.\nPreserve the second line.",
+ })
+ ).catch((error: unknown) => error);
+ let socket: Socket | undefined;
+ let closing: Promise<void> | undefined;
+ try {
+ [socket] = (await Promise.race([
+ connected,
+ operation.then((error) => {
+ throw error;
+ }),
+ ])) as [Socket];
+ expect(
+ String((await once(socket, "data", { signal: deadline }))[0]),
+ ).toBe("ready");
+ expect(
+ await readFile(join(root, "registration-input.json"), "utf8"),
+ ).toBe(submitted!);
+ const closed = once(socket, "close", { signal: deadline });
+ if (cancel === "close") closing = client.close();
+ else controller.abort();
+ if (boundary === "commit" || boundary === "rollback") {
+ // This child has already renamed the directory. A request/response
+ // proves it remains alive after cancellation until we let it settle.
+ const alive = once(socket, "data", { signal: deadline });
+ socket.write("p");
+ expect(String((await alive)[0])).toBe("alive");
+ socket.end("r");
+ }
+ await closed;
+ const error = await operation;
+ expect(error).toBeInstanceOf(Error);
+ expect((error as Error).message).toMatch(/closed|interrupted/);
+ await closing;
+ expect(await runWorkbench(workbenchOptions, ["database-info"])).toEqual(
+ databaseInfo,
+ );
+ const scans = (await runWorkbench(workbenchOptions, ["list-scans"]))[
+ "scans"
+ ] as Array<{ scanId: string; scanDir: string }>;
+ expect(scans).toHaveLength(
+ boundary === "empty" ? 0 : boundary === "commit" ? 2 : 1,
+ );
+ if (previousId !== undefined) {
+ const previous = scans.find((scan) => scan.scanId === previousId)!;
+ expect(previous.scanDir === output).toBe(boundary !== "commit");
+ expect(
+ await readFile(join(previous.scanDir, "previous.txt"), "utf8"),
+ ).toBe("previous scan\n");
+ }
+ expect(
+ (await readdir(root)).filter((name) =>
+ name.startsWith("scan.previous-"),
+ ),
+ ).toHaveLength(boundary === "commit" ? 1 : 0);
+ } finally {
+ cleaningUp = true;
+ clearTimeout(guardTimer);
+ controller.abort();
+ for (const connection of connections) connection.destroy();
+ await operation;
+ await client.close();
+ await new Promise<void>((resolve, reject) =>
+ server.close((error) => (error ? reject(error) : resolve())),
+ );
+ }
+ },
+ );
test("reports the real scan failure when scan cleanup also fails", async () => {
const root = await temporaryDirectory();
diff --git a/sdk/typescript/tests-ts/import-scan.test.ts b/sdk/typescript/tests-ts/import-scan.test.ts
index ab8478184..ff12a15aa 100644
--- a/sdk/typescript/tests-ts/import-scan.test.ts
+++ b/sdk/typescript/tests-ts/import-scan.test.ts
@@ -1,5 +1,6 @@
import { execFileSync } from "node:child_process";
import {
+ cp,
mkdir,
mkdtemp,
readFile,
@@ -132,6 +133,30 @@ async function fixture(format: "csv" | "json" = "csv") {
};
}
+async function legacyArchivePlugin(
+ context: Awaited<ReturnType<typeof fixture>>,
+) {
+ const plugin = join(context.root, "legacy-plugin");
+ await cp(PLUGIN_ROOT, plugin, { recursive: true });
+ await writeFile(
+ join(plugin, "scripts", "workbench_db.py"),
+ [
+ "import runpy, sys",
+ "from pathlib import Path",
+ "args = sys.argv[1:]",
+ 'if args[:2] == ["register-cli-scan", "--help"]:',
+ ' print("--archive-existing --archived-scan-dir")',
+ " raise SystemExit(0)",
+ 'if args and args[0] == "register-cli-scan":',
+ ' scan_dir = Path(args[args.index("--scan-dir") + 1])',
+ " if next(scan_dir.iterdir(), None) is not None:",
+ ' raise SystemExit("The scan artifact directory must be empty before the scan starts.")',
+ `runpy.run_path(${JSON.stringify(join(PLUGIN_ROOT, "scripts", "workbench_db.py"))}, run_name="__main__")`,
+ ].join("\n"),
+ );
+ return plugin;
+}
+
function completed(result: Awaited<ReturnType<typeof importScan>>): ScanResult {
expect(result).toBeInstanceOf(ScanResult);
if (!(result instanceof ScanResult)) throw new Error("Expected a saved scan");
@@ -165,6 +190,27 @@ async function storedScans(context: Awaited<ReturnType<typeof fixture>>) {
}>;
}
+test("a rejected import registration keeps existing output in place", async () => {
+ const context = await fixture();
+ const outputDir = join(context.root, "scan");
+ await mkdir(outputDir, { mode: 0o700 });
+ await writeFile(join(outputDir, "previous.txt"), "previous scan\n");
+ await expect(
+ importScan(
+ { ...context.options, outputDir, archiveExisting: true },
+ {
+ ...context.dependencies,
+ runWorkbench: async () => {
+ throw new Error("fixture registration rejected");
+ },
+ },
+ ),
+ ).rejects.toThrow("fixture registration rejected");
+ expect(await readFile(join(outputDir, "previous.txt"), "utf8")).toBe(
+ "previous scan\n",
+ );
+});
+
test.each(["csv", "json"] as const)(
"%s import seals every source occurrence and indexes a separate dataset scan",
async (format) => {
@@ -466,35 +512,145 @@ test("dry run validates source rows without creating database state or invoking
});
});
-test("imports archive prior output without replacing its saved findings", async () => {
- const context = await fixture();
- const options = {
- ...context.options,
- outputDir: join(context.root, "results"),
- };
- const first = completed(await importScan(options, context.dependencies));
- const originalManifest = await readFile(first.manifestPath, "utf8");
- await expect(importScan(options, context.dependencies)).rejects.toThrow();
- expect(await readFile(first.manifestPath, "utf8")).toBe(originalManifest);
- const second = completed(
- await importScan(
- { ...options, archiveExisting: true },
- context.dependencies,
- ),
- );
- const scans = await storedScans(context);
- expect(scans).toHaveLength(2);
- const archived = scans.find((scan) => scan.id === first.manifest.scan.id)!;
- expect(archived.scan_dir).not.toBe(second.scanDir);
- expect(archived.occurrence_count).toBe(2);
- expect(
- await readFile(join(archived.scan_dir, "scan-manifest.json"), "utf8"),
- ).toBe(originalManifest);
- expect(
- (await loadContract(archived.scan_dir, { pluginRoot: PLUGIN_ROOT }))
- .findings.findings,
- ).toHaveLength(2);
-});
+test.each(["bundled", "legacy"])(
+ "imports with %s helpers archive prior output without replacing its saved findings",
+ async (helper) => {
+ const context = await fixture();
+ const options = {
+ ...context.options,
+ ...(helper === "legacy"
+ ? {
+ config: {
+ ...context.options.config,
+ pluginPath: await legacyArchivePlugin(context),
+ },
+ }
+ : {}),
+ outputDir: join(context.root, "results"),
+ };
+ const first = completed(await importScan(options, context.dependencies));
+ const originalManifest = await readFile(first.manifestPath, "utf8");
+ await expect(importScan(options, context.dependencies)).rejects.toThrow();
+ expect(await readFile(first.manifestPath, "utf8")).toBe(originalManifest);
+ const second = completed(
+ await importScan(
+ { ...options, archiveExisting: true },
+ context.dependencies,
+ ),
+ );
+ const scans = await storedScans(context);
+ expect(scans).toHaveLength(2);
+ const archived = scans.find((scan) => scan.id === first.manifest.scan.id)!;
+ expect(archived.scan_dir).not.toBe(second.scanDir);
+ expect(archived.occurrence_count).toBe(2);
+ expect(
+ await readFile(join(archived.scan_dir, "scan-manifest.json"), "utf8"),
+ ).toBe(originalManifest);
+ expect(
+ (await loadContract(archived.scan_dir, { pluginRoot: PLUGIN_ROOT }))
+ .findings.findings,
+ ).toHaveLength(2);
+ },
+);
+
+test.each(["active scan", "missing parent"])(
+ "legacy registration restores output after rejecting %s",
+ async (reason) => {
+ const context = await fixture();
+ const options = {
+ ...context.options,
+ config: {
+ ...context.options.config,
+ pluginPath: await legacyArchivePlugin(context),
+ },
+ outputDir: join(context.root, "results"),
+ };
+ const first = completed(await importScan(options, context.dependencies));
+ const manifest = await readFile(first.manifestPath, "utf8");
+ if (reason === "active scan") {
+ const update = await runCommand(
+ context.python,
+ [
+ "-c",
+ "import sqlite3,sys; c=sqlite3.connect(sys.argv[1]); c.execute(\"UPDATE scans SET status='running'\"); c.commit()",
+ join(context.stateDirectory, "workbench.sqlite3"),
+ ],
+ { timeout: 30_000 },
+ );
+ expect(update.status, update.stderr).toBe(0);
+ }
+ await expect(
+ importScan(
+ {
+ ...options,
+ archiveExisting: true,
+ ...(reason === "missing parent"
+ ? { parentScanId: "00000000-0000-4000-8000-000000000001" }
+ : {}),
+ },
+ context.dependencies,
+ ),
+ ).rejects.toThrow(
+ reason === "active scan" ? "running scan" : "scan not found",
+ );
+ expect(await readFile(first.manifestPath, "utf8")).toBe(manifest);
+ expect(await storedScans(context)).toHaveLength(1);
+ expect((await storedScans(context))[0]?.scan_dir).toBe(first.scanDir);
+ },
+);
+
+test.each(["response lost", "state unreadable"])(
+ "legacy registration preserves its archive when %s after commit",
+ async (failure) => {
+ const context = await fixture();
+ const pluginPath = await legacyArchivePlugin(context);
+ const options = {
+ ...context.options,
+ config: { ...context.options.config, pluginPath },
+ outputDir: join(context.root, "results"),
+ };
+ const first = completed(await importScan(options, context.dependencies));
+ const manifest = await readFile(first.manifestPath, "utf8");
+ const helper = join(pluginPath, "scripts", "workbench_db.py");
+ const original = await readFile(helper, "utf8");
+ await writeFile(
+ helper,
+ original
+ .replace(
+ "import runpy, sys",
+ [
+ "import contextlib, io, runpy, sys",
+ "from pathlib import Path",
+ "args = sys.argv[1:]",
+ ...(failure === "state unreadable"
+ ? [
+ 'if args and args[0] == "list-scans" and Path(__file__).with_suffix(".committed").exists():',
+ ' raise SystemExit("Synthetic state read failure")',
+ ]
+ : []),
+ ].join("\n"),
+ )
+ .replace(
+ "runpy.run_path(",
+ 'lost_response = args and args[0] == "register-cli-scan"\nwith contextlib.redirect_stdout(io.StringIO()) if lost_response else contextlib.nullcontext():\n runpy.run_path(',
+ ) +
+ '\nif lost_response:\n Path(__file__).with_suffix(".committed").touch()\n raise SystemExit("Synthetic response lost after commit")\n',
+ );
+ await expect(
+ importScan({ ...options, archiveExisting: true }, context.dependencies),
+ ).rejects.toThrow("Synthetic response lost after commit");
+ const scans = await storedScans(context);
+ expect(scans).toHaveLength(2);
+ const archived = scans.find((scan) => scan.id === first.manifest.scan.id)!;
+ expect(archived.scan_dir).not.toBe(first.scanDir);
+ expect(
+ await readFile(join(archived.scan_dir, "scan-manifest.json"), "utf8"),
+ ).toBe(manifest);
+ expect(
+ await stat(join(first.scanDir, "scan-manifest.json")).catch(() => null),
+ ).toBeNull();
+ },
+);
test.each(["failure", "abort"] as const)(
"an import %s after registration leaves a terminal saved scan",
diff --git a/sdk/typescript/tests-ts/runtime.test.ts b/sdk/typescript/tests-ts/runtime.test.ts
index 57e6fd798..0934a1266 100644
--- a/sdk/typescript/tests-ts/runtime.test.ts
+++ b/sdk/typescript/tests-ts/runtime.test.ts
@@ -78,6 +78,7 @@ import {
prepareCodexSecurityCredentialHome,
preparePersistentOutputRoot,
prepareScanArtifactRestorer,
+ prepareScanRegistrationOutput,
preserveCodexSecurityPluginRegistration,
environmentWithGit,
requirePrivateCredentialHome,
@@ -5749,7 +5750,8 @@ describe("runtime directories and plugin Python boundary", () => {
"artifacts = {'coverage': 'coverage.json', 'findings': 'findings.json', 'manifest': 'scan-manifest.json', 'markdownReport': 'report.md'}",
"connection.executemany('INSERT INTO scan_artifacts VALUES (?, ?, ?)', [('previous-scan', kind, str(scan_dir / path)) for kind, path in artifacts.items()])",
"args = argparse.Namespace(archive_existing=True, archived_scan_dir=str(archived_scan_dir))",
- "archive_scan(connection, args, scan_dir, 'after', lambda path: path.resolve(strict=True))",
+ "with archive_scan(connection, args, scan_dir, 'after', lambda path: path.resolve(strict=True)):",
+ " pass",
"scan = connection.execute('SELECT scan_dir FROM scans WHERE id = ?', ('previous-scan',)).fetchone()",
"rows = connection.execute('SELECT kind, path FROM scan_artifacts WHERE scan_id = ? ORDER BY kind', ('previous-scan',))",
"print(json.dumps({'scanDir': scan['scan_dir'], 'artifacts': [dict(row) for row in rows]}))",
@@ -5800,7 +5802,8 @@ describe("runtime directories and plugin Python boundary", () => {
"connection.execute('INSERT INTO scans VALUES (?, ?, ?, ?)', ('previous-scan', 'complete', str(scan_dir), 'before'))",
"connection.execute('INSERT INTO scan_artifacts VALUES (?, ?, ?)', ('previous-scan', 'coverage', str(scan_dir / 'coverage.json')))",
"args = argparse.Namespace(archive_existing=True, archived_scan_dir=None)",
- "archive_scan(connection, args, scan_dir, 'after', lambda path: path.resolve(strict=True))",
+ "with archive_scan(connection, args, scan_dir, 'after', lambda path: path.resolve(strict=True)):",
+ " pass",
].join("\n"),
join(PLUGIN_ROOT, "scripts"),
scanDir,
@@ -5947,7 +5950,7 @@ describe("runtime directories and plugin Python boundary", () => {
).not.toThrow();
});
- test("archives a non-empty private output directory", async () => {
+ test("prepares a private output directory without moving prior results", async () => {
const root = await temporaryDirectory();
const output = join(root, "scan");
await mkdir(output, { mode: 0o700 });
@@ -5964,40 +5967,75 @@ describe("runtime directories and plugin Python boundary", () => {
);
await expect(stat(preview!)).rejects.toThrow();
- const onOutputArchived = mock((_archiveDir: string) => {});
expect(
- await prepareOutputDir(
+ await prepareScanRegistrationOutput(
output,
"repo",
undefined,
undefined,
true,
- onOutputArchived,
),
).toBe(output);
- const archived = onOutputArchived.mock.lastCall?.[0];
- expect(archived?.startsWith(`${output}.previous-`)).toBe(true);
- expect(await readFile(join(archived!, "previous.txt"), "utf8")).toBe(
+ expect(await readFile(join(output, "previous.txt"), "utf8")).toBe(
"previous scan\n",
);
- expect(await readdir(output)).toEqual([]);
+ expect(await readdir(root)).toEqual(["scan"]);
if (process.platform !== "win32") {
expect((await stat(output)).mode & 0o777).toBe(0o700);
const linkedOutput = join(root, "linked-scan");
- await symlink(archived!, linkedOutput);
+ await symlink(output, linkedOutput);
await expect(validateOutputDir(linkedOutput, true)).rejects.toThrow(
"not a directory",
);
- await chmod(archived!, 0o770);
- await expect(validateOutputDir(archived!, true)).rejects.toThrow(
+ await chmod(output, 0o770);
+ await expect(validateOutputDir(output, true)).rejects.toThrow(
"must not be accessible to other users",
);
- await chmod(archived!, 0o700);
+ await chmod(output, 0o700);
}
- expect(await planOutputArchive(output)).toBeNull();
+ expect(await planOutputArchive(output)).not.toBeNull();
+ });
+
+ test("archives prior output through the exported preparation helper", async () => {
+ const root = await temporaryDirectory();
+ const output = join(root, "scan");
+ await mkdir(output, { mode: 0o700 });
+ await writeFile(join(output, "previous.txt"), "previous scan\n");
+ const archived: string[] = [];
+
+ expect(
+ await prepareOutputDir(
+ output,
+ "repo",
+ undefined,
+ undefined,
+ true,
+ (path) => {
+ archived.push(path);
+ },
+ ),
+ ).toBe(output);
+ expect(archived).toHaveLength(1);
+ expect(await readFile(join(archived[0]!, "previous.txt"), "utf8")).toBe(
+ "previous scan\n",
+ );
+ expect(await readdir(output)).toEqual([]);
+ expect(
+ await prepareOutputDir(
+ output,
+ "repo",
+ undefined,
+ undefined,
+ true,
+ (path) => {
+ archived.push(path);
+ },
+ ),
+ ).toBe(output);
+ expect(archived).toHaveLength(1);
});
test("validates explicit output directories and creates private temporary paths", async () => {