Skip to content

OpenAI's Codex Security CLI and TypeScript SDK for finding, validating, and fixing security vulnerabilities. npm: https://www.npmjs.com/package/@openai/codex-security

fix(deep-scan): preserve coordinator ownership and source novelty (#1503)

mldangelo-oai8ea9763a6e44
diff --git a/plugins/codex-security/mcp-app/src/deep-scan/artifact-validation.ts b/plugins/codex-security/mcp-app/src/deep-scan/artifact-validation.ts
index 224974618..d3bdd6ee8 100644
--- a/plugins/codex-security/mcp-app/src/deep-scan/artifact-validation.ts
+++ b/plugins/codex-security/mcp-app/src/deep-scan/artifact-validation.ts
@@ -69,7 +69,7 @@ export async function validateDiscoveryArtifacts(
return result;
}
-/** Validate the complete aggregate and derive convergence from stable finding identities. */
+/** Validate the complete aggregate and derive convergence from retained source ancestry. */
export async function validateReducerArtifacts(
input: {
artifacts: DeepScanArtifacts;
@@ -127,12 +127,18 @@ export async function validateReducerArtifacts(
validateRetainedFindings(result, [], previous);
}
const previousFindingIds = new Set(
- (previous?.findings ?? []).map(scanFindingIdentity),
+ input.sources
+ ? (previous?.findings ?? [])
+ .flatMap(retainedFindingSources)
+ .map((source) => source.id)
+ : (previous?.findings ?? []).map(scanFindingIdentity),
);
return {
result,
- newFindings: result.findings.filter(
- (finding) => !previousFindingIds.has(scanFindingIdentity(finding)),
+ newFindings: result.findings.filter((finding) =>
+ input.sources
+ ? !findingSourceIds(finding).some((id) => previousFindingIds.has(id))
+ : !previousFindingIds.has(scanFindingIdentity(finding)),
).length,
};
}
@@ -161,22 +167,18 @@ export function reconcileDeepReduction(
previous ?? undefined,
);
retainSourceFindings(result, { discoveries, previous });
- const unmatched = new Set(result.findings);
- for (const finding of previous?.findings ?? []) {
- const previousRefs = findingSourceIds(finding);
- const retained =
- (previousRefs.length > 0
- ? result.findings.find((current) =>
- findingSourceIds(current).some((ref) => previousRefs.includes(ref)),
- )
- : undefined) ??
- [...unmatched].find(
- (current) =>
- scanFindingIdentity(current) === scanFindingIdentity(finding),
- );
+ for (const [index, finding] of (previous?.findings ?? []).entries()) {
+ const previousRefs = retainedFindingSources(finding, index).map(
+ (source) => source.id,
+ );
+ const retained = result.findings.find((current) =>
+ findingSourceIds(current).some((ref) => previousRefs.includes(ref)),
+ );
if (retained) {
+ const sourceIds = findingSourceIds(retained);
preserveFindingDetails(retained, finding);
- unmatched.delete(retained);
+ (retained.provenance as Record<string, unknown>).sourceFindingIds =
+ sourceIds;
}
}
retainSourceFindings(result, { discoveries, previous });
@@ -211,6 +213,16 @@ function findingSourceIds(finding: Record<string, unknown>): string[] {
return sourceFindingIds ?? sourceFindings?.map((source) => source.id) ?? [];
}
+function retainedFindingSources(
+ finding: Record<string, unknown>,
+ index: number,
+) {
+ const originals = (finding.provenance as Record<string, unknown>)
+ .sourceFindings as
+ Array<{ id: string; finding: Record<string, unknown> }> | undefined;
+ return originals?.length ? originals : [{ id: `previous:${index}`, finding }];
+}
+
function retainSourceFindings(
result: DeepReductionInput,
inputs: DeepReductionSources,
@@ -224,17 +236,10 @@ function retainSourceFindings(
sources.set(`${discovery.workerId}:${index}`, original);
}
}
- for (const [index, finding] of (inputs.previous?.findings ?? []).entries()) {
- const provenance = finding.provenance as Finding;
- const originals = provenance.sourceFindings as
- Array<{ id: string; finding: Finding }> | undefined;
- if (originals?.length) {
- for (const original of originals)
- sources.set(original.id, original.finding);
- } else {
- sources.set(`previous:${index}`, finding);
- }
- }
+ for (const original of (inputs.previous?.findings ?? []).flatMap(
+ retainedFindingSources,
+ ))
+ sources.set(original.id, original.finding);
const claimed = new Set<string>();
for (const finding of result.findings) {
const provenance = finding.provenance as Finding;
diff --git a/plugins/codex-security/mcp-app/src/deep-scan/coordinator.ts b/plugins/codex-security/mcp-app/src/deep-scan/coordinator.ts
index 88879e99a..2287efe56 100644
--- a/plugins/codex-security/mcp-app/src/deep-scan/coordinator.ts
+++ b/plugins/codex-security/mcp-app/src/deep-scan/coordinator.ts
@@ -25,11 +25,11 @@ import type {
} from "./worker-runner.js";
import {
boundedDeepScanErrorPair,
+ confirmedOwnershipChange,
errorNameWithCode,
abortError,
errorMessage,
boundedDeepScanErrorMessage,
- isStaleCoordinatorGenerationError,
} from "./errors.js";
import type {
CodexWorkerExecutor,
@@ -324,12 +324,7 @@ export class DeepScanCoordinator {
if (this.canceled || this.externallyFailed) {
return;
}
- if (
- await this.stopAfterOwnershipChange(
- this.options.threadId,
- isStaleCoordinatorGenerationError(error),
- )
- ) {
+ if (await this.stopAfterOwnershipChange(this.options.threadId, error)) {
return;
}
const message = errorMessage(error);
@@ -354,6 +349,14 @@ export class DeepScanCoordinator {
this.state = failed;
} catch (persistError) {
if (this.canceled || this.externallyFailed) return;
+ if (
+ confirmedOwnershipChange(persistError, this.state.scanId) &&
+ (await this.stopAfterOwnershipChange(
+ this.options.threadId,
+ persistError,
+ ))
+ )
+ return;
this.state = {
...this.state,
status: "failed",
@@ -536,7 +539,7 @@ export class DeepScanCoordinator {
}
this.scheduleHeartbeat();
if (this.terminal || this.ownershipCheck) return;
- const ownershipCheck = this.stopAfterOwnershipChange(threadId, false);
+ const ownershipCheck = this.stopAfterOwnershipChange(threadId);
this.ownershipCheck = ownershipCheck;
try {
await ownershipCheck;
@@ -548,29 +551,27 @@ export class DeepScanCoordinator {
private async stopAfterOwnershipChange(
threadId: string | undefined,
- leaseLossConfirmed: boolean,
+ error?: unknown,
): Promise<boolean> {
if (this.externallyFailed || this.terminal || !threadId)
return this.externallyFailed;
- let current: DeepScanRunState;
+ let current = confirmedOwnershipChange(error, this.state.scanId)?.run;
try {
- current = await this.options.store.get(this.state.scanId, threadId);
+ current ??= await this.options.store.get(this.state.scanId, threadId);
} catch (readError) {
this.log({
event: "coordinator_ownership_read_failed",
scanId: this.state.scanId,
reason: errorKind(readError),
});
- if (!leaseLossConfirmed) return this.externallyFailed;
- current = this.state;
+ return this.externallyFailed;
}
if (this.externallyFailed || this.terminal) return this.externallyFailed;
const replacementConfirmed =
- leaseLossConfirmed ||
- (current.status === "running" &&
- current.coordinatorGeneration !== undefined &&
- this.state.coordinatorGeneration !== undefined &&
- current.coordinatorGeneration > this.state.coordinatorGeneration);
+ current.status === "running" &&
+ current.coordinatorGeneration !== undefined &&
+ this.state.coordinatorGeneration !== undefined &&
+ current.coordinatorGeneration > this.state.coordinatorGeneration;
if (current.status === "running" && !replacementConfirmed) return false;
this.externallyFailed = true;
@@ -730,6 +731,8 @@ export class DeepScanCoordinator {
let firstFailure: unknown | undefined;
for (const result of results) {
if (result.status === "rejected") {
+ if (confirmedOwnershipChange(result.reason, this.state.scanId))
+ throw result.reason;
firstFailure ??= result.reason;
continue;
}
@@ -754,6 +757,26 @@ export class DeepScanCoordinator {
return undefined;
};
+ const settleFailure = async (error: unknown): Promise<never> => {
+ this.abortController.abort(errorMessage(error));
+ const reducerFailure = await reconcileReducerSettlement();
+ const discoveries = await Promise.allSettled([...active.values()]);
+ // SDK iterator cleanup can let a later generic rejection settle first.
+ // Inspect every active worker's result before discarding confirmed ownership.
+ const failures = [
+ error,
+ reducerFailure,
+ ...discoveries.flatMap((result) =>
+ result.status === "rejected" ? [result.reason] : [],
+ ),
+ ];
+ throw (
+ failures.find((failure) =>
+ confirmedOwnershipChange(failure, this.state.scanId),
+ ) ?? error
+ );
+ };
+
await this.options.store.updateProgress({
scanId: this.state.scanId,
phase: "discovery",
@@ -830,10 +853,7 @@ export class DeepScanCoordinator {
const settlement = await nextSettlement();
if (settlement.status === "rejected") {
- this.abortController.abort(errorMessage(settlement.error));
- await reconcileReducerSettlement();
- await this.settleSchedulerWork();
- throw settlement.error;
+ return await settleFailure(settlement.error);
}
const outcome = settlement.outcome;
if (outcome.type === "discovery") {
@@ -862,15 +882,9 @@ export class DeepScanCoordinator {
outcome.error.message,
outcome.error,
);
- this.abortController.abort(thresholdError.message);
- await reconcileReducerSettlement();
- await this.settleSchedulerWork();
- throw thresholdError;
+ return await settleFailure(thresholdError);
}
- this.abortController.abort(outcome.error.message);
- await reconcileReducerSettlement();
- await this.settleSchedulerWork();
- throw outcome.error;
+ return await settleFailure(outcome.error);
}
if (outcome.status === "canceled") {
if (
@@ -908,9 +922,7 @@ export class DeepScanCoordinator {
outcome.error.message,
outcome.error,
);
- this.abortController.abort(thresholdError.message);
- await this.settleSchedulerWork();
- throw thresholdError;
+ return await settleFailure(thresholdError);
}
reducerFailures = 0;
if (!this.canceled && !this.externallyFailed) this.state = outcome.run;
@@ -1083,6 +1095,8 @@ export class DeepScanCoordinator {
try {
return await this.options.store.finish(input);
} catch (firstError) {
+ if (confirmedOwnershipChange(firstError, this.state.scanId))
+ throw firstError;
this.log({
event: "coordinator_finish_replay",
scanId: this.state.scanId,
@@ -1091,6 +1105,8 @@ export class DeepScanCoordinator {
try {
return await this.options.store.finish(input);
} catch (replayError) {
+ if (confirmedOwnershipChange(replayError, this.state.scanId))
+ throw replayError;
throw new Error(
`Deep Scan terminal persistence replay failed: ${errorMessage(replayError)}`,
{ cause: firstError },
diff --git a/plugins/codex-security/mcp-app/src/deep-scan/errors.ts b/plugins/codex-security/mcp-app/src/deep-scan/errors.ts
index 9b480c6a8..4385dbe72 100644
--- a/plugins/codex-security/mcp-app/src/deep-scan/errors.ts
+++ b/plugins/codex-security/mcp-app/src/deep-scan/errors.ts
@@ -1,4 +1,5 @@
import { createHash } from "node:crypto";
+import type { DeepScanRunState } from "./types.js";
const MAX_PERSISTED_ERROR_LENGTH = 2_400;
@@ -25,6 +26,31 @@ export class DeepScanNonRetryableError extends Error {
}
}
+/** A store read confirmed that this coordinator no longer owns the scan. */
+export class DeepScanOwnershipChangedError extends Error {
+ constructor(
+ readonly run: DeepScanRunState,
+ cause: unknown,
+ ) {
+ super(errorMessage(cause), { cause });
+ this.name = "DeepScanOwnershipChangedError";
+ }
+}
+
+/** Only a store-confirmed state for this scan establishes ownership loss. */
+export function confirmedOwnershipChange(
+ error: unknown,
+ scanId: string,
+): DeepScanOwnershipChangedError | undefined {
+ for (let cause = error; cause instanceof Error; cause = cause.cause) {
+ if (
+ cause instanceof DeepScanOwnershipChangedError &&
+ cause.run.scanId === scanId
+ )
+ return cause;
+ }
+}
+
/** Keep SQLite's bounded diagnostic useful while the manifest retains the full error. */
export function boundedDeepScanErrorMessage(error: unknown): string {
return boundedDeepScanErrorText(
diff --git a/plugins/codex-security/mcp-app/src/deep-scan/store.ts b/plugins/codex-security/mcp-app/src/deep-scan/store.ts
index 4928a3691..05c1166d6 100644
--- a/plugins/codex-security/mcp-app/src/deep-scan/store.ts
+++ b/plugins/codex-security/mcp-app/src/deep-scan/store.ts
@@ -5,6 +5,7 @@ import { writeJsonAtomic } from "./artifacts.js";
import {
boundedDeepScanErrorMessage,
DeepScanNonRetryableError,
+ DeepScanOwnershipChangedError,
isStaleCoordinatorGenerationError,
} from "./errors.js";
import type {
@@ -488,8 +489,23 @@ export class WorkbenchDeepScanStore {
: await this.runWorkbench(args, input);
} catch (error) {
const scanId = argumentValue(args, "--scan-id");
- if (scanId && isStaleCoordinatorGenerationError(error)) {
- this.coordinatorLeases.delete(scanId);
+ const lease = scanId ? this.coordinatorLeases.get(scanId) : undefined;
+ if (lease && isStaleCoordinatorGenerationError(error)) {
+ // Diagnostics can contain paths or user text; only state establishes ownership.
+ const current = await this.get(
+ lease.input.scanId,
+ lease.input.threadId,
+ ).catch(() => undefined);
+ if (
+ current &&
+ (current.status !== "running" ||
+ (current.coordinatorGeneration !== undefined &&
+ lease.run.coordinatorGeneration !== undefined &&
+ current.coordinatorGeneration >
+ lease.run.coordinatorGeneration))
+ ) {
+ throw new DeepScanOwnershipChangedError(current, error);
+ }
}
throw error;
}
diff --git a/plugins/codex-security/mcp-app/src/deep-scan/worker-runner.ts b/plugins/codex-security/mcp-app/src/deep-scan/worker-runner.ts
index fc3db2204..e95d546c7 100644
--- a/plugins/codex-security/mcp-app/src/deep-scan/worker-runner.ts
+++ b/plugins/codex-security/mcp-app/src/deep-scan/worker-runner.ts
@@ -17,6 +17,7 @@ import {
boundedDeepScanErrorMessage,
classifyCodexWorkerError as asError,
DeepScanNonRetryableError,
+ confirmedOwnershipChange,
isCodexCybersecurityPolicyRefusal,
} from "./errors.js";
import { renderDedupPrompt, renderDiscoveryPrompt } from "./templates.js";
@@ -212,6 +213,7 @@ export class DeepScanWorkerRunner {
async () => await this.options.store.updateWorker(acceptance),
);
} catch (error) {
+ if (confirmedOwnershipChange(error, run.scanId)) throw error;
if (!this.options.signal.aborted) throw error;
return { type: "discovery", status: "canceled", workerId };
}
@@ -464,6 +466,7 @@ export class DeepScanWorkerRunner {
threadId: result.threadId ?? activeThreadId,
};
} catch (error) {
+ if (confirmedOwnershipChange(error, run.scanId)) throw error;
if (signal.aborted) {
return await this.cancelAttempt(input, attempt, activeThreadId);
}
@@ -612,6 +615,8 @@ export class DeepScanWorkerRunner {
try {
return await operation();
} catch (firstError) {
+ if (confirmedOwnershipChange(firstError, this.options.run.scanId))
+ throw firstError;
this.options.log({
event,
scanId: this.options.run.scanId,
@@ -621,6 +626,8 @@ export class DeepScanWorkerRunner {
try {
return await operation();
} catch (replayError) {
+ if (confirmedOwnershipChange(replayError, this.options.run.scanId))
+ throw replayError;
throw new Error(
`Deep Scan persistence replay failed: ${asError(replayError).message}`,
{ cause: firstError },
diff --git a/plugins/codex-security/mcp-app/tests/deep_scan_coordinator_fixture.ts b/plugins/codex-security/mcp-app/tests/deep_scan_coordinator_fixture.ts
index 879e8de59..0daa7aeb3 100644
--- a/plugins/codex-security/mcp-app/tests/deep_scan_coordinator_fixture.ts
+++ b/plugins/codex-security/mcp-app/tests/deep_scan_coordinator_fixture.ts
@@ -25,6 +25,7 @@ import { importModule } from "./import-module.ts";
export const {
DeepScanCoordinator,
DeepScanCoordinatorRegistry,
+ WorkbenchDeepScanStore,
DeepScanNonRetryableError,
DeepScanRemoteCoordinator,
AsyncLock,
@@ -34,6 +35,7 @@ export const {
stdin: {
contents: `
export * from "./registry.ts";
+ export { WorkbenchDeepScanStore } from "./store.ts";
export { classifyCodexWorkerError } from "./errors.ts";
`,
resolveDir: path.join(import.meta.dirname, "../src/deep-scan/"),
diff --git a/plugins/codex-security/mcp-app/tests/test_deep_scan_artifact_validation.ts b/plugins/codex-security/mcp-app/tests/test_deep_scan_artifact_validation.ts
index 110f7bbf8..6fafbfb3f 100644
--- a/plugins/codex-security/mcp-app/tests/test_deep_scan_artifact_validation.ts
+++ b/plugins/codex-security/mcp-app/tests/test_deep_scan_artifact_validation.ts
@@ -390,6 +390,79 @@ async function testReducerValidation(root: string) {
),
[previousCollisionA.summary, previousCollisionB.summary],
);
+ const referenced = (original: Record<string, unknown>, refs: string[]) => ({
+ ...original,
+ provenance: { source: "local_plugin", sourceFindingIds: refs },
+ });
+ for (const [label, previous, output, expected] of [
+ [
+ "independent",
+ [previousCollisionA],
+ [
+ referenced(collidingOriginalA, ["origin:a"]),
+ referenced(collidingOriginalB, ["fresh:0"]),
+ ],
+ 1,
+ ],
+ [
+ "repeat",
+ [previousCollisionA],
+ [referenced(collidingOriginalA, ["origin:a", "fresh:0"])],
+ 0,
+ ],
+ [
+ "consolidate",
+ [previousCollisionA, previousCollisionB],
+ [
+ referenced(collidingOriginalA, ["origin:a", "origin:b"]),
+ referenced(collidingOriginalB, ["fresh:0"]),
+ ],
+ 1,
+ ],
+ [
+ "legacy",
+ [
+ {
+ ...referenced(collidingOriginalA, ["legacy-id-without-body"]),
+ summary: "Previously accepted legacy evidence.",
+ },
+ ],
+ [
+ referenced(collidingOriginalB, ["fresh:0"]),
+ referenced(collidingOriginalA, ["previous:0"]),
+ ],
+ 1,
+ ],
+ ] as const) {
+ const snapshot = {
+ discoveries: [{ workerId: "fresh", result: draft([collidingOriginalB]) }],
+ previous: draft([...previous]),
+ };
+ await writeResult(resultPath, draft([...output]));
+ const accepted = await validateSnapshot(`dedup-${label}`, snapshot);
+ assert.equal(accepted.newFindings, expected, label);
+ if (label === "legacy") {
+ assert.equal(
+ accepted.result.findings[0].provenance.previousFindings,
+ undefined,
+ );
+ assert.equal(
+ accepted.result.findings[1].provenance.previousFindings[0].title,
+ collidingOriginalA.title,
+ );
+ }
+ // Reopening the accepted JSON keeps the same ancestry and adds no finding.
+ assert.equal(
+ (
+ await validateSnapshot(`resume-${label}`, {
+ discoveries: [],
+ previous: await readJson(resultPath),
+ })
+ ).newFindings,
+ 0,
+ label,
+ );
+ }
await writeResult(first.resultPath, draft([firstFinding]));
await writeResult(resultPath, draft([firstFinding]));
diff --git a/plugins/codex-security/mcp-app/tests/test_deep_scan_coordinator.ts b/plugins/codex-security/mcp-app/tests/test_deep_scan_coordinator.ts
index d4bb261f6..bb7769e59 100644
--- a/plugins/codex-security/mcp-app/tests/test_deep_scan_coordinator.ts
+++ b/plugins/codex-security/mcp-app/tests/test_deep_scan_coordinator.ts
@@ -22,6 +22,7 @@ import path from "node:path";
import { testDeepScanLifecycle } from "./deep_scan_lifecycle_cases.ts";
import {
DeepScanCoordinatorRegistry,
+ WorkbenchDeepScanStore,
DeepScanNonRetryableError,
DeepScanRemoteCoordinator,
AsyncLock,
@@ -357,6 +358,54 @@ async function testConsumedSourceWithToolDiagnosticIsNotRereadAfterReducerWrites
assert.deepEqual((await readJson(terminal.manifestPath)).findings, []);
}
+async function testIndependentSourceGroupsDoNotSaturate() {
+ const { fixture, store } = await coordinatorFixture({
+ workers: 1,
+ stopAfterNoNew: 1,
+ maxDiscoveryRuns: 4,
+ });
+ const executor = new FakeExecutor({
+ discoveryCandidateId: "shared-identity",
+ });
+ const run = executor.run.bind(executor);
+ executor.run = async (request) => {
+ const result = await run(request);
+ if (request.kind === "dedup") {
+ const assigned = request.artifactContext!.deepReducer!;
+ const previous = assigned.previousReducerResultPath
+ ? (await readJson(assigned.previousReducerResultPath)).findings
+ : [];
+ const discoveries = await Promise.all(
+ assigned.claimedWorkers.map(async (worker) =>
+ (await readJson(worker.resultPath)).findings.map(
+ (finding: Record<string, unknown>, index: number) => ({
+ ...finding,
+ provenance: {
+ source: "local_plugin",
+ sourceFindingIds: [`${worker.id}:${index}`],
+ },
+ }),
+ ),
+ ),
+ );
+ const resultPath = path.join(
+ request.artifactContext!.root,
+ "result.json",
+ );
+ await writeJson(resultPath, {
+ ...(await readJson(resultPath)),
+ findings: [...previous, ...discoveries.flat()],
+ });
+ }
+ return result;
+ };
+ const terminal = await runCoordinator(fixture, store, executor);
+ assert.equal(terminal?.terminalReason, "capped", terminal?.error);
+ assert.equal(store.run.noNewStreak, 0);
+ assert.equal(executor.discoveryCalls, 4);
+ assert.equal((await readJson(terminal.manifestPath)).findings.length, 4);
+}
+
async function testRetryKeepsLogicalWorker() {
const { fixture, store } = await coordinatorFixture(
twoRunConfig({ workers: 2, subagents: 1 }),
@@ -2353,6 +2402,266 @@ async function testStaleMutationObservesReplacement() {
);
}
+async function testStoreConfirmedReplacementSurvivesReadFailure() {
+ for (const confirmation of ["mutation", "failure-write"])
+ for (const replaced of [false, true]) {
+ const fixture = await fixtureRun();
+ const run = { ...fixture.run, coordinatorGeneration: 2 };
+ let current: DeepScanRunState = run;
+ const stale =
+ "Deep Scan coordinator lease belongs to a newer generation.";
+ const diagnostic = replaced
+ ? stale
+ : `Permission denied: /fixture/${stale}/lock`;
+ let reads = 0;
+ const failures: string[][] = [];
+ const store = new WorkbenchDeepScanStore(async (args: string[]) => {
+ switch (args[0]) {
+ case "claim-deep-scan-coordinator":
+ return { deepScan: current, coordinatorDisposition: "claimed" };
+ case "update-progress":
+ if (replaced) current = { ...current, coordinatorGeneration: 3 };
+ throw new Error(diagnostic);
+ case "get-deep-scan": {
+ reads += 1;
+ if (
+ reads === 2 ||
+ (reads === 1 && (!replaced || confirmation === "failure-write"))
+ )
+ throw new Error("sqlite3.OperationalError: database is locked");
+ const snapshot = current;
+ if (replaced)
+ current = {
+ ...current,
+ status: "succeeded",
+ terminalReason: "capped",
+ };
+ return { deepScan: snapshot };
+ }
+ case "fail-deep-scan":
+ failures.push(args);
+ if (replaced)
+ throw new Error(
+ args.includes("--coordinator-generation")
+ ? stale
+ : "Deep Scan mutation requires the current coordinator lease.",
+ );
+ current = {
+ ...current,
+ status: "failed",
+ error: args
+ .find((arg) => arg.startsWith("--message="))!
+ .slice("--message=".length),
+ };
+ return { deepScan: current };
+ default:
+ throw new Error(`Unexpected fixture operation: ${args[0]}`);
+ }
+ });
+ await store.claimCoordinator({
+ scanId: run.scanId,
+ threadId: "fixture-thread",
+ });
+ const coordinator = new DeepScanCoordinatorRegistry().start({
+ run,
+ store,
+ executor: new FakeExecutor(),
+ pluginRoot: fixture.pluginRoot,
+ clock: immediateClock,
+ threadId: "fixture-thread",
+ heartbeatIntervalMs: 60_000,
+ });
+ const terminal = await coordinator.wait(undefined, 2_500);
+ assert.equal(
+ terminal?.status,
+ replaced ? "succeeded" : "failed",
+ confirmation,
+ );
+ assert.equal(terminal?.coordinatorGeneration, replaced ? 3 : 2);
+ assert.equal(current.status, terminal?.status);
+ assert.equal(
+ failures.length,
+ replaced && confirmation === "mutation" ? 0 : 1,
+ );
+ if (replaced) {
+ assert.equal(reads, confirmation === "mutation" ? 3 : 4);
+ assert.deepEqual(store.coordinatorLeaseArgs(run.scanId), []);
+ } else {
+ assert.equal(
+ reads,
+ 2,
+ "unconfirmed diagnostics must not add ownership reads",
+ );
+ assert.equal(terminal?.error, diagnostic);
+ const generationFlag = failures[0].indexOf("--coordinator-generation");
+ assert.equal(failures[0][generationFlag + 1], "2");
+ }
+ }
+}
+
+async function testStoreConfirmationSurvivesReplayAndThreadMetadata() {
+ for (const operation of [
+ "finish",
+ "worker-acceptance",
+ "thread-started",
+ ] as const)
+ for (const replaced of [false, true]) {
+ const fixture = await fixtureRun();
+ const run = { ...fixture.run, coordinatorGeneration: 2 };
+ let current: DeepScanRunState = run;
+ const store = new FakeStore(run);
+ const stale =
+ "Deep Scan coordinator lease belongs to a newer generation.";
+ const diagnostic = replaced
+ ? stale
+ : `Permission denied: /fixture/${stale}/lock`;
+ let mutations = 0;
+ let reads = 0;
+ let failures = 0;
+ const persisted = new WorkbenchDeepScanStore(async (args: string[]) => {
+ switch (args[0]) {
+ case "claim-deep-scan-coordinator":
+ return { deepScan: current, coordinatorDisposition: "claimed" };
+ case "finish-deep-scan":
+ case "upsert-deep-scan-worker":
+ mutations += 1;
+ if (operation !== "thread-started" && mutations === 1)
+ throw new Error("Synthetic initial persistence failure.");
+ if (replaced) current = { ...current, coordinatorGeneration: 3 };
+ throw new Error(diagnostic);
+ case "get-deep-scan": {
+ reads += 1;
+ if (reads === 2)
+ throw new Error("sqlite3.OperationalError: database is locked");
+ const snapshot = current;
+ if (replaced)
+ current = {
+ ...current,
+ status: "succeeded",
+ terminalReason: "capped",
+ };
+ return { deepScan: snapshot };
+ }
+ case "fail-deep-scan":
+ failures += 1;
+ if (replaced)
+ throw new Error(
+ "Deep Scan mutation requires the current coordinator lease.",
+ );
+ current = {
+ ...current,
+ status: "failed",
+ error: args
+ .find((arg) => arg.startsWith("--message="))!
+ .slice("--message=".length),
+ };
+ return { deepScan: current };
+ default:
+ throw new Error(`Unexpected fixture operation: ${args[0]}`);
+ }
+ });
+ await persisted.claimCoordinator({
+ scanId: run.scanId,
+ threadId: "fixture-thread",
+ });
+ store.get = persisted.get.bind(persisted);
+ store.fail = persisted.fail.bind(persisted);
+ if (operation === "finish")
+ store.finish = persisted.finish.bind(persisted);
+ else {
+ const updateWorker = store.updateWorker.bind(store);
+ store.updateWorker = async (update) =>
+ (operation === "worker-acceptance" &&
+ update.status === "succeeded") ||
+ (operation === "thread-started" && update.threadId !== undefined)
+ ? persisted.updateWorker(update)
+ : updateWorker(update);
+ }
+ const coordinator = new DeepScanCoordinatorRegistry().start({
+ run,
+ store,
+ executor: new FakeExecutor(),
+ pluginRoot: fixture.pluginRoot,
+ clock: immediateClock,
+ threadId: "fixture-thread",
+ heartbeatIntervalMs: 60_000,
+ retryDelaysMs: [],
+ });
+ const terminal = await coordinator.wait(undefined, 2_500);
+ assert.equal(
+ terminal?.status,
+ replaced ? "succeeded" : "failed",
+ operation,
+ );
+ assert.equal(terminal?.coordinatorGeneration, replaced ? 3 : 2);
+ assert.equal(current.status, terminal?.status);
+ assert.equal(failures, replaced ? 0 : 1);
+ if (replaced) {
+ assert.equal(mutations, operation === "thread-started" ? 1 : 2);
+ assert.equal(reads, 3);
+ } else {
+ assert.ok(terminal?.error?.includes(diagnostic));
+ }
+ }
+}
+
+async function testReducerDiagnosticDoesNotOverrideOwnership() {
+ for (const failRead of [false, true])
+ for (const reference of [
+ "missing-reference",
+ "Deep Scan coordinator lease belongs to a newer generation.",
+ ]) {
+ const { fixture, store } = await coordinatorFixture({
+ workers: 1,
+ maxDiscoveryRuns: 1,
+ stopAfterConsecutiveErrors: 1,
+ });
+ fixture.run.coordinatorGeneration = store.run.coordinatorGeneration = 2;
+ const worker = new FakeExecutor({ discoveryCandidateId: "candidate-1" });
+ const executor = {
+ async run(request: Parameters<FakeExecutor["run"]>[0]) {
+ const result = await worker.run(request);
+ if (request.kind === "dedup") {
+ const resultPath = path.join(
+ request.artifactContext!.root,
+ "result.json",
+ );
+ const draft = await readJson(resultPath);
+ draft.findings[0].provenance.sourceFindingIds = [reference];
+ await writeJson(resultPath, draft);
+ }
+ return result;
+ },
+ };
+ if (failRead) {
+ const get = store.get.bind(store);
+ let failed = false;
+ store.get = async (...args) => {
+ if (!failed) {
+ failed = true;
+ throw new Error("Synthetic authoritative ownership read failed");
+ }
+ return get(...args);
+ };
+ }
+ let observations = 0;
+ const terminal = await runCoordinator(fixture, store, executor, {
+ retryDelaysMs: [],
+ threadId: "fixture-thread",
+ heartbeatIntervalMs: 60_000,
+ observeReplacement: async (run) => {
+ observations += 1;
+ return run;
+ },
+ });
+ assert.equal(observations, 0);
+ assert.equal(terminal?.status, "failed");
+ assert.equal(store.run.status, "failed");
+ assert.equal(terminal?.coordinatorGeneration, 2);
+ assert.ok(store.failureMessages[0].includes(reference));
+ }
+}
+
async function testJoinAndOrphanRules() {
const fixture = await fixtureRun(twoRunConfig({ workers: 2, subagents: 1 }));
const existingCoordinator = { marker: "existing" };
@@ -3379,7 +3688,9 @@ async function testSuccessfulDeepCoverageIgnoresWorkerAndReducerReviewStatus() {
}
}
-async function testSaturationIgnoresDiscoveryCancellationWriteFailure() {
+async function testSaturationIgnoresDiscoveryCancellationWriteFailure(
+ replaced = false,
+) {
const { fixture, store } = await coordinatorFixture({
workers: 2,
stopAfterNoNew: 2,
@@ -3391,18 +3702,54 @@ async function testSaturationIgnoresDiscoveryCancellationWriteFailure() {
executor.dedupGate = Promise.withResolvers<void>();
const updateWorker = store.updateWorker.bind(store);
const rejectedCancellations = new Set();
+ fixture.run.coordinatorGeneration = store.run.coordinatorGeneration = 2;
+ let ownershipReads = 0;
+ const persisted = new WorkbenchDeepScanStore(async (args: string[]) => {
+ if (args[0] === "claim-deep-scan-coordinator")
+ return { deepScan: store.run, coordinatorDisposition: "claimed" };
+ if (args[0] === "get-deep-scan") {
+ ownershipReads += 1;
+ if (ownershipReads > 1)
+ throw new Error("sqlite3.OperationalError: database is locked");
+ return {
+ deepScan: {
+ ...store.run,
+ status: "succeeded",
+ coordinatorGeneration: 3,
+ terminalReason: "capped",
+ },
+ };
+ }
+ throw new Error(
+ "Deep Scan coordinator lease belongs to a newer generation.",
+ );
+ });
+ if (replaced) {
+ await persisted.claimCoordinator({
+ scanId: fixture.run.scanId,
+ threadId: "fixture-thread",
+ });
+ store.get = persisted.get.bind(persisted);
+ }
store.updateWorker = async (update) => {
if (update.kind === "discovery" && update.status === "canceled") {
assert.equal(executor.dedupSignal?.aborted, true);
assert.equal(store.run.noNewStreak, 2);
rejectedCancellations.add(update.id);
+ if (replaced) return await persisted.updateWorker(update);
throw new Error("fixture cancellation persistence failure");
}
return updateWorker(update);
};
const completed: ScanDraftInput[] = [];
const coordinator = createCoordinator(fixture, store, executor, {
- onComplete: async (draft) => void completed.push(structuredClone(draft)),
+ threadId: "fixture-thread",
+ heartbeatIntervalMs: 60_000,
+ onComplete: async (draft) => {
+ if (replaced)
+ throw new Error("Only a running scan can update artifacts.");
+ completed.push(structuredClone(draft));
+ },
});
coordinator.start();
await executor.dedupStarted.promise;
@@ -3418,6 +3765,15 @@ async function testSaturationIgnoresDiscoveryCancellationWriteFailure() {
"the redundant discoveries reached the failing cancellation write",
);
assert.equal(terminal?.status, "succeeded", terminal?.error);
+ if (replaced) {
+ assert.equal(terminal.coordinatorGeneration, 3);
+ assert.equal(terminal.terminalReason, "capped");
+ assert.equal(store.failureMessages.length, 0);
+ assert.equal(store.finishCalls.length, 0);
+ assert.equal(completed.length, 0);
+ assert.equal(executor.runningDiscovery, 0);
+ return;
+ }
assert.equal(terminal.terminalReason, "saturated");
assert.equal(store.failureMessages.length, 0);
assert.equal(store.finishCalls.length, 1);
@@ -3738,6 +4094,7 @@ try {
await testWorkerScopedCandidateSourceAggregation();
await testConsumedSourceIsNotRereadAfterReducerWritesResult();
await testConsumedSourceWithToolDiagnosticIsNotRereadAfterReducerWritesResult();
+ await testIndependentSourceGroupsDoNotSaturate();
await testRetryKeepsLogicalWorker();
await testSandboxDiagnosticSurvivesArtifactRetries();
await testCompletionOrdering();
@@ -3745,6 +4102,7 @@ try {
await testSaturationOmitsWorkerAcceptedDuringCancellation();
await testSuccessfulDeepCoverageIgnoresWorkerAndReducerReviewStatus();
await testSaturationIgnoresDiscoveryCancellationWriteFailure();
+ await testSaturationIgnoresDiscoveryCancellationWriteFailure(true);
await testPublicationUsesAcceptedReducerSnapshot();
await testDirectReducerCannotDropAcceptedFinding();
await testSaturationDrainsBufferedAndCancelsInflight();
@@ -3811,6 +4169,9 @@ try {
await testStoppedPublicationFailureBoundsPrefixedDiagnostic();
await testTerminalReadFailureIsNotRecordedAsPublicationFailure();
await testStaleMutationObservesReplacement();
+ await testStoreConfirmedReplacementSurvivesReadFailure();
+ await testStoreConfirmationSurvivesReplayAndThreadMetadata();
+ await testReducerDiagnosticDoesNotOverrideOwnership();
await testCoordinatorHeartbeatsStopAfterOwnershipChanges();
await testCoordinatorHeartbeatsContinueDuringBlockedOwnershipRead();
await testRemoteObserverRetriesTransientPersistenceFailures();
diff --git a/plugins/codex-security/mcp-app/tests/test_deep_scan_handoff_races.ts b/plugins/codex-security/mcp-app/tests/test_deep_scan_handoff_races.ts
new file mode 100644
index 000000000..43af8d961
--- /dev/null
+++ b/plugins/codex-security/mcp-app/tests/test_deep_scan_handoff_races.ts
@@ -0,0 +1,271 @@
+import assert from "node:assert/strict";
+import { mock, test } from "node:test";
+import { Thread } from "@openai/codex-sdk";
+import type {
+ CodexWorkerRequest,
+ CodexWorkerResult,
+ DeepScanRunState,
+ DeepScanWorkerMutation,
+} from "../src/deep-scan/types.js";
+import type { DeepScanCoordinator } from "../src/deep-scan/coordinator.js";
+import {
+ DeepScanCoordinatorRegistry,
+ DeepScanNonRetryableError,
+ FakeExecutor,
+ WorkbenchDeepScanStore,
+ fixtureRun,
+ temporaryDirectories,
+} from "./deep_scan_coordinator_fixture.ts";
+
+try {
+ for (const scenario of [
+ "deadline",
+ "concurrent",
+ "acceptance",
+ "acceptance canceled",
+ "user cancellation",
+ "terminal state",
+ "ordinary failure",
+ ] as const) {
+ await test(`preserves coordinator authority through ${scenario}`, async () => {
+ const concurrent = scenario === "concurrent";
+ const acceptance = scenario.startsWith("acceptance");
+ const userCanceled = [
+ "user cancellation",
+ "acceptance canceled",
+ ].includes(scenario);
+ const fixture = await fixtureRun({
+ workers: concurrent || acceptance ? 2 : 1,
+ maxDiscoveryRuns: concurrent || acceptance ? 2 : 1,
+ });
+ const run = { ...fixture.run, coordinatorGeneration: 2 };
+ const replacement: DeepScanRunState =
+ scenario === "terminal state"
+ ? { ...run, status: "succeeded", terminalReason: "capped" }
+ : { ...run, coordinatorGeneration: 3 };
+ const diagnostic =
+ scenario === "ordinary failure"
+ ? "Permission denied: /fixture/Deep Scan coordinator lease belongs to a newer generation./lock\nSynthetic diagnostic detail."
+ : "Deep Scan coordinator lease belongs to a newer generation.";
+ let reads = 0;
+ let failureWrites = 0;
+ let metadataMutations = 0;
+ let queuedFailureSeen = false;
+ const workers = new Map<string, Record<string, unknown>>();
+ const store = new WorkbenchDeepScanStore(async (args: string[]) => {
+ const value = (flag: string) => args[args.indexOf(flag) + 1];
+ switch (args[0]) {
+ case "claim-deep-scan-coordinator":
+ return { deepScan: run, coordinatorDisposition: "claimed" };
+ case "update-progress":
+ return { deepScan: run };
+ case "upsert-deep-scan-worker": {
+ if (
+ acceptance
+ ? value("--status") === "succeeded"
+ : args.includes("--sdk-thread-id") ||
+ (concurrent && metadataMutations > 0)
+ ) {
+ if (
+ args.includes("--sdk-thread-id") &&
+ value("--status") === "running"
+ )
+ metadataMutations += 1;
+ else if (concurrent && !args.includes("--sdk-thread-id"))
+ queuedFailureSeen = true;
+ throw new Error(diagnostic);
+ }
+ workers.set(value("--worker-id"), {
+ id: value("--worker-id"),
+ kind: "discovery",
+ status: value("--status"),
+ mergeState: "none",
+ promptPath: value("--prompt-path"),
+ artifactDir: value("--artifact-dir"),
+ attempt: 1,
+ });
+ return { deepScan: { ...run, workers: [...workers.values()] } };
+ }
+ case "get-deep-scan":
+ reads += 1;
+ if (reads === (scenario === "ordinary failure" ? 1 : 2))
+ throw new Error("sqlite3.OperationalError: database is locked");
+ return {
+ deepScan:
+ scenario === "ordinary failure"
+ ? run
+ : reads === 1
+ ? replacement
+ : {
+ ...replacement,
+ status: "succeeded",
+ terminalReason: "capped",
+ },
+ };
+ case "fail-deep-scan":
+ failureWrites += 1;
+ if (scenario !== "ordinary failure")
+ throw new Error(
+ "Deep Scan mutation requires the current coordinator lease.",
+ );
+ return {
+ deepScan: {
+ ...run,
+ status: "failed",
+ error: args
+ .find((arg) => arg.startsWith("--message="))!
+ .slice("--message=".length),
+ },
+ };
+ default:
+ throw new Error(`Unexpected fixture operation: ${args[0]}`);
+ }
+ });
+ await store.claimCoordinator({
+ scanId: run.scanId,
+ threadId: "fixture-owner",
+ });
+ const secondQueued = Promise.withResolvers<void>();
+ const firstMetadata = Promise.withResolvers<void>();
+ const acceptanceConfirmed = Promise.withResolvers<void>();
+ let acceptanceSignal: AbortSignal;
+ const updateWorker = store.updateWorker.bind(store);
+ let queued = 0;
+ store.updateWorker = async (update: DeepScanWorkerMutation) => {
+ if (concurrent && update.status === "queued" && ++queued === 2) {
+ secondQueued.resolve();
+ await firstMetadata.promise;
+ }
+ try {
+ return await updateWorker(update);
+ } catch (error) {
+ if (acceptance && update.status === "succeeded") {
+ acceptanceConfirmed.resolve();
+ await new Promise<void>((resolve) => {
+ if (acceptanceSignal.aborted) resolve();
+ else
+ acceptanceSignal.addEventListener("abort", () => resolve(), {
+ once: true,
+ });
+ });
+ }
+ throw error;
+ }
+ };
+ const executorErrors: Error[] = [];
+ let executions = 0;
+ let coordinator: DeepScanCoordinator;
+ const executor = {
+ async run(request: CodexWorkerRequest): Promise<CodexWorkerResult> {
+ const sequence = ++executions;
+ if (acceptance) {
+ if (sequence === 1) {
+ acceptanceSignal = request.signal;
+ return new FakeExecutor().run(request);
+ }
+ await acceptanceConfirmed.promise;
+ if (userCanceled)
+ coordinator.cancel("Synthetic user cancellation.");
+ throw new DeepScanNonRetryableError(
+ "Synthetic independent worker failure.",
+ );
+ }
+ if (concurrent) await secondQueued.promise;
+ // Only the process event source is synthetic; SDK iterator cleanup is real.
+ const thread: Thread = Reflect.construct(Thread, [
+ {
+ async *run() {
+ yield JSON.stringify({
+ type: "thread.started",
+ thread_id: `fixture-worker-${sequence}`,
+ });
+ },
+ },
+ {},
+ {},
+ ]);
+ try {
+ const { events } = await thread.runStreamed("Synthetic worker");
+ for await (const event of events) {
+ assert.equal(event.type, "thread.started");
+ if (event.type !== "thread.started") continue;
+ const mutation = request.onThreadStarted!(event.thread_id);
+ if (sequence === 1) firstMetadata.resolve();
+ await mutation;
+ }
+ } catch (error) {
+ assert.ok(error instanceof Error);
+ executorErrors.push(error);
+ if (scenario === "deadline") {
+ mock.timers.tick(1);
+ assert.equal(request.signal.aborted, true);
+ mock.timers.reset();
+ } else if (scenario === "user cancellation") {
+ coordinator.cancel("Synthetic user cancellation.");
+ }
+ throw error;
+ }
+ throw new Error("The synthetic metadata mutation must reject.");
+ },
+ };
+ if (scenario === "deadline") mock.timers.enable({ apis: ["setTimeout"] });
+ try {
+ coordinator = new DeepScanCoordinatorRegistry().start({
+ run,
+ store,
+ executor,
+ pluginRoot: fixture.pluginRoot,
+ threadId: "fixture-owner",
+ heartbeatIntervalMs: 60_000,
+ discoveryTimeoutMs: scenario === "deadline" ? 1 : 60_000,
+ retryDelaysMs: [],
+ });
+ const terminal = await coordinator.wait(undefined, 5_000);
+ assert.equal(
+ terminal?.status,
+ userCanceled
+ ? "canceled"
+ : scenario === "ordinary failure"
+ ? "failed"
+ : "succeeded",
+ );
+ assert.equal(
+ terminal?.coordinatorGeneration,
+ userCanceled ||
+ ["terminal state", "ordinary failure"].includes(scenario)
+ ? 2
+ : 3,
+ );
+ assert.equal(failureWrites, scenario === "ordinary failure" ? 1 : 0);
+ if (scenario === "ordinary failure") {
+ assert.ok(terminal?.error?.includes(diagnostic));
+ assert.equal(
+ reads,
+ 2,
+ "diagnostic text must not establish ownership loss",
+ );
+ } else {
+ assert.equal(
+ reads,
+ scenario === "terminal state" || userCanceled ? 1 : 3,
+ );
+ if (acceptance) assert.equal(executions, 2);
+ if (concurrent) {
+ assert.equal(queuedFailureSeen, true);
+ assert.equal(executions, 1);
+ assert.equal(metadataMutations, 1);
+ assert.equal(executorErrors.length, 1);
+ assert.equal(
+ executorErrors[0].name,
+ "DeepScanOwnershipChangedError",
+ );
+ }
+ }
+ } finally {
+ mock.timers.reset();
+ }
+ });
+ }
+} finally {
+ await temporaryDirectories.cleanup();
+}
diff --git a/plugins/codex-security/mcp-app/tests/test_deep_scan_store.ts b/plugins/codex-security/mcp-app/tests/test_deep_scan_store.ts
index 188522809..f153cd0d5 100644
--- a/plugins/codex-security/mcp-app/tests/test_deep_scan_store.ts
+++ b/plugins/codex-security/mcp-app/tests/test_deep_scan_store.ts
@@ -237,11 +237,12 @@ async function testOwnershipReadClearsStaleLease() {
const store = new WorkbenchDeepScanStore(async (args: string[]) => {
calls.push(args);
if (args[0] === "update-progress") {
+ generation = 4;
throw new Error(
"Deep Scan coordinator lease belongs to a newer generation.",
);
}
- if (args[0] === "get-deep-scan") generation = 3;
+ if (args[0] === "get-deep-scan" && generation === 2) generation = 3;
return {
...stateResult(scanId, {
deepScan: { coordinatorGeneration: generation },
@@ -266,13 +267,36 @@ async function testOwnershipReadClearsStaleLease() {
store.updateProgress({ scanId, phase: "discovery" }),
/newer generation/,
);
- generation = 4;
await store.claimCoordinator(lease);
assert.equal(
- calls[4].includes("--coordinator-generation"),
+ calls.at(-1)!.includes("--coordinator-generation"),
false,
- "a fenced mutation must also clear the cached lease",
+ "a confirmed newer generation after a fenced mutation must clear the cached lease",
);
+
+ for (const readFails of [false, true]) {
+ const original = new Error(
+ "Permission denied: /fixture/Deep Scan coordinator lease belongs to a newer generation./lock",
+ );
+ const owned = new WorkbenchDeepScanStore(async (args: string[]) => {
+ if (args[0] === "update-progress") throw original;
+ if (args[0] === "get-deep-scan" && readFails)
+ throw new Error("read failed");
+ return {
+ ...stateResult(scanId, { deepScan: { coordinatorGeneration: 2 } }),
+ coordinatorDisposition: "claimed",
+ };
+ });
+ await owned.claimCoordinator(lease);
+ await assert.rejects(
+ owned.updateProgress({ scanId, phase: "discovery" }),
+ (error: unknown) => error === original,
+ );
+ assert.deepEqual(owned.coordinatorLeaseArgs(scanId), [
+ "--coordinator-generation",
+ "2",
+ ]);
+ }
}
async function testWorkerResponseParsing() {