Skip to content

OpenAI's Codex Security CLI and TypeScript SDK for finding, validating, and fixing security vulnerabilities. npm: https://www.npmjs.com/package/@openai/codex-security

fix(deep-scan): checkpoint accepted reviews before honoring stop conditions (#2318)

Michael D'Angelo832c46d0cdcd
diff --git a/plugins/codex-security/mcp-app/src/deep-scan/coordinator.ts b/plugins/codex-security/mcp-app/src/deep-scan/coordinator.ts
index cfa33feed..d5caf3cda 100644
--- a/plugins/codex-security/mcp-app/src/deep-scan/coordinator.ts
+++ b/plugins/codex-security/mcp-app/src/deep-scan/coordinator.ts
@@ -735,18 +735,36 @@ export class DeepScanCoordinator {
let dispatched = this.state.dispatchedCount;
let streak = this.state.noNewStreak;
let errors = this.state.consecutiveErrors;
+ let failure = latestPersistedReplaceableFailure(persisted);
+ const recordOutcome = (error?: typeof failure) => {
+ if (errors >= config.stopAfterConsecutiveErrors) return;
+ errors = error ? errors + 1 : 0;
+ if (error) failure = error;
+ };
if (batch.length) {
- // SQLite recorded in finish order. Restore the streak in dispatch order.
- const lastSuccess = discoverySequence(
- dirname(accepted.at(-1)!.resultPath),
- );
- errors = persisted.filter(
- (worker) =>
- discoverySequence(worker.artifactDir) > lastSuccess &&
- persistedReplaceableFailure(worker),
- ).length;
+ // SQLite observed completion order. Replay the uncheckpointed outcomes
+ // in dispatch order, including failures before a later accepted worker.
+ const lastMerged = accepted.findLast((worker) => merged.has(worker.id));
+ const boundary = lastMerged
+ ? discoverySequence(dirname(lastMerged.resultPath))
+ : -1n;
+ const pending = persisted
+ .filter(
+ (w) =>
+ w.kind === "discovery" &&
+ discoverySequence(w.artifactDir) > boundary,
+ )
+ .sort((a, b) =>
+ Number(
+ discoverySequence(a.artifactDir) - discoverySequence(b.artifactDir),
+ ),
+ );
+ errors = 0;
+ for (const worker of pending) {
+ const error = persistedReplaceableFailure(worker);
+ if (error || worker.status === "succeeded") recordOutcome(error);
+ }
}
- let failure = latestPersistedReplaceableFailure(persisted);
await this.options.store.updateProgress({
scanId,
phase: "discovery",
@@ -759,7 +777,10 @@ export class DeepScanCoordinator {
signal.throwIfAborted();
if (batch.length) {
// Newly dispatched scans arrive in launch order. Recovery uses the same labels.
- const newCounts = await this.mergeAcceptedBatch(result, batch);
+ const { counts: newCounts, stop } = await this.mergeAcceptedBatch(
+ result,
+ batch,
+ );
for (const count of newCounts) {
streak = count ? 0 : streak + 1;
}
@@ -793,6 +814,7 @@ export class DeepScanCoordinator {
retainedFindings: result.findings.length,
});
this.logProgress(acceptedCount);
+ if (stop) throw stop;
}
if (errors >= config.stopAfterConsecutiveErrors && failure) {
throw discoveryErrorLimitError(
@@ -837,21 +859,20 @@ export class DeepScanCoordinator {
batch = [];
for (const outcome of outcomes) {
if (outcome.status === "succeeded") {
- errors = 0;
+ recordOutcome();
batch.push(outcome.worker);
acceptedCount++;
} else if (outcome.status === "failed") {
if (!outcome.replaceableFailureKind) throw outcome.error;
- errors++;
- failure = {
+ recordOutcome({
kind: outcome.replaceableFailureKind,
message: outcome.error.message,
- };
+ });
this.log({
event: "discovery_worker_replaced",
scanId,
workerId: outcome.workerId,
- reason: failure.kind,
+ reason: outcome.replaceableFailureKind,
});
} else if (!this.discoveryDeadlineReached) {
throw new Error(
@@ -862,21 +883,32 @@ export class DeepScanCoordinator {
}
}
- private mergeAcceptedBatch(result: ScanUnion, batch: AcceptedDiscovery[]) {
- return mergeBatch(
+ private async mergeAcceptedBatch(
+ result: ScanUnion,
+ batch: AcceptedDiscovery[],
+ ) {
+ let stop: DeepScanNonRetryableError | undefined;
+ const counts = await mergeBatch(
result,
batch,
this.artifacts.scanDir,
- (prompt, schema) => {
+ async (prompt, schema) => {
+ if (stop) throw stop;
const { executor } = this.options;
if (!executor.compareFindings)
throw new Error("No Deep Scan finding comparison executor");
- return executor.compareFindings(
- prompt,
- this.artifacts.deepRoot,
- this.discoveryAbortController.signal,
- schema,
- );
+ try {
+ return await executor.compareFindings(
+ prompt,
+ this.artifacts.deepRoot,
+ this.abortController.signal,
+ schema,
+ );
+ } catch (error) {
+ // Finish checkpointing accepted workers before stopping discovery.
+ if (error instanceof DeepScanNonRetryableError) stop = error;
+ throw error;
+ }
},
(error) => {
this.abortController.signal.throwIfAborted();
@@ -887,6 +919,7 @@ export class DeepScanCoordinator {
});
},
);
+ return { counts, stop };
}
private async restoreUnion(
diff --git a/plugins/codex-security/mcp-app/tests/test_deep_scan_coordinator.ts b/plugins/codex-security/mcp-app/tests/test_deep_scan_coordinator.ts
index bc834a0e4..fd4d25dd6 100644
--- a/plugins/codex-security/mcp-app/tests/test_deep_scan_coordinator.ts
+++ b/plugins/codex-security/mcp-app/tests/test_deep_scan_coordinator.ts
@@ -2269,11 +2269,13 @@ async function testPersistedErrorLimitStopsBeforeRescheduling() {
}
}
-async function testRestartRestoresDispatchOrderAfterMixedResults() {
+async function testRestartRestoresDispatchOrderAfterMixedResults(
+ limit: number,
+) {
const { fixture, store } = await coordinatorFixture({
workers: 2,
maxDiscoveryRuns: 2,
- stopAfterConsecutiveErrors: 1,
+ stopAfterConsecutiveErrors: limit,
});
const workers = path.join(
fixture.run.scanDir,
@@ -2313,12 +2315,84 @@ async function testRestartRestoresDispatchOrderAfterMixedResults() {
const terminal = await runCoordinator(fixture, store, executor, {
run: store.run,
});
- assert.equal(terminal?.status, "succeeded", terminal?.error);
- assert.equal(terminal.consecutiveErrors, 0);
+ assert.equal(
+ terminal?.status,
+ limit === 1 ? "failed" : "succeeded",
+ terminal?.error,
+ );
+ assert.equal(store.iterations[0].consecutiveErrors, limit === 1 ? 1 : 0);
assert.deepEqual(store.iterations[0].workerIds, [second.id]);
assert.equal(executor.discoveryCalls, 0);
}
+async function testComparisonStopCondition(mode: "quota" | "deadline") {
+ const { fixture, store } = await coordinatorFixture({
+ workers: 2,
+ stopAfterNoNew: 3,
+ maxDiscoveryRuns: 6,
+ });
+ const worker = new FakeExecutor({ discoveryCandidateId: "candidate-1" });
+ const deadline = Promise.withResolvers<void>();
+ let comparisons = 0;
+ const terminal = await runCoordinator(
+ fixture,
+ store,
+ {
+ async run(request) {
+ const result = await worker.run(request);
+ const resultPath = path.join(
+ request.artifactContext!.root,
+ "result.json",
+ );
+ const draft = await readJson(resultPath);
+ draft.findings[0].title = (
+ await promptContext(request.promptPath)
+ ).workerLabel;
+ await writeJson(resultPath, draft);
+ return result;
+ },
+ async compareFindings(_prompt, _cwd, signal) {
+ comparisons++;
+ if (mode === "quota")
+ throw new DeepScanNonRetryableError(
+ "Quota exceeded. Check your plan and billing details.",
+ );
+ await deadline.promise;
+ signal.throwIfAborted();
+ return JSON.stringify({
+ uncertain: [],
+ defects: [
+ {
+ path: "fixture.py",
+ operation: "query",
+ failure: "check",
+ reports: [0, 1].map((id) => ({ id, evidence: 0 })),
+ },
+ ],
+ });
+ },
+ },
+ {
+ ...(mode === "deadline" && { discoveryTimeoutMs: 500 }),
+ log: (event) => {
+ if (event.event === "discovery_deadline_reached") deadline.resolve();
+ },
+ },
+ );
+ assert.equal(comparisons, 1);
+ assert.equal(worker.discoveryCalls, 2);
+ assert.equal(store.iterations.length, 1);
+ assert.equal(store.iterations[0].workerIds.length, 2);
+ assert.equal(
+ terminal?.status,
+ mode === "quota" ? "failed" : "succeeded",
+ terminal?.error,
+ );
+ const checkpoint = await readJson(store.iterations[0].resultPath);
+ assert.equal(checkpoint.findings.length, mode === "quota" ? 2 : 1);
+ if (mode === "quota") assert.match(terminal.error ?? "", /Quota exceeded/);
+}
+
async function testDiscoveryDeadlineReducesSingleBufferedFinding() {
const { fixture, store } = await coordinatorFixture({
stopAfterNoNew: 99,
@@ -2676,6 +2750,7 @@ test(testWorkerScopedCandidateSourceAggregation);
test(testRetryKeepsLogicalWorker);
test(testSandboxDiagnosticSurvivesArtifactRetries);
test(testSuccessfulDeepCoverageRetainsIndependentWorkerReviewStatus);
+registerCases(testComparisonStopCondition, [["quota"], ["deadline"]]);
test(testDiscoveryDeadlineReducesSingleBufferedFinding);
test(testDiscoveryAcceptedAtDeadlineIsReduced);
test(testDiscoveryDeadlineWithoutAcceptedWorkersReturnsPartialEvidence);
@@ -2737,4 +2812,4 @@ registerCases(testResumedManifestPreservesCompletedReducer, [
[false, true, ["api", "background jobs"]],
]);
test(testPersistedErrorLimitStopsBeforeRescheduling);
-test(testRestartRestoresDispatchOrderAfterMixedResults);
+registerCases(testRestartRestoresDispatchOrderAfterMixedResults, [[1], [2]]);