Skip to content

OpenAI's Codex Security CLI and TypeScript SDK for finding, validating, and fixing security vulnerabilities. npm: https://www.npmjs.com/package/@openai/codex-security

fix(deep-scan): reserve worker directories across coordinator handoffs (#2322)

Michael D'Angelo2ef08b544502
diff --git a/plugins/codex-security/mcp-app/src/deep-scan/coordinator.ts b/plugins/codex-security/mcp-app/src/deep-scan/coordinator.ts
index 86bb54017..29001f96d 100644
--- a/plugins/codex-security/mcp-app/src/deep-scan/coordinator.ts
+++ b/plugins/codex-security/mcp-app/src/deep-scan/coordinator.ts
@@ -739,10 +739,13 @@ export class DeepScanCoordinator {
const scanLabel = await nextWorkerLabel(
this.artifacts.workersRoot,
"discovery",
+ persisted,
+ this.state.dispatchedCount,
);
const checkpointLabel = await nextWorkerLabel(
this.artifacts.dedupRoot,
"dedup",
+ persisted,
);
let pass = persisted.filter(
(w) => w.kind === "dedup" && w.status === "succeeded",
@@ -800,7 +803,7 @@ export class DeepScanCoordinator {
streak = count ? 0 : streak + 1;
}
resultPath = join(
- await reserveCheckpointDirectory(
+ await reserveWorkerDirectory(
this.artifacts.dedupRoot,
checkpointLabel,
),
@@ -963,11 +966,21 @@ export class DeepScanCoordinator {
nextLabel: () => string,
previousFindings: string[],
): Promise<DiscoveryOutcome[]> {
- const scans = Array.from({ length: count }, async () => {
+ // Reserve in dispatch order. Another coordinator may create one of these
+ // labels after our directory listing but before we write its prompt.
+ const labels: string[] = [];
+ for (let i = 0; i < count; i++) {
+ labels.push(
+ basename(
+ await reserveWorkerDirectory(this.artifacts.workersRoot, nextLabel),
+ ),
+ );
+ }
+ const scans = labels.map(async (label) => {
try {
const outcome = await this.discoveryWorkers.runDiscoveryWorker(
randomUUID(),
- nextLabel(),
+ label,
previousFindings,
);
if (outcome.status === "failed" && !outcome.replaceableFailureKind)
@@ -1045,14 +1058,14 @@ const systemClock: DeepScanClock = {
},
};
-async function reserveCheckpointDirectory(
+async function reserveWorkerDirectory(
root: string,
nextLabel: () => string,
): Promise<string> {
for (;;) {
const directory = join(root, nextLabel());
try {
- // An old coordinator can finish a batch after its replacement checkpointed.
+ // A coordinator being replaced can still be reserving its last directory.
await fs.mkdir(directory);
return directory;
} catch (error) {
@@ -1064,11 +1077,19 @@ async function reserveCheckpointDirectory(
async function nextWorkerLabel(
root: string,
kind: "dedup" | "discovery",
+ persisted: PersistedDeepScanWorker[],
+ minimum = 0,
): Promise<() => string> {
- let last = (await fs.readdir(root)).reduce((n, name) => {
+ const labels = [
+ ...(await fs.readdir(root)),
+ ...persisted
+ .filter((worker) => worker.kind === kind)
+ .map((worker) => basename(dirname(worker.artifactDir))),
+ ];
+ let last = labels.reduce((n, name) => {
const sequence = workerLabelSequence(name, kind);
return sequence > n ? sequence : n;
- }, 0n);
+ }, BigInt(minimum));
return () => `${kind}-${String(++last).padStart(4, "0")}`;
}
diff --git a/plugins/codex-security/mcp-app/tests/test_deep_scan_coordinator.ts b/plugins/codex-security/mcp-app/tests/test_deep_scan_coordinator.ts
index 230e7ad03..37b3e8c48 100644
--- a/plugins/codex-security/mcp-app/tests/test_deep_scan_coordinator.ts
+++ b/plugins/codex-security/mcp-app/tests/test_deep_scan_coordinator.ts
@@ -30,6 +30,7 @@ import {
} from "./deep_scan_lifecycle_cases.ts";
import {
DeepScanCoordinatorRegistry,
+ DeepScanNonRetryableError,
WorkbenchDeepScanStore,
DeepScanRemoteCoordinator,
DeepScanNonRetryableError,
@@ -100,6 +101,48 @@ async function testStandardWorkersReceiveExistingFalsePositiveFeedback() {
);
}
+async function testDiscoveryAllocatesUniqueLabelsAfterRecovery() {
+ const { fixture, store } = await coordinatorFixture({
+ maxDiscoveryRuns: 12,
+ stopAfterNoNew: 1,
+ });
+ const root = path.join(
+ fixture.run.scanDir,
+ "artifacts",
+ "deep_discovery",
+ "workers",
+ );
+ Object.assign(store.run, {
+ dispatchedCount: 1,
+ persistedWorkers: [
+ {
+ id: randomUUID(),
+ kind: "discovery",
+ status: "canceled",
+ attempt: 1,
+ mergeState: "none",
+ artifactDir: path.join(root, "discovery-0010", "output"),
+ promptPath: path.join(root, "discovery-0010", "prompt.md"),
+ },
+ ],
+ });
+ // A stale coordinator writes its final prompt after the new coordinator's listing.
+ const occupied = path.join(root, "discovery-0011", "prompt.md");
+ const updateProgress = store.updateProgress.bind(store);
+ store.updateProgress = async (input) => {
+ await mkdir(path.dirname(occupied), { recursive: true });
+ await writeFile(occupied, "old coordinator");
+ await updateProgress(input);
+ };
+ const executor = new FakeExecutor();
+ const terminal = await runCoordinator(fixture, store, executor, {
+ run: store.run,
+ });
+ assert.equal(terminal?.status, "succeeded", terminal?.error);
+ assert.deepEqual([...executor.discoveryAttempts.keys()], ["discovery-0012"]);
+ assert.equal(await readFile(occupied, "utf8"), "old coordinator");
+}
+
async function testConcurrentCheckpointKeepsExistingIteration() {
const { fixture, store } = await coordinatorFixture({ maxDiscoveryRuns: 1 });
const gate = Promise.withResolvers<void>();
@@ -2792,6 +2835,7 @@ registerCases(testTerminalFailureStopsDispatchBeforeWorkerShutdown, [
]);
test(testStandardWorkersReceiveExistingFalsePositiveFeedback);
test(testConcurrentCheckpointKeepsExistingIteration);
+test(testDiscoveryAllocatesUniqueLabelsAfterRecovery);
test(testDiscoveryWorkersKeepOneContextAfterPersistedUpdate);
test(testPersistedContextDoesNotChangeAnotherProcessDiscoverySnapshot);
test(testWorkerScopedCandidateSourceAggregation);