Skip to content

OpenAI's Codex Security CLI and TypeScript SDK for finding, validating, and fixing security vulnerabilities. npm: https://www.npmjs.com/package/@openai/codex-security

fix: stop Deep Scan workers when usage is exhausted (#1638)

ianw-oaia8f142f5d434
diff --git a/plugins/codex-security/mcp-app/src/deep-scan/coordinator.ts b/plugins/codex-security/mcp-app/src/deep-scan/coordinator.ts
index b19fb6383..982f06526 100644
--- a/plugins/codex-security/mcp-app/src/deep-scan/coordinator.ts
+++ b/plugins/codex-security/mcp-app/src/deep-scan/coordinator.ts
@@ -32,6 +32,7 @@ import {
abortError,
errorMessage,
boundedDeepScanErrorMessage,
+ DeepScanNonRetryableError,
} from "./errors.js";
import type {
CodexWorkerExecutor,
@@ -139,6 +140,7 @@ export class DeepScanCoordinator {
random: options.random ?? Math.random,
log: this.log,
retryDelaysMs: options.retryDelaysMs ?? RETRY_DELAYS_MS,
+ onNonRetryableError: (error: Error) => this.abortController.abort(error),
} satisfies Omit<
ConstructorParameters<typeof DeepScanWorkerRunner>[0],
"signal"
@@ -423,6 +425,11 @@ export class DeepScanCoordinator {
if (await this.stopAfterOwnershipChange(this.options.threadId, error)) {
return;
}
+ if (
+ this.abortController.signal.reason instanceof DeepScanNonRetryableError
+ ) {
+ error = this.abortController.signal.reason;
+ }
const message = errorMessage(error);
const persistedMessage = boundedDeepScanErrorMessage(error);
if (!this.setupComplete) {
diff --git a/plugins/codex-security/mcp-app/src/deep-scan/errors.ts b/plugins/codex-security/mcp-app/src/deep-scan/errors.ts
index 4385dbe72..3af49bfc8 100644
--- a/plugins/codex-security/mcp-app/src/deep-scan/errors.ts
+++ b/plugins/codex-security/mcp-app/src/deep-scan/errors.ts
@@ -26,6 +26,26 @@ export class DeepScanNonRetryableError extends Error {
}
}
+const USAGE_LIMIT_MESSAGES = new Set([
+ "Quota exceeded. Check your plan and billing details.",
+ "Your workspace is out of credits. Add credits to continue.",
+ "Your workspace is out of credits. Ask your workspace owner to refill in order to continue.",
+ "You hit your spend cap set in your workspace. Increase your spend cap to continue.",
+ "You hit your spend cap set by the owner of your workspace. Ask an owner to increase your spend cap to continue.",
+]);
+
+// Codex exec exposes only text for failed turns. Match its native quota messages here.
+export function codexWorkerTurnError(message: string): Error {
+ if (
+ USAGE_LIMIT_MESSAGES.has(message) ||
+ message.startsWith("You’ve hit your usage limit.") ||
+ message.startsWith("You’ve hit your usage limit for ")
+ ) {
+ return new DeepScanNonRetryableError(message);
+ }
+ return new Error(message);
+}
+
/** A store read confirmed that this coordinator no longer owns the scan. */
export class DeepScanOwnershipChangedError extends Error {
constructor(
diff --git a/plugins/codex-security/mcp-app/src/deep-scan/executor.ts b/plugins/codex-security/mcp-app/src/deep-scan/executor.ts
index 10b3eb1e8..0a43cd25b 100644
--- a/plugins/codex-security/mcp-app/src/deep-scan/executor.ts
+++ b/plugins/codex-security/mcp-app/src/deep-scan/executor.ts
@@ -35,6 +35,7 @@ import { executablePathForSpawn } from "./executable-path.js";
import { writePrivateFile } from "./artifacts.js";
import {
classifyCodexWorkerError,
+ codexWorkerTurnError,
DeepScanNonRetryableError,
} from "./errors.js";
import {
@@ -239,7 +240,10 @@ export class CodexSdkWorkerExecutor implements CodexWorkerExecutor {
} else if (event.type === "turn.completed") {
request.signal.removeEventListener("abort", forwardAbort);
} else if (event.type === "turn.failed") {
- throw new Error(event.error.message);
+ const error = codexWorkerTurnError(event.error.message);
+ if (error instanceof DeepScanNonRetryableError)
+ request.onNonRetryableError?.(error);
+ throw error;
} else if (event.type === "error") {
// Codex exec emits retry-in-progress notifications as error events.
appendStreamDiagnostic(diagnostics, event.message);
diff --git a/plugins/codex-security/mcp-app/src/deep-scan/types.ts b/plugins/codex-security/mcp-app/src/deep-scan/types.ts
index 5b68ef075..03279af6f 100644
--- a/plugins/codex-security/mcp-app/src/deep-scan/types.ts
+++ b/plugins/codex-security/mcp-app/src/deep-scan/types.ts
@@ -124,6 +124,7 @@ export interface CodexWorkerRequest {
continuationPrompt?: string;
artifactContext?: CodexWorkerArtifactContext;
onThreadStarted?: (threadId: string) => Promise<void> | void;
+ onNonRetryableError?: (error: Error) => void;
}
export interface CodexWorkerResult {
diff --git a/plugins/codex-security/mcp-app/src/deep-scan/worker-runner.ts b/plugins/codex-security/mcp-app/src/deep-scan/worker-runner.ts
index cfa691ba7..0dfe5ac91 100644
--- a/plugins/codex-security/mcp-app/src/deep-scan/worker-runner.ts
+++ b/plugins/codex-security/mcp-app/src/deep-scan/worker-runner.ts
@@ -87,6 +87,7 @@ export interface DeepScanWorkerRunnerOptions {
log: DeepScanLogger;
retryDelaysMs: readonly number[];
signal: AbortSignal;
+ onNonRetryableError?: (error: Error) => void;
}
type WorkerAttemptOutcome =
@@ -410,6 +411,7 @@ export class DeepScanWorkerRunner {
: join(run.scanDir, "artifacts"),
subagents: input.subagents,
signal,
+ onNonRetryableError: this.options.onNonRetryableError,
resumeThreadId: resumableThreadId,
continuationPrompt: resumableThreadId
? (continuationPrompt ??
diff --git a/plugins/codex-security/mcp-app/tests/test_deep_scan_coordinator.ts b/plugins/codex-security/mcp-app/tests/test_deep_scan_coordinator.ts
index 98ab8d708..3cfc97c4c 100644
--- a/plugins/codex-security/mcp-app/tests/test_deep_scan_coordinator.ts
+++ b/plugins/codex-security/mcp-app/tests/test_deep_scan_coordinator.ts
@@ -4185,6 +4185,63 @@ async function testNonRetryableReducerAbortsScanWithoutRetry(
assert.equal(store.dedupCommits.length, 0);
}
+async function testQuotaFailureStopsDispatchBeforeWorkerShutdown(
+ kind: "discovery" | "dedup",
+) {
+ const { fixture, store } = await coordinatorFixture({
+ workers: 2,
+ stopAfterNoNew: 10,
+ stopAfterConsecutiveErrors: 4,
+ maxDiscoveryRuns: 10,
+ });
+ const normal = new FakeExecutor({
+ blockDiscoveryAfterCalls: kind === "discovery" ? 0 : 2,
+ });
+ const notified = Promise.withResolvers<void>();
+ const shutdown = Promise.withResolvers<void>();
+ const signals: AbortSignal[] = [];
+ const sleeps: number[] = [];
+ let discoveryCalls = 0;
+ const terminal = runCoordinator(
+ fixture,
+ store,
+ {
+ async run(request) {
+ signals.push(request.signal);
+ if (request.kind === "discovery") discoveryCalls++;
+ if (
+ request.kind === kind &&
+ (kind === "dedup" || discoveryCalls === 2)
+ ) {
+ const error = new DeepScanNonRetryableError(
+ "Synthetic usage limit reached",
+ );
+ request.onNonRetryableError?.(error);
+ notified.resolve();
+ await shutdown.promise;
+ throw error;
+ }
+ return normal.run(request);
+ },
+ },
+ { clock: recordingClock(sleeps) },
+ );
+ try {
+ await notified.promise;
+ assert.ok(signals.every((signal) => signal.aborted));
+ const callsAtFailure = signals.length;
+ shutdown.resolve();
+ const result = await terminal;
+ assert.equal(result?.status, "failed");
+ assert.equal(result?.error, "Synthetic usage limit reached");
+ assert.equal(signals.length, callsAtFailure);
+ assert.deepEqual(sleeps, []);
+ } finally {
+ shutdown.resolve();
+ await terminal;
+ }
+}
+
try {
await testDeepScanLifecycle({
fixtureRun,
@@ -4194,6 +4251,8 @@ try {
DeepScanCoordinatorRegistry,
immediateClock,
});
+ await testQuotaFailureStopsDispatchBeforeWorkerShutdown("discovery");
+ await testQuotaFailureStopsDispatchBeforeWorkerShutdown("dedup");
await testCappedQueueAndSerialDedup();
await testStandardWorkersReceiveExistingFalsePositiveFeedback();
await testDiscoveryWorkersKeepOneContextAfterPersistedUpdate();
diff --git a/plugins/codex-security/mcp-app/tests/test_deep_scan_executor.ts b/plugins/codex-security/mcp-app/tests/test_deep_scan_executor.ts
index f08cb1d19..dbfba3ebc 100644
--- a/plugins/codex-security/mcp-app/tests/test_deep_scan_executor.ts
+++ b/plugins/codex-security/mcp-app/tests/test_deep_scan_executor.ts
@@ -55,6 +55,7 @@ const {
snapshotWorkerEnvironment,
appendItemDiagnostic,
classifyCodexWorkerError,
+ codexWorkerTurnError,
DeepScanNonRetryableError,
isCodexCybersecurityPolicyRefusal,
} = await importModule({
@@ -116,7 +117,8 @@ try {
await testLargeScopeTransport();
await testOpenAiCredentialsReachWorker();
await testWorkerCancellation();
- await testNullUsageCompletion();
+ testUsageLimitClassification();
+ await testWorkerTerminalEvents();
await testWorkerRuntimeSettings();
await testUnsupportedProviderSnapshotFailsBeforeLaunch();
await testWorkerCyberAccessSettings();
@@ -3853,7 +3855,31 @@ async function testWorkerCancellation() {
}
}
-async function testNullUsageCompletion() {
+function testUsageLimitClassification() {
+ for (const message of [
+ "Quota exceeded. Check your plan and billing details.",
+ "Your workspace is out of credits. Add credits to continue.",
+ "Your workspace is out of credits. Ask your workspace owner to refill in order to continue.",
+ "You hit your spend cap set in your workspace. Increase your spend cap to continue.",
+ "You hit your spend cap set by the owner of your workspace. Ask an owner to increase your spend cap to continue.",
+ "You’ve hit your usage limit. Try again later.",
+ "You’ve hit your usage limit for the selected model. Switch to another model to continue.",
+ ]) {
+ const error = codexWorkerTurnError(message);
+ assert.ok(error instanceof DeepScanNonRetryableError);
+ assert.equal(error.message, message);
+ }
+ for (const message of [
+ "429 Too Many Requests",
+ "Command failed: Quota exceeded. Check your plan and billing details.",
+ ]) {
+ const error = codexWorkerTurnError(message);
+ assert.equal(error.name, "Error");
+ assert.equal(error.message, message);
+ }
+}
+
+async function testWorkerTerminalEvents() {
const fixture = await fakeCodexFixture();
const previousPath = process.env.CODEX_CLI_PATH;
const originalSpawn = childProcess.spawn;
@@ -3874,20 +3900,55 @@ async function testNullUsageCompletion() {
process.env.CODEX_CLI_PATH = process.execPath;
try {
const promptPath = path.join(fixture.root, "prompt.md");
- await writeFile(promptPath, "NULL_USAGE\n");
- for (const kind of ["discovery", "dedup"] as const) {
- for (const resumeThreadId of [undefined, "fixture-resumed-thread"]) {
- const result = await new CodexSdkWorkerExecutor({
- parentSandbox: trustedParentSandbox,
- }).run({
- kind,
- promptPath,
- workingDirectory: fixture.root,
- subagents: 0,
- resumeThreadId,
- signal: new AbortController().signal,
- });
- assert.equal(result.threadId, resumeThreadId ?? "fixture-thread-id");
+ const quotaMessage = "Quota exceeded. Check your plan and billing details.";
+ for (const event of [
+ undefined,
+ { type: "turn.failed", error: { message: quotaMessage } },
+ { type: "turn.failed", error: { message: "429 Too Many Requests" } },
+ { type: "error", message: quotaMessage },
+ {
+ type: "item.completed",
+ item: { id: "quota-text", type: "agent_message", text: quotaMessage },
+ },
+ ] as const) {
+ await writeFile(
+ promptPath,
+ event
+ ? `IPC_DIAGNOSTIC_EVENT\n${JSON.stringify(event)}\n`
+ : "NULL_USAGE\n",
+ );
+ for (const kind of ["discovery", "dedup"] as const) {
+ for (const resumeThreadId of [undefined, "fixture-resumed-thread"]) {
+ const onNonRetryableError = mock.fn();
+ const execution = new CodexSdkWorkerExecutor({
+ parentSandbox: trustedParentSandbox,
+ }).run({
+ kind,
+ promptPath,
+ workingDirectory: fixture.root,
+ subagents: 0,
+ resumeThreadId,
+ signal: new AbortController().signal,
+ onNonRetryableError,
+ });
+ if (event?.type === "turn.failed") {
+ const isQuota = event.error.message === quotaMessage;
+ await assert.rejects(execution, {
+ name: isQuota ? "DeepScanNonRetryableError" : "Error",
+ message: event.error.message,
+ });
+ assert.deepEqual(
+ onNonRetryableError.mock.calls.map(
+ ({ arguments: [error] }) => error.message,
+ ),
+ isQuota ? [quotaMessage] : [],
+ );
+ continue;
+ }
+ const result = await execution;
+ assert.equal(onNonRetryableError.mock.callCount(), 0);
+ assert.equal(result.threadId, resumeThreadId ?? "fixture-thread-id");
+ }
}
}
} finally {
diff --git a/sdk/typescript/docs/cli.md b/sdk/typescript/docs/cli.md
index 87b0dde58..4a1f76392 100644
--- a/sdk/typescript/docs/cli.md
+++ b/sdk/typescript/docs/cli.md
@@ -723,6 +723,9 @@ work is needed, the scan is preserved and you can run `scans match --all`.
When Deep discovery has finished at a cost stop, the scan returns a sealed
partial report and lists unvalidated candidates as follow-up work.
+When Codex reports an exhausted usage limit, Deep Scan cancels active workers
+and stops retrying or launching replacements. Completed worker artifacts are retained.
+
At 80% of a single interactive scan's limit, the dashboard offers a higher total
budget. Enter a larger total, or leave it blank/press Escape to keep the limit.
Work continues while you decide; the old limit remains active until saved.