Skip to content

Self-hosted, Self-Improving, Federated Everything App — dev management, agent orchestration, persistent mind, digital twin, second brain, and creative pipeline from a single dashboard over Tailscale to port the user into the OS

fix: diagnose Codex capacity failures without scanning echoed prompts

Adam Eivy21e219d6329a
diff --git a/server/lib/aiToolkit/errorDetection.js b/server/lib/aiToolkit/errorDetection.js
index 1bb0273c79..3a902167fc 100644
--- a/server/lib/aiToolkit/errorDetection.js
+++ b/server/lib/aiToolkit/errorDetection.js
@@ -118,6 +118,15 @@ const ERROR_PATTERNS = [
actionable: false,
suggestedFix: 'The local inference runtime ran out of GPU memory. Retry once the device drains, shrink the context/KV cache, or run a smaller model.'
},
+ {
+ // Codex's transient model-capacity rejection is not a credential or
+ // billing problem. Let the existing rate-limit recovery choose a fallback.
+ pattern: /^(?:ERROR:\s*)?Selected model is at capacity\. Please try a different model\.[ \t]*$/im,
+ category: ERROR_CATEGORIES.RATE_LIMIT,
+ requiresFallback: true,
+ actionable: false,
+ suggestedFix: 'Selected model is temporarily at capacity. Retry later or use a different model.'
+ },
{
pattern: /billing|payment|credit|insufficient funds/i,
category: ERROR_CATEGORIES.QUOTA_EXCEEDED,
@@ -782,7 +791,7 @@ export function extractWaitTime(text) {
return null;
}
-export function analyzeError(errorText, exitCode = null) {
+export function analyzeError(errorText, exitCode = null, { prompt } = {}) {
if (!errorText && exitCode === 0) {
return {
hasError: false,
@@ -795,7 +804,16 @@ export function analyzeError(errorText, exitCode = null) {
};
}
- const text = String(errorText || '');
+ let text = String(errorText || '');
+ if (typeof prompt === 'string' && prompt.trim()) {
+ // Codex prints `user` followed by the complete input on its diagnostic
+ // stream. Match the KNOWN submitted body, not guessed transcript markers:
+ // tool schemas and quoted errors in that body are not provider failures.
+ // Keep the original transcript on disk and normalize only this scan.
+ text = text.replace(/\r\n/g, '\n');
+ const echo = `\nuser\n${prompt.replace(/\r\n/g, '\n').replace(/\n+$/, '')}\n`;
+ text = text.replace(echo, '\n');
+ }
for (const errorPattern of ERROR_PATTERNS) {
if (errorPattern.pattern.test(text)) {
diff --git a/server/lib/aiToolkit/errorDetection.test.js b/server/lib/aiToolkit/errorDetection.test.js
index bab1097d49..2914b3ecde 100644
--- a/server/lib/aiToolkit/errorDetection.test.js
+++ b/server/lib/aiToolkit/errorDetection.test.js
@@ -30,6 +30,17 @@ const QUOTA_BANNER_LINE = '⚠ Individual quota reached. Please upgrade your sub
describe('Error Detection', () => {
describe('analyzeError', () => {
+ it('excludes only a complete known prompt echo, preserving real errors with the same words', () => {
+ const prompt = 'authentication\nuser\ncredit \n';
+ const echoed = `OpenAI Codex v1\nuser\n${prompt}\nERROR: Unrecognized provider failure\n`;
+ expect(analyzeError(echoed, 1, { prompt }).category).toBe(ERROR_CATEGORIES.UNKNOWN);
+ // No framing or an incomplete body is not proof of an input echo.
+ expect(analyzeError('ERROR: authentication failed', 1, { prompt: 'authentication' })).toMatchObject({
+ category: ERROR_CATEGORIES.AUTH_ERROR, message: 'authentication failed',
+ });
+ expect(analyzeError('OpenAI Codex v1\nuser\nauthentication failed\n', 1, { prompt }).category).toBe(ERROR_CATEGORIES.AUTH_ERROR);
+ });
+
it('detects a Codex/OpenAI content-safety refusal', () => {
const result = analyzeError(
"Invalid prompt: we've limited access to this content for safety reasons. This type of information may be used to benefit or to harm people."
diff --git a/server/lib/aiToolkit/runner.js b/server/lib/aiToolkit/runner.js
index 83eab213cb..a7ccd23c2b 100644
--- a/server/lib/aiToolkit/runner.js
+++ b/server/lib/aiToolkit/runner.js
@@ -558,7 +558,7 @@ export function createRunnerService(config = {}) {
metadata.success = code === 0;
metadata.outputSize = Buffer.byteLength(output);
if (!metadata.success) {
- const errorAnalysis = analyzeError(output, code);
+ const errorAnalysis = analyzeError(output, code, { prompt });
metadata.error = errorAnalysis.message || `Process exited with code ${code}`;
metadata.errorCategory = errorAnalysis.category;
metadata.errorAnalysis = errorAnalysis;
diff --git a/server/lib/aiToolkit/runner.test.js b/server/lib/aiToolkit/runner.test.js
index b37087203f..f18ecf4444 100644
--- a/server/lib/aiToolkit/runner.test.js
+++ b/server/lib/aiToolkit/runner.test.js
@@ -1975,6 +1975,32 @@ describe('AI Toolkit runner — built-in executeCliRun spawn (#1865)', () => {
return child;
}
+ it('classifies a capacity rejection without scanning the echoed input', async () => {
+ const dataDir = await mkdtemp(join(tmpdir(), 'ai-toolkit-cli-capacity-'));
+ try {
+ const runner = createRunnerService({ dataDir });
+ const child = makeChild();
+ spawn.mockReturnValue(child);
+ let resolveComplete;
+ const completed = new Promise(resolve => { resolveComplete = resolve; });
+ const prompt = 'Example schema describes authentication, billing, and credit.';
+ await runner.executeCliRun({
+ runId: 'capacity', provider: { id: 'codex', command: 'codex', args: [] },
+ prompt, onComplete: resolveComplete,
+ });
+ const transcript = `OpenAI Codex v1\r\nuser\r\n${prompt}\r\nERROR: Selected model is at capacity. Please try a different model.\r\n`;
+ child.stderr.emit('data', Buffer.from(transcript));
+ child.emit('close', 1);
+ expect(await completed).toMatchObject({
+ success: false, errorCategory: 'rate-limit',
+ error: 'Selected model is at capacity. Please try a different model.',
+ });
+ expect(await readFile(join(dataDir, 'runs', 'capacity', 'output.txt'), 'utf8')).toBe(transcript);
+ } finally {
+ await rm(dataDir, { recursive: true, force: true });
+ }
+ });
+
it('never enables shell:true — resolveWindowsExecutable (not a shell) is the Windows fix', async () => {
// The shared leaf's input matrix lives in bufferedSpawn.test.js; this
// caller contract pins shell-free spawning regardless of platform.
diff --git a/server/services/runner.js b/server/services/runner.js
index d3210b098e..45ce02997e 100644
--- a/server/services/runner.js
+++ b/server/services/runner.js
@@ -624,7 +624,7 @@ async function executeAdmittedCliRun({ runId, provider, prompt, workspacePath, s
const timedOut = !!timeoutError && !immediateFallbackAnalysis;
if (timedOut) metadata.completionReason = 'timeout';
const errorAnalysis = immediateFallbackAnalysis
- || toolkit.services.errorDetection.analyzeError(timedOut ? timeoutError : output, exitCode);
+ || toolkit.services.errorDetection.analyzeError(timedOut ? timeoutError : output, exitCode, { prompt: promptInput });
metadata.error = errorAnalysis.message || `Process exited with code ${exitCode}`;
metadata.errorCategory = errorAnalysis.category;
metadata.errorAnalysis = errorAnalysis;
diff --git a/server/services/runner.test.js b/server/services/runner.test.js
index ddc3d9d160..0549939337 100644
--- a/server/services/runner.test.js
+++ b/server/services/runner.test.js
@@ -366,6 +366,35 @@ describe('patchRunMetadata — serialized merges', () => {
});
describe('executeCliRun — Codex response channel', () => {
+ it.each([
+ ['Selected model is at capacity. Please try a different model.', ERROR_CATEGORIES.RATE_LIMIT, false],
+ ['Authentication failed: invalid API key', ERROR_CATEGORIES.AUTH_ERROR, true],
+ ])('reports the provider rejection after the echoed prompt: %s', async (message, category, actionable) => {
+ const child = makeChild();
+ spawn.mockReturnValue(child);
+ const onRunFailed = vi.fn();
+ setAIToolkit(fakeToolkit({ analyzeError }), { dataDir: '/tmp/test-runner', hooks: { onRunFailed } });
+ const onComplete = vi.fn();
+ const prompt = 'Example tool schema: {"description":"authentication and billing", "message":"not an error"}';
+ await executeCliRun({
+ runId: 'run-rejected-model',
+ provider: { id: 'codex', command: 'codex', args: [], timeout: 5000 },
+ prompt, workspacePath: TEST_WORKSPACE, onComplete,
+ });
+ const transcript = `OpenAI Codex v1\nuser\n${prompt}\nERROR: ${message}\n`;
+ // The input echo can span arbitrary pipe chunks.
+ child.stderr.emit('data', Buffer.from(transcript.slice(0, 70)));
+ child.stderr.emit('data', Buffer.from(transcript.slice(70)));
+ child.emit('close', 1);
+ await flushMicrotasks();
+ expect(onComplete).toHaveBeenCalledWith(expect.objectContaining({
+ success: false, error: message, errorCategory: category,
+ errorAnalysis: expect.objectContaining({ requiresFallback: true, actionable }),
+ }));
+ expect(onRunFailed).toHaveBeenCalledWith(expect.objectContaining({ errorCategory: category }), message, transcript);
+ expect(writeFile).toHaveBeenCalledWith(expect.stringContaining('output.txt'), transcript);
+ });
+
it('returns only stdout while retaining diagnostic chunks for observers', async () => {
const child = makeChild();
spawn.mockReturnValue(child);