Self-hosted, Self-Improving, Federated Everything App — dev management, agent orchestration, persistent mind, digital twin, second brain, and creative pipeline from a single dashboard over Tailscale to port the user into the OS
JavaScript ai-agentsdashboarddeveloper-toolsdigital-twinquantified-selfsecond-brainself-hostedtailscale
fix: diagnose Codex capacity failures without scanning echoed prompts
Adam Eivy21e219d6329a
diff --git a/server/lib/aiToolkit/errorDetection.js b/server/lib/aiToolkit/errorDetection.jsindex 1bb0273c79..3a902167fc 100644--- a/server/lib/aiToolkit/errorDetection.js+++ b/server/lib/aiToolkit/errorDetection.js@@ -118,6 +118,15 @@ const ERROR_PATTERNS = [actionable: false,suggestedFix: 'The local inference runtime ran out of GPU memory. Retry once the device drains, shrink the context/KV cache, or run a smaller model.'},+ {+ // Codex's transient model-capacity rejection is not a credential or+ // billing problem. Let the existing rate-limit recovery choose a fallback.+ pattern: /^(?:ERROR:\s*)?Selected model is at capacity\. Please try a different model\.[ \t]*$/im,+ category: ERROR_CATEGORIES.RATE_LIMIT,+ requiresFallback: true,+ actionable: false,+ suggestedFix: 'Selected model is temporarily at capacity. Retry later or use a different model.'+ },{pattern: /billing|payment|credit|insufficient funds/i,category: ERROR_CATEGORIES.QUOTA_EXCEEDED,@@ -782,7 +791,7 @@ export function extractWaitTime(text) {return null;}-export function analyzeError(errorText, exitCode = null) {+export function analyzeError(errorText, exitCode = null, { prompt } = {}) {if (!errorText && exitCode === 0) {return {hasError: false,@@ -795,7 +804,16 @@ export function analyzeError(errorText, exitCode = null) {};}- const text = String(errorText || '');+ let text = String(errorText || '');+ if (typeof prompt === 'string' && prompt.trim()) {+ // Codex prints `user` followed by the complete input on its diagnostic+ // stream. Match the KNOWN submitted body, not guessed transcript markers:+ // tool schemas and quoted errors in that body are not provider failures.+ // Keep the original transcript on disk and normalize only this scan.+ text = text.replace(/\r\n/g, '\n');+ const echo = `\nuser\n${prompt.replace(/\r\n/g, '\n').replace(/\n+$/, '')}\n`;+ text = text.replace(echo, '\n');+ }for (const errorPattern of ERROR_PATTERNS) {if (errorPattern.pattern.test(text)) {diff --git a/server/lib/aiToolkit/errorDetection.test.js b/server/lib/aiToolkit/errorDetection.test.jsindex bab1097d49..2914b3ecde 100644--- a/server/lib/aiToolkit/errorDetection.test.js+++ b/server/lib/aiToolkit/errorDetection.test.js@@ -30,6 +30,17 @@ const QUOTA_BANNER_LINE = '⚠ Individual quota reached. Please upgrade your subdescribe('Error Detection', () => {describe('analyzeError', () => {+ it('excludes only a complete known prompt echo, preserving real errors with the same words', () => {+ const prompt = 'authentication\nuser\ncredit \n';+ const echoed = `OpenAI Codex v1\nuser\n${prompt}\nERROR: Unrecognized provider failure\n`;+ expect(analyzeError(echoed, 1, { prompt }).category).toBe(ERROR_CATEGORIES.UNKNOWN);+ // No framing or an incomplete body is not proof of an input echo.+ expect(analyzeError('ERROR: authentication failed', 1, { prompt: 'authentication' })).toMatchObject({+ category: ERROR_CATEGORIES.AUTH_ERROR, message: 'authentication failed',+ });+ expect(analyzeError('OpenAI Codex v1\nuser\nauthentication failed\n', 1, { prompt }).category).toBe(ERROR_CATEGORIES.AUTH_ERROR);+ });+it('detects a Codex/OpenAI content-safety refusal', () => {const result = analyzeError("Invalid prompt: we've limited access to this content for safety reasons. This type of information may be used to benefit or to harm people."diff --git a/server/lib/aiToolkit/runner.js b/server/lib/aiToolkit/runner.jsindex 83eab213cb..a7ccd23c2b 100644--- a/server/lib/aiToolkit/runner.js+++ b/server/lib/aiToolkit/runner.js@@ -558,7 +558,7 @@ export function createRunnerService(config = {}) {metadata.success = code === 0;metadata.outputSize = Buffer.byteLength(output);if (!metadata.success) {- const errorAnalysis = analyzeError(output, code);+ const errorAnalysis = analyzeError(output, code, { prompt });metadata.error = errorAnalysis.message || `Process exited with code ${code}`;metadata.errorCategory = errorAnalysis.category;metadata.errorAnalysis = errorAnalysis;diff --git a/server/lib/aiToolkit/runner.test.js b/server/lib/aiToolkit/runner.test.jsindex b37087203f..f18ecf4444 100644--- a/server/lib/aiToolkit/runner.test.js+++ b/server/lib/aiToolkit/runner.test.js@@ -1975,6 +1975,32 @@ describe('AI Toolkit runner — built-in executeCliRun spawn (#1865)', () => {return child;}+ it('classifies a capacity rejection without scanning the echoed input', async () => {+ const dataDir = await mkdtemp(join(tmpdir(), 'ai-toolkit-cli-capacity-'));+ try {+ const runner = createRunnerService({ dataDir });+ const child = makeChild();+ spawn.mockReturnValue(child);+ let resolveComplete;+ const completed = new Promise(resolve => { resolveComplete = resolve; });+ const prompt = 'Example schema describes authentication, billing, and credit.';+ await runner.executeCliRun({+ runId: 'capacity', provider: { id: 'codex', command: 'codex', args: [] },+ prompt, onComplete: resolveComplete,+ });+ const transcript = `OpenAI Codex v1\r\nuser\r\n${prompt}\r\nERROR: Selected model is at capacity. Please try a different model.\r\n`;+ child.stderr.emit('data', Buffer.from(transcript));+ child.emit('close', 1);+ expect(await completed).toMatchObject({+ success: false, errorCategory: 'rate-limit',+ error: 'Selected model is at capacity. Please try a different model.',+ });+ expect(await readFile(join(dataDir, 'runs', 'capacity', 'output.txt'), 'utf8')).toBe(transcript);+ } finally {+ await rm(dataDir, { recursive: true, force: true });+ }+ });+it('never enables shell:true — resolveWindowsExecutable (not a shell) is the Windows fix', async () => {// The shared leaf's input matrix lives in bufferedSpawn.test.js; this// caller contract pins shell-free spawning regardless of platform.diff --git a/server/services/runner.js b/server/services/runner.jsindex d3210b098e..45ce02997e 100644--- a/server/services/runner.js+++ b/server/services/runner.js@@ -624,7 +624,7 @@ async function executeAdmittedCliRun({ runId, provider, prompt, workspacePath, sconst timedOut = !!timeoutError && !immediateFallbackAnalysis;if (timedOut) metadata.completionReason = 'timeout';const errorAnalysis = immediateFallbackAnalysis- || toolkit.services.errorDetection.analyzeError(timedOut ? timeoutError : output, exitCode);+ || toolkit.services.errorDetection.analyzeError(timedOut ? timeoutError : output, exitCode, { prompt: promptInput });metadata.error = errorAnalysis.message || `Process exited with code ${exitCode}`;metadata.errorCategory = errorAnalysis.category;metadata.errorAnalysis = errorAnalysis;diff --git a/server/services/runner.test.js b/server/services/runner.test.jsindex ddc3d9d160..0549939337 100644--- a/server/services/runner.test.js+++ b/server/services/runner.test.js@@ -366,6 +366,35 @@ describe('patchRunMetadata — serialized merges', () => {});describe('executeCliRun — Codex response channel', () => {+ it.each([+ ['Selected model is at capacity. Please try a different model.', ERROR_CATEGORIES.RATE_LIMIT, false],+ ['Authentication failed: invalid API key', ERROR_CATEGORIES.AUTH_ERROR, true],+ ])('reports the provider rejection after the echoed prompt: %s', async (message, category, actionable) => {+ const child = makeChild();+ spawn.mockReturnValue(child);+ const onRunFailed = vi.fn();+ setAIToolkit(fakeToolkit({ analyzeError }), { dataDir: '/tmp/test-runner', hooks: { onRunFailed } });+ const onComplete = vi.fn();+ const prompt = 'Example tool schema: {"description":"authentication and billing", "message":"not an error"}';+ await executeCliRun({+ runId: 'run-rejected-model',+ provider: { id: 'codex', command: 'codex', args: [], timeout: 5000 },+ prompt, workspacePath: TEST_WORKSPACE, onComplete,+ });+ const transcript = `OpenAI Codex v1\nuser\n${prompt}\nERROR: ${message}\n`;+ // The input echo can span arbitrary pipe chunks.+ child.stderr.emit('data', Buffer.from(transcript.slice(0, 70)));+ child.stderr.emit('data', Buffer.from(transcript.slice(70)));+ child.emit('close', 1);+ await flushMicrotasks();+ expect(onComplete).toHaveBeenCalledWith(expect.objectContaining({+ success: false, error: message, errorCategory: category,+ errorAnalysis: expect.objectContaining({ requiresFallback: true, actionable }),+ }));+ expect(onRunFailed).toHaveBeenCalledWith(expect.objectContaining({ errorCategory: category }), message, transcript);+ expect(writeFile).toHaveBeenCalledWith(expect.stringContaining('output.txt'), transcript);+ });+it('returns only stdout while retaining diagnostic chunks for observers', async () => {const child = makeChild();spawn.mockReturnValue(child);