Self-hosted, Self-Improving, Federated Everything App — dev management, agent orchestration, persistent mind, digital twin, second brain, and creative pipeline from a single dashboard over Tailscale to port the user into the OS
JavaScript ai-agentsdashboarddeveloper-toolsdigital-twinquantified-selfsecond-brainself-hostedtailscale
fix: nudge a Codex session parked by a model-at-capacity rejection
[._.]/ Adam Eivyabcdf74f4c40
diff --git a/server/lib/agentOutputMarkers.js b/server/lib/agentOutputMarkers.jsindex 92fefbdb93..c3a55da16c 100644--- a/server/lib/agentOutputMarkers.js+++ b/server/lib/agentOutputMarkers.js@@ -110,6 +110,8 @@ export const LIFECYCLE_LINES = Object.freeze({permissionDeclineNudged: lifecycleLine`🔁 Nudged the session to continue after declined permission prompt ${'declined'}`,localOomPending: lifecycleLine`⏳ Local runtime out of GPU memory — will nudge the session to continue if it goes quiet`,localOomNudged: lifecycleLine`🔁 Local runtime OOM — nudged the session to continue (attempt ${'attempt'}/${'max'})`,+ capacityPending: lifecycleLine`⏳ Selected model is at capacity — will nudge the session to continue after it has been quiet a while`,+ capacityNudged: lifecycleLine`🔁 Selected model was at capacity — nudged the session to continue (attempt ${'attempt'}/${'max'})`,truncationPending: lifecycleLine`⏳ Response was truncated before completion — will nudge the session to continue if it goes quiet`,truncationNudged: lifecycleLine`🔁 Response was truncated before completion — nudged the session to continue (attempt ${'attempt'}/${'max'})`,stallNudged: lifecycleLine`🔁 Session idle with the task unfinished — nudged it to continue (attempt ${'attempt'}/${'max'}, ${'sent'}/${'total'} this run)`,diff --git a/server/lib/aiToolkit/errorDetection.js b/server/lib/aiToolkit/errorDetection.jsindex 06b11030cc..8e25203c5d 100644--- a/server/lib/aiToolkit/errorDetection.js+++ b/server/lib/aiToolkit/errorDetection.js@@ -737,6 +737,48 @@ export function createTruncatedResponseDetector({ maxBuffer = 512 } = {}) {};}+// A provider that rejected the turn because the selected model is overloaded+// (Codex: `Selected model is at capacity. Please try a different model.`). The+// turn is dead but the session still holds the conversation, and a human typing+// `continue` a little later resumes it. Unattended, nothing types it: the run+// sits at the composer holding its lane (agent-9322c230). Same remedy as the+// truncation banner, but the cause is load on the provider, so the consumer's+// gate waits much longer between nudges (createModelCapacityNudgeGate).+//+// `\s+` joins survive a TUI hard-wrap. Not an IMMEDIATE_FALLBACK_SIGNALS entry+// for the same reason as the truncation banner: the remedy is a continuation,+// not a restart or an instant provider swap.+const MODEL_AT_CAPACITY_PATTERN = /selected\s+model\s+is\s+at\s+capacity/i;++export function detectModelAtCapacity(text) {+ if (!text) return null;+ if (!MODEL_AT_CAPACITY_PATTERN.test(String(text))) return null;+ return {+ hasError: true,+ category: ERROR_CATEGORIES.RATE_LIMIT,+ // FIXED sentence, not the banner wording: it becomes the run error and a+ // task description can quote it back through this detector.+ message: 'Provider reported the selected model is overloaded',+ waitTime: null,+ requiresFallback: true,+ actionable: false,+ graceMs: 0,+ suggestedFix: 'The selected model is temporarily at capacity. A delayed continue nudge resumes the session; if it stays overloaded, route the task to a fallback provider.',+ origin: 'provider',+ };+}++export function createModelAtCapacityDetector({ maxBuffer = 512 } = {}) {+ let buffer = '';+ const cap = Number.isFinite(maxBuffer) && maxBuffer > 0 ? maxBuffer : 512;++ return (chunk) => {+ if (!chunk) return null;+ buffer = `${buffer}${String(chunk)}`.slice(-cap);+ return detectModelAtCapacity(buffer);+ };+}+// Claude Code's in-TUI retry banner, painted while it re-sends a failed request:// `API error · Retrying in 0s · attempt 1/10` (the prefix varies — `Request timed// out`, `Overloaded` — so only the retry suffix is matched). The whitespace isdiff --git a/server/lib/aiToolkit/errorDetection.test.js b/server/lib/aiToolkit/errorDetection.test.jsindex aae426b543..88b357cfcc 100644--- a/server/lib/aiToolkit/errorDetection.test.js+++ b/server/lib/aiToolkit/errorDetection.test.js@@ -1105,3 +1105,13 @@ describe('terminal content refusals', () => {.toMatchObject({ category: 'content-refusal', requiresFallback: false, permanent: true });});});++describe('detectModelAtCapacity', () => {+ it('flags the Codex capacity rejection, even hard-wrapped, with a fixed non-echoing message', async () => {+ const { detectModelAtCapacity } = await import('./errorDetection.js');+ const hit = detectModelAtCapacity('■ Selected model is at capacity. Please try\na different model.');+ expect(hit).toMatchObject({ category: 'rate-limit', requiresFallback: true, actionable: false });+ expect(detectModelAtCapacity(hit.message)).toBeNull();+ expect(detectModelAtCapacity('the parking lot is at capacity')).toBeNull();+ });+});diff --git a/server/lib/tuiHandshake.js b/server/lib/tuiHandshake.jsindex 1b6626002a..8c907e0423 100644--- a/server/lib/tuiHandshake.js+++ b/server/lib/tuiHandshake.js@@ -1394,6 +1394,37 @@ export function createTruncationNudgeGate() {return createOomNudgeGate();}+// ─── Model-at-capacity nudge — "the provider is overloaded, try again later" ──+//+// Codex halts the turn on `Selected model is at capacity.` and waits at the+// composer for a human to type `continue`. Same dead-turn/intact-session shape+// as the OOM gate, but the cause is provider load, which clears in minutes+// rather than seconds — so the session must be quiet far longer before the+// nudge, the window waiting for that quiet is wider, and more attempts are+// allowed before the run fails over to a fallback provider.+export const CAPACITY_NUDGE_SETTLE_MS = 90000;+export const CAPACITY_NUDGE_ARM_WINDOW_MS = 300000;+export const CAPACITY_NUDGE_COOLDOWN_MS = 180000;+export const CAPACITY_NUDGE_MAX_ATTEMPTS = 6;+export const CAPACITY_NUDGE_TEXT = OOM_NUDGE_TEXT;++/**+ * State machine for "nudge a TUI session a model-at-capacity rejection parked".+ * See `createSilenceNudgeGate` for the semantics of `arm` / `takeNudge`.+ */+export function createModelCapacityNudgeGate() {+ const gate = createSilenceNudgeGate({+ cooldownMs: CAPACITY_NUDGE_COOLDOWN_MS,+ settleMs: CAPACITY_NUDGE_SETTLE_MS,+ armWindowMs: CAPACITY_NUDGE_ARM_WINDOW_MS,+ maxAttempts: CAPACITY_NUDGE_MAX_ATTEMPTS,+ });+ return {+ arm: (analysis, nowMs) => (analysis ? gate.arm(nowMs) : null),+ takeNudge: gate.takeNudge,+ };+}+/*** The "wait for silence, then nudge" policy the OOM gate and the* tool-permission gate share: dedupe sightings inside `cooldownMs` (a TUIdiff --git a/server/lib/tuiHandshake.test.js b/server/lib/tuiHandshake.test.jsindex 7983763bbe..1c54428e30 100644--- a/server/lib/tuiHandshake.test.js+++ b/server/lib/tuiHandshake.test.js@@ -2238,3 +2238,22 @@ describe('screenShowsInFlightChrome', () => {expect(screenShowsInFlightChrome(screen)).toBe(false);});});++describe('createModelCapacityNudgeGate', () => {+ it('waits for a long quiet spell, nudges, then reports exhaustion after the budget', async () => {+ const { createModelCapacityNudgeGate, CAPACITY_NUDGE_SETTLE_MS, CAPACITY_NUDGE_COOLDOWN_MS, CAPACITY_NUDGE_MAX_ATTEMPTS } = await import('./tuiHandshake.js');+ const gate = createModelCapacityNudgeGate();+ let now = 1_000_000;+ expect(gate.arm({ hasError: true }, now)).toBe('armed');+ // Output still fresh: no nudge yet.+ expect(gate.takeNudge(now + 1000, now)).toBe(0);+ expect(gate.takeNudge(now + CAPACITY_NUDGE_SETTLE_MS + 1, now)).toBe(1);+ for (let i = 2; i <= CAPACITY_NUDGE_MAX_ATTEMPTS; i += 1) {+ now += CAPACITY_NUDGE_COOLDOWN_MS + 1;+ expect(gate.arm({ hasError: true }, now)).toBe('armed');+ expect(gate.takeNudge(now + CAPACITY_NUDGE_SETTLE_MS + 1, now)).toBe(i);+ }+ now += CAPACITY_NUDGE_COOLDOWN_MS + 1;+ expect(gate.arm({ hasError: true }, now)).toBe('exhausted');+ });+});diff --git a/server/services/agentTuiSpawning/sessionController.js b/server/services/agentTuiSpawning/sessionController.jsindex 984e50b32e..338aaf1afe 100644--- a/server/services/agentTuiSpawning/sessionController.js+++ b/server/services/agentTuiSpawning/sessionController.js@@ -37,7 +37,7 @@ import { missingSentinelLogMessage, parseSentinelPayload } from '../../lib/agentimport { LIFECYCLE_LINES, SENTINEL_COMPLETION_MARKER } from '../../lib/agentOutputMarkers.js';import { resolveMergeGateVerdict, buildMergeGateReprompt } from '../../lib/mergeGateContract.js';import { createStreamingAnsiStripper, stripAnsi } from '../../lib/ansiStrip.js';-import { createContentRefusalDetector, createClaudeSessionLimitBannerDetector, createImmediateFallbackSignalDetector, createLocalRuntimeOomDetector, createTruncatedResponseDetector } from '../../lib/aiToolkit/errorDetection.js';+import { createContentRefusalDetector, createClaudeSessionLimitBannerDetector, createImmediateFallbackSignalDetector, createLocalRuntimeOomDetector, createTruncatedResponseDetector, createModelAtCapacityDetector } from '../../lib/aiToolkit/errorDetection.js';import { isAntigravityCommand } from '../../lib/antigravity.js';import { isCodexCommand } from '../../lib/codex.js';import { isClaudeCommand } from '../../lib/providerModels.js';@@ -56,6 +56,9 @@ import {OOM_NUDGE_MAX_ATTEMPTS,OOM_NUDGE_TEXT,createTruncationNudgeGate,+ createModelCapacityNudgeGate,+ CAPACITY_NUDGE_MAX_ATTEMPTS,+ CAPACITY_NUDGE_TEXT,TRUNCATION_NUDGE_MAX_ATTEMPTS,TRUNCATION_NUDGE_TEXT,createStallNudgeGate,@@ -559,6 +562,10 @@ export function createTuiSessionController({// createTruncationNudgeGate.const detectTruncatedResponse = createTruncatedResponseDetector();const truncationNudgeGate = createTruncationNudgeGate();+ // A provider-side `model is at capacity` rejection (Codex). Dead turn, intact+ // session — nudged with `continue`, but only after a much longer quiet spell.+ const detectModelAtCapacity = createModelAtCapacityDetector();+ const capacityNudgeGate = createModelCapacityNudgeGate();// agy quitting back to its launch shell (see createAgyResumeGate). Only a// login-shell launch has a shell to fall back to; runner/direct PTYs die with// agy and finish through handleExit instead.@@ -1489,6 +1496,18 @@ export function createTuiSessionController({}}+ // Model at capacity: arm a delayed nudge; fail over once they run out.+ const capacitySignal = promptSubmittedAt ? detectModelAtCapacity(stripped) : null;+ if (capacitySignal) {+ const armed = capacityNudgeGate.arm(capacitySignal, now);+ if (armed === 'armed') {+ appendLine(LIFECYCLE_LINES.capacityPending());+ } else if (armed === 'exhausted') {+ await failOverToFallback(capacitySignal);+ return;+ }+ }+// Same gating as the OOM nudge above: before the prompt is in there is no// request of ours for the provider to be retrying. Acted on by the// provider-signal timer, on its own poll, like the other gates.@@ -1878,6 +1897,14 @@ export function createTuiSessionController({}return;}+ // Nudge a session parked by a model-at-capacity rejection, after a long quiet.+ const capacityNudge = capacityNudgeGate.takeNudge(now, lastOutputAt);+ if (capacityNudge) {+ if (pasteController?.resubmit({ text: CAPACITY_NUDGE_TEXT, label: 'model-capacity nudge' })) {+ appendLine(LIFECYCLE_LINES.capacityNudged({ attempt: capacityNudge, max: CAPACITY_NUDGE_MAX_ATTEMPTS }));+ }+ return;+ }// Last, and only once no gate above has a say: every one of them describes a// session stalled for a KNOWN reason, and their own silence tests would be// pre-empted by a nudge sent on plain quiet.