Skip to content

Self-hosted, Self-Improving, Federated Everything App — dev management, agent orchestration, persistent mind, digital twin, second brain, and creative pipeline from a single dashboard over Tailscale to port the user into the OS

fix: nudge a Codex session parked by a model-at-capacity rejection

[._.]/ Adam Eivyabcdf74f4c40
diff --git a/server/lib/agentOutputMarkers.js b/server/lib/agentOutputMarkers.js
index 92fefbdb93..c3a55da16c 100644
--- a/server/lib/agentOutputMarkers.js
+++ b/server/lib/agentOutputMarkers.js
@@ -110,6 +110,8 @@ export const LIFECYCLE_LINES = Object.freeze({
permissionDeclineNudged: lifecycleLine`🔁 Nudged the session to continue after declined permission prompt ${'declined'}`,
localOomPending: lifecycleLine`⏳ Local runtime out of GPU memory — will nudge the session to continue if it goes quiet`,
localOomNudged: lifecycleLine`🔁 Local runtime OOM — nudged the session to continue (attempt ${'attempt'}/${'max'})`,
+ capacityPending: lifecycleLine`⏳ Selected model is at capacity — will nudge the session to continue after it has been quiet a while`,
+ capacityNudged: lifecycleLine`🔁 Selected model was at capacity — nudged the session to continue (attempt ${'attempt'}/${'max'})`,
truncationPending: lifecycleLine`⏳ Response was truncated before completion — will nudge the session to continue if it goes quiet`,
truncationNudged: lifecycleLine`🔁 Response was truncated before completion — nudged the session to continue (attempt ${'attempt'}/${'max'})`,
stallNudged: lifecycleLine`🔁 Session idle with the task unfinished — nudged it to continue (attempt ${'attempt'}/${'max'}, ${'sent'}/${'total'} this run)`,
diff --git a/server/lib/aiToolkit/errorDetection.js b/server/lib/aiToolkit/errorDetection.js
index 06b11030cc..8e25203c5d 100644
--- a/server/lib/aiToolkit/errorDetection.js
+++ b/server/lib/aiToolkit/errorDetection.js
@@ -737,6 +737,48 @@ export function createTruncatedResponseDetector({ maxBuffer = 512 } = {}) {
};
}
+// A provider that rejected the turn because the selected model is overloaded
+// (Codex: `Selected model is at capacity. Please try a different model.`). The
+// turn is dead but the session still holds the conversation, and a human typing
+// `continue` a little later resumes it. Unattended, nothing types it: the run
+// sits at the composer holding its lane (agent-9322c230). Same remedy as the
+// truncation banner, but the cause is load on the provider, so the consumer's
+// gate waits much longer between nudges (createModelCapacityNudgeGate).
+//
+// `\s+` joins survive a TUI hard-wrap. Not an IMMEDIATE_FALLBACK_SIGNALS entry
+// for the same reason as the truncation banner: the remedy is a continuation,
+// not a restart or an instant provider swap.
+const MODEL_AT_CAPACITY_PATTERN = /selected\s+model\s+is\s+at\s+capacity/i;
+
+export function detectModelAtCapacity(text) {
+ if (!text) return null;
+ if (!MODEL_AT_CAPACITY_PATTERN.test(String(text))) return null;
+ return {
+ hasError: true,
+ category: ERROR_CATEGORIES.RATE_LIMIT,
+ // FIXED sentence, not the banner wording: it becomes the run error and a
+ // task description can quote it back through this detector.
+ message: 'Provider reported the selected model is overloaded',
+ waitTime: null,
+ requiresFallback: true,
+ actionable: false,
+ graceMs: 0,
+ suggestedFix: 'The selected model is temporarily at capacity. A delayed continue nudge resumes the session; if it stays overloaded, route the task to a fallback provider.',
+ origin: 'provider',
+ };
+}
+
+export function createModelAtCapacityDetector({ maxBuffer = 512 } = {}) {
+ let buffer = '';
+ const cap = Number.isFinite(maxBuffer) && maxBuffer > 0 ? maxBuffer : 512;
+
+ return (chunk) => {
+ if (!chunk) return null;
+ buffer = `${buffer}${String(chunk)}`.slice(-cap);
+ return detectModelAtCapacity(buffer);
+ };
+}
+
// Claude Code's in-TUI retry banner, painted while it re-sends a failed request:
// `API error · Retrying in 0s · attempt 1/10` (the prefix varies — `Request timed
// out`, `Overloaded` — so only the retry suffix is matched). The whitespace is
diff --git a/server/lib/aiToolkit/errorDetection.test.js b/server/lib/aiToolkit/errorDetection.test.js
index aae426b543..88b357cfcc 100644
--- a/server/lib/aiToolkit/errorDetection.test.js
+++ b/server/lib/aiToolkit/errorDetection.test.js
@@ -1105,3 +1105,13 @@ describe('terminal content refusals', () => {
.toMatchObject({ category: 'content-refusal', requiresFallback: false, permanent: true });
});
});
+
+describe('detectModelAtCapacity', () => {
+ it('flags the Codex capacity rejection, even hard-wrapped, with a fixed non-echoing message', async () => {
+ const { detectModelAtCapacity } = await import('./errorDetection.js');
+ const hit = detectModelAtCapacity('■ Selected model is at capacity. Please try\na different model.');
+ expect(hit).toMatchObject({ category: 'rate-limit', requiresFallback: true, actionable: false });
+ expect(detectModelAtCapacity(hit.message)).toBeNull();
+ expect(detectModelAtCapacity('the parking lot is at capacity')).toBeNull();
+ });
+});
diff --git a/server/lib/tuiHandshake.js b/server/lib/tuiHandshake.js
index 1b6626002a..8c907e0423 100644
--- a/server/lib/tuiHandshake.js
+++ b/server/lib/tuiHandshake.js
@@ -1394,6 +1394,37 @@ export function createTruncationNudgeGate() {
return createOomNudgeGate();
}
+// ─── Model-at-capacity nudge — "the provider is overloaded, try again later" ──
+//
+// Codex halts the turn on `Selected model is at capacity.` and waits at the
+// composer for a human to type `continue`. Same dead-turn/intact-session shape
+// as the OOM gate, but the cause is provider load, which clears in minutes
+// rather than seconds — so the session must be quiet far longer before the
+// nudge, the window waiting for that quiet is wider, and more attempts are
+// allowed before the run fails over to a fallback provider.
+export const CAPACITY_NUDGE_SETTLE_MS = 90000;
+export const CAPACITY_NUDGE_ARM_WINDOW_MS = 300000;
+export const CAPACITY_NUDGE_COOLDOWN_MS = 180000;
+export const CAPACITY_NUDGE_MAX_ATTEMPTS = 6;
+export const CAPACITY_NUDGE_TEXT = OOM_NUDGE_TEXT;
+
+/**
+ * State machine for "nudge a TUI session a model-at-capacity rejection parked".
+ * See `createSilenceNudgeGate` for the semantics of `arm` / `takeNudge`.
+ */
+export function createModelCapacityNudgeGate() {
+ const gate = createSilenceNudgeGate({
+ cooldownMs: CAPACITY_NUDGE_COOLDOWN_MS,
+ settleMs: CAPACITY_NUDGE_SETTLE_MS,
+ armWindowMs: CAPACITY_NUDGE_ARM_WINDOW_MS,
+ maxAttempts: CAPACITY_NUDGE_MAX_ATTEMPTS,
+ });
+ return {
+ arm: (analysis, nowMs) => (analysis ? gate.arm(nowMs) : null),
+ takeNudge: gate.takeNudge,
+ };
+}
+
/**
* The "wait for silence, then nudge" policy the OOM gate and the
* tool-permission gate share: dedupe sightings inside `cooldownMs` (a TUI
diff --git a/server/lib/tuiHandshake.test.js b/server/lib/tuiHandshake.test.js
index 7983763bbe..1c54428e30 100644
--- a/server/lib/tuiHandshake.test.js
+++ b/server/lib/tuiHandshake.test.js
@@ -2238,3 +2238,22 @@ describe('screenShowsInFlightChrome', () => {
expect(screenShowsInFlightChrome(screen)).toBe(false);
});
});
+
+describe('createModelCapacityNudgeGate', () => {
+ it('waits for a long quiet spell, nudges, then reports exhaustion after the budget', async () => {
+ const { createModelCapacityNudgeGate, CAPACITY_NUDGE_SETTLE_MS, CAPACITY_NUDGE_COOLDOWN_MS, CAPACITY_NUDGE_MAX_ATTEMPTS } = await import('./tuiHandshake.js');
+ const gate = createModelCapacityNudgeGate();
+ let now = 1_000_000;
+ expect(gate.arm({ hasError: true }, now)).toBe('armed');
+ // Output still fresh: no nudge yet.
+ expect(gate.takeNudge(now + 1000, now)).toBe(0);
+ expect(gate.takeNudge(now + CAPACITY_NUDGE_SETTLE_MS + 1, now)).toBe(1);
+ for (let i = 2; i <= CAPACITY_NUDGE_MAX_ATTEMPTS; i += 1) {
+ now += CAPACITY_NUDGE_COOLDOWN_MS + 1;
+ expect(gate.arm({ hasError: true }, now)).toBe('armed');
+ expect(gate.takeNudge(now + CAPACITY_NUDGE_SETTLE_MS + 1, now)).toBe(i);
+ }
+ now += CAPACITY_NUDGE_COOLDOWN_MS + 1;
+ expect(gate.arm({ hasError: true }, now)).toBe('exhausted');
+ });
+});
diff --git a/server/services/agentTuiSpawning/sessionController.js b/server/services/agentTuiSpawning/sessionController.js
index 984e50b32e..338aaf1afe 100644
--- a/server/services/agentTuiSpawning/sessionController.js
+++ b/server/services/agentTuiSpawning/sessionController.js
@@ -37,7 +37,7 @@ import { missingSentinelLogMessage, parseSentinelPayload } from '../../lib/agent
import { LIFECYCLE_LINES, SENTINEL_COMPLETION_MARKER } from '../../lib/agentOutputMarkers.js';
import { resolveMergeGateVerdict, buildMergeGateReprompt } from '../../lib/mergeGateContract.js';
import { createStreamingAnsiStripper, stripAnsi } from '../../lib/ansiStrip.js';
-import { createContentRefusalDetector, createClaudeSessionLimitBannerDetector, createImmediateFallbackSignalDetector, createLocalRuntimeOomDetector, createTruncatedResponseDetector } from '../../lib/aiToolkit/errorDetection.js';
+import { createContentRefusalDetector, createClaudeSessionLimitBannerDetector, createImmediateFallbackSignalDetector, createLocalRuntimeOomDetector, createTruncatedResponseDetector, createModelAtCapacityDetector } from '../../lib/aiToolkit/errorDetection.js';
import { isAntigravityCommand } from '../../lib/antigravity.js';
import { isCodexCommand } from '../../lib/codex.js';
import { isClaudeCommand } from '../../lib/providerModels.js';
@@ -56,6 +56,9 @@ import {
OOM_NUDGE_MAX_ATTEMPTS,
OOM_NUDGE_TEXT,
createTruncationNudgeGate,
+ createModelCapacityNudgeGate,
+ CAPACITY_NUDGE_MAX_ATTEMPTS,
+ CAPACITY_NUDGE_TEXT,
TRUNCATION_NUDGE_MAX_ATTEMPTS,
TRUNCATION_NUDGE_TEXT,
createStallNudgeGate,
@@ -559,6 +562,10 @@ export function createTuiSessionController({
// createTruncationNudgeGate.
const detectTruncatedResponse = createTruncatedResponseDetector();
const truncationNudgeGate = createTruncationNudgeGate();
+ // A provider-side `model is at capacity` rejection (Codex). Dead turn, intact
+ // session — nudged with `continue`, but only after a much longer quiet spell.
+ const detectModelAtCapacity = createModelAtCapacityDetector();
+ const capacityNudgeGate = createModelCapacityNudgeGate();
// agy quitting back to its launch shell (see createAgyResumeGate). Only a
// login-shell launch has a shell to fall back to; runner/direct PTYs die with
// agy and finish through handleExit instead.
@@ -1489,6 +1496,18 @@ export function createTuiSessionController({
}
}
+ // Model at capacity: arm a delayed nudge; fail over once they run out.
+ const capacitySignal = promptSubmittedAt ? detectModelAtCapacity(stripped) : null;
+ if (capacitySignal) {
+ const armed = capacityNudgeGate.arm(capacitySignal, now);
+ if (armed === 'armed') {
+ appendLine(LIFECYCLE_LINES.capacityPending());
+ } else if (armed === 'exhausted') {
+ await failOverToFallback(capacitySignal);
+ return;
+ }
+ }
+
// Same gating as the OOM nudge above: before the prompt is in there is no
// request of ours for the provider to be retrying. Acted on by the
// provider-signal timer, on its own poll, like the other gates.
@@ -1878,6 +1897,14 @@ export function createTuiSessionController({
}
return;
}
+ // Nudge a session parked by a model-at-capacity rejection, after a long quiet.
+ const capacityNudge = capacityNudgeGate.takeNudge(now, lastOutputAt);
+ if (capacityNudge) {
+ if (pasteController?.resubmit({ text: CAPACITY_NUDGE_TEXT, label: 'model-capacity nudge' })) {
+ appendLine(LIFECYCLE_LINES.capacityNudged({ attempt: capacityNudge, max: CAPACITY_NUDGE_MAX_ATTEMPTS }));
+ }
+ return;
+ }
// Last, and only once no gate above has a say: every one of them describes a
// session stalled for a KNOWN reason, and their own silence tests would be
// pre-empted by a nudge sent on plain quiet.