Self-hosted, Self-Improving, Federated Everything App — dev management, agent orchestration, persistent mind, digital twin, second brain, and creative pipeline from a single dashboard over Tailscale to port the user into the OS
JavaScript ai-agentsdashboarddeveloper-toolsdigital-twinquantified-selfsecond-brainself-hostedtailscale
import { isLoopbackEndpoint } from '../lib/localEndpoint.js';
import { maintenance } from '../lib/maintenanceAdmission.js';
/**
* Shared LLM runner wrapper.
*
* Four near-identical implementations of "create run → branch on
* provider.type → executeCliRun/executeApiRun → accumulate streamed text
* → reject on error" had drifted in failure-handling:
*
* - universeBuilderExpand#callLLM — CLI: `error || success === false`, API: `error` only
* - stageRunner#awaitRunnerCall — CLI: `error || success === false`, API: `error` only
* - mediaPromptRefiner#runRefinePrompt — both: `error || success === false`
* - messageEvaluator#runPrompt — CLI: `error || success === false`, API: `error` only
*
* This unified runner picks the strictest discriminator: reject on
* `success === false` OR truthy `error` for BOTH CLI and API. The
* per-site drift was the bug — silent API "soft failures" (e.g. an
* empty completion that doesn't set `error` but does set
* `success: false`) used to flow through as a successful empty string.
*
* Returns `{ text, runId, model }`. `text` is the full streamed body;
* `runId` is the persisted run id so callers can log it and surface
* data/runs/<runId>/output.txt for offline debugging; `model` is the
* effective model that actually executed after the per-provider
* override gate (null when neither the caller's model nor
* provider.defaultModel applies). Callers should log/return THIS
* `model`, not the value they passed in, so logs and run records
* stay honest about what the runner actually executed.
*/
import { LOCAL_LLM_MAX_CONCURRENCY, acquireLocalSlot, releaseLocalSlot } from '../lib/localLlmConcurrency.js';
import { isVisionCapableCodexTuiProvider } from '../lib/codex.js';
import { createRun, executeApiRun, executeCliRun, extractBakedModel, hasModelFlag, stopRun, patchRunMetadata, finalizeRunRecord } from './runner.js';
import { getActiveProvider, getProviderById, getAllProviders } from './providers.js';
import { isCompositeProviderId } from '../lib/providerRef.js';
import { peekFallbackChain } from '../lib/providerFallbackChainCache.js';
// `./tuiPromptRunner.js` (which drags node-pty in through `./shell.js`) and
// `./providerExecutionReadiness.js` are imported lazily on the branches that
// actually execute a run — see their call sites below. promptRunner.js is reached
// by ~160 suites that only build or classify a run, and a static edge instantiated
// both subtrees in every one of them.
import { ServerError } from '../lib/errorHandler.js';
import { PROVIDER_TYPES } from '../lib/aiToolkit/constants.js';
import { analyzeError, ERROR_CATEGORIES, isRunCanceledError, isRuntimeBudgetError, isContentRefusalError } from '../lib/aiToolkit/errorDetection.js';
import { isSchemaTypeCategory, resolveProviderBench } from '../lib/providerCooldown.js';
import { isGenerationModel, isVisionCapableCliProvider } from '../lib/localModelHeuristics.js';
import { contextWindowRejection, knownContextWindow } from '../lib/aiToolkit/providerStatus.js';
import { isOllamaBackedProvider } from '../lib/aiToolkit/internal/ollamaBacked.js';
import { apiRunAbsoluteTimeoutMs } from '../lib/aiToolkit/internal/runTimeouts.js';
import { withOllamaRuntimeContextWindow } from '../lib/ollamaContext.js';
import { getAIToolkitInstance } from '../lib/aiToolkitState.js';
import { createSingleFlight } from '../lib/singleFlight.js';
import { extractJson } from '../lib/jsonExtract.js';
import { isCreativeRunSource, withCreativeLatitude } from '../lib/creativeLatitude.js';
import { DEFAULT_OUTPUT_RESERVE_TOKENS, estimateTokens, localOutputReserveTokens } from '../lib/contextBudget.js';
import { allowedModesFor, callerModeRejection } from '../lib/callerModePolicy.js';
import { attachGatewaySiblingKey } from '../lib/providerGateways.js';
// The fallback-lifecycle notifiers live in services/autoFixer.js, which
// pulls in services/cos.js. initAutoFixer() registers them through this leaf;
// neither a static nor a lazy import of autoFixer.js belongs here, because
// `vitest related` follows both into the CoS cluster (#12285).
import { getProviderRecoveryHandlers } from './providerRecoveryHooks.js';
// Deferred so the static closure stays small: `observedContextWindows.js`
// reaches `lib/localProviderRuntime.js` and the endpoint probe, which
// promptRunner.js itself never needs, and ~160 suites only build or classify a
// run. A static edge instantiated that subtree in every one of them and pushed
// the server suite past its import budget (see "Import scoping" in
// server/AGENTS.md).
let observedContextWindowsLoadPromise;
function loadObservedContextWindows() {
if (!observedContextWindowsLoadPromise) {
observedContextWindowsLoadPromise = import('./observedContextWindows.js').catch((err) => {
observedContextWindowsLoadPromise = null;
throw err;
});
}
return observedContextWindowsLoadPromise;
}
/**
* `withObservedContextWindows(provider)` behind that deferred load — the ONE
* operation both readers of the module want (the dispatch gate below, and
* `stageRunner.resolveStageContext` for the budgeter). Exported as the operation
* rather than the loader so neither caller hard-codes which named export to
* destructure, and both share one dynamic import and one settled promise.
*
* Rejects on a load or probe failure; error policy is the caller's, because the
* two differ deliberately — the gate lets it surface, the budgeter degrades to
* the provider as stored rather than turning planning into a new failure mode.
*/
export async function withObservedContextWindowsLazy(provider) {
const { withObservedContextWindows } = await loadObservedContextWindows();
return withObservedContextWindows(provider);
}
// The category an oversized prompt already carries everywhere else in PortOS —
// `agentErrorAnalysis.js` extracts it from a failed agent's output and
// `codexTurn.js` maps Codex's `contextWindowExceeded` tag to it. It is a plain
// string rather than an `ERROR_CATEGORIES` member on purpose: the toolkit's
// enum is for provider-health categories, and this one is REQUEST-specific
// (`providerCooldown.isSchemaTypeCategory` covers it, so it never benches).
const CONTEXT_LENGTH_CATEGORY = 'context-length';
export const DEFAULT_TIMEOUT_MS = 300000;
// Grace window past the runner's own API deadline before promptRunner's
// backstop timer fires. Lets executeApiRun's internal timers win (aborting +
// finalizing as ERROR_CATEGORIES.TIMEOUT) so the normal path is classified
// correctly; the backstop only intervenes if the runner never reports
// completion at all.
const API_TIMEOUT_BACKSTOP_GRACE_MS = 2000;
const APPEND_CHUNK = (acc, chunk) => acc + (typeof chunk === 'string' ? chunk : (chunk?.text || ''));
export function buildRequestCapabilities({ prompt, screenshots, outputReserveTokens, callerPolicy = null } = {}) {
const reserve = Number.isFinite(Number(outputReserveTokens))
? Math.max(0, Number(outputReserveTokens))
: DEFAULT_OUTPUT_RESERVE_TOKENS;
return {
requiredContextTokens: estimateTokens(prompt) + reserve,
hasImages: Array.isArray(screenshots) && screenshots.length > 0,
// Only present when the caller declared a mode policy, so an undeclared
// caller keeps routing across every mode exactly as it always has.
...(callerPolicy ? { allowedModes: allowedModesFor(callerPolicy) } : {}),
};
}
// ── Local-backend concurrency gate ─────────────────────────────────────────
// A local LLM server (Ollama / LM Studio on localhost) runs inference on one
// GPU. Firing N stage calls at it concurrently (e.g. the 3 parallel
// canon-extraction kinds) gains nothing — the backend serializes on the GPU
// regardless — and, if the backend is configured for parallelism
// (OLLAMA_NUM_PARALLEL > 1), it loads N model contexts at once and can spike
// VRAM into an OOM/thrash. Cloud HTTP and CLI/TUI providers have no such
// constraint. Local TUI providers still share this gate because their readiness
// hook can start the same daemon before the PTY is spawned. So we cap concurrent
// IN-FLIGHT calls per LOCAL endpoint and queue
// the rest (FIFO). Default 1 (serialize); a beefy box can lift it with
// LOCAL_LLM_MAX_CONCURRENCY. Keyed by endpoint so two distinct local servers
// still run in parallel with each other.
//
// The gate lives HERE (the actual execution layer) rather than at the
// stageRunner call site so it covers EVERY local execution: the initially
// selected provider, a proactive createRun swap, AND a runtime fallback that
// lands on a local backend after a remote/CLI primary fails. Gating only the
// outer stageRunner call missed the fallback path — a failure storm could still
// fire N concurrent calls at one local endpoint.
export const isLocalEndpoint = isLoopbackEndpoint;
export { LOCAL_LLM_MAX_CONCURRENCY };
// Run `fn` under the local-endpoint gate when `provider` is a local API or TUI
// backend; otherwise run it immediately (cloud / non-local CLI/TUI are
// unconstrained).
export async function withLocalConcurrencyGate(provider, fn) {
const endpoint = provider?.endpoint;
const isLocalProvider = provider?.type === PROVIDER_TYPES.API || provider?.type === PROVIDER_TYPES.TUI;
if (!(isLocalProvider && isLocalEndpoint(endpoint))) return fn();
await acquireLocalSlot(endpoint);
try {
return await fn();
} finally {
releaseLocalSlot(endpoint);
}
}
/**
* Returns true when the runner+provider pair will actually honor a
* per-call `model` override. For API providers the model is a
* first-class arg to `executeApiRun`. For CLI/TUI providers,
* `runner.js#buildCliArgs` / `tuiHandshake.js#buildTuiInvocation`
* translate the resolved `defaultModel` into a `--model`/`-m` flag — BUT
* only when the user hasn't already baked a model flag into `provider.args`.
* If a flag is baked in, the runner-injected one is suppressed and the
* args-baked model wins; so claim "doesn't honor" for that case to keep
* the run-record honest.
*/
export const providerHonorsModelOverride = (provider) => (
provider?.type === PROVIDER_TYPES.API
|| ((provider?.type === PROVIDER_TYPES.CLI || provider?.type === PROVIDER_TYPES.TUI) && !hasModelFlag(provider?.args))
);
/**
* Resolve the model id that will ACTUALLY execute against the provider,
* for accurate logging + run-record persistence.
*
* Decision table:
* - Provider honors per-call override (API or CLI w/o baked args flag)
* → callerModel || provider.defaultModel || provider.models[0]
* - CLI with a baked --model/-m in provider.args (runner.js will
* suppress its own injection and let the args-pinned model win)
* → extractBakedModel(args) || provider.defaultModel || models[0]
*
* Returns null when no fallback resolves (so logs read "(default)" instead
* of an inaccurate value).
*
* @param {object} provider
* @param {string} [callerModel] — the per-call model the caller asked for
* @returns {string|null}
*/
export function resolveEffectiveModel(provider, callerModel) {
if (providerHonorsModelOverride(provider)) {
// A stale config can leave an embedding-only id in callerModel/defaultModel
// (the UI only hides+clears embedding models when the provider is edited);
// skip those so an unchanged older provider can't route a generation/fallback
// run to e.g. nomic-embed-text. Falls through to the first generation model.
return firstNonEmbedding(callerModel, provider?.defaultModel)
|| firstGenerationModel(provider) || null;
}
// Non-honoring CLI/TUI path: args-baked model id wins over defaultModel.
const baked = (provider?.type === PROVIDER_TYPES.CLI || provider?.type === PROVIDER_TYPES.TUI) && hasModelFlag(provider?.args)
? extractBakedModel(provider.args)
: null;
return firstNonEmbedding(baked, provider?.defaultModel)
|| firstGenerationModel(provider) || null;
}
/**
* First of the given candidate model ids that is a usable generation model
* (skips null/empty and embedding-only ids). Returns null when none qualify, so
* callers can fall through to `firstGenerationModel(provider)`.
*/
function firstNonEmbedding(...candidates) {
return candidates.find((m) => m && isGenerationModel(m)) || null;
}
/**
* First model in `provider.models` usable for generation (skips embedding-only
* models). Without this, a local provider whose `models[0]` is an embedding
* model (e.g. Ollama with `nomic-embed-text:latest` listed first and a null
* defaultModel) would resolve the embedding model for a chat/fallback run and
* fail — the exact nomic-embed-text fallback bug. Falls back to `models[0]`
* only when every listed model looks like an embedding model (better to try
* something than resolve null).
*/
function firstGenerationModel(provider) {
const models = provider?.models || [];
return models.find((m) => m && isGenerationModel(m)) || models[0] || null;
}
/**
* Tier-1 (config/env) deterministic fix for a `model-not-found` /
* `model-not-supported` failure (issue #2342): the request named a model id the
* (reachable) provider doesn't serve, so pick a DIFFERENT valid generation
* model that the provider DOES list and retry the SAME provider with it —
* cheaper than benching the provider and switching to a fallback (Tier 3).
*
* Pure — no I/O. Returns null (⇒ Tier 1 declines, cascade falls through) when:
* - the provider carries no enumerable `models` list (nothing to correct to), OR
* - the only listed models are the failed one and/or embedding-only ids.
*
* The failed id is excluded so we never re-issue the same broken request, and
* embedding-only ids are skipped (mirrors `firstGenerationModel`) so a
* generation call can't get corrected onto e.g. `nomic-embed-text`.
*
* @param {object} provider
* @param {string} [failedModel] — the model id that just failed model-not-found
* @returns {string|null}
*/
export function pickConfigCorrectedModel(provider, failedModel) {
const models = provider?.models;
if (!Array.isArray(models)) return null;
return models.find((m) => m && m !== failedModel && isGenerationModel(m)) || null;
}
// ── Tier-2 (schema/type) request/response correction (issue #2350) ─────────
// At the runner layer a schema/type failure never arrives as a provider ERROR —
// errorDetection.js only categorizes transport/auth/model failures, so a
// malformed *response* surfaces as a run that SUCCEEDED at the transport layer
// but whose text doesn't satisfy the caller's declared schema. Tier-2 therefore
// validates a successful response against a caller-supplied `responseSchema`
// (or `repair` normalizer), attempts a bounded deterministic JSON coercion in
// place, and — only when that can't recover the shape — re-requests the SAME
// provider with a schema-strengthened prompt before the cascade escalates to a
// Tier-3 fallback / Tier-4 investigation task.
// Re-exported from providerCooldown.js, which owns it alongside the cooldown
// table that consults it (the tier classification and the "don't bench a healthy
// provider over one off-shape response" rule are the same judgement).
export { isSchemaTypeCategory };
// Deterministic instruction appended to the prompt on a schema/type re-request
// when the caller declared a schema but supplied no custom `repair`. Names the
// two failure modes coercion couldn't fix (fenced/prose-wrapped output) so the
// model returns a bare, parseable value on the retry.
const SCHEMA_RETRY_INSTRUCTION = '\n\n---\nIMPORTANT: Your previous response could not be parsed into the required structured format. Respond with ONLY the raw JSON value that satisfies the required schema — no markdown code fences, and no explanatory text before or after the JSON.';
/**
* Normalize a caller `responseSchema` into a predicate over a PARSED JSON value
* (issue #2350). Accepts a Zod-style schema (duck-typed via `.safeParse`/`.parse`
* — promptRunner takes no zod dependency), a bare predicate `(value)=>boolean`,
* or null/undefined (⇒ null, feature off). An unrecognized truthy value degrades
* to "any JSON parses" so a loose schema hint still enables fence/prose stripping
* without throwing. Pure.
* @returns {((value:unknown)=>boolean)|null}
*/
export function normalizeResponseSchema(schema) {
if (!schema) return null;
if (typeof schema === 'function') return (value) => { try { return !!schema(value); } catch { return false; } };
if (typeof schema.safeParse === 'function') return (value) => schema.safeParse(value).success;
if (typeof schema.parse === 'function') return (value) => { try { schema.parse(value); return true; } catch { return false; } };
return () => true;
}
// Discriminated JSON parse: `{ ok:true, value }` (value may legitimately be
// `null`) vs `{ ok:false }`, so a top-level `null` response isn't conflated with
// a parse failure.
const safeJsonParse = (text) => { try { return { ok: true, value: JSON.parse(text) }; } catch { return { ok: false }; } };
/**
* True when `text` already parses as JSON that satisfies the schema predicate —
* i.e. no correction is needed. A non-JSON or off-shape response returns false.
*/
function responseSatisfiesSchema(text, predicate) {
if (typeof text !== 'string' || !text) return false;
const parsed = safeJsonParse(text);
return parsed.ok && (!predicate || predicate(parsed.value));
}
// Pick which top-level JSON shape to try first based on the first structural
// char, so a top-level array isn't mis-picked as its first inner object (and
// vice-versa). Deterministic; always returns both types so the other is still
// tried. `[`/`{` never appear inside a ```json fence marker, so scanning the raw
// text is sufficient without stripping fences first.
function detectBlockOrder(text) {
const firstBrace = text.indexOf('{');
const firstBracket = text.indexOf('[');
if (firstBracket !== -1 && (firstBrace === -1 || firstBracket < firstBrace)) return ['array', 'object'];
return ['object', 'array'];
}
/**
* Deterministic Tier-2 JSON coercion keyed on the caller's schema (issue #2350).
* Reuses jsonExtract.extractJson to strip markdown fences / prose and pull the
* first balanced JSON block whose shape satisfies the schema predicate — the
* "JSON coercion, field-shape fix keyed on that schema" the issue calls for.
* Returns the coerced value plus its canonical `JSON.stringify` text. Pure.
* @param {string} text — raw response text
* @param {*} schema — a responseSchema (see normalizeResponseSchema)
* @returns {{ ok:true, value:unknown, text:string } | { ok:false }}
*/
export function coerceResponseToSchema(text, schema) {
if (typeof text !== 'string' || !text.trim()) return { ok: false };
const predicate = normalizeResponseSchema(schema);
for (const blockType of detectBlockOrder(text)) {
// extractJson's `parsedHolder` fallback returns the first PARSEABLE block
// even when it doesn't match the shape predicate (so a top-level `null` can
// flow through) — so re-check the predicate here and only accept a block
// that actually satisfies the schema.
const { value } = extractJson(text, { shapePredicate: predicate || undefined, blockType });
if (value !== undefined && (!predicate || predicate(value))) {
return { ok: true, value, text: JSON.stringify(value) };
}
}
return { ok: false };
}
// Best-effort call to a caller `repair` normalizer for a given phase. Never
// throws (a repair-callback error is logged + treated as a decline); returns the
// corrected string for the requested field, or null.
async function callRepair(repair, payload, field) {
if (typeof repair !== 'function') return null;
let out = null;
try { out = await repair(payload); } catch (e) {
console.error(`❌ Tier 2 ${payload.phase} repair callback threw: ${e.message}`);
return null;
}
const v = out?.[field];
return typeof v === 'string' && v ? v : null;
}
/**
* Validate a run RESULT against the caller's declared `responseSchema` (issue
* #2350) and, when it doesn't match, attempt a bounded deterministic correction
* IN PLACE: first the caller's `repair({ phase:'response' })` normalizer, then
* the schema-keyed JSON coercion. Never throws. Returns:
* { ok:true, result } — no schema declared, or already valid
* { ok:true, result, coerced:true } — recovered in place (⇒ mark Tier-2)
* { ok:false } — response is off-shape and uncoercible
*/
async function correctResponseToSchema(result, { responseSchema, repair, prompt }) {
const predicate = normalizeResponseSchema(responseSchema);
// Response validation requires a declared schema — a `repair`-only caller has
// no way for the runner to know a response is malformed, so leave it untouched.
if (!predicate) return { ok: true, result };
const text = typeof result?.text === 'string' ? result.text : '';
if (responseSatisfiesSchema(text, predicate)) return { ok: true, result };
const repaired = await callRepair(repair, { phase: 'response', text, prompt }, 'text');
if (repaired && responseSatisfiesSchema(repaired, predicate)) {
return { ok: true, result: { ...result, text: repaired }, coerced: true };
}
const coerced = coerceResponseToSchema(text, responseSchema);
if (coerced.ok) return { ok: true, result: { ...result, text: coerced.text }, coerced: true };
return { ok: false };
}
/**
* Build the Tier-2 corrected REQUEST for a schema/type failure. Prefers the
* caller's `repair({ phase:'request' })` normalizer; otherwise, when a
* `responseSchema` is declared, appends a deterministic schema-conformance
* instruction to the original prompt. Returns null (⇒ Tier-2 declines, cascade
* falls through to Tier-3) when no correction is available or it wouldn't change
* the prompt.
*/
async function buildSchemaCorrectedPrompt({ prompt, responseSchema, repair, category, error }) {
const repaired = await callRepair(repair, { phase: 'request', prompt, category, error }, 'prompt');
if (repaired && repaired !== prompt) return repaired;
if (responseSchema) {
const corrected = `${prompt}${SCHEMA_RETRY_INSTRUCTION}`;
if (corrected !== prompt) return corrected;
}
return null;
}
/**
* Synthesize a schema/type failure Error so an uncoercible response re-enters
* the SAME fallback cascade as a transport failure (Tier-2 re-request → Tier-3
* fallback → Tier-4 escalate) with the identical noteFallback* reconciliation.
* Carries `effectiveProvider`/`effectiveModel` (the annotations the cascade
* keys on) from the run that actually produced the bad response, a 'parse-error'
* category (⇒ isSchemaTypeCategory, and skipped by markProviderUnavailableFromError
* so a healthy provider isn't benched for one off-shape response), and a
* `schemaFailure` marker for diagnostics.
*/
function buildSchemaFailureError(result) {
const err = new Error('AI response did not match the declared schema and could not be coerced');
err.errorAnalysis = { category: 'parse-error' };
err.effectiveProvider = result?.provider;
err.effectiveModel = result?.model ?? null;
err.schemaFailure = true;
return err;
}
/**
* Resolve `{provider, selectedModel}` for an LLM caller. Prefers
* `providerId` — any `getProviderById` failure (stale id, lookup
* error, network blip) falls through to `getActiveProvider`. Returns
* `{provider: null, selectedModel: null}` when neither resolves a
* provider (e.g. no providers configured), so callers throw their own
* typed error.
*
* Note: errors from `getActiveProvider` (e.g. toolkit not initialized)
* still propagate — only `getProviderById` failures are swallowed.
* This mirrors the inline pattern this helper replaced. If a caller
* wants total "always-null on failure" semantics, wrap the call in
* their own try/catch.
*
* @param {object} args
* @param {string} [rawArgs.providerId]
* @param {string} [args.model]
* @returns {Promise<{ provider: object|null, selectedModel: string|null }>}
*/
export async function resolveProviderAndModel({ providerId, model } = {}) {
let provider = providerId ? await getProviderById(providerId).catch(() => null) : null;
if (!provider) provider = await getActiveProvider();
const selectedModel = provider ? resolveEffectiveModel(provider, model) : null;
return { provider, selectedModel };
}
/**
* Throw a typed "no AI provider available" error when `provider` is falsy.
*
* Sites that surface to HTTP route handlers should pass `code` (+ optional
* `status`, defaults to 503) so the centralized error middleware emits a
* structured response. Sites that are internal-only (service-to-service)
* can omit both and get a plain `Error` — matching the legacy shape.
*
* @param {object|null} provider
* @param {object} opts
* @param {string} opts.message — human-readable error message
* @param {string} [opts.code] — error code constant (e.g. `NO_PROVIDER`)
* @param {number} [opts.status=503] — HTTP status; only used when `code` is set
*/
export function assertProvider(provider, { message, code, status = 503 } = {}) {
if (provider) return;
if (code) throw new ServerError(message, { status, code });
throw new Error(message);
}
/**
* Guard a vision run against a transport that did not receive images.
*
* API providers base64-inline images, while the shared CLI lifecycle stages
* file attachments for Codex and Claude Code. Codex TUI forwards native image
* arguments. Other TUI and CLI providers must
* not be accepted because they would answer from the text prompt alone.
* `runPromptThroughProvider` can swap the provider two ways — a proactive swap
* inside createRun (`result.provider`) or a retry fallback after failure
* (`result.fallbackProvider`) — so the provider that ACTUALLY ran is the first
* of those, else the requested one. Throw
* VISION_FALLBACK_DROPPED_IMAGES when it isn't an API provider so callers don't
* report image-grounded output that was really text-only.
*
* Returns the provider that ran so callers can read its id/type.
*
* @param {object} result — the runPromptThroughProvider result
* @param {object} requestedProvider — the provider the caller asked to run
* @returns {object} the effective provider that ran
*/
export function assertVisionRunUsedImages(result, requestedProvider) {
const ran = result?.provider || result?.fallbackProvider || requestedProvider;
const usedImages = ran?.type === 'api' || isVisionCapableCliProvider(ran) || isVisionCapableCodexTuiProvider(ran);
if (ran?.type && !usedImages) {
// Name both providers so the cause is actionable. The usual trigger is a
// proactive/retry swap because the requested API provider is in a temporary
// cooldown (e.g. a prior model-not-found benched it for several minutes) —
// NOT that the user picked a non-vision provider. Point them at the real fix.
const requestedName = requestedProvider?.name || requestedProvider?.id || 'the selected provider';
const ranName = ran?.name || ran?.id || 'a non-vision provider';
const swapped = ran?.id && requestedProvider?.id && ran.id !== requestedProvider.id;
const cause = swapped
? `"${requestedName}" was unavailable (likely a temporary cooldown after an earlier failed request), so the run fell back to "${ranName}", which can't read images.`
: `"${ranName}" can't read images.`;
throw new ServerError(
`${cause} Retry in a few minutes, or pick a different vision-capable API provider/model.`,
{ status: 502, code: 'VISION_FALLBACK_DROPPED_IMAGES' },
);
}
return ran;
}
/** Own deferred investigations for one retry cascade, independently of retry policy. */
function createRecoveryInvestigationScope({ failed, failedModel, firstError }) {
// Handlers registered by initAutoFixer(); null when the auto-fixer is not
// running in this process, in which case it has no deferred tasks to settle.
const getAutoFixer = async () => getProviderRecoveryHandlers();
// Keys must match what bootstrap.js's onRunFailed hook published
// (metadata.providerName + metadata.model). Best-effort throughout: a
// notifier failure must never turn a working retry into a user-visible error.
const noteStarted = async (key) => { const a = await getAutoFixer(); try { a?.noteFallbackStarted(key); } catch { /* best-effort */ } };
const noteFailed = async (key) => { const a = await getAutoFixer(); try { a?.noteFallbackFailed(key); } catch { /* best-effort */ } };
const noteHandled = async (key) => {
const a = await getAutoFixer();
try { a?.noteFallbackHandled(key); } catch (suppressErr) {
console.error(`❌ noteFallbackHandled failed (investigation task not suppressed): ${suppressErr.message}`);
}
};
// Track every provider+model key whose deferred investigation task we
// suppress across the cascade, so (a) a recovery at ANY tier cancels ALL of
// them, (b) a give-up releases them without leaking autoFixer's in-flight
// set, and (c) the `finally` safety-net releases any left unresolved by an
// unexpected throw (e.g. coalesceFallbackMarkAndPick) — which would
// otherwise suppress every future identical failure for the process
// lifetime. Keyed by a NUL-joined string (provider names / model ids both
// contain '-'); the value carries the original key object + resolution state.
const suppressed = new Map();
const keyStr = (k) => `${k.provider}\x00${k.model}`;
const suppress = async (key) => {
const id = keyStr(key);
if (suppressed.has(id)) return; // idempotent across tiers
suppressed.set(id, { key, resolved: false });
await noteStarted(key);
};
const recoverAll = async () => {
for (const entry of suppressed.values()) {
if (!entry.resolved) { entry.resolved = true; await noteHandled(entry.key); }
}
};
const releaseForExistingInvestigation = async () => {
for (const entry of suppressed.values()) {
if (!entry.resolved) { entry.resolved = true; await noteFailed(entry.key); }
}
};
// Explicitly escalate the primary failure to a Tier-4 investigation task —
// used only on a give-up where every attempted key's incidental task was
// suppressed, so nothing else would surface the unrecovered failure. Honors
// the circuit breaker. Best-effort: a failure here must not mask the rethrow.
const escalatePrimaryFailure = async () => {
const a = await getAutoFixer();
try {
await a?.escalateProviderFailure?.({
code: 'AI_PROVIDER_EXECUTION_FAILED',
message: firstError.message,
timestamp: Date.now(),
context: {
provider: failed.name || failed.id,
providerId: failed.id,
model: failedModel,
errorDetails: firstError.message,
errorAnalysis: firstError.errorAnalysis,
},
});
} catch (escalateErr) {
console.error(`❌ escalateProviderFailure failed (unrecovered failure not escalated): ${escalateErr.message}`);
}
};
const escalateAndRelease = async () => {
// With no suppression, the execution layer still owns the original event.
// Preserve that path without loading autoFixer or creating a replacement.
if (suppressed.size === 0) return;
await escalatePrimaryFailure();
await releaseForExistingInvestigation();
};
const closeUnexpectedFailure = async () => {
// Explicit terminal operations settle every key; only an unexpected throw
// can leave canceled backstops without a surviving investigation here.
if ([...suppressed.values()].some((entry) => !entry.resolved)) {
await escalateAndRelease();
}
};
return { suppress, recoverAll, releaseForExistingInvestigation, escalateAndRelease, closeUnexpectedFailure };
}
/**
* Run a prompt through a provider and resolve with the streamed text +
* run id. Rejects (via the strictest discriminator) on any runner-
* reported failure.
*
* Per-call `model` overrides are silently dropped for providers that
* don't honor them (see `providerHonorsModelOverride`). This keeps the
* persisted run record honest about which model actually ran — passing
* the user's selection downstream when the runner can't apply it would
* make `/runs` and SSE status events lie about model usage.
*
* @param {object} args
* @param {object} rawArgs.provider — { id, type: 'cli'|'api'|'tui', timeout?, ... }
* @param {string} rawArgs.prompt — full text to send to the LLM
* @param {string} rawArgs.source — run-record tag (`'universe-builder-expansion'`,
* `'media-prompt-refine'`, `'messages-triage'`, `'staged-llm'`, etc.)
* @param {string} [args.model] — model id hint; ignored when the
* provider doesn't honor it (claude-code, antigravity-cli today).
* @param {string} [args.effort] — reasoning-effort override for effort-capable
* CLI/TUI providers (claude `--effort`, agy `--effort`, codex
* `-c model_reasoning_effort=`). Clamped to the provider's (and, for agy, the
* selected model's) ladder and dropped entirely for providers with no effort
* control, so it is safe to pass unconditionally — including through the
* fallback cascade, which may land on a provider that has none.
* @param {string} [args.runId] — caller-supplied run id (skip createRun
* round-trip when the caller has already created the run)
* @param {(chunk: string) => void} [args.onData] — incremental stream
* callback; receives each output chunk as it arrives. Useful for live
* progress UI (loops, live transcripts). Does NOT change the resolved
* `text` value — callers receive the full buffered text either way.
* For TUI providers the stripped chunks are emitted; the final `text`
* is the cleaned response with the prompt-echo elided.
* @param {(runId:string,meta:object)=>void} [args.onRunCreated] — called immediately after
* each concrete primary/correction/fallback run id is known. Observational;
* callback errors are logged and ignored.
* @param {(meta:object)=>void} [args.onRunReady] — called when an interactive
* TUI run has been registered as an attachable shell session.
* @param {(runId:string)=>void} [args.onRunSettled] — called when that concrete
* attempt resolves or rejects. Observational; callback errors are ignored.
* @param {(meta:object)=>Promise<void>} [args.beforeExecute] — enforced caller
* admission immediately before provider dispatch, after concurrency/readiness
* and attachment preparation. A rejection cancels this attempt without
* retrying, benching the provider, or investigating a provider failure.
* @param {string[]} [args.screenshots] — image paths for a vision/multimodal
* call (relative to the runner's screenshots dir, or absolute). API providers
* base64-encode each into `image_url` blocks. Codex and Claude Code CLI
* providers receive staged file attachments through executeCliRun. Other CLI
* and all TUI providers are rejected before a run record is created.
* @param {number|((provider: object) => number)} [args.timeout] — per-call
* timeout in ms; falls back to `provider.timeout`, then DEFAULT_TIMEOUT_MS.
* Callers like the loop runner expose a user-configurable timeout that isn't
* a provider attr. A function is resolved against the provider that actually
* runs (fallbacks included), for callers whose bound depends on its type:
* CLI/TUI treat it as a total cap, API runs as a no-progress (stall) bound.
* @param {number} [args.outputReserveTokens=8000] — expected completion budget
* included in fallback context admission. Callers with a tighter known
* output budget may lower it.
* @param {string} [args.cwd] — working directory for the spawned process.
* Defaults to `process.cwd()`. Callers that run AI against external
* directories (loops with `loop.cwd`, pm2Standardizer with a repo path)
* must pass this — without it, the CLI/TUI spawn lands in PortOS's own
* cwd and the analysis runs against the wrong files. No-op for API
* providers (no spawn).
* @param {string|string[]|object} [args.callerPolicy] — this caller's
* EXECUTION-MODE policy (a name from `lib/callerModePolicy.js`, a mode array,
* or an inline policy object). Set it when the call site genuinely cannot run
* every mode — a context with no PTY to drive a TUI, or one that must stay on
* a harness-free direct API provider. The policy gates the EXPLICIT provider
* (422 `PROVIDER_MODE_NOT_PERMITTED`, before any run record is written) and
* rides along on `requestCapabilities` so createRun's proactive swap and the
* Tier-3 retry fallback skip ineligible candidates too. Omit (the default) and
* nothing is constrained.
* @param {boolean} [args.allowFallback=true] — set false when provider/model
* identity is part of the feature contract. Disables proactive provider
* substitution and every model/provider retry tier after the first attempt.
* It does NOT disable provider-health bookkeeping: a failure that benches the
* provider still benches it, so the Providers page and every unpinned caller
* learn that this provider is sick. The one exception is a usage limit, whose
* observed-block ledger a pinned caller records itself.
* @param {boolean} [args.toolFree=false] — the prompt carries text this caller
* does not control (AI app detection, #9008). A headless CLI run then gets no
* approval-bypass flag and, where its vendor has one, its tool-disable switch
* (`toolFreeOneShotArgs` in lib/providerVendors.js). Rides every fallback tier.
* It does not reach a TUI run, whose interactive argv has no such posture —
* a caller that must stay tool-free gates on `isToolFreeOneShotProvider`
* and disables fallback.
* @param {*} [args.responseSchema] — the caller's declared response schema
* (issue #2350). A Zod-style schema (`.safeParse`/`.parse`) or a bare
* predicate `(parsedValue) => boolean`. When set, the runner enables Tier-2
* (schema/type): it validates a SUCCESSFUL run's response against the schema
* and, when it doesn't match, attempts a deterministic JSON coercion in place
* (strip fences/prose, pull the first schema-matching block); if that can't
* recover the shape it re-requests the SAME provider once with a
* schema-strengthened prompt before falling through to a Tier-3 fallback and
* Tier-4 investigation task. Left unset (the default), the runner behaves
* exactly as before — no validation, no coercion.
* @param {(ctx: { phase: 'response'|'request', text?: string, prompt?: string,
* category?: string, error?: Error }) => ({ text?: string, prompt?: string }|null|undefined)}
* [args.repair] — optional caller-owned deterministic normalizer (issue #2350).
* `phase:'response'` fires when a successful response fails `responseSchema`;
* return `{ text }` to substitute a corrected response (tried before the
* built-in JSON coercion). `phase:'request'` fires on a schema/type failure;
* return `{ prompt }` to re-request the SAME provider with a corrected prompt
* (tried before the built-in schema-strengthened prompt). Must be deterministic
* (no LLM calls); may be async. Errors are logged and treated as a decline.
* @returns {Promise<{ text: string, runId: string, model: string|null, provider: object, usedFallback?: boolean, fixTier?: number, fixStrategy?: string, coercedResponse?: boolean, fallbackFrom?: { id: string, name: string }, fallbackProvider?: object }>}
* — `model` is the resolved model that actually executed (null when
* neither override nor provider.defaultModel applies). `provider` is the
* provider object that actually ran, reflecting createRun's proactive swap
* (read this to detect a proactive fallback; `usedFallback`/`fallbackProvider`
* only cover the retry-fallback path). `runId` always
* points to the run record that *actually* ran — on fallback recovery
* this is the fresh fallback runId, NOT the failed primary's. When
* `usedFallback` is true, `fallbackFrom` identifies the failed primary
* and `fallbackProvider` is the full provider object that actually ran
* (so callers persisting run attribution can write the correct
* providerId without re-picking the fallback themselves). `fixTier` /
* `fixStrategy` (issue #2342/#2350) record WHICH fallback tier recovered the
* failure — `1`/`'config/env'` when a same-provider model correction
* recovered it, `2`/`'schema/type'` when the response was coerced in place or
* re-requested to match the caller's declared schema (`coercedResponse:true`
* flags the pure in-place coercion, which needs no retry — `usedFallback`
* stays unset there since the same provider succeeded), `3`/
* `'constrained-agent-retry'` for a fallback-provider retry — so callers can
* log/attribute the deterministic recovery path.
*/
export async function runPromptThroughProvider(rawArgs) {
return maintenance.run('provider', rawArgs?.source ?? 'prompt', () => runPromptThroughProviderAdmitted(rawArgs), { continuation: true });
}
async function runPromptThroughProviderAdmitted(rawArgs) {
// Validate inputs up front so an accidentally-null `provider` (or one
// missing `id`/`type`) surfaces a clear error here instead of throwing
// a downstream TypeError on `provider.id` inside createRun or on the
// provider.type dispatch below.
if (!rawArgs?.provider || typeof rawArgs.provider !== 'object') {
throw new Error('runPromptThroughProvider: provider is required');
}
if (typeof rawArgs.provider.id !== 'string' || !rawArgs.provider.id) {
throw new Error('runPromptThroughProvider: provider.id must be a non-empty string');
}
if (rawArgs.provider.type !== PROVIDER_TYPES.CLI && rawArgs.provider.type !== PROVIDER_TYPES.API && rawArgs.provider.type !== PROVIDER_TYPES.TUI) {
throw new Error(`Unsupported provider type: ${rawArgs.provider.type}`);
}
if (typeof rawArgs.prompt !== 'string' || !rawArgs.prompt.length) {
throw new Error('runPromptThroughProvider: prompt must be a non-empty string');
}
if (typeof rawArgs.source !== 'string' || !rawArgs.source.length) {
throw new Error('runPromptThroughProvider: source must be a non-empty string');
}
// Creative requests carry the IP-latitude clause (lib/creativeLatitude.js) so a
// model doesn't quietly genericize a reference the artist put there on purpose.
// Keyed on `source` — the one stable per-call-site tag every caller already
// has to pass — which covers every hand-rolled creative prompt from one place.
// Stage-template prompts are stamped upstream in promptService.buildPrompt; the
// stamp is idempotent, so a pre-stamped prompt passes through untouched.
const args = isCreativeRunSource(rawArgs.source)
? { ...rawArgs, prompt: withCreativeLatitude(rawArgs.prompt) }
: rawArgs;
// Execute once, then decide whether the deterministic fallback cascade runs.
// A transport-layer failure is caught into `firstError`. A transport SUCCESS
// whose response fails the caller's declared schema (Tier-2, issue #2350) is
// validated/coerced in place and returned; only an UNCOERCIBLE off-schema
// response synthesizes a schema/type `firstError` so it re-enters the SAME
// cascade (with the identical noteFallback* reconciliation) below.
let firstError;
{
let firstResult;
try {
// The local-endpoint concurrency gate lives INSIDE executeProviderRunOnce,
// keyed on the EFFECTIVE provider after createRun's proactive swap — so a
// remote/CLI primary that swaps to a local backend is still serialized.
// Gating here on the requested provider would miss that swap (the gate
// would no-op for a remote primary and the swapped-in local run would
// dispatch ungated).
firstResult = await executeProviderRunOnce(args);
} catch (err) {
firstError = err;
}
if (!firstError) {
const validated = await correctResponseToSchema(firstResult, args);
if (validated.ok) {
// Either the response already matched the schema (return as-is) or a
// deterministic in-place coercion recovered its shape (Tier-2, no retry).
return validated.coerced
? { ...validated.result, fixTier: 2, fixStrategy: 'schema/type', coercedResponse: true }
: validated.result;
}
firstError = buildSchemaFailureError(firstResult);
}
}
// Stops and caller spending ceilings carry no provider-health verdict.
// Neither enters correction/fallback, benching, or fault investigation.
if (isRunCanceledError(firstError) || isRuntimeBudgetError(firstError) || isContentRefusalError(firstError)) {
throw stripFallbackContext(firstError);
}
if (rawArgs.allowFallback === false) {
// A pinned caller takes no retry tier — but provider HEALTH is not routing.
// Skipping the mark here left a provider that had failed every call for
// hours still reading green on the Providers page, and every *other*
// (unpinned) caller still routing to it, because the only code path that
// benches lived inside the fallback cascade this caller opts out of.
//
// It also kept the pinned caller itself in a burn loop: the persistent mind
// is pinned, so ten consecutive runs each spent the provider's full
// configured timeout before failing. With the mark in place the next pinned
// call is refused by createRun in milliseconds ("unavailable … and fallback
// is disabled") until the cooldown expires, so a sick local backend costs
// one timeout per cooldown window instead of one per wake.
//
// `resolveProviderBench` still declines request-specific categories (a
// content refusal, a bad model id, an off-schema response), so a pinned
// caller's prompt-specific failure never benches a healthy provider.
//
// Only the `unavailable` markers. A usage limit is left exactly as it is
// today: `markProviderUsageLimit` writes the Usage page's observed-block
// ledger, and the pinned caller that cares already calls it for this same
// failure (persistentMindSupervisor autopauses and probes for recovery), so
// benching it here would record the one block twice.
if (firstError?.effectiveProvider) {
await markProviderUnavailableFromError(
firstError.effectiveProvider,
firstError.message,
firstError.errorAnalysis,
firstError.effectiveModel ?? null,
{ skipUsageLimit: true },
).catch((err) => {
console.error(`❌ markUnavailable failed for ${firstError.effectiveProvider.id}: ${err.message}`);
});
}
throw stripFallbackContext(firstError);
}
{
// Only retry when the failure came from the execution layer (annotated
// by safeReject with effectiveProvider) or is a synthetic schema/type
// failure (also annotated). Pre-execution throws — createRun rejecting on a
// disk error / disabled provider / unsupported type — never fire the
// AI_PROVIDER_EXECUTION_FAILED hook, so there's no deferred investigation
// task to suppress and marking the provider unavailable would punish it for
// a disk/config problem. Rethrow those as-is so the caller sees the original.
if (!firstError?.effectiveProvider) {
throw stripFallbackContext(firstError);
}
// The runner already wrote a failed metadata.json and the onRunFailed
// hook in server/index.js queued a deferred investigation task keyed on
// (providerName, model). We now walk the deterministic fallback cascade
// (issue #2342), attempting the CHEAPEST tier that can plausibly recover
// the failure and escalating to the queued investigation task (Tier 4)
// only when every deterministic tier declines or fails:
// Tier 1 config/env — retry the SAME provider with a valid model
// (deterministic; no persisted config change)
// Tier 2 schema/type — re-request the SAME provider with a schema-
// corrected prompt (issue #2350), when the caller
// declared a `responseSchema`/`repair`; else falls
// through (the in-place response coercion already
// ran on the success path above)
// Tier 3 constrained-agent-retry — bounded retry via a fallback provider
// Tier 4 escalate — the deferred investigation task the hook queued
//
// `failed` is the provider that ACTUALLY ran — usually equals
// rawArgs.provider, but createRun may have proactively swapped to a
// different fallback if the requested primary was already marked
// unavailable. Always dedupe against what actually ran so the
// task-suppression notifiers cancel the right queued task.
const failed = firstError.effectiveProvider;
const failedModel = firstError.effectiveModel || resolveEffectiveModel(failed, args.model);
const category = firstError.errorAnalysis?.category;
// The primary's queued investigation task is keyed on (providerName, model).
// Every tier that retries suppresses this key up front so a slow retry
// (a CLI fallback can take 20–30s) can't outrun the backstop timer
// (autoFixer.TASK_DEFER_MS) and leave a task behind for a recovered failure.
const primaryKey = { provider: failed.name || failed.id, model: failedModel };
const investigation = createRecoveryInvestigationScope({ failed, failedModel, firstError });
const stopOnTerminalRequest = async (error) => {
if (!isRuntimeBudgetError(error) && !isContentRefusalError(error)) return;
// Caller ceilings settle ownership without declaring a provider fault.
await investigation.recoverAll();
throw stripFallbackContext(error);
};
try {
// ── Tier 1 — config/env correction (issue #2342) ───────────────────────
// A model-not-found/model-not-supported failure is request-specific: the
// provider is reachable but doesn't serve the requested model id. Before
// escalating to a whole-provider fallback (Tier 3), attempt the cheapest
// deterministic fix — retry the SAME provider with a valid model it lists.
// No config is persisted; on success the user's own provider keeps serving.
//
// Gated on `providerHonorsModelOverride`: a CLI/TUI provider with a model
// flag baked into its args ignores the per-call `model`, so the "corrected"
// model would silently be the same failed one — a pointless retry that
// would also mis-key the corrected task onto the primary key. Skip Tier 1
// for those and let the cascade fall through to a real fallback provider.
// Only a wrong-model failure is config-correctable here. Compare against
// string literals, not `ERROR_CATEGORIES.MODEL_NOT_SUPPORTED` — that member
// does NOT exist (it would be `undefined`, matching a category-less failure
// and wrongly engaging Tier 1). 'model-not-supported' is autoFixer's own
// Tier-1 category (from CoS agent analysis), matched here for parity.
if ((category === ERROR_CATEGORIES.MODEL_NOT_FOUND || category === 'model-not-supported')
&& providerHonorsModelOverride(failed)) {
const correctedModel = pickConfigCorrectedModel(failed, failedModel);
if (correctedModel) {
console.log(`🔧 Tier 1 (config/env) retry: ${rawArgs.source} on ${failed.name} with model ${correctedModel} (requested ${failedModel} → ${category})`);
// Suppress the primary's investigation task while the corrected retry
// runs so a slow (>TASK_DEFER_MS) but SUCCESSFUL retry can't leave a
// task behind for a recovered failure.
await investigation.suppress(primaryKey);
let tier1Result;
try {
tier1Result = await executeProviderRunOnce({
...args,
provider: failed,
model: correctedModel,
runId: undefined, // fresh run so the failed primary's record stays intact
});
} catch (tier1Error) {
await stopOnTerminalRequest(tier1Error);
// The corrected retry failed. If it reached the execution layer, its
// onRunFailed queued a task keyed on the EFFECTIVE provider/model
// (createRun may have proactively swapped) — record and suppress that
// key too, so a slow Tier-3 recovery below can't let its backstop
// fire. A pre-execution throw (no effectiveProvider) queued NO task,
// so leave ownership with the primary for explicit escalation.
if (tier1Error?.effectiveProvider) {
await investigation.suppress({
provider: tier1Error.effectiveProvider.name || tier1Error.effectiveProvider.id,
model: tier1Error.effectiveModel || correctedModel,
});
}
console.log(`↪️ Tier 1 correction failed on ${failed.name} (${correctedModel}): ${tier1Error.message} — escalating to constrained-agent-retry`);
}
if (tier1Result) {
await investigation.recoverAll();
return {
...tier1Result,
usedFallback: true,
fixTier: 1, // FIX_TIERS.CONFIG_ENV (mirrored; kept decoupled from the lazy autoFixer import)
fixStrategy: 'config/env',
fallbackFrom: { id: failed.id, name: failed.name },
fallbackProvider: failed,
};
}
}
}
// ── Tier 2 — schema/type request correction (issue #2350) ──────────────
// Reached for a schema/type-category failure — the synthetic
// response-schema failure raised on the success path above, or an
// agent-analysis parse/bad-request/output-length category. When the caller
// declared a `responseSchema` (or supplied a `repair`), re-request the SAME
// provider once with a schema-corrected prompt before benching it for a
// Tier-3 fallback. Mirrors Tier 1: suppress the primary's task up front,
// retry once, cancel the task on a recovered + schema-valid response.
if (isSchemaTypeCategory(category)) {
const correctedPrompt = await buildSchemaCorrectedPrompt({
prompt: rawArgs.prompt,
responseSchema: args.responseSchema,
repair: args.repair,
category,
error: firstError,
});
if (correctedPrompt) {
console.log(`🧩 Tier 2 (schema/type) retry: ${rawArgs.source} on ${failed.name} (category ${category})`);
await investigation.suppress(primaryKey);
let tier2Result;
try {
tier2Result = await executeProviderRunOnce({
...args,
provider: failed,
prompt: correctedPrompt,
runId: undefined, // fresh run so the failed primary's record stays intact
});
} catch (tier2Error) {
await stopOnTerminalRequest(tier2Error);
// Same bookkeeping as Tier 1's failed corrected retry: a retry that
// reached the execution layer queued its own task keyed on the
// EFFECTIVE provider/model — record + suppress it so a slow Tier-3
// recovery can't let its backstop fire. A pre-execution throw queued
// no task, so leave ownership with the primary for explicit escalation.
if (tier2Error?.effectiveProvider) {
await investigation.suppress({
provider: tier2Error.effectiveProvider.name || tier2Error.effectiveProvider.id,
model: tier2Error.effectiveModel || failedModel,
});
}
console.log(`↪️ Tier 2 correction failed on ${failed.name}: ${tier2Error.message} — escalating to constrained-agent-retry`);
}
if (tier2Result) {
// The re-requested response must ALSO satisfy the schema (or coerce)
// — a retry that returns fresh-but-still-malformed output is not a
// recovery. On success cancel the primary task; otherwise fall
// through to the Tier-3 fallback provider.
const revalidated = await correctResponseToSchema(tier2Result, args);
if (revalidated.ok) {
await investigation.recoverAll();
return {
...revalidated.result,
usedFallback: true,
fixTier: 2, // FIX_TIERS.SCHEMA_TYPE (mirrored; kept decoupled from the lazy autoFixer import)
fixStrategy: 'schema/type',
fallbackFrom: { id: failed.id, name: failed.name },
fallbackProvider: failed,
};
}
console.log(`↪️ Tier 2 re-request still off-schema on ${failed.name} — escalating to constrained-agent-retry`);
}
}
}
// ── Tier 3 — constrained-agent-retry via a fallback provider ───────────
// Coalesce the per-provider mark-and-pick during an N-way failure storm:
// the 2nd…Nth simultaneous failure for the same provider awaits the first's
// result instead of independently re-reading providers.json
// (pickFallbackProvider) and re-writing provider-status.json
// (markUnavailable). The fallback *run* below is NOT coalesced — each failed
// call still executes its own fallback to get its own result.
const picked = await coalesceFallbackMarkAndPick(failed, firstError, buildRequestCapabilities(args));
if (!picked) {
// ── Tier 4 — escalate ──: no recovery path left. Every attempted key's
// incidental task was suppressed, so escalate exactly one investigation
// task explicitly (honors the circuit breaker), then release the keys.
await investigation.escalateAndRelease();
throw stripFallbackContext(firstError);
}
const fallback = picked.provider;
console.log(`⚡ Tier 3 (constrained-agent-retry): ${rawArgs.source} with fallback ${fallback.name} (primary ${failed.name} failed: ${firstError.message})`);
// Suppress the primary key (idempotent across tiers) for the fallback run.
await investigation.suppress(primaryKey);
// Run the fallback as a fresh attempt. Pass the configured `fallbackModel`
// when one is set (so the user's chosen fallback provider+model pair is
// honored); otherwise `undefined` lets the fallback pick its own default.
// Either way we never inherit the primary's model id, which usually
// doesn't exist on the fallback.
let fallbackResult;
try {
// The fallback is gated inside executeProviderRunOnce on its effective
// provider, so a fail-over that lands on a local backend still serializes
// against that endpoint (the failure-storm case) — with no double-gate
// (wrapping here too would deadlock at MAX_CONCURRENCY=1).
fallbackResult = await executeProviderRunOnce({
...args,
provider: fallback,
model: picked.model ?? undefined,
// A step in the global fallback list may pin its own effort; the
// primary's effort belongs to the primary's model, so it is not inherited.
...(picked.effort ? { effort: picked.effort } : {}),
runId: undefined, // fresh runId so the failed primary's record stays intact
});
} catch (fallbackError) {
await stopOnTerminalRequest(fallbackError);
// ── Tier 4 — escalate ──: every deterministic tier failed. If the
// fallback reached the execution layer, its OWN onRunFailed already
// queued an investigation task (its key was never suppressed), so just
// release the suppressed keys — one surviving task. But if the fallback
// threw BEFORE execution (e.g. createRun error), no task was queued and
// the suppressed primary/corrected keys would leave ZERO tasks — so
// escalate one explicitly first (deduped inside escalateProviderFailure).
if (fallbackError?.effectiveProvider) {
await investigation.releaseForExistingInvestigation();
} else {
await investigation.escalateAndRelease();
}
throw stripFallbackContext(fallbackError);
}
// A fallback provider can ALSO return an off-schema response (issue #2350)
// — validate/coerce it before declaring recovery. If it can't be coerced,
// no deterministic tier is left: escalate one investigation task and give
// up (a fresh-but-unusable fallback response is not a recovery).
const fallbackValidated = await correctResponseToSchema(fallbackResult, args);
if (!fallbackValidated.ok) {
await investigation.escalateAndRelease();
throw stripFallbackContext(buildSchemaFailureError(fallbackResult));
}
// Fallback succeeded — cancel EVERY suppressed key (the primary and any
// Tier-1/Tier-2 corrected-retry key) so a fully-recovered action creates
// ZERO investigation tasks (issue #2342 acceptance: only UNRECOVERED
// failures escalate to Tier 4).
await investigation.recoverAll();
return {
...fallbackValidated.result,
usedFallback: true,
fixTier: 3, // FIX_TIERS.CONSTRAINED_RETRY
fixStrategy: 'constrained-agent-retry',
fallbackFrom: { id: failed.id, name: failed.name },
fallbackProvider: fallback,
};
} finally {
await investigation.closeUnexpectedFailure();
}
}
}
// In-flight mark-and-pick work, keyed by the failed provider id. During an
// N-way failure storm (one stuck provider, many simultaneous in-flight
// calls) every failure would otherwise independently re-read providers.json
// (pickFallbackProvider) and re-write provider-status.json (markUnavailable).
// Sharing the first failure's promise collapses that to a single read +
// single write. The slot clears as soon as it settles, so a genuinely new
// failure after the storm re-picks against fresh provider status.
const _fallbackMarkAndPick = createSingleFlight();
/**
* Mark the failed provider unavailable and pick its fallback, coalescing
* concurrent calls for the same provider id onto one shared promise.
* Resolves to the same `{ provider, model }` shape as `pickFallbackProvider`
* (or null when no usable fallback exists). The mark is skipped when no
* fallback is available — matching the prior ordering where the original
* error is rethrown without benching a provider that has no recovery path.
*
* @param {object} failed — the provider that actually ran and failed
* @param {Error} firstError — the execution error (carries message + errorAnalysis)
* @returns {Promise<{ provider: object, model: string|null }|null>}
*/
function coalesceFallbackMarkAndPick(failed, firstError, requestCapabilities) {
const capabilityKey = `${requestCapabilities?.hasImages === true ? 'images' : 'text'}:${requestCapabilities?.requiredContextTokens || 0}`;
return _fallbackMarkAndPick.run(`${failed.id}:${capabilityKey}`, async () => {
const picked = await pickFallbackProvider(failed, requestCapabilities);
if (!picked) return null;
await markProviderUnavailableFromError(failed, firstError.message, firstError.errorAnalysis, firstError?.effectiveModel ?? null).catch(err => {
console.error(`❌ markUnavailable failed for ${failed.id}: ${err.message}`);
});
return picked;
});
}
/**
* Pick a fallback provider for `failed`. Honors the failed provider's
* `fallbackProvider` field first, then the toolkit's system priority
* list. Returns `{ provider, model }` (or null when no usable fallback
* exists). `model` is the configured `fallbackModel` hint for the chosen
* fallback (null for system-priority picks, meaning "use the fallback's
* own default") — never the failed provider's model.
*
* The toolkit's `getFallbackProvider` reads `providers[failed.id]` to
* look up the `fallbackProvider` field, so the primary MUST stay in the
* map. Self-loop protection (`fallbackProvider === self`) lives in
* `getFallbackProvider`; the system priority loop already excludes
* `failed.id` by construction.
*/
async function pickFallbackProvider(failed, requestCapabilities) {
const toolkit = getAIToolkitInstance();
const providerStatus = toolkit?.services?.providerStatus;
if (!providerStatus) return null;
const all = await getAllProviders().catch(() => null);
if (!all?.providers) return null;
const providersMap = {};
for (const p of all.providers) providersMap[p.id] = p;
// A composite (`harness.method@service`, #7564) never sits in the stored map;
// the failed provider's own id may be one, and it must be in the map for the
// toolkit's self-loop guard and `fallbackProvider` read to see it. The
// toolkit's boot-registered resolver already materializes composite ids;
// reusing it keeps the host resolver graph out of request-path CI selection.
if (isCompositeProviderId(failed.id) && !providersMap[failed.id]) {
const record = await getProviderById(failed.id);
if (record) providersMap[failed.id] = record;
}
const chainComposites = peekFallbackChain().map((e) => e.providerId).filter(isCompositeProviderId);
for (const id of chainComposites) {
if (providersMap[id]) continue;
const record = await getProviderById(id);
if (record) providersMap[id] = record;
}
const picked = providerStatus.getFallbackProvider(failed.id, providersMap, null, null, requestCapabilities);
if (!picked?.provider) return null;
// The pick comes from the RAW provider map (`getAllProviders`), which carries
// no gateway-inherited key — a gateway-backed wrapper picked here would execute
// without its sibling's apiKey and fail auth (NVIDIA NIM 401s "Header of type
// `authorization` was missing") despite a stored key. Attach it from the same
// map (synchronous — no re-read to go stale).
return { provider: attachGatewaySiblingKey(picked.provider, providersMap), model: picked.model ?? null, effort: picked.effort ?? null };
}
/**
* Translate the runner's failure into an availability marker on the
* failed provider. Reuses the runner's precomputed error analysis when
* present, otherwise falls back to analyzeError to categorize and pick a
* cooldown; usage-limit failures route through `markUsageLimit` so the
* parsed wait time (e.g. "reset 5pm") is honored.
*
* Skips the mark when the toolkit's `executeApiRun` has already marked
* the provider unavailable inline (it does this for RATE_LIMIT and
* USAGE_LIMIT before firing onComplete) — re-marking would double-
* increment `failureCount` and re-write the status file for no gain.
*
* `skipUsageLimit` declines the usage-limit marker entirely (bench AND ledger),
* for a caller that records that marker itself. Everything else still benches.
*/
async function markProviderUnavailableFromError(failed, errorMessage, runnerAnalysis, runModel = null, { skipUsageLimit = false } = {}) {
const toolkit = getAIToolkitInstance();
const providerStatus = toolkit?.services?.providerStatus;
if (!providerStatus) return;
const analysis = runnerAnalysis && typeof runnerAnalysis === 'object'
? runnerAnalysis
: analyzeError(errorMessage || '');
// Request/response-specific categories (a content refusal, a bad model id, an
// off-shape response) resolve to null — the provider is healthy and the single
// failing call still falls back via the caller's retry path.
const bench = resolveProviderBench(analysis);
if (!bench) return;
if (bench.marker === 'usage-limit') {
// `skipUsageLimit` callers own this marker themselves — see the pinned
// branch in runPromptThroughProvider. Returning before the ledger write is
// the point: recordLimitBlock is deliberately NOT deduped (a repeated
// refusal refreshes "since last block"), so a second call for the same
// failure would show up as a second observed block.
if (skipUsageLimit) return;
// Observed-block ledger for the Usage page's free-tier estimated-quota
// signal — recorded even when the provider is already benched (a repeated
// refusal refreshes "since last block"). Transient 429 retries never reach
// this branch: they classify as `rate-limit`, not `usage-limit`.
await import('./usage.js')
.then(({ recordLimitBlock }) => recordLimitBlock({
providerId: failed.id,
model: runModel,
category: 'usage-limit',
message: bench.message || errorMessage,
resetHint: bench.waitTime
}))
.catch((err) => {
console.error(`❌ Failed to record limit block for ${failed.id}: ${err.message}`);
});
if (!providerStatus.isAvailable(failed.id)) return;
await providerStatus.markUsageLimit(failed.id, {
message: bench.message || errorMessage,
waitTime: bench.waitTime,
});
return;
}
if (!providerStatus.isAvailable(failed.id)) return;
await providerStatus.markUnavailable(failed.id, {
reason: bench.category,
message: bench.message || errorMessage || `Provider ${failed.name || failed.id} failed`,
waitTimeMs: bench.waitTimeMs,
});
}
// Strip the fallback-context fields off an error before rethrowing so
// callers don't see internal metadata they didn't ask for. The
// `.effectiveProvider`/`.effectiveModel` annotations exist solely for
// runPromptThroughProvider's retry path.
function stripFallbackContext(err) {
if (err && typeof err === 'object') {
delete err.effectiveProvider;
delete err.effectiveModel;
}
return err;
}
/**
* Whether `provider` generates on a local inference server — an API provider
* served by an Ollama daemon, flagged as llama.cpp / LM Studio / MTPLX / vLLM /
* SGLang, or pointed at a loopback endpoint. These are the windows a cloud-sized
* output reserve can swallow whole.
*/
function isLocalInferenceProvider(provider) {
if (provider?.type !== PROVIDER_TYPES.API) return false;
return isOllamaBackedProvider(provider)
|| provider.llamaBacked === true
|| provider.lmstudioBacked === true
|| provider.mtplxBacked === true
|| provider.vllmBacked === true
|| provider.sglangBacked === true
|| isLocalEndpoint(provider.endpoint);
}
/**
* The output reserve (and generation cap) for a run on a LOCAL model.
*
* A caller's declared `outputReserveTokens` is a cloud-sized number (the
* persistent mind declares 8,192). On a 16K local daemon that reserve alone is
* half the window, so since #9437 made the mind declare it on every wake, a
* ~13.5K-token prompt that fits was refused before dispatch ("known 16384-token
* context is below the 21744-token request budget"). For a local provider with
* a KNOWN window the reserve shrinks to `localOutputReserveTokens` (floor 2,048,
* ⅛ of the window, never above what was declared), and the request's
* `max_tokens` / `num_predict` is capped at the same number so generation
* cannot outgrow the room the gate planned for.
*
* Unchanged when the caller named its own `maxTokens` (it sized the answer
* deliberately), declared no reserve, the window is unknown, or the provider
* is not local.
*
* @returns {Promise<{ outputReserveTokens: number|undefined, maxTokens: number|undefined }>}
*/
async function resolveLocalOutputBudget(provider, model, { outputReserveTokens, maxTokens }) {
const unchanged = { outputReserveTokens, maxTokens };
if (Number.isInteger(maxTokens) && maxTokens > 0) return unchanged;
if (!Number.isFinite(Number(outputReserveTokens)) || !isLocalInferenceProvider(provider)) return unchanged;
const observed = withOllamaRuntimeContextWindow(await withObservedContextWindowsLazy(provider));
const reserve = localOutputReserveTokens(outputReserveTokens, knownContextWindow(observed, model));
return reserve ? { outputReserveTokens: reserve, maxTokens: reserve } : unchanged;
}
/**
* The prompt room a LOCAL provider leaves once its output reserve is planned,
* for callers that build their own prompt and want to fit it rather than be
* refused (the persistent mind's tool rounds). Uses the same window lookup and
* the same `localOutputReserveTokens` sizing as the dispatch gate, so a prompt
* kept under `maxPromptTokens` (chars/4) is one the gate admits.
*
* Returns `null` for a cloud / CLI provider, an undeclared reserve, or an
* unknown window — callers then build exactly as before.
*
* @param {{ provider: object, model?: string|null, outputReserveTokens: number }} args
* @returns {Promise<{ contextWindow: number, outputReserveTokens: number, maxPromptTokens: number }|null>}
*/
export async function resolveLocalPromptBudget({ provider, model = null, outputReserveTokens }) {
if (!isLocalInferenceProvider(provider)) return null;
const resolvedModel = model || provider?.defaultModel || null;
const observed = withOllamaRuntimeContextWindow(await withObservedContextWindowsLazy(provider));
const contextWindow = knownContextWindow(observed, resolvedModel);
const reserve = localOutputReserveTokens(outputReserveTokens, contextWindow);
if (!reserve) return null;
return { contextWindow, outputReserveTokens: reserve, maxPromptTokens: Math.max(0, contextWindow - reserve) };
}
/**
* The budget a pre-dispatch REFUSAL is allowed to act on.
*
* `requiredContextTokens` is the prompt plus an output reserve, and when the
* caller named no reserve that is `DEFAULT_OUTPUT_RESERVE_TOKENS` — 8,000
* tokens nobody asked for. Ranking fallback candidates on that optimistic
* budget is fine (it prefers a provider with room to answer), but REFUSING on
* it turns "provably cannot fit" into "probably cannot fit": a pinned caller
* sending a 26K-token prompt to a 32K endpoint and expecting three sentences
* back would be refused, though it succeeds today — these daemons size the
* answer from what the window has left.
*
* So a refusal uses the provable floor — the prompt alone — unless the caller
* declared its own `outputReserveTokens`, which is it telling us how much room
* the answer genuinely needs. Candidate ranking in `getFallbackProvider` keeps
* using the full budget, exactly as it did before this gate existed.
*/
function refusalCapabilities({ requestCapabilities, prompt, outputReserveTokens }) {
if (Number.isFinite(Number(outputReserveTokens))) return requestCapabilities;
return { ...requestCapabilities, requiredContextTokens: estimateTokens(prompt) };
}
/**
* Refuse a prompt that provably cannot fit the provider about to run it.
*
* Before #7441 the context check ran ONLY inside `getFallbackProvider`, so it
* judged the candidates a failure had already escalated to and never the
* provider on the happy path — see `services/observedContextWindows.js` for the
* stall that cost.
*
* Four rules, all deliberate:
*
* - An UNKNOWN window is not a rejection. `contextWindowRejection` answers
* `null` whenever nothing declared a window, and nothing infers one from a
* model id — so a provider PortOS has no data about dispatches exactly as
* it does today.
* - What it compares is `refusalCapabilities`, not the raw routing budget:
* a reserve the caller never asked for must not manufacture a refusal.
* - The refusal is an ORDINARY failure, not a new category or exception class.
* `context-length` is what `agentErrorAnalysis.js` and `codexTurn.js`
* already tag an oversized prompt, and it is request-specific
* (`providerCooldown.isSchemaTypeCategory`), so it never benches a server
* that is perfectly healthy for smaller prompts, and `autoFixer` already
* knows its tier. It carries `effectiveProvider`, so the cascade in
* `runPromptThroughProvider` routes it to a fallback with a wider window
* (Tier 3, which applies this same rule to every candidate). A pinned
* (`allowFallback: false`) caller gets the message in milliseconds instead
* of the ten-minute stall — which is the whole point.
* - It judges the REQUESTED provider, ahead of `createRun`'s proactive swap.
* A provider that swap lands on was already asked the same question by
* `capabilityRejection`, so the only cost is a cached listing spent on a
* primary that turns out to be benched.
*/
async function assertRequestFitsContext(provider, model, requestCapabilities, { runId = null, startTime = null } = {}) {
const required = Number(requestCapabilities?.requiredContextTokens);
if (!Number.isFinite(required) || required <= 0) return;
// Two in-memory projections, both describing what the daemon is doing RIGHT
// NOW rather than what the record remembers: the windows it is serving, and
// the ceiling PortOS launched it at — including the ambient
// `OLLAMA_CONTEXT_LENGTH` rung the toolkit cannot reach for itself (#7472).
const observed = withOllamaRuntimeContextWindow(await withObservedContextWindowsLazy(provider));
// `reason` already names both numbers, which is what the operator needs to
// decide between a smaller prompt and a wider provider — so it IS the
// message rather than being re-worded beside it.
const reason = contextWindowRejection(observed, model, requestCapabilities);
if (!reason) return;
const message = `${provider.name || provider.id}${model ? ` (${model})` : ''}: ${reason}`;
// A caller that created its own run record (stageRunner, the loops) has one
// sitting at `success: null, endTime: null`. Throwing past it leaves the Runs
// page showing an attempt that never ends and every `success === null`
// reconciler seeing a phantom active run, so settle it here — the same thing
// the TUI readiness check below does for its own pre-execution refusal.
if (runId) {
await finalizeRunRecord({ runId, output: '', exitCode: 1, success: false, error: message, startTime });
}
const err = new Error(message);
// The cascade only retries a failure annotated with the provider that ran
// (see runPromptThroughProvider) — without these the refusal would be
// rethrown straight to the caller and a wider fallback would never be tried.
err.effectiveProvider = provider;
err.effectiveModel = model;
err.errorAnalysis = { category: CONTEXT_LENGTH_CATEGORY, message };
console.log(`📏 Refused before dispatch: ${message}`);
throw err;
}
/**
* Inner helper: execute one attempt against `rawArgs.provider`. Returns
* { text, runId, model } on success. On failure, throws an Error with
* `effectiveProvider` and `effectiveModel` attached so the retry path
* knows which provider actually ran (createRun may have proactively
* swapped to a fallback when the requested provider was already marked
* unavailable).
*/
async function executeProviderRunOnce({
provider,
prompt,
source,
model,
effort = null,
runId: callerRunId,
onData: onDataCallback,
onRunCreated,
onRunReady,
onRunSettled,
beforeExecute,
timeout: timeoutOverride,
absoluteTimeoutMs,
maxTokens,
cwd: cwdOverride,
screenshots = [],
outputReserveTokens,
callerPolicy = null,
allowFallback = true,
toolFree = false,
}) {
// Caller EXECUTION-MODE policy, enforced on the EXPLICIT provider before a run
// record exists and carried into fallback selection below, so the pin and the
// route that might replace it are judged by one rule.
const pinRejection = callerPolicy ? callerModeRejection(provider, callerPolicy) : null;
if (pinRejection) {
throw new ServerError(
`Provider "${provider.id}" ${pinRejection.reason} — this caller cannot run it.`,
{ status: 422, code: 'PROVIDER_MODE_NOT_PERMITTED' },
);
}
if (screenshots.length > 0
&& provider.type !== PROVIDER_TYPES.API
&& !isVisionCapableCliProvider(provider)
&& !isVisionCapableCodexTuiProvider(provider)) {
throw new ServerError(
'The selected provider cannot receive image attachments. Choose Codex, Claude Code, or a vision API provider.',
{ status: 422, code: 'VISION_PROVIDER_UNSUPPORTED' },
);
}
// Resolve the model that'll actually run BEFORE creating the run record
// so the record reflects reality. resolveEffectiveModel handles both
// the override-honored fallback chain AND the args-baked-CLI case
// (extract the args-pinned model id rather than logging defaultModel).
let effectiveProvider = provider;
let effectiveModel = resolveEffectiveModel(effectiveProvider, model);
// `??` only catches null/undefined; an empty-string override would leak
// through to spawn() and resolve to `/`. Match tuiPromptRunner.js's
// string-truthy gate so an empty override falls back to process.cwd().
const effectiveCwd = (typeof cwdOverride === 'string' && cwdOverride) ? cwdOverride : process.cwd();
// Built here rather than inside the createRun branch below because the
// context gate needs it on EVERY attempt — a caller that supplies its own
// runId (stageRunner, the loops) skips that branch entirely and would
// otherwise dispatch ungated.
const localBudget = await resolveLocalOutputBudget(provider, effectiveModel, { outputReserveTokens, maxTokens });
const requestCapabilities = buildRequestCapabilities({ prompt, screenshots, outputReserveTokens: localBudget.outputReserveTokens, callerPolicy });
await assertRequestFitsContext(
effectiveProvider,
effectiveModel,
refusalCapabilities({ requestCapabilities, prompt, outputReserveTokens: localBudget.outputReserveTokens }),
{ runId: callerRunId, startTime: Date.now() },
);
// Some call sites (stageRunner, loops) create the run themselves so
// they can log the runId before the LLM call starts. When provided,
// reuse it. Otherwise create one here so callers always get a runId
// back. Pass `workspacePath` so /runs metadata reflects the directory
// the spawn ran in.
//
// When we create the run ourselves, capture the FULL result — the
// toolkit's createRun may switch to a fallback provider when the
// requested one is unavailable (providerStatusService), and
// `runResult.provider` is the effective provider after that switch.
// Dispatch must use the fallback (otherwise we'd execute against the
// unavailable provider while the run record claims the fallback ran)
// and `effectiveModel` must be re-resolved against the fallback's
// defaults so the response value reflects what actually ran.
let runId = callerRunId;
if (!runId) {
const runResult = await createRun({
providerId: provider.id,
model: effectiveModel,
prompt,
source,
workspacePath: effectiveCwd,
effort,
requestCapabilities,
allowFallback,
});
runId = runResult.runId;
if (runResult.provider && runResult.provider.id !== provider.id) {
effectiveProvider = runResult.provider;
// Re-resolve against the FALLBACK provider using the configured
// `fallbackModel` createRun surfaced — NOT the caller's `model`, which
// was resolved against the (now-benched) primary and almost never
// exists on the fallback. Forwarding it is the leak that sent
// `codex-configured-default` to LM Studio (mirrors stageRunner.js).
effectiveModel = resolveEffectiveModel(effectiveProvider, runResult.fallbackModel ?? null);
// createRun persisted `metadata.model = effectiveModel || provider.defaultModel`
// using the ORIGINAL provider's resolved value — so /runs would
// attribute a model that doesn't belong to the fallback (e.g. an
// API model id recorded on a CLI fallback). Await the patch so a
// fast-failing run can't fire onRunFailed with stale model/provider
// info — `noteFallbackHandled` keys on metadata.providerName +
// metadata.model and would miss if the patch hadn't landed yet.
await patchRunMetadata(runId, {
model: effectiveModel,
providerId: effectiveProvider.id,
providerName: effectiveProvider.name,
}).catch(() => { /* best-effort; metadata patch is not load-bearing */ });
}
}
// Lifecycle hooks let a parent orchestrator expose and stop the concrete
// provider run that is active inside a larger workflow. They fire for the
// caller-supplied primary run AND for every fresh correction/fallback run.
// Hook failures are observational only and must not break provider work.
if (typeof onRunCreated === 'function') {
try {
onRunCreated(runId, {
runId,
providerId: effectiveProvider.id,
providerName: effectiveProvider.name || effectiveProvider.id,
model: effectiveModel,
providerType: effectiveProvider.type,
});
} catch (err) {
console.error(`❌ run lifecycle start hook failed: ${err.message}`);
}
}
// Compute timeout AFTER the possible fallback switch so it reflects
// the provider that actually runs — providers can have wildly different
// `timeout` settings (a 5-min CLI vs a 30-s API), and using the
// original provider's timeout against a fallback would either time out
// a still-working run early or let a stuck one hang past its
// intended cap.
const resolvedTimeoutOverride = typeof timeoutOverride === 'function' ? timeoutOverride(effectiveProvider) : timeoutOverride;
const effectiveTimeout = resolvedTimeoutOverride ?? effectiveProvider?.timeout ?? DEFAULT_TIMEOUT_MS;
// Gate concurrent in-flight calls to a LOCAL endpoint on the EFFECTIVE
// provider — the one that ACTUALLY runs after createRun's proactive swap
// (above) and any retry-fallback. This is the single chokepoint the
// module comment promises: gating the *requested* provider at the call
// site missed a remote/CLI primary that swaps/fails over to a local
// backend, defeating the VRAM/OOM serialization. Non-local CLI/TUI providers
// remain unconstrained.
const attemptStartedAt = Date.now();
return withLocalConcurrencyGate(effectiveProvider, async () => {
// API providers run through the shared toolkit readiness hook. TUI
// providers spawn OpenCode directly, so they need the same hook here or
// MTPLX remains stopped until after the TUI has already failed its request.
if (effectiveProvider.type === PROVIDER_TYPES.TUI) {
const ready = await import('./providerExecutionReadiness.js')
.then(({ ensureProviderReadyForExecution }) => ensureProviderReadyForExecution(effectiveProvider))
.catch((err) => ({
success: false,
error: err.message,
}));
if (!ready.success) {
const message = ready.error || 'Provider readiness check failed';
await finalizeRunRecord({
runId,
output: '',
exitCode: 1,
success: false,
error: message,
startTime: attemptStartedAt,
});
const error = new Error(message);
error.effectiveProvider = effectiveProvider;
error.effectiveModel = effectiveModel;
throw error;
}
}
return new Promise((resolve, reject) => {
let text = '';
let settled = false;
let apiTimeoutHandle = null;
let dispatchRefusal = null;
const admitDispatch = typeof beforeExecute === 'function' ? async () => {
try {
await beforeExecute({ runId, provider: effectiveProvider, model: effectiveModel });
} catch (error) {
dispatchRefusal = error instanceof Error ? error : new Error(String(error));
dispatchRefusal.canceled = true;
throw dispatchRefusal;
}
} : undefined;
const safeResolve = (value) => { if (!settled) { settled = true; if (apiTimeoutHandle) clearTimeout(apiTimeoutHandle); resolve(value); } };
const safeReject = (err) => {
if (settled) return;
settled = true;
if (apiTimeoutHandle) clearTimeout(apiTimeoutHandle);
// Annotate so runPromptThroughProvider's retry path knows which
// provider actually ran (createRun may have swapped to a fallback
// before this attempt) and which model is the dedupe key for
// suppressing the queued investigation task on a successful retry.
if (err && typeof err === 'object') {
err.effectiveProvider = effectiveProvider;
err.effectiveModel = effectiveModel;
}
reject(err);
};
// TUI runs discard `text` and use `result.text` from executeTuiRun (see
// onComplete below), so the per-chunk APPEND_CHUNK concat is pure waste —
// TUI streams can emit hundreds of KB of screen redraws per run.
const isTui = effectiveProvider.type === PROVIDER_TYPES.TUI;
const onData = (chunk) => {
if (!isTui) text = APPEND_CHUNK(text, chunk);
if (onDataCallback) {
const chunkText = typeof chunk === 'string' ? chunk : (chunk?.text || '');
if (chunkText) onDataCallback(chunkText);
}
};
// Strictest discriminator: reject on either truthy `error` OR
// explicit `success === false`. Per-site drift was the bug.
const labelByType = { cli: 'CLI', api: 'API', tui: 'TUI' };
const onComplete = (result) => {
if (dispatchRefusal) { safeReject(dispatchRefusal); return; }
if (result?.error || result?.success === false) {
const err = new Error(result?.error || `${labelByType[effectiveProvider.type] || effectiveProvider.type} execution failed`);
if (result?.canceled === true) {
err.code = 'RUN_CANCELED';
err.canceled = true;
}
if (result?.errorAnalysis && typeof result.errorAnalysis === 'object') {
err.errorAnalysis = result.errorAnalysis;
}
for (const key of ['timeoutBound', 'timeoutOrigin', 'runtimeBudgetMs']) {
if (result?.[key] !== undefined) err[key] = result[key];
}
if (isRuntimeBudgetError(result)) {
err.code = 'RUN_RUNTIME_BUDGET_EXHAUSTED';
err.runId = runId;
}
safeReject(err);
} else {
// TUI runs do their own cleanup inside executeTuiRun (preferring
// the response file the model was directed to write, falling back
// to cleanTuiResponse on the screen scrape). Trust `result.text`
// — the accumulated `text` here is the raw chrome-laden stream.
// Codex CLI likewise supplies stdout as its authoritative final text;
// its diagnostic stream includes prompt examples that are not answers.
const finalText = effectiveProvider.type === PROVIDER_TYPES.TUI
? (typeof result?.text === 'string' ? result.text : '')
: effectiveProvider.type === PROVIDER_TYPES.CLI && typeof result?.text === 'string'
? result.text
: text;
// Report the provider that ACTUALLY ran. `effectiveProvider` reflects
// createRun's proactive swap (when the requested provider was already
// benched) — distinct from the retry-fallback path, which the outer
// runPromptThroughProvider annotates separately. Callers that care
// about the effective provider's type (e.g. vision, which only works
// on API providers) must read this, since a proactive swap leaves
// `usedFallback`/`fallbackProvider` unset.
safeResolve({ text: finalText, runId, model: effectiveModel, provider: effectiveProvider,
...(result?.finishReason ? { finishReason: result.finishReason } : {}),
});
}
};
// executeCliRun / executeTuiRun both read `provider.defaultModel` for
// arg construction AND the run-started metadata hook. Hand them a
// clone with effectiveModel pinned so a per-call model override
// actually picks up (and the hook reports the right model). The
// guard skips the clone when effectiveModel already equals
// provider.defaultModel — typical for non-codex CLI providers where
// resolveEffectiveModel falls through to defaultModel anyway.
//
// A per-call reasoning-effort override rides the same clone (there is no
// `effort` argument on executeCliRun/executeTuiRun): `buildCliArgs` and
// `buildTuiInvocation` both read `provider.effort`. It is a no-op for
// providers with no effort control, so a caller that always passes one
// can't corrupt an opencode/grok/API run.
const needsModelPin = effectiveModel && effectiveModel !== effectiveProvider.defaultModel;
const providerForRun = (needsModelPin || effort)
? {
...effectiveProvider,
...(needsModelPin ? { defaultModel: effectiveModel } : {}),
...(effort ? { effort } : {}),
}
: effectiveProvider;
// `withGatewayApiKey` attaches a gateway-backed wrapper's sibling key as a
// NON-ENUMERABLE property (so it never persists or leaks) — which the spread
// above drops. A model-pinned or effort-overridden CLI/TUI run would then go
// out with no Authorization header (NVIDIA NIM 401s "Header of type
// `authorization` was missing") despite a stored key. Re-carry it with the
// same enumerability so the clone executes with identical credentials.
if (providerForRun !== effectiveProvider && !providerForRun.apiKey && effectiveProvider?.apiKey) {
Object.defineProperty(providerForRun, 'apiKey', {
value: effectiveProvider.apiKey,
enumerable: false,
configurable: true,
});
}
if (effectiveProvider.type === PROVIDER_TYPES.CLI) {
executeCliRun({ runId, provider: providerForRun, prompt, workspacePath: effectiveCwd, screenshots, onData, onComplete, timeout: effectiveTimeout, toolFree, ...(admitDispatch ? { beforeExecute: admitDispatch } : {}) }).catch(safeReject);
} else if (effectiveProvider.type === PROVIDER_TYPES.API) {
// API runs take model as a first-class arg — no clone needed. The
// toolkit's executeApiRun now owns the primary wall-clock timeout (it
// aborts AND finalizes the run as an ERROR_CATEGORIES.TIMEOUT, firing
// onComplete → our reject below with a proper timeout classification).
// We keep a SECONDARY backstop timer here — delayed past the runner's
// own deadline by a grace window — purely for the pathological case
// where the runner neither throws nor ever calls onComplete (e.g. a
// future override): it best-effort stops the run and rejects so callers
// (meatspacePostLlm, pm2Standardizer, brain) never hang indefinitely.
// The grace lets the runner's timeout win classification in the normal
// path; safeReject/safeResolve clear this timer once onComplete lands.
//
// Armed against the runner's ABSOLUTE bound, not `effectiveTimeout`.
// Since #7560 `effectiveTimeout` is the runner's re-armable NO-PROGRESS
// bound — a run that keeps streaming legitimately outlives it, up to the
// 30-minute total-runtime cap. This timer still fired at
// `effectiveTimeout + 2s` and killed it anyway, which put back the exact
// ceiling #7560 removed and made the new cap unreachable. Worse, it kills
// via `stopRun`, so the runner finalized the run as a USER CANCEL:
// `onRunFailed` never fired (no provider bench, no investigation task)
// and `isRunCanceledError` callers skipped the fallback cascade — a
// healthy, productive run silently produced nothing. Sixteen NVIDIA NIM
// nemotron persistent-mind runs died this way at exactly 302s in the two
// days after #7560 merged, and none before it (#7665).
const backstopTimeout = apiRunAbsoluteTimeoutMs(effectiveTimeout, absoluteTimeoutMs);
if (backstopTimeout > 0) apiTimeoutHandle = setTimeout(() => {
stopRun(runId).catch(() => { /* best-effort cancel */ });
safeReject(new Error(`API execution timed out after ${backstopTimeout}ms`));
}, backstopTimeout + API_TIMEOUT_BACKSTOP_GRACE_MS);
// Thread the caller's effectiveTimeout into the runner's internal timer so
// a per-call override (e.g. the importer's long stage timeout) governs the
// ceiling instead of the runner's provider/default fallback — same
// caller-override precedence CLI/TUI runs already get.
executeApiRun({ runId, provider: effectiveProvider, model: effectiveModel, prompt, workspacePath: effectiveCwd, screenshots: Array.isArray(screenshots) ? screenshots : [], onData, onComplete, timeout: effectiveTimeout, absoluteTimeoutMs, maxTokens: effectiveProvider === provider ? localBudget.maxTokens : maxTokens, ...(admitDispatch ? { beforeExecute: admitDispatch } : {}) }).catch(safeReject);
} else if (effectiveProvider.type === PROVIDER_TYPES.TUI) {
// `source` (e.g. 'pipeline-manuscript-completeness') labels the live,
// interactive view this TUI run surfaces in the Shell page.
import('./tuiPromptRunner.js')
.then(({ executeTuiRun }) => executeTuiRun({ runId, provider: providerForRun, prompt, screenshots, workspacePath: effectiveCwd, onData, onComplete, onReady: onRunReady, timeout: effectiveTimeout, label: source, ...(admitDispatch ? { beforeExecute: admitDispatch } : {}) }))
.catch(safeReject);
} else {
safeReject(new Error(`Unsupported provider type: ${effectiveProvider.type}`));
}
});
}).finally(() => {
if (typeof onRunSettled !== 'function') return;
try { onRunSettled(runId); } catch (err) {
console.error(`❌ run lifecycle settle hook failed: ${err.message}`);
}
});
}