Never stop coding. Free MIT AI gateway: one endpoint, 375 providers (150+ free), 1,400+ models Kimi, Claude, GPT, Gemini, GLM, DeepSeek, MiniMax. Works with Claude Code, Codex, Cursor, OpenCode, Cline & Copilot. Quota-aware auto-fallback, RTK+Caveman compression saves 15-95% tokens, MCP/A2A, Desktop/PWA. Built by 900+ contributors
TypeScript a2aai-agentsai-gatewayanthropicclaudeclaude-codeclinecodexcopilotcursordeepseekfree-aigeminikimillm-gatewaymcpopenaiopenai-proxyqwentoken-saver
/**
* Set-try + speculative dispatch loop for handleComboChatInner.
* Extracted from combo.ts dispatchWithCooldownRetry (#11804 finally lives here).
*
* @internal — not part of the public combo.ts barrel.
*/
import {
formatRetryAfter,
getEarliestRateLimitedUntil,
getModelLockoutInfo,
scanTransientModelLock,
} from "../accountFallback.ts";
import {
getCachedProviderConnectionById,
getCachedProviderConnections,
} from "../../../src/lib/db/readCache.ts";
import { getProviderAlias, resolveProviderId } from "../../../src/shared/constants/providers.ts";
import {
errorResponse,
errorResponseWithComboDiagnostics,
unavailableResponse,
} from "../../utils/error.ts";
import { estimateSizeFast } from "../../utils/estimateSize.ts";
import { jsonLength } from "../../utils/jsonSize.ts";
import { COMBO_FAILURE_THRESHOLD, recordComboFailure } from "./failureTracker.ts";
import {
buildAllTargetsCoolingDownResponse,
buildNoUpstreamResponseDiagnostics,
formatPreDispatchExclusions,
} from "./pinRecovery.ts";
import { collectQuotaWindowExclusions, formatQuotaSkipMessage } from "./quotaSkipDiagnostics.ts";
import { recordComboRequest } from "../comboMetrics.ts";
import { notifyWebhookEvent } from "../../../src/lib/webhookDispatcher.ts";
import { parseModel } from "../model.ts";
import {
formatComboOutcomes,
buildRedactedSummary,
resolveComboTerminalStatus,
resolveComboTerminalCode,
} from "./comboErrorAggregation.ts";
import {
resolveComboCooldownWaitDecision,
resolveCircuitOpenWaitDecision,
isTransientComboCooldownReason,
type ResolveComboCooldownDecisionResult,
} from "./comboCooldownRetry.ts";
import {
computeClosestRetryAfter,
waitForCooldownAwareRetry,
} from "../../../src/sse/services/cooldownAwareRetry.ts";
import { toRetryAfterDisplayValue } from "./validateQuality.ts";
import { finalizeComboTrace, finishComboTrace } from "./decisionTrace.ts";
import { isRetryAfterEligibleStatus } from "./unavailableRetryGate.ts";
import { withQuotaExhaustionClassification } from "./quotaExhaustion.ts";
import {
COMBO_LOOP_SAFETY_TIMEOUT_MS,
COMBO_SAFETY_DRAIN_MS,
IDENTICAL_MODEL_ERROR_STREAK,
hasIdenticalModelErrorStreak,
resolveDelayMs,
requestScopedReplayKey,
} from "./comboPredicates.ts";
import { collectCircuitOpenExclusions, evaluateExecuteTargetGates } from "./executeTargetGates.ts";
import { executeTargetAttempt } from "./executeTargetAttempt.ts";
import { buildComboDiag } from "./executeTargetClassify.ts";
import type { AttemptLoopDeps, AttemptLoopState, ExecuteTargetResult } from "./attemptLoopTypes.ts";
import { getStrategyTraits } from "./strategyRegistry.ts";
import { resolveSkippedConnectionCooldownTerminal } from "./skippedConnectionCooldown.ts";
/** A second in-flight copy of a body larger than this sits in the TLS send buffer. */
const HEDGE_MAX_BODY_BYTES = 256 * 1024;
/**
* Hedging sends the body twice. The fast estimate rejects oversized values
* without allocating a JSON string. `jsonLength` then confirms the exact
* serialized size; a cycle or BigInt makes it throw, and that unknown size
* must not hedge — recording 0 bytes would send the body twice anyway.
*/
function isBodySmallEnoughToHedge(body: unknown): boolean {
try {
const estimated = estimateSizeFast(body, HEDGE_MAX_BODY_BYTES);
if (!Number.isFinite(estimated) || estimated > HEDGE_MAX_BODY_BYTES) return false;
return jsonLength(body) <= HEDGE_MAX_BODY_BYTES;
} catch {
return false;
}
}
/**
* Resolve the earliest known connection cooldown across every target's eligible
* pool. A target may pin one connection, allowlist several, or leave selection to
* the provider pool; all three shapes must participate or provider-level combos
* lose the reset time when every account is exhausted. Best-effort by design:
* terminal error construction must never become a new database failure.
*/
async function computeEarliestSkippedRateLimitedUntil(
orderedTargets: AttemptLoopState["orderedTargets"]
): Promise<string | null> {
try {
const connectionIds = new Set<string>();
const unrestrictedProviders = new Set<string>();
for (const target of orderedTargets) {
if (target.connectionId) {
connectionIds.add(target.connectionId);
continue;
}
if (Array.isArray(target.allowedConnectionIds) && target.allowedConnectionIds.length > 0) {
for (const id of target.allowedConnectionIds) {
if (typeof id === "string" && id.trim()) connectionIds.add(id.trim());
}
continue;
}
if (target.provider && target.provider !== "unknown") {
unrestrictedProviders.add(target.provider);
}
}
const connections = await Promise.all(
[...connectionIds].map((id) => getCachedProviderConnectionById(id))
);
const providerConnections = await Promise.all(
[...unrestrictedProviders].map(async (provider) => {
const canonical = resolveProviderId(provider);
const alias = getProviderAlias(canonical);
const ids = new Set([provider, canonical, alias].filter(Boolean));
const rows = await Promise.all(
[...ids].map((id) => getCachedProviderConnections({ provider: id, isActive: true }))
);
return rows.flat();
})
);
const accounts = [...connections, ...providerConnections.flat()]
.filter((connection): connection is Record<string, unknown> => Boolean(connection))
.map((connection) => ({
rateLimitedUntil:
typeof connection.rateLimitedUntil === "string" ? connection.rateLimitedUntil : null,
}));
return getEarliestRateLimitedUntil(accounts);
} catch {
return null;
}
}
/** Pinned lookups stay exact. An empty connection id scans every account. */
function lookupComboModelLockout(provider: string, connectionId: string, rawModel: string) {
if (!connectionId) return scanTransientModelLock(provider, rawModel);
return getModelLockoutInfo(provider, connectionId, rawModel);
}
/**
* Resolve the earliest TRANSIENT model-lockout expiry across the skipped
* targets (#16319). `computeEarliestSkippedRateLimitedUntil` only reads
* connection-level `rateLimitedUntil`, so a model-scoped cooldown never
* surfaced a recovery hint: a single-target combo whose only model sat in a
* short lock answered a hint-less 503 the client reads as an outage. Only
* reasons the cooldown-wait helper classifies as transient qualify — a
* `quota_exhausted` lock runs to midnight and auth/not-found locks never
* clear on their own, so neither may advertise a retry-after. Best-effort by
* design, like its connection-level sibling: terminal error construction must
* never become a new failure.
*/
async function computeEarliestSkippedModelLockUntil(
orderedTargets: AttemptLoopState["orderedTargets"]
): Promise<string | null> {
try {
let earliestMs: number | null = null;
for (const target of orderedTargets) {
const provider = typeof target?.provider === "string" ? target.provider : "";
if (!provider) continue;
const connectionId = typeof target?.connectionId === "string" ? target.connectionId : "";
const rawModel = parseModel(target?.modelStr ?? "").model || "";
if (!rawModel) continue;
const info = lookupComboModelLockout(provider, connectionId, rawModel);
if (!info || !isTransientComboCooldownReason(info.reason)) continue;
if (typeof info.remainingMs !== "number" || !(info.remainingMs > 0)) continue;
const untilMs = Date.now() + info.remainingMs;
if (earliestMs === null || untilMs < earliestMs) earliestMs = untilMs;
}
return earliestMs === null ? null : new Date(earliestMs).toISOString();
} catch {
return null;
}
}
/** Earliest of two optional ISO instants (null-safe). */
function earliestIsoInstant(a: string | null, b: string | null): string | null {
if (a && b) return Date.parse(a) <= Date.parse(b) ? a : b;
return a || b;
}
export type DispatchWithCooldownRetryExtra = {
maxSetRetries: number;
setRetryDelayMs: number;
comboTimeoutMs: number;
comboStartTime: number;
comboCooldownWaitEnabled: boolean;
comboCooldownAttempt: { current: number };
comboCooldownBudgetLeftMs: { current: number };
evaluateGates: typeof evaluateExecuteTargetGates;
executeAttempt: typeof executeTargetAttempt;
};
export async function dispatchWithCooldownRetry(opts: {
state: AttemptLoopState;
deps: AttemptLoopDeps;
extra: DispatchWithCooldownRetryExtra;
}): Promise<Response> {
const { state, deps, extra } = opts;
// #7360: persist lastStatus/earliestRetryAfter across set retries; reset
// only on a fresh dispatch (including cooldown-aware re-dispatch).
state.lastError = null;
state.earliestRetryAfter = null;
state.lastStatus = null;
state.skippedForCircuitOpen = false;
state.earliestCircuitOpenRetryMs = 0;
// #11804: the loop-safety timer is armed per setTry iteration but must be
// cleared on EVERY exit path, not just the happy one. Hoisted to function
// scope so the `finally` at the end of this function always reaches it.
let activeLoopSafetyTimer: ReturnType<typeof setTimeout> | null = null;
try {
for (let setTry = 0; setTry <= extra.maxSetRetries; setTry++) {
// #1731: Per-set-iteration set of providers whose quota is fully exhausted.
// Reset each retry so providers excluded in a previous attempt get another chance.
state.exhaustedProviders = new Set<string>();
state.exhaustedConnections = new Set<string>();
state.transientRateLimitedProviders = new Set<string>();
state.skippedForCircuitOpen = false;
state.earliestCircuitOpenRetryMs = 0;
state.skippedConnectionCooldownUntil = undefined;
if (setTry > 0) {
deps.log.info(
"COMBO",
`All targets failed — retrying set (${setTry}/${extra.maxSetRetries})`
);
await new Promise((resolve) => {
const timer = setTimeout(resolve, extra.setRetryDelayMs);
deps.signal?.addEventListener(
"abort",
() => {
clearTimeout(timer);
resolve(undefined);
},
{ once: true }
);
});
if (deps.signal?.aborted) {
deps.log.info("COMBO", "Client disconnected during set retry delay — aborting");
return errorResponse(499, "Client disconnected");
}
}
deps.startTime = Date.now();
// Fallbacks within this set try: one per ordered target abandoned before
// the one that serves the request. Abandoning the FIRST target counts
// too — it equals the index of the target that ends up serving, i.e. the
// value exposed as `X-OmniRoute-Fallback-Attempts`.
state.fallbackCount = 0;
state.recordedAttempts = 0;
state.comboErrors = [];
// QA P0: the sanitized diagnostic trace (pool size + this set-try's exhausted
// providers/connections + attempt order + terminal reason + recovery hint) is
// assembled by the shared `buildComboDiag` helper — see executeTargetClassify.ts.
let globalResolve: ((res: Response) => void) | null = null;
const globalPromise = new Promise<Response>((res) => {
globalResolve = res;
});
// G1 (silent-stop fix): the speculative loop's `Promise.race` waits on
// `globalPromise`, which is ONLY resolved from inside a task (success or
// fatal error). If a target hangs — e.g. the operator disabled the per-model
// timeout (`targetTimeoutMs: 0`) and the upstream never settles — the race
// never resolves and the request hangs forever with no response. This safety
// promise force-resolves after the combo budget (extra.comboTimeoutMs when set,
// otherwise a hard ceiling) so the request ALWAYS terminates with an
// actionable 504 instead of dying silently. `state.comboExpired` is flipped so the
// target loop stops launching new work; the existing state.comboExpired branch
// returns the aggregated 504.
const loopSafetyMs =
extra.comboTimeoutMs > 0 ? extra.comboTimeoutMs : COMBO_LOOP_SAFETY_TIMEOUT_MS;
let loopSafetyFired = false;
let loopSafetyTimer: ReturnType<typeof setTimeout> | null = null;
const loopSafetyPromise = new Promise<Response>((resolve) => {
loopSafetyTimer = setTimeout(() => {
loopSafetyFired = true;
deps.log.warn(
"COMBO",
`Combo loop safety timeout (${loopSafetyMs}ms) reached without a terminal response — force-terminating`
);
resolve(
errorResponseWithComboDiagnostics(
504,
`Combo global timeout (${loopSafetyMs}ms) without a terminal response`,
buildComboDiag(state, deps.traceInvocationId, "combo_timeout"),
{ code: "COMBO_TIMEOUT", type: "server_error" }
)
);
}, loopSafetyMs);
loopSafetyTimer.unref?.();
activeLoopSafetyTimer = loopSafetyTimer;
});
const runningTasks = new Set<Promise<void>>();
let anySuccess = false;
// Flipped once the last IDENTICAL_MODEL_ERROR_STREAK targets have all
// failed with the exact same request-shape error — see comboPredicates.ts.
// Stops this set-try's target loop early AND skips the whole-set retry
// below, since a malformed request fails identically no matter how many
// more times it's replayed against the remaining fallbacks.
let comboRequestMalformed = false;
// #10681: steps already recorded as dispatched (so per-target retries do not
// duplicate the decision).
state.dispatchedTargets = new Set<string>();
// G1: flip state.comboExpired as soon as the safety timer fires so the next loop
// iteration breaks instead of launching more targets after the budget, and
// abort every in-flight target so a hung upstream actually gets cancelled
// (not just "response stops").
const markLoopExpiredIfSafetyFired = () => {
if (loopSafetyFired) {
state.comboExpired = true;
for (const [, ac] of state.abortControllers.entries()) ac.abort();
}
};
state.abortControllers = new Map<number, AbortController>();
const rejectedModelKeys = (state.requestScopedRejectedModelKeys ??= new Set<string>());
const zeroLatencyOptimizationsEnabled = deps.config.zeroLatencyOptimizationsEnabled === true;
// A hedged target sends the same body a second time. Bodies past this
// size sit in the native TLS send buffer until the network drains them,
// and a slow upstream holds both copies for the whole headers wait.
const bodySmallEnoughToHedge = isBodySmallEnoughToHedge(deps.body);
const hasProtectedPriorityTarget =
getStrategyTraits(deps.strategy).honorsFallbackOnlyTargets &&
state.orderedTargets.some((target) => target.fallbackOnlyOnQuotaExhaustion === true);
const executeTarget = async (i: number): Promise<ExecuteTargetResult> => {
const gate = await extra.evaluateGates({ index: i, state, deps });
if (gate.kind === "skip") return gate.result;
return extra.executeAttempt({
index: i,
state,
deps,
targetForAttempt: gate.targetForAttempt,
profile: gate.profile,
protectedPriorityTarget: gate.protectedPriorityTarget,
});
};
for (let i = 0; i < state.orderedTargets.length; i++) {
if (anySuccess || state.comboExpired || comboRequestMalformed) break;
// Another connection of the SAME model rejects the request identically:
// a request-scoped refusal is not fixable by another account.
if (rejectedModelKeys.has(requestScopedReplayKey(state.orderedTargets[i].modelStr))) {
deps.log.info(
"COMBO",
`Skipping ${state.orderedTargets[i].modelStr} — same request already refused as request-scoped`
);
continue;
}
const abortController = new AbortController();
state.abortControllers.set(i, abortController);
const onClientAbort = () => abortController.abort();
deps.signal?.addEventListener("abort", onClientAbort);
const task = (async () => {
try {
const res = await executeTarget(i);
if (res && !anySuccess) {
if (res.ok) {
anySuccess = true;
globalResolve!(res.response!);
for (const [idx, ac] of state.abortControllers.entries()) {
if (idx !== i) ac.abort();
}
} else if (res.response) {
// Fatal error, abort combo
anySuccess = true;
globalResolve!(res.response);
}
}
} finally {
deps.signal?.removeEventListener("abort", onClientAbort);
}
})().catch((err) => {
const logError = deps.log.error ?? deps.log.warn;
logError("COMBO", `Speculative task error for target ${i}`, err);
// G2 (silent-stop fix): never leave the speculative loop waiting on an
// unresolved globalPromise. If a task throws unexpectedly (outside
// executeTarget's error handling) and no other task succeeds, the post-loop
// `Promise.race([globalPromise, ...])` would hang forever. Resolve with a
// 502 so the request terminates with an actionable error.
if (!anySuccess && globalResolve) {
anySuccess = true;
globalResolve(errorResponse(502, `Combo target ${i} failed with an unexpected error`));
}
});
runningTasks.add(task);
task.finally(() => runningTasks.delete(task));
if (
zeroLatencyOptimizationsEnabled &&
deps.config.hedging &&
bodySmallEnoughToHedge &&
!hasProtectedPriorityTarget &&
i + 1 < state.orderedTargets.length
) {
const hedgeDelay = resolveDelayMs(deps.config.hedgeDelayMs, 500);
const timeoutPromise = new Promise<void>((r) => {
setTimeout(r, hedgeDelay);
});
await Promise.race([task, globalPromise, timeoutPromise, loopSafetyPromise]);
} else {
await Promise.race([task, globalPromise, loopSafetyPromise]);
}
markLoopExpiredIfSafetyFired();
// Global combo timeout check: after each target completes, stop trying
// further targets if the total elapsed time exceeds extra.comboTimeoutMs.
if (
!anySuccess &&
extra.comboTimeoutMs > 0 &&
Date.now() - extra.comboStartTime >= extra.comboTimeoutMs
) {
state.comboExpired = true;
deps.log.info(
"COMBO",
`Combo global timeout (${extra.comboTimeoutMs}ms) reached after ` +
`${i + 1}/${state.orderedTargets.length} targets (${state.recordedAttempts} attempted) — stopping`
);
}
if (!anySuccess && !state.comboExpired && hasIdenticalModelErrorStreak(state.comboErrors)) {
comboRequestMalformed = true;
const last = state.comboErrors[state.comboErrors.length - 1];
deps.log.warn(
"COMBO",
`The last ${IDENTICAL_MODEL_ERROR_STREAK} targets all failed with the identical ` +
`request-shape error (status ${last.status}) after ${i + 1}/${state.orderedTargets.length} ` +
`targets (${state.recordedAttempts} attempted) — stopping instead of retrying the same ` +
`malformed request against remaining fallbacks or set-retries`
);
}
}
if (!anySuccess && runningTasks.size > 0) {
// G1: include loopSafetyPromise so a hung last task (per-model timeout
// disabled) cannot freeze this post-loop race forever.
await Promise.race([globalPromise, Promise.all([...runningTasks]), loopSafetyPromise]);
markLoopExpiredIfSafetyFired();
}
// G1: if the safety timer won the race (request would otherwise hang), give
// in-flight tasks a short drain window to land their per-model errors into
// state.comboErrors so the 504 carries the same "tried: a (500)" summary the
// regular state.comboExpired branch produces — then return the safety 504.
if (loopSafetyFired && !anySuccess) {
if (runningTasks.size > 0) {
await Promise.race([
Promise.allSettled([...runningTasks]),
new Promise((resolve) => setTimeout(resolve, COMBO_SAFETY_DRAIN_MS)),
]);
}
const summary = state.comboErrors
.slice(0, 5)
.map((e) => `${e.model} (${e.status})`)
.join(", ");
const msg =
`Combo global timeout (${loopSafetyMs}ms) after ${state.recordedAttempts}/${state.orderedTargets.length} targets` +
(state.comboErrors.length > 0
? ` | tried: ${summary}${state.comboErrors.length > 5 ? `... (+${state.comboErrors.length - 5})` : ""}`
: "") +
" without a terminal response";
return errorResponseWithComboDiagnostics(
504,
msg,
buildComboDiag(state, deps.traceInvocationId, "combo_timeout"),
{
code: "COMBO_TIMEOUT",
type: "server_error",
}
);
}
// A resolved fatal response also sets anySuccess to stop dispatching.
// Record its actual HTTP status, not an unconditional success.
finalizeComboTrace(deps.traceInvocationId, state.orderedTargets);
if (anySuccess) {
// G1: clear the safety timer on the happy path so a successful combo does
// not leave a 10-minute timer alive per request.
if (loopSafetyTimer) {
clearTimeout(loopSafetyTimer);
loopSafetyTimer = null;
}
const response = await globalPromise;
finishComboTrace(deps.traceInvocationId, { status: response.status });
return response;
}
// #10681: finalize the decision trace (global timeout).
finalizeComboTrace(deps.traceInvocationId, state.orderedTargets);
finishComboTrace(deps.traceInvocationId, { status: 504 });
// Global combo timeout: return aggregated error immediately, skipping set retries.
if (state.comboExpired) {
const summary = buildRedactedSummary(state.comboErrors);
const msg =
`Combo global timeout (${extra.comboTimeoutMs}ms) after ${state.recordedAttempts}/${state.orderedTargets.length} targets` +
(state.comboErrors.length > 0 ? ` | tried: ${summary}` : "");
const latencyMs = Date.now() - deps.startTime;
if (state.recordedAttempts === 0) {
recordComboRequest(deps.combo.name, null, {
success: false,
latencyMs,
fallbackCount: state.fallbackCount,
strategy: deps.strategy,
});
}
notifyWebhookEvent("request.failed", {
combo: deps.combo.name,
reason: "COMBO_TIMEOUT",
latencyMs,
fallbackCount: state.fallbackCount,
});
return errorResponseWithComboDiagnostics(
504,
msg,
buildComboDiag(state, deps.traceInvocationId, "combo_timeout"),
{
code: "COMBO_TIMEOUT",
type: "server_error",
}
);
}
// All models failed in this set try
const latencyMs = Date.now() - deps.startTime;
if (state.recordedAttempts === 0) {
recordComboRequest(deps.combo.name, null, {
success: false,
latencyMs,
fallbackCount: state.fallbackCount,
strategy: deps.strategy,
});
}
// Retry the entire set if more attempts remain -- unless the identical-
// error streak already proved the request itself is malformed, in which
// case a fresh set-try would just reproduce the same streak.
if (
setTry < extra.maxSetRetries &&
!comboRequestMalformed &&
!state.requestScopedFailureSeen
) {
continue;
}
if (!state.lastStatus && state.recordedAttempts === 0 && extra.comboCooldownWaitEnabled) {
const circuitOpenWait = resolveCircuitOpenWaitDecision({
skippedForCircuitOpen: state.skippedForCircuitOpen,
retryAfterMs: state.earliestCircuitOpenRetryMs,
attempt: extra.comboCooldownAttempt.current,
budgetLeftMs: extra.comboCooldownBudgetLeftMs.current,
settings: deps.resilienceSettings.comboCooldownWait,
});
if (circuitOpenWait.wait) {
deps.log.info(
"COMBO",
`${deps.strategy} circuit-open wait: waiting ${Math.ceil(circuitOpenWait.waitMs / 1000)}s (reason=${circuitOpenWait.reason ?? "circuit_open"}) then retrying (attempt ${extra.comboCooldownAttempt.current + 1}/${deps.resilienceSettings.comboCooldownWait.maxAttempts})`
);
const completed = await waitForCooldownAwareRetry(circuitOpenWait.waitMs, deps.signal);
if (!completed) {
return errorResponse(499, "Request aborted");
}
extra.comboCooldownAttempt.current += 1;
extra.comboCooldownBudgetLeftMs.current = Math.max(
0,
extra.comboCooldownBudgetLeftMs.current - circuitOpenWait.waitMs
);
return dispatchWithCooldownRetry({ state, deps, extra });
}
// #16319: a model-scoped cooldown is not a circuit-open skip. With zero
// dispatched attempts nothing populates state.earliestRetryAfter, so the
// general cooldown wait below never runs and a single-target combo
// crystallized a hint-less 503 while the lock had seconds left. Ask the
// same helper the post-dispatch path uses — it derives the wait from each
// target's own lock and rejects quota/auth/not-found reasons — and
// re-dispatch when a SHORT transient lock is about to clear.
const modelLockWait = resolveComboCooldownWaitDecision({
targets: state.orderedTargets,
earliestRetryAfter: null,
attempt: extra.comboCooldownAttempt.current,
budgetLeftMs: extra.comboCooldownBudgetLeftMs.current,
settings: deps.resilienceSettings.comboCooldownWait,
lookupLock: (provider, connectionId, target) => {
const rawModel = parseModel(target?.modelStr ?? "").model || "";
if (!rawModel) return null;
return lookupComboModelLockout(provider, connectionId, rawModel);
},
computeWaitMs: () => null,
});
if (modelLockWait.wait) {
deps.log.info(
"COMBO",
`${deps.strategy} model-lock wait: waiting ${Math.ceil(modelLockWait.waitMs / 1000)}s (reason=${modelLockWait.reason ?? "model_lockout"}) then retrying (attempt ${extra.comboCooldownAttempt.current + 1}/${deps.resilienceSettings.comboCooldownWait.maxAttempts})`
);
const completed = await waitForCooldownAwareRetry(modelLockWait.waitMs, deps.signal);
if (!completed) {
return errorResponse(499, "Request aborted");
}
extra.comboCooldownAttempt.current += 1;
extra.comboCooldownBudgetLeftMs.current = Math.max(
0,
extra.comboCooldownBudgetLeftMs.current - modelLockWait.waitMs
);
return dispatchWithCooldownRetry({ state, deps, extra });
}
}
// All set retries exhausted — return the final error
// #10681: finalize the decision trace (all targets failed or skipped).
finalizeComboTrace(deps.traceInvocationId, state.orderedTargets);
finishComboTrace(deps.traceInvocationId, { status: 503 });
if (!state.lastStatus) {
if (state.recordedAttempts === 0) {
notifyWebhookEvent("request.failed", {
combo: deps.combo.name,
reason: "ALL_TARGETS_SKIPPED",
latencyMs,
fallbackCount: state.fallbackCount,
});
// Every target sat behind an OPEN breaker: say so, with the providers and
// the time until the next probe, instead of the generic skip whose recovery
// hint points at quota and top-ups the breaker has nothing to do with.
const circuitOpen = state.skippedForCircuitOpen
? collectCircuitOpenExclusions(state.orderedTargets)
: null;
const circuitOpenResponse = circuitOpen
? buildAllTargetsCoolingDownResponse(circuitOpen)
: null;
if (circuitOpenResponse) {
deps.log.warn(
"COMBO",
`All targets skipped: circuit breaker open — ${formatPreDispatchExclusions(circuitOpen!)}`
);
return circuitOpenResponse;
}
const quotaSkip = formatQuotaSkipMessage(
collectQuotaWindowExclusions(state.orderedTargets)
);
// #16319: the connection-level scan misses model-scoped cooldowns —
// fold the earliest transient model-lock expiry into the same hint.
const skippedRateLimitedUntil = earliestIsoInstant(
await computeEarliestSkippedRateLimitedUntil(state.orderedTargets),
await computeEarliestSkippedModelLockUntil(state.orderedTargets)
);
const retryHuman = skippedRateLimitedUntil
? formatRetryAfter(toRetryAfterDisplayValue(skippedRateLimitedUntil))
: "";
return withQuotaExhaustionClassification(
errorResponseWithComboDiagnostics(
skippedRateLimitedUntil ? 429 : 503,
[
quotaSkip
? `Service temporarily unavailable: all targets were skipped by pre-dispatch filters (${quotaSkip})`
: "Service temporarily unavailable: all targets were skipped by pre-dispatch filters",
retryHuman,
]
.filter(Boolean)
.join(" "),
buildComboDiag(state, deps.traceInvocationId, "all_targets_skipped"),
{
code: "ALL_TARGETS_SKIPPED",
type: skippedRateLimitedUntil ? "rate_limit_error" : "service_unavailable",
retryAfter: skippedRateLimitedUntil,
}
),
state.observedFailure ? state.allObservedFailuresQuota : null
);
}
notifyWebhookEvent("request.failed", {
combo: deps.combo.name,
reason: "ALL_ACCOUNTS_INACTIVE",
latencyMs,
fallbackCount: state.fallbackCount,
});
recordComboFailure(deps.effectiveSessionId, deps.combo.name);
const inactiveRateLimitedUntil = await computeEarliestSkippedRateLimitedUntil(
state.orderedTargets
);
const retryHuman = inactiveRateLimitedUntil
? formatRetryAfter(toRetryAfterDisplayValue(inactiveRateLimitedUntil))
: "";
return errorResponseWithComboDiagnostics(
inactiveRateLimitedUntil ? 429 : 503,
["Service temporarily unavailable: all upstream accounts are inactive", retryHuman]
.filter(Boolean)
.join(" "),
buildComboDiag(state, deps.traceInvocationId, "all_accounts_inactive"),
{
code: "ALL_ACCOUNTS_INACTIVE",
type: inactiveRateLimitedUntil ? "rate_limit_error" : "service_unavailable",
retryAfter: inactiveRateLimitedUntil,
}
);
}
// #10501: derive the terminal HTTP status from the structured per-target
// outcomes instead of `state.lastStatus` (whichever target happened to fail
// LAST). A 4xx is preserved only when the request itself is genuinely
// invalid across every eligible target; a heterogeneous mix of failure
// classes (e.g. a quality failure + a sibling's 401) normalizes to a
// 5xx-class status reflecting an infra/provider problem, not a client
// error. See comboErrorAggregation.ts::resolveComboTerminalStatus.
const terminal = resolveSkippedConnectionCooldownTerminal(
resolveComboTerminalStatus(state.comboErrors, state.lastStatus),
state
);
const status = terminal.status;
state.earliestRetryAfter = terminal.retryAfter ?? state.earliestRetryAfter;
// #10314: build the terminal message from the structured per-target
// outcomes (each distinct class+reason listed separately) instead of
// mashing a single state.lastError with raw `[model (status)]` markers. Connection
// identifiers are redacted. Falls back to state.lastError when no target recorded
// a structured outcome.
const msg =
formatComboOutcomes(state.comboErrors) || state.lastError || "All combo models unavailable";
// Cooldown-aware retry: instead of crystallizing a transient failure, wait
// out a SHORT cooldown and re-run the whole set loop. Guarded by the helper
// (quota_exhausted/auth/not-found excluded, ceiling, attempts, budget).
// MAX_GLOBAL_ATTEMPTS still bounds total dispatches. Available to ALL combo
// strategies when enabled — entry is driven by earliestRetryAfter + the
// real model-lockout reason, NOT by whichever target last overwrote
// `status` (a later 403 must not skip the allow-list check for an earlier
// 429's retry-after hint). SECURITY (see comboCooldownRetry.ts header): the
// allow-list is the PRIMARY barrier and `maxWaitMs` only the SECOND one.
// Hardcoding reason:"rate_limit" would drop the primary barrier and leave
// only the ceiling — which does NOT cover a quota_exhausted lock carrying a
// SHORT upstream retry-after. Model lockouts are recorded for all strategies,
// so the real reason is always available.
if (extra.comboCooldownWaitEnabled && state.earliestRetryAfter) {
const decision: ResolveComboCooldownDecisionResult = resolveComboCooldownWaitDecision({
targets: state.orderedTargets,
earliestRetryAfter: state.earliestRetryAfter,
attempt: extra.comboCooldownAttempt.current,
budgetLeftMs: extra.comboCooldownBudgetLeftMs.current,
settings: deps.resilienceSettings.comboCooldownWait,
// Key each lookup on the TARGET's own model: quota-share combos are
// single-model/multi-account (so this is identical to the previous
// state.orderedTargets[0] behavior), but heterogeneous combos carry a
// different model per target.
lookupLock: (provider, connectionId, target) => {
const rawModel = parseModel(target?.modelStr ?? "").model || "";
if (!rawModel) return null;
return lookupComboModelLockout(provider, connectionId, rawModel);
},
computeWaitMs: (retryAfter) => computeClosestRetryAfter(retryAfter).waitMs,
});
if (decision.wait) {
deps.log.info(
"COMBO",
`${deps.strategy} cooldown wait: ${msg} — waiting ${Math.ceil(
decision.waitMs / 1000
)}s (reason=${decision.reason ?? "?"}) then retrying (attempt ${
extra.comboCooldownAttempt.current + 1
}/${deps.resilienceSettings.comboCooldownWait.maxAttempts})`
);
const completed = await waitForCooldownAwareRetry(decision.waitMs, deps.signal);
if (!completed) {
deps.log.info("COMBO", `${deps.strategy} cooldown wait aborted by client disconnect`);
return errorResponse(499, "Request aborted");
}
extra.comboCooldownAttempt.current += 1;
extra.comboCooldownBudgetLeftMs.current = Math.max(
0,
extra.comboCooldownBudgetLeftMs.current - decision.waitMs
);
return dispatchWithCooldownRetry({ state, deps, extra });
}
}
// #10681: finalize the decision trace with the aggregated terminal status.
finalizeComboTrace(deps.traceInvocationId, state.orderedTargets);
finishComboTrace(deps.traceInvocationId, { status });
// Retry-after decoration is separate from the wait decision above: only
// rate-limit-class final statuses may carry a `(reset after ...)` suffix
// (see unavailableRetryGate.ts — do not stitch a peer target's window onto
// a config-class status like 403/422).
if (state.earliestRetryAfter && isRetryAfterEligibleStatus(status)) {
const retryHuman = formatRetryAfter(toRetryAfterDisplayValue(state.earliestRetryAfter));
deps.log.warn("COMBO", `All models failed | ${msg} (${retryHuman})`);
return withQuotaExhaustionClassification(
unavailableResponse(status, msg, state.earliestRetryAfter, retryHuman),
terminal.quotaExhausted
);
}
// Silent-stop fix: bump the failure counter (pin clears on 3rd consecutive) and emit
// `try-auto` recovery action via buildRecoveryHint so the OC plugin can show "→ Try
// model: auto" instead of an opaque 5xx. We pass the upstream retry-after seconds to
// the hint so the client can render a precise "wait Ns and retry" message.
deps.log.warn("COMBO", `All models failed | ${msg}`);
const { pinClearedNow } = recordComboFailure(deps.effectiveSessionId, deps.combo.name);
if (pinClearedNow) {
deps.log.info(
"COMBO",
`Auto-cleared session_model_history pin for combo "${deps.combo.name}" after ${COMBO_FAILURE_THRESHOLD} consecutive failures to break the silent-stop loop`
);
}
const retryAfterSeconds = undefined;
// #10966: when every observed failure was independently classified as quota/
// balance exhaustion (isQuotaExhaustionResponse, tracked via state.observeFailure's
// state.allObservedFailuresQuota accumulator), stamp a stable `quota_exhausted`
// terminalReason instead of forwarding the raw upstream error string. The raw
// string falls through buildRecoveryHint's default branch ("retry" / "failed
// transiently"), which is actively misleading for a durable wallet/quota
// exhaustion — retrying the same combo will never refill it.
const terminalReason =
state.observedFailure && state.allObservedFailuresQuota
? "quota_exhausted"
: (state.lastError ?? "all_models_failed");
return withQuotaExhaustionClassification(
errorResponseWithComboDiagnostics(
status,
msg,
buildComboDiag(state, deps.traceInvocationId, terminalReason, retryAfterSeconds),
{ code: resolveComboTerminalCode(state.comboErrors, status) }
),
state.observedFailure ? state.allObservedFailuresQuota : null
);
}
// Final fallback — when the dispatch returned without crystallizing a status (rare).
// Surface the recovery hint with a generic retry recommendation so the client at least
// gets a non-opaque message instead of "Combo routing completed without an upstream response".
recordComboFailure(deps.effectiveSessionId, deps.combo.name);
return errorResponseWithComboDiagnostics(
503,
"Combo routing completed without an upstream response",
buildNoUpstreamResponseDiagnostics(state.orderedTargets.length)
);
} finally {
// #11804: always release the loop-safety timer. Covering every exit path by
// construction here means a future `return` added to this function cannot
// silently reintroduce the leak.
if (activeLoopSafetyTimer) {
clearTimeout(activeLoopSafetyTimer);
activeLoopSafetyTimer = null;
}
}
}