7.4 billion tokens per month. 34 free LLM providers. 635 free model endpoints. All behind one /v1 endpoint, plus any custom OpenAI-compatible endpoint. Smart routing, automatic failover, encrypted keys. Personal experimentation only.
TypeScript
feat(fallback): abort stalled attempts when the retry time budget expires (hedging) (#828)
suantea (asuan_suan)1d2226a80999
diff --git a/server/src/__tests__/lib/fallback-loop-hedge.test.ts b/server/src/__tests__/lib/fallback-loop-hedge.test.tsnew file mode 100644index 00000000..3ce06113--- /dev/null+++ b/server/src/__tests__/lib/fallback-loop-hedge.test.ts@@ -0,0 +1,238 @@+import { describe, it, expect, beforeEach, afterEach, vi } from 'vitest';+import { initDb, getDb } from '../../db/index.js';+import { runFallbackLoop, newFallbackState, type FallbackState } from '../../lib/fallback-loop.js';+import { isHedgeAbortError, newHedgeAbortError, isClientAbortError, isRetryableError } from '../../lib/error-classify.js';+import { acquireLease, releaseLease, resetLeases, inFlightForKey } from '../../services/ratelimit.js';+import type { RouteResult } from '../../services/router.js';++/**+ * Fallback-v2 hedging: when the wall-clock retry budget expires MID-FLIGHT,+ * the loop calls abortInFlight() so a stalled attempt is abandoned instead of+ * only refusing to start the next retry behind it. The surface aborts its+ * composed fetch signal with newHedgeAbortError(); the loop must render+ * timedOut exhaustion WITHOUT any provider-health bookkeeping — no cooldown,+ * no skip entry, no logFailure row — because the budget is spent, not the+ * provider broken.+ */++let keySeq = 900;+function leasedRoute(): { route: RouteResult; keyId: number } {+ const keyId = ++keySeq;+ const id = acquireLease('fake', 'fake-model', keyId, 100);+ return {+ keyId,+ route: {+ provider: {} as any,+ modelId: 'fake-model',+ modelDbId: 900000 + keyId,+ apiKey: 'test-key',+ keyId,+ platform: 'fake',+ displayName: 'Fake Model',+ rpdLimit: null,+ tpdLimit: null,+ release: () => releaseLease(id),+ },+ };+}++function hooks(state: FallbackState, overrides: Record<string, unknown>) {+ return {+ state,+ timeBudgetMs: 0,+ logFailure: vi.fn(),+ onFatal: vi.fn(),+ onRoutingExhausted: vi.fn(),+ onExhausted: vi.fn(),+ ...overrides,+ } as any;+}++// An upstream that hangs until the composed signal aborts, then rejects with+// the marked hedge error — mirrors what a real fetch does when the surface+// aborts its signal via abortInFlight().+function stalledDispatch(hedgeAbort: AbortController) {+ return async (): Promise<never> => {+ return await new Promise((_resolve, reject) => {+ hedgeAbort.signal.addEventListener('abort', () => reject(newHedgeAbortError()), { once: true });+ });+ };+}++describe('fallback loop time-budget hedging', () => {+ beforeEach(() => {+ initDb(':memory:');+ resetLeases();+ vi.spyOn(console, 'log').mockImplementation(() => {});+ });++ afterEach(() => {+ resetLeases();+ vi.restoreAllMocks();+ });++ it('classifies the hedge error as budget-abort, not retryable or client-abort', () => {+ const err = newHedgeAbortError();+ expect(isHedgeAbortError(err)).toBe(true);+ // Must never enter the retryable ladder as a provider failure.+ expect(isRetryableError(err)).toBe(false);+ // Distinct from the client-disconnect abort — different branch in the loop.+ expect(isClientAbortError(err)).toBe(false);+ });++ it('aborts the stalled in-flight attempt when the budget expires, rendering timedOut exhaustion', async () => {+ const state = newFallbackState();+ const hedgeAbort = new AbortController();+ // First two attempts fail quickly and retryably (each ~30ms < the 100ms+ // budget, so the third attempt gets to start); the third hangs and is+ // aborted when the remaining budget (~40ms) expires mid-flight.+ const dispatch = vi.fn()+ .mockImplementationOnce(async () => {+ await new Promise(r => setTimeout(r, 30));+ throw Object.assign(new Error('fake API error 429: rate limit'), { status: 429 });+ })+ .mockImplementationOnce(async () => {+ await new Promise(r => setTimeout(r, 30));+ throw Object.assign(new Error('fake API error 429: rate limit'), { status: 429 });+ })+ .mockImplementation(stalledDispatch(hedgeAbort));++ const h = hooks(state, {+ maxRetries: 5,+ timeBudgetMs: 100,+ abortInFlight: () => hedgeAbort.abort(newHedgeAbortError()),+ route: () => leasedRoute().route,+ dispatch,+ });++ await runFallbackLoop(h);++ // The stalled attempt was abandoned mid-flight; the budget is spent, so+ // no fourth attempt starts.+ expect(dispatch).toHaveBeenCalledTimes(3);+ expect(h.onExhausted).toHaveBeenCalledTimes(1);+ expect(h.onExhausted.mock.calls[0][1].timedOut).toBe(true);+ // The two quick 429 failures were logged as failures; the hedge abort of+ // the third attempt must NOT add a third logFailure row, and there is no+ // fatal render or routing-exhaustion render.+ expect(h.logFailure).toHaveBeenCalledTimes(2);+ expect(h.onFatal).not.toHaveBeenCalled();+ expect(h.onRoutingExhausted).not.toHaveBeenCalled();+ });++ it('the hedge abort adds no cooldown, no skip, no extra failure beyond the retryable attempts', async () => {+ const state = newFallbackState();+ const hedgeAbort = new AbortController();+ // Same ladder shape as above: two quick retryable failures let the third+ // attempt start, then the budget expires mid-flight and the hedge abort+ // abandons it. Hedging only arms for attempt > 1 (the first attempt and+ // the first retry always run — #751).+ const dispatch = vi.fn()+ .mockImplementationOnce(async () => {+ await new Promise(r => setTimeout(r, 30));+ throw Object.assign(new Error('fake API error 429: rate limit'), { status: 429 });+ })+ .mockImplementationOnce(async () => {+ await new Promise(r => setTimeout(r, 30));+ throw Object.assign(new Error('fake API error 429: rate limit'), { status: 429 });+ })+ .mockImplementation(stalledDispatch(hedgeAbort));++ const h = hooks(state, {+ maxRetries: 5,+ timeBudgetMs: 100,+ abortInFlight: () => hedgeAbort.abort(newHedgeAbortError()),+ route: () => leasedRoute().route,+ dispatch,+ });+ await runFallbackLoop(h);++ // The two 429s each produced their normal failure bookkeeping (a cooldown+ // row and a skip entry); the hedge abort must NOT add a third of either —+ // the budget expiring is not a provider-health signal.+ const cooldowns = getDb().prepare('SELECT COUNT(*) AS n FROM rate_limit_cooldowns').get() as { n: number };+ expect(cooldowns.n).toBe(2);+ expect(state.skipKeys.size).toBe(2);+ expect(state.skipModels.size).toBe(0);+ expect(h.logFailure).toHaveBeenCalledTimes(2);+ // And the request rendered as timedOut exhaustion, not a third failure.+ expect(h.onExhausted).toHaveBeenCalledTimes(1);+ expect(h.onExhausted.mock.calls[0][1].timedOut).toBe(true);+ });++ it('never cancels an attempt that already committed, however long it then runs', async () => {+ // The whole point of the hedge is to kill a SILENT attempt. Once a stream+ // has flushed its headers the answer is already reaching the client and the+ // loop could not fail over anyway, so cancelling would truncate a healthy+ // response for nothing. Slow != stalled.+ const state = newFallbackState();+ const abortInFlight = vi.fn();+ const dispatch = vi.fn()+ .mockImplementationOnce(async () => {+ await new Promise(r => setTimeout(r, 30));+ throw Object.assign(new Error('fake API error 429: rate limit'), { status: 429 });+ })+ .mockImplementationOnce(async () => {+ await new Promise(r => setTimeout(r, 30));+ throw Object.assign(new Error('fake API error 429: rate limit'), { status: 429 });+ })+ // Third attempt commits immediately, then keeps streaming well past the+ // 100ms budget — exactly the slow-but-healthy generation that used to be+ // truncated at the budget.+ .mockImplementation(async (_route: unknown, _attempt: number, ctx: { disarmHedge: () => void }) => {+ ctx.disarmHedge();+ await new Promise(r => setTimeout(r, 250));+ return 'committed';+ });++ const h = hooks(state, {+ maxRetries: 5,+ timeBudgetMs: 100,+ abortInFlight,+ route: () => leasedRoute().route,+ dispatch,+ });++ await runFallbackLoop(h);++ expect(dispatch).toHaveBeenCalledTimes(3);+ // The committed attempt outlived the budget and was left alone.+ expect(abortInFlight).not.toHaveBeenCalled();+ expect(h.onExhausted).not.toHaveBeenCalled();+ expect(h.onFatal).not.toHaveBeenCalled();+ // Only the two real 429s were booked as failures.+ expect(h.logFailure).toHaveBeenCalledTimes(2);+ });++ it('still aborts when the attempt stays silent past the budget', async () => {+ // The complement of the test above: no disarmHedge call (nothing was ever+ // flushed), so the stalled attempt is still cancelled.+ const state = newFallbackState();+ const hedgeAbort = new AbortController();+ const abortInFlight = vi.fn(() => hedgeAbort.abort(newHedgeAbortError()));+ const dispatch = vi.fn()+ .mockImplementationOnce(async () => {+ await new Promise(r => setTimeout(r, 30));+ throw Object.assign(new Error('fake API error 429: rate limit'), { status: 429 });+ })+ .mockImplementationOnce(async () => {+ await new Promise(r => setTimeout(r, 30));+ throw Object.assign(new Error('fake API error 429: rate limit'), { status: 429 });+ })+ .mockImplementation(stalledDispatch(hedgeAbort));++ const h = hooks(state, {+ maxRetries: 5,+ timeBudgetMs: 100,+ abortInFlight,+ route: () => leasedRoute().route,+ dispatch,+ });++ await runFallbackLoop(h);++ expect(abortInFlight).toHaveBeenCalledTimes(1);+ expect(h.onExhausted).toHaveBeenCalledTimes(1);+ expect(h.onExhausted.mock.calls[0][1].timedOut).toBe(true);+ });+});diff --git a/server/src/lib/error-classify.ts b/server/src/lib/error-classify.tsindex 72d60090..2d2df22a 100644--- a/server/src/lib/error-classify.ts+++ b/server/src/lib/error-classify.ts@@ -175,6 +175,28 @@ export function isClientAbortError(err: any): boolean {return msg.includes('client disconnected');}+// ── Fallback time-budget hedging (fallback-v2) ──────────────────────────────+// When the wall-clock retry budget expires MID-FLIGHT (the current attempt is+// still waiting on a stalled upstream), the fallback loop aborts the composed+// fetch signal with this marked error. Same rationale as the client abort: it+// is NOT a provider-health signal, so it must never bench/cooldown/penalize+// the model+key, and enrichAbort must pass it through untouched. The loop+// catches it, renders timedOut exhaustion (the budget is spent — nothing left+// to try), and stops without failure bookkeeping.+export function newHedgeAbortError(): Error {+ const err = new Error('fallback time budget expired — upstream request canceled');+ (err as Error & { hedgeAbort?: boolean }).hedgeAbort = true;+ return err;+}++/** True when an error is (or wraps) the time-budget hedge abort above. */+export function isHedgeAbortError(err: any): boolean {+ if (err?.hedgeAbort === true) return true;+ if (err?.cause && (err.cause as { hedgeAbort?: boolean }).hedgeAbort === true) return true;+ const msg = (err?.message ?? '').toLowerCase();+ return msg.includes('fallback time budget expired');+}+/** True for any fetch-abort rejection surfacing out of a body read — the* per-attempt timeout ('request'-bounds deadline in fetchWithTimeout), an* AbortSignal.timeout, or the client disconnect above. Adapters that wrapdiff --git a/server/src/lib/fallback-loop.ts b/server/src/lib/fallback-loop.tsindex ece1273d..f4f6170d 100644--- a/server/src/lib/fallback-loop.ts+++ b/server/src/lib/fallback-loop.ts@@ -35,6 +35,7 @@ import {isRateLimitSignal,isKeyAuthError,isClientAbortError,+ isHedgeAbortError,isDailyQuotaExhaustedError,isPaymentRequiredError,isModelNotFoundError,@@ -677,7 +678,7 @@ export function exhaustedRetryError(lastError: any, maxRetries?: number, ctx?: Econst attempts = ctx?.attempts ?? [];const trail = attempts.length > 0 ? ` Attempt trail: ${formatAttemptTrail(attempts)}.` : '';const budgetNote = ctx?.timedOut- ? ` (stopped early: retry time budget ${Math.round((ctx.budgetMs ?? 0) / 1000)}s exceeded — the attempt in flight is never aborted mid-flight and one failover hop is always allowed, the budget only stops STARTING further retries; raise FALLBACK_TIME_BUDGET_MS or the fallback_time_budget_ms setting to allow a longer failover chain)`+ ? ` (stopped early: retry time budget ${Math.round((ctx.budgetMs ?? 0) / 1000)}s exceeded — one failover hop is always allowed, and past that the budget stops starting further retries and cancels an attempt still waiting on its first byte; raise FALLBACK_TIME_BUDGET_MS or the fallback_time_budget_ms setting to allow a longer failover chain)`: '';const everyAttempt = (cls: AttemptErrorClass | ReadonlySet<AttemptErrorClass>): boolean =>attempts.length > 0 && attempts.every(a => (cls instanceof Set ? cls.has(a.errorClass) : a.errorClass === cls));@@ -883,6 +884,15 @@ export function routingExhaustionBody(routeErr: any): ExhaustionBody {// possible, so stop without recording another retry.export type DispatchOutcome = 'done' | 'committed';+/** Per-attempt handles the loop hands to dispatch. */+export interface DispatchContext {+ /**+ * Cancel this attempt's time-budget hedge. Idempotent and always safe to+ * call, including when hedging is not armed at all. See FallbackHooks.dispatch.+ */+ disarmHedge(): void;+}+// Per-request exhaustion metadata handed to the exhaustion hooks, so each// surface can stamp X-Fallback-Attempts on error responses (previously// success-only) without re-deriving the count.@@ -912,6 +922,14 @@ export interface FallbackHooks {// resulting client-abort throw stops the loop without any failure// bookkeeping (see the isClientAbortError branch below).clientGone?: () => boolean;+ // Fallback-v2 hedging: when provided, the loop starts a per-attempt timer+ // (remaining wall-clock budget) and calls this to ABORT the in-flight+ // upstream instead of just refusing to start the next retry behind a+ // stalled attempt. The surface aborts its composed fetch signal with+ // newHedgeAbortError() — a non-provider-health signal, so the loop renders+ // timedOut exhaustion without benching the model+key (see the+ // isHedgeAbortError branch below). Absent = pre-v2 behavior.+ abortInFlight?: () => void;// Skip state; recordRetryableFailure / recordAuthFailure (called by the loop)// mutate it, and the surface's route() reads it to exclude failed keys/models.state: FallbackState;@@ -934,8 +952,15 @@ export interface FallbackHooks {* return 'committed') so the loop can fail over invisibly. The loop enforces* this contract: any other return value is a programming error and fails* loudly instead of silently swallowing the request.+ *+ * `ctx.disarmHedge()` cancels the time-budget hedge for THIS attempt. Call it+ * the moment the attempt proves it is alive (first byte / headers flushed):+ * past that point the budget must not cancel it, because the answer is+ * already on its way and killing it would truncate a healthy response for no+ * failover benefit. Streaming surfaces are expected to call it; a+ * non-streaming attempt has nothing to disarm until it returns.*/- dispatch(route: RouteResult, attempt: number): Promise<DispatchOutcome>;+ dispatch(route: RouteResult, attempt: number, ctx: DispatchContext): Promise<DispatchOutcome>;/** Trace + log a per-attempt failure (per-surface scope + logRequest args). */logFailure(route: RouteResult, err: any, attempt: number): void;@@ -1085,6 +1110,32 @@ async function runFallbackLoopAttempts(hooks: FallbackHooks, trace: RequestTrace// it, so no path can leak a lease and leave the key's concurrency budget short.// Success accounting happens inside dispatch, so the persisted counters are// already written by the time the provisional lease goes away.+ // Fallback-v2 hedging: arm a timer for the remaining wall-clock budget so a+ // STALLED attempt is aborted mid-flight (abortInFlight) instead of only+ // refusing to start the next retry behind it. Mirrors the loop-top budget+ // check — attempt 0 and the first retry always run (#751).+ //+ // The timer only covers the silent window. dispatch calls ctx.disarmHedge()+ // as soon as the attempt proves it is alive (first byte / headers flushed),+ // because past that point cancelling would truncate a healthy response and+ // buy nothing: a committed stream can no longer fail over anyway. Slow is+ // not the same as stalled, and only stalled is worth killing.+ let hedgeTimer: NodeJS.Timeout | undefined;+ const disarmHedge = () => {+ if (hedgeTimer) {+ clearTimeout(hedgeTimer);+ hedgeTimer = undefined;+ }+ };+ if (attempt > 1 && budgetMs > 0 && hooks.abortInFlight) {+ const remaining = budgetMs - (Date.now() - startedAt);+ if (remaining > 0) {+ hedgeTimer = setTimeout(() => {+ console.log(`[FallbackLoop] retry time budget (${budgetMs}ms) expired with no first byte on ${route.platform}/${route.modelId} — aborting stalled upstream`);+ hooks.abortInFlight?.();+ }, remaining);+ }+ }try {let outcome: DispatchOutcome;try {@@ -1093,7 +1144,7 @@ async function runFallbackLoopAttempts(hooks: FallbackHooks, trace: RequestTrace// otherwise the global proxy / direct path applies as before. The URL// arrives already decrypted on the route (services/router.ts), so an// attempt costs nothing extra — no query, no decrypt.- outcome = await withKeyProxy(route.proxyUrl, () => hooks.dispatch(route, attempt));+ outcome = await withKeyProxy(route.proxyUrl, () => hooks.dispatch(route, attempt, { disarmHedge }));} catch (err: any) {// Client-caused abort: the composed fetch signal fired because OUR// client hung up mid-attempt (see newClientAbortError). Not a@@ -1114,6 +1165,21 @@ async function runFallbackLoopAttempts(hooks: FallbackHooks, trace: RequestTracetraceAttempt('client_abort');return;}+ // Time-budget hedge abort: the wall-clock retry budget expired while this+ // attempt was still in flight, and the surface aborted the composed fetch+ // signal (see newHedgeAbortError). Not a provider-health signal — no+ // cooldown, no penalty, no failure stats — and the budget is spent, so+ // render timedOut exhaustion exactly like the loop-top budget check does.+ if (isHedgeAbortError(err)) {+ const elapsedMs = Date.now() - startedAt;+ console.log(`[FallbackLoop] retry time budget expired mid-attempt on ${route.platform}/${route.modelId} after ${(elapsedMs / 1000).toFixed(1)}s — rendering timedOut exhaustion without benching`);+ hooks.onExhausted(+ exhaustedRetryError(lastError, maxRetries, { attempts, timedOut: true, budgetMs }),+ { attempts, timedOut: true },+ );+ traceAttempt('timeout', err);+ return;+ }hooks.logFailure(route, err, attempt);if (isKeyAuthError(err)) {// KEY-fatal, not request-fatal: rotate past the bad key and revalidate@@ -1170,6 +1236,7 @@ async function runFallbackLoopAttempts(hooks: FallbackHooks, trace: RequestTracetraceAttempt(outcome === 'done' ? 'ok' : 'committed');return;} finally {+ if (hedgeTimer) clearTimeout(hedgeTimer);route.release?.();}}diff --git a/server/src/routes/anthropic.ts b/server/src/routes/anthropic.tsindex 9b66aea5..feec3631 100644--- a/server/src/routes/anthropic.ts+++ b/server/src/routes/anthropic.ts@@ -17,7 +17,7 @@ import { invalidToolArgumentsError, invalidToolCallReasons, isToolArgumentValidaimport { rescueInlineToolCalls, startsWithDialectMarker, couldBecomeDialectMarker, containsDialectMarker } from '../lib/tool-call-rescue.js';import { sanitizeProviderErrorMessage } from '../lib/error-redaction.js';import { convertDocumentBlock, documentRejectionMessage } from '../lib/anthropic-documents.js';-import { isClientAbortError, newClientAbortError, isUpstreamClassificationOutput } from '../lib/error-classify.js';+import { isClientAbortError, newClientAbortError, newHedgeAbortError, isUpstreamClassificationOutput } from '../lib/error-classify.js';import { logRequest } from '../lib/request-log.js';import { extractApiToken, timingSafeStringEqual, getStickyModel, setStickyModel } from './proxy.js';import { runFallbackLoop, newFallbackState, recordUpstreamSuccess, type ExhaustionBody, setFallbackHeaders, setExhaustionHeaders, type AttemptRecord } from '../lib/fallback-loop.js';@@ -551,26 +551,31 @@ anthropicRouter.post('/messages', async (req: Request, res: Response) => {// writableEnded distinguishes a real disconnect.let clientGone = false;const clientAbort = new AbortController();+ // Fallback-v2 hedging: the loop aborts this controller (via abortInFlight)+ // when the wall-clock retry budget expires mid-attempt, canceling the+ // in-flight upstream instead of waiting for a stalled attempt to time out.+ const hedgeAbort = new AbortController();res.on('close', () => {if (!res.writableEnded) {clientGone = true;clientAbort.abort(newClientAbortError());}});- const dispatchOptions = { ...completionOptions, signal: clientAbort.signal };+ const dispatchOptions = { ...completionOptions, signal: AbortSignal.any([clientAbort.signal, hedgeAbort.signal]) };await runFallbackLoop({maxRetries: MAX_RETRIES,state,attemptLog,clientGone: () => clientGone,+ abortInFlight: () => hedgeAbort.abort(newHedgeAbortError()),route: () => routeRequest(estimatedTotal, state.skipKeys.size > 0 ? state.skipKeys : undefined, preferredModel, hasImage, wantsTools, state.skipModels.size > 0 ? state.skipModels : undefined, undefined, false, state.skipPlatforms.size > 0 ? state.skipPlatforms : undefined),- dispatch: async (route, attempt) => {+ dispatch: async (route, attempt, dispatchCtx) => {if (stream) {try {await streamCompletion(res, route, messages, dispatchOptions, {start, attempt, attemptLog, clientGone: () => clientGone, requestedModel, estimatedInputTokens, tools, pinnedModelId,- sessionId, pinned: resolved.pinned,+ sessionId, pinned: resolved.pinned, disarmHedge: dispatchCtx.disarmHedge,});return 'done';} catch (err: any) {@@ -703,6 +708,8 @@ interface StreamCtx {pinnedModelId: string | null;sessionId?: string;pinned: boolean;+ /** Cancel this attempt's time-budget hedge once the stream commits. */+ disarmHedge: () => void;}// Consume the provider's OpenAI-style stream and re-emit it as the Anthropic@@ -745,6 +752,9 @@ async function streamCompletion(res.setHeader('Connection', 'keep-alive');res.setHeader('X-Routed-Via', routedViaValue(route.platform, route.modelId));setFallbackHeaders(res, ctx.attempt, ctx.attemptLog);+ // Committed: the answer is on its way, so the retry budget must no longer+ // cancel this attempt (it could not fail over now anyway).+ ctx.disarmHedge();writeSse(res, 'message_start', {type: 'message_start',message: {diff --git a/server/src/routes/proxy.ts b/server/src/routes/proxy.tsindex 6ceb9fab..1e2730db 100644--- a/server/src/routes/proxy.ts+++ b/server/src/routes/proxy.ts@@ -18,7 +18,7 @@ import { sanitizeProviderErrorMessage } from '../lib/error-redaction.js';import { rescueInlineToolCalls, startsWithDialectMarker, couldBecomeDialectMarker, containsDialectMarker } from '../lib/tool-call-rescue.js';import { getContextHandoffMode, recordIncomingMessages, maybeInjectContextHandoff, recordSuccessfulModel, hasPriorModel, HANDOFF_MAX_TOKENS } from '../services/context-handoff.js';import { isFusionModel, runFusion, fusionConfigSchema, FusionError, FUSION_MODEL_ID } from '../services/fusion.js';-import { isRetryableError, isPaymentRequiredError, isModelNotFoundError, isModelAccessForbiddenError, isClientAbortError, newClientAbortError, isUpstreamClassificationOutput } from '../lib/error-classify.js';+import { isRetryableError, isPaymentRequiredError, isModelNotFoundError, isModelAccessForbiddenError, isClientAbortError, newClientAbortError, newHedgeAbortError, isUpstreamClassificationOutput } from '../lib/error-classify.js';import { logRequest } from '../lib/request-log.js';import { observeServedModel } from '../lib/served-model.js';import { parseCacheDirective, cacheActive, isCacheableTemperature, computeCacheKey, getCachedResponse, storeCachedResponse } from '../services/cache.js';@@ -961,6 +961,10 @@ proxyRouter.post('/completions', async (req: Request, res: Response) => {// writableEnded distinguishes a real disconnect.let clientGone = false;const clientAbort = new AbortController();+ // Fallback-v2 hedging: the loop aborts this controller (via abortInFlight)+ // when the wall-clock retry budget expires mid-attempt, canceling the+ // in-flight upstream instead of waiting for a stalled attempt to time out.+ const hedgeAbort = new AbortController();res.on('close', () => {if (!res.writableEnded) {clientGone = true;@@ -976,6 +980,7 @@ proxyRouter.post('/completions', async (req: Request, res: Response) => {state,attemptLog,clientGone: () => clientGone,+ abortInFlight: () => hedgeAbort.abort(newHedgeAbortError()),route: () => routeRequest(estimatedTotal,state.skipKeys.size > 0 ? state.skipKeys : undefined,@@ -987,7 +992,7 @@ proxyRouter.post('/completions', async (req: Request, res: Response) => {false,state.skipPlatforms.size > 0 ? state.skipPlatforms : undefined,),- dispatch: async (route, attempt) => {+ dispatch: async (route, attempt, ctx) => {traceRouteEvent('Proxy', {event: attempt === 0 ? 'start' : 'next',requestId: requestGroupId,@@ -1017,6 +1022,9 @@ proxyRouter.post('/completions', async (req: Request, res: Response) => {res.setHeader('X-Routed-Via', routedViaValue(route.platform, route.modelId));setFallbackHeaders(res, attempt, attemptLog);headerSent = true;+ // Committed: the answer is on its way, so the retry budget must no+ // longer cancel this attempt (it could not fail over now anyway).+ ctx.disarmHedge();for (const frame of buffered) res.write(`data: ${JSON.stringify(frame)}\n\n`);buffered.length = 0;};@@ -1026,7 +1034,7 @@ proxyRouter.post('/completions', async (req: Request, res: Response) => {route.apiKey,messages,route.modelId,- { temperature, max_tokens, top_p, stop, signal: clientAbort.signal },+ { temperature, max_tokens, top_p, stop, signal: AbortSignal.any([clientAbort.signal, hedgeAbort.signal]) },quotaContextForRoute(route, 'chat/completions'),);@@ -1126,7 +1134,7 @@ proxyRouter.post('/completions', async (req: Request, res: Response) => {route.apiKey,messages,route.modelId,- { temperature, max_tokens, top_p, stop, signal: clientAbort.signal },+ { temperature, max_tokens, top_p, stop, signal: AbortSignal.any([clientAbort.signal, hedgeAbort.signal]) },quotaContextForRoute(route, 'chat/completions'),);@@ -1777,6 +1785,10 @@ proxyRouter.post('/chat/completions', async (req: Request, res: Response) => {// writableEnded distinguishes a real disconnect.let clientGone = false;const clientAbort = new AbortController();+ // Fallback-v2 hedging: the loop aborts this controller (via abortInFlight)+ // when the wall-clock retry budget expires mid-attempt, canceling the+ // in-flight upstream instead of waiting for a stalled attempt to time out.+ const hedgeAbort = new AbortController();res.on('close', () => {if (!res.writableEnded) {clientGone = true;@@ -1789,6 +1801,7 @@ proxyRouter.post('/chat/completions', async (req: Request, res: Response) => {state,attemptLog,clientGone: () => clientGone,+ abortInFlight: () => hedgeAbort.abort(newHedgeAbortError()),route: () => {// When a handoff could fire this turn, pad the token estimate so the router's// context-window and TPM checks account for the extra system message overhead.@@ -1799,7 +1812,7 @@ proxyRouter.post('/chat/completions', async (req: Request, res: Response) => {const routingEstimate = handoffPossible ? estimatedTotal + HANDOFF_MAX_TOKENS : estimatedTotal;return routeRequest(routingEstimate, state.skipKeys.size > 0 ? state.skipKeys : undefined, preferredModel, hasImage, wantsTools, state.skipModels.size > 0 ? state.skipModels : undefined, groupChain ?? resolvedChain?.chain, samplingParams.response_format !== undefined, state.skipPlatforms.size > 0 ? state.skipPlatforms : undefined);},- dispatch: async (route, attempt) => {+ dispatch: async (route, attempt, ctx) => {const modelKey = `${route.platform}:${route.modelId}`;traceRouteEvent('Proxy', {event: attempt === 0 ? 'start' : 'next',@@ -1883,6 +1896,9 @@ proxyRouter.post('/chat/completions', async (req: Request, res: Response) => {res.setHeader('X-Routed-Via', routedViaValue(route.platform, route.modelId));setFallbackHeaders(res, attempt, attemptLog);headerSent = true;+ // Committed: the answer is on its way, so the retry budget must no+ // longer cancel this attempt (it could not fail over now anyway).+ ctx.disarmHedge();for (const p of preamble) res.write(`data: ${JSON.stringify(p)}\n\n`);preamble.length = 0;};@@ -1898,7 +1914,7 @@ proxyRouter.post('/chat/completions', async (req: Request, res: Response) => {try {const gen = route.provider.streamChatCompletion(route.apiKey, outboundMessages, route.modelId,- { temperature, max_tokens, top_p, stop, tools, tool_choice, parallel_tool_calls, stream_options: parsed.data.stream_options, ...samplingParams, signal: clientAbort.signal },+ { temperature, max_tokens, top_p, stop, tools, tool_choice, parallel_tool_calls, stream_options: parsed.data.stream_options, ...samplingParams, signal: AbortSignal.any([clientAbort.signal, hedgeAbort.signal]) },quotaContextForRoute(route, 'chat/completions'),);@@ -2182,7 +2198,7 @@ proxyRouter.post('/chat/completions', async (req: Request, res: Response) => {} else {const result = await route.provider.chatCompletion(route.apiKey, outboundMessages, route.modelId,- { temperature, max_tokens, top_p, stop, tools, tool_choice, parallel_tool_calls, ...samplingParams, signal: clientAbort.signal },+ { temperature, max_tokens, top_p, stop, tools, tool_choice, parallel_tool_calls, ...samplingParams, signal: AbortSignal.any([clientAbort.signal, hedgeAbort.signal]) },quotaContextForRoute(route, 'chat/completions'),);diff --git a/server/src/routes/responses.ts b/server/src/routes/responses.tsindex 99d20569..ec718a99 100644--- a/server/src/routes/responses.ts+++ b/server/src/routes/responses.ts@@ -34,7 +34,7 @@ import { applyTokenBudget, tokenBudgetMessage } from '../lib/guardrails.js';import { samplingParamSchemaFields, pickSamplingParams, type ResponseFormat } from '../lib/sampling-params.js';import { enforceJsonContent } from '../lib/structured-output.js';import { sanitizeProviderErrorMessage } from '../lib/error-redaction.js';-import { isClientAbortError, newClientAbortError } from '../lib/error-classify.js';+import { isClientAbortError, newClientAbortError, newHedgeAbortError } from '../lib/error-classify.js';import { inferQuotaPoolKey, type QuotaObservationContext } from '../services/provider-quota.js';import { compressRequest, formatCompressionHeader } from '../services/compression/pipeline.js';@@ -738,13 +738,17 @@ responsesRouter.post('/responses', async (req: Request, res: Response) => {// writableEnded distinguishes a real disconnect.let clientGone = false;const clientAbort = new AbortController();+ // Fallback-v2 hedging: the loop aborts this controller (via abortInFlight)+ // when the wall-clock retry budget expires mid-attempt, canceling the+ // in-flight upstream instead of waiting for a stalled attempt to time out.+ const hedgeAbort = new AbortController();res.on('close', () => {if (!res.writableEnded) {clientGone = true;clientAbort.abort(newClientAbortError());}});- const dispatchOpts = { ...completionOpts, signal: clientAbort.signal };+ const dispatchOpts = { ...completionOpts, signal: AbortSignal.any([clientAbort.signal, hedgeAbort.signal]) };// Stream bookkeeping (used only when stream === true). `streamStarted` is the// commit flag: true once the response.created/in_progress skeleton has left,@@ -763,8 +767,9 @@ responsesRouter.post('/responses', async (req: Request, res: Response) => {state,attemptLog,clientGone: () => clientGone,+ abortInFlight: () => hedgeAbort.abort(newHedgeAbortError()),route: () => routeRequest(estimatedTotal, state.skipKeys.size > 0 ? state.skipKeys : undefined, preferredModel, hasImage, wantsTools, state.skipModels.size > 0 ? state.skipModels : undefined, groupChain, completionOpts.response_format !== undefined, state.skipPlatforms.size > 0 ? state.skipPlatforms : undefined),- dispatch: async (route, attempt) => {+ dispatch: async (route, attempt, ctx) => {traceRouteEvent('Responses', {event: attempt === 0 ? 'start' : 'next',requestId: requestGroupId,@@ -826,6 +831,9 @@ responsesRouter.post('/responses', async (req: Request, res: Response) => {sse('response.created', { response: skeleton });sse('response.in_progress', { response: skeleton });streamStarted = true;+ // Committed: the answer is on its way, so the retry budget must no+ // longer cancel this attempt (it could not fail over now anyway).+ ctx.disarmHedge();};// Open the text output item and stream `text` as its first delta.