Skip to content

7.4 billion tokens per month. 34 free LLM providers. 635 free model endpoints. All behind one /v1 endpoint, plus any custom OpenAI-compatible endpoint. Smart routing, automatic failover, encrypted keys. Personal experimentation only.

TypeScript

feat(fallback): abort stalled attempts when the retry time budget expires (hedging) (#828)

suantea (asuan_suan)1d2226a80999
diff --git a/server/src/__tests__/lib/fallback-loop-hedge.test.ts b/server/src/__tests__/lib/fallback-loop-hedge.test.ts
new file mode 100644
index 00000000..3ce06113
--- /dev/null
+++ b/server/src/__tests__/lib/fallback-loop-hedge.test.ts
@@ -0,0 +1,238 @@
+import { describe, it, expect, beforeEach, afterEach, vi } from 'vitest';
+import { initDb, getDb } from '../../db/index.js';
+import { runFallbackLoop, newFallbackState, type FallbackState } from '../../lib/fallback-loop.js';
+import { isHedgeAbortError, newHedgeAbortError, isClientAbortError, isRetryableError } from '../../lib/error-classify.js';
+import { acquireLease, releaseLease, resetLeases, inFlightForKey } from '../../services/ratelimit.js';
+import type { RouteResult } from '../../services/router.js';
+
+/**
+ * Fallback-v2 hedging: when the wall-clock retry budget expires MID-FLIGHT,
+ * the loop calls abortInFlight() so a stalled attempt is abandoned instead of
+ * only refusing to start the next retry behind it. The surface aborts its
+ * composed fetch signal with newHedgeAbortError(); the loop must render
+ * timedOut exhaustion WITHOUT any provider-health bookkeeping — no cooldown,
+ * no skip entry, no logFailure row — because the budget is spent, not the
+ * provider broken.
+ */
+
+let keySeq = 900;
+function leasedRoute(): { route: RouteResult; keyId: number } {
+ const keyId = ++keySeq;
+ const id = acquireLease('fake', 'fake-model', keyId, 100);
+ return {
+ keyId,
+ route: {
+ provider: {} as any,
+ modelId: 'fake-model',
+ modelDbId: 900000 + keyId,
+ apiKey: 'test-key',
+ keyId,
+ platform: 'fake',
+ displayName: 'Fake Model',
+ rpdLimit: null,
+ tpdLimit: null,
+ release: () => releaseLease(id),
+ },
+ };
+}
+
+function hooks(state: FallbackState, overrides: Record<string, unknown>) {
+ return {
+ state,
+ timeBudgetMs: 0,
+ logFailure: vi.fn(),
+ onFatal: vi.fn(),
+ onRoutingExhausted: vi.fn(),
+ onExhausted: vi.fn(),
+ ...overrides,
+ } as any;
+}
+
+// An upstream that hangs until the composed signal aborts, then rejects with
+// the marked hedge error — mirrors what a real fetch does when the surface
+// aborts its signal via abortInFlight().
+function stalledDispatch(hedgeAbort: AbortController) {
+ return async (): Promise<never> => {
+ return await new Promise((_resolve, reject) => {
+ hedgeAbort.signal.addEventListener('abort', () => reject(newHedgeAbortError()), { once: true });
+ });
+ };
+}
+
+describe('fallback loop time-budget hedging', () => {
+ beforeEach(() => {
+ initDb(':memory:');
+ resetLeases();
+ vi.spyOn(console, 'log').mockImplementation(() => {});
+ });
+
+ afterEach(() => {
+ resetLeases();
+ vi.restoreAllMocks();
+ });
+
+ it('classifies the hedge error as budget-abort, not retryable or client-abort', () => {
+ const err = newHedgeAbortError();
+ expect(isHedgeAbortError(err)).toBe(true);
+ // Must never enter the retryable ladder as a provider failure.
+ expect(isRetryableError(err)).toBe(false);
+ // Distinct from the client-disconnect abort — different branch in the loop.
+ expect(isClientAbortError(err)).toBe(false);
+ });
+
+ it('aborts the stalled in-flight attempt when the budget expires, rendering timedOut exhaustion', async () => {
+ const state = newFallbackState();
+ const hedgeAbort = new AbortController();
+ // First two attempts fail quickly and retryably (each ~30ms < the 100ms
+ // budget, so the third attempt gets to start); the third hangs and is
+ // aborted when the remaining budget (~40ms) expires mid-flight.
+ const dispatch = vi.fn()
+ .mockImplementationOnce(async () => {
+ await new Promise(r => setTimeout(r, 30));
+ throw Object.assign(new Error('fake API error 429: rate limit'), { status: 429 });
+ })
+ .mockImplementationOnce(async () => {
+ await new Promise(r => setTimeout(r, 30));
+ throw Object.assign(new Error('fake API error 429: rate limit'), { status: 429 });
+ })
+ .mockImplementation(stalledDispatch(hedgeAbort));
+
+ const h = hooks(state, {
+ maxRetries: 5,
+ timeBudgetMs: 100,
+ abortInFlight: () => hedgeAbort.abort(newHedgeAbortError()),
+ route: () => leasedRoute().route,
+ dispatch,
+ });
+
+ await runFallbackLoop(h);
+
+ // The stalled attempt was abandoned mid-flight; the budget is spent, so
+ // no fourth attempt starts.
+ expect(dispatch).toHaveBeenCalledTimes(3);
+ expect(h.onExhausted).toHaveBeenCalledTimes(1);
+ expect(h.onExhausted.mock.calls[0][1].timedOut).toBe(true);
+ // The two quick 429 failures were logged as failures; the hedge abort of
+ // the third attempt must NOT add a third logFailure row, and there is no
+ // fatal render or routing-exhaustion render.
+ expect(h.logFailure).toHaveBeenCalledTimes(2);
+ expect(h.onFatal).not.toHaveBeenCalled();
+ expect(h.onRoutingExhausted).not.toHaveBeenCalled();
+ });
+
+ it('the hedge abort adds no cooldown, no skip, no extra failure beyond the retryable attempts', async () => {
+ const state = newFallbackState();
+ const hedgeAbort = new AbortController();
+ // Same ladder shape as above: two quick retryable failures let the third
+ // attempt start, then the budget expires mid-flight and the hedge abort
+ // abandons it. Hedging only arms for attempt > 1 (the first attempt and
+ // the first retry always run — #751).
+ const dispatch = vi.fn()
+ .mockImplementationOnce(async () => {
+ await new Promise(r => setTimeout(r, 30));
+ throw Object.assign(new Error('fake API error 429: rate limit'), { status: 429 });
+ })
+ .mockImplementationOnce(async () => {
+ await new Promise(r => setTimeout(r, 30));
+ throw Object.assign(new Error('fake API error 429: rate limit'), { status: 429 });
+ })
+ .mockImplementation(stalledDispatch(hedgeAbort));
+
+ const h = hooks(state, {
+ maxRetries: 5,
+ timeBudgetMs: 100,
+ abortInFlight: () => hedgeAbort.abort(newHedgeAbortError()),
+ route: () => leasedRoute().route,
+ dispatch,
+ });
+ await runFallbackLoop(h);
+
+ // The two 429s each produced their normal failure bookkeeping (a cooldown
+ // row and a skip entry); the hedge abort must NOT add a third of either —
+ // the budget expiring is not a provider-health signal.
+ const cooldowns = getDb().prepare('SELECT COUNT(*) AS n FROM rate_limit_cooldowns').get() as { n: number };
+ expect(cooldowns.n).toBe(2);
+ expect(state.skipKeys.size).toBe(2);
+ expect(state.skipModels.size).toBe(0);
+ expect(h.logFailure).toHaveBeenCalledTimes(2);
+ // And the request rendered as timedOut exhaustion, not a third failure.
+ expect(h.onExhausted).toHaveBeenCalledTimes(1);
+ expect(h.onExhausted.mock.calls[0][1].timedOut).toBe(true);
+ });
+
+ it('never cancels an attempt that already committed, however long it then runs', async () => {
+ // The whole point of the hedge is to kill a SILENT attempt. Once a stream
+ // has flushed its headers the answer is already reaching the client and the
+ // loop could not fail over anyway, so cancelling would truncate a healthy
+ // response for nothing. Slow != stalled.
+ const state = newFallbackState();
+ const abortInFlight = vi.fn();
+ const dispatch = vi.fn()
+ .mockImplementationOnce(async () => {
+ await new Promise(r => setTimeout(r, 30));
+ throw Object.assign(new Error('fake API error 429: rate limit'), { status: 429 });
+ })
+ .mockImplementationOnce(async () => {
+ await new Promise(r => setTimeout(r, 30));
+ throw Object.assign(new Error('fake API error 429: rate limit'), { status: 429 });
+ })
+ // Third attempt commits immediately, then keeps streaming well past the
+ // 100ms budget — exactly the slow-but-healthy generation that used to be
+ // truncated at the budget.
+ .mockImplementation(async (_route: unknown, _attempt: number, ctx: { disarmHedge: () => void }) => {
+ ctx.disarmHedge();
+ await new Promise(r => setTimeout(r, 250));
+ return 'committed';
+ });
+
+ const h = hooks(state, {
+ maxRetries: 5,
+ timeBudgetMs: 100,
+ abortInFlight,
+ route: () => leasedRoute().route,
+ dispatch,
+ });
+
+ await runFallbackLoop(h);
+
+ expect(dispatch).toHaveBeenCalledTimes(3);
+ // The committed attempt outlived the budget and was left alone.
+ expect(abortInFlight).not.toHaveBeenCalled();
+ expect(h.onExhausted).not.toHaveBeenCalled();
+ expect(h.onFatal).not.toHaveBeenCalled();
+ // Only the two real 429s were booked as failures.
+ expect(h.logFailure).toHaveBeenCalledTimes(2);
+ });
+
+ it('still aborts when the attempt stays silent past the budget', async () => {
+ // The complement of the test above: no disarmHedge call (nothing was ever
+ // flushed), so the stalled attempt is still cancelled.
+ const state = newFallbackState();
+ const hedgeAbort = new AbortController();
+ const abortInFlight = vi.fn(() => hedgeAbort.abort(newHedgeAbortError()));
+ const dispatch = vi.fn()
+ .mockImplementationOnce(async () => {
+ await new Promise(r => setTimeout(r, 30));
+ throw Object.assign(new Error('fake API error 429: rate limit'), { status: 429 });
+ })
+ .mockImplementationOnce(async () => {
+ await new Promise(r => setTimeout(r, 30));
+ throw Object.assign(new Error('fake API error 429: rate limit'), { status: 429 });
+ })
+ .mockImplementation(stalledDispatch(hedgeAbort));
+
+ const h = hooks(state, {
+ maxRetries: 5,
+ timeBudgetMs: 100,
+ abortInFlight,
+ route: () => leasedRoute().route,
+ dispatch,
+ });
+
+ await runFallbackLoop(h);
+
+ expect(abortInFlight).toHaveBeenCalledTimes(1);
+ expect(h.onExhausted).toHaveBeenCalledTimes(1);
+ expect(h.onExhausted.mock.calls[0][1].timedOut).toBe(true);
+ });
+});
diff --git a/server/src/lib/error-classify.ts b/server/src/lib/error-classify.ts
index 72d60090..2d2df22a 100644
--- a/server/src/lib/error-classify.ts
+++ b/server/src/lib/error-classify.ts
@@ -175,6 +175,28 @@ export function isClientAbortError(err: any): boolean {
return msg.includes('client disconnected');
}
+// ── Fallback time-budget hedging (fallback-v2) ──────────────────────────────
+// When the wall-clock retry budget expires MID-FLIGHT (the current attempt is
+// still waiting on a stalled upstream), the fallback loop aborts the composed
+// fetch signal with this marked error. Same rationale as the client abort: it
+// is NOT a provider-health signal, so it must never bench/cooldown/penalize
+// the model+key, and enrichAbort must pass it through untouched. The loop
+// catches it, renders timedOut exhaustion (the budget is spent — nothing left
+// to try), and stops without failure bookkeeping.
+export function newHedgeAbortError(): Error {
+ const err = new Error('fallback time budget expired — upstream request canceled');
+ (err as Error & { hedgeAbort?: boolean }).hedgeAbort = true;
+ return err;
+}
+
+/** True when an error is (or wraps) the time-budget hedge abort above. */
+export function isHedgeAbortError(err: any): boolean {
+ if (err?.hedgeAbort === true) return true;
+ if (err?.cause && (err.cause as { hedgeAbort?: boolean }).hedgeAbort === true) return true;
+ const msg = (err?.message ?? '').toLowerCase();
+ return msg.includes('fallback time budget expired');
+}
+
/** True for any fetch-abort rejection surfacing out of a body read — the
* per-attempt timeout ('request'-bounds deadline in fetchWithTimeout), an
* AbortSignal.timeout, or the client disconnect above. Adapters that wrap
diff --git a/server/src/lib/fallback-loop.ts b/server/src/lib/fallback-loop.ts
index ece1273d..f4f6170d 100644
--- a/server/src/lib/fallback-loop.ts
+++ b/server/src/lib/fallback-loop.ts
@@ -35,6 +35,7 @@ import {
isRateLimitSignal,
isKeyAuthError,
isClientAbortError,
+ isHedgeAbortError,
isDailyQuotaExhaustedError,
isPaymentRequiredError,
isModelNotFoundError,
@@ -677,7 +678,7 @@ export function exhaustedRetryError(lastError: any, maxRetries?: number, ctx?: E
const attempts = ctx?.attempts ?? [];
const trail = attempts.length > 0 ? ` Attempt trail: ${formatAttemptTrail(attempts)}.` : '';
const budgetNote = ctx?.timedOut
- ? ` (stopped early: retry time budget ${Math.round((ctx.budgetMs ?? 0) / 1000)}s exceeded — the attempt in flight is never aborted mid-flight and one failover hop is always allowed, the budget only stops STARTING further retries; raise FALLBACK_TIME_BUDGET_MS or the fallback_time_budget_ms setting to allow a longer failover chain)`
+ ? ` (stopped early: retry time budget ${Math.round((ctx.budgetMs ?? 0) / 1000)}s exceeded — one failover hop is always allowed, and past that the budget stops starting further retries and cancels an attempt still waiting on its first byte; raise FALLBACK_TIME_BUDGET_MS or the fallback_time_budget_ms setting to allow a longer failover chain)`
: '';
const everyAttempt = (cls: AttemptErrorClass | ReadonlySet<AttemptErrorClass>): boolean =>
attempts.length > 0 && attempts.every(a => (cls instanceof Set ? cls.has(a.errorClass) : a.errorClass === cls));
@@ -883,6 +884,15 @@ export function routingExhaustionBody(routeErr: any): ExhaustionBody {
// possible, so stop without recording another retry.
export type DispatchOutcome = 'done' | 'committed';
+/** Per-attempt handles the loop hands to dispatch. */
+export interface DispatchContext {
+ /**
+ * Cancel this attempt's time-budget hedge. Idempotent and always safe to
+ * call, including when hedging is not armed at all. See FallbackHooks.dispatch.
+ */
+ disarmHedge(): void;
+}
+
// Per-request exhaustion metadata handed to the exhaustion hooks, so each
// surface can stamp X-Fallback-Attempts on error responses (previously
// success-only) without re-deriving the count.
@@ -912,6 +922,14 @@ export interface FallbackHooks {
// resulting client-abort throw stops the loop without any failure
// bookkeeping (see the isClientAbortError branch below).
clientGone?: () => boolean;
+ // Fallback-v2 hedging: when provided, the loop starts a per-attempt timer
+ // (remaining wall-clock budget) and calls this to ABORT the in-flight
+ // upstream instead of just refusing to start the next retry behind a
+ // stalled attempt. The surface aborts its composed fetch signal with
+ // newHedgeAbortError() — a non-provider-health signal, so the loop renders
+ // timedOut exhaustion without benching the model+key (see the
+ // isHedgeAbortError branch below). Absent = pre-v2 behavior.
+ abortInFlight?: () => void;
// Skip state; recordRetryableFailure / recordAuthFailure (called by the loop)
// mutate it, and the surface's route() reads it to exclude failed keys/models.
state: FallbackState;
@@ -934,8 +952,15 @@ export interface FallbackHooks {
* return 'committed') so the loop can fail over invisibly. The loop enforces
* this contract: any other return value is a programming error and fails
* loudly instead of silently swallowing the request.
+ *
+ * `ctx.disarmHedge()` cancels the time-budget hedge for THIS attempt. Call it
+ * the moment the attempt proves it is alive (first byte / headers flushed):
+ * past that point the budget must not cancel it, because the answer is
+ * already on its way and killing it would truncate a healthy response for no
+ * failover benefit. Streaming surfaces are expected to call it; a
+ * non-streaming attempt has nothing to disarm until it returns.
*/
- dispatch(route: RouteResult, attempt: number): Promise<DispatchOutcome>;
+ dispatch(route: RouteResult, attempt: number, ctx: DispatchContext): Promise<DispatchOutcome>;
/** Trace + log a per-attempt failure (per-surface scope + logRequest args). */
logFailure(route: RouteResult, err: any, attempt: number): void;
@@ -1085,6 +1110,32 @@ async function runFallbackLoopAttempts(hooks: FallbackHooks, trace: RequestTrace
// it, so no path can leak a lease and leave the key's concurrency budget short.
// Success accounting happens inside dispatch, so the persisted counters are
// already written by the time the provisional lease goes away.
+ // Fallback-v2 hedging: arm a timer for the remaining wall-clock budget so a
+ // STALLED attempt is aborted mid-flight (abortInFlight) instead of only
+ // refusing to start the next retry behind it. Mirrors the loop-top budget
+ // check — attempt 0 and the first retry always run (#751).
+ //
+ // The timer only covers the silent window. dispatch calls ctx.disarmHedge()
+ // as soon as the attempt proves it is alive (first byte / headers flushed),
+ // because past that point cancelling would truncate a healthy response and
+ // buy nothing: a committed stream can no longer fail over anyway. Slow is
+ // not the same as stalled, and only stalled is worth killing.
+ let hedgeTimer: NodeJS.Timeout | undefined;
+ const disarmHedge = () => {
+ if (hedgeTimer) {
+ clearTimeout(hedgeTimer);
+ hedgeTimer = undefined;
+ }
+ };
+ if (attempt > 1 && budgetMs > 0 && hooks.abortInFlight) {
+ const remaining = budgetMs - (Date.now() - startedAt);
+ if (remaining > 0) {
+ hedgeTimer = setTimeout(() => {
+ console.log(`[FallbackLoop] retry time budget (${budgetMs}ms) expired with no first byte on ${route.platform}/${route.modelId} — aborting stalled upstream`);
+ hooks.abortInFlight?.();
+ }, remaining);
+ }
+ }
try {
let outcome: DispatchOutcome;
try {
@@ -1093,7 +1144,7 @@ async function runFallbackLoopAttempts(hooks: FallbackHooks, trace: RequestTrace
// otherwise the global proxy / direct path applies as before. The URL
// arrives already decrypted on the route (services/router.ts), so an
// attempt costs nothing extra — no query, no decrypt.
- outcome = await withKeyProxy(route.proxyUrl, () => hooks.dispatch(route, attempt));
+ outcome = await withKeyProxy(route.proxyUrl, () => hooks.dispatch(route, attempt, { disarmHedge }));
} catch (err: any) {
// Client-caused abort: the composed fetch signal fired because OUR
// client hung up mid-attempt (see newClientAbortError). Not a
@@ -1114,6 +1165,21 @@ async function runFallbackLoopAttempts(hooks: FallbackHooks, trace: RequestTrace
traceAttempt('client_abort');
return;
}
+ // Time-budget hedge abort: the wall-clock retry budget expired while this
+ // attempt was still in flight, and the surface aborted the composed fetch
+ // signal (see newHedgeAbortError). Not a provider-health signal — no
+ // cooldown, no penalty, no failure stats — and the budget is spent, so
+ // render timedOut exhaustion exactly like the loop-top budget check does.
+ if (isHedgeAbortError(err)) {
+ const elapsedMs = Date.now() - startedAt;
+ console.log(`[FallbackLoop] retry time budget expired mid-attempt on ${route.platform}/${route.modelId} after ${(elapsedMs / 1000).toFixed(1)}s — rendering timedOut exhaustion without benching`);
+ hooks.onExhausted(
+ exhaustedRetryError(lastError, maxRetries, { attempts, timedOut: true, budgetMs }),
+ { attempts, timedOut: true },
+ );
+ traceAttempt('timeout', err);
+ return;
+ }
hooks.logFailure(route, err, attempt);
if (isKeyAuthError(err)) {
// KEY-fatal, not request-fatal: rotate past the bad key and revalidate
@@ -1170,6 +1236,7 @@ async function runFallbackLoopAttempts(hooks: FallbackHooks, trace: RequestTrace
traceAttempt(outcome === 'done' ? 'ok' : 'committed');
return;
} finally {
+ if (hedgeTimer) clearTimeout(hedgeTimer);
route.release?.();
}
}
diff --git a/server/src/routes/anthropic.ts b/server/src/routes/anthropic.ts
index 9b66aea5..feec3631 100644
--- a/server/src/routes/anthropic.ts
+++ b/server/src/routes/anthropic.ts
@@ -17,7 +17,7 @@ import { invalidToolArgumentsError, invalidToolCallReasons, isToolArgumentValida
import { rescueInlineToolCalls, startsWithDialectMarker, couldBecomeDialectMarker, containsDialectMarker } from '../lib/tool-call-rescue.js';
import { sanitizeProviderErrorMessage } from '../lib/error-redaction.js';
import { convertDocumentBlock, documentRejectionMessage } from '../lib/anthropic-documents.js';
-import { isClientAbortError, newClientAbortError, isUpstreamClassificationOutput } from '../lib/error-classify.js';
+import { isClientAbortError, newClientAbortError, newHedgeAbortError, isUpstreamClassificationOutput } from '../lib/error-classify.js';
import { logRequest } from '../lib/request-log.js';
import { extractApiToken, timingSafeStringEqual, getStickyModel, setStickyModel } from './proxy.js';
import { runFallbackLoop, newFallbackState, recordUpstreamSuccess, type ExhaustionBody, setFallbackHeaders, setExhaustionHeaders, type AttemptRecord } from '../lib/fallback-loop.js';
@@ -551,26 +551,31 @@ anthropicRouter.post('/messages', async (req: Request, res: Response) => {
// writableEnded distinguishes a real disconnect.
let clientGone = false;
const clientAbort = new AbortController();
+ // Fallback-v2 hedging: the loop aborts this controller (via abortInFlight)
+ // when the wall-clock retry budget expires mid-attempt, canceling the
+ // in-flight upstream instead of waiting for a stalled attempt to time out.
+ const hedgeAbort = new AbortController();
res.on('close', () => {
if (!res.writableEnded) {
clientGone = true;
clientAbort.abort(newClientAbortError());
}
});
- const dispatchOptions = { ...completionOptions, signal: clientAbort.signal };
+ const dispatchOptions = { ...completionOptions, signal: AbortSignal.any([clientAbort.signal, hedgeAbort.signal]) };
await runFallbackLoop({
maxRetries: MAX_RETRIES,
state,
attemptLog,
clientGone: () => clientGone,
+ abortInFlight: () => hedgeAbort.abort(newHedgeAbortError()),
route: () => routeRequest(estimatedTotal, state.skipKeys.size > 0 ? state.skipKeys : undefined, preferredModel, hasImage, wantsTools, state.skipModels.size > 0 ? state.skipModels : undefined, undefined, false, state.skipPlatforms.size > 0 ? state.skipPlatforms : undefined),
- dispatch: async (route, attempt) => {
+ dispatch: async (route, attempt, dispatchCtx) => {
if (stream) {
try {
await streamCompletion(res, route, messages, dispatchOptions, {
start, attempt, attemptLog, clientGone: () => clientGone, requestedModel, estimatedInputTokens, tools, pinnedModelId,
- sessionId, pinned: resolved.pinned,
+ sessionId, pinned: resolved.pinned, disarmHedge: dispatchCtx.disarmHedge,
});
return 'done';
} catch (err: any) {
@@ -703,6 +708,8 @@ interface StreamCtx {
pinnedModelId: string | null;
sessionId?: string;
pinned: boolean;
+ /** Cancel this attempt's time-budget hedge once the stream commits. */
+ disarmHedge: () => void;
}
// Consume the provider's OpenAI-style stream and re-emit it as the Anthropic
@@ -745,6 +752,9 @@ async function streamCompletion(
res.setHeader('Connection', 'keep-alive');
res.setHeader('X-Routed-Via', routedViaValue(route.platform, route.modelId));
setFallbackHeaders(res, ctx.attempt, ctx.attemptLog);
+ // Committed: the answer is on its way, so the retry budget must no longer
+ // cancel this attempt (it could not fail over now anyway).
+ ctx.disarmHedge();
writeSse(res, 'message_start', {
type: 'message_start',
message: {
diff --git a/server/src/routes/proxy.ts b/server/src/routes/proxy.ts
index 6ceb9fab..1e2730db 100644
--- a/server/src/routes/proxy.ts
+++ b/server/src/routes/proxy.ts
@@ -18,7 +18,7 @@ import { sanitizeProviderErrorMessage } from '../lib/error-redaction.js';
import { rescueInlineToolCalls, startsWithDialectMarker, couldBecomeDialectMarker, containsDialectMarker } from '../lib/tool-call-rescue.js';
import { getContextHandoffMode, recordIncomingMessages, maybeInjectContextHandoff, recordSuccessfulModel, hasPriorModel, HANDOFF_MAX_TOKENS } from '../services/context-handoff.js';
import { isFusionModel, runFusion, fusionConfigSchema, FusionError, FUSION_MODEL_ID } from '../services/fusion.js';
-import { isRetryableError, isPaymentRequiredError, isModelNotFoundError, isModelAccessForbiddenError, isClientAbortError, newClientAbortError, isUpstreamClassificationOutput } from '../lib/error-classify.js';
+import { isRetryableError, isPaymentRequiredError, isModelNotFoundError, isModelAccessForbiddenError, isClientAbortError, newClientAbortError, newHedgeAbortError, isUpstreamClassificationOutput } from '../lib/error-classify.js';
import { logRequest } from '../lib/request-log.js';
import { observeServedModel } from '../lib/served-model.js';
import { parseCacheDirective, cacheActive, isCacheableTemperature, computeCacheKey, getCachedResponse, storeCachedResponse } from '../services/cache.js';
@@ -961,6 +961,10 @@ proxyRouter.post('/completions', async (req: Request, res: Response) => {
// writableEnded distinguishes a real disconnect.
let clientGone = false;
const clientAbort = new AbortController();
+ // Fallback-v2 hedging: the loop aborts this controller (via abortInFlight)
+ // when the wall-clock retry budget expires mid-attempt, canceling the
+ // in-flight upstream instead of waiting for a stalled attempt to time out.
+ const hedgeAbort = new AbortController();
res.on('close', () => {
if (!res.writableEnded) {
clientGone = true;
@@ -976,6 +980,7 @@ proxyRouter.post('/completions', async (req: Request, res: Response) => {
state,
attemptLog,
clientGone: () => clientGone,
+ abortInFlight: () => hedgeAbort.abort(newHedgeAbortError()),
route: () => routeRequest(
estimatedTotal,
state.skipKeys.size > 0 ? state.skipKeys : undefined,
@@ -987,7 +992,7 @@ proxyRouter.post('/completions', async (req: Request, res: Response) => {
false,
state.skipPlatforms.size > 0 ? state.skipPlatforms : undefined,
),
- dispatch: async (route, attempt) => {
+ dispatch: async (route, attempt, ctx) => {
traceRouteEvent('Proxy', {
event: attempt === 0 ? 'start' : 'next',
requestId: requestGroupId,
@@ -1017,6 +1022,9 @@ proxyRouter.post('/completions', async (req: Request, res: Response) => {
res.setHeader('X-Routed-Via', routedViaValue(route.platform, route.modelId));
setFallbackHeaders(res, attempt, attemptLog);
headerSent = true;
+ // Committed: the answer is on its way, so the retry budget must no
+ // longer cancel this attempt (it could not fail over now anyway).
+ ctx.disarmHedge();
for (const frame of buffered) res.write(`data: ${JSON.stringify(frame)}\n\n`);
buffered.length = 0;
};
@@ -1026,7 +1034,7 @@ proxyRouter.post('/completions', async (req: Request, res: Response) => {
route.apiKey,
messages,
route.modelId,
- { temperature, max_tokens, top_p, stop, signal: clientAbort.signal },
+ { temperature, max_tokens, top_p, stop, signal: AbortSignal.any([clientAbort.signal, hedgeAbort.signal]) },
quotaContextForRoute(route, 'chat/completions'),
);
@@ -1126,7 +1134,7 @@ proxyRouter.post('/completions', async (req: Request, res: Response) => {
route.apiKey,
messages,
route.modelId,
- { temperature, max_tokens, top_p, stop, signal: clientAbort.signal },
+ { temperature, max_tokens, top_p, stop, signal: AbortSignal.any([clientAbort.signal, hedgeAbort.signal]) },
quotaContextForRoute(route, 'chat/completions'),
);
@@ -1777,6 +1785,10 @@ proxyRouter.post('/chat/completions', async (req: Request, res: Response) => {
// writableEnded distinguishes a real disconnect.
let clientGone = false;
const clientAbort = new AbortController();
+ // Fallback-v2 hedging: the loop aborts this controller (via abortInFlight)
+ // when the wall-clock retry budget expires mid-attempt, canceling the
+ // in-flight upstream instead of waiting for a stalled attempt to time out.
+ const hedgeAbort = new AbortController();
res.on('close', () => {
if (!res.writableEnded) {
clientGone = true;
@@ -1789,6 +1801,7 @@ proxyRouter.post('/chat/completions', async (req: Request, res: Response) => {
state,
attemptLog,
clientGone: () => clientGone,
+ abortInFlight: () => hedgeAbort.abort(newHedgeAbortError()),
route: () => {
// When a handoff could fire this turn, pad the token estimate so the router's
// context-window and TPM checks account for the extra system message overhead.
@@ -1799,7 +1812,7 @@ proxyRouter.post('/chat/completions', async (req: Request, res: Response) => {
const routingEstimate = handoffPossible ? estimatedTotal + HANDOFF_MAX_TOKENS : estimatedTotal;
return routeRequest(routingEstimate, state.skipKeys.size > 0 ? state.skipKeys : undefined, preferredModel, hasImage, wantsTools, state.skipModels.size > 0 ? state.skipModels : undefined, groupChain ?? resolvedChain?.chain, samplingParams.response_format !== undefined, state.skipPlatforms.size > 0 ? state.skipPlatforms : undefined);
},
- dispatch: async (route, attempt) => {
+ dispatch: async (route, attempt, ctx) => {
const modelKey = `${route.platform}:${route.modelId}`;
traceRouteEvent('Proxy', {
event: attempt === 0 ? 'start' : 'next',
@@ -1883,6 +1896,9 @@ proxyRouter.post('/chat/completions', async (req: Request, res: Response) => {
res.setHeader('X-Routed-Via', routedViaValue(route.platform, route.modelId));
setFallbackHeaders(res, attempt, attemptLog);
headerSent = true;
+ // Committed: the answer is on its way, so the retry budget must no
+ // longer cancel this attempt (it could not fail over now anyway).
+ ctx.disarmHedge();
for (const p of preamble) res.write(`data: ${JSON.stringify(p)}\n\n`);
preamble.length = 0;
};
@@ -1898,7 +1914,7 @@ proxyRouter.post('/chat/completions', async (req: Request, res: Response) => {
try {
const gen = route.provider.streamChatCompletion(
route.apiKey, outboundMessages, route.modelId,
- { temperature, max_tokens, top_p, stop, tools, tool_choice, parallel_tool_calls, stream_options: parsed.data.stream_options, ...samplingParams, signal: clientAbort.signal },
+ { temperature, max_tokens, top_p, stop, tools, tool_choice, parallel_tool_calls, stream_options: parsed.data.stream_options, ...samplingParams, signal: AbortSignal.any([clientAbort.signal, hedgeAbort.signal]) },
quotaContextForRoute(route, 'chat/completions'),
);
@@ -2182,7 +2198,7 @@ proxyRouter.post('/chat/completions', async (req: Request, res: Response) => {
} else {
const result = await route.provider.chatCompletion(
route.apiKey, outboundMessages, route.modelId,
- { temperature, max_tokens, top_p, stop, tools, tool_choice, parallel_tool_calls, ...samplingParams, signal: clientAbort.signal },
+ { temperature, max_tokens, top_p, stop, tools, tool_choice, parallel_tool_calls, ...samplingParams, signal: AbortSignal.any([clientAbort.signal, hedgeAbort.signal]) },
quotaContextForRoute(route, 'chat/completions'),
);
diff --git a/server/src/routes/responses.ts b/server/src/routes/responses.ts
index 99d20569..ec718a99 100644
--- a/server/src/routes/responses.ts
+++ b/server/src/routes/responses.ts
@@ -34,7 +34,7 @@ import { applyTokenBudget, tokenBudgetMessage } from '../lib/guardrails.js';
import { samplingParamSchemaFields, pickSamplingParams, type ResponseFormat } from '../lib/sampling-params.js';
import { enforceJsonContent } from '../lib/structured-output.js';
import { sanitizeProviderErrorMessage } from '../lib/error-redaction.js';
-import { isClientAbortError, newClientAbortError } from '../lib/error-classify.js';
+import { isClientAbortError, newClientAbortError, newHedgeAbortError } from '../lib/error-classify.js';
import { inferQuotaPoolKey, type QuotaObservationContext } from '../services/provider-quota.js';
import { compressRequest, formatCompressionHeader } from '../services/compression/pipeline.js';
@@ -738,13 +738,17 @@ responsesRouter.post('/responses', async (req: Request, res: Response) => {
// writableEnded distinguishes a real disconnect.
let clientGone = false;
const clientAbort = new AbortController();
+ // Fallback-v2 hedging: the loop aborts this controller (via abortInFlight)
+ // when the wall-clock retry budget expires mid-attempt, canceling the
+ // in-flight upstream instead of waiting for a stalled attempt to time out.
+ const hedgeAbort = new AbortController();
res.on('close', () => {
if (!res.writableEnded) {
clientGone = true;
clientAbort.abort(newClientAbortError());
}
});
- const dispatchOpts = { ...completionOpts, signal: clientAbort.signal };
+ const dispatchOpts = { ...completionOpts, signal: AbortSignal.any([clientAbort.signal, hedgeAbort.signal]) };
// Stream bookkeeping (used only when stream === true). `streamStarted` is the
// commit flag: true once the response.created/in_progress skeleton has left,
@@ -763,8 +767,9 @@ responsesRouter.post('/responses', async (req: Request, res: Response) => {
state,
attemptLog,
clientGone: () => clientGone,
+ abortInFlight: () => hedgeAbort.abort(newHedgeAbortError()),
route: () => routeRequest(estimatedTotal, state.skipKeys.size > 0 ? state.skipKeys : undefined, preferredModel, hasImage, wantsTools, state.skipModels.size > 0 ? state.skipModels : undefined, groupChain, completionOpts.response_format !== undefined, state.skipPlatforms.size > 0 ? state.skipPlatforms : undefined),
- dispatch: async (route, attempt) => {
+ dispatch: async (route, attempt, ctx) => {
traceRouteEvent('Responses', {
event: attempt === 0 ? 'start' : 'next',
requestId: requestGroupId,
@@ -826,6 +831,9 @@ responsesRouter.post('/responses', async (req: Request, res: Response) => {
sse('response.created', { response: skeleton });
sse('response.in_progress', { response: skeleton });
streamStarted = true;
+ // Committed: the answer is on its way, so the retry budget must no
+ // longer cancel this attempt (it could not fail over now anyway).
+ ctx.disarmHedge();
};
// Open the text output item and stream `text` as its first delta.