7.4 billion tokens per month. 34 free LLM providers. 635 free model endpoints. All behind one /v1 endpoint, plus any custom OpenAI-compatible endpoint. Smart routing, automatic failover, encrypted keys. Personal experimentation only.
TypeScript
feat(providers): honor a back-off stated in the error body, not just the header (#798)
OhOkThisIsFine6f13332ae2b1
diff --git a/server/src/__tests__/providers/stated-retry.test.ts b/server/src/__tests__/providers/stated-retry.test.tsnew file mode 100644index 00000000..5eacddf7--- /dev/null+++ b/server/src/__tests__/providers/stated-retry.test.ts@@ -0,0 +1,145 @@+import { describe, it, expect } from 'vitest';+import { parseStatedRetryMs, providerHttpError } from '../../providers/base.js';++// Not every provider uses the Retry-After header. Gemini answers a 429 with a+// google.rpc.RetryInfo in error.details[], and several OpenAI-compatible tiers+// only say it in prose. Those hints used to be thrown away, so a provider that+// told us exactly when to come back got the same heuristic cooldown ladder as+// one that said nothing.++const response = (status: number, headers: Record<string, string> = {}) =>+ new Response(null, { status, headers });++describe('parseStatedRetryMs — google.rpc.RetryInfo', () => {+ // The shape Gemini actually returns on a free-tier 429.+ const geminiQuotaBody = {+ error: {+ code: 429,+ message: 'You exceeded your current quota, please check your plan and billing details.',+ status: 'RESOURCE_EXHAUSTED',+ details: [+ { '@type': 'type.googleapis.com/google.rpc.QuotaFailure', violations: [{ quotaMetric: 'generate_requests_per_model_per_day' }] },+ { '@type': 'type.googleapis.com/google.rpc.RetryInfo', retryDelay: '17s' },+ ],+ },+ };++ it('reads the retryDelay out of the details array', () => {+ expect(parseStatedRetryMs(geminiQuotaBody)).toBe(17_000);+ });++ it('handles a fractional protobuf duration', () => {+ expect(parseStatedRetryMs({ error: { details: [{ retryDelay: '1.5s' }] } })).toBe(1_500);+ expect(parseStatedRetryMs({ error: { details: [{ retryDelay: '0.25s' }] } })).toBe(250);+ });+});++describe('parseStatedRetryMs — other stated shapes', () => {+ it('accepts snake_case and camelCase spellings', () => {+ expect(parseStatedRetryMs({ retry_after: 30 })).toBe(30_000);+ expect(parseStatedRetryMs({ retryAfter: 30 })).toBe(30_000);+ expect(parseStatedRetryMs({ error: { retry_after_seconds: 45 } })).toBe(45_000);+ });++ it('accepts a numeric value that arrived as a string', () => {+ expect(parseStatedRetryMs({ retry_after: '30' })).toBe(30_000);+ });++ it('finds the field however deeply the provider buried it', () => {+ expect(parseStatedRetryMs({ a: { b: { c: { retryDelay: '5s' } } } })).toBe(5_000);+ });+});++describe('parseStatedRetryMs — prose', () => {+ it('reads the Groq-style sentence', () => {+ const body = { error: { message: 'Rate limit reached for model X. Please try again in 7.66s.' } };+ expect(parseStatedRetryMs(body)).toBe(7_660);+ });++ it('understands the units providers actually use', () => {+ expect(parseStatedRetryMs('please retry after 30 seconds')).toBe(30_000);+ expect(parseStatedRetryMs('try again in 2m')).toBe(120_000);+ expect(parseStatedRetryMs('try again in 1 hour')).toBe(3_600_000);+ expect(parseStatedRetryMs('try again in 500ms')).toBe(500);+ });++ it('requires an explicit retry phrase, so a stray number is never mistaken for a back-off', () => {+ expect(parseStatedRetryMs('You have used 30 seconds of compute')).toBeUndefined();+ expect(parseStatedRetryMs({ error: { message: 'model gpt-4 is 30s slower than usual' } })).toBeUndefined();+ expect(parseStatedRetryMs({ error: { message: 'Invalid API Key' } })).toBeUndefined();+ });+});++describe('parseStatedRetryMs — refusals', () => {+ it('returns undefined for nothing, and never throws', () => {+ expect(parseStatedRetryMs(undefined)).toBeUndefined();+ expect(parseStatedRetryMs(null)).toBeUndefined();+ expect(parseStatedRetryMs({})).toBeUndefined();+ expect(parseStatedRetryMs(42)).toBeUndefined();+ });++ it('survives a circular body', () => {+ const circular: any = { error: {} };+ circular.error.self = circular;+ expect(() => parseStatedRetryMs(circular)).not.toThrow();+ });++ it('does not spin on a deeply nested body', () => {+ let deep: any = { retryDelay: '5s' };+ for (let i = 0; i < 50; i++) deep = { nested: deep };+ expect(() => parseStatedRetryMs(deep)).not.toThrow();+ });++ it('clamps a hostile value to a day, like the header path', () => {+ expect(parseStatedRetryMs({ retry_after: 99_999_999_999 })).toBe(24 * 60 * 60 * 1000);+ });++ it('ignores a negative delay rather than benching into the past', () => {+ expect(parseStatedRetryMs({ retry_after: -5 })).toBeUndefined();+ });+});++describe('providerHttpError', () => {+ it('picks up a body-stated delay when there is no Retry-After header', () => {+ const err = providerHttpError(response(429), 'Google API error 429: quota', {+ error: { details: [{ '@type': 'type.googleapis.com/google.rpc.RetryInfo', retryDelay: '17s' }] },+ });++ expect(err.status).toBe(429);+ expect(err.retryAfterMs).toBe(17_000);+ });++ it('lets the Retry-After header win, so existing behavior is unchanged', () => {+ // Both are provider-stated; the header is the standard channel.+ const err = providerHttpError(response(429, { 'retry-after': '60' }), 'rate limited', {+ error: { details: [{ retryDelay: '17s' }] },+ });++ expect(err.retryAfterMs).toBe(60_000);+ });++ it('is unchanged when neither states a delay', () => {+ const err = providerHttpError(response(500), 'upstream boom', { error: { message: 'boom' } });++ expect(err.status).toBe(500);+ expect(err.retryAfterMs).toBeUndefined();+ });++ it('still works when no body is passed at all', () => {+ const err = providerHttpError(response(429, { 'retry-after': '5' }), 'rate limited');++ expect(err.retryAfterMs).toBe(5_000);+ });++ it('keeps only the number — the body itself is never retained', () => {+ // Nothing from the payload should be able to reach a log or the trace+ // through this object.+ const err = providerHttpError(response(429), 'rate limited', {+ error: { retryDelay: '3s', apiKey: 'sk-SECRET', message: 'quota' },+ });++ expect(err.retryAfterMs).toBe(3_000);+ expect(JSON.stringify(err)).not.toContain('SECRET');+ expect(Object.keys(err)).toEqual(expect.not.arrayContaining(['body']));+ });+});diff --git a/server/src/providers/aihorde.ts b/server/src/providers/aihorde.tsindex 196ee34a..23120d2c 100644--- a/server/src/providers/aihorde.ts+++ b/server/src/providers/aihorde.ts@@ -154,7 +154,7 @@ export class AIHordeProvider extends BaseProvider {if (!res.ok) {const err = await res.json().catch(() => ({}));- throw providerHttpError(res, `${this.name} API error ${res.status}: ${this.parseError(err, res.status, res.statusText)}`);+ throw providerHttpError(res, `${this.name} API error ${res.status}: ${this.parseError(err, res.status, res.statusText)}`, err);}const data = await res.json() as ChatCompletionResponse;diff --git a/server/src/providers/base.ts b/server/src/providers/base.tsindex 93a49b9f..d27f60f9 100644--- a/server/src/providers/base.ts+++ b/server/src/providers/base.ts@@ -12,9 +12,10 @@ import { proxyFetch } from '../lib/proxy.js';import { providerTimeoutMs, streamStallTimeoutMs } from '../lib/provider-timeout.js';import { extractThinkTagsFromStream } from '../lib/think-tags.js';-/** A provider HTTP error carrying the upstream status and, when the response- * included a Retry-After header, the parsed delay so the router can bench the- * key for at least that long. */+/** A provider HTTP error carrying the upstream status and, when the provider+ * stated one, the parsed back-off so the router can bench the key for at least+ * that long. The delay may come from the Retry-After header or from the error+ * body — see providerHttpError. */export interface ProviderHttpError extends Error {status?: number;retryAfterMs?: number;@@ -40,13 +41,109 @@ export function parseRetryAfterMs(value: string | null | undefined): number | unreturn undefined;}+/** A protobuf Duration, as google.rpc.RetryInfo spells its `retryDelay`:+ * digits, an optional fraction, then a literal `s` ("17s", "1.5s", "0.25s"). */+const PROTOBUF_DURATION = /^(\d+(?:\.\d+)?)s$/;++/** "Please try again in 7.66s" / "retry after 30 seconds" / "try again in 2m".+ * Anchored on an explicit retry phrase so an unrelated number in an error+ * message can never be mistaken for a back-off. */+const PROSE_RETRY = /(?:try again|retry)\s+(?:in|after)\s+(\d+(?:\.\d+)?)\s*(ms|s|sec|secs|second|seconds|m|min|mins|minute|minutes|h|hour|hours)\b/i;++const UNIT_MS: Record<string, number> = {+ ms: 1,+ s: 1000, sec: 1000, secs: 1000, second: 1000, seconds: 1000,+ m: 60_000, min: 60_000, mins: 60_000, minute: 60_000, minutes: 60_000,+ h: 3_600_000, hour: 3_600_000, hours: 3_600_000,+};++function clampRetryMs(ms: number): number | undefined {+ if (!Number.isFinite(ms) || ms < 0) return undefined;+ return Math.min(ms, MAX_RETRY_AFTER_MS);+}++/** Depth-first search for the first `retryDelay`/`retry_after`-shaped field. The+ * shape varies by provider and by endpoint within a provider, so walking is+ * more durable than a list of exact paths — and cheaper to maintain than one+ * parser per adapter. Depth-capped so a pathological body cannot spin. */+function findStatedDelayMs(node: unknown, depth = 0): number | undefined {+ if (depth > 6 || node === null || typeof node !== 'object') return undefined;++ if (Array.isArray(node)) {+ for (const item of node) {+ const found = findStatedDelayMs(item, depth + 1);+ if (found !== undefined) return found;+ }+ return undefined;+ }++ for (const [key, value] of Object.entries(node as Record<string, unknown>)) {+ const normalized = key.toLowerCase().replace(/[_-]/g, '');+ if (normalized === 'retrydelay' || normalized === 'retryafter' || normalized === 'retryafterseconds') {+ // google.rpc.RetryInfo states "17s"; other providers state bare seconds.+ if (typeof value === 'string') {+ const duration = PROTOBUF_DURATION.exec(value.trim());+ if (duration) return clampRetryMs(Number(duration[1]) * 1000);+ if (/^\d+(\.\d+)?$/.test(value.trim())) return clampRetryMs(Number(value) * 1000);+ }+ if (typeof value === 'number') return clampRetryMs(value * 1000);+ }+ const found = findStatedDelayMs(value, depth + 1);+ if (found !== undefined) return found;+ }+ return undefined;+}++/**+ * The back-off a provider stated inside its error BODY, in milliseconds.+ *+ * Not every provider uses the Retry-After header. Gemini answers a 429 with+ * `error.details[]` carrying a `google.rpc.RetryInfo` whose `retryDelay` is+ * "17s", and several OpenAI-compatible tiers only say it in prose. Those hints+ * were being thrown away: the body is flattened to a single message string+ * before it reaches the router, so a provider that told us exactly when to come+ * back got the same heuristic cooldown ladder as one that said nothing.+ */+export function parseStatedRetryMs(body: unknown): number | undefined {+ if (body == null) return undefined;++ if (typeof body !== 'string') {+ const structured = findStatedDelayMs(body);+ if (structured !== undefined) return structured;+ }++ // Prose last: a stated field is a promise, a sentence is an observation.+ const text = typeof body === 'string' ? body : safeStringify(body);+ const prose = text ? PROSE_RETRY.exec(text) : null;+ if (prose) {+ const unit = UNIT_MS[prose[2].toLowerCase()];+ if (unit) return clampRetryMs(Number(prose[1]) * unit);+ }+ return undefined;+}++function safeStringify(value: unknown): string {+ try {+ return JSON.stringify(value) ?? '';+ } catch {+ return ''; // Circular or otherwise unserialisable — no prose to read.+ }+}+/** Build an error for a non-OK upstream response, capturing the status and any- * Retry-After hint. Used by every provider adapter so the proxy can honor a- * provider's explicit back-off when it sets the cooldown. */-export function providerHttpError(res: Response, message: string): ProviderHttpError {+ * stated back-off. Used by every provider adapter so the proxy can honor a+ * provider's explicit hint when it sets the cooldown.+ *+ * `body` is the already-parsed error payload the caller used to build+ * `message`. Only a delay is read out of it and only a number is kept — the+ * body itself is never retained, so nothing extra reaches a log or the+ * attempt trace. The Retry-After header still wins when both are present: it+ * is the standard channel, and preferring it keeps existing behavior exactly+ * as it was. */+export function providerHttpError(res: Response, message: string, body?: unknown): ProviderHttpError {const err = new Error(message) as ProviderHttpError;err.status = res.status;- const retryAfterMs = parseRetryAfterMs(res.headers?.get('retry-after'));+ const retryAfterMs = parseRetryAfterMs(res.headers?.get('retry-after')) ?? parseStatedRetryMs(body);if (retryAfterMs !== undefined) err.retryAfterMs = retryAfterMs;return err;}diff --git a/server/src/providers/cloudflare.ts b/server/src/providers/cloudflare.tsindex db76f2eb..bc681b9f 100644--- a/server/src/providers/cloudflare.ts+++ b/server/src/providers/cloudflare.ts@@ -91,7 +91,7 @@ export class CloudflareProvider extends BaseProvider {if (!res.ok) {const err = await res.json().catch(() => ({}));- throw providerHttpError(res, `Cloudflare API error ${res.status}: ${(err as any).error?.message ?? (err as any).errors?.[0]?.message ?? res.statusText}`);+ throw providerHttpError(res, `Cloudflare API error ${res.status}: ${(err as any).error?.message ?? (err as any).errors?.[0]?.message ?? res.statusText}`, err);}const data = await res.json() as ChatCompletionResponse;@@ -148,7 +148,7 @@ export class CloudflareProvider extends BaseProvider {if (!res.ok) {const err = await res.json().catch(() => ({}));- throw providerHttpError(res, `Cloudflare API error ${res.status}: ${(err as any).error?.message ?? (err as any).errors?.[0]?.message ?? res.statusText}`);+ throw providerHttpError(res, `Cloudflare API error ${res.status}: ${(err as any).error?.message ?? (err as any).errors?.[0]?.message ?? res.statusText}`, err);}// First-byte grace (#584): reuse the per-model chat timeout (GLM 4.7diff --git a/server/src/providers/cohere.ts b/server/src/providers/cohere.tsindex 9e1ef29c..b134a232 100644--- a/server/src/providers/cohere.ts+++ b/server/src/providers/cohere.ts@@ -73,7 +73,7 @@ export class CohereProvider extends BaseProvider {if (!res.ok) {const err = await res.json().catch(() => ({}));- throw providerHttpError(res, `Cohere API error ${res.status}: ${(err as any).error?.message ?? res.statusText}`);+ throw providerHttpError(res, `Cohere API error ${res.status}: ${(err as any).error?.message ?? res.statusText}`, err);}const data = await res.json() as ChatCompletionResponse;@@ -122,7 +122,7 @@ export class CohereProvider extends BaseProvider {if (!res.ok) {const err = await res.json().catch(() => ({}));- throw providerHttpError(res, `Cohere API error ${res.status}: ${(err as any).error?.message ?? res.statusText}`);+ throw providerHttpError(res, `Cohere API error ${res.status}: ${(err as any).error?.message ?? res.statusText}`, err);}yield* this.readSseStream(res);diff --git a/server/src/providers/google.ts b/server/src/providers/google.tsindex 7a32ba4e..3c10e28b 100644--- a/server/src/providers/google.ts+++ b/server/src/providers/google.ts@@ -557,7 +557,7 @@ export class GoogleProvider extends BaseProvider {if (!res.ok) {const err = await res.json().catch(() => ({}));- throw providerHttpError(res, `Google API error ${res.status}: ${(err as any).error?.message ?? res.statusText}`);+ throw providerHttpError(res, `Google API error ${res.status}: ${(err as any).error?.message ?? res.statusText}`, err);}const data = await res.json() as GeminiResponse;@@ -638,7 +638,7 @@ export class GoogleProvider extends BaseProvider {if (!res.ok) {const err = await res.json().catch(() => ({}));- throw providerHttpError(res, `Google API error ${res.status}: ${(err as any).error?.message ?? res.statusText}`);+ throw providerHttpError(res, `Google API error ${res.status}: ${(err as any).error?.message ?? res.statusText}`, err);}const reader = res.body?.getReader();diff --git a/server/src/providers/openai-compat.ts b/server/src/providers/openai-compat.tsindex 423700db..5fabb5f6 100644--- a/server/src/providers/openai-compat.ts+++ b/server/src/providers/openai-compat.ts@@ -224,7 +224,7 @@ export class OpenAICompatProvider extends BaseProvider {out._routed_via = { platform: this.platform, model: modelId };return out;}- throw providerHttpError(res, `${this.name} API error ${res.status}: ${this.upstreamErrorText(err, res)}`);+ throw providerHttpError(res, `${this.name} API error ${res.status}: ${this.upstreamErrorText(err, res)}`, err);}let data: ChatCompletionResponse;@@ -339,7 +339,7 @@ export class OpenAICompatProvider extends BaseProvider {yield { ...base, choices: [{ index: 0, delta: {}, finish_reason: 'tool_calls' }] };return;}- throw providerHttpError(res, `${this.name} API error ${res.status}: ${this.upstreamErrorText(err, res)}`);+ throw providerHttpError(res, `${this.name} API error ${res.status}: ${this.upstreamErrorText(err, res)}`, err);}// First-byte grace (#584): the same chat timeout that bounded the headers