Skip to content

7.4 billion tokens per month. 34 free LLM providers. 635 free model endpoints. All behind one /v1 endpoint, plus any custom OpenAI-compatible endpoint. Smart routing, automatic failover, encrypted keys. Personal experimentation only.

TypeScript

feat(providers): honor a back-off stated in the error body, not just the header (#798)

OhOkThisIsFine6f13332ae2b1
diff --git a/server/src/__tests__/providers/stated-retry.test.ts b/server/src/__tests__/providers/stated-retry.test.ts
new file mode 100644
index 00000000..5eacddf7
--- /dev/null
+++ b/server/src/__tests__/providers/stated-retry.test.ts
@@ -0,0 +1,145 @@
+import { describe, it, expect } from 'vitest';
+import { parseStatedRetryMs, providerHttpError } from '../../providers/base.js';
+
+// Not every provider uses the Retry-After header. Gemini answers a 429 with a
+// google.rpc.RetryInfo in error.details[], and several OpenAI-compatible tiers
+// only say it in prose. Those hints used to be thrown away, so a provider that
+// told us exactly when to come back got the same heuristic cooldown ladder as
+// one that said nothing.
+
+const response = (status: number, headers: Record<string, string> = {}) =>
+ new Response(null, { status, headers });
+
+describe('parseStatedRetryMs — google.rpc.RetryInfo', () => {
+ // The shape Gemini actually returns on a free-tier 429.
+ const geminiQuotaBody = {
+ error: {
+ code: 429,
+ message: 'You exceeded your current quota, please check your plan and billing details.',
+ status: 'RESOURCE_EXHAUSTED',
+ details: [
+ { '@type': 'type.googleapis.com/google.rpc.QuotaFailure', violations: [{ quotaMetric: 'generate_requests_per_model_per_day' }] },
+ { '@type': 'type.googleapis.com/google.rpc.RetryInfo', retryDelay: '17s' },
+ ],
+ },
+ };
+
+ it('reads the retryDelay out of the details array', () => {
+ expect(parseStatedRetryMs(geminiQuotaBody)).toBe(17_000);
+ });
+
+ it('handles a fractional protobuf duration', () => {
+ expect(parseStatedRetryMs({ error: { details: [{ retryDelay: '1.5s' }] } })).toBe(1_500);
+ expect(parseStatedRetryMs({ error: { details: [{ retryDelay: '0.25s' }] } })).toBe(250);
+ });
+});
+
+describe('parseStatedRetryMs — other stated shapes', () => {
+ it('accepts snake_case and camelCase spellings', () => {
+ expect(parseStatedRetryMs({ retry_after: 30 })).toBe(30_000);
+ expect(parseStatedRetryMs({ retryAfter: 30 })).toBe(30_000);
+ expect(parseStatedRetryMs({ error: { retry_after_seconds: 45 } })).toBe(45_000);
+ });
+
+ it('accepts a numeric value that arrived as a string', () => {
+ expect(parseStatedRetryMs({ retry_after: '30' })).toBe(30_000);
+ });
+
+ it('finds the field however deeply the provider buried it', () => {
+ expect(parseStatedRetryMs({ a: { b: { c: { retryDelay: '5s' } } } })).toBe(5_000);
+ });
+});
+
+describe('parseStatedRetryMs — prose', () => {
+ it('reads the Groq-style sentence', () => {
+ const body = { error: { message: 'Rate limit reached for model X. Please try again in 7.66s.' } };
+ expect(parseStatedRetryMs(body)).toBe(7_660);
+ });
+
+ it('understands the units providers actually use', () => {
+ expect(parseStatedRetryMs('please retry after 30 seconds')).toBe(30_000);
+ expect(parseStatedRetryMs('try again in 2m')).toBe(120_000);
+ expect(parseStatedRetryMs('try again in 1 hour')).toBe(3_600_000);
+ expect(parseStatedRetryMs('try again in 500ms')).toBe(500);
+ });
+
+ it('requires an explicit retry phrase, so a stray number is never mistaken for a back-off', () => {
+ expect(parseStatedRetryMs('You have used 30 seconds of compute')).toBeUndefined();
+ expect(parseStatedRetryMs({ error: { message: 'model gpt-4 is 30s slower than usual' } })).toBeUndefined();
+ expect(parseStatedRetryMs({ error: { message: 'Invalid API Key' } })).toBeUndefined();
+ });
+});
+
+describe('parseStatedRetryMs — refusals', () => {
+ it('returns undefined for nothing, and never throws', () => {
+ expect(parseStatedRetryMs(undefined)).toBeUndefined();
+ expect(parseStatedRetryMs(null)).toBeUndefined();
+ expect(parseStatedRetryMs({})).toBeUndefined();
+ expect(parseStatedRetryMs(42)).toBeUndefined();
+ });
+
+ it('survives a circular body', () => {
+ const circular: any = { error: {} };
+ circular.error.self = circular;
+ expect(() => parseStatedRetryMs(circular)).not.toThrow();
+ });
+
+ it('does not spin on a deeply nested body', () => {
+ let deep: any = { retryDelay: '5s' };
+ for (let i = 0; i < 50; i++) deep = { nested: deep };
+ expect(() => parseStatedRetryMs(deep)).not.toThrow();
+ });
+
+ it('clamps a hostile value to a day, like the header path', () => {
+ expect(parseStatedRetryMs({ retry_after: 99_999_999_999 })).toBe(24 * 60 * 60 * 1000);
+ });
+
+ it('ignores a negative delay rather than benching into the past', () => {
+ expect(parseStatedRetryMs({ retry_after: -5 })).toBeUndefined();
+ });
+});
+
+describe('providerHttpError', () => {
+ it('picks up a body-stated delay when there is no Retry-After header', () => {
+ const err = providerHttpError(response(429), 'Google API error 429: quota', {
+ error: { details: [{ '@type': 'type.googleapis.com/google.rpc.RetryInfo', retryDelay: '17s' }] },
+ });
+
+ expect(err.status).toBe(429);
+ expect(err.retryAfterMs).toBe(17_000);
+ });
+
+ it('lets the Retry-After header win, so existing behavior is unchanged', () => {
+ // Both are provider-stated; the header is the standard channel.
+ const err = providerHttpError(response(429, { 'retry-after': '60' }), 'rate limited', {
+ error: { details: [{ retryDelay: '17s' }] },
+ });
+
+ expect(err.retryAfterMs).toBe(60_000);
+ });
+
+ it('is unchanged when neither states a delay', () => {
+ const err = providerHttpError(response(500), 'upstream boom', { error: { message: 'boom' } });
+
+ expect(err.status).toBe(500);
+ expect(err.retryAfterMs).toBeUndefined();
+ });
+
+ it('still works when no body is passed at all', () => {
+ const err = providerHttpError(response(429, { 'retry-after': '5' }), 'rate limited');
+
+ expect(err.retryAfterMs).toBe(5_000);
+ });
+
+ it('keeps only the number — the body itself is never retained', () => {
+ // Nothing from the payload should be able to reach a log or the trace
+ // through this object.
+ const err = providerHttpError(response(429), 'rate limited', {
+ error: { retryDelay: '3s', apiKey: 'sk-SECRET', message: 'quota' },
+ });
+
+ expect(err.retryAfterMs).toBe(3_000);
+ expect(JSON.stringify(err)).not.toContain('SECRET');
+ expect(Object.keys(err)).toEqual(expect.not.arrayContaining(['body']));
+ });
+});
diff --git a/server/src/providers/aihorde.ts b/server/src/providers/aihorde.ts
index 196ee34a..23120d2c 100644
--- a/server/src/providers/aihorde.ts
+++ b/server/src/providers/aihorde.ts
@@ -154,7 +154,7 @@ export class AIHordeProvider extends BaseProvider {
if (!res.ok) {
const err = await res.json().catch(() => ({}));
- throw providerHttpError(res, `${this.name} API error ${res.status}: ${this.parseError(err, res.status, res.statusText)}`);
+ throw providerHttpError(res, `${this.name} API error ${res.status}: ${this.parseError(err, res.status, res.statusText)}`, err);
}
const data = await res.json() as ChatCompletionResponse;
diff --git a/server/src/providers/base.ts b/server/src/providers/base.ts
index 93a49b9f..d27f60f9 100644
--- a/server/src/providers/base.ts
+++ b/server/src/providers/base.ts
@@ -12,9 +12,10 @@ import { proxyFetch } from '../lib/proxy.js';
import { providerTimeoutMs, streamStallTimeoutMs } from '../lib/provider-timeout.js';
import { extractThinkTagsFromStream } from '../lib/think-tags.js';
-/** A provider HTTP error carrying the upstream status and, when the response
- * included a Retry-After header, the parsed delay so the router can bench the
- * key for at least that long. */
+/** A provider HTTP error carrying the upstream status and, when the provider
+ * stated one, the parsed back-off so the router can bench the key for at least
+ * that long. The delay may come from the Retry-After header or from the error
+ * body — see providerHttpError. */
export interface ProviderHttpError extends Error {
status?: number;
retryAfterMs?: number;
@@ -40,13 +41,109 @@ export function parseRetryAfterMs(value: string | null | undefined): number | un
return undefined;
}
+/** A protobuf Duration, as google.rpc.RetryInfo spells its `retryDelay`:
+ * digits, an optional fraction, then a literal `s` ("17s", "1.5s", "0.25s"). */
+const PROTOBUF_DURATION = /^(\d+(?:\.\d+)?)s$/;
+
+/** "Please try again in 7.66s" / "retry after 30 seconds" / "try again in 2m".
+ * Anchored on an explicit retry phrase so an unrelated number in an error
+ * message can never be mistaken for a back-off. */
+const PROSE_RETRY = /(?:try again|retry)\s+(?:in|after)\s+(\d+(?:\.\d+)?)\s*(ms|s|sec|secs|second|seconds|m|min|mins|minute|minutes|h|hour|hours)\b/i;
+
+const UNIT_MS: Record<string, number> = {
+ ms: 1,
+ s: 1000, sec: 1000, secs: 1000, second: 1000, seconds: 1000,
+ m: 60_000, min: 60_000, mins: 60_000, minute: 60_000, minutes: 60_000,
+ h: 3_600_000, hour: 3_600_000, hours: 3_600_000,
+};
+
+function clampRetryMs(ms: number): number | undefined {
+ if (!Number.isFinite(ms) || ms < 0) return undefined;
+ return Math.min(ms, MAX_RETRY_AFTER_MS);
+}
+
+/** Depth-first search for the first `retryDelay`/`retry_after`-shaped field. The
+ * shape varies by provider and by endpoint within a provider, so walking is
+ * more durable than a list of exact paths — and cheaper to maintain than one
+ * parser per adapter. Depth-capped so a pathological body cannot spin. */
+function findStatedDelayMs(node: unknown, depth = 0): number | undefined {
+ if (depth > 6 || node === null || typeof node !== 'object') return undefined;
+
+ if (Array.isArray(node)) {
+ for (const item of node) {
+ const found = findStatedDelayMs(item, depth + 1);
+ if (found !== undefined) return found;
+ }
+ return undefined;
+ }
+
+ for (const [key, value] of Object.entries(node as Record<string, unknown>)) {
+ const normalized = key.toLowerCase().replace(/[_-]/g, '');
+ if (normalized === 'retrydelay' || normalized === 'retryafter' || normalized === 'retryafterseconds') {
+ // google.rpc.RetryInfo states "17s"; other providers state bare seconds.
+ if (typeof value === 'string') {
+ const duration = PROTOBUF_DURATION.exec(value.trim());
+ if (duration) return clampRetryMs(Number(duration[1]) * 1000);
+ if (/^\d+(\.\d+)?$/.test(value.trim())) return clampRetryMs(Number(value) * 1000);
+ }
+ if (typeof value === 'number') return clampRetryMs(value * 1000);
+ }
+ const found = findStatedDelayMs(value, depth + 1);
+ if (found !== undefined) return found;
+ }
+ return undefined;
+}
+
+/**
+ * The back-off a provider stated inside its error BODY, in milliseconds.
+ *
+ * Not every provider uses the Retry-After header. Gemini answers a 429 with
+ * `error.details[]` carrying a `google.rpc.RetryInfo` whose `retryDelay` is
+ * "17s", and several OpenAI-compatible tiers only say it in prose. Those hints
+ * were being thrown away: the body is flattened to a single message string
+ * before it reaches the router, so a provider that told us exactly when to come
+ * back got the same heuristic cooldown ladder as one that said nothing.
+ */
+export function parseStatedRetryMs(body: unknown): number | undefined {
+ if (body == null) return undefined;
+
+ if (typeof body !== 'string') {
+ const structured = findStatedDelayMs(body);
+ if (structured !== undefined) return structured;
+ }
+
+ // Prose last: a stated field is a promise, a sentence is an observation.
+ const text = typeof body === 'string' ? body : safeStringify(body);
+ const prose = text ? PROSE_RETRY.exec(text) : null;
+ if (prose) {
+ const unit = UNIT_MS[prose[2].toLowerCase()];
+ if (unit) return clampRetryMs(Number(prose[1]) * unit);
+ }
+ return undefined;
+}
+
+function safeStringify(value: unknown): string {
+ try {
+ return JSON.stringify(value) ?? '';
+ } catch {
+ return ''; // Circular or otherwise unserialisable — no prose to read.
+ }
+}
+
/** Build an error for a non-OK upstream response, capturing the status and any
- * Retry-After hint. Used by every provider adapter so the proxy can honor a
- * provider's explicit back-off when it sets the cooldown. */
-export function providerHttpError(res: Response, message: string): ProviderHttpError {
+ * stated back-off. Used by every provider adapter so the proxy can honor a
+ * provider's explicit hint when it sets the cooldown.
+ *
+ * `body` is the already-parsed error payload the caller used to build
+ * `message`. Only a delay is read out of it and only a number is kept — the
+ * body itself is never retained, so nothing extra reaches a log or the
+ * attempt trace. The Retry-After header still wins when both are present: it
+ * is the standard channel, and preferring it keeps existing behavior exactly
+ * as it was. */
+export function providerHttpError(res: Response, message: string, body?: unknown): ProviderHttpError {
const err = new Error(message) as ProviderHttpError;
err.status = res.status;
- const retryAfterMs = parseRetryAfterMs(res.headers?.get('retry-after'));
+ const retryAfterMs = parseRetryAfterMs(res.headers?.get('retry-after')) ?? parseStatedRetryMs(body);
if (retryAfterMs !== undefined) err.retryAfterMs = retryAfterMs;
return err;
}
diff --git a/server/src/providers/cloudflare.ts b/server/src/providers/cloudflare.ts
index db76f2eb..bc681b9f 100644
--- a/server/src/providers/cloudflare.ts
+++ b/server/src/providers/cloudflare.ts
@@ -91,7 +91,7 @@ export class CloudflareProvider extends BaseProvider {
if (!res.ok) {
const err = await res.json().catch(() => ({}));
- throw providerHttpError(res, `Cloudflare API error ${res.status}: ${(err as any).error?.message ?? (err as any).errors?.[0]?.message ?? res.statusText}`);
+ throw providerHttpError(res, `Cloudflare API error ${res.status}: ${(err as any).error?.message ?? (err as any).errors?.[0]?.message ?? res.statusText}`, err);
}
const data = await res.json() as ChatCompletionResponse;
@@ -148,7 +148,7 @@ export class CloudflareProvider extends BaseProvider {
if (!res.ok) {
const err = await res.json().catch(() => ({}));
- throw providerHttpError(res, `Cloudflare API error ${res.status}: ${(err as any).error?.message ?? (err as any).errors?.[0]?.message ?? res.statusText}`);
+ throw providerHttpError(res, `Cloudflare API error ${res.status}: ${(err as any).error?.message ?? (err as any).errors?.[0]?.message ?? res.statusText}`, err);
}
// First-byte grace (#584): reuse the per-model chat timeout (GLM 4.7
diff --git a/server/src/providers/cohere.ts b/server/src/providers/cohere.ts
index 9e1ef29c..b134a232 100644
--- a/server/src/providers/cohere.ts
+++ b/server/src/providers/cohere.ts
@@ -73,7 +73,7 @@ export class CohereProvider extends BaseProvider {
if (!res.ok) {
const err = await res.json().catch(() => ({}));
- throw providerHttpError(res, `Cohere API error ${res.status}: ${(err as any).error?.message ?? res.statusText}`);
+ throw providerHttpError(res, `Cohere API error ${res.status}: ${(err as any).error?.message ?? res.statusText}`, err);
}
const data = await res.json() as ChatCompletionResponse;
@@ -122,7 +122,7 @@ export class CohereProvider extends BaseProvider {
if (!res.ok) {
const err = await res.json().catch(() => ({}));
- throw providerHttpError(res, `Cohere API error ${res.status}: ${(err as any).error?.message ?? res.statusText}`);
+ throw providerHttpError(res, `Cohere API error ${res.status}: ${(err as any).error?.message ?? res.statusText}`, err);
}
yield* this.readSseStream(res);
diff --git a/server/src/providers/google.ts b/server/src/providers/google.ts
index 7a32ba4e..3c10e28b 100644
--- a/server/src/providers/google.ts
+++ b/server/src/providers/google.ts
@@ -557,7 +557,7 @@ export class GoogleProvider extends BaseProvider {
if (!res.ok) {
const err = await res.json().catch(() => ({}));
- throw providerHttpError(res, `Google API error ${res.status}: ${(err as any).error?.message ?? res.statusText}`);
+ throw providerHttpError(res, `Google API error ${res.status}: ${(err as any).error?.message ?? res.statusText}`, err);
}
const data = await res.json() as GeminiResponse;
@@ -638,7 +638,7 @@ export class GoogleProvider extends BaseProvider {
if (!res.ok) {
const err = await res.json().catch(() => ({}));
- throw providerHttpError(res, `Google API error ${res.status}: ${(err as any).error?.message ?? res.statusText}`);
+ throw providerHttpError(res, `Google API error ${res.status}: ${(err as any).error?.message ?? res.statusText}`, err);
}
const reader = res.body?.getReader();
diff --git a/server/src/providers/openai-compat.ts b/server/src/providers/openai-compat.ts
index 423700db..5fabb5f6 100644
--- a/server/src/providers/openai-compat.ts
+++ b/server/src/providers/openai-compat.ts
@@ -224,7 +224,7 @@ export class OpenAICompatProvider extends BaseProvider {
out._routed_via = { platform: this.platform, model: modelId };
return out;
}
- throw providerHttpError(res, `${this.name} API error ${res.status}: ${this.upstreamErrorText(err, res)}`);
+ throw providerHttpError(res, `${this.name} API error ${res.status}: ${this.upstreamErrorText(err, res)}`, err);
}
let data: ChatCompletionResponse;
@@ -339,7 +339,7 @@ export class OpenAICompatProvider extends BaseProvider {
yield { ...base, choices: [{ index: 0, delta: {}, finish_reason: 'tool_calls' }] };
return;
}
- throw providerHttpError(res, `${this.name} API error ${res.status}: ${this.upstreamErrorText(err, res)}`);
+ throw providerHttpError(res, `${this.name} API error ${res.status}: ${this.upstreamErrorText(err, res)}`, err);
}
// First-byte grace (#584): the same chat timeout that bounded the headers