Skip to content

7.4 billion tokens per month. 34 free LLM providers. 635 free model endpoints. All behind one /v1 endpoint, plus any custom OpenAI-compatible endpoint. Smart routing, automatic failover, encrypted keys. Personal experimentation only.

TypeScript

feat(routing): honor upstream Retry-After when benching a key

dansc67006c542268
diff --git a/server/src/__tests__/services/ratelimit.test.ts b/server/src/__tests__/services/ratelimit.test.ts
index 8b88e98b..d8f996da 100644
--- a/server/src/__tests__/services/ratelimit.test.ts
+++ b/server/src/__tests__/services/ratelimit.test.ts
@@ -12,6 +12,7 @@ import {
providerDailyRequestCount,
getProviderDailyRequestCap,
} from '../../services/ratelimit.js';
+import { parseRetryAfterMs } from '../../providers/base.js';
function removeDbFile(dbPath: string) {
for (const suffix of ['', '-shm', '-wal']) {
@@ -229,3 +230,44 @@ describe('Rate Limiter', () => {
});
});
});
+
+describe('Cooldown duration with upstream Retry-After', () => {
+ const noLimits = { rpd: null, tpd: null };
+ let testId: number;
+ beforeEach(() => { testId = Math.floor(Math.random() * 1_000_000); });
+
+ it('uses the transient cooldown when no Retry-After is given', () => {
+ expect(getCooldownDurationForLimit('groq', 'm', testId, noLimits)).toBe(90_000);
+ });
+
+ it('never benches shorter than the heuristic (ignores a shorter Retry-After)', () => {
+ expect(getCooldownDurationForLimit('groq', 'm', testId, noLimits, 30_000)).toBe(90_000);
+ });
+
+ it('honors a Retry-After longer than the heuristic', () => {
+ expect(getCooldownDurationForLimit('groq', 'm', testId, noLimits, 300_000)).toBe(300_000);
+ });
+
+ it('caps an absurd Retry-After at a day', () => {
+ expect(getCooldownDurationForLimit('groq', 'm', testId, noLimits, 5 * 86_400_000)).toBe(86_400_000);
+ });
+});
+
+describe('parseRetryAfterMs', () => {
+ it('parses delta-seconds', () => {
+ expect(parseRetryAfterMs('120')).toBe(120_000);
+ expect(parseRetryAfterMs('0')).toBe(0);
+ });
+
+ it('parses an HTTP-date into a positive future delay', () => {
+ const ms = parseRetryAfterMs(new Date(Date.now() + 60_000).toUTCString());
+ expect(ms).toBeGreaterThan(50_000);
+ expect(ms).toBeLessThanOrEqual(60_000);
+ });
+
+ it('returns undefined for absent or unparseable values', () => {
+ expect(parseRetryAfterMs(null)).toBeUndefined();
+ expect(parseRetryAfterMs('')).toBeUndefined();
+ expect(parseRetryAfterMs('soon')).toBeUndefined();
+ });
+});
diff --git a/server/src/providers/base.ts b/server/src/providers/base.ts
index 24d6d56f..6c396561 100644
--- a/server/src/providers/base.ts
+++ b/server/src/providers/base.ts
@@ -7,6 +7,36 @@ import type {
Platform,
} from '@freellmapi/shared/types.js';
+/** A provider HTTP error carrying the upstream status and, when the response
+ * included a Retry-After header, the parsed delay so the router can bench the
+ * key for at least that long. */
+export interface ProviderHttpError extends Error {
+ status?: number;
+ retryAfterMs?: number;
+}
+
+/** Parse an HTTP `Retry-After` header (delta-seconds or an HTTP-date) into a
+ * millisecond delay. Returns undefined when absent or unparseable. */
+export function parseRetryAfterMs(value: string | null | undefined): number | undefined {
+ if (!value) return undefined;
+ const trimmed = value.trim();
+ if (/^\d+$/.test(trimmed)) return Number(trimmed) * 1000;
+ const when = Date.parse(trimmed);
+ if (!Number.isNaN(when)) return Math.max(0, when - Date.now());
+ return undefined;
+}
+
+/** Build an error for a non-OK upstream response, capturing the status and any
+ * Retry-After hint. Used by every provider adapter so the proxy can honor a
+ * provider's explicit back-off when it sets the cooldown. */
+export function providerHttpError(res: Response, message: string): ProviderHttpError {
+ const err = new Error(message) as ProviderHttpError;
+ err.status = res.status;
+ const retryAfterMs = parseRetryAfterMs(res.headers?.get('retry-after'));
+ if (retryAfterMs !== undefined) err.retryAfterMs = retryAfterMs;
+ return err;
+}
+
export interface CompletionOptions {
model?: string;
temperature?: number;
diff --git a/server/src/providers/cloudflare.ts b/server/src/providers/cloudflare.ts
index a80ecf67..0a9074ff 100644
--- a/server/src/providers/cloudflare.ts
+++ b/server/src/providers/cloudflare.ts
@@ -3,7 +3,7 @@ import type {
ChatCompletionResponse,
ChatCompletionChunk,
} from '@freellmapi/shared/types.js';
-import { BaseProvider, type CompletionOptions } from './base.js';
+import { BaseProvider, providerHttpError, type CompletionOptions } from './base.js';
import { contentToString } from '../lib/content.js';
/**
@@ -58,7 +58,7 @@ export class CloudflareProvider extends BaseProvider {
if (!res.ok) {
const err = await res.json().catch(() => ({}));
- throw new Error(`Cloudflare API error ${res.status}: ${(err as any).error?.message ?? (err as any).errors?.[0]?.message ?? res.statusText}`);
+ throw providerHttpError(res, `Cloudflare API error ${res.status}: ${(err as any).error?.message ?? (err as any).errors?.[0]?.message ?? res.statusText}`);
}
const data = await res.json() as ChatCompletionResponse;
@@ -96,7 +96,7 @@ export class CloudflareProvider extends BaseProvider {
if (!res.ok) {
const err = await res.json().catch(() => ({}));
- throw new Error(`Cloudflare API error ${res.status}: ${(err as any).error?.message ?? (err as any).errors?.[0]?.message ?? res.statusText}`);
+ throw providerHttpError(res, `Cloudflare API error ${res.status}: ${(err as any).error?.message ?? (err as any).errors?.[0]?.message ?? res.statusText}`);
}
yield* this.readSseStream(res);
diff --git a/server/src/providers/cohere.ts b/server/src/providers/cohere.ts
index 5fcb0a4b..73737c9d 100644
--- a/server/src/providers/cohere.ts
+++ b/server/src/providers/cohere.ts
@@ -3,7 +3,7 @@ import type {
ChatCompletionResponse,
ChatCompletionChunk,
} from '@freellmapi/shared/types.js';
-import { BaseProvider, type CompletionOptions } from './base.js';
+import { BaseProvider, providerHttpError, type CompletionOptions } from './base.js';
import { flattenMessageContent } from '../lib/content.js';
const API_BASE = 'https://api.cohere.ai/compatibility/v1';
@@ -39,7 +39,7 @@ export class CohereProvider extends BaseProvider {
if (!res.ok) {
const err = await res.json().catch(() => ({}));
- throw new Error(`Cohere API error ${res.status}: ${(err as any).error?.message ?? res.statusText}`);
+ throw providerHttpError(res, `Cohere API error ${res.status}: ${(err as any).error?.message ?? res.statusText}`);
}
const data = await res.json() as ChatCompletionResponse;
@@ -75,7 +75,7 @@ export class CohereProvider extends BaseProvider {
if (!res.ok) {
const err = await res.json().catch(() => ({}));
- throw new Error(`Cohere API error ${res.status}: ${(err as any).error?.message ?? res.statusText}`);
+ throw providerHttpError(res, `Cohere API error ${res.status}: ${(err as any).error?.message ?? res.statusText}`);
}
yield* this.readSseStream(res);
diff --git a/server/src/providers/google.ts b/server/src/providers/google.ts
index 7273787c..05d629cc 100644
--- a/server/src/providers/google.ts
+++ b/server/src/providers/google.ts
@@ -7,7 +7,7 @@ import type {
ChatToolDefinition,
TokenUsage,
} from '@freellmapi/shared/types.js';
-import { BaseProvider, type CompletionOptions } from './base.js';
+import { BaseProvider, providerHttpError, type CompletionOptions } from './base.js';
import { contentToString } from '../lib/content.js';
const API_BASE = 'https://generativelanguage.googleapis.com/v1beta';
@@ -350,7 +350,7 @@ export class GoogleProvider extends BaseProvider {
if (!res.ok) {
const err = await res.json().catch(() => ({}));
- throw new Error(`Google API error ${res.status}: ${(err as any).error?.message ?? res.statusText}`);
+ throw providerHttpError(res, `Google API error ${res.status}: ${(err as any).error?.message ?? res.statusText}`);
}
const data = await res.json() as GeminiResponse;
@@ -413,7 +413,7 @@ export class GoogleProvider extends BaseProvider {
if (!res.ok) {
const err = await res.json().catch(() => ({}));
- throw new Error(`Google API error ${res.status}: ${(err as any).error?.message ?? res.statusText}`);
+ throw providerHttpError(res, `Google API error ${res.status}: ${(err as any).error?.message ?? res.statusText}`);
}
const reader = res.body?.getReader();
diff --git a/server/src/providers/openai-compat.ts b/server/src/providers/openai-compat.ts
index e632e4db..e441865b 100644
--- a/server/src/providers/openai-compat.ts
+++ b/server/src/providers/openai-compat.ts
@@ -4,7 +4,7 @@ import type {
ChatCompletionChunk,
Platform,
} from '@freellmapi/shared/types.js';
-import { BaseProvider, type CompletionOptions } from './base.js';
+import { BaseProvider, providerHttpError, type CompletionOptions } from './base.js';
/**
* Generic provider for platforms that use an OpenAI-compatible API.
@@ -74,7 +74,7 @@ export class OpenAICompatProvider extends BaseProvider {
if (!res.ok) {
const err = await res.json().catch(() => ({}));
- throw new Error(`${this.name} API error ${res.status}: ${(err as any).error?.message ?? res.statusText}`);
+ throw providerHttpError(res, `${this.name} API error ${res.status}: ${(err as any).error?.message ?? res.statusText}`);
}
let data: ChatCompletionResponse;
@@ -123,7 +123,7 @@ export class OpenAICompatProvider extends BaseProvider {
if (!res.ok) {
const err = await res.json().catch(() => ({}));
- throw new Error(`${this.name} API error ${res.status}: ${(err as any).error?.message ?? res.statusText}`);
+ throw providerHttpError(res, `${this.name} API error ${res.status}: ${(err as any).error?.message ?? res.statusText}`);
}
yield* this.readSseStream(res);
diff --git a/server/src/routes/proxy.ts b/server/src/routes/proxy.ts
index 44b98cd9..8b6a0b84 100644
--- a/server/src/routes/proxy.ts
+++ b/server/src/routes/proxy.ts
@@ -973,7 +973,7 @@ proxyRouter.post('/chat/completions', async (req: Request, res: Response) => {
: getCooldownDurationForLimit(route.platform, route.modelId, route.keyId, {
rpd: route.rpdLimit,
tpd: route.tpdLimit,
- }),
+ }, err.retryAfterMs),
);
recordRateLimitHit(route.modelDbId);
lastError = err;
diff --git a/server/src/services/ratelimit.ts b/server/src/services/ratelimit.ts
index 62344cb6..252ed09c 100644
--- a/server/src/services/ratelimit.ts
+++ b/server/src/services/ratelimit.ts
@@ -326,16 +326,21 @@ export function getCooldownDurationForLimit(
modelId: string,
keyId: number,
limits: { rpd: number | null; tpd: number | null },
+ retryAfterMs?: number | null,
): number {
const now = Date.now();
const rpdExhausted =
limits.rpd !== null && requestCount(platform, modelId, keyId, DAY, now) >= limits.rpd;
const tpdExhausted =
limits.tpd !== null && tokenCount(platform, modelId, keyId, DAY, now) >= limits.tpd;
- if (rpdExhausted || tpdExhausted) {
- return getNextCooldownDuration(platform, modelId, keyId);
- }
- return TRANSIENT_COOLDOWN_MS;
+ const base = (rpdExhausted || tpdExhausted)
+ ? getNextCooldownDuration(platform, modelId, keyId)
+ : TRANSIENT_COOLDOWN_MS;
+ // Honor an upstream Retry-After as a floor: never bench shorter than our own
+ // heuristic, but extend (capped at a day) when the provider explicitly asks
+ // to wait longer than we otherwise would.
+ if (retryAfterMs != null && retryAfterMs > base) return Math.min(retryAfterMs, DAY);
+ return base;
}
function persistedCooldownExpiry(