7.4 billion tokens per month. 34 free LLM providers. 635 free model endpoints. All behind one /v1 endpoint, plus any custom OpenAI-compatible endpoint. Smart routing, automatic failover, encrypted keys. Personal experimentation only.
TypeScript
feat(routing): honor upstream Retry-After when benching a key
dansc67006c542268
diff --git a/server/src/__tests__/services/ratelimit.test.ts b/server/src/__tests__/services/ratelimit.test.tsindex 8b88e98b..d8f996da 100644--- a/server/src/__tests__/services/ratelimit.test.ts+++ b/server/src/__tests__/services/ratelimit.test.ts@@ -12,6 +12,7 @@ import {providerDailyRequestCount,getProviderDailyRequestCap,} from '../../services/ratelimit.js';+import { parseRetryAfterMs } from '../../providers/base.js';function removeDbFile(dbPath: string) {for (const suffix of ['', '-shm', '-wal']) {@@ -229,3 +230,44 @@ describe('Rate Limiter', () => {});});});++describe('Cooldown duration with upstream Retry-After', () => {+ const noLimits = { rpd: null, tpd: null };+ let testId: number;+ beforeEach(() => { testId = Math.floor(Math.random() * 1_000_000); });++ it('uses the transient cooldown when no Retry-After is given', () => {+ expect(getCooldownDurationForLimit('groq', 'm', testId, noLimits)).toBe(90_000);+ });++ it('never benches shorter than the heuristic (ignores a shorter Retry-After)', () => {+ expect(getCooldownDurationForLimit('groq', 'm', testId, noLimits, 30_000)).toBe(90_000);+ });++ it('honors a Retry-After longer than the heuristic', () => {+ expect(getCooldownDurationForLimit('groq', 'm', testId, noLimits, 300_000)).toBe(300_000);+ });++ it('caps an absurd Retry-After at a day', () => {+ expect(getCooldownDurationForLimit('groq', 'm', testId, noLimits, 5 * 86_400_000)).toBe(86_400_000);+ });+});++describe('parseRetryAfterMs', () => {+ it('parses delta-seconds', () => {+ expect(parseRetryAfterMs('120')).toBe(120_000);+ expect(parseRetryAfterMs('0')).toBe(0);+ });++ it('parses an HTTP-date into a positive future delay', () => {+ const ms = parseRetryAfterMs(new Date(Date.now() + 60_000).toUTCString());+ expect(ms).toBeGreaterThan(50_000);+ expect(ms).toBeLessThanOrEqual(60_000);+ });++ it('returns undefined for absent or unparseable values', () => {+ expect(parseRetryAfterMs(null)).toBeUndefined();+ expect(parseRetryAfterMs('')).toBeUndefined();+ expect(parseRetryAfterMs('soon')).toBeUndefined();+ });+});diff --git a/server/src/providers/base.ts b/server/src/providers/base.tsindex 24d6d56f..6c396561 100644--- a/server/src/providers/base.ts+++ b/server/src/providers/base.ts@@ -7,6 +7,36 @@ import type {Platform,} from '@freellmapi/shared/types.js';+/** A provider HTTP error carrying the upstream status and, when the response+ * included a Retry-After header, the parsed delay so the router can bench the+ * key for at least that long. */+export interface ProviderHttpError extends Error {+ status?: number;+ retryAfterMs?: number;+}++/** Parse an HTTP `Retry-After` header (delta-seconds or an HTTP-date) into a+ * millisecond delay. Returns undefined when absent or unparseable. */+export function parseRetryAfterMs(value: string | null | undefined): number | undefined {+ if (!value) return undefined;+ const trimmed = value.trim();+ if (/^\d+$/.test(trimmed)) return Number(trimmed) * 1000;+ const when = Date.parse(trimmed);+ if (!Number.isNaN(when)) return Math.max(0, when - Date.now());+ return undefined;+}++/** Build an error for a non-OK upstream response, capturing the status and any+ * Retry-After hint. Used by every provider adapter so the proxy can honor a+ * provider's explicit back-off when it sets the cooldown. */+export function providerHttpError(res: Response, message: string): ProviderHttpError {+ const err = new Error(message) as ProviderHttpError;+ err.status = res.status;+ const retryAfterMs = parseRetryAfterMs(res.headers?.get('retry-after'));+ if (retryAfterMs !== undefined) err.retryAfterMs = retryAfterMs;+ return err;+}+export interface CompletionOptions {model?: string;temperature?: number;diff --git a/server/src/providers/cloudflare.ts b/server/src/providers/cloudflare.tsindex a80ecf67..0a9074ff 100644--- a/server/src/providers/cloudflare.ts+++ b/server/src/providers/cloudflare.ts@@ -3,7 +3,7 @@ import type {ChatCompletionResponse,ChatCompletionChunk,} from '@freellmapi/shared/types.js';-import { BaseProvider, type CompletionOptions } from './base.js';+import { BaseProvider, providerHttpError, type CompletionOptions } from './base.js';import { contentToString } from '../lib/content.js';/**@@ -58,7 +58,7 @@ export class CloudflareProvider extends BaseProvider {if (!res.ok) {const err = await res.json().catch(() => ({}));- throw new Error(`Cloudflare API error ${res.status}: ${(err as any).error?.message ?? (err as any).errors?.[0]?.message ?? res.statusText}`);+ throw providerHttpError(res, `Cloudflare API error ${res.status}: ${(err as any).error?.message ?? (err as any).errors?.[0]?.message ?? res.statusText}`);}const data = await res.json() as ChatCompletionResponse;@@ -96,7 +96,7 @@ export class CloudflareProvider extends BaseProvider {if (!res.ok) {const err = await res.json().catch(() => ({}));- throw new Error(`Cloudflare API error ${res.status}: ${(err as any).error?.message ?? (err as any).errors?.[0]?.message ?? res.statusText}`);+ throw providerHttpError(res, `Cloudflare API error ${res.status}: ${(err as any).error?.message ?? (err as any).errors?.[0]?.message ?? res.statusText}`);}yield* this.readSseStream(res);diff --git a/server/src/providers/cohere.ts b/server/src/providers/cohere.tsindex 5fcb0a4b..73737c9d 100644--- a/server/src/providers/cohere.ts+++ b/server/src/providers/cohere.ts@@ -3,7 +3,7 @@ import type {ChatCompletionResponse,ChatCompletionChunk,} from '@freellmapi/shared/types.js';-import { BaseProvider, type CompletionOptions } from './base.js';+import { BaseProvider, providerHttpError, type CompletionOptions } from './base.js';import { flattenMessageContent } from '../lib/content.js';const API_BASE = 'https://api.cohere.ai/compatibility/v1';@@ -39,7 +39,7 @@ export class CohereProvider extends BaseProvider {if (!res.ok) {const err = await res.json().catch(() => ({}));- throw new Error(`Cohere API error ${res.status}: ${(err as any).error?.message ?? res.statusText}`);+ throw providerHttpError(res, `Cohere API error ${res.status}: ${(err as any).error?.message ?? res.statusText}`);}const data = await res.json() as ChatCompletionResponse;@@ -75,7 +75,7 @@ export class CohereProvider extends BaseProvider {if (!res.ok) {const err = await res.json().catch(() => ({}));- throw new Error(`Cohere API error ${res.status}: ${(err as any).error?.message ?? res.statusText}`);+ throw providerHttpError(res, `Cohere API error ${res.status}: ${(err as any).error?.message ?? res.statusText}`);}yield* this.readSseStream(res);diff --git a/server/src/providers/google.ts b/server/src/providers/google.tsindex 7273787c..05d629cc 100644--- a/server/src/providers/google.ts+++ b/server/src/providers/google.ts@@ -7,7 +7,7 @@ import type {ChatToolDefinition,TokenUsage,} from '@freellmapi/shared/types.js';-import { BaseProvider, type CompletionOptions } from './base.js';+import { BaseProvider, providerHttpError, type CompletionOptions } from './base.js';import { contentToString } from '../lib/content.js';const API_BASE = 'https://generativelanguage.googleapis.com/v1beta';@@ -350,7 +350,7 @@ export class GoogleProvider extends BaseProvider {if (!res.ok) {const err = await res.json().catch(() => ({}));- throw new Error(`Google API error ${res.status}: ${(err as any).error?.message ?? res.statusText}`);+ throw providerHttpError(res, `Google API error ${res.status}: ${(err as any).error?.message ?? res.statusText}`);}const data = await res.json() as GeminiResponse;@@ -413,7 +413,7 @@ export class GoogleProvider extends BaseProvider {if (!res.ok) {const err = await res.json().catch(() => ({}));- throw new Error(`Google API error ${res.status}: ${(err as any).error?.message ?? res.statusText}`);+ throw providerHttpError(res, `Google API error ${res.status}: ${(err as any).error?.message ?? res.statusText}`);}const reader = res.body?.getReader();diff --git a/server/src/providers/openai-compat.ts b/server/src/providers/openai-compat.tsindex e632e4db..e441865b 100644--- a/server/src/providers/openai-compat.ts+++ b/server/src/providers/openai-compat.ts@@ -4,7 +4,7 @@ import type {ChatCompletionChunk,Platform,} from '@freellmapi/shared/types.js';-import { BaseProvider, type CompletionOptions } from './base.js';+import { BaseProvider, providerHttpError, type CompletionOptions } from './base.js';/*** Generic provider for platforms that use an OpenAI-compatible API.@@ -74,7 +74,7 @@ export class OpenAICompatProvider extends BaseProvider {if (!res.ok) {const err = await res.json().catch(() => ({}));- throw new Error(`${this.name} API error ${res.status}: ${(err as any).error?.message ?? res.statusText}`);+ throw providerHttpError(res, `${this.name} API error ${res.status}: ${(err as any).error?.message ?? res.statusText}`);}let data: ChatCompletionResponse;@@ -123,7 +123,7 @@ export class OpenAICompatProvider extends BaseProvider {if (!res.ok) {const err = await res.json().catch(() => ({}));- throw new Error(`${this.name} API error ${res.status}: ${(err as any).error?.message ?? res.statusText}`);+ throw providerHttpError(res, `${this.name} API error ${res.status}: ${(err as any).error?.message ?? res.statusText}`);}yield* this.readSseStream(res);diff --git a/server/src/routes/proxy.ts b/server/src/routes/proxy.tsindex 44b98cd9..8b6a0b84 100644--- a/server/src/routes/proxy.ts+++ b/server/src/routes/proxy.ts@@ -973,7 +973,7 @@ proxyRouter.post('/chat/completions', async (req: Request, res: Response) => {: getCooldownDurationForLimit(route.platform, route.modelId, route.keyId, {rpd: route.rpdLimit,tpd: route.tpdLimit,- }),+ }, err.retryAfterMs),);recordRateLimitHit(route.modelDbId);lastError = err;diff --git a/server/src/services/ratelimit.ts b/server/src/services/ratelimit.tsindex 62344cb6..252ed09c 100644--- a/server/src/services/ratelimit.ts+++ b/server/src/services/ratelimit.ts@@ -326,16 +326,21 @@ export function getCooldownDurationForLimit(modelId: string,keyId: number,limits: { rpd: number | null; tpd: number | null },+ retryAfterMs?: number | null,): number {const now = Date.now();const rpdExhausted =limits.rpd !== null && requestCount(platform, modelId, keyId, DAY, now) >= limits.rpd;const tpdExhausted =limits.tpd !== null && tokenCount(platform, modelId, keyId, DAY, now) >= limits.tpd;- if (rpdExhausted || tpdExhausted) {- return getNextCooldownDuration(platform, modelId, keyId);- }- return TRANSIENT_COOLDOWN_MS;+ const base = (rpdExhausted || tpdExhausted)+ ? getNextCooldownDuration(platform, modelId, keyId)+ : TRANSIENT_COOLDOWN_MS;+ // Honor an upstream Retry-After as a floor: never bench shorter than our own+ // heuristic, but extend (capped at a day) when the provider explicitly asks+ // to wait longer than we otherwise would.+ if (retryAfterMs != null && retryAfterMs > base) return Math.min(retryAfterMs, DAY);+ return base;}function persistedCooldownExpiry(