7.4 billion tokens per month. 34 free LLM providers. 635 free model endpoints. All behind one /v1 endpoint, plus any custom OpenAI-compatible endpoint. Smart routing, automatic failover, encrypted keys. Personal experimentation only.
TypeScript
Don't escalate transient TPM/RPM 429s to a 24h quarantine (#158)
Tashfeen1eb9abdb9202
diff --git a/server/src/__tests__/services/ratelimit.test.ts b/server/src/__tests__/services/ratelimit.test.tsindex e2f51ce8..b63b867f 100644--- a/server/src/__tests__/services/ratelimit.test.ts+++ b/server/src/__tests__/services/ratelimit.test.ts@@ -7,6 +7,7 @@ import {recordTokens,getRateLimitStatus,getNextCooldownDuration,+ getCooldownDurationForLimit,} from '../../services/ratelimit.js';function removeDbFile(dbPath: string) {@@ -115,6 +116,39 @@ describe('Rate Limiter', () => {});});+ describe('getCooldownDurationForLimit (daily vs transient 429)', () => {+ it('uses a short, non-escalating cooldown when the daily quota is NOT exhausted', () => {+ const id = Math.floor(Math.random() * 1_000_000);+ const args = ['groq', `transient-${id}`, id] as const;+ // groq-like: large daily quota, no requests recorded yet → transient (TPM/RPM)+ // 429s must stay at the short fixed cooldown and never escalate.+ expect(getCooldownDurationForLimit(...args, { rpd: 1000, tpd: null })).toBe(90 * 1000);+ expect(getCooldownDurationForLimit(...args, { rpd: 1000, tpd: null })).toBe(90 * 1000);+ expect(getCooldownDurationForLimit(...args, { rpd: 1000, tpd: null })).toBe(90 * 1000);+ });++ it('treats a null daily limit as never-exhausted (always transient)', () => {+ const id = Math.floor(Math.random() * 1_000_000);+ for (let i = 0; i < 50; i++) recordRequest('mistral', `nolimit-${id}`, id);+ expect(+ getCooldownDurationForLimit('mistral', `nolimit-${id}`, id, { rpd: null, tpd: null }),+ ).toBe(90 * 1000);+ });++ it('escalates only once the daily request limit is actually reached', () => {+ const id = Math.floor(Math.random() * 1_000_000);+ const platform = 'openrouter';+ const model = `daily-${id}`;+ // Below the daily limit → still transient.+ recordRequest(platform, model, id);+ expect(getCooldownDurationForLimit(platform, model, id, { rpd: 5, tpd: null })).toBe(90 * 1000);+ // Reach the daily limit → now it escalates (2m, then 10m, ...).+ for (let i = 0; i < 5; i++) recordRequest(platform, model, id);+ expect(getCooldownDurationForLimit(platform, model, id, { rpd: 5, tpd: null })).toBe(2 * 60 * 1000);+ expect(getCooldownDurationForLimit(platform, model, id, { rpd: 5, tpd: null })).toBe(10 * 60 * 1000);+ });+ });+describe('persistent state', () => {it('preserves per-key usage and cooldowns after the limiter module reloads', async () => {process.env.ENCRYPTION_KEY = '0'.repeat(64);diff --git a/server/src/routes/proxy.ts b/server/src/routes/proxy.tsindex afdbadc6..23391384 100644--- a/server/src/routes/proxy.ts+++ b/server/src/routes/proxy.ts@@ -4,7 +4,7 @@ import type { Request, Response } from 'express';import { z } from 'zod';import type { ChatMessage } from '@freellmapi/shared/types.js';import { routeRequest, recordRateLimitHit, recordSuccess, hasEnabledVisionModel, type RouteResult } from '../services/router.js';-import { recordRequest, recordTokens, setCooldown, getNextCooldownDuration } from '../services/ratelimit.js';+import { recordRequest, recordTokens, setCooldown, getCooldownDurationForLimit } from '../services/ratelimit.js';import { getDb, getUnifiedApiKey } from '../db/index.js';import { contentToString, messageHasImage } from '../lib/content.js';@@ -491,7 +491,10 @@ proxyRouter.post('/chat/completions', async (req: Request, res: Response) => {route.platform,route.modelId,route.keyId,- getNextCooldownDuration(route.platform, route.modelId, route.keyId),+ getCooldownDurationForLimit(route.platform, route.modelId, route.keyId, {+ rpd: route.rpdLimit,+ tpd: route.tpdLimit,+ }),);recordRateLimitHit(route.modelDbId);lastError = err;diff --git a/server/src/routes/responses.ts b/server/src/routes/responses.tsindex 25404539..eb37adda 100644--- a/server/src/routes/responses.ts+++ b/server/src/routes/responses.ts@@ -9,7 +9,7 @@ import type {ChatToolChoice,} from '@freellmapi/shared/types.js';import { routeRequest, recordRateLimitHit, recordSuccess, type RouteResult } from '../services/router.js';-import { recordRequest, recordTokens, setCooldown, getNextCooldownDuration } from '../services/ratelimit.js';+import { recordRequest, recordTokens, setCooldown, getCooldownDurationForLimit } from '../services/ratelimit.js';import { getUnifiedApiKey } from '../db/index.js';import { contentToString } from '../lib/content.js';import {@@ -479,7 +479,7 @@ responsesRouter.post('/responses', async (req: Request, res: Response) => {if (isRetryableError(err)) {skipKeys.add(`${route.platform}:${route.modelId}:${route.keyId}`);- setCooldown(route.platform, route.modelId, route.keyId, getNextCooldownDuration(route.platform, route.modelId, route.keyId));+ setCooldown(route.platform, route.modelId, route.keyId, getCooldownDurationForLimit(route.platform, route.modelId, route.keyId, { rpd: route.rpdLimit, tpd: route.tpdLimit }));recordRateLimitHit(route.modelDbId);lastError = err;continue;diff --git a/server/src/services/ratelimit.ts b/server/src/services/ratelimit.tsindex 3f342ae4..5da48b94 100644--- a/server/src/services/ratelimit.ts+++ b/server/src/services/ratelimit.ts@@ -233,6 +233,39 @@ export function getNextCooldownDuration(platform: string, modelId: string, keyIdreturn COOLDOWN_DURATIONS[idx]!;}+// Short cooldown for a transient (per-minute) 429 — recovers within ~one window.+const TRANSIENT_COOLDOWN_MS = 90 * 1000;++// Decide how long to bench a model+key after an upstream 429. Escalate to the+// long quarantine (getNextCooldownDuration, up to 24h) ONLY when the model is+// genuinely at its DAILY limit (RPD or TPD) — that won't recover until the+// provider's daily reset, so a long bench avoids hammering a truly-dead key.+//+// A transient RPM/TPM 429 gets a short fixed cooldown and does NOT count toward+// escalation. This is the common case for providers with a tight per-minute+// token budget but a large daily quota — e.g. groq gpt-oss-120b has rpd=1000+// yet tpm=8000, so a single burst of large prompts 429s on TPM while the daily+// quota is barely touched. Without this split, those transient bursts escalated+// (2m → 10m → 1h → 24h) and quarantined a perfectly healthy provider for the+// rest of the day. Daily counters are persisted (countPersistedRequests /+// sumPersistedTokens), so this verdict is stable across restarts.+export function getCooldownDurationForLimit(+ platform: string,+ modelId: string,+ keyId: number,+ limits: { rpd: number | null; tpd: number | null },+): number {+ const now = Date.now();+ const rpdExhausted =+ limits.rpd !== null && requestCount(platform, modelId, keyId, DAY, now) >= limits.rpd;+ const tpdExhausted =+ limits.tpd !== null && tokenCount(platform, modelId, keyId, DAY, now) >= limits.tpd;+ if (rpdExhausted || tpdExhausted) {+ return getNextCooldownDuration(platform, modelId, keyId);+ }+ return TRANSIENT_COOLDOWN_MS;+}+function persistedCooldownExpiry(platform: string,modelId: string,diff --git a/server/src/services/router.ts b/server/src/services/router.tsindex 3c90a59a..8aa7bef8 100644--- a/server/src/services/router.ts+++ b/server/src/services/router.ts@@ -41,6 +41,10 @@ export interface RouteResult {keyId: number;platform: string;displayName: string;+ // Daily limits for this model, so a 429 handler can tell a genuine daily+ // exhaustion (escalate the cooldown) from a transient per-minute spike.+ rpdLimit: number | null;+ tpdLimit: number | null;}// Round-robin index per platform@@ -233,6 +237,8 @@ export function routeRequest(estimatedTokens = 1000, skipKeys?: Set<string>, prekeyId: key.id,platform: model.platform,displayName: model.display_name,+ rpdLimit: limits.rpd,+ tpdLimit: limits.tpd,};}