Skip to content

7.4 billion tokens per month. 34 free LLM providers. 635 free model endpoints. All behind one /v1 endpoint, plus any custom OpenAI-compatible endpoint. Smart routing, automatic failover, encrypted keys. Personal experimentation only.

TypeScript

Don't escalate transient TPM/RPM 429s to a 24h quarantine (#158)

Tashfeen1eb9abdb9202
diff --git a/server/src/__tests__/services/ratelimit.test.ts b/server/src/__tests__/services/ratelimit.test.ts
index e2f51ce8..b63b867f 100644
--- a/server/src/__tests__/services/ratelimit.test.ts
+++ b/server/src/__tests__/services/ratelimit.test.ts
@@ -7,6 +7,7 @@ import {
recordTokens,
getRateLimitStatus,
getNextCooldownDuration,
+ getCooldownDurationForLimit,
} from '../../services/ratelimit.js';
function removeDbFile(dbPath: string) {
@@ -115,6 +116,39 @@ describe('Rate Limiter', () => {
});
});
+ describe('getCooldownDurationForLimit (daily vs transient 429)', () => {
+ it('uses a short, non-escalating cooldown when the daily quota is NOT exhausted', () => {
+ const id = Math.floor(Math.random() * 1_000_000);
+ const args = ['groq', `transient-${id}`, id] as const;
+ // groq-like: large daily quota, no requests recorded yet → transient (TPM/RPM)
+ // 429s must stay at the short fixed cooldown and never escalate.
+ expect(getCooldownDurationForLimit(...args, { rpd: 1000, tpd: null })).toBe(90 * 1000);
+ expect(getCooldownDurationForLimit(...args, { rpd: 1000, tpd: null })).toBe(90 * 1000);
+ expect(getCooldownDurationForLimit(...args, { rpd: 1000, tpd: null })).toBe(90 * 1000);
+ });
+
+ it('treats a null daily limit as never-exhausted (always transient)', () => {
+ const id = Math.floor(Math.random() * 1_000_000);
+ for (let i = 0; i < 50; i++) recordRequest('mistral', `nolimit-${id}`, id);
+ expect(
+ getCooldownDurationForLimit('mistral', `nolimit-${id}`, id, { rpd: null, tpd: null }),
+ ).toBe(90 * 1000);
+ });
+
+ it('escalates only once the daily request limit is actually reached', () => {
+ const id = Math.floor(Math.random() * 1_000_000);
+ const platform = 'openrouter';
+ const model = `daily-${id}`;
+ // Below the daily limit → still transient.
+ recordRequest(platform, model, id);
+ expect(getCooldownDurationForLimit(platform, model, id, { rpd: 5, tpd: null })).toBe(90 * 1000);
+ // Reach the daily limit → now it escalates (2m, then 10m, ...).
+ for (let i = 0; i < 5; i++) recordRequest(platform, model, id);
+ expect(getCooldownDurationForLimit(platform, model, id, { rpd: 5, tpd: null })).toBe(2 * 60 * 1000);
+ expect(getCooldownDurationForLimit(platform, model, id, { rpd: 5, tpd: null })).toBe(10 * 60 * 1000);
+ });
+ });
+
describe('persistent state', () => {
it('preserves per-key usage and cooldowns after the limiter module reloads', async () => {
process.env.ENCRYPTION_KEY = '0'.repeat(64);
diff --git a/server/src/routes/proxy.ts b/server/src/routes/proxy.ts
index afdbadc6..23391384 100644
--- a/server/src/routes/proxy.ts
+++ b/server/src/routes/proxy.ts
@@ -4,7 +4,7 @@ import type { Request, Response } from 'express';
import { z } from 'zod';
import type { ChatMessage } from '@freellmapi/shared/types.js';
import { routeRequest, recordRateLimitHit, recordSuccess, hasEnabledVisionModel, type RouteResult } from '../services/router.js';
-import { recordRequest, recordTokens, setCooldown, getNextCooldownDuration } from '../services/ratelimit.js';
+import { recordRequest, recordTokens, setCooldown, getCooldownDurationForLimit } from '../services/ratelimit.js';
import { getDb, getUnifiedApiKey } from '../db/index.js';
import { contentToString, messageHasImage } from '../lib/content.js';
@@ -491,7 +491,10 @@ proxyRouter.post('/chat/completions', async (req: Request, res: Response) => {
route.platform,
route.modelId,
route.keyId,
- getNextCooldownDuration(route.platform, route.modelId, route.keyId),
+ getCooldownDurationForLimit(route.platform, route.modelId, route.keyId, {
+ rpd: route.rpdLimit,
+ tpd: route.tpdLimit,
+ }),
);
recordRateLimitHit(route.modelDbId);
lastError = err;
diff --git a/server/src/routes/responses.ts b/server/src/routes/responses.ts
index 25404539..eb37adda 100644
--- a/server/src/routes/responses.ts
+++ b/server/src/routes/responses.ts
@@ -9,7 +9,7 @@ import type {
ChatToolChoice,
} from '@freellmapi/shared/types.js';
import { routeRequest, recordRateLimitHit, recordSuccess, type RouteResult } from '../services/router.js';
-import { recordRequest, recordTokens, setCooldown, getNextCooldownDuration } from '../services/ratelimit.js';
+import { recordRequest, recordTokens, setCooldown, getCooldownDurationForLimit } from '../services/ratelimit.js';
import { getUnifiedApiKey } from '../db/index.js';
import { contentToString } from '../lib/content.js';
import {
@@ -479,7 +479,7 @@ responsesRouter.post('/responses', async (req: Request, res: Response) => {
if (isRetryableError(err)) {
skipKeys.add(`${route.platform}:${route.modelId}:${route.keyId}`);
- setCooldown(route.platform, route.modelId, route.keyId, getNextCooldownDuration(route.platform, route.modelId, route.keyId));
+ setCooldown(route.platform, route.modelId, route.keyId, getCooldownDurationForLimit(route.platform, route.modelId, route.keyId, { rpd: route.rpdLimit, tpd: route.tpdLimit }));
recordRateLimitHit(route.modelDbId);
lastError = err;
continue;
diff --git a/server/src/services/ratelimit.ts b/server/src/services/ratelimit.ts
index 3f342ae4..5da48b94 100644
--- a/server/src/services/ratelimit.ts
+++ b/server/src/services/ratelimit.ts
@@ -233,6 +233,39 @@ export function getNextCooldownDuration(platform: string, modelId: string, keyId
return COOLDOWN_DURATIONS[idx]!;
}
+// Short cooldown for a transient (per-minute) 429 — recovers within ~one window.
+const TRANSIENT_COOLDOWN_MS = 90 * 1000;
+
+// Decide how long to bench a model+key after an upstream 429. Escalate to the
+// long quarantine (getNextCooldownDuration, up to 24h) ONLY when the model is
+// genuinely at its DAILY limit (RPD or TPD) — that won't recover until the
+// provider's daily reset, so a long bench avoids hammering a truly-dead key.
+//
+// A transient RPM/TPM 429 gets a short fixed cooldown and does NOT count toward
+// escalation. This is the common case for providers with a tight per-minute
+// token budget but a large daily quota — e.g. groq gpt-oss-120b has rpd=1000
+// yet tpm=8000, so a single burst of large prompts 429s on TPM while the daily
+// quota is barely touched. Without this split, those transient bursts escalated
+// (2m → 10m → 1h → 24h) and quarantined a perfectly healthy provider for the
+// rest of the day. Daily counters are persisted (countPersistedRequests /
+// sumPersistedTokens), so this verdict is stable across restarts.
+export function getCooldownDurationForLimit(
+ platform: string,
+ modelId: string,
+ keyId: number,
+ limits: { rpd: number | null; tpd: number | null },
+): number {
+ const now = Date.now();
+ const rpdExhausted =
+ limits.rpd !== null && requestCount(platform, modelId, keyId, DAY, now) >= limits.rpd;
+ const tpdExhausted =
+ limits.tpd !== null && tokenCount(platform, modelId, keyId, DAY, now) >= limits.tpd;
+ if (rpdExhausted || tpdExhausted) {
+ return getNextCooldownDuration(platform, modelId, keyId);
+ }
+ return TRANSIENT_COOLDOWN_MS;
+}
+
function persistedCooldownExpiry(
platform: string,
modelId: string,
diff --git a/server/src/services/router.ts b/server/src/services/router.ts
index 3c90a59a..8aa7bef8 100644
--- a/server/src/services/router.ts
+++ b/server/src/services/router.ts
@@ -41,6 +41,10 @@ export interface RouteResult {
keyId: number;
platform: string;
displayName: string;
+ // Daily limits for this model, so a 429 handler can tell a genuine daily
+ // exhaustion (escalate the cooldown) from a transient per-minute spike.
+ rpdLimit: number | null;
+ tpdLimit: number | null;
}
// Round-robin index per platform
@@ -233,6 +237,8 @@ export function routeRequest(estimatedTokens = 1000, skipKeys?: Set<string>, pre
keyId: key.id,
platform: model.platform,
displayName: model.display_name,
+ rpdLimit: limits.rpd,
+ tpdLimit: limits.tpd,
};
}