Skip to content

7.4 billion tokens per month. 34 free LLM providers. 635 free model endpoints. All behind one /v1 endpoint, plus any custom OpenAI-compatible endpoint. Smart routing, automatic failover, encrypted keys. Personal experimentation only.

TypeScript

feat(router): enforce provider-wide daily request caps (#162)

Tashfeen03480d86c697
diff --git a/server/src/__tests__/services/ratelimit.test.ts b/server/src/__tests__/services/ratelimit.test.ts
index b63b867f..8b88e98b 100644
--- a/server/src/__tests__/services/ratelimit.test.ts
+++ b/server/src/__tests__/services/ratelimit.test.ts
@@ -1,5 +1,5 @@
import fs from 'fs';
-import { describe, it, expect, beforeEach, vi } from 'vitest';
+import { describe, it, expect, beforeEach, afterEach, vi } from 'vitest';
import {
canMakeRequest,
canUseTokens,
@@ -8,6 +8,9 @@ import {
getRateLimitStatus,
getNextCooldownDuration,
getCooldownDurationForLimit,
+ canUseProvider,
+ providerDailyRequestCount,
+ getProviderDailyRequestCap,
} from '../../services/ratelimit.js';
function removeDbFile(dbPath: string) {
@@ -186,4 +189,43 @@ describe('Rate Limiter', () => {
}
});
});
+
+ describe('provider-wide daily request cap (#162)', () => {
+ const ENV = 'PROVIDER_DAILY_REQUEST_CAP_OPENROUTER';
+ let original: string | undefined;
+
+ beforeEach(() => { original = process.env[ENV]; });
+ afterEach(() => {
+ if (original === undefined) delete process.env[ENV];
+ else process.env[ENV] = original;
+ });
+
+ it('defaults to OpenRouter ~1000/day and allows env override / disable', () => {
+ delete process.env[ENV];
+ expect(getProviderDailyRequestCap('openrouter')).toBe(1000);
+ expect(getProviderDailyRequestCap('groq')).toBeNull(); // no shared cap
+ process.env[ENV] = '50';
+ expect(getProviderDailyRequestCap('openrouter')).toBe(50);
+ process.env[ENV] = '0'; // 0 disables the cap
+ expect(getProviderDailyRequestCap('openrouter')).toBeNull();
+ });
+
+ it('counts requests across ALL of a provider\'s models for one key', () => {
+ recordRequest('openrouter', 'deepseek/deepseek-v3.1:free', testId);
+ recordRequest('openrouter', 'deepseek/deepseek-v3.1:free', testId);
+ recordRequest('openrouter', 'qwen/qwen3-coder:free', testId);
+ // Same key on a different provider must not bleed into the count.
+ recordRequest('groq', 'llama-70b', testId);
+ expect(providerDailyRequestCount('openrouter', testId)).toBe(3);
+ });
+
+ it('blocks the whole provider once the shared daily cap is hit', () => {
+ process.env[ENV] = '3';
+ recordRequest('openrouter', 'model-a', testId);
+ recordRequest('openrouter', 'model-b', testId);
+ expect(canUseProvider('openrouter', testId)).toBe(true); // 2 < 3
+ recordRequest('openrouter', 'model-c', testId);
+ expect(canUseProvider('openrouter', testId)).toBe(false); // 3 >= 3
+ });
+ });
});
diff --git a/server/src/services/ratelimit.ts b/server/src/services/ratelimit.ts
index 5da48b94..0c4c778f 100644
--- a/server/src/services/ratelimit.ts
+++ b/server/src/services/ratelimit.ts
@@ -176,6 +176,71 @@ export function canUseTokens(
return true;
}
+// ── Provider-wide daily request caps (#162) ──
+// Some providers enforce one daily REQUEST quota across the WHOLE account,
+// shared by every model — not per model. OpenRouter's free tier is the classic
+// case: ~1000 requests/day total (50/day if you've bought <10 credits) no
+// matter how many different free models you spread them across. The
+// per-(platform,model,key) rpd ledger can't see that, so without a provider-wide
+// gate the router happily fires (models × rpd) requests and earns surprise 429s.
+//
+// Defaults below; override per provider with an env var, e.g.
+// PROVIDER_DAILY_REQUEST_CAP_OPENROUTER=50 (set 0 to disable the cap)
+const DEFAULT_PROVIDER_DAILY_REQUEST_CAPS: Record<string, number> = {
+ openrouter: 1000,
+};
+
+export function getProviderDailyRequestCap(platform: string): number | null {
+ const raw = process.env[`PROVIDER_DAILY_REQUEST_CAP_${platform.toUpperCase()}`];
+ if (raw !== undefined && raw.trim() !== '') {
+ const n = Number(raw);
+ if (Number.isFinite(n) && n >= 0) return n === 0 ? null : n;
+ }
+ return DEFAULT_PROVIDER_DAILY_REQUEST_CAPS[platform] ?? null;
+}
+
+function countPersistedProviderRequests(
+ platform: string,
+ keyId: number,
+ windowMs: number,
+ now: number,
+): number | undefined {
+ return withDb(db => {
+ const row = db.prepare(`
+ SELECT COUNT(*) AS used
+ FROM rate_limit_usage
+ WHERE platform = ?
+ AND key_id = ?
+ AND kind = 'request'
+ AND created_at_ms > ?
+ `).get(platform, keyId, now - windowMs) as { used: number };
+ return row.used;
+ });
+}
+
+// Total requests today for a provider account+key, summed across every model.
+export function providerDailyRequestCount(platform: string, keyId: number, now = Date.now()): number {
+ const persisted = countPersistedProviderRequests(platform, keyId, DAY, now);
+ if (persisted !== undefined) return persisted;
+ // DB-unavailable fallback: sum the per-model rpd windows for this platform+key.
+ // Window key format is "platform:modelId:keyId:rpd" (modelId may contain ':').
+ let total = 0;
+ for (const [key, w] of windows) {
+ if (key.startsWith(`${platform}:`) && key.endsWith(`:${keyId}:rpd`)) {
+ total += pruneTimestamps(w.timestamps, DAY, now).length;
+ }
+ }
+ return total;
+}
+
+// False when this provider account+key has hit its shared daily request cap, so
+// the router skips every model on that provider for this key until UTC-ish reset.
+export function canUseProvider(platform: string, keyId: number, now = Date.now()): boolean {
+ const cap = getProviderDailyRequestCap(platform);
+ if (cap === null) return true;
+ return providerDailyRequestCount(platform, keyId, now) < cap;
+}
+
export function recordRequest(platform: string, modelId: string, keyId: number) {
const now = Date.now();
diff --git a/server/src/services/router.ts b/server/src/services/router.ts
index 20d8d966..10257d77 100644
--- a/server/src/services/router.ts
+++ b/server/src/services/router.ts
@@ -1,7 +1,7 @@
import { getDb, getSetting, setSetting } from '../db/index.js';
import { getProvider, resolveProvider } from '../providers/index.js';
import { decrypt } from '../lib/crypto.js';
-import { canMakeRequest, canUseTokens, isOnCooldown } from './ratelimit.js';
+import { canMakeRequest, canUseTokens, isOnCooldown, canUseProvider } from './ratelimit.js';
import {
BANDIT_PRESETS, DEFAULT_STRATEGY, type RoutingStrategy, type RoutingWeights,
reliabilityPosterior, expectedReliability, sampleBeta,
@@ -419,6 +419,11 @@ export function routeRequest(estimatedTokens = 1000, skipKeys?: Set<string>, pre
// Check cooldown (from previous 429s)
if (isOnCooldown(entry.platform, entry.model_id, key.id)) continue;
+ // Provider-wide daily request cap (#162): providers like OpenRouter cap
+ // total requests/day across ALL their models for the account, not per
+ // model — skip every model on this provider once that key hits the cap.
+ if (!canUseProvider(entry.platform, key.id)) continue;
+
if (!canMakeRequest(entry.platform, entry.model_id, key.id, limits)) continue;
if (!canUseTokens(entry.platform, entry.model_id, key.id, estimatedTokens, limits)) continue;