7.4 billion tokens per month. 34 free LLM providers. 635 free model endpoints. All behind one /v1 endpoint, plus any custom OpenAI-compatible endpoint. Smart routing, automatic failover, encrypted keys. Personal experimentation only.
TypeScript
feat(router): enforce provider-wide daily request caps (#162)
Tashfeen03480d86c697
diff --git a/server/src/__tests__/services/ratelimit.test.ts b/server/src/__tests__/services/ratelimit.test.tsindex b63b867f..8b88e98b 100644--- a/server/src/__tests__/services/ratelimit.test.ts+++ b/server/src/__tests__/services/ratelimit.test.ts@@ -1,5 +1,5 @@import fs from 'fs';-import { describe, it, expect, beforeEach, vi } from 'vitest';+import { describe, it, expect, beforeEach, afterEach, vi } from 'vitest';import {canMakeRequest,canUseTokens,@@ -8,6 +8,9 @@ import {getRateLimitStatus,getNextCooldownDuration,getCooldownDurationForLimit,+ canUseProvider,+ providerDailyRequestCount,+ getProviderDailyRequestCap,} from '../../services/ratelimit.js';function removeDbFile(dbPath: string) {@@ -186,4 +189,43 @@ describe('Rate Limiter', () => {}});});++ describe('provider-wide daily request cap (#162)', () => {+ const ENV = 'PROVIDER_DAILY_REQUEST_CAP_OPENROUTER';+ let original: string | undefined;++ beforeEach(() => { original = process.env[ENV]; });+ afterEach(() => {+ if (original === undefined) delete process.env[ENV];+ else process.env[ENV] = original;+ });++ it('defaults to OpenRouter ~1000/day and allows env override / disable', () => {+ delete process.env[ENV];+ expect(getProviderDailyRequestCap('openrouter')).toBe(1000);+ expect(getProviderDailyRequestCap('groq')).toBeNull(); // no shared cap+ process.env[ENV] = '50';+ expect(getProviderDailyRequestCap('openrouter')).toBe(50);+ process.env[ENV] = '0'; // 0 disables the cap+ expect(getProviderDailyRequestCap('openrouter')).toBeNull();+ });++ it('counts requests across ALL of a provider\'s models for one key', () => {+ recordRequest('openrouter', 'deepseek/deepseek-v3.1:free', testId);+ recordRequest('openrouter', 'deepseek/deepseek-v3.1:free', testId);+ recordRequest('openrouter', 'qwen/qwen3-coder:free', testId);+ // Same key on a different provider must not bleed into the count.+ recordRequest('groq', 'llama-70b', testId);+ expect(providerDailyRequestCount('openrouter', testId)).toBe(3);+ });++ it('blocks the whole provider once the shared daily cap is hit', () => {+ process.env[ENV] = '3';+ recordRequest('openrouter', 'model-a', testId);+ recordRequest('openrouter', 'model-b', testId);+ expect(canUseProvider('openrouter', testId)).toBe(true); // 2 < 3+ recordRequest('openrouter', 'model-c', testId);+ expect(canUseProvider('openrouter', testId)).toBe(false); // 3 >= 3+ });+ });});diff --git a/server/src/services/ratelimit.ts b/server/src/services/ratelimit.tsindex 5da48b94..0c4c778f 100644--- a/server/src/services/ratelimit.ts+++ b/server/src/services/ratelimit.ts@@ -176,6 +176,71 @@ export function canUseTokens(return true;}+// ── Provider-wide daily request caps (#162) ──+// Some providers enforce one daily REQUEST quota across the WHOLE account,+// shared by every model — not per model. OpenRouter's free tier is the classic+// case: ~1000 requests/day total (50/day if you've bought <10 credits) no+// matter how many different free models you spread them across. The+// per-(platform,model,key) rpd ledger can't see that, so without a provider-wide+// gate the router happily fires (models × rpd) requests and earns surprise 429s.+//+// Defaults below; override per provider with an env var, e.g.+// PROVIDER_DAILY_REQUEST_CAP_OPENROUTER=50 (set 0 to disable the cap)+const DEFAULT_PROVIDER_DAILY_REQUEST_CAPS: Record<string, number> = {+ openrouter: 1000,+};++export function getProviderDailyRequestCap(platform: string): number | null {+ const raw = process.env[`PROVIDER_DAILY_REQUEST_CAP_${platform.toUpperCase()}`];+ if (raw !== undefined && raw.trim() !== '') {+ const n = Number(raw);+ if (Number.isFinite(n) && n >= 0) return n === 0 ? null : n;+ }+ return DEFAULT_PROVIDER_DAILY_REQUEST_CAPS[platform] ?? null;+}++function countPersistedProviderRequests(+ platform: string,+ keyId: number,+ windowMs: number,+ now: number,+): number | undefined {+ return withDb(db => {+ const row = db.prepare(`+ SELECT COUNT(*) AS used+ FROM rate_limit_usage+ WHERE platform = ?+ AND key_id = ?+ AND kind = 'request'+ AND created_at_ms > ?+ `).get(platform, keyId, now - windowMs) as { used: number };+ return row.used;+ });+}++// Total requests today for a provider account+key, summed across every model.+export function providerDailyRequestCount(platform: string, keyId: number, now = Date.now()): number {+ const persisted = countPersistedProviderRequests(platform, keyId, DAY, now);+ if (persisted !== undefined) return persisted;+ // DB-unavailable fallback: sum the per-model rpd windows for this platform+key.+ // Window key format is "platform:modelId:keyId:rpd" (modelId may contain ':').+ let total = 0;+ for (const [key, w] of windows) {+ if (key.startsWith(`${platform}:`) && key.endsWith(`:${keyId}:rpd`)) {+ total += pruneTimestamps(w.timestamps, DAY, now).length;+ }+ }+ return total;+}++// False when this provider account+key has hit its shared daily request cap, so+// the router skips every model on that provider for this key until UTC-ish reset.+export function canUseProvider(platform: string, keyId: number, now = Date.now()): boolean {+ const cap = getProviderDailyRequestCap(platform);+ if (cap === null) return true;+ return providerDailyRequestCount(platform, keyId, now) < cap;+}+export function recordRequest(platform: string, modelId: string, keyId: number) {const now = Date.now();diff --git a/server/src/services/router.ts b/server/src/services/router.tsindex 20d8d966..10257d77 100644--- a/server/src/services/router.ts+++ b/server/src/services/router.ts@@ -1,7 +1,7 @@import { getDb, getSetting, setSetting } from '../db/index.js';import { getProvider, resolveProvider } from '../providers/index.js';import { decrypt } from '../lib/crypto.js';-import { canMakeRequest, canUseTokens, isOnCooldown } from './ratelimit.js';+import { canMakeRequest, canUseTokens, isOnCooldown, canUseProvider } from './ratelimit.js';import {BANDIT_PRESETS, DEFAULT_STRATEGY, type RoutingStrategy, type RoutingWeights,reliabilityPosterior, expectedReliability, sampleBeta,@@ -419,6 +419,11 @@ export function routeRequest(estimatedTokens = 1000, skipKeys?: Set<string>, pre// Check cooldown (from previous 429s)if (isOnCooldown(entry.platform, entry.model_id, key.id)) continue;+ // Provider-wide daily request cap (#162): providers like OpenRouter cap+ // total requests/day across ALL their models for the account, not per+ // model — skip every model on this provider once that key hits the cap.+ if (!canUseProvider(entry.platform, key.id)) continue;+if (!canMakeRequest(entry.platform, entry.model_id, key.id, limits)) continue;if (!canUseTokens(entry.platform, entry.model_id, key.id, estimatedTokens, limits)) continue;