7.4 billion tokens per month. 34 free LLM providers. 635 free model endpoints. All behind one /v1 endpoint, plus any custom OpenAI-compatible endpoint. Smart routing, automatic failover, encrypted keys. Personal experimentation only.
TypeScript
feat(router): exhaust all keys before falling back + tests (#42, supersedes #33)
Tashfeen9b972192ef12
diff --git a/server/src/__tests__/services/routing-exhaustion.test.ts b/server/src/__tests__/services/routing-exhaustion.test.tsnew file mode 100644index 00000000..a7f5fde9--- /dev/null+++ b/server/src/__tests__/services/routing-exhaustion.test.ts@@ -0,0 +1,91 @@+import { describe, it, expect, beforeEach, vi } from 'vitest';+import { routeRequest } from '../../services/router.js';+import * as ratelimit from '../../services/ratelimit.js';+import { getDb, initDb } from '../../db/index.js';+import * as crypto from '../../lib/crypto.js';++// Mock ratelimit to control quota availability+vi.mock('../../services/ratelimit.js', async () => {+ const actual = await vi.importActual('../../services/ratelimit.js');+ return {+ ...actual,+ canMakeRequest: vi.fn(),+ canUseTokens: vi.fn(),+ isOnCooldown: vi.fn(() => false),+ };+});++// Mock crypto to avoid IV errors+vi.mock('../../lib/crypto.js', async () => {+ const actual = await vi.importActual('../../lib/crypto.js');+ return {+ ...actual,+ decrypt: vi.fn(() => 'mocked-api-key'),+ };+});++describe('Routing Key Exhaustion', () => {+ beforeEach(() => {+ initDb(':memory:');+ const db = getDb();++ // Setup: 2 models (Pro and Flash)+ // Pro is higher priority (priority 1), Flash is lower (priority 2)+ db.prepare("INSERT INTO models (platform, model_id, display_name, intelligence_rank, speed_rank, enabled) VALUES ('google', 'gemini-1.5-pro', 'Pro', 1, 1, 1)").run();+ db.prepare("INSERT INTO models (platform, model_id, display_name, intelligence_rank, speed_rank, enabled) VALUES ('google', 'gemini-1.5-flash', 'Flash', 2, 2, 1)").run();++ const proId = db.prepare("SELECT id FROM models WHERE model_id = 'gemini-1.5-pro'").get().id;+ const flashId = db.prepare("SELECT id FROM models WHERE model_id = 'gemini-1.5-flash'").get().id;++ db.prepare("INSERT INTO fallback_config (model_db_id, priority, enabled) VALUES (?, 1, 1)").run(proId);+ db.prepare("INSERT INTO fallback_config (model_db_id, priority, enabled) VALUES (?, 2, 1)").run(flashId);++ // Setup: 2 keys for Google+ db.prepare("INSERT INTO api_keys (platform, label, encrypted_key, iv, auth_tag, status, enabled) VALUES ('google', 'Key A', 'enc', 'iv', 'tag', 'healthy', 1)").run();+ db.prepare("INSERT INTO api_keys (platform, label, encrypted_key, iv, auth_tag, status, enabled) VALUES ('google', 'Key B', 'enc', 'iv', 'tag', 'healthy', 1)").run();++ vi.clearAllMocks();+ });++ it('should skip exhausted Key B and use functional Key A for the same high-priority model', () => {+ const db = getDb();+ const keys = db.prepare("SELECT id, label FROM api_keys").all();+ const keyA = keys.find(k => k.label === 'Key A');+ const keyB = keys.find(k => k.label === 'Key B');++ // Mock behavior:+ // Key B is exhausted (returns false for canMakeRequest)+ // Key A is functional (returns true)+ (ratelimit.canMakeRequest as any).mockImplementation((platform, modelId, keyId) => {+ if (keyId === keyB.id) return false;+ if (keyId === keyA.id) return true;+ return true;+ });+ (ratelimit.canUseTokens as any).mockReturnValue(true);++ // Act: Route request+ const result = routeRequest(100);++ // Assert: It should have picked the Pro model despite Key B being exhausted+ expect(result.modelId).toBe('gemini-1.5-pro');+ expect(result.keyId).toBe(keyA.id);+ expect(ratelimit.canMakeRequest).toHaveBeenCalled();+ });++ it('should throw 429 when every key on every model is exhausted', () => {+ (ratelimit.canMakeRequest as any).mockReturnValue(false);+ expect(() => routeRequest(100)).toThrow(/All models exhausted/);+ });++ it('should fall back to Flash when Pro is exhausted but Flash has quota', () => {+ (ratelimit.canMakeRequest as any).mockImplementation((_platform: string, modelId: string) => {+ if (modelId === 'gemini-1.5-pro') return false;+ if (modelId === 'gemini-1.5-flash') return true;+ return true;+ });+ (ratelimit.canUseTokens as any).mockReturnValue(true);++ const result = routeRequest(100);+ expect(result.modelId).toBe('gemini-1.5-flash');+ });+});diff --git a/server/src/services/router.ts b/server/src/services/router.tsindex be8db232..ce540467 100644--- a/server/src/services/router.ts+++ b/server/src/services/router.ts@@ -174,7 +174,15 @@ export function routeRequest(estimatedTokens = 1000, skipKeys?: Set<string>, preif (keys.length === 0) continue;- // Round-robin across keys+ // Get limits once for this model+ const limits = {+ rpm: model.rpm_limit,+ rpd: model.rpd_limit,+ tpm: model.tpm_limit,+ tpd: model.tpd_limit,+ };++ // Try all keys for this model before giving up on itconst rrKey = `${model.platform}:${model.model_id}`;let idx = roundRobinIndex.get(rrKey) ?? 0;@@ -188,18 +196,11 @@ export function routeRequest(estimatedTokens = 1000, skipKeys?: Set<string>, pre// Check cooldown (from previous 429s)if (isOnCooldown(model.platform, model.model_id, key.id)) continue;- const limits = {- rpm: model.rpm_limit,- rpd: model.rpd_limit,- tpm: model.tpm_limit,- tpd: model.tpd_limit,- };-if (!canMakeRequest(model.platform, model.model_id, key.id, limits)) continue;if (!canUseTokens(model.platform, model.model_id, key.id, estimatedTokens, limits)) continue;+ // We found a working key for this model!roundRobinIndex.set(rrKey, idx);-const decryptedKey = decrypt(key.encrypted_key, key.iv, key.auth_tag);return {@@ -213,7 +214,13 @@ export function routeRequest(estimatedTokens = 1000, skipKeys?: Set<string>, pre};}+ // If we reach here, this specific model has NO available keys.+ // Update round-robin index even if we failed so we don't get stuck.roundRobinIndex.set(rrKey, idx);++ // We don't explicitly penalize the model here because the fact that we+ // couldn't find a key means we will naturally move to the next model+ // in the `sortedChain` for THIS specific request.}const err = new Error('All models exhausted. Add more API keys or wait for rate limits to reset.') as any;