Skip to content

7.4 billion tokens per month. 34 free LLM providers. 635 free model endpoints. All behind one /v1 endpoint, plus any custom OpenAI-compatible endpoint. Smart routing, automatic failover, encrypted keys. Personal experimentation only.

TypeScript

feat(router): exhaust all keys before falling back + tests (#42, supersedes #33)

Tashfeen9b972192ef12
diff --git a/server/src/__tests__/services/routing-exhaustion.test.ts b/server/src/__tests__/services/routing-exhaustion.test.ts
new file mode 100644
index 00000000..a7f5fde9
--- /dev/null
+++ b/server/src/__tests__/services/routing-exhaustion.test.ts
@@ -0,0 +1,91 @@
+import { describe, it, expect, beforeEach, vi } from 'vitest';
+import { routeRequest } from '../../services/router.js';
+import * as ratelimit from '../../services/ratelimit.js';
+import { getDb, initDb } from '../../db/index.js';
+import * as crypto from '../../lib/crypto.js';
+
+// Mock ratelimit to control quota availability
+vi.mock('../../services/ratelimit.js', async () => {
+ const actual = await vi.importActual('../../services/ratelimit.js');
+ return {
+ ...actual,
+ canMakeRequest: vi.fn(),
+ canUseTokens: vi.fn(),
+ isOnCooldown: vi.fn(() => false),
+ };
+});
+
+// Mock crypto to avoid IV errors
+vi.mock('../../lib/crypto.js', async () => {
+ const actual = await vi.importActual('../../lib/crypto.js');
+ return {
+ ...actual,
+ decrypt: vi.fn(() => 'mocked-api-key'),
+ };
+});
+
+describe('Routing Key Exhaustion', () => {
+ beforeEach(() => {
+ initDb(':memory:');
+ const db = getDb();
+
+ // Setup: 2 models (Pro and Flash)
+ // Pro is higher priority (priority 1), Flash is lower (priority 2)
+ db.prepare("INSERT INTO models (platform, model_id, display_name, intelligence_rank, speed_rank, enabled) VALUES ('google', 'gemini-1.5-pro', 'Pro', 1, 1, 1)").run();
+ db.prepare("INSERT INTO models (platform, model_id, display_name, intelligence_rank, speed_rank, enabled) VALUES ('google', 'gemini-1.5-flash', 'Flash', 2, 2, 1)").run();
+
+ const proId = db.prepare("SELECT id FROM models WHERE model_id = 'gemini-1.5-pro'").get().id;
+ const flashId = db.prepare("SELECT id FROM models WHERE model_id = 'gemini-1.5-flash'").get().id;
+
+ db.prepare("INSERT INTO fallback_config (model_db_id, priority, enabled) VALUES (?, 1, 1)").run(proId);
+ db.prepare("INSERT INTO fallback_config (model_db_id, priority, enabled) VALUES (?, 2, 1)").run(flashId);
+
+ // Setup: 2 keys for Google
+ db.prepare("INSERT INTO api_keys (platform, label, encrypted_key, iv, auth_tag, status, enabled) VALUES ('google', 'Key A', 'enc', 'iv', 'tag', 'healthy', 1)").run();
+ db.prepare("INSERT INTO api_keys (platform, label, encrypted_key, iv, auth_tag, status, enabled) VALUES ('google', 'Key B', 'enc', 'iv', 'tag', 'healthy', 1)").run();
+
+ vi.clearAllMocks();
+ });
+
+ it('should skip exhausted Key B and use functional Key A for the same high-priority model', () => {
+ const db = getDb();
+ const keys = db.prepare("SELECT id, label FROM api_keys").all();
+ const keyA = keys.find(k => k.label === 'Key A');
+ const keyB = keys.find(k => k.label === 'Key B');
+
+ // Mock behavior:
+ // Key B is exhausted (returns false for canMakeRequest)
+ // Key A is functional (returns true)
+ (ratelimit.canMakeRequest as any).mockImplementation((platform, modelId, keyId) => {
+ if (keyId === keyB.id) return false;
+ if (keyId === keyA.id) return true;
+ return true;
+ });
+ (ratelimit.canUseTokens as any).mockReturnValue(true);
+
+ // Act: Route request
+ const result = routeRequest(100);
+
+ // Assert: It should have picked the Pro model despite Key B being exhausted
+ expect(result.modelId).toBe('gemini-1.5-pro');
+ expect(result.keyId).toBe(keyA.id);
+ expect(ratelimit.canMakeRequest).toHaveBeenCalled();
+ });
+
+ it('should throw 429 when every key on every model is exhausted', () => {
+ (ratelimit.canMakeRequest as any).mockReturnValue(false);
+ expect(() => routeRequest(100)).toThrow(/All models exhausted/);
+ });
+
+ it('should fall back to Flash when Pro is exhausted but Flash has quota', () => {
+ (ratelimit.canMakeRequest as any).mockImplementation((_platform: string, modelId: string) => {
+ if (modelId === 'gemini-1.5-pro') return false;
+ if (modelId === 'gemini-1.5-flash') return true;
+ return true;
+ });
+ (ratelimit.canUseTokens as any).mockReturnValue(true);
+
+ const result = routeRequest(100);
+ expect(result.modelId).toBe('gemini-1.5-flash');
+ });
+});
diff --git a/server/src/services/router.ts b/server/src/services/router.ts
index be8db232..ce540467 100644
--- a/server/src/services/router.ts
+++ b/server/src/services/router.ts
@@ -174,7 +174,15 @@ export function routeRequest(estimatedTokens = 1000, skipKeys?: Set<string>, pre
if (keys.length === 0) continue;
- // Round-robin across keys
+ // Get limits once for this model
+ const limits = {
+ rpm: model.rpm_limit,
+ rpd: model.rpd_limit,
+ tpm: model.tpm_limit,
+ tpd: model.tpd_limit,
+ };
+
+ // Try all keys for this model before giving up on it
const rrKey = `${model.platform}:${model.model_id}`;
let idx = roundRobinIndex.get(rrKey) ?? 0;
@@ -188,18 +196,11 @@ export function routeRequest(estimatedTokens = 1000, skipKeys?: Set<string>, pre
// Check cooldown (from previous 429s)
if (isOnCooldown(model.platform, model.model_id, key.id)) continue;
- const limits = {
- rpm: model.rpm_limit,
- rpd: model.rpd_limit,
- tpm: model.tpm_limit,
- tpd: model.tpd_limit,
- };
-
if (!canMakeRequest(model.platform, model.model_id, key.id, limits)) continue;
if (!canUseTokens(model.platform, model.model_id, key.id, estimatedTokens, limits)) continue;
+ // We found a working key for this model!
roundRobinIndex.set(rrKey, idx);
-
const decryptedKey = decrypt(key.encrypted_key, key.iv, key.auth_tag);
return {
@@ -213,7 +214,13 @@ export function routeRequest(estimatedTokens = 1000, skipKeys?: Set<string>, pre
};
}
+ // If we reach here, this specific model has NO available keys.
+ // Update round-robin index even if we failed so we don't get stuck.
roundRobinIndex.set(rrKey, idx);
+
+ // We don't explicitly penalize the model here because the fact that we
+ // couldn't find a key means we will naturally move to the next model
+ // in the `sortedChain` for THIS specific request.
}
const err = new Error('All models exhausted. Add more API keys or wait for rate limits to reset.') as any;