7.4 billion tokens per month. 34 free LLM providers. 635 free model endpoints. All behind one /v1 endpoint, plus any custom OpenAI-compatible endpoint. Smart routing, automatic failover, encrypted keys. Personal experimentation only.
TypeScript
fix(router): drop keyless-platform models before the routing walk (#1105)
Chong Jia Zhen625add04845a
diff --git a/server/src/__tests__/services/exhaustion-summary.test.ts b/server/src/__tests__/services/exhaustion-summary.test.tsindex 525b0762..10b15cd9 100644--- a/server/src/__tests__/services/exhaustion-summary.test.ts+++ b/server/src/__tests__/services/exhaustion-summary.test.ts@@ -51,6 +51,38 @@ describe('summarizeExhaustion', () => {expect(msg).toContain('1 rate-limited or on cooldown');});+ // Keyless models are dropped before the routing walk, so they must be+ // reported OUTSIDE the "routes checked" total: counting them inflated the+ // pool the caller thinks it has (37 reported for a 22-candidate clean tier).+ describe('keyless-skipped count', () => {+ // Separator selectKeyForModel really emits, built by code point so the+ // fixture stays byte-identical to production output.+ const SEP = String.fromCharCode(0x2014);+ const diag = [`groq/llama: 1 key(s) ${SEP} cooldown:1`];++ it('reports skipped models separately from the route total', () => {+ const msg = summarizeExhaustion(diag, null, now, 14);+ expect(msg).toContain('1 route checked');+ expect(msg).not.toContain('15 routes checked');+ expect(msg).toContain('14 models skipped: no key configured for their platform.');+ });++ it('singularizes one skipped model', () => {+ expect(summarizeExhaustion(diag, null, now, 1)).toContain('1 model skipped:');+ });++ it('says nothing when none were skipped', () => {+ expect(summarizeExhaustion(diag, null, now, 0)).not.toContain('skipped');+ expect(summarizeExhaustion(diag, null, now)).not.toContain('skipped');+ });++ it('still reports skips when every candidate was dropped (empty diag)', () => {+ const msg = summarizeExhaustion([], null, now, 14);+ expect(msg).toMatch(/^All models exhausted\./);+ expect(msg).toContain('14 models skipped: no key configured for their platform.');+ });+ });+it('classifies prompt-too-large lines, not as rate limits', () => {const diag = ['groq/gpt-oss-120b: tpm_limit 8000 < estimated 33476',diff --git a/server/src/__tests__/services/routing-keyless-filter.test.ts b/server/src/__tests__/services/routing-keyless-filter.test.tsnew file mode 100644index 00000000..8f01c436--- /dev/null+++ b/server/src/__tests__/services/routing-keyless-filter.test.ts@@ -0,0 +1,144 @@+import { describe, it, expect, beforeEach, afterEach, vi } from 'vitest';+import { routeRequest, setRoutingStrategy } from '../../services/router.js';+import * as ratelimit from '../../services/ratelimit.js';+import { getDb, initDb } from '../../db/index.js';+import { routingExhaustionBody } from '../../lib/fallback-loop.js';++// A model whose PLATFORM has no enabled+healthy key can never produce a route:+// selectKeyForModel's first query comes back empty for it, every request. So it+// is dropped before the walk instead of burning an iteration and padding the+// exhaustion diagnostic with a constant that says nothing about why THIS+// request failed. Live case: the clean tier reported "37 routes checked+// (... 14 no usable key configured ...)" when only 22 rows were ever candidates.++vi.mock('../../services/ratelimit.js', async () => {+ const actual = await vi.importActual('../../services/ratelimit.js');+ return {+ ...actual,+ canMakeRequest: vi.fn(() => true),+ canUseTokens: vi.fn(() => true),+ isOnCooldown: vi.fn(() => false),+ };+});++vi.mock('../../lib/crypto.js', async () => {+ const actual = await vi.importActual('../../lib/crypto.js');+ return { ...actual, decrypt: vi.fn(() => 'mocked-api-key') };+});++const ORIGINAL_DEV_MODE = process.env.DEV_MODE;+const ORIGINAL_NODE_ENV = process.env.NODE_ENV;++function insertModel(platform: string, modelId: string, priority: number): number {+ const db = getDb();+ db.prepare(+ "INSERT INTO models (platform, model_id, display_name, intelligence_rank, speed_rank, enabled) VALUES (?, ?, ?, 1, 1, 1)"+ ).run(platform, modelId, modelId);+ const id = (db.prepare('SELECT id FROM models WHERE model_id = ?').get(modelId) as any).id;+ db.prepare('INSERT INTO fallback_config (model_db_id, priority, enabled) VALUES (?, ?, 1)').run(id, priority);+ return id;+}++function routeError(): any {+ try {+ routeRequest(100);+ } catch (e) {+ return e;+ }+ throw new Error('expected routeRequest to throw');+}++describe('keyless-platform prefilter', () => {+ beforeEach(() => {+ process.env.DEV_MODE = 'true';+ process.env.NODE_ENV = 'test';+ initDb(':memory:');+ setRoutingStrategy('priority');+ const db = getDb();+ db.prepare("DELETE FROM settings WHERE key = 'active_profile_id'").run();+ db.prepare('DELETE FROM fallback_config').run();+ db.prepare('DELETE FROM profile_models').run();+ db.prepare('DELETE FROM models').run();+ db.prepare('DELETE FROM api_keys').run();++ // google is keyed; cohere and mistral are in the catalog with no key at all.+ insertModel('google', 'gemini-1.5-pro', 1);+ insertModel('cohere', 'command-r', 2);+ insertModel('mistral', 'mistral-small', 3);+ db.prepare(+ "INSERT INTO api_keys (platform, label, encrypted_key, iv, auth_tag, status, enabled) VALUES ('google', 'Key A', 'enc', 'iv', 'tag', 'healthy', 1)"+ ).run();++ vi.clearAllMocks();+ (ratelimit.canMakeRequest as any).mockReturnValue(true);+ (ratelimit.canUseTokens as any).mockReturnValue(true);+ (ratelimit.isOnCooldown as any).mockReturnValue(false);+ });++ afterEach(() => {+ if (ORIGINAL_DEV_MODE === undefined) delete process.env.DEV_MODE;+ else process.env.DEV_MODE = ORIGINAL_DEV_MODE;+ if (ORIGINAL_NODE_ENV === undefined) delete process.env.NODE_ENV;+ else process.env.NODE_ENV = ORIGINAL_NODE_ENV;+ });++ it('still routes to the keyed model (no behavior change on the happy path)', () => {+ expect(routeRequest(100).modelId).toBe('gemini-1.5-pro');+ });++ // The absent diag line IS the proof the row was never walked: a keyless model+ // that reaches selectKeyForModel always pushes "no enabled+healthy key for+ // platform" before returning null.+ it('keeps keyless models out of the walk, the diagnostics, and the route count', () => {+ (ratelimit.canMakeRequest as any).mockReturnValue(false); // exhaust the keyed model++ const err = routeError();+ // Walked rows + exactly ONE aggregate line for the two keyless platforms.+ expect(err.diagnostics).toHaveLength(2);+ expect(err.diagnostics[0]).toContain('google/gemini-1.5-pro');+ expect(err.diagnostics[1]).toBe(+ '2 model(s) skipped: no enabled+healthy key for platform (cohere, mistral)'+ );+ expect(err.message).toContain('1 route checked');+ expect(err.message).toContain('2 models skipped: no key configured for their platform.');+ expect(err.message).not.toContain('no usable key configured');+ });++ it('exempts an explicitly pinned model so it reports against its own label', () => {+ const db = getDb();+ const cohereId = (db.prepare("SELECT id FROM models WHERE model_id = 'command-r'").get() as any).id;++ let err: any;+ try {+ routeRequest(100, undefined, cohereId);+ } catch (e) {+ err = e;+ }+ // The pin is walked, fails honestly, and the keyed google model is still+ // reached as fallback - so this must NOT throw.+ expect(err).toBeUndefined();++ (ratelimit.canMakeRequest as any).mockReturnValue(false);+ try {+ routeRequest(100, undefined, cohereId);+ } catch (e) {+ err = e;+ }+ expect(err.diagnostics.some((d: string) => d.includes('cohere/command-r'))).toBe(true);+ expect(err.message).toContain('1 no usable key configured');+ // mistral stays filtered; only cohere was exempted.+ expect(err.message).toContain('1 model skipped: no key configured for their platform.');+ });++ it('reports the skip count even when every candidate is keyless', () => {+ getDb().prepare('DELETE FROM api_keys').run();+ const err = routeError();+ expect(err.message).toContain('3 models skipped: no key configured for their platform.');+ // A fully-unconfigured pool must still classify as config (503), which+ // routingExhaustionBody reads off the diagnostics, not the message.+ expect(err.diagnostics).toEqual([+ '3 model(s) skipped: no enabled+healthy key for platform (cohere, google, mistral)',+ ]);+ expect(routingExhaustionBody(err).status).toBe(503);+ });+});diff --git a/server/src/services/router.ts b/server/src/services/router.tsindex 2da4fb41..95b804e3 100644--- a/server/src/services/router.ts+++ b/server/src/services/router.ts@@ -81,11 +81,18 @@ export function summarizeExhaustion(diag: string[] | undefined,soonestResetMs?: number | null,now = Date.now(),+ keylessSkipped = 0,): string {const eta = formatResetEta(soonestResetMs, now);const etaSuffix = eta ? ` Soonest reset ${eta}.` : '';+ // Models dropped before the walk (#423 follow-up) are reported separately and+ // never counted as "routes checked": they were never candidates, and folding+ // them into the total inflates the pool the caller thinks it has.+ const keylessSuffix = keylessSkipped > 0+ ? ` ${keylessSkipped} model${keylessSkipped === 1 ? '' : 's'} skipped: no key configured for their platform.`+ : '';if (!diag || diag.length === 0) {- return `All models exhausted. ${EXHAUSTION_ADVICE}${etaSuffix}`;+ return `All models exhausted. ${EXHAUSTION_ADVICE}${etaSuffix}${keylessSuffix}`;}const counts: Record<string, number> = {};@@ -116,7 +123,7 @@ export function summarizeExhaustion(];const parts = order.filter(b => counts[b]).map(b => `${counts[b]} ${b}`);const total = diag.length;- return `All models exhausted: ${total} route${total === 1 ? '' : 's'} checked (${parts.join(', ')}). ${EXHAUSTION_ADVICE}${etaSuffix}`;+ return `All models exhausted: ${total} route${total === 1 ? '' : 's'} checked (${parts.join(', ')}). ${EXHAUSTION_ADVICE}${etaSuffix}${keylessSuffix}`;}interface KeyRow {@@ -2024,6 +2031,31 @@ export function routeRequest(estimatedTokens = 1000, skipKeys?: Set<string>, pre}}+ // Drop models whose platform has NO enabled+healthy key before the walk. Such+ // a row can never produce a route (selectKeyForModel's first query returns+ // empty for it), so walking it is pure overhead on every request, and its diag+ // line pads the exhaustion summary with a constant that has nothing to do with+ // why THIS request failed. On the clean tier that was 14 of 36 rows, reported+ // as "37 routes checked" when only 22 were ever candidates.+ //+ // An explicit pin is exempt: the client named that model, so it still gets+ // walked and still reports "no enabled+healthy key for platform" against its+ // own label rather than vanishing into an aggregate.+ const keyCounts = usableKeyCountsByPlatform(db);+ const isRoutable = (e: ChainRow) =>+ e.model_db_id === preferredModelDbId || (keyCounts.get(e.platform) ?? 0) > 0;+ const routableChain = sortedChain.filter(isRoutable);+ const keylessSkipped = sortedChain.length - routableChain.length;+ // One aggregate line, not one per model: the platforms stay visible to anyone+ // reading RouteError.diagnostics (and keep routingExhaustionBody classifying a+ // fully-unconfigured pool as 503 config, not a 429 rate limit), without N+ // near-identical rows drowning the request's real reasons.+ const keylessLine = keylessSkipped > 0+ ? `${keylessSkipped} model(s) skipped: no enabled+healthy key for platform (${+ [...new Set(sortedChain.filter(e => !isRoutable(e)).map(e => e.platform))].sort().join(', ')+ })`+ : null;+// Per-model disposition, attached to the exhaustion error when the loop falls// through with no route — the only record of WHY the pool was empty on the// synchronous "all exhausted" path (nothing downstream logs it). See issue _1.@@ -2038,7 +2070,7 @@ export function routeRequest(estimatedTokens = 1000, skipKeys?: Set<string>, pre// hop instead of no route at all.const servingChain: ChainRow[] = [];const marginDeferred: ChainRow[] = [];- for (const e of sortedChain) {+ for (const e of routableChain) {(fitsContextWindow(e.platform, e.context_window, estimatedTokens, exactOutputReserve) ? servingChain : marginDeferred).push(e);}servingChain.push(...marginDeferred);@@ -2115,7 +2147,13 @@ export function routeRequest(estimatedTokens = 1000, skipKeys?: Set<string>, preif (route) return route;}- throw new RouteError(summarizeExhaustion(diag, getSoonestCooldownExpiry()), 429, diag);+ // The aggregate keyless line rides in diagnostics but NOT in the summary's+ // route count: those models were never candidates for this request.+ throw new RouteError(+ summarizeExhaustion(diag, getSoonestCooldownExpiry(), Date.now(), keylessSkipped),+ 429,+ keylessLine ? [...diag, keylessLine] : diag,+ );}/**