Skip to content

7.4 billion tokens per month. 34 free LLM providers. 635 free model endpoints. All behind one /v1 endpoint, plus any custom OpenAI-compatible endpoint. Smart routing, automatic failover, encrypted keys. Personal experimentation only.

TypeScript

fix(router): drop keyless-platform models before the routing walk (#1105)

Chong Jia Zhen625add04845a
diff --git a/server/src/__tests__/services/exhaustion-summary.test.ts b/server/src/__tests__/services/exhaustion-summary.test.ts
index 525b0762..10b15cd9 100644
--- a/server/src/__tests__/services/exhaustion-summary.test.ts
+++ b/server/src/__tests__/services/exhaustion-summary.test.ts
@@ -51,6 +51,38 @@ describe('summarizeExhaustion', () => {
expect(msg).toContain('1 rate-limited or on cooldown');
});
+ // Keyless models are dropped before the routing walk, so they must be
+ // reported OUTSIDE the "routes checked" total: counting them inflated the
+ // pool the caller thinks it has (37 reported for a 22-candidate clean tier).
+ describe('keyless-skipped count', () => {
+ // Separator selectKeyForModel really emits, built by code point so the
+ // fixture stays byte-identical to production output.
+ const SEP = String.fromCharCode(0x2014);
+ const diag = [`groq/llama: 1 key(s) ${SEP} cooldown:1`];
+
+ it('reports skipped models separately from the route total', () => {
+ const msg = summarizeExhaustion(diag, null, now, 14);
+ expect(msg).toContain('1 route checked');
+ expect(msg).not.toContain('15 routes checked');
+ expect(msg).toContain('14 models skipped: no key configured for their platform.');
+ });
+
+ it('singularizes one skipped model', () => {
+ expect(summarizeExhaustion(diag, null, now, 1)).toContain('1 model skipped:');
+ });
+
+ it('says nothing when none were skipped', () => {
+ expect(summarizeExhaustion(diag, null, now, 0)).not.toContain('skipped');
+ expect(summarizeExhaustion(diag, null, now)).not.toContain('skipped');
+ });
+
+ it('still reports skips when every candidate was dropped (empty diag)', () => {
+ const msg = summarizeExhaustion([], null, now, 14);
+ expect(msg).toMatch(/^All models exhausted\./);
+ expect(msg).toContain('14 models skipped: no key configured for their platform.');
+ });
+ });
+
it('classifies prompt-too-large lines, not as rate limits', () => {
const diag = [
'groq/gpt-oss-120b: tpm_limit 8000 < estimated 33476',
diff --git a/server/src/__tests__/services/routing-keyless-filter.test.ts b/server/src/__tests__/services/routing-keyless-filter.test.ts
new file mode 100644
index 00000000..8f01c436
--- /dev/null
+++ b/server/src/__tests__/services/routing-keyless-filter.test.ts
@@ -0,0 +1,144 @@
+import { describe, it, expect, beforeEach, afterEach, vi } from 'vitest';
+import { routeRequest, setRoutingStrategy } from '../../services/router.js';
+import * as ratelimit from '../../services/ratelimit.js';
+import { getDb, initDb } from '../../db/index.js';
+import { routingExhaustionBody } from '../../lib/fallback-loop.js';
+
+// A model whose PLATFORM has no enabled+healthy key can never produce a route:
+// selectKeyForModel's first query comes back empty for it, every request. So it
+// is dropped before the walk instead of burning an iteration and padding the
+// exhaustion diagnostic with a constant that says nothing about why THIS
+// request failed. Live case: the clean tier reported "37 routes checked
+// (... 14 no usable key configured ...)" when only 22 rows were ever candidates.
+
+vi.mock('../../services/ratelimit.js', async () => {
+ const actual = await vi.importActual('../../services/ratelimit.js');
+ return {
+ ...actual,
+ canMakeRequest: vi.fn(() => true),
+ canUseTokens: vi.fn(() => true),
+ isOnCooldown: vi.fn(() => false),
+ };
+});
+
+vi.mock('../../lib/crypto.js', async () => {
+ const actual = await vi.importActual('../../lib/crypto.js');
+ return { ...actual, decrypt: vi.fn(() => 'mocked-api-key') };
+});
+
+const ORIGINAL_DEV_MODE = process.env.DEV_MODE;
+const ORIGINAL_NODE_ENV = process.env.NODE_ENV;
+
+function insertModel(platform: string, modelId: string, priority: number): number {
+ const db = getDb();
+ db.prepare(
+ "INSERT INTO models (platform, model_id, display_name, intelligence_rank, speed_rank, enabled) VALUES (?, ?, ?, 1, 1, 1)"
+ ).run(platform, modelId, modelId);
+ const id = (db.prepare('SELECT id FROM models WHERE model_id = ?').get(modelId) as any).id;
+ db.prepare('INSERT INTO fallback_config (model_db_id, priority, enabled) VALUES (?, ?, 1)').run(id, priority);
+ return id;
+}
+
+function routeError(): any {
+ try {
+ routeRequest(100);
+ } catch (e) {
+ return e;
+ }
+ throw new Error('expected routeRequest to throw');
+}
+
+describe('keyless-platform prefilter', () => {
+ beforeEach(() => {
+ process.env.DEV_MODE = 'true';
+ process.env.NODE_ENV = 'test';
+ initDb(':memory:');
+ setRoutingStrategy('priority');
+ const db = getDb();
+ db.prepare("DELETE FROM settings WHERE key = 'active_profile_id'").run();
+ db.prepare('DELETE FROM fallback_config').run();
+ db.prepare('DELETE FROM profile_models').run();
+ db.prepare('DELETE FROM models').run();
+ db.prepare('DELETE FROM api_keys').run();
+
+ // google is keyed; cohere and mistral are in the catalog with no key at all.
+ insertModel('google', 'gemini-1.5-pro', 1);
+ insertModel('cohere', 'command-r', 2);
+ insertModel('mistral', 'mistral-small', 3);
+ db.prepare(
+ "INSERT INTO api_keys (platform, label, encrypted_key, iv, auth_tag, status, enabled) VALUES ('google', 'Key A', 'enc', 'iv', 'tag', 'healthy', 1)"
+ ).run();
+
+ vi.clearAllMocks();
+ (ratelimit.canMakeRequest as any).mockReturnValue(true);
+ (ratelimit.canUseTokens as any).mockReturnValue(true);
+ (ratelimit.isOnCooldown as any).mockReturnValue(false);
+ });
+
+ afterEach(() => {
+ if (ORIGINAL_DEV_MODE === undefined) delete process.env.DEV_MODE;
+ else process.env.DEV_MODE = ORIGINAL_DEV_MODE;
+ if (ORIGINAL_NODE_ENV === undefined) delete process.env.NODE_ENV;
+ else process.env.NODE_ENV = ORIGINAL_NODE_ENV;
+ });
+
+ it('still routes to the keyed model (no behavior change on the happy path)', () => {
+ expect(routeRequest(100).modelId).toBe('gemini-1.5-pro');
+ });
+
+ // The absent diag line IS the proof the row was never walked: a keyless model
+ // that reaches selectKeyForModel always pushes "no enabled+healthy key for
+ // platform" before returning null.
+ it('keeps keyless models out of the walk, the diagnostics, and the route count', () => {
+ (ratelimit.canMakeRequest as any).mockReturnValue(false); // exhaust the keyed model
+
+ const err = routeError();
+ // Walked rows + exactly ONE aggregate line for the two keyless platforms.
+ expect(err.diagnostics).toHaveLength(2);
+ expect(err.diagnostics[0]).toContain('google/gemini-1.5-pro');
+ expect(err.diagnostics[1]).toBe(
+ '2 model(s) skipped: no enabled+healthy key for platform (cohere, mistral)'
+ );
+ expect(err.message).toContain('1 route checked');
+ expect(err.message).toContain('2 models skipped: no key configured for their platform.');
+ expect(err.message).not.toContain('no usable key configured');
+ });
+
+ it('exempts an explicitly pinned model so it reports against its own label', () => {
+ const db = getDb();
+ const cohereId = (db.prepare("SELECT id FROM models WHERE model_id = 'command-r'").get() as any).id;
+
+ let err: any;
+ try {
+ routeRequest(100, undefined, cohereId);
+ } catch (e) {
+ err = e;
+ }
+ // The pin is walked, fails honestly, and the keyed google model is still
+ // reached as fallback - so this must NOT throw.
+ expect(err).toBeUndefined();
+
+ (ratelimit.canMakeRequest as any).mockReturnValue(false);
+ try {
+ routeRequest(100, undefined, cohereId);
+ } catch (e) {
+ err = e;
+ }
+ expect(err.diagnostics.some((d: string) => d.includes('cohere/command-r'))).toBe(true);
+ expect(err.message).toContain('1 no usable key configured');
+ // mistral stays filtered; only cohere was exempted.
+ expect(err.message).toContain('1 model skipped: no key configured for their platform.');
+ });
+
+ it('reports the skip count even when every candidate is keyless', () => {
+ getDb().prepare('DELETE FROM api_keys').run();
+ const err = routeError();
+ expect(err.message).toContain('3 models skipped: no key configured for their platform.');
+ // A fully-unconfigured pool must still classify as config (503), which
+ // routingExhaustionBody reads off the diagnostics, not the message.
+ expect(err.diagnostics).toEqual([
+ '3 model(s) skipped: no enabled+healthy key for platform (cohere, google, mistral)',
+ ]);
+ expect(routingExhaustionBody(err).status).toBe(503);
+ });
+});
diff --git a/server/src/services/router.ts b/server/src/services/router.ts
index 2da4fb41..95b804e3 100644
--- a/server/src/services/router.ts
+++ b/server/src/services/router.ts
@@ -81,11 +81,18 @@ export function summarizeExhaustion(
diag: string[] | undefined,
soonestResetMs?: number | null,
now = Date.now(),
+ keylessSkipped = 0,
): string {
const eta = formatResetEta(soonestResetMs, now);
const etaSuffix = eta ? ` Soonest reset ${eta}.` : '';
+ // Models dropped before the walk (#423 follow-up) are reported separately and
+ // never counted as "routes checked": they were never candidates, and folding
+ // them into the total inflates the pool the caller thinks it has.
+ const keylessSuffix = keylessSkipped > 0
+ ? ` ${keylessSkipped} model${keylessSkipped === 1 ? '' : 's'} skipped: no key configured for their platform.`
+ : '';
if (!diag || diag.length === 0) {
- return `All models exhausted. ${EXHAUSTION_ADVICE}${etaSuffix}`;
+ return `All models exhausted. ${EXHAUSTION_ADVICE}${etaSuffix}${keylessSuffix}`;
}
const counts: Record<string, number> = {};
@@ -116,7 +123,7 @@ export function summarizeExhaustion(
];
const parts = order.filter(b => counts[b]).map(b => `${counts[b]} ${b}`);
const total = diag.length;
- return `All models exhausted: ${total} route${total === 1 ? '' : 's'} checked (${parts.join(', ')}). ${EXHAUSTION_ADVICE}${etaSuffix}`;
+ return `All models exhausted: ${total} route${total === 1 ? '' : 's'} checked (${parts.join(', ')}). ${EXHAUSTION_ADVICE}${etaSuffix}${keylessSuffix}`;
}
interface KeyRow {
@@ -2024,6 +2031,31 @@ export function routeRequest(estimatedTokens = 1000, skipKeys?: Set<string>, pre
}
}
+ // Drop models whose platform has NO enabled+healthy key before the walk. Such
+ // a row can never produce a route (selectKeyForModel's first query returns
+ // empty for it), so walking it is pure overhead on every request, and its diag
+ // line pads the exhaustion summary with a constant that has nothing to do with
+ // why THIS request failed. On the clean tier that was 14 of 36 rows, reported
+ // as "37 routes checked" when only 22 were ever candidates.
+ //
+ // An explicit pin is exempt: the client named that model, so it still gets
+ // walked and still reports "no enabled+healthy key for platform" against its
+ // own label rather than vanishing into an aggregate.
+ const keyCounts = usableKeyCountsByPlatform(db);
+ const isRoutable = (e: ChainRow) =>
+ e.model_db_id === preferredModelDbId || (keyCounts.get(e.platform) ?? 0) > 0;
+ const routableChain = sortedChain.filter(isRoutable);
+ const keylessSkipped = sortedChain.length - routableChain.length;
+ // One aggregate line, not one per model: the platforms stay visible to anyone
+ // reading RouteError.diagnostics (and keep routingExhaustionBody classifying a
+ // fully-unconfigured pool as 503 config, not a 429 rate limit), without N
+ // near-identical rows drowning the request's real reasons.
+ const keylessLine = keylessSkipped > 0
+ ? `${keylessSkipped} model(s) skipped: no enabled+healthy key for platform (${
+ [...new Set(sortedChain.filter(e => !isRoutable(e)).map(e => e.platform))].sort().join(', ')
+ })`
+ : null;
+
// Per-model disposition, attached to the exhaustion error when the loop falls
// through with no route — the only record of WHY the pool was empty on the
// synchronous "all exhausted" path (nothing downstream logs it). See issue _1.
@@ -2038,7 +2070,7 @@ export function routeRequest(estimatedTokens = 1000, skipKeys?: Set<string>, pre
// hop instead of no route at all.
const servingChain: ChainRow[] = [];
const marginDeferred: ChainRow[] = [];
- for (const e of sortedChain) {
+ for (const e of routableChain) {
(fitsContextWindow(e.platform, e.context_window, estimatedTokens, exactOutputReserve) ? servingChain : marginDeferred).push(e);
}
servingChain.push(...marginDeferred);
@@ -2115,7 +2147,13 @@ export function routeRequest(estimatedTokens = 1000, skipKeys?: Set<string>, pre
if (route) return route;
}
- throw new RouteError(summarizeExhaustion(diag, getSoonestCooldownExpiry()), 429, diag);
+ // The aggregate keyless line rides in diagnostics but NOT in the summary's
+ // route count: those models were never candidates for this request.
+ throw new RouteError(
+ summarizeExhaustion(diag, getSoonestCooldownExpiry(), Date.now(), keylessSkipped),
+ 429,
+ keylessLine ? [...diag, keylessLine] : diag,
+ );
}
/**