Skip to content

7.4 billion tokens per month. 34 free LLM providers. 635 free model endpoints. All behind one /v1 endpoint, plus any custom OpenAI-compatible endpoint. Smart routing, automatic failover, encrypted keys. Personal experimentation only.

TypeScript

fix(router): surface per-model disposition on auto-routing exhaustion

jiazhenf96a604630f1
diff --git a/server/src/__tests__/services/router.test.ts b/server/src/__tests__/services/router.test.ts
index e3007aaf..5e81d433 100644
--- a/server/src/__tests__/services/router.test.ts
+++ b/server/src/__tests__/services/router.test.ts
@@ -7,6 +7,7 @@ import {
routeRequest,
setRoutingStrategy,
} from '../../services/router.js';
+import { setCooldown } from '../../services/ratelimit.js';
describe('Router', () => {
beforeAll(() => {
@@ -185,3 +186,57 @@ describe('Router', () => {
});
});
});
+
+describe('Router exhaustion diagnostics (issue _1)', () => {
+ beforeAll(() => {
+ process.env.ENCRYPTION_KEY = '0'.repeat(64);
+ initDb(':memory:');
+ });
+
+ beforeEach(() => {
+ const db = getDb();
+ setRoutingStrategy('priority');
+ db.prepare('DELETE FROM api_keys').run();
+ db.prepare("DELETE FROM settings WHERE key = 'active_profile_id'").run();
+ db.prepare('DELETE FROM rate_limit_cooldowns').run();
+ });
+
+ afterEach(() => {
+ vi.useRealTimers();
+ });
+
+ it('attaches a non-empty per-model disposition to the exhaustion error', () => {
+ // No keys configured → every chain model is unroutable. The thrown error
+ // must carry diagnostics so the synchronous routing_error is debuggable
+ // instead of opaque (the failure that NOTHING else logs).
+ let caught: any;
+ try { routeRequest(); } catch (e) { caught = e; }
+ expect(caught).toBeDefined();
+ expect(Array.isArray(caught.diagnostics)).toBe(true);
+ expect(caught.diagnostics.length).toBeGreaterThan(0);
+ // Every entry is "<platform>/<model>: <reason>"; with no keys the reason is
+ // the platform having no enabled+healthy key.
+ expect(caught.diagnostics.every((d: string) => d.includes(': '))).toBe(true);
+ expect(caught.diagnostics.some((d: string) => /no enabled.*key/i.test(d))).toBe(true);
+ });
+
+ it('records cooldown as the skip reason for a benched key', () => {
+ const db = getDb();
+ const groqKey = encrypt('test-groq-key');
+ db.prepare(`
+ INSERT INTO api_keys (platform, label, encrypted_key, iv, auth_tag, status, enabled)
+ VALUES (?, ?, ?, ?, ?, ?, ?)
+ `).run('groq', 'test', groqKey.encrypted, groqKey.iv, groqKey.authTag, 'healthy', 1);
+
+ // Bench every groq model on this key, so the only configured provider is
+ // fully cooled down and the pool empties with a key present (not absent).
+ const keyId = (db.prepare("SELECT id FROM api_keys WHERE platform='groq'").get() as { id: number }).id;
+ const groqModels = db.prepare("SELECT model_id FROM models WHERE platform='groq' AND enabled=1").all() as { model_id: string }[];
+ for (const m of groqModels) setCooldown('groq', m.model_id, keyId, 5 * 60 * 1000);
+
+ let caught: any;
+ try { routeRequest(); } catch (e) { caught = e; }
+ expect(caught).toBeDefined();
+ expect(caught.diagnostics.some((d: string) => /cooldown/.test(d))).toBe(true);
+ });
+});
diff --git a/server/src/routes/proxy.ts b/server/src/routes/proxy.ts
index 6e5a5173..1210a096 100644
--- a/server/src/routes/proxy.ts
+++ b/server/src/routes/proxy.ts
@@ -916,6 +916,16 @@ proxyRouter.post('/chat/completions', async (req: Request, res: Response) => {
},
});
} else {
+ // Synchronous exhaustion: the router rejected every candidate before any
+ // upstream was tried, so this is the ONLY place the per-model disposition
+ // is recorded. Without it a routing_error 429 is opaque — you can't tell a
+ // genuinely dry pool from cooldowns/quota/context narrowing (issue _1).
+ const disposition: string[] = Array.isArray(err.diagnostics) ? err.diagnostics : [];
+ console.warn(
+ `[Proxy] routing exhausted (no upstream tried) req=${shortRequestId(requestGroupId)} ` +
+ `requested=${requestedModelLabel} candidates=${disposition.length}` +
+ (disposition.length ? `:\n ${disposition.join('\n ')}` : ''),
+ );
res.status(err.status ?? 503).json({
error: { message: err.message, type: 'routing_error' },
});
diff --git a/server/src/services/router.ts b/server/src/services/router.ts
index 6aeff47b..2565abaf 100644
--- a/server/src/services/router.ts
+++ b/server/src/services/router.ts
@@ -15,9 +15,16 @@ import type { Database } from 'better-sqlite3';
class RouteError extends Error {
status: number;
- constructor(message: string, status: number) {
+ // Per-model disposition of the chain at the moment routing gave up: one line
+ // per considered model with the reason it could not serve (no key, cooldown,
+ // provider cap, rpm/rpd, tpm/tpd, context too small, …). Populated only on the
+ // synchronous "all exhausted" throw, where NO upstream was tried and nothing
+ // else logs WHY the pool was empty (issue _1: opaque routing_error 429).
+ diagnostics?: string[];
+ constructor(message: string, status: number, diagnostics?: string[]) {
super(message);
this.status = status;
+ this.diagnostics = diagnostics;
}
}
@@ -543,16 +550,28 @@ export function resolveRoutingChain(modelString: string | undefined): ResolvedCh
* context window) stay in the caller; this only does key selection + accounting
* pre-checks.
*/
-function selectKeyForModel(entry: ChainRow, estimatedTokens: number, skipKeys?: Set<string>): RouteResult | null {
+function selectKeyForModel(entry: ChainRow, estimatedTokens: number, skipKeys?: Set<string>, diag?: string[]): RouteResult | null {
const db = getDb();
+ const label = `${entry.platform}/${entry.model_id}`;
- if (!hasProvider(entry.platform as Platform)) return null;
+ if (!hasProvider(entry.platform as Platform)) {
+ diag?.push(`${label}: no provider registered`);
+ return null;
+ }
const provider = getProvider(entry.platform as Platform)!;
const keys = db.prepare(
"SELECT * FROM api_keys WHERE platform = ? AND enabled = 1 AND status IN ('healthy', 'unknown')"
).all(entry.platform) as KeyRow[];
- if (keys.length === 0) return null;
+ if (keys.length === 0) {
+ diag?.push(`${label}: no enabled+healthy key for platform`);
+ return null;
+ }
+
+ // Tally the gate that rejected each key, so the exhaustion diagnostic can say
+ // *why* a model with keys still couldn't serve (all on cooldown vs over quota).
+ const skipTally: Record<string, number> = {};
+ const note = (reason: string) => { skipTally[reason] = (skipTally[reason] ?? 0) + 1; };
const limits = {
rpm: entry.rpm_limit,
@@ -570,15 +589,15 @@ function selectKeyForModel(entry: ChainRow, estimatedTokens: number, skipKeys?:
// A custom model belongs to exactly one endpoint (#212); legacy rows
// (key_id NULL) keep the old any-key match.
- if (entry.platform === 'custom' && entry.key_id != null && key.id !== entry.key_id) continue;
+ if (entry.platform === 'custom' && entry.key_id != null && key.id !== entry.key_id) { note('custom-key-mismatch'); continue; }
const skipId = `${entry.platform}:${entry.model_id}:${key.id}`;
- if (skipKeys?.has(skipId)) continue;
+ if (skipKeys?.has(skipId)) { note('already-failed-this-request'); continue; }
- if (isOnCooldown(entry.platform, entry.model_id, key.id)) continue;
- if (!canUseProvider(entry.platform, key.id)) continue;
- if (!canMakeRequest(entry.platform, entry.model_id, key.id, limits)) continue;
- if (!canUseTokens(entry.platform, entry.model_id, key.id, estimatedTokens, limits)) continue;
+ if (isOnCooldown(entry.platform, entry.model_id, key.id)) { note('cooldown'); continue; }
+ if (!canUseProvider(entry.platform, key.id)) { note('provider-daily-cap'); continue; }
+ if (!canMakeRequest(entry.platform, entry.model_id, key.id, limits)) { note('rpm/rpd-limit'); continue; }
+ if (!canUseTokens(entry.platform, entry.model_id, key.id, estimatedTokens, limits)) { note('tpm/tpd-limit'); continue; }
let decryptedKey: string;
try {
@@ -586,13 +605,14 @@ function selectKeyForModel(entry: ChainRow, estimatedTokens: number, skipKeys?:
} catch {
db.prepare("UPDATE api_keys SET status = 'error', last_checked_at = datetime('now') WHERE id = ?")
.run(key.id);
+ note('decrypt-error');
continue;
}
const resolvedProvider = entry.platform === 'custom'
? resolveProvider('custom', key.base_url)
: provider;
- if (!resolvedProvider) continue;
+ if (!resolvedProvider) { note('no-resolved-provider'); continue; }
roundRobinIndex.set(rrKey, idx);
return {
@@ -611,6 +631,8 @@ function selectKeyForModel(entry: ChainRow, estimatedTokens: number, skipKeys?:
// No usable key for this model. Advance the round-robin index anyway so we
// don't get stuck re-trying the same exhausted key first next time.
roundRobinIndex.set(rrKey, idx);
+ const summary = Object.entries(skipTally).map(([r, n]) => `${r}:${n}`).join(', ') || 'no usable key';
+ diag?.push(`${label}: ${keys.length} key(s) — ${summary}`);
return null;
}
@@ -845,23 +867,29 @@ export function routeRequest(estimatedTokens = 1000, skipKeys?: Set<string>, pre
}
}
+ // Per-model disposition, attached to the exhaustion error when the loop falls
+ // through with no route — the only record of WHY the pool was empty on the
+ // synchronous "all exhausted" path (nothing downstream logs it). See issue _1.
+ const diag: string[] = [];
+
for (const entry of sortedChain) {
+ const label = `${entry.platform}/${entry.model_id}`;
// Models the caller has ruled out for this request — e.g. a 404
// "model removed upstream" already seen this request: trying the same
// model again on a different key would just burn another attempt on the
// same dead route (PR #111, credits @barbotkonv).
- if (skipModels?.has(entry.model_db_id)) continue;
+ if (skipModels?.has(entry.model_db_id)) { diag.push(`${label}: ruled out earlier this request`); continue; }
// Vision requests skip text-only models — including a sticky/preferred one,
// which is correct: don't pin an image turn to a model that can't see it.
- if (requireVision && !entry.supports_vision) continue;
+ if (requireVision && !entry.supports_vision) { diag.push(`${label}: no vision support`); continue; }
// Tool-bearing requests skip models that can't emit structured tool_calls.
// A model that "answers" a tool request with the call serialized as text
// looks successful at the transport level while the client's harness sees
// nothing — worse than a failover. Applies to sticky models too, same
// reasoning as vision above.
- if (requireTools && !entry.supports_tools) continue;
+ if (requireTools && !entry.supports_tools) { diag.push(`${label}: no tool-calling support`); continue; }
// Context-aware routing: skip a model whose context window can't hold the
// request, so a large prompt never selects a small-context model and burns
@@ -872,23 +900,23 @@ export function routeRequest(estimatedTokens = 1000, skipKeys?: Set<string>, pre
// failed model is put on cooldown — so this is a fast-path, not the only
// guard. If every model is too small, the loop falls through and the caller
// gets the normal "all models exhausted" error rather than a wasted sweep.
- if (entry.context_window != null && estimatedTokens > entry.context_window) continue;
+ if (entry.context_window != null && estimatedTokens > entry.context_window) { diag.push(`${label}: context ${entry.context_window} < estimated ${estimatedTokens}`); continue; }
// Same guard for a model with a small per-minute token budget: a single
// request that alone exceeds tpm_limit can never fit one minute of quota and
// returns a guaranteed 413 (e.g. Groq gpt-oss-120b: 131k context but 8k TPM).
// estimatedTokens already includes reserved output, mirroring the check above.
- if (entry.tpm_limit != null && estimatedTokens > entry.tpm_limit) continue;
+ if (entry.tpm_limit != null && estimatedTokens > entry.tpm_limit) { diag.push(`${label}: tpm_limit ${entry.tpm_limit} < estimated ${estimatedTokens}`); continue; }
// Key selection + accounting pre-checks for this one model. Returns the
// first usable key's RouteResult, or null when the model has no key that
// can serve right now — in which case we fall through to the next model in
// the sorted chain for THIS request (no explicit penalty needed).
- const route = selectKeyForModel(entry, estimatedTokens, skipKeys);
+ const route = selectKeyForModel(entry, estimatedTokens, skipKeys, diag);
if (route) return route;
}
- throw new RouteError('All models exhausted. Add more API keys or wait for rate limits to reset.', 429);
+ throw new RouteError('All models exhausted. Add more API keys or wait for rate limits to reset.', 429, diag);
}
/**