7.4 billion tokens per month. 34 free LLM providers. 635 free model endpoints. All behind one /v1 endpoint, plus any custom OpenAI-compatible endpoint. Smart routing, automatic failover, encrypted keys. Personal experimentation only.
TypeScript
feat(router): explain WHY the pool is empty on exhaustion (#423) (#444)
Tashfeen34948eece727
diff --git a/server/src/__tests__/services/exhaustion-summary.test.ts b/server/src/__tests__/services/exhaustion-summary.test.tsnew file mode 100644index 00000000..525b0762--- /dev/null+++ b/server/src/__tests__/services/exhaustion-summary.test.ts@@ -0,0 +1,80 @@+import { describe, it, expect } from 'vitest';+import { summarizeExhaustion, formatResetEta } from '../../services/router.js';++// #423: "All models exhausted" gave the caller nothing to act on. The summary+// rolls the per-model routing diagnostics into aggregate, client-safe buckets+// plus a soonest-reset ETA.++describe('formatResetEta', () => {+ const now = 1_000_000_000_000;+ it('formats seconds under 90s', () => {+ expect(formatResetEta(now + 12_000, now)).toBe('~12s');+ });+ it('formats minutes from 90s up', () => {+ expect(formatResetEta(now + 4 * 60_000, now)).toBe('~4m');+ });+ it('formats hours past 90m', () => {+ expect(formatResetEta(now + 3 * 60 * 60_000, now)).toBe('~3h');+ });+ it('returns null for lapsed or missing timestamps', () => {+ expect(formatResetEta(now - 1000, now)).toBeNull();+ expect(formatResetEta(null, now)).toBeNull();+ expect(formatResetEta(undefined, now)).toBeNull();+ });+});++describe('summarizeExhaustion', () => {+ const now = 1_000_000_000_000;++ it('keeps the "All models exhausted" prefix (backwards-compatible)', () => {+ expect(summarizeExhaustion([], null, now)).toMatch(/^All models exhausted/);+ expect(summarizeExhaustion(undefined, null, now)).toMatch(/^All models exhausted/);+ });++ it('buckets rate-limit and cooldown reasons together', () => {+ const diag = [+ 'groq/llama-3.3-70b: 2 key(s) — cooldown:1, rpm/rpd-limit:1',+ 'google/gemini-1.5-flash: 1 key(s) — tpm/tpd-limit:1',+ ];+ const msg = summarizeExhaustion(diag, null, now);+ expect(msg).toContain('2 routes checked');+ expect(msg).toContain('2 rate-limited or on cooldown');+ });++ it('distinguishes "no usable key" from rate limits', () => {+ const diag = [+ 'cohere/command-r: no enabled+healthy key for platform',+ 'groq/llama: 1 key(s) — cooldown:1',+ ];+ const msg = summarizeExhaustion(diag, null, now);+ expect(msg).toContain('1 no usable key configured');+ expect(msg).toContain('1 rate-limited or on cooldown');+ });++ it('classifies prompt-too-large lines, not as rate limits', () => {+ const diag = [+ 'groq/gpt-oss-120b: tpm_limit 8000 < estimated 33476',+ 'google/flash: context 32768 < estimated 40000',+ ];+ const msg = summarizeExhaustion(diag, null, now);+ expect(msg).toContain('2 prompt too large for the model');+ expect(msg).not.toContain('rate-limited');+ });++ it('handles model ids that contain a colon', () => {+ const diag = ['custom/qwen3:4b: no vision support'];+ const msg = summarizeExhaustion(diag, null, now);+ expect(msg).toContain('1 model lacks vision');+ });++ it('appends the soonest-reset ETA when a cooldown is active', () => {+ const diag = ['groq/llama: 1 key(s) — cooldown:1'];+ const msg = summarizeExhaustion(diag, now + 3 * 60_000, now);+ expect(msg).toContain('Soonest reset ~3m.');+ });++ it('omits the ETA when nothing is cooling down', () => {+ const diag = ['cohere/command-r: no enabled+healthy key for platform'];+ expect(summarizeExhaustion(diag, null, now)).not.toContain('Soonest reset');+ });+});diff --git a/server/src/services/ratelimit.ts b/server/src/services/ratelimit.tsindex 1741f099..ccdfac21 100644--- a/server/src/services/ratelimit.ts+++ b/server/src/services/ratelimit.ts@@ -477,6 +477,22 @@ export function isOnCooldown(platform: string, modelId: string, keyId: number):return true;}+/**+ * Soonest moment any active cooldown expires, in ms since epoch, or null when+ * nothing is cooling down. Used to tell an exhausted caller roughly when to+ * retry (#423) instead of the bare "wait for rate limits to reset".+ */+export function getSoonestCooldownExpiry(now = Date.now()): number | null {+ return withDb(db => {+ const row = db.prepare(`+ SELECT MIN(expires_at_ms) AS soonest+ FROM rate_limit_cooldowns+ WHERE expires_at_ms > ?+ `).get(now) as { soonest: number | null } | undefined;+ return row?.soonest ?? null;+ }) ?? null;+}+export function getRateLimitStatus(platform: string,modelId: string,diff --git a/server/src/services/router.ts b/server/src/services/router.tsindex 2565abaf..9191662e 100644--- a/server/src/services/router.ts+++ b/server/src/services/router.ts@@ -1,7 +1,7 @@import { getDb, getSetting, setSetting } from '../db/index.js';import { getProvider, hasProvider, resolveProvider } from '../providers/index.js';import { decrypt } from '../lib/crypto.js';-import { canMakeRequest, canUseTokens, isOnCooldown, canUseProvider } from './ratelimit.js';+import { canMakeRequest, canUseTokens, isOnCooldown, canUseProvider, getSoonestCooldownExpiry } from './ratelimit.js';import {BANDIT_PRESETS, DEFAULT_STRATEGY, type RoutingStrategy, type RoutingWeights,reliabilityPosterior, expectedReliability, sampleBeta,@@ -28,6 +28,66 @@ class RouteError extends Error {}}+// Human-readable retry ETA from a cooldown expiry timestamp (#423). Null when+// nothing is cooling down or it already lapsed.+export function formatResetEta(soonestResetMs: number | null | undefined, now = Date.now()): string | null {+ if (soonestResetMs == null) return null;+ const deltaMs = soonestResetMs - now;+ if (deltaMs <= 0) return null;+ const secs = Math.round(deltaMs / 1000);+ if (secs < 90) return `~${secs}s`;+ const mins = Math.round(secs / 60);+ if (mins < 90) return `~${mins}m`;+ return `~${Math.round(mins / 60)}h`;+}++const EXHAUSTION_ADVICE = 'Add more API keys or wait for rate limits to reset.';++// Roll the per-model diagnostics (see RouteError.diagnostics) up into a short,+// client-safe summary so an exhausted caller learns WHY the pool was empty+// instead of a bare "All models exhausted" (#423). Buckets are aggregate+// counts only — no key material, no per-key detail. Classifies off the whole+// line (model ids can contain ':' so splitting label from reason is unsafe).+export function summarizeExhaustion(+ diag: string[] | undefined,+ soonestResetMs?: number | null,+ now = Date.now(),+): string {+ const eta = formatResetEta(soonestResetMs, now);+ const etaSuffix = eta ? ` Soonest reset ${eta}.` : '';+ if (!diag || diag.length === 0) {+ return `All models exhausted. ${EXHAUSTION_ADVICE}${etaSuffix}`;+ }++ const counts: Record<string, number> = {};+ const bump = (bucket: string) => { counts[bucket] = (counts[bucket] ?? 0) + 1; };+ for (const line of diag) {+ const l = line.toLowerCase();+ if (l.includes('no provider registered')) bump('unsupported provider');+ else if (/no enabled\+healthy key|no usable key|decrypt-error/.test(l)) bump('no usable key configured');+ else if (l.includes('< estimated')) bump('prompt too large for the model');+ else if (l.includes('no vision support')) bump('model lacks vision');+ else if (l.includes('no tool-calling support')) bump('model lacks tool-calling');+ else if (/ruled out|already-failed/.test(l)) bump('failed earlier this request');+ else if (/cooldown|rpm|rpd|tpm|tpd|provider-daily-cap/.test(l)) bump('rate-limited or on cooldown');+ else bump('unavailable');+ }+ // Most actionable buckets first.+ const order = [+ 'rate-limited or on cooldown',+ 'no usable key configured',+ 'prompt too large for the model',+ 'model lacks vision',+ 'model lacks tool-calling',+ 'failed earlier this request',+ 'unsupported provider',+ 'unavailable',+ ];+ const parts = order.filter(b => counts[b]).map(b => `${counts[b]} ${b}`);+ const total = diag.length;+ return `All models exhausted: ${total} route${total === 1 ? '' : 's'} checked (${parts.join(', ')}). ${EXHAUSTION_ADVICE}${etaSuffix}`;+}+interface KeyRow {id: number;platform: string;@@ -916,7 +976,7 @@ export function routeRequest(estimatedTokens = 1000, skipKeys?: Set<string>, preif (route) return route;}- throw new RouteError('All models exhausted. Add more API keys or wait for rate limits to reset.', 429, diag);+ throw new RouteError(summarizeExhaustion(diag, getSoonestCooldownExpiry()), 429, diag);}/**