Skip to content

7.4 billion tokens per month. 34 free LLM providers. 635 free model endpoints. All behind one /v1 endpoint, plus any custom OpenAI-compatible endpoint. Smart routing, automatic failover, encrypted keys. Personal experimentation only.

TypeScript

feat(router): explain WHY the pool is empty on exhaustion (#423) (#444)

Tashfeen34948eece727
diff --git a/server/src/__tests__/services/exhaustion-summary.test.ts b/server/src/__tests__/services/exhaustion-summary.test.ts
new file mode 100644
index 00000000..525b0762
--- /dev/null
+++ b/server/src/__tests__/services/exhaustion-summary.test.ts
@@ -0,0 +1,80 @@
+import { describe, it, expect } from 'vitest';
+import { summarizeExhaustion, formatResetEta } from '../../services/router.js';
+
+// #423: "All models exhausted" gave the caller nothing to act on. The summary
+// rolls the per-model routing diagnostics into aggregate, client-safe buckets
+// plus a soonest-reset ETA.
+
+describe('formatResetEta', () => {
+ const now = 1_000_000_000_000;
+ it('formats seconds under 90s', () => {
+ expect(formatResetEta(now + 12_000, now)).toBe('~12s');
+ });
+ it('formats minutes from 90s up', () => {
+ expect(formatResetEta(now + 4 * 60_000, now)).toBe('~4m');
+ });
+ it('formats hours past 90m', () => {
+ expect(formatResetEta(now + 3 * 60 * 60_000, now)).toBe('~3h');
+ });
+ it('returns null for lapsed or missing timestamps', () => {
+ expect(formatResetEta(now - 1000, now)).toBeNull();
+ expect(formatResetEta(null, now)).toBeNull();
+ expect(formatResetEta(undefined, now)).toBeNull();
+ });
+});
+
+describe('summarizeExhaustion', () => {
+ const now = 1_000_000_000_000;
+
+ it('keeps the "All models exhausted" prefix (backwards-compatible)', () => {
+ expect(summarizeExhaustion([], null, now)).toMatch(/^All models exhausted/);
+ expect(summarizeExhaustion(undefined, null, now)).toMatch(/^All models exhausted/);
+ });
+
+ it('buckets rate-limit and cooldown reasons together', () => {
+ const diag = [
+ 'groq/llama-3.3-70b: 2 key(s) — cooldown:1, rpm/rpd-limit:1',
+ 'google/gemini-1.5-flash: 1 key(s) — tpm/tpd-limit:1',
+ ];
+ const msg = summarizeExhaustion(diag, null, now);
+ expect(msg).toContain('2 routes checked');
+ expect(msg).toContain('2 rate-limited or on cooldown');
+ });
+
+ it('distinguishes "no usable key" from rate limits', () => {
+ const diag = [
+ 'cohere/command-r: no enabled+healthy key for platform',
+ 'groq/llama: 1 key(s) — cooldown:1',
+ ];
+ const msg = summarizeExhaustion(diag, null, now);
+ expect(msg).toContain('1 no usable key configured');
+ expect(msg).toContain('1 rate-limited or on cooldown');
+ });
+
+ it('classifies prompt-too-large lines, not as rate limits', () => {
+ const diag = [
+ 'groq/gpt-oss-120b: tpm_limit 8000 < estimated 33476',
+ 'google/flash: context 32768 < estimated 40000',
+ ];
+ const msg = summarizeExhaustion(diag, null, now);
+ expect(msg).toContain('2 prompt too large for the model');
+ expect(msg).not.toContain('rate-limited');
+ });
+
+ it('handles model ids that contain a colon', () => {
+ const diag = ['custom/qwen3:4b: no vision support'];
+ const msg = summarizeExhaustion(diag, null, now);
+ expect(msg).toContain('1 model lacks vision');
+ });
+
+ it('appends the soonest-reset ETA when a cooldown is active', () => {
+ const diag = ['groq/llama: 1 key(s) — cooldown:1'];
+ const msg = summarizeExhaustion(diag, now + 3 * 60_000, now);
+ expect(msg).toContain('Soonest reset ~3m.');
+ });
+
+ it('omits the ETA when nothing is cooling down', () => {
+ const diag = ['cohere/command-r: no enabled+healthy key for platform'];
+ expect(summarizeExhaustion(diag, null, now)).not.toContain('Soonest reset');
+ });
+});
diff --git a/server/src/services/ratelimit.ts b/server/src/services/ratelimit.ts
index 1741f099..ccdfac21 100644
--- a/server/src/services/ratelimit.ts
+++ b/server/src/services/ratelimit.ts
@@ -477,6 +477,22 @@ export function isOnCooldown(platform: string, modelId: string, keyId: number):
return true;
}
+/**
+ * Soonest moment any active cooldown expires, in ms since epoch, or null when
+ * nothing is cooling down. Used to tell an exhausted caller roughly when to
+ * retry (#423) instead of the bare "wait for rate limits to reset".
+ */
+export function getSoonestCooldownExpiry(now = Date.now()): number | null {
+ return withDb(db => {
+ const row = db.prepare(`
+ SELECT MIN(expires_at_ms) AS soonest
+ FROM rate_limit_cooldowns
+ WHERE expires_at_ms > ?
+ `).get(now) as { soonest: number | null } | undefined;
+ return row?.soonest ?? null;
+ }) ?? null;
+}
+
export function getRateLimitStatus(
platform: string,
modelId: string,
diff --git a/server/src/services/router.ts b/server/src/services/router.ts
index 2565abaf..9191662e 100644
--- a/server/src/services/router.ts
+++ b/server/src/services/router.ts
@@ -1,7 +1,7 @@
import { getDb, getSetting, setSetting } from '../db/index.js';
import { getProvider, hasProvider, resolveProvider } from '../providers/index.js';
import { decrypt } from '../lib/crypto.js';
-import { canMakeRequest, canUseTokens, isOnCooldown, canUseProvider } from './ratelimit.js';
+import { canMakeRequest, canUseTokens, isOnCooldown, canUseProvider, getSoonestCooldownExpiry } from './ratelimit.js';
import {
BANDIT_PRESETS, DEFAULT_STRATEGY, type RoutingStrategy, type RoutingWeights,
reliabilityPosterior, expectedReliability, sampleBeta,
@@ -28,6 +28,66 @@ class RouteError extends Error {
}
}
+// Human-readable retry ETA from a cooldown expiry timestamp (#423). Null when
+// nothing is cooling down or it already lapsed.
+export function formatResetEta(soonestResetMs: number | null | undefined, now = Date.now()): string | null {
+ if (soonestResetMs == null) return null;
+ const deltaMs = soonestResetMs - now;
+ if (deltaMs <= 0) return null;
+ const secs = Math.round(deltaMs / 1000);
+ if (secs < 90) return `~${secs}s`;
+ const mins = Math.round(secs / 60);
+ if (mins < 90) return `~${mins}m`;
+ return `~${Math.round(mins / 60)}h`;
+}
+
+const EXHAUSTION_ADVICE = 'Add more API keys or wait for rate limits to reset.';
+
+// Roll the per-model diagnostics (see RouteError.diagnostics) up into a short,
+// client-safe summary so an exhausted caller learns WHY the pool was empty
+// instead of a bare "All models exhausted" (#423). Buckets are aggregate
+// counts only — no key material, no per-key detail. Classifies off the whole
+// line (model ids can contain ':' so splitting label from reason is unsafe).
+export function summarizeExhaustion(
+ diag: string[] | undefined,
+ soonestResetMs?: number | null,
+ now = Date.now(),
+): string {
+ const eta = formatResetEta(soonestResetMs, now);
+ const etaSuffix = eta ? ` Soonest reset ${eta}.` : '';
+ if (!diag || diag.length === 0) {
+ return `All models exhausted. ${EXHAUSTION_ADVICE}${etaSuffix}`;
+ }
+
+ const counts: Record<string, number> = {};
+ const bump = (bucket: string) => { counts[bucket] = (counts[bucket] ?? 0) + 1; };
+ for (const line of diag) {
+ const l = line.toLowerCase();
+ if (l.includes('no provider registered')) bump('unsupported provider');
+ else if (/no enabled\+healthy key|no usable key|decrypt-error/.test(l)) bump('no usable key configured');
+ else if (l.includes('< estimated')) bump('prompt too large for the model');
+ else if (l.includes('no vision support')) bump('model lacks vision');
+ else if (l.includes('no tool-calling support')) bump('model lacks tool-calling');
+ else if (/ruled out|already-failed/.test(l)) bump('failed earlier this request');
+ else if (/cooldown|rpm|rpd|tpm|tpd|provider-daily-cap/.test(l)) bump('rate-limited or on cooldown');
+ else bump('unavailable');
+ }
+ // Most actionable buckets first.
+ const order = [
+ 'rate-limited or on cooldown',
+ 'no usable key configured',
+ 'prompt too large for the model',
+ 'model lacks vision',
+ 'model lacks tool-calling',
+ 'failed earlier this request',
+ 'unsupported provider',
+ 'unavailable',
+ ];
+ const parts = order.filter(b => counts[b]).map(b => `${counts[b]} ${b}`);
+ const total = diag.length;
+ return `All models exhausted: ${total} route${total === 1 ? '' : 's'} checked (${parts.join(', ')}). ${EXHAUSTION_ADVICE}${etaSuffix}`;
+}
+
interface KeyRow {
id: number;
platform: string;
@@ -916,7 +976,7 @@ export function routeRequest(estimatedTokens = 1000, skipKeys?: Set<string>, pre
if (route) return route;
}
- throw new RouteError('All models exhausted. Add more API keys or wait for rate limits to reset.', 429, diag);
+ throw new RouteError(summarizeExhaustion(diag, getSoonestCooldownExpiry()), 429, diag);
}
/**