7.4 billion tokens per month. 34 free LLM providers. 635 free model endpoints. All behind one /v1 endpoint, plus any custom OpenAI-compatible endpoint. Smart routing, automatic failover, encrypted keys. Personal experimentation only.
TypeScript
fix(routing): multi-key quota fixes for #470, #454, #456, #453 (#479)
Tashfeena2d2a54508e7
diff --git a/server/src/__tests__/routes/fallback-keycount-parity.test.ts b/server/src/__tests__/routes/fallback-keycount-parity.test.tsnew file mode 100644index 00000000..047e7f4c--- /dev/null+++ b/server/src/__tests__/routes/fallback-keycount-parity.test.ts@@ -0,0 +1,70 @@+import { describe, it, expect, beforeAll } from 'vitest';+import type { Express } from 'express';+import { createApp } from '../../app.js';+import { getDb, initDb } from '../../db/index.js';+import { encrypt } from '../../lib/crypto.js';+import { mintDashboardToken } from '../helpers/auth.js';++let dashToken = '';++async function get(app: Express, path: string) {+ const server = app.listen(0);+ const addr = server.address() as any;+ const res = await fetch(`http://127.0.0.1:${addr.port}${path}`, {+ headers: { Authorization: `Bearer ${dashToken}` },+ });+ const body = await res.json().catch(() => null);+ server.close();+ return { status: res.status, body };+}++// PR #459 scaled /token-usage budgets by healthy+enabled keys, but the pre-existing+// GET / handler counted enabled=1 only — so the two dashboards disagreed on pooled+// capacity. Both must now use the SAME filter (enabled AND status healthy/unknown),+// matching the routing scorer (#456).+describe('fallback key-count filter parity (#456)', () => {+ let app: Express;+ let modelId: string;++ beforeAll(() => {+ process.env.ENCRYPTION_KEY = '0'.repeat(64);+ initDb(':memory:');+ app = createApp();+ dashToken = mintDashboardToken();++ const db = getDb();+ // One usable (healthy) groq key and one that must NOT add capacity (invalid).+ const good = encrypt('good'); const bad = encrypt('bad');+ db.prepare("INSERT INTO api_keys (platform, label, encrypted_key, iv, auth_tag, status, enabled) VALUES ('groq','good',?,?,?,'healthy',1)").run(good.encrypted, good.iv, good.authTag);+ db.prepare("INSERT INTO api_keys (platform, label, encrypted_key, iv, auth_tag, status, enabled) VALUES ('groq','bad',?,?,?,'invalid',1)").run(bad.encrypted, bad.iv, bad.authTag);++ const m = db.prepare("SELECT model_id FROM models WHERE platform = 'groq' ORDER BY id LIMIT 1").get() as { model_id: string };+ modelId = m.model_id;+ // Non-zero parseable budget so the key multiplier is observable.+ db.prepare("UPDATE models SET monthly_token_budget = '~1M' WHERE platform = 'groq' AND model_id = ?").run(modelId);+ });++ it('GET / counts only enabled healthy/unknown keys (invalid excluded)', async () => {+ const { status, body } = await get(app, '/api/fallback');+ expect(status).toBe(200);+ const row = body.find((r: any) => r.platform === 'groq' && r.modelId === modelId);+ expect(row).toBeDefined();+ // Two enabled groq keys exist, but one is invalid → usable count is 1.+ expect(row.keyCount).toBe(1);+ });++ it('GET / and GET /token-usage agree on the pooled budget for the same model', async () => {+ const listing = await get(app, '/api/fallback');+ const usage = await get(app, '/api/fallback/token-usage');++ const listRow = listing.body.find((r: any) => r.platform === 'groq' && r.modelId === modelId);+ const usageRow = usage.body.models.find((r: any) => r.platform === 'groq' && r.modelId === modelId);++ expect(listRow).toBeDefined();+ expect(usageRow).toBeDefined();+ // Both scale parseBudget('~1M') by the SAME usable key count (1), so the+ // pooled budgets match. Before the fix, GET / used the enabled=2 count.+ expect(listRow.monthlyTokenBudgetTokens).toBe(usageRow.budget);+ expect(usageRow.budget).toBe(1_000_000);+ });+});diff --git a/server/src/__tests__/routes/proxy-max-tokens-routing.test.ts b/server/src/__tests__/routes/proxy-max-tokens-routing.test.tsnew file mode 100644index 00000000..db599868--- /dev/null+++ b/server/src/__tests__/routes/proxy-max-tokens-routing.test.ts@@ -0,0 +1,118 @@+import { describe, it, expect, beforeAll, beforeEach, vi } from 'vitest';+import type { Express } from 'express';++// Mock the provider so routing is exercised without real upstream calls.+const chatCompletion = vi.fn();+const streamChatCompletion = vi.fn();+const fakeProvider = { name: 'fake', chatCompletion, streamChatCompletion } as any;++vi.mock('../../providers/index.js', async (importOriginal) => {+ const actual = await importOriginal() as any;+ return {+ ...actual,+ getProvider: () => fakeProvider,+ resolveProvider: () => fakeProvider,+ };+});++const { createApp } = await import('../../app.js');+const { initDb, getDb, getUnifiedApiKey } = await import('../../db/index.js');+const { encrypt } = await import('../../lib/crypto.js');+const { setRoutingStrategy, routingReserveTokens, OUTPUT_RESERVE_CAP } = await import('../../services/router.js');++async function post(app: Express, body: any, key: string) {+ const server = app.listen(0);+ const addr = server.address() as any;+ const res = await fetch(`http://127.0.0.1:${addr.port}/v1/chat/completions`, {+ method: 'POST',+ headers: { 'Content-Type': 'application/json', Authorization: `Bearer ${key}` },+ body: JSON.stringify(body),+ });+ const raw = await res.text();+ server.close();+ let json: any = null;+ try { json = JSON.parse(raw); } catch {}+ return { status: res.status, body: json };+}++const GOOD_RESULT = {+ choices: [{ message: { role: 'assistant', content: 'ok' } }],+ usage: { prompt_tokens: 5, completion_tokens: 2, total_tokens: 7 },+};++describe('routingReserveTokens (#470)', () => {+ it('caps the reserved output at OUTPUT_RESERVE_CAP', () => {+ expect(routingReserveTokens(32000)).toBe(OUTPUT_RESERVE_CAP);+ expect(routingReserveTokens(50)).toBe(50);+ // Omitted / non-positive max_tokens keeps the historical 1000 default (capped).+ expect(routingReserveTokens(undefined)).toBe(Math.min(1000, OUTPUT_RESERVE_CAP));+ expect(routingReserveTokens(0)).toBe(Math.min(1000, OUTPUT_RESERVE_CAP));+ expect(routingReserveTokens(-1)).toBe(Math.min(1000, OUTPUT_RESERVE_CAP));+ });+});++describe('max_tokens no longer starves routing (#470)', () => {+ let app: Express;+ let key: string;++ beforeAll(() => {+ process.env.ENCRYPTION_KEY = '0'.repeat(64);+ initDb(':memory:');+ app = createApp();+ key = getUnifiedApiKey();++ const db = getDb();+ setRoutingStrategy('priority');+ db.prepare("DELETE FROM settings WHERE key = 'active_profile_id'").run();++ const { encrypted, iv, authTag } = encrypt('groq-key');+ db.prepare(`+ INSERT INTO api_keys (platform, label, encrypted_key, iv, auth_tag, status, enabled)+ VALUES ('groq', 'test', ?, ?, ?, 'healthy', 1)+ `).run(encrypted, iv, authTag);++ // Only groq is keyed, so make groq the ONLY routable platform and give every+ // groq model a small per-minute TOKEN budget (6k) with a large context+ // window, so the sole thing that can exclude a groq model is the token+ // estimate: a huge client max_tokens must NOT exceed 6k once the reserved+ // output is capped, while a huge INPUT still must.+ db.prepare("UPDATE models SET enabled = 0 WHERE platform != 'groq'").run();+ db.prepare("UPDATE models SET tpm_limit = 6000, tpd_limit = NULL, context_window = 200000 WHERE platform = 'groq'").run();+ });++ beforeEach(() => {+ chatCompletion.mockReset();+ streamChatCompletion.mockReset();+ getDb().prepare('DELETE FROM rate_limit_cooldowns').run();+ getDb().prepare('DELETE FROM rate_limit_usage').run();+ });++ it('routes a tiny prompt with a huge max_tokens (was a false 429 with zero upstream calls)', async () => {+ chatCompletion.mockResolvedValueOnce(GOOD_RESULT);++ const { status, body } = await post(app, {+ messages: [{ role: 'user', content: 'hi' }],+ max_tokens: 32000, // >> the 6k TPM; capped reserve keeps it routable.+ }, key);++ expect(status).toBe(200);+ expect(body.choices[0].message.content).toBe('ok');+ // The model was actually dispatched — not synchronously excluded.+ expect(chatCompletion).toHaveBeenCalledTimes(1);+ });++ it('still excludes the model when the INPUT itself exceeds the TPM budget', async () => {+ chatCompletion.mockResolvedValueOnce(GOOD_RESULT);++ // ~8k input tokens (32k chars / 4) > the 6k TPM budget → the only enabled+ // model is filtered out before any upstream call.+ const bigPrompt = 'x'.repeat(32_000);+ const { status } = await post(app, {+ messages: [{ role: 'user', content: bigPrompt }],+ max_tokens: 50,+ }, key);++ expect(status).toBe(429);+ expect(chatCompletion).not.toHaveBeenCalled();+ });+});diff --git a/server/src/__tests__/routes/proxy-multikey-penalty.test.ts b/server/src/__tests__/routes/proxy-multikey-penalty.test.tsnew file mode 100644index 00000000..e5b48589--- /dev/null+++ b/server/src/__tests__/routes/proxy-multikey-penalty.test.ts@@ -0,0 +1,148 @@+import { describe, it, expect, beforeAll, beforeEach, vi } from 'vitest';+import type { Express } from 'express';++// Mock the provider so we can script 429s without real upstream calls.+const chatCompletion = vi.fn();+const streamChatCompletion = vi.fn();+const fakeProvider = { name: 'fake', chatCompletion, streamChatCompletion } as any;++vi.mock('../../providers/index.js', async (importOriginal) => {+ const actual = await importOriginal() as any;+ return {+ ...actual,+ getProvider: () => fakeProvider,+ resolveProvider: () => fakeProvider,+ };+});++const { createApp } = await import('../../app.js');+const { initDb, getDb, getUnifiedApiKey } = await import('../../db/index.js');+const { encrypt } = await import('../../lib/crypto.js');+const { setRoutingStrategy, hasOtherUsableKey, getAllPenalties } = await import('../../services/router.js');+const { setCooldown } = await import('../../services/ratelimit.js');++async function post(app: Express, body: any, key: string) {+ const server = app.listen(0);+ const addr = server.address() as any;+ const res = await fetch(`http://127.0.0.1:${addr.port}/v1/chat/completions`, {+ method: 'POST',+ headers: { 'Content-Type': 'application/json', Authorization: `Bearer ${key}` },+ body: JSON.stringify(body),+ });+ const raw = await res.text();+ server.close();+ let json: any = null;+ try { json = JSON.parse(raw); } catch {}+ return { status: res.status, body: json };+}++const GOOD_RESULT = {+ choices: [{ message: { role: 'assistant', content: 'ok' } }],+ usage: { prompt_tokens: 5, completion_tokens: 2, total_tokens: 7 },+};+const err429 = () => Object.assign(new Error('Groq API error 429: rate limit'), { status: 429 });++interface Setup { modelDbId: number; modelId: string; keyA: number; keyB: number }++// Each test uses a DISTINCT groq model so the module-global in-memory rate-limit+// state (cooldowns, round-robin, penalties — keyed by model) can't leak between+// tests, mirroring how ratelimit.test.ts uses unique identifiers per case.+function setup(rank: number, onlyModel = false): Setup {+ const db = getDb();+ setRoutingStrategy('priority');+ db.prepare("DELETE FROM settings WHERE key = 'active_profile_id'").run();+ db.prepare('DELETE FROM api_keys').run();+ db.prepare('DELETE FROM rate_limit_cooldowns').run();+ db.prepare('DELETE FROM rate_limit_usage').run();++ const models = db.prepare("SELECT id, model_id FROM models WHERE platform = 'groq' ORDER BY id").all() as { id: number; model_id: string }[];+ const model = models[rank];+ if (onlyModel) db.prepare('UPDATE models SET enabled = 0').run();+ db.prepare('UPDATE models SET enabled = 1, rpm_limit = NULL, rpd_limit = NULL, tpm_limit = NULL, tpd_limit = NULL WHERE id = ?').run(model.id);++ const ids: number[] = [];+ for (const label of ['a', 'b']) {+ const { encrypted, iv, authTag } = encrypt(`groq-${rank}-${label}`);+ const info = db.prepare(`+ INSERT INTO api_keys (platform, label, encrypted_key, iv, auth_tag, status, enabled)+ VALUES ('groq', ?, ?, ?, ?, 'healthy', 1)+ `).run(`${rank}-${label}`, encrypted, iv, authTag);+ ids.push(Number(info.lastInsertRowid));+ }+ return { modelDbId: model.id, modelId: model.model_id, keyA: ids[0], keyB: ids[1] };+}++describe('hasOtherUsableKey (#454)', () => {+ beforeAll(() => {+ process.env.ENCRYPTION_KEY = '0'.repeat(64);+ initDb(':memory:');+ });++ it('returns true when a sibling key is free', () => {+ const s = setup(0);+ expect(hasOtherUsableKey(s.modelDbId, s.keyA)).toBe(true);+ });++ it('returns false when the only sibling is on cooldown', () => {+ const s = setup(1);+ setCooldown('groq', s.modelId, s.keyB, 5 * 60 * 1000);+ expect(hasOtherUsableKey(s.modelDbId, s.keyA)).toBe(false);+ });++ it('returns false when the only sibling is already ruled out this request (skipKeys)', () => {+ const s = setup(2);+ const skip = new Set<string>([`groq:${s.modelId}:${s.keyB}`]);+ expect(hasOtherUsableKey(s.modelDbId, s.keyA, skip)).toBe(false);+ });++ it('is symmetric: excluding one key finds the other when it is free, none when it is benched', () => {+ const s = setup(3);+ expect(hasOtherUsableKey(s.modelDbId, s.keyB)).toBe(true); // keyA free+ setCooldown('groq', s.modelId, s.keyA, 5 * 60 * 1000);+ expect(hasOtherUsableKey(s.modelDbId, s.keyB)).toBe(false); // keyA benched, keyB excluded+ });+});++describe('one key\'s 429 must not demote the whole model (#454)', () => {+ let app: Express;+ let key: string;++ beforeAll(() => {+ process.env.ENCRYPTION_KEY = '0'.repeat(64);+ initDb(':memory:');+ app = createApp();+ key = getUnifiedApiKey();+ });++ beforeEach(() => {+ chatCompletion.mockReset();+ streamChatCompletion.mockReset();+ });++ it('does NOT record a model-level penalty when the first key 429s but a sibling succeeds', async () => {+ const s = setup(4, true);+ chatCompletion+ .mockRejectedValueOnce(err429()) // key A+ .mockResolvedValueOnce(GOOD_RESULT); // key B++ const { status } = await post(app, { messages: [{ role: 'user', content: 'hi' }] }, key);++ expect(status).toBe(200);+ expect(chatCompletion).toHaveBeenCalledTimes(2);+ // The model must NOT be penalized — only one key hit the limit.+ expect(getAllPenalties().some(p => p.modelDbId === s.modelDbId)).toBe(false);+ });++ it('DOES record the model-level penalty once the last usable key 429s', async () => {+ const s = setup(5, true);+ chatCompletion.mockRejectedValue(err429()); // both keys 429++ const { status } = await post(app, { messages: [{ role: 'user', content: 'hi' }] }, key);++ // Chain exhausts after both keys fail.+ expect(status).toBe(429);+ expect(chatCompletion).toHaveBeenCalledTimes(2);+ // The model IS penalized: the 429 exhausted it, not just one key.+ expect(getAllPenalties().some(p => p.modelDbId === s.modelDbId)).toBe(true);+ });+});diff --git a/server/src/__tests__/services/provider-quota.test.ts b/server/src/__tests__/services/provider-quota.test.tsnew file mode 100644index 00000000..b11a69dd--- /dev/null+++ b/server/src/__tests__/services/provider-quota.test.ts@@ -0,0 +1,161 @@+import { describe, it, expect, beforeAll, beforeEach } from 'vitest';+import { initDb, getDb } from '../../db/index.js';+import {+ recordQuotaObservation,+ getQuotaStateForKeys,+ parseQuotaObservationsFromResponse,+ inferQuotaPoolKey,+} from '../../services/provider-quota.js';++function insertState(row: {+ platform: string;+ keyId: number;+ pool: string;+ metric: string;+ limit: number | null;+ remaining: number | null;+ resetAt: string | null;+}) {+ getDb().prepare(`+ INSERT INTO provider_quota_state+ (platform, key_id, quota_pool_key, metric, limit_value, remaining_value, reset_at)+ VALUES (?, ?, ?, ?, ?, ?, ?)+ `).run(row.platform, row.keyId, row.pool, row.metric, row.limit, row.remaining, row.resetAt);+}++function readState(platform: string, keyId: number, pool: string, metric: string) {+ return getDb().prepare(`+ SELECT limit_value AS lim, remaining_value AS remaining, reset_at AS resetAt+ FROM provider_quota_state+ WHERE platform = ? AND key_id = ? AND quota_pool_key = ? AND metric = ?+ `).get(platform, keyId, pool, metric) as { lim: number | null; remaining: number | null; resetAt: string | null } | undefined;+}++describe('provider-quota: pool inference', () => {+ beforeAll(() => {+ process.env.ENCRYPTION_KEY = '0'.repeat(64);+ initDb(':memory:');+ });++ it('buckets shared-pool providers per account and openrouter free vs account', () => {+ expect(inferQuotaPoolKey('groq')).toBe('groq::account');+ expect(inferQuotaPoolKey('openrouter', 'meta-llama/llama-3.1-8b-instruct:free')).toBe('openrouter::free');+ expect(inferQuotaPoolKey('openrouter', 'openai/gpt-4o')).toBe('openrouter::account');+ // Unknown platform falls back to platform::model or platform::account.+ expect(inferQuotaPoolKey('acme' as any, 'x')).toBe('acme::x');+ expect(inferQuotaPoolKey('acme' as any)).toBe('acme::account');+ });+});++describe('provider-quota: record + read round-trip', () => {+ beforeAll(() => {+ process.env.ENCRYPTION_KEY = '0'.repeat(64);+ initDb(':memory:');+ });++ beforeEach(() => {+ getDb().prepare('DELETE FROM provider_quota_state').run();+ getDb().prepare('DELETE FROM provider_quota_observations').run();+ });++ it('records an observation and surfaces it via getQuotaStateForKeys', () => {+ const rec = recordQuotaObservation({+ platform: 'groq',+ keyId: 7,+ quotaPoolKey: 'groq::account',+ metric: 'requests',+ limit: 1000,+ remaining: 950,+ source: 'header',+ });+ expect(rec).not.toBeNull();++ const states = getQuotaStateForKeys();+ const row = states.find(s => s.platform === 'groq' && s.keyId === 7 && s.metric === 'requests');+ expect(row).toBeDefined();+ expect(row!.limit).toBe(1000);+ expect(row!.remaining).toBe(950);+ });+});++describe('provider-quota: parse from response headers (shared parseRetryAfterMs)', () => {+ beforeAll(() => {+ process.env.ENCRYPTION_KEY = '0'.repeat(64);+ initDb(':memory:');+ });++ it('parses Groq ratelimit headers into a requests observation', () => {+ const resp = new Response(null, {+ status: 200,+ headers: {+ 'x-ratelimit-limit-requests': '100',+ 'x-ratelimit-remaining-requests': '90',+ 'x-ratelimit-reset-requests': '60',+ },+ });+ const obs = parseQuotaObservationsFromResponse(resp, { platform: 'groq', keyId: 1 });+ const requests = obs.find(o => o.metric === 'requests');+ expect(requests).toBeDefined();+ expect(requests!.limit).toBe(100);+ expect(requests!.remaining).toBe(90);+ });++ it('reads Retry-After on a 429 via the shared parser (dedup of base.ts)', () => {+ const resp = new Response(null, { status: 429, headers: { 'retry-after': '30' } });+ const obs = parseQuotaObservationsFromResponse(resp, { platform: 'groq', keyId: 1 });+ // The shared parseRetryAfterMs turns "30" seconds into 30000 ms.+ expect(obs.some(o => o.retryAfterMs === 30_000)).toBe(true);+ // A 429 always marks the pool as remaining 0.+ expect(obs.some(o => o.remaining === 0)).toBe(true);+ });+});++describe('provider-quota: reset_at replenishment on read (#453)', () => {+ beforeAll(() => {+ process.env.ENCRYPTION_KEY = '0'.repeat(64);+ initDb(':memory:');+ });++ beforeEach(() => {+ getDb().prepare('DELETE FROM provider_quota_state').run();+ getDb().prepare('DELETE FROM provider_quota_observations').run();+ });++ const past = () => new Date(Date.now() - 60 * 60 * 1000).toISOString();+ const future = () => new Date(Date.now() + 60 * 60 * 1000).toISOString();++ it('restores remaining to the known limit once reset_at has passed, and persists it', () => {+ insertState({ platform: 'groq', keyId: 1, pool: 'groq::account', metric: 'requests', limit: 100, remaining: 0, resetAt: past() });++ const states = getQuotaStateForKeys();+ const row = states.find(s => s.platform === 'groq' && s.keyId === 1);+ expect(row!.remaining).toBe(100); // replenished to the limit+ expect(row!.resetAt).toBeNull(); // stale reset dropped++ // Persisted so it does not recur (the exhausted 0 is gone from the table).+ const persisted = readState('groq', 1, 'groq::account', 'requests');+ expect(persisted!.remaining).toBe(100);+ expect(persisted!.resetAt).toBeNull();+ });++ it('clears remaining to unknown when the limit is unknown and reset_at passed', () => {+ insertState({ platform: 'ollama', keyId: 2, pool: 'ollama::cloud', metric: 'requests', limit: null, remaining: 0, resetAt: past() });++ const states = getQuotaStateForKeys();+ const row = states.find(s => s.platform === 'ollama' && s.keyId === 2);+ expect(row!.remaining).toBeNull(); // no known limit → clear the 0+ expect(row!.resetAt).toBeNull();++ const persisted = readState('ollama', 2, 'ollama::cloud', 'requests');+ expect(persisted!.remaining).toBeNull();+ });++ it('leaves a still-active window (reset_at in the future) untouched', () => {+ insertState({ platform: 'groq', keyId: 3, pool: 'groq::account', metric: 'requests', limit: 100, remaining: 0, resetAt: future() });++ const states = getQuotaStateForKeys();+ const row = states.find(s => s.platform === 'groq' && s.keyId === 3);+ expect(row!.remaining).toBe(0); // still exhausted until it resets+ expect(row!.resetAt).not.toBeNull();+ });+});diff --git a/server/src/__tests__/services/router-keycount-budget.test.ts b/server/src/__tests__/services/router-keycount-budget.test.tsnew file mode 100644index 00000000..6d5d598e--- /dev/null+++ b/server/src/__tests__/services/router-keycount-budget.test.ts@@ -0,0 +1,74 @@+import { describe, it, expect, beforeAll, beforeEach } from 'vitest';+import { initDb, getDb } from '../../db/index.js';+import { encrypt } from '../../lib/crypto.js';+import { getRoutingScores, refreshStatsCache } from '../../services/router.js';++// The monthly free-tier budget is PER KEY, and monthly usage is pooled across+// all keys — so the headroom guardrail must scale the budget by the usable key+// count or a multi-key model gets damped to the floor after one account's worth+// of tokens (#456).+describe('routing headroom scales the monthly budget by usable key count (#456)', () => {+ let modelDbId: number;+ let modelId: string;++ beforeAll(() => {+ process.env.ENCRYPTION_KEY = '0'.repeat(64);+ initDb(':memory:');+ const db = getDb();+ const m = db.prepare("SELECT id, model_id FROM models WHERE platform = 'groq' ORDER BY id LIMIT 1").get() as { id: number; model_id: string };+ modelDbId = m.id;+ modelId = m.model_id;+ // A known, parseable per-key budget so headroom math is deterministic.+ db.prepare("UPDATE models SET monthly_token_budget = '~1M' WHERE id = ?").run(modelDbId);+ // ~950k tokens used this month (pooled across keys): 95% of a single key's 1M.+ db.prepare(`+ INSERT INTO requests (platform, model_id, key_id, status, input_tokens, output_tokens, latency_ms, error, request_type, created_at)+ VALUES (?, ?, NULL, 'success', 500000, 450000, 10, NULL, 'chat', datetime('now'))+ `).run('groq', modelId);+ });++ beforeEach(() => {+ getDb().prepare('DELETE FROM api_keys').run();+ });++ function headroomFor(): number {+ refreshStatsCache(getDb(), true); // force so the inserted usage is picked up+ const { scores } = getRoutingScores();+ const row = scores.find(s => s.modelDbId === modelDbId);+ if (!row) throw new Error('model missing from routing scores');+ return row.headroom;+ }++ function addGroqKeys(n: number) {+ const db = getDb();+ for (let i = 0; i < n; i++) {+ const { encrypted, iv, authTag } = encrypt(`groq-${i}`);+ db.prepare(`+ INSERT INTO api_keys (platform, label, encrypted_key, iv, auth_tag, status, enabled)+ VALUES ('groq', ?, ?, ?, ?, 'healthy', 1)+ `).run(`k${i}`, encrypted, iv, authTag);+ }+ }++ it('damps a nearly-spent single-key model below full headroom', () => {+ addGroqKeys(1);+ expect(headroomFor()).toBeLessThan(1); // 950k / 1M → <20% left → protecting+ });++ it('restores full headroom once three usable keys pool 3x the budget', () => {+ addGroqKeys(3);+ // 950k / 3M → ~68% left → the guardrail stops protecting.+ expect(headroomFor()).toBe(1);+ });++ it('ignores disabled and unhealthy keys when scaling (same filter as the endpoints)', () => {+ addGroqKeys(1); // one healthy key+ const db = getDb();+ // An invalid and a disabled key add NO pooled capacity.+ const bad = encrypt('bad'); const off = encrypt('off');+ db.prepare("INSERT INTO api_keys (platform, label, encrypted_key, iv, auth_tag, status, enabled) VALUES ('groq','bad',?,?,?,'invalid',1)").run(bad.encrypted, bad.iv, bad.authTag);+ db.prepare("INSERT INTO api_keys (platform, label, encrypted_key, iv, auth_tag, status, enabled) VALUES ('groq','off',?,?,?,'healthy',0)").run(off.encrypted, off.iv, off.authTag);+ // Still effectively single-key → still damped.+ expect(headroomFor()).toBeLessThan(1);+ });+});diff --git a/server/src/routes/anthropic.ts b/server/src/routes/anthropic.tsindex bf39df55..aaf64503 100644--- a/server/src/routes/anthropic.ts+++ b/server/src/routes/anthropic.ts@@ -10,7 +10,7 @@ import type {ChatContent,ChatContentBlock,} from '@freellmapi/shared/types.js';-import { routeRequest, recordRateLimitHit, recordSuccess, type RouteResult } from '../services/router.js';+import { routeRequest, recordRateLimitHit, recordSuccess, hasOtherUsableKey, routingReserveTokens, type RouteResult } from '../services/router.js';import {recordRequest, recordTokens, setCooldown, getCooldownDurationForLimit,PAYMENT_REQUIRED_COOLDOWN_MS, MODEL_FORBIDDEN_COOLDOWN_MS, learnLimitFromError,@@ -359,7 +359,9 @@ anthropicRouter.post('/messages', async (req: Request, res: Response) => {const estimatedInputTokens = estimateTokens(messages);const imageCount = messages.reduce((n, m) =>n + (Array.isArray(m.content) ? m.content.filter(b => (b as any)?.type === 'image_url').length : 0), 0);- const estimatedTotal = estimatedInputTokens + imageCount * IMAGE_TOKEN_ESTIMATE + max_tokens;+ // Capped output reserve so a large max_tokens can't falsely exclude the model+ // pool (#470); input + images count in full.+ const estimatedTotal = estimatedInputTokens + imageCount * IMAGE_TOKEN_ESTIMATE + routingReserveTokens(max_tokens);// Resolve the model through the operator's Claude-family map (opus/sonnet/// haiku/default → auto | a pinned catalog model). A concrete catalog id pins@@ -411,7 +413,9 @@ anthropicRouter.post('/messages', async (req: Request, res: Response) => {logRequest(route.platform, route.modelId, route.keyId, 'error', estimatedInputTokens, 0, Date.now() - start, 'empty completion (no content, no tool_calls)', null, pinnedModelId);skipKeys.add(`${route.platform}:${route.modelId}:${route.keyId}`);setCooldown(route.platform, route.modelId, route.keyId, cooldownFor(route, {}));- recordRateLimitHit(route.modelDbId);+ // Only demote the whole model when no sibling key can still serve; the+ // per-key cooldown already isolates this key's failure (#454).+ if (!hasOtherUsableKey(route.modelDbId, route.keyId, skipKeys)) recordRateLimitHit(route.modelDbId);lastError = new Error(`empty completion from ${route.displayName}`);continue;}@@ -465,7 +469,8 @@ anthropicRouter.post('/messages', async (req: Request, res: Response) => {if (isModelNotFoundError(err) || isModelAccessForbiddenError(err)) skipModels.add(route.modelDbId);skipKeys.add(`${route.platform}:${route.modelId}:${route.keyId}`);setCooldown(route.platform, route.modelId, route.keyId, cooldownFor(route, err));- recordRateLimitHit(route.modelDbId);+ // Model-level penalty only when no sibling key can still serve (#454).+ if (!hasOtherUsableKey(route.modelDbId, route.keyId, skipKeys)) recordRateLimitHit(route.modelDbId);learnLimitFromError(route.modelDbId, err);lastError = err;continue;diff --git a/server/src/routes/fallback.ts b/server/src/routes/fallback.tsindex 49093dd8..4ebc069e 100644--- a/server/src/routes/fallback.ts+++ b/server/src/routes/fallback.ts@@ -79,10 +79,12 @@ fallbackRouter.get('/', (_req: Request, res: Response) => {ORDER BY fc.priority ASC`).all() as any[];- // Count enabled keys per platform+ // Count usable keys per platform — enabled AND healthy/unknown status. Unified+ // with /token-usage and the routing scorer (#456) so budget pooling is computed+ // from the same key set everywhere (a disabled or invalid key adds no capacity).const keyCounts = db.prepare(`SELECT platform, COUNT(*) as count- FROM api_keys WHERE enabled = 1+ FROM api_keys WHERE enabled = 1 AND status IN ('healthy', 'unknown')GROUP BY platform`).all() as { platform: string; count: number }[];const keyCountMap = new Map(keyCounts.map(k => [k.platform, k.count]));diff --git a/server/src/routes/proxy.ts b/server/src/routes/proxy.tsindex 5934d4b8..8ec837b2 100644--- a/server/src/routes/proxy.ts+++ b/server/src/routes/proxy.ts@@ -3,7 +3,7 @@ import { Router } from 'express';import type { Request, Response } from 'express';import { z } from 'zod';import type { ChatMessage, ChatToolCall, ModelListRow } from '@freellmapi/shared/types.js';-import { routeRequest, resolveRoutingChain, resolveModelGroupCandidates, recordRateLimitHit, recordSuccess, hasEnabledVisionModel, hasEnabledToolsModel, type RouteResult, type ResolvedChain, type ChainRow } from '../services/router.js';+import { routeRequest, resolveRoutingChain, resolveModelGroupCandidates, recordRateLimitHit, recordSuccess, hasEnabledVisionModel, hasEnabledToolsModel, hasOtherUsableKey, routingReserveTokens, type RouteResult, type ResolvedChain, type ChainRow } from '../services/router.js';import { recordRequest, recordTokens, setCooldown, getCooldownDurationForLimit, PAYMENT_REQUIRED_COOLDOWN_MS, MODEL_FORBIDDEN_COOLDOWN_MS, learnLimitFromError } from '../services/ratelimit.js';import { runEmbeddings, EmbeddingsError } from '../services/embeddings.js';import { runImageGeneration, runSpeech, MediaError } from '../services/media.js';@@ -660,7 +660,9 @@ proxyRouter.post('/completions', async (req: Request, res: Response) => {const stop = providerSafeStop(parsed.data.stop);const messages = completionPromptToMessages(prompt, suffix);const estimatedInputTokens = messages.reduce((sum, m) => sum + Math.ceil(contentToString(m.content).length / 4), 0);- const estimatedTotal = estimatedInputTokens + max_tokens;+ // Cap the reserved output so a huge client-set max_tokens doesn't falsely+ // exclude the whole model pool (#470); input is still counted in full.+ const estimatedTotal = estimatedInputTokens + routingReserveTokens(max_tokens);let resolvedChain: ResolvedChain | undefined;if (isAutoModel(requestedModel)) {@@ -920,7 +922,11 @@ proxyRouter.post('/completions', async (req: Request, res: Response) => {tpd: route.tpdLimit,}, err.retryAfterMs),);- recordRateLimitHit(route.modelDbId);+ // Only demote the whole model when this 429 exhausted it. If another+ // enabled, non-cooldown key can still serve, the per-key cooldown alone+ // isolates the failure; a model-level penalty would wrongly sink a+ // model that's healthy on its other keys (#454).+ if (!hasOtherUsableKey(route.modelDbId, route.keyId, skipKeys)) recordRateLimitHit(route.modelDbId);learnLimitFromError(route.modelDbId, err);lastError = err;continue;@@ -1111,7 +1117,9 @@ proxyRouter.post('/chat/completions', async (req: Request, res: Response) => {const IMAGE_TOKEN_ESTIMATE = 1000;const imageCount = messages.reduce((n, m) =>n + (Array.isArray(m.content) ? m.content.filter(b => (b as { type?: string })?.type === 'image_url' || (b as { type?: string })?.type === 'image').length : 0), 0);- const estimatedTotal = estimatedInputTokens + imageCount * IMAGE_TOKEN_ESTIMATE + (max_tokens ?? 1000);+ // The reserved output is capped (routingReserveTokens, #470) so an oversized+ // client max_tokens can't starve routing; input + images count in full.+ const estimatedTotal = estimatedInputTokens + imageCount * IMAGE_TOKEN_ESTIMATE + routingReserveTokens(max_tokens);// Tool-bearing requests must route to a model that emits STRUCTURED// tool_calls. A model without real function-calling support serializes the@@ -1676,7 +1684,8 @@ proxyRouter.post('/chat/completions', async (req: Request, res: Response) => {logRequest(route.platform, route.modelId, route.keyId, 'error', estimatedInputTokens, 0, Date.now() - start, 'empty completion (no content, no tool_calls)', null, pinnedModelId);skipKeys.add(`${route.platform}:${route.modelId}:${route.keyId}`);setCooldown(route.platform, route.modelId, route.keyId, getCooldownDurationForLimit(route.platform, route.modelId, route.keyId, { rpd: route.rpdLimit, tpd: route.tpdLimit }));- recordRateLimitHit(route.modelDbId);+ // Model-level penalty only when no sibling key can still serve (#454).+ if (!hasOtherUsableKey(route.modelDbId, route.keyId, skipKeys)) recordRateLimitHit(route.modelDbId);lastError = new Error(`empty completion from ${route.displayName}`);continue;}@@ -1790,7 +1799,8 @@ proxyRouter.post('/chat/completions', async (req: Request, res: Response) => {tpd: route.tpdLimit,}, err.retryAfterMs),);- recordRateLimitHit(route.modelDbId);+ // Model-level penalty only when no sibling key can still serve (#454).+ if (!hasOtherUsableKey(route.modelDbId, route.keyId, skipKeys)) recordRateLimitHit(route.modelDbId);// Self-correct the catalog: if the provider reported its real ceiling in// the error body (e.g. a Groq 413 "tokens per minute (TPM): Limit 30000"),// persist it so the next request's pre-check fails over BEFORE the 413diff --git a/server/src/routes/responses.ts b/server/src/routes/responses.tsindex 7535425e..64aa8a1c 100644--- a/server/src/routes/responses.ts+++ b/server/src/routes/responses.ts@@ -9,7 +9,7 @@ import type {ChatToolChoice,Platform,} from '@freellmapi/shared/types.js';-import { routeRequest, recordRateLimitHit, recordSuccess, hasEnabledToolsModel, type RouteResult } from '../services/router.js';+import { routeRequest, recordRateLimitHit, recordSuccess, hasEnabledToolsModel, hasOtherUsableKey, routingReserveTokens, type RouteResult } from '../services/router.js';import { recordRequest, recordTokens, setCooldown, getCooldownDurationForLimit, PAYMENT_REQUIRED_COOLDOWN_MS, learnLimitFromError } from '../services/ratelimit.js';import { getUnifiedApiKey } from '../db/index.js';import { contentToString } from '../lib/content.js';@@ -341,7 +341,9 @@ responsesRouter.post('/responses', async (req: Request, res: Response) => {(sum, m) => sum + Math.ceil(contentToString(m.content).length / 4),0,);- const estimatedTotal = estimatedInputTokens + (reqData.max_output_tokens ?? 1000);+ // Capped output reserve so a large max_output_tokens can't falsely exclude the+ // model pool (#470); input counts in full.+ const estimatedTotal = estimatedInputTokens + routingReserveTokens(reqData.max_output_tokens);// Optional client-managed session affinity (mirrors /chat/completions).const rawSessionId = req.headers['x-session-id'];const sessionIdHeader = Array.isArray(rawSessionId) ? rawSessionId[0] : rawSessionId;@@ -559,7 +561,9 @@ responsesRouter.post('/responses', async (req: Request, res: Response) => {logRequest(route.platform, route.modelId, route.keyId, 'error', estimatedInputTokens, 0, Date.now() - start, `unparseable inline tool-call dialect: ${heldText.slice(0, 120)}`);skipKeys.add(`${route.platform}:${route.modelId}:${route.keyId}`);setCooldown(route.platform, route.modelId, route.keyId, getCooldownDurationForLimit(route.platform, route.modelId, route.keyId, { rpd: route.rpdLimit, tpd: route.tpdLimit }));- recordRateLimitHit(route.modelDbId);+ // Only demote the whole model when no sibling key can still serve;+ // the per-key cooldown already isolates this key's failure (#454).+ if (!hasOtherUsableKey(route.modelDbId, route.keyId, skipKeys)) recordRateLimitHit(route.modelDbId);lastError = new Error(`unparseable inline tool-call dialect from ${route.displayName}`);continue;}@@ -609,7 +613,8 @@ responsesRouter.post('/responses', async (req: Request, res: Response) => {logRequest(route.platform, route.modelId, route.keyId, 'error', estimatedInputTokens, 0, Date.now() - start, 'empty completion (no content, no tool_calls)');skipKeys.add(`${route.platform}:${route.modelId}:${route.keyId}`);setCooldown(route.platform, route.modelId, route.keyId, getCooldownDurationForLimit(route.platform, route.modelId, route.keyId, { rpd: route.rpdLimit, tpd: route.tpdLimit }));- recordRateLimitHit(route.modelDbId);+ // Model-level penalty only when no sibling key can still serve (#454).+ if (!hasOtherUsableKey(route.modelDbId, route.keyId, skipKeys)) recordRateLimitHit(route.modelDbId);lastError = new Error(`empty completion from ${route.displayName}`);continue;}@@ -706,7 +711,8 @@ responsesRouter.post('/responses', async (req: Request, res: Response) => {logRequest(route.platform, route.modelId, route.keyId, 'error', promptTokens, 0, Date.now() - start, 'empty completion (no content, no tool_calls)');skipKeys.add(`${route.platform}:${route.modelId}:${route.keyId}`);setCooldown(route.platform, route.modelId, route.keyId, getCooldownDurationForLimit(route.platform, route.modelId, route.keyId, { rpd: route.rpdLimit, tpd: route.tpdLimit }));- recordRateLimitHit(route.modelDbId);+ // Model-level penalty only when no sibling key can still serve (#454).+ if (!hasOtherUsableKey(route.modelDbId, route.keyId, skipKeys)) recordRateLimitHit(route.modelDbId);lastError = new Error(`empty completion from ${route.displayName}`);continue;}@@ -765,7 +771,8 @@ responsesRouter.post('/responses', async (req: Request, res: Response) => {setCooldown(route.platform, route.modelId, route.keyId, isPaymentRequiredError(err)? PAYMENT_REQUIRED_COOLDOWN_MS: getCooldownDurationForLimit(route.platform, route.modelId, route.keyId, { rpd: route.rpdLimit, tpd: route.tpdLimit }));- recordRateLimitHit(route.modelDbId);+ // Model-level penalty only when no sibling key can still serve (#454).+ if (!hasOtherUsableKey(route.modelDbId, route.keyId, skipKeys)) recordRateLimitHit(route.modelDbId);// Learn a provider-reported ceiling (e.g. a 413 TPM limit) so the next// request's pre-check fails over before the 413. Mirrors the chat path.learnLimitFromError(route.modelDbId, err);diff --git a/server/src/services/provider-quota.ts b/server/src/services/provider-quota.tsindex d6113ce0..ff98158d 100644--- a/server/src/services/provider-quota.ts+++ b/server/src/services/provider-quota.ts@@ -1,6 +1,8 @@import crypto from 'crypto';import { AsyncLocalStorage } from 'node:async_hooks';import { getDb } from '../db/index.js';+// Single shared Retry-After parser (was duplicated here and in providers/base.ts).+import { parseRetryAfterMs } from '../providers/base.js';import type {Platform,QuotaMetric,@@ -102,16 +104,6 @@ function parseResetAtFromHeader(raw: string | null, now = Date.now()): string |return new Date(now + parsed * 1000).toISOString();}-function parseRetryAfterMs(raw: string | null): number | null {- if (!raw) return null;- const seconds = Number(raw.trim());- if (Number.isFinite(seconds)) return Math.max(0, Math.round(seconds * 1000));- const asDate = new Date(raw);- const ms = asDate.getTime();- if (Number.isNaN(ms)) return null;- return Math.max(0, ms - Date.now());-}-function pickBetterSource(existing: QuotaObservationSource | null | undefined, next: QuotaObservationSource): QuotaObservationSource {if (!existing) return next;return SOURCE_PRIORITY[next] >= SOURCE_PRIORITY[existing] ? next : existing;@@ -228,7 +220,7 @@ export function parseQuotaObservationsFromResponse(}}- const retryAfterMs = parseRetryAfterMs(get('retry-after'));+ const retryAfterMs = parseRetryAfterMs(get('retry-after')) ?? null;if (retryAfterMs !== null) {observations.push({...base,@@ -399,6 +391,24 @@ export function recordQuotaObservationsFromResponse(.filter((row): row is ProviderQuotaObservation => row !== null);}+// A quota window whose reset_at has passed has replenished at the provider, but+// remaining_value is only ever written on a fresh observation — so a key that hit+// remaining=0 reads as "exhausted" forever on the dashboard health view until the+// next live call (#453). Restore remaining to the known limit (or clear it to+// unknown when the limit isn't known — `= limit_value` yields NULL in that case)+// and drop the stale reset_at so the row stops reading as exhausted and this+// fix-up doesn't recur. Runs on read; a new observation re-populates reset_at.+function normalizeExpiredQuotaState(db: ReturnType<typeof getDb>): void {+ db.prepare(`+ UPDATE provider_quota_state+ SET remaining_value = limit_value,+ reset_at = NULL,+ updated_at = datetime('now')+ WHERE reset_at IS NOT NULL+ AND julianday(reset_at) < julianday('now')+ `).run();+}+export function getQuotaStateForKeys(): QuotaObservationView[] {let db;try {@@ -406,6 +416,7 @@ export function getQuotaStateForKeys(): QuotaObservationView[] {} catch {return [];}+ normalizeExpiredQuotaState(db);return db.prepare(`WITH latest AS (SELECTdiff --git a/server/src/services/router.ts b/server/src/services/router.tsindex 91f5f371..37329d4b 100644--- a/server/src/services/router.ts+++ b/server/src/services/router.ts@@ -136,6 +136,29 @@ export interface RouteResult {tpdLimit: number | null;}+// ── Routing token estimate: cap the reserved OUTPUT, not the full max_tokens ──+// A client can request a huge max_tokens (e.g. 32000) it will never actually+// emit. Reserving that full amount against every model's context window and TPM+// budget falsely excludes the entire free pool (TPM 6k-30k) and returns a bogus+// "all models exhausted" 429 with ZERO upstream calls (#470). Providers meter+// ACTUAL tokens, so under-reserving only risks an upstream 429/413 the retry+// loop already handles, whereas over-reserving starves routing. For routing and+// the context-window / TPM filters we therefore reserve at most this many output+// tokens; the INPUT estimate is still counted in full so a genuinely large+// prompt still (correctly) skips a too-small model.+export const OUTPUT_RESERVE_CAP = 2000;++/**+ * Output tokens to reserve for routing/filter purposes: the requested max_tokens+ * clamped to OUTPUT_RESERVE_CAP (default 1000 when the client omitted it, matching+ * the historical fallback). Callers add this to their INPUT estimate before+ * calling routeRequest / routePinnedModel.+ */+export function routingReserveTokens(requestedMaxTokens: number | null | undefined): number {+ const requested = requestedMaxTokens != null && requestedMaxTokens > 0 ? requestedMaxTokens : 1000;+ return Math.min(requested, OUTPUT_RESERVE_CAP);+}+// Round-robin index per platformconst roundRobinIndex = new Map<string, number>();@@ -388,12 +411,26 @@ interface ScoredEntry {score: number;}+// Enabled + healthy/unknown key count per platform, for pooled-budget scaling.+// This is the SAME filter both /api/fallback endpoints use (issue #456): the+// monthly budget is a PER-KEY free-tier allowance, so N usable keys pool N× the+// capacity. `monthlyUsedTokens` is already summed across all keys, so budget+// must scale to match or the headroom guardrail damps a multi-key model to the+// floor after just one account's worth of tokens.+function usableKeyCountsByPlatform(db: Db): Map<string, number> {+ const rows = db.prepare(+ "SELECT platform, COUNT(*) AS count FROM api_keys WHERE enabled = 1 AND status IN ('healthy', 'unknown') GROUP BY platform"+ ).all() as { platform: string; count: number }[];+ return new Map(rows.map(r => [r.platform, r.count]));+}+function scoreChainEntry(entry: ChainRow,weights: RoutingWeights,intelMin: number,intelMax: number,sampled: boolean,+ keyCounts: Map<string, number>,): ScoredEntry {const stats = statsCache?.get(`${entry.platform}:${entry.model_id}`);const successes = stats?.successes ?? 0;@@ -412,7 +449,10 @@ function scoreChainEntry(intelligenceComposite(entry.size_label, entry.intelligence_rank), intelMin, intelMax,);- const budget = parseBudget(entry.monthly_token_budget);+ // Scale the per-key monthly budget by the usable key count for this platform,+ // matching the pooled `monthlyUsedTokens` aggregate (#456). Math.max(1, …) so a+ // model whose platform currently has no usable key isn't handed a 0 budget.+ const budget = parseBudget(entry.monthly_token_budget) * Math.max(1, keyCounts.get(entry.platform) ?? 1);const headroom = headroomFactor(stats?.monthlyUsedTokens ?? 0, budget);const rl = rateLimitFactor(getPenalty(entry.model_db_id));@@ -446,9 +486,10 @@ function orderChain(chain: ChainRow[], strategy: RoutingStrategy, sampled = trueconst composites = chain.map(e => intelligenceComposite(e.size_label, e.intelligence_rank));const intelMin = composites.length ? Math.min(...composites) : 0;const intelMax = composites.length ? Math.max(...composites) : 0;+ const keyCounts = usableKeyCountsByPlatform(getDb());return chain- .map(e => ({ e, s: scoreChainEntry(e, weights, intelMin, intelMax, sampled).score }))+ .map(e => ({ e, s: scoreChainEntry(e, weights, intelMin, intelMax, sampled, keyCounts).score }))// Higher score first; manual priority breaks ties so the chain still matters..sort((a, b) => b.s - a.s || a.e.priority - b.e.priority).map(x => x.e);@@ -696,6 +737,56 @@ function selectKeyForModel(entry: ChainRow, estimatedTokens: number, skipKeys?:return null;}+/**+ * Whether the model still has ANOTHER key that could serve it right now, given+ * the key that just failed (excludingKeyId) and any keys already ruled out this+ * request (skipKeys, in the "platform:modelId:keyId" form). Applies the same+ * gates selectKeyForModel uses — enabled + healthy status, not on cooldown,+ * under the provider daily cap, and under rpm/rpd/tpm/tpd — so the answer means+ * "a real, dispatchable alternative exists".+ *+ * Used by the retry loops to decide whether a single key's 429 should demote the+ * WHOLE model (the model-level 429 penalty). It should not: the per-key cooldown+ * already isolates the failing key, so demoting the model while a sibling key can+ * still serve it wrongly sinks a healthy model in the scorer (#454). We only+ * record the model-level hit when this returns false — i.e. the 429 exhausted the+ * model, not just one of its keys.+ */+export function hasOtherUsableKey(modelDbId: number, excludingKeyId: number, skipKeys?: Set<string>): boolean {+ const db = getDb();+ const m = db.prepare(`+ SELECT platform, model_id, rpm_limit, rpd_limit, tpm_limit, tpd_limit, key_id+ FROM models WHERE id = ?+ `).get(modelDbId) as {+ platform: string; model_id: string;+ rpm_limit: number | null; rpd_limit: number | null;+ tpm_limit: number | null; tpd_limit: number | null; key_id: number | null;+ } | undefined;+ if (!m) return false;++ const limits = { rpm: m.rpm_limit, rpd: m.rpd_limit, tpm: m.tpm_limit, tpd: m.tpd_limit };+ const keys = db.prepare(+ "SELECT id FROM api_keys WHERE platform = ? AND enabled = 1 AND status IN ('healthy', 'unknown')"+ ).all(m.platform) as { id: number }[];++ for (const k of keys) {+ if (k.id === excludingKeyId) continue;+ // A custom model binds to exactly one endpoint key (#212); a sibling custom+ // key cannot serve it, so it doesn't count as an alternative.+ if (m.platform === 'custom' && m.key_id != null && k.id !== m.key_id) continue;+ if (skipKeys?.has(`${m.platform}:${m.model_id}:${k.id}`)) continue;+ if (isOnCooldown(m.platform, m.model_id, k.id)) continue;+ if (!canUseProvider(m.platform, k.id)) continue;+ if (!canMakeRequest(m.platform, m.model_id, k.id, limits)) continue;+ // A per-minute token spike on the failed key doesn't mean a fresh key lacks+ // headroom; a nominal 1-token probe only rules out a key already at its+ // TPM/TPD ceiling.+ if (!canUseTokens(m.platform, m.model_id, k.id, 1, limits)) continue;+ return true;+ }+ return false;+}+/*** Fetch a single enabled model's chain row by its db id.*/@@ -954,18 +1045,21 @@ export function routeRequest(estimatedTokens = 1000, skipKeys?: Set<string>, pre// Context-aware routing: skip a model whose context window can't hold the// request, so a large prompt never selects a small-context model and burns// a failover hop on a 413 "request too large" (#167). Only enforced when we- // know the model's window; estimatedTokens already includes the reserved- // output (max_tokens), so this is the total-context check the model must- // satisfy. A 413 that slips through is still retryable downstream, and the- // failed model is put on cooldown — so this is a fast-path, not the only- // guard. If every model is too small, the loop falls through and the caller- // gets the normal "all models exhausted" error rather than a wasted sweep.+ // know the model's window; estimatedTokens is the INPUT estimate plus a+ // CAPPED output reserve (routingReserveTokens, #470), so a huge client-set+ // max_tokens no longer excludes the model — the input must fit, not+ // input+full max_tokens. A 413 that slips through is still retryable+ // downstream, and the failed model is put on cooldown — so this is a+ // fast-path, not the only guard. If every model is too small, the loop falls+ // through and the caller gets the normal "all models exhausted" error rather+ // than a wasted sweep.if (entry.context_window != null && estimatedTokens > entry.context_window) { diag.push(`${label}: context ${entry.context_window} < estimated ${estimatedTokens}`); continue; }- // Same guard for a model with a small per-minute token budget: a single- // request that alone exceeds tpm_limit can never fit one minute of quota and+ // Same guard for a model with a small per-minute token budget: a request+ // whose input alone exceeds tpm_limit can never fit one minute of quota and// returns a guaranteed 413 (e.g. Groq gpt-oss-120b: 131k context but 8k TPM).- // estimatedTokens already includes reserved output, mirroring the check above.+ // estimatedTokens carries the same capped output reserve, mirroring the+ // check above (#470).if (entry.tpm_limit != null && estimatedTokens > entry.tpm_limit) { diag.push(`${label}: tpm_limit ${entry.tpm_limit} < estimated ${estimatedTokens}`); continue; }// Key selection + accounting pre-checks for this one model. Returns the@@ -1021,9 +1115,10 @@ export function getRoutingScores(): { strategy: RoutingStrategy; weights: Routinconst composites = chain.map(e => intelligenceComposite(e.size_label, e.intelligence_rank));const intelMin = composites.length ? Math.min(...composites) : 0;const intelMax = composites.length ? Math.max(...composites) : 0;+ const keyCounts = usableKeyCountsByPlatform(db);const scores: RoutingScore[] = chain.map(entry => {- const scored = scoreChainEntry(entry, weights, intelMin, intelMax, false);+ const scored = scoreChainEntry(entry, weights, intelMin, intelMax, false, keyCounts);const stats = statsCache?.get(`${entry.platform}:${entry.model_id}`);return {modelDbId: entry.model_db_id,