Skip to content

7.4 billion tokens per month. 34 free LLM providers. 635 free model endpoints. All behind one /v1 endpoint, plus any custom OpenAI-compatible endpoint. Smart routing, automatic failover, encrypted keys. Personal experimentation only.

TypeScript

fix(routing): multi-key quota fixes for #470, #454, #456, #453 (#479)

Tashfeena2d2a54508e7
diff --git a/server/src/__tests__/routes/fallback-keycount-parity.test.ts b/server/src/__tests__/routes/fallback-keycount-parity.test.ts
new file mode 100644
index 00000000..047e7f4c
--- /dev/null
+++ b/server/src/__tests__/routes/fallback-keycount-parity.test.ts
@@ -0,0 +1,70 @@
+import { describe, it, expect, beforeAll } from 'vitest';
+import type { Express } from 'express';
+import { createApp } from '../../app.js';
+import { getDb, initDb } from '../../db/index.js';
+import { encrypt } from '../../lib/crypto.js';
+import { mintDashboardToken } from '../helpers/auth.js';
+
+let dashToken = '';
+
+async function get(app: Express, path: string) {
+ const server = app.listen(0);
+ const addr = server.address() as any;
+ const res = await fetch(`http://127.0.0.1:${addr.port}${path}`, {
+ headers: { Authorization: `Bearer ${dashToken}` },
+ });
+ const body = await res.json().catch(() => null);
+ server.close();
+ return { status: res.status, body };
+}
+
+// PR #459 scaled /token-usage budgets by healthy+enabled keys, but the pre-existing
+// GET / handler counted enabled=1 only — so the two dashboards disagreed on pooled
+// capacity. Both must now use the SAME filter (enabled AND status healthy/unknown),
+// matching the routing scorer (#456).
+describe('fallback key-count filter parity (#456)', () => {
+ let app: Express;
+ let modelId: string;
+
+ beforeAll(() => {
+ process.env.ENCRYPTION_KEY = '0'.repeat(64);
+ initDb(':memory:');
+ app = createApp();
+ dashToken = mintDashboardToken();
+
+ const db = getDb();
+ // One usable (healthy) groq key and one that must NOT add capacity (invalid).
+ const good = encrypt('good'); const bad = encrypt('bad');
+ db.prepare("INSERT INTO api_keys (platform, label, encrypted_key, iv, auth_tag, status, enabled) VALUES ('groq','good',?,?,?,'healthy',1)").run(good.encrypted, good.iv, good.authTag);
+ db.prepare("INSERT INTO api_keys (platform, label, encrypted_key, iv, auth_tag, status, enabled) VALUES ('groq','bad',?,?,?,'invalid',1)").run(bad.encrypted, bad.iv, bad.authTag);
+
+ const m = db.prepare("SELECT model_id FROM models WHERE platform = 'groq' ORDER BY id LIMIT 1").get() as { model_id: string };
+ modelId = m.model_id;
+ // Non-zero parseable budget so the key multiplier is observable.
+ db.prepare("UPDATE models SET monthly_token_budget = '~1M' WHERE platform = 'groq' AND model_id = ?").run(modelId);
+ });
+
+ it('GET / counts only enabled healthy/unknown keys (invalid excluded)', async () => {
+ const { status, body } = await get(app, '/api/fallback');
+ expect(status).toBe(200);
+ const row = body.find((r: any) => r.platform === 'groq' && r.modelId === modelId);
+ expect(row).toBeDefined();
+ // Two enabled groq keys exist, but one is invalid → usable count is 1.
+ expect(row.keyCount).toBe(1);
+ });
+
+ it('GET / and GET /token-usage agree on the pooled budget for the same model', async () => {
+ const listing = await get(app, '/api/fallback');
+ const usage = await get(app, '/api/fallback/token-usage');
+
+ const listRow = listing.body.find((r: any) => r.platform === 'groq' && r.modelId === modelId);
+ const usageRow = usage.body.models.find((r: any) => r.platform === 'groq' && r.modelId === modelId);
+
+ expect(listRow).toBeDefined();
+ expect(usageRow).toBeDefined();
+ // Both scale parseBudget('~1M') by the SAME usable key count (1), so the
+ // pooled budgets match. Before the fix, GET / used the enabled=2 count.
+ expect(listRow.monthlyTokenBudgetTokens).toBe(usageRow.budget);
+ expect(usageRow.budget).toBe(1_000_000);
+ });
+});
diff --git a/server/src/__tests__/routes/proxy-max-tokens-routing.test.ts b/server/src/__tests__/routes/proxy-max-tokens-routing.test.ts
new file mode 100644
index 00000000..db599868
--- /dev/null
+++ b/server/src/__tests__/routes/proxy-max-tokens-routing.test.ts
@@ -0,0 +1,118 @@
+import { describe, it, expect, beforeAll, beforeEach, vi } from 'vitest';
+import type { Express } from 'express';
+
+// Mock the provider so routing is exercised without real upstream calls.
+const chatCompletion = vi.fn();
+const streamChatCompletion = vi.fn();
+const fakeProvider = { name: 'fake', chatCompletion, streamChatCompletion } as any;
+
+vi.mock('../../providers/index.js', async (importOriginal) => {
+ const actual = await importOriginal() as any;
+ return {
+ ...actual,
+ getProvider: () => fakeProvider,
+ resolveProvider: () => fakeProvider,
+ };
+});
+
+const { createApp } = await import('../../app.js');
+const { initDb, getDb, getUnifiedApiKey } = await import('../../db/index.js');
+const { encrypt } = await import('../../lib/crypto.js');
+const { setRoutingStrategy, routingReserveTokens, OUTPUT_RESERVE_CAP } = await import('../../services/router.js');
+
+async function post(app: Express, body: any, key: string) {
+ const server = app.listen(0);
+ const addr = server.address() as any;
+ const res = await fetch(`http://127.0.0.1:${addr.port}/v1/chat/completions`, {
+ method: 'POST',
+ headers: { 'Content-Type': 'application/json', Authorization: `Bearer ${key}` },
+ body: JSON.stringify(body),
+ });
+ const raw = await res.text();
+ server.close();
+ let json: any = null;
+ try { json = JSON.parse(raw); } catch {}
+ return { status: res.status, body: json };
+}
+
+const GOOD_RESULT = {
+ choices: [{ message: { role: 'assistant', content: 'ok' } }],
+ usage: { prompt_tokens: 5, completion_tokens: 2, total_tokens: 7 },
+};
+
+describe('routingReserveTokens (#470)', () => {
+ it('caps the reserved output at OUTPUT_RESERVE_CAP', () => {
+ expect(routingReserveTokens(32000)).toBe(OUTPUT_RESERVE_CAP);
+ expect(routingReserveTokens(50)).toBe(50);
+ // Omitted / non-positive max_tokens keeps the historical 1000 default (capped).
+ expect(routingReserveTokens(undefined)).toBe(Math.min(1000, OUTPUT_RESERVE_CAP));
+ expect(routingReserveTokens(0)).toBe(Math.min(1000, OUTPUT_RESERVE_CAP));
+ expect(routingReserveTokens(-1)).toBe(Math.min(1000, OUTPUT_RESERVE_CAP));
+ });
+});
+
+describe('max_tokens no longer starves routing (#470)', () => {
+ let app: Express;
+ let key: string;
+
+ beforeAll(() => {
+ process.env.ENCRYPTION_KEY = '0'.repeat(64);
+ initDb(':memory:');
+ app = createApp();
+ key = getUnifiedApiKey();
+
+ const db = getDb();
+ setRoutingStrategy('priority');
+ db.prepare("DELETE FROM settings WHERE key = 'active_profile_id'").run();
+
+ const { encrypted, iv, authTag } = encrypt('groq-key');
+ db.prepare(`
+ INSERT INTO api_keys (platform, label, encrypted_key, iv, auth_tag, status, enabled)
+ VALUES ('groq', 'test', ?, ?, ?, 'healthy', 1)
+ `).run(encrypted, iv, authTag);
+
+ // Only groq is keyed, so make groq the ONLY routable platform and give every
+ // groq model a small per-minute TOKEN budget (6k) with a large context
+ // window, so the sole thing that can exclude a groq model is the token
+ // estimate: a huge client max_tokens must NOT exceed 6k once the reserved
+ // output is capped, while a huge INPUT still must.
+ db.prepare("UPDATE models SET enabled = 0 WHERE platform != 'groq'").run();
+ db.prepare("UPDATE models SET tpm_limit = 6000, tpd_limit = NULL, context_window = 200000 WHERE platform = 'groq'").run();
+ });
+
+ beforeEach(() => {
+ chatCompletion.mockReset();
+ streamChatCompletion.mockReset();
+ getDb().prepare('DELETE FROM rate_limit_cooldowns').run();
+ getDb().prepare('DELETE FROM rate_limit_usage').run();
+ });
+
+ it('routes a tiny prompt with a huge max_tokens (was a false 429 with zero upstream calls)', async () => {
+ chatCompletion.mockResolvedValueOnce(GOOD_RESULT);
+
+ const { status, body } = await post(app, {
+ messages: [{ role: 'user', content: 'hi' }],
+ max_tokens: 32000, // >> the 6k TPM; capped reserve keeps it routable.
+ }, key);
+
+ expect(status).toBe(200);
+ expect(body.choices[0].message.content).toBe('ok');
+ // The model was actually dispatched — not synchronously excluded.
+ expect(chatCompletion).toHaveBeenCalledTimes(1);
+ });
+
+ it('still excludes the model when the INPUT itself exceeds the TPM budget', async () => {
+ chatCompletion.mockResolvedValueOnce(GOOD_RESULT);
+
+ // ~8k input tokens (32k chars / 4) > the 6k TPM budget → the only enabled
+ // model is filtered out before any upstream call.
+ const bigPrompt = 'x'.repeat(32_000);
+ const { status } = await post(app, {
+ messages: [{ role: 'user', content: bigPrompt }],
+ max_tokens: 50,
+ }, key);
+
+ expect(status).toBe(429);
+ expect(chatCompletion).not.toHaveBeenCalled();
+ });
+});
diff --git a/server/src/__tests__/routes/proxy-multikey-penalty.test.ts b/server/src/__tests__/routes/proxy-multikey-penalty.test.ts
new file mode 100644
index 00000000..e5b48589
--- /dev/null
+++ b/server/src/__tests__/routes/proxy-multikey-penalty.test.ts
@@ -0,0 +1,148 @@
+import { describe, it, expect, beforeAll, beforeEach, vi } from 'vitest';
+import type { Express } from 'express';
+
+// Mock the provider so we can script 429s without real upstream calls.
+const chatCompletion = vi.fn();
+const streamChatCompletion = vi.fn();
+const fakeProvider = { name: 'fake', chatCompletion, streamChatCompletion } as any;
+
+vi.mock('../../providers/index.js', async (importOriginal) => {
+ const actual = await importOriginal() as any;
+ return {
+ ...actual,
+ getProvider: () => fakeProvider,
+ resolveProvider: () => fakeProvider,
+ };
+});
+
+const { createApp } = await import('../../app.js');
+const { initDb, getDb, getUnifiedApiKey } = await import('../../db/index.js');
+const { encrypt } = await import('../../lib/crypto.js');
+const { setRoutingStrategy, hasOtherUsableKey, getAllPenalties } = await import('../../services/router.js');
+const { setCooldown } = await import('../../services/ratelimit.js');
+
+async function post(app: Express, body: any, key: string) {
+ const server = app.listen(0);
+ const addr = server.address() as any;
+ const res = await fetch(`http://127.0.0.1:${addr.port}/v1/chat/completions`, {
+ method: 'POST',
+ headers: { 'Content-Type': 'application/json', Authorization: `Bearer ${key}` },
+ body: JSON.stringify(body),
+ });
+ const raw = await res.text();
+ server.close();
+ let json: any = null;
+ try { json = JSON.parse(raw); } catch {}
+ return { status: res.status, body: json };
+}
+
+const GOOD_RESULT = {
+ choices: [{ message: { role: 'assistant', content: 'ok' } }],
+ usage: { prompt_tokens: 5, completion_tokens: 2, total_tokens: 7 },
+};
+const err429 = () => Object.assign(new Error('Groq API error 429: rate limit'), { status: 429 });
+
+interface Setup { modelDbId: number; modelId: string; keyA: number; keyB: number }
+
+// Each test uses a DISTINCT groq model so the module-global in-memory rate-limit
+// state (cooldowns, round-robin, penalties — keyed by model) can't leak between
+// tests, mirroring how ratelimit.test.ts uses unique identifiers per case.
+function setup(rank: number, onlyModel = false): Setup {
+ const db = getDb();
+ setRoutingStrategy('priority');
+ db.prepare("DELETE FROM settings WHERE key = 'active_profile_id'").run();
+ db.prepare('DELETE FROM api_keys').run();
+ db.prepare('DELETE FROM rate_limit_cooldowns').run();
+ db.prepare('DELETE FROM rate_limit_usage').run();
+
+ const models = db.prepare("SELECT id, model_id FROM models WHERE platform = 'groq' ORDER BY id").all() as { id: number; model_id: string }[];
+ const model = models[rank];
+ if (onlyModel) db.prepare('UPDATE models SET enabled = 0').run();
+ db.prepare('UPDATE models SET enabled = 1, rpm_limit = NULL, rpd_limit = NULL, tpm_limit = NULL, tpd_limit = NULL WHERE id = ?').run(model.id);
+
+ const ids: number[] = [];
+ for (const label of ['a', 'b']) {
+ const { encrypted, iv, authTag } = encrypt(`groq-${rank}-${label}`);
+ const info = db.prepare(`
+ INSERT INTO api_keys (platform, label, encrypted_key, iv, auth_tag, status, enabled)
+ VALUES ('groq', ?, ?, ?, ?, 'healthy', 1)
+ `).run(`${rank}-${label}`, encrypted, iv, authTag);
+ ids.push(Number(info.lastInsertRowid));
+ }
+ return { modelDbId: model.id, modelId: model.model_id, keyA: ids[0], keyB: ids[1] };
+}
+
+describe('hasOtherUsableKey (#454)', () => {
+ beforeAll(() => {
+ process.env.ENCRYPTION_KEY = '0'.repeat(64);
+ initDb(':memory:');
+ });
+
+ it('returns true when a sibling key is free', () => {
+ const s = setup(0);
+ expect(hasOtherUsableKey(s.modelDbId, s.keyA)).toBe(true);
+ });
+
+ it('returns false when the only sibling is on cooldown', () => {
+ const s = setup(1);
+ setCooldown('groq', s.modelId, s.keyB, 5 * 60 * 1000);
+ expect(hasOtherUsableKey(s.modelDbId, s.keyA)).toBe(false);
+ });
+
+ it('returns false when the only sibling is already ruled out this request (skipKeys)', () => {
+ const s = setup(2);
+ const skip = new Set<string>([`groq:${s.modelId}:${s.keyB}`]);
+ expect(hasOtherUsableKey(s.modelDbId, s.keyA, skip)).toBe(false);
+ });
+
+ it('is symmetric: excluding one key finds the other when it is free, none when it is benched', () => {
+ const s = setup(3);
+ expect(hasOtherUsableKey(s.modelDbId, s.keyB)).toBe(true); // keyA free
+ setCooldown('groq', s.modelId, s.keyA, 5 * 60 * 1000);
+ expect(hasOtherUsableKey(s.modelDbId, s.keyB)).toBe(false); // keyA benched, keyB excluded
+ });
+});
+
+describe('one key\'s 429 must not demote the whole model (#454)', () => {
+ let app: Express;
+ let key: string;
+
+ beforeAll(() => {
+ process.env.ENCRYPTION_KEY = '0'.repeat(64);
+ initDb(':memory:');
+ app = createApp();
+ key = getUnifiedApiKey();
+ });
+
+ beforeEach(() => {
+ chatCompletion.mockReset();
+ streamChatCompletion.mockReset();
+ });
+
+ it('does NOT record a model-level penalty when the first key 429s but a sibling succeeds', async () => {
+ const s = setup(4, true);
+ chatCompletion
+ .mockRejectedValueOnce(err429()) // key A
+ .mockResolvedValueOnce(GOOD_RESULT); // key B
+
+ const { status } = await post(app, { messages: [{ role: 'user', content: 'hi' }] }, key);
+
+ expect(status).toBe(200);
+ expect(chatCompletion).toHaveBeenCalledTimes(2);
+ // The model must NOT be penalized — only one key hit the limit.
+ expect(getAllPenalties().some(p => p.modelDbId === s.modelDbId)).toBe(false);
+ });
+
+ it('DOES record the model-level penalty once the last usable key 429s', async () => {
+ const s = setup(5, true);
+ chatCompletion.mockRejectedValue(err429()); // both keys 429
+
+ const { status } = await post(app, { messages: [{ role: 'user', content: 'hi' }] }, key);
+
+ // Chain exhausts after both keys fail.
+ expect(status).toBe(429);
+ expect(chatCompletion).toHaveBeenCalledTimes(2);
+ // The model IS penalized: the 429 exhausted it, not just one key.
+ expect(getAllPenalties().some(p => p.modelDbId === s.modelDbId)).toBe(true);
+ });
+});
diff --git a/server/src/__tests__/services/provider-quota.test.ts b/server/src/__tests__/services/provider-quota.test.ts
new file mode 100644
index 00000000..b11a69dd
--- /dev/null
+++ b/server/src/__tests__/services/provider-quota.test.ts
@@ -0,0 +1,161 @@
+import { describe, it, expect, beforeAll, beforeEach } from 'vitest';
+import { initDb, getDb } from '../../db/index.js';
+import {
+ recordQuotaObservation,
+ getQuotaStateForKeys,
+ parseQuotaObservationsFromResponse,
+ inferQuotaPoolKey,
+} from '../../services/provider-quota.js';
+
+function insertState(row: {
+ platform: string;
+ keyId: number;
+ pool: string;
+ metric: string;
+ limit: number | null;
+ remaining: number | null;
+ resetAt: string | null;
+}) {
+ getDb().prepare(`
+ INSERT INTO provider_quota_state
+ (platform, key_id, quota_pool_key, metric, limit_value, remaining_value, reset_at)
+ VALUES (?, ?, ?, ?, ?, ?, ?)
+ `).run(row.platform, row.keyId, row.pool, row.metric, row.limit, row.remaining, row.resetAt);
+}
+
+function readState(platform: string, keyId: number, pool: string, metric: string) {
+ return getDb().prepare(`
+ SELECT limit_value AS lim, remaining_value AS remaining, reset_at AS resetAt
+ FROM provider_quota_state
+ WHERE platform = ? AND key_id = ? AND quota_pool_key = ? AND metric = ?
+ `).get(platform, keyId, pool, metric) as { lim: number | null; remaining: number | null; resetAt: string | null } | undefined;
+}
+
+describe('provider-quota: pool inference', () => {
+ beforeAll(() => {
+ process.env.ENCRYPTION_KEY = '0'.repeat(64);
+ initDb(':memory:');
+ });
+
+ it('buckets shared-pool providers per account and openrouter free vs account', () => {
+ expect(inferQuotaPoolKey('groq')).toBe('groq::account');
+ expect(inferQuotaPoolKey('openrouter', 'meta-llama/llama-3.1-8b-instruct:free')).toBe('openrouter::free');
+ expect(inferQuotaPoolKey('openrouter', 'openai/gpt-4o')).toBe('openrouter::account');
+ // Unknown platform falls back to platform::model or platform::account.
+ expect(inferQuotaPoolKey('acme' as any, 'x')).toBe('acme::x');
+ expect(inferQuotaPoolKey('acme' as any)).toBe('acme::account');
+ });
+});
+
+describe('provider-quota: record + read round-trip', () => {
+ beforeAll(() => {
+ process.env.ENCRYPTION_KEY = '0'.repeat(64);
+ initDb(':memory:');
+ });
+
+ beforeEach(() => {
+ getDb().prepare('DELETE FROM provider_quota_state').run();
+ getDb().prepare('DELETE FROM provider_quota_observations').run();
+ });
+
+ it('records an observation and surfaces it via getQuotaStateForKeys', () => {
+ const rec = recordQuotaObservation({
+ platform: 'groq',
+ keyId: 7,
+ quotaPoolKey: 'groq::account',
+ metric: 'requests',
+ limit: 1000,
+ remaining: 950,
+ source: 'header',
+ });
+ expect(rec).not.toBeNull();
+
+ const states = getQuotaStateForKeys();
+ const row = states.find(s => s.platform === 'groq' && s.keyId === 7 && s.metric === 'requests');
+ expect(row).toBeDefined();
+ expect(row!.limit).toBe(1000);
+ expect(row!.remaining).toBe(950);
+ });
+});
+
+describe('provider-quota: parse from response headers (shared parseRetryAfterMs)', () => {
+ beforeAll(() => {
+ process.env.ENCRYPTION_KEY = '0'.repeat(64);
+ initDb(':memory:');
+ });
+
+ it('parses Groq ratelimit headers into a requests observation', () => {
+ const resp = new Response(null, {
+ status: 200,
+ headers: {
+ 'x-ratelimit-limit-requests': '100',
+ 'x-ratelimit-remaining-requests': '90',
+ 'x-ratelimit-reset-requests': '60',
+ },
+ });
+ const obs = parseQuotaObservationsFromResponse(resp, { platform: 'groq', keyId: 1 });
+ const requests = obs.find(o => o.metric === 'requests');
+ expect(requests).toBeDefined();
+ expect(requests!.limit).toBe(100);
+ expect(requests!.remaining).toBe(90);
+ });
+
+ it('reads Retry-After on a 429 via the shared parser (dedup of base.ts)', () => {
+ const resp = new Response(null, { status: 429, headers: { 'retry-after': '30' } });
+ const obs = parseQuotaObservationsFromResponse(resp, { platform: 'groq', keyId: 1 });
+ // The shared parseRetryAfterMs turns "30" seconds into 30000 ms.
+ expect(obs.some(o => o.retryAfterMs === 30_000)).toBe(true);
+ // A 429 always marks the pool as remaining 0.
+ expect(obs.some(o => o.remaining === 0)).toBe(true);
+ });
+});
+
+describe('provider-quota: reset_at replenishment on read (#453)', () => {
+ beforeAll(() => {
+ process.env.ENCRYPTION_KEY = '0'.repeat(64);
+ initDb(':memory:');
+ });
+
+ beforeEach(() => {
+ getDb().prepare('DELETE FROM provider_quota_state').run();
+ getDb().prepare('DELETE FROM provider_quota_observations').run();
+ });
+
+ const past = () => new Date(Date.now() - 60 * 60 * 1000).toISOString();
+ const future = () => new Date(Date.now() + 60 * 60 * 1000).toISOString();
+
+ it('restores remaining to the known limit once reset_at has passed, and persists it', () => {
+ insertState({ platform: 'groq', keyId: 1, pool: 'groq::account', metric: 'requests', limit: 100, remaining: 0, resetAt: past() });
+
+ const states = getQuotaStateForKeys();
+ const row = states.find(s => s.platform === 'groq' && s.keyId === 1);
+ expect(row!.remaining).toBe(100); // replenished to the limit
+ expect(row!.resetAt).toBeNull(); // stale reset dropped
+
+ // Persisted so it does not recur (the exhausted 0 is gone from the table).
+ const persisted = readState('groq', 1, 'groq::account', 'requests');
+ expect(persisted!.remaining).toBe(100);
+ expect(persisted!.resetAt).toBeNull();
+ });
+
+ it('clears remaining to unknown when the limit is unknown and reset_at passed', () => {
+ insertState({ platform: 'ollama', keyId: 2, pool: 'ollama::cloud', metric: 'requests', limit: null, remaining: 0, resetAt: past() });
+
+ const states = getQuotaStateForKeys();
+ const row = states.find(s => s.platform === 'ollama' && s.keyId === 2);
+ expect(row!.remaining).toBeNull(); // no known limit → clear the 0
+ expect(row!.resetAt).toBeNull();
+
+ const persisted = readState('ollama', 2, 'ollama::cloud', 'requests');
+ expect(persisted!.remaining).toBeNull();
+ });
+
+ it('leaves a still-active window (reset_at in the future) untouched', () => {
+ insertState({ platform: 'groq', keyId: 3, pool: 'groq::account', metric: 'requests', limit: 100, remaining: 0, resetAt: future() });
+
+ const states = getQuotaStateForKeys();
+ const row = states.find(s => s.platform === 'groq' && s.keyId === 3);
+ expect(row!.remaining).toBe(0); // still exhausted until it resets
+ expect(row!.resetAt).not.toBeNull();
+ });
+});
diff --git a/server/src/__tests__/services/router-keycount-budget.test.ts b/server/src/__tests__/services/router-keycount-budget.test.ts
new file mode 100644
index 00000000..6d5d598e
--- /dev/null
+++ b/server/src/__tests__/services/router-keycount-budget.test.ts
@@ -0,0 +1,74 @@
+import { describe, it, expect, beforeAll, beforeEach } from 'vitest';
+import { initDb, getDb } from '../../db/index.js';
+import { encrypt } from '../../lib/crypto.js';
+import { getRoutingScores, refreshStatsCache } from '../../services/router.js';
+
+// The monthly free-tier budget is PER KEY, and monthly usage is pooled across
+// all keys — so the headroom guardrail must scale the budget by the usable key
+// count or a multi-key model gets damped to the floor after one account's worth
+// of tokens (#456).
+describe('routing headroom scales the monthly budget by usable key count (#456)', () => {
+ let modelDbId: number;
+ let modelId: string;
+
+ beforeAll(() => {
+ process.env.ENCRYPTION_KEY = '0'.repeat(64);
+ initDb(':memory:');
+ const db = getDb();
+ const m = db.prepare("SELECT id, model_id FROM models WHERE platform = 'groq' ORDER BY id LIMIT 1").get() as { id: number; model_id: string };
+ modelDbId = m.id;
+ modelId = m.model_id;
+ // A known, parseable per-key budget so headroom math is deterministic.
+ db.prepare("UPDATE models SET monthly_token_budget = '~1M' WHERE id = ?").run(modelDbId);
+ // ~950k tokens used this month (pooled across keys): 95% of a single key's 1M.
+ db.prepare(`
+ INSERT INTO requests (platform, model_id, key_id, status, input_tokens, output_tokens, latency_ms, error, request_type, created_at)
+ VALUES (?, ?, NULL, 'success', 500000, 450000, 10, NULL, 'chat', datetime('now'))
+ `).run('groq', modelId);
+ });
+
+ beforeEach(() => {
+ getDb().prepare('DELETE FROM api_keys').run();
+ });
+
+ function headroomFor(): number {
+ refreshStatsCache(getDb(), true); // force so the inserted usage is picked up
+ const { scores } = getRoutingScores();
+ const row = scores.find(s => s.modelDbId === modelDbId);
+ if (!row) throw new Error('model missing from routing scores');
+ return row.headroom;
+ }
+
+ function addGroqKeys(n: number) {
+ const db = getDb();
+ for (let i = 0; i < n; i++) {
+ const { encrypted, iv, authTag } = encrypt(`groq-${i}`);
+ db.prepare(`
+ INSERT INTO api_keys (platform, label, encrypted_key, iv, auth_tag, status, enabled)
+ VALUES ('groq', ?, ?, ?, ?, 'healthy', 1)
+ `).run(`k${i}`, encrypted, iv, authTag);
+ }
+ }
+
+ it('damps a nearly-spent single-key model below full headroom', () => {
+ addGroqKeys(1);
+ expect(headroomFor()).toBeLessThan(1); // 950k / 1M → <20% left → protecting
+ });
+
+ it('restores full headroom once three usable keys pool 3x the budget', () => {
+ addGroqKeys(3);
+ // 950k / 3M → ~68% left → the guardrail stops protecting.
+ expect(headroomFor()).toBe(1);
+ });
+
+ it('ignores disabled and unhealthy keys when scaling (same filter as the endpoints)', () => {
+ addGroqKeys(1); // one healthy key
+ const db = getDb();
+ // An invalid and a disabled key add NO pooled capacity.
+ const bad = encrypt('bad'); const off = encrypt('off');
+ db.prepare("INSERT INTO api_keys (platform, label, encrypted_key, iv, auth_tag, status, enabled) VALUES ('groq','bad',?,?,?,'invalid',1)").run(bad.encrypted, bad.iv, bad.authTag);
+ db.prepare("INSERT INTO api_keys (platform, label, encrypted_key, iv, auth_tag, status, enabled) VALUES ('groq','off',?,?,?,'healthy',0)").run(off.encrypted, off.iv, off.authTag);
+ // Still effectively single-key → still damped.
+ expect(headroomFor()).toBeLessThan(1);
+ });
+});
diff --git a/server/src/routes/anthropic.ts b/server/src/routes/anthropic.ts
index bf39df55..aaf64503 100644
--- a/server/src/routes/anthropic.ts
+++ b/server/src/routes/anthropic.ts
@@ -10,7 +10,7 @@ import type {
ChatContent,
ChatContentBlock,
} from '@freellmapi/shared/types.js';
-import { routeRequest, recordRateLimitHit, recordSuccess, type RouteResult } from '../services/router.js';
+import { routeRequest, recordRateLimitHit, recordSuccess, hasOtherUsableKey, routingReserveTokens, type RouteResult } from '../services/router.js';
import {
recordRequest, recordTokens, setCooldown, getCooldownDurationForLimit,
PAYMENT_REQUIRED_COOLDOWN_MS, MODEL_FORBIDDEN_COOLDOWN_MS, learnLimitFromError,
@@ -359,7 +359,9 @@ anthropicRouter.post('/messages', async (req: Request, res: Response) => {
const estimatedInputTokens = estimateTokens(messages);
const imageCount = messages.reduce((n, m) =>
n + (Array.isArray(m.content) ? m.content.filter(b => (b as any)?.type === 'image_url').length : 0), 0);
- const estimatedTotal = estimatedInputTokens + imageCount * IMAGE_TOKEN_ESTIMATE + max_tokens;
+ // Capped output reserve so a large max_tokens can't falsely exclude the model
+ // pool (#470); input + images count in full.
+ const estimatedTotal = estimatedInputTokens + imageCount * IMAGE_TOKEN_ESTIMATE + routingReserveTokens(max_tokens);
// Resolve the model through the operator's Claude-family map (opus/sonnet/
// haiku/default → auto | a pinned catalog model). A concrete catalog id pins
@@ -411,7 +413,9 @@ anthropicRouter.post('/messages', async (req: Request, res: Response) => {
logRequest(route.platform, route.modelId, route.keyId, 'error', estimatedInputTokens, 0, Date.now() - start, 'empty completion (no content, no tool_calls)', null, pinnedModelId);
skipKeys.add(`${route.platform}:${route.modelId}:${route.keyId}`);
setCooldown(route.platform, route.modelId, route.keyId, cooldownFor(route, {}));
- recordRateLimitHit(route.modelDbId);
+ // Only demote the whole model when no sibling key can still serve; the
+ // per-key cooldown already isolates this key's failure (#454).
+ if (!hasOtherUsableKey(route.modelDbId, route.keyId, skipKeys)) recordRateLimitHit(route.modelDbId);
lastError = new Error(`empty completion from ${route.displayName}`);
continue;
}
@@ -465,7 +469,8 @@ anthropicRouter.post('/messages', async (req: Request, res: Response) => {
if (isModelNotFoundError(err) || isModelAccessForbiddenError(err)) skipModels.add(route.modelDbId);
skipKeys.add(`${route.platform}:${route.modelId}:${route.keyId}`);
setCooldown(route.platform, route.modelId, route.keyId, cooldownFor(route, err));
- recordRateLimitHit(route.modelDbId);
+ // Model-level penalty only when no sibling key can still serve (#454).
+ if (!hasOtherUsableKey(route.modelDbId, route.keyId, skipKeys)) recordRateLimitHit(route.modelDbId);
learnLimitFromError(route.modelDbId, err);
lastError = err;
continue;
diff --git a/server/src/routes/fallback.ts b/server/src/routes/fallback.ts
index 49093dd8..4ebc069e 100644
--- a/server/src/routes/fallback.ts
+++ b/server/src/routes/fallback.ts
@@ -79,10 +79,12 @@ fallbackRouter.get('/', (_req: Request, res: Response) => {
ORDER BY fc.priority ASC
`).all() as any[];
- // Count enabled keys per platform
+ // Count usable keys per platform — enabled AND healthy/unknown status. Unified
+ // with /token-usage and the routing scorer (#456) so budget pooling is computed
+ // from the same key set everywhere (a disabled or invalid key adds no capacity).
const keyCounts = db.prepare(`
SELECT platform, COUNT(*) as count
- FROM api_keys WHERE enabled = 1
+ FROM api_keys WHERE enabled = 1 AND status IN ('healthy', 'unknown')
GROUP BY platform
`).all() as { platform: string; count: number }[];
const keyCountMap = new Map(keyCounts.map(k => [k.platform, k.count]));
diff --git a/server/src/routes/proxy.ts b/server/src/routes/proxy.ts
index 5934d4b8..8ec837b2 100644
--- a/server/src/routes/proxy.ts
+++ b/server/src/routes/proxy.ts
@@ -3,7 +3,7 @@ import { Router } from 'express';
import type { Request, Response } from 'express';
import { z } from 'zod';
import type { ChatMessage, ChatToolCall, ModelListRow } from '@freellmapi/shared/types.js';
-import { routeRequest, resolveRoutingChain, resolveModelGroupCandidates, recordRateLimitHit, recordSuccess, hasEnabledVisionModel, hasEnabledToolsModel, type RouteResult, type ResolvedChain, type ChainRow } from '../services/router.js';
+import { routeRequest, resolveRoutingChain, resolveModelGroupCandidates, recordRateLimitHit, recordSuccess, hasEnabledVisionModel, hasEnabledToolsModel, hasOtherUsableKey, routingReserveTokens, type RouteResult, type ResolvedChain, type ChainRow } from '../services/router.js';
import { recordRequest, recordTokens, setCooldown, getCooldownDurationForLimit, PAYMENT_REQUIRED_COOLDOWN_MS, MODEL_FORBIDDEN_COOLDOWN_MS, learnLimitFromError } from '../services/ratelimit.js';
import { runEmbeddings, EmbeddingsError } from '../services/embeddings.js';
import { runImageGeneration, runSpeech, MediaError } from '../services/media.js';
@@ -660,7 +660,9 @@ proxyRouter.post('/completions', async (req: Request, res: Response) => {
const stop = providerSafeStop(parsed.data.stop);
const messages = completionPromptToMessages(prompt, suffix);
const estimatedInputTokens = messages.reduce((sum, m) => sum + Math.ceil(contentToString(m.content).length / 4), 0);
- const estimatedTotal = estimatedInputTokens + max_tokens;
+ // Cap the reserved output so a huge client-set max_tokens doesn't falsely
+ // exclude the whole model pool (#470); input is still counted in full.
+ const estimatedTotal = estimatedInputTokens + routingReserveTokens(max_tokens);
let resolvedChain: ResolvedChain | undefined;
if (isAutoModel(requestedModel)) {
@@ -920,7 +922,11 @@ proxyRouter.post('/completions', async (req: Request, res: Response) => {
tpd: route.tpdLimit,
}, err.retryAfterMs),
);
- recordRateLimitHit(route.modelDbId);
+ // Only demote the whole model when this 429 exhausted it. If another
+ // enabled, non-cooldown key can still serve, the per-key cooldown alone
+ // isolates the failure; a model-level penalty would wrongly sink a
+ // model that's healthy on its other keys (#454).
+ if (!hasOtherUsableKey(route.modelDbId, route.keyId, skipKeys)) recordRateLimitHit(route.modelDbId);
learnLimitFromError(route.modelDbId, err);
lastError = err;
continue;
@@ -1111,7 +1117,9 @@ proxyRouter.post('/chat/completions', async (req: Request, res: Response) => {
const IMAGE_TOKEN_ESTIMATE = 1000;
const imageCount = messages.reduce((n, m) =>
n + (Array.isArray(m.content) ? m.content.filter(b => (b as { type?: string })?.type === 'image_url' || (b as { type?: string })?.type === 'image').length : 0), 0);
- const estimatedTotal = estimatedInputTokens + imageCount * IMAGE_TOKEN_ESTIMATE + (max_tokens ?? 1000);
+ // The reserved output is capped (routingReserveTokens, #470) so an oversized
+ // client max_tokens can't starve routing; input + images count in full.
+ const estimatedTotal = estimatedInputTokens + imageCount * IMAGE_TOKEN_ESTIMATE + routingReserveTokens(max_tokens);
// Tool-bearing requests must route to a model that emits STRUCTURED
// tool_calls. A model without real function-calling support serializes the
@@ -1676,7 +1684,8 @@ proxyRouter.post('/chat/completions', async (req: Request, res: Response) => {
logRequest(route.platform, route.modelId, route.keyId, 'error', estimatedInputTokens, 0, Date.now() - start, 'empty completion (no content, no tool_calls)', null, pinnedModelId);
skipKeys.add(`${route.platform}:${route.modelId}:${route.keyId}`);
setCooldown(route.platform, route.modelId, route.keyId, getCooldownDurationForLimit(route.platform, route.modelId, route.keyId, { rpd: route.rpdLimit, tpd: route.tpdLimit }));
- recordRateLimitHit(route.modelDbId);
+ // Model-level penalty only when no sibling key can still serve (#454).
+ if (!hasOtherUsableKey(route.modelDbId, route.keyId, skipKeys)) recordRateLimitHit(route.modelDbId);
lastError = new Error(`empty completion from ${route.displayName}`);
continue;
}
@@ -1790,7 +1799,8 @@ proxyRouter.post('/chat/completions', async (req: Request, res: Response) => {
tpd: route.tpdLimit,
}, err.retryAfterMs),
);
- recordRateLimitHit(route.modelDbId);
+ // Model-level penalty only when no sibling key can still serve (#454).
+ if (!hasOtherUsableKey(route.modelDbId, route.keyId, skipKeys)) recordRateLimitHit(route.modelDbId);
// Self-correct the catalog: if the provider reported its real ceiling in
// the error body (e.g. a Groq 413 "tokens per minute (TPM): Limit 30000"),
// persist it so the next request's pre-check fails over BEFORE the 413
diff --git a/server/src/routes/responses.ts b/server/src/routes/responses.ts
index 7535425e..64aa8a1c 100644
--- a/server/src/routes/responses.ts
+++ b/server/src/routes/responses.ts
@@ -9,7 +9,7 @@ import type {
ChatToolChoice,
Platform,
} from '@freellmapi/shared/types.js';
-import { routeRequest, recordRateLimitHit, recordSuccess, hasEnabledToolsModel, type RouteResult } from '../services/router.js';
+import { routeRequest, recordRateLimitHit, recordSuccess, hasEnabledToolsModel, hasOtherUsableKey, routingReserveTokens, type RouteResult } from '../services/router.js';
import { recordRequest, recordTokens, setCooldown, getCooldownDurationForLimit, PAYMENT_REQUIRED_COOLDOWN_MS, learnLimitFromError } from '../services/ratelimit.js';
import { getUnifiedApiKey } from '../db/index.js';
import { contentToString } from '../lib/content.js';
@@ -341,7 +341,9 @@ responsesRouter.post('/responses', async (req: Request, res: Response) => {
(sum, m) => sum + Math.ceil(contentToString(m.content).length / 4),
0,
);
- const estimatedTotal = estimatedInputTokens + (reqData.max_output_tokens ?? 1000);
+ // Capped output reserve so a large max_output_tokens can't falsely exclude the
+ // model pool (#470); input counts in full.
+ const estimatedTotal = estimatedInputTokens + routingReserveTokens(reqData.max_output_tokens);
// Optional client-managed session affinity (mirrors /chat/completions).
const rawSessionId = req.headers['x-session-id'];
const sessionIdHeader = Array.isArray(rawSessionId) ? rawSessionId[0] : rawSessionId;
@@ -559,7 +561,9 @@ responsesRouter.post('/responses', async (req: Request, res: Response) => {
logRequest(route.platform, route.modelId, route.keyId, 'error', estimatedInputTokens, 0, Date.now() - start, `unparseable inline tool-call dialect: ${heldText.slice(0, 120)}`);
skipKeys.add(`${route.platform}:${route.modelId}:${route.keyId}`);
setCooldown(route.platform, route.modelId, route.keyId, getCooldownDurationForLimit(route.platform, route.modelId, route.keyId, { rpd: route.rpdLimit, tpd: route.tpdLimit }));
- recordRateLimitHit(route.modelDbId);
+ // Only demote the whole model when no sibling key can still serve;
+ // the per-key cooldown already isolates this key's failure (#454).
+ if (!hasOtherUsableKey(route.modelDbId, route.keyId, skipKeys)) recordRateLimitHit(route.modelDbId);
lastError = new Error(`unparseable inline tool-call dialect from ${route.displayName}`);
continue;
}
@@ -609,7 +613,8 @@ responsesRouter.post('/responses', async (req: Request, res: Response) => {
logRequest(route.platform, route.modelId, route.keyId, 'error', estimatedInputTokens, 0, Date.now() - start, 'empty completion (no content, no tool_calls)');
skipKeys.add(`${route.platform}:${route.modelId}:${route.keyId}`);
setCooldown(route.platform, route.modelId, route.keyId, getCooldownDurationForLimit(route.platform, route.modelId, route.keyId, { rpd: route.rpdLimit, tpd: route.tpdLimit }));
- recordRateLimitHit(route.modelDbId);
+ // Model-level penalty only when no sibling key can still serve (#454).
+ if (!hasOtherUsableKey(route.modelDbId, route.keyId, skipKeys)) recordRateLimitHit(route.modelDbId);
lastError = new Error(`empty completion from ${route.displayName}`);
continue;
}
@@ -706,7 +711,8 @@ responsesRouter.post('/responses', async (req: Request, res: Response) => {
logRequest(route.platform, route.modelId, route.keyId, 'error', promptTokens, 0, Date.now() - start, 'empty completion (no content, no tool_calls)');
skipKeys.add(`${route.platform}:${route.modelId}:${route.keyId}`);
setCooldown(route.platform, route.modelId, route.keyId, getCooldownDurationForLimit(route.platform, route.modelId, route.keyId, { rpd: route.rpdLimit, tpd: route.tpdLimit }));
- recordRateLimitHit(route.modelDbId);
+ // Model-level penalty only when no sibling key can still serve (#454).
+ if (!hasOtherUsableKey(route.modelDbId, route.keyId, skipKeys)) recordRateLimitHit(route.modelDbId);
lastError = new Error(`empty completion from ${route.displayName}`);
continue;
}
@@ -765,7 +771,8 @@ responsesRouter.post('/responses', async (req: Request, res: Response) => {
setCooldown(route.platform, route.modelId, route.keyId, isPaymentRequiredError(err)
? PAYMENT_REQUIRED_COOLDOWN_MS
: getCooldownDurationForLimit(route.platform, route.modelId, route.keyId, { rpd: route.rpdLimit, tpd: route.tpdLimit }));
- recordRateLimitHit(route.modelDbId);
+ // Model-level penalty only when no sibling key can still serve (#454).
+ if (!hasOtherUsableKey(route.modelDbId, route.keyId, skipKeys)) recordRateLimitHit(route.modelDbId);
// Learn a provider-reported ceiling (e.g. a 413 TPM limit) so the next
// request's pre-check fails over before the 413. Mirrors the chat path.
learnLimitFromError(route.modelDbId, err);
diff --git a/server/src/services/provider-quota.ts b/server/src/services/provider-quota.ts
index d6113ce0..ff98158d 100644
--- a/server/src/services/provider-quota.ts
+++ b/server/src/services/provider-quota.ts
@@ -1,6 +1,8 @@
import crypto from 'crypto';
import { AsyncLocalStorage } from 'node:async_hooks';
import { getDb } from '../db/index.js';
+// Single shared Retry-After parser (was duplicated here and in providers/base.ts).
+import { parseRetryAfterMs } from '../providers/base.js';
import type {
Platform,
QuotaMetric,
@@ -102,16 +104,6 @@ function parseResetAtFromHeader(raw: string | null, now = Date.now()): string |
return new Date(now + parsed * 1000).toISOString();
}
-function parseRetryAfterMs(raw: string | null): number | null {
- if (!raw) return null;
- const seconds = Number(raw.trim());
- if (Number.isFinite(seconds)) return Math.max(0, Math.round(seconds * 1000));
- const asDate = new Date(raw);
- const ms = asDate.getTime();
- if (Number.isNaN(ms)) return null;
- return Math.max(0, ms - Date.now());
-}
-
function pickBetterSource(existing: QuotaObservationSource | null | undefined, next: QuotaObservationSource): QuotaObservationSource {
if (!existing) return next;
return SOURCE_PRIORITY[next] >= SOURCE_PRIORITY[existing] ? next : existing;
@@ -228,7 +220,7 @@ export function parseQuotaObservationsFromResponse(
}
}
- const retryAfterMs = parseRetryAfterMs(get('retry-after'));
+ const retryAfterMs = parseRetryAfterMs(get('retry-after')) ?? null;
if (retryAfterMs !== null) {
observations.push({
...base,
@@ -399,6 +391,24 @@ export function recordQuotaObservationsFromResponse(
.filter((row): row is ProviderQuotaObservation => row !== null);
}
+// A quota window whose reset_at has passed has replenished at the provider, but
+// remaining_value is only ever written on a fresh observation — so a key that hit
+// remaining=0 reads as "exhausted" forever on the dashboard health view until the
+// next live call (#453). Restore remaining to the known limit (or clear it to
+// unknown when the limit isn't known — `= limit_value` yields NULL in that case)
+// and drop the stale reset_at so the row stops reading as exhausted and this
+// fix-up doesn't recur. Runs on read; a new observation re-populates reset_at.
+function normalizeExpiredQuotaState(db: ReturnType<typeof getDb>): void {
+ db.prepare(`
+ UPDATE provider_quota_state
+ SET remaining_value = limit_value,
+ reset_at = NULL,
+ updated_at = datetime('now')
+ WHERE reset_at IS NOT NULL
+ AND julianday(reset_at) < julianday('now')
+ `).run();
+}
+
export function getQuotaStateForKeys(): QuotaObservationView[] {
let db;
try {
@@ -406,6 +416,7 @@ export function getQuotaStateForKeys(): QuotaObservationView[] {
} catch {
return [];
}
+ normalizeExpiredQuotaState(db);
return db.prepare(`
WITH latest AS (
SELECT
diff --git a/server/src/services/router.ts b/server/src/services/router.ts
index 91f5f371..37329d4b 100644
--- a/server/src/services/router.ts
+++ b/server/src/services/router.ts
@@ -136,6 +136,29 @@ export interface RouteResult {
tpdLimit: number | null;
}
+// ── Routing token estimate: cap the reserved OUTPUT, not the full max_tokens ──
+// A client can request a huge max_tokens (e.g. 32000) it will never actually
+// emit. Reserving that full amount against every model's context window and TPM
+// budget falsely excludes the entire free pool (TPM 6k-30k) and returns a bogus
+// "all models exhausted" 429 with ZERO upstream calls (#470). Providers meter
+// ACTUAL tokens, so under-reserving only risks an upstream 429/413 the retry
+// loop already handles, whereas over-reserving starves routing. For routing and
+// the context-window / TPM filters we therefore reserve at most this many output
+// tokens; the INPUT estimate is still counted in full so a genuinely large
+// prompt still (correctly) skips a too-small model.
+export const OUTPUT_RESERVE_CAP = 2000;
+
+/**
+ * Output tokens to reserve for routing/filter purposes: the requested max_tokens
+ * clamped to OUTPUT_RESERVE_CAP (default 1000 when the client omitted it, matching
+ * the historical fallback). Callers add this to their INPUT estimate before
+ * calling routeRequest / routePinnedModel.
+ */
+export function routingReserveTokens(requestedMaxTokens: number | null | undefined): number {
+ const requested = requestedMaxTokens != null && requestedMaxTokens > 0 ? requestedMaxTokens : 1000;
+ return Math.min(requested, OUTPUT_RESERVE_CAP);
+}
+
// Round-robin index per platform
const roundRobinIndex = new Map<string, number>();
@@ -388,12 +411,26 @@ interface ScoredEntry {
score: number;
}
+// Enabled + healthy/unknown key count per platform, for pooled-budget scaling.
+// This is the SAME filter both /api/fallback endpoints use (issue #456): the
+// monthly budget is a PER-KEY free-tier allowance, so N usable keys pool N× the
+// capacity. `monthlyUsedTokens` is already summed across all keys, so budget
+// must scale to match or the headroom guardrail damps a multi-key model to the
+// floor after just one account's worth of tokens.
+function usableKeyCountsByPlatform(db: Db): Map<string, number> {
+ const rows = db.prepare(
+ "SELECT platform, COUNT(*) AS count FROM api_keys WHERE enabled = 1 AND status IN ('healthy', 'unknown') GROUP BY platform"
+ ).all() as { platform: string; count: number }[];
+ return new Map(rows.map(r => [r.platform, r.count]));
+}
+
function scoreChainEntry(
entry: ChainRow,
weights: RoutingWeights,
intelMin: number,
intelMax: number,
sampled: boolean,
+ keyCounts: Map<string, number>,
): ScoredEntry {
const stats = statsCache?.get(`${entry.platform}:${entry.model_id}`);
const successes = stats?.successes ?? 0;
@@ -412,7 +449,10 @@ function scoreChainEntry(
intelligenceComposite(entry.size_label, entry.intelligence_rank), intelMin, intelMax,
);
- const budget = parseBudget(entry.monthly_token_budget);
+ // Scale the per-key monthly budget by the usable key count for this platform,
+ // matching the pooled `monthlyUsedTokens` aggregate (#456). Math.max(1, …) so a
+ // model whose platform currently has no usable key isn't handed a 0 budget.
+ const budget = parseBudget(entry.monthly_token_budget) * Math.max(1, keyCounts.get(entry.platform) ?? 1);
const headroom = headroomFactor(stats?.monthlyUsedTokens ?? 0, budget);
const rl = rateLimitFactor(getPenalty(entry.model_db_id));
@@ -446,9 +486,10 @@ function orderChain(chain: ChainRow[], strategy: RoutingStrategy, sampled = true
const composites = chain.map(e => intelligenceComposite(e.size_label, e.intelligence_rank));
const intelMin = composites.length ? Math.min(...composites) : 0;
const intelMax = composites.length ? Math.max(...composites) : 0;
+ const keyCounts = usableKeyCountsByPlatform(getDb());
return chain
- .map(e => ({ e, s: scoreChainEntry(e, weights, intelMin, intelMax, sampled).score }))
+ .map(e => ({ e, s: scoreChainEntry(e, weights, intelMin, intelMax, sampled, keyCounts).score }))
// Higher score first; manual priority breaks ties so the chain still matters.
.sort((a, b) => b.s - a.s || a.e.priority - b.e.priority)
.map(x => x.e);
@@ -696,6 +737,56 @@ function selectKeyForModel(entry: ChainRow, estimatedTokens: number, skipKeys?:
return null;
}
+/**
+ * Whether the model still has ANOTHER key that could serve it right now, given
+ * the key that just failed (excludingKeyId) and any keys already ruled out this
+ * request (skipKeys, in the "platform:modelId:keyId" form). Applies the same
+ * gates selectKeyForModel uses — enabled + healthy status, not on cooldown,
+ * under the provider daily cap, and under rpm/rpd/tpm/tpd — so the answer means
+ * "a real, dispatchable alternative exists".
+ *
+ * Used by the retry loops to decide whether a single key's 429 should demote the
+ * WHOLE model (the model-level 429 penalty). It should not: the per-key cooldown
+ * already isolates the failing key, so demoting the model while a sibling key can
+ * still serve it wrongly sinks a healthy model in the scorer (#454). We only
+ * record the model-level hit when this returns false — i.e. the 429 exhausted the
+ * model, not just one of its keys.
+ */
+export function hasOtherUsableKey(modelDbId: number, excludingKeyId: number, skipKeys?: Set<string>): boolean {
+ const db = getDb();
+ const m = db.prepare(`
+ SELECT platform, model_id, rpm_limit, rpd_limit, tpm_limit, tpd_limit, key_id
+ FROM models WHERE id = ?
+ `).get(modelDbId) as {
+ platform: string; model_id: string;
+ rpm_limit: number | null; rpd_limit: number | null;
+ tpm_limit: number | null; tpd_limit: number | null; key_id: number | null;
+ } | undefined;
+ if (!m) return false;
+
+ const limits = { rpm: m.rpm_limit, rpd: m.rpd_limit, tpm: m.tpm_limit, tpd: m.tpd_limit };
+ const keys = db.prepare(
+ "SELECT id FROM api_keys WHERE platform = ? AND enabled = 1 AND status IN ('healthy', 'unknown')"
+ ).all(m.platform) as { id: number }[];
+
+ for (const k of keys) {
+ if (k.id === excludingKeyId) continue;
+ // A custom model binds to exactly one endpoint key (#212); a sibling custom
+ // key cannot serve it, so it doesn't count as an alternative.
+ if (m.platform === 'custom' && m.key_id != null && k.id !== m.key_id) continue;
+ if (skipKeys?.has(`${m.platform}:${m.model_id}:${k.id}`)) continue;
+ if (isOnCooldown(m.platform, m.model_id, k.id)) continue;
+ if (!canUseProvider(m.platform, k.id)) continue;
+ if (!canMakeRequest(m.platform, m.model_id, k.id, limits)) continue;
+ // A per-minute token spike on the failed key doesn't mean a fresh key lacks
+ // headroom; a nominal 1-token probe only rules out a key already at its
+ // TPM/TPD ceiling.
+ if (!canUseTokens(m.platform, m.model_id, k.id, 1, limits)) continue;
+ return true;
+ }
+ return false;
+}
+
/**
* Fetch a single enabled model's chain row by its db id.
*/
@@ -954,18 +1045,21 @@ export function routeRequest(estimatedTokens = 1000, skipKeys?: Set<string>, pre
// Context-aware routing: skip a model whose context window can't hold the
// request, so a large prompt never selects a small-context model and burns
// a failover hop on a 413 "request too large" (#167). Only enforced when we
- // know the model's window; estimatedTokens already includes the reserved
- // output (max_tokens), so this is the total-context check the model must
- // satisfy. A 413 that slips through is still retryable downstream, and the
- // failed model is put on cooldown — so this is a fast-path, not the only
- // guard. If every model is too small, the loop falls through and the caller
- // gets the normal "all models exhausted" error rather than a wasted sweep.
+ // know the model's window; estimatedTokens is the INPUT estimate plus a
+ // CAPPED output reserve (routingReserveTokens, #470), so a huge client-set
+ // max_tokens no longer excludes the model — the input must fit, not
+ // input+full max_tokens. A 413 that slips through is still retryable
+ // downstream, and the failed model is put on cooldown — so this is a
+ // fast-path, not the only guard. If every model is too small, the loop falls
+ // through and the caller gets the normal "all models exhausted" error rather
+ // than a wasted sweep.
if (entry.context_window != null && estimatedTokens > entry.context_window) { diag.push(`${label}: context ${entry.context_window} < estimated ${estimatedTokens}`); continue; }
- // Same guard for a model with a small per-minute token budget: a single
- // request that alone exceeds tpm_limit can never fit one minute of quota and
+ // Same guard for a model with a small per-minute token budget: a request
+ // whose input alone exceeds tpm_limit can never fit one minute of quota and
// returns a guaranteed 413 (e.g. Groq gpt-oss-120b: 131k context but 8k TPM).
- // estimatedTokens already includes reserved output, mirroring the check above.
+ // estimatedTokens carries the same capped output reserve, mirroring the
+ // check above (#470).
if (entry.tpm_limit != null && estimatedTokens > entry.tpm_limit) { diag.push(`${label}: tpm_limit ${entry.tpm_limit} < estimated ${estimatedTokens}`); continue; }
// Key selection + accounting pre-checks for this one model. Returns the
@@ -1021,9 +1115,10 @@ export function getRoutingScores(): { strategy: RoutingStrategy; weights: Routin
const composites = chain.map(e => intelligenceComposite(e.size_label, e.intelligence_rank));
const intelMin = composites.length ? Math.min(...composites) : 0;
const intelMax = composites.length ? Math.max(...composites) : 0;
+ const keyCounts = usableKeyCountsByPlatform(db);
const scores: RoutingScore[] = chain.map(entry => {
- const scored = scoreChainEntry(entry, weights, intelMin, intelMax, false);
+ const scored = scoreChainEntry(entry, weights, intelMin, intelMax, false, keyCounts);
const stats = statsCache?.get(`${entry.platform}:${entry.model_id}`);
return {
modelDbId: entry.model_db_id,