7.4 billion tokens per month. 34 free LLM providers. 635 free model endpoints. All behind one /v1 endpoint, plus any custom OpenAI-compatible endpoint. Smart routing, automatic failover, encrypted keys. Personal experimentation only.
TypeScript
Quota discipline: in-flight leases, provider RPM gate, health false negatives (#598)
Tashfeenc911ded8a8fd
diff --git a/server/src/__tests__/lib/fallback-loop-lease.test.ts b/server/src/__tests__/lib/fallback-loop-lease.test.tsnew file mode 100644index 00000000..dc1f67d6--- /dev/null+++ b/server/src/__tests__/lib/fallback-loop-lease.test.ts@@ -0,0 +1,131 @@+import { describe, it, expect, beforeEach, afterEach, vi } from 'vitest';+import { initDb } from '../../db/index.js';+import { runFallbackLoop, newFallbackState } from '../../lib/fallback-loop.js';+import { acquireLease, releaseLease, resetLeases, inFlightForKey } from '../../services/ratelimit.js';+import type { RouteResult } from '../../services/router.js';++/**+ * The loop must free a route's in-flight lease on every exit path. A leak would+ * leave the key's concurrency budget permanently short (until the age-out+ * backstop), so this covers each way an iteration can end.+ */+function leasedRoute(keyId: number): RouteResult {+ const id = acquireLease('groq', 'llama-3.3-70b', keyId, 100);+ return {+ provider: {} as any,+ modelId: 'llama-3.3-70b',+ modelDbId: 1,+ apiKey: 'test-key',+ keyId,+ platform: 'groq',+ displayName: 'Llama 3.3 70B',+ rpdLimit: null,+ tpdLimit: null,+ release: () => releaseLease(id),+ };+}++function hooks(overrides: Record<string, unknown> = {}) {+ return {+ state: newFallbackState(),+ attemptLog: [],+ route: () => leasedRoute(1),+ dispatch: async () => 'done' as const,+ logFailure: () => undefined,+ onFatal: () => undefined,+ onRoutingExhausted: () => undefined,+ onExhausted: () => undefined,+ ...overrides,+ } as any;+}++describe('fallback loop releases in-flight leases', () => {+ beforeEach(() => {+ initDb(':memory:');+ resetLeases();+ vi.spyOn(console, 'log').mockImplementation(() => {});+ vi.spyOn(console, 'error').mockImplementation(() => {});+ vi.spyOn(console, 'warn').mockImplementation(() => {});+ });++ afterEach(() => {+ resetLeases();+ vi.restoreAllMocks();+ });++ it('releases after a successful dispatch', async () => {+ await runFallbackLoop(hooks());+ expect(inFlightForKey('groq', 1)).toBe(0);+ });++ it('releases after a committed stream', async () => {+ await runFallbackLoop(hooks({ dispatch: async () => 'committed' as const }));+ expect(inFlightForKey('groq', 1)).toBe(0);+ });++ it('releases every attempt when retryable failures exhaust the budget', async () => {+ let dispatched = 0;+ await runFallbackLoop(hooks({+ maxRetries: 4,+ dispatch: async () => {+ dispatched++;+ throw Object.assign(new Error('Groq API error 429: rate limit'), { status: 429 });+ },+ }));++ expect(dispatched).toBe(4);+ // Four leases were taken across the four attempts; none may survive.+ expect(inFlightForKey('groq', 1)).toBe(0);+ });++ it('releases on an auth failure that rotates the key', async () => {+ await runFallbackLoop(hooks({+ maxRetries: 2,+ dispatch: async () => {+ throw Object.assign(new Error('Groq API error 401: Invalid API Key'), { status: 401 });+ },+ }));+ expect(inFlightForKey('groq', 1)).toBe(0);+ });++ it('releases on a non-retryable fatal error', async () => {+ const onFatal = vi.fn();+ await runFallbackLoop(hooks({+ onFatal,+ dispatch: async () => { throw Object.assign(new Error('boom'), { status: 418 }); },+ }));+ expect(onFatal).toHaveBeenCalled();+ expect(inFlightForKey('groq', 1)).toBe(0);+ });++ it('releases when dispatch violates the outcome contract', async () => {+ const onFatal = vi.fn();+ await runFallbackLoop(hooks({ onFatal, dispatch: async () => 'nonsense' as any }));+ expect(onFatal).toHaveBeenCalled();+ expect(inFlightForKey('groq', 1)).toBe(0);+ });++ it('does not mask the provider error if release itself throws', async () => {+ const onFatal = vi.fn();+ const bad = {+ ...leasedRoute(1),+ release: () => { throw new Error('release exploded'); },+ };+ await runFallbackLoop(hooks({+ route: () => bad,+ onFatal,+ dispatch: async () => { throw Object.assign(new Error('real provider failure'), { status: 418 }); },+ })).catch(() => undefined);++ // onFatal must have seen the provider error, not the release error.+ expect(onFatal).toHaveBeenCalled();+ const err = onFatal.mock.calls[0]![1] as Error;+ expect(err.message).toContain('real provider failure');+ });++ it('tolerates a route with no release function', async () => {+ const noRelease = { ...leasedRoute(2) } as RouteResult;+ delete (noRelease as { release?: unknown }).release;+ await expect(runFallbackLoop(hooks({ route: () => noRelease }))).resolves.toBeUndefined();+ });+});diff --git a/server/src/__tests__/routes/proxy-auth-rotation.test.ts b/server/src/__tests__/routes/proxy-auth-rotation.test.tsindex a3900dd7..32e1d4e7 100644--- a/server/src/__tests__/routes/proxy-auth-rotation.test.ts+++ b/server/src/__tests__/routes/proxy-auth-rotation.test.ts@@ -22,8 +22,17 @@ vi.mock('../../providers/index.js', async (importOriginal) => {};});-const { mockCheckKeyHealth } = vi.hoisted(() => ({ mockCheckKeyHealth: vi.fn() }));-vi.mock('../../services/health.js', () => ({ checkKeyHealth: mockCheckKeyHealth }));+const { mockCheckKeyHealth, mockMarkKeyHealthy } = vi.hoisted(() => ({+ mockCheckKeyHealth: vi.fn(),+ mockMarkKeyHealthy: vi.fn(),+}));+// Both exports must be stubbed: recordUpstreamSuccess calls+// markKeyHealthyFromRequest, and a missing stub would throw on the success path+// and surface as a 502 rather than the 200 under test.+vi.mock('../../services/health.js', () => ({+ checkKeyHealth: mockCheckKeyHealth,+ markKeyHealthyFromRequest: mockMarkKeyHealthy,+}));const { createApp } = await import('../../app.js');const { initDb, getDb, getUnifiedApiKey } = await import('../../db/index.js');diff --git a/server/src/__tests__/services/health-error.test.ts b/server/src/__tests__/services/health-error.test.tsindex 48413990..3bc3fc80 100644--- a/server/src/__tests__/services/health-error.test.ts+++ b/server/src/__tests__/services/health-error.test.ts@@ -62,8 +62,13 @@ describe('persisted key health diagnostics', () => {validateKey.mockRejectedValue(new Error('Bearer secret-token-value-1234567890 failed at https://api.example.test/models'));const error = vi.spyOn(console, 'error').mockImplementation(() => {});- expect(await checkKeyHealth(id)).toBe('error');- expect(row(id).last_health_error).toBe('Bearer [redacted] failed at [redacted-url]');+ // Seeded as 'unknown', and a transport error preserves that rather than+ // demoting to 'error' — the diagnostic is still recorded either way.+ expect(await checkKeyHealth(id)).toBe('unknown');+ expect(row(id)).toMatchObject({+ status: 'unknown',+ last_health_error: 'Bearer [redacted] failed at [redacted-url]',+ });expect(String(error.mock.calls[0][0])).not.toContain('secret-token-value');});diff --git a/server/src/__tests__/services/health-log.test.ts b/server/src/__tests__/services/health-log.test.tsindex 56b25b7c..4c3ded55 100644--- a/server/src/__tests__/services/health-log.test.ts+++ b/server/src/__tests__/services/health-log.test.ts@@ -19,7 +19,9 @@ const { checkKeyHealth } = await import('../../services/health.js');// that drops the leading "[Health] Key N (" prefix or removes the platform/base// context will silently break attribution. These tests pin the format.-const ERROR_RE = /^\[Health\] Key (\d+) \(([^,]+), base=([^)]+)\) transport error: (.+)$/;+// The trailing "— status preserved as '<status>'" records that a transport error+// deliberately no longer demotes the key; the error message is captured up to it.+const ERROR_RE = /^\[Health\] Key (\d+) \(([^,]+), base=([^)]+)\) transport error: (.+?)(?: — status preserved as '(\w+)')?$/;beforeAll(() => {process.env.ENCRYPTION_KEY = '0'.repeat(64);@@ -61,7 +63,12 @@ describe('checkKeyHealth transport-error log format', () => {it('emits a single line including platform and base_url', async () => {const id = seedKey('openai-compat', 'https://api.example.com/v1');const status = await checkKeyHealth(id);- expect(status).toBe('error');+ // A transport error is evidence about the network, not the key: the prior+ // status is kept so the key stays routable (selectKeyForModel only accepts+ // 'healthy'/'unknown', so demoting here would silently drop its capacity).+ expect(status).toBe('healthy');+ const persisted = getDb().prepare('SELECT status FROM api_keys WHERE id = ?').get(id) as { status: string };+ expect(persisted.status).toBe('healthy');expect(consoleSpy).toHaveBeenCalledTimes(1);const line = firstLine();@@ -78,7 +85,7 @@ describe('checkKeyHealth transport-error log format', () => {it('falls back to base=default when base_url is null', async () => {const id = seedKey('nvidia', null);const status = await checkKeyHealth(id);- expect(status).toBe('error');+ expect(status).toBe('healthy');const line = firstLine();expect(line).toMatch(ERROR_RE);diff --git a/server/src/__tests__/services/health-transport-preserve.test.ts b/server/src/__tests__/services/health-transport-preserve.test.tsnew file mode 100644index 00000000..61018c2e--- /dev/null+++ b/server/src/__tests__/services/health-transport-preserve.test.ts@@ -0,0 +1,135 @@+import { describe, it, expect, beforeAll, beforeEach, vi } from 'vitest';++const validateKey = vi.fn();+vi.mock('../../providers/index.js', async (importOriginal) => {+ const actual = await importOriginal() as any;+ return {+ ...actual,+ resolveProvider: () => ({ name: 'fake', validateKey }),+ };+});++const { initDb, getDb } = await import('../../db/index.js');+const { encrypt } = await import('../../lib/crypto.js');+const { checkKeyHealth, checkAllKeys, markKeyHealthyFromRequest } = await import('../../services/health.js');++/**+ * selectKeyForModel only accepts keys with status IN ('healthy','unknown'), so+ * writing status='error' removes a key's capacity entirely. A transport error+ * (DNS, TLS, timeout, suspended laptop) says nothing about the credential, so it+ * must not demote — otherwise one blip benches every key on a provider at once.+ */+describe('transport errors do not demote a key', () => {+ let nextId = 7000;++ beforeAll(() => {+ process.env.ENCRYPTION_KEY = '0'.repeat(64);+ initDb(':memory:');+ });++ beforeEach(() => {+ validateKey.mockReset();+ vi.spyOn(console, 'error').mockImplementation(() => {});+ vi.spyOn(console, 'warn').mockImplementation(() => {});+ vi.spyOn(console, 'log').mockImplementation(() => {});+ });++ function seedKey(status: string): number {+ const id = ++nextId;+ const enc = encrypt(`health-preserve-${id}`);+ getDb().prepare(`+ INSERT INTO api_keys (id, platform, label, encrypted_key, iv, auth_tag, enabled, status)+ VALUES (?, 'mistral', ?, ?, ?, ?, 1, ?)+ `).run(id, `preserve-${id}`, enc.encrypted, enc.iv, enc.authTag, status);+ return id;+ }++ function statusOf(id: number): string {+ return (getDb().prepare('SELECT status FROM api_keys WHERE id = ?').get(id) as { status: string }).status;+ }++ it('keeps a healthy key healthy through a transport failure', async () => {+ const id = seedKey('healthy');+ validateKey.mockRejectedValue(new Error('getaddrinfo ENOTFOUND api.mistral.ai'));++ expect(await checkKeyHealth(id)).toBe('healthy');+ expect(statusOf(id)).toBe('healthy');+ });++ it('records the diagnostic and timestamp even though status is unchanged', async () => {+ const id = seedKey('healthy');+ validateKey.mockRejectedValue(new Error('socket hang up'));++ await checkKeyHealth(id);+ const row = getDb().prepare(+ 'SELECT last_health_error, last_checked_at FROM api_keys WHERE id = ?',+ ).get(id) as { last_health_error: string; last_checked_at: string | null };++ expect(row.last_health_error).toContain('socket hang up');+ expect(row.last_checked_at).not.toBeNull();+ });++ it('still demotes on a confirmed bad credential', async () => {+ const id = seedKey('healthy');+ validateKey.mockResolvedValue({ valid: false, error: 'HTTP 401 invalid key' });++ expect(await checkKeyHealth(id)).toBe('invalid');+ expect(statusOf(id)).toBe('invalid');+ });++ it('an offline burst leaves every key routable', async () => {+ const ids = [seedKey('healthy'), seedKey('healthy'), seedKey('unknown')];+ validateKey.mockRejectedValue(new Error('fetch failed'));++ await checkAllKeys();++ // Previously all three would have gone to 'error' and the router would have+ // had zero usable keys until a later pass succeeded.+ expect(ids.map(statusOf)).toEqual(['healthy', 'healthy', 'unknown']);+ });+});++describe('markKeyHealthyFromRequest', () => {+ let nextId = 8000;++ beforeAll(() => {+ process.env.ENCRYPTION_KEY = '0'.repeat(64);+ initDb(':memory:');+ });++ function seedKey(status: string): number {+ const id = ++nextId;+ const enc = encrypt(`heal-${id}`);+ getDb().prepare(`+ INSERT INTO api_keys (id, platform, label, encrypted_key, iv, auth_tag, enabled, status, last_health_error)+ VALUES (?, 'groq', ?, ?, ?, ?, 1, ?, 'stale reason')+ `).run(id, `heal-${id}`, enc.encrypted, enc.iv, enc.authTag, status);+ return id;+ }++ function row(id: number): { status: string; last_health_error: string | null } {+ return getDb().prepare('SELECT status, last_health_error FROM api_keys WHERE id = ?').get(id) as any;+ }++ it("promotes an 'error' key back to healthy and clears the stale reason", () => {+ const id = seedKey('error');+ markKeyHealthyFromRequest(id);+ expect(row(id)).toEqual({ status: 'healthy', last_health_error: null });+ });++ it("never resurrects an 'invalid' key — only a real probe clears that", () => {+ const id = seedKey('invalid');+ markKeyHealthyFromRequest(id);+ expect(row(id).status).toBe('invalid');+ });++ it('leaves an already-healthy key untouched', () => {+ const id = seedKey('healthy');+ markKeyHealthyFromRequest(id);+ expect(row(id).status).toBe('healthy');+ });++ it('is silent for an unknown key id', () => {+ expect(() => markKeyHealthyFromRequest(999_999)).not.toThrow();+ });+});diff --git a/server/src/__tests__/services/key-concurrency.test.ts b/server/src/__tests__/services/key-concurrency.test.tsnew file mode 100644index 00000000..af931640--- /dev/null+++ b/server/src/__tests__/services/key-concurrency.test.ts@@ -0,0 +1,135 @@+import { describe, it, expect, beforeEach, afterEach, vi } from 'vitest';+import { initDb } from '../../db/index.js';+import {+ acquireLease,+ releaseLease,+ resetLeases,+ inFlightForKey,+ canUseKeyConcurrency,+ getKeyConcurrencyLimit,+} from '../../services/ratelimit.js';++/**+ * In-flight leases make the window between key selection and usage recording+ * visible. They back the per-key concurrency cap, which stops parallel streams+ * from all picking one key and 429-ing each other on providers that meter+ * concurrency per credential.+ */+describe('in-flight leases', () => {+ beforeEach(() => {+ initDb(':memory:');+ resetLeases();+ });++ afterEach(() => {+ resetLeases();+ delete process.env.MAX_CONCURRENT_REQUESTS_PER_KEY;+ delete process.env.MAX_CONCURRENT_REQUESTS_PER_KEY_GROQ;+ vi.useRealTimers();+ });++ it('counts only the matching platform and key', () => {+ acquireLease('groq', 'llama-3.3-70b', 1, 100);+ acquireLease('groq', 'llama-3.1-8b', 1, 100);+ acquireLease('groq', 'llama-3.3-70b', 2, 100);+ acquireLease('cerebras', 'qwen-3-coder', 1, 100);++ expect(inFlightForKey('groq', 1)).toBe(2);+ expect(inFlightForKey('groq', 2)).toBe(1);+ expect(inFlightForKey('cerebras', 1)).toBe(1);+ expect(inFlightForKey('groq', 99)).toBe(0);+ });++ it('release frees the slot and is idempotent', () => {+ const id = acquireLease('groq', 'llama-3.3-70b', 1, 100);+ expect(inFlightForKey('groq', 1)).toBe(1);++ releaseLease(id);+ expect(inFlightForKey('groq', 1)).toBe(0);+ // Overlapping cleanup paths must not double-decrement someone else's slot.+ releaseLease(id);+ releaseLease(id);+ expect(inFlightForKey('groq', 1)).toBe(0);+ });++ it('ages out a leaked lease so a key cannot be blocked forever', () => {+ vi.useFakeTimers();+ acquireLease('groq', 'llama-3.3-70b', 1, 100);+ expect(inFlightForKey('groq', 1)).toBe(1);++ // Past LEASE_MAX_AGE_MS (2 min) — the backstop for a release that never came.+ vi.advanceTimersByTime(3 * 60 * 1000);+ expect(inFlightForKey('groq', 1)).toBe(0);+ });+});++describe('per-key concurrency cap', () => {+ beforeEach(() => {+ initDb(':memory:');+ resetLeases();+ });++ afterEach(() => {+ resetLeases();+ delete process.env.MAX_CONCURRENT_REQUESTS_PER_KEY;+ delete process.env.MAX_CONCURRENT_REQUESTS_PER_KEY_GROQ;+ });++ it('is unlimited by default — capping every provider would cost throughput', () => {+ expect(getKeyConcurrencyLimit('groq')).toBeNull();+ for (let i = 0; i < 25; i++) acquireLease('groq', 'llama-3.3-70b', 1, 100);+ expect(canUseKeyConcurrency('groq', 1)).toBe(true);+ });++ it('reads a per-platform env cap ahead of the global one', () => {+ process.env.MAX_CONCURRENT_REQUESTS_PER_KEY = '5';+ process.env.MAX_CONCURRENT_REQUESTS_PER_KEY_GROQ = '2';+ expect(getKeyConcurrencyLimit('groq')).toBe(2);+ expect(getKeyConcurrencyLimit('cerebras')).toBe(5);+ });++ it('ignores a malformed cap rather than blocking everything', () => {+ process.env.MAX_CONCURRENT_REQUESTS_PER_KEY_GROQ = 'not-a-number';+ expect(getKeyConcurrencyLimit('groq')).toBeNull();+ process.env.MAX_CONCURRENT_REQUESTS_PER_KEY_GROQ = '0';+ expect(getKeyConcurrencyLimit('groq')).toBeNull();+ });++ it('blocks a key at its cap and frees it on release', () => {+ process.env.MAX_CONCURRENT_REQUESTS_PER_KEY_GROQ = '1';++ expect(canUseKeyConcurrency('groq', 1)).toBe(true);+ const id = acquireLease('groq', 'llama-3.3-70b', 1, 100);+ expect(canUseKeyConcurrency('groq', 1)).toBe(false);++ releaseLease(id);+ expect(canUseKeyConcurrency('groq', 1)).toBe(true);+ });++ it('caps each key independently, so a sibling key stays available', () => {+ process.env.MAX_CONCURRENT_REQUESTS_PER_KEY_GROQ = '1';+ acquireLease('groq', 'llama-3.3-70b', 1, 100);++ // This is the point: the request fails over to key 2 instead of queueing+ // behind key 1 and collecting a 429.+ expect(canUseKeyConcurrency('groq', 1)).toBe(false);+ expect(canUseKeyConcurrency('groq', 2)).toBe(true);+ });++ it('counts a key busy on one model against a request for another', () => {+ process.env.MAX_CONCURRENT_REQUESTS_PER_KEY_GROQ = '1';+ acquireLease('groq', 'llama-3.3-70b', 1, 100);+ // Concurrency is metered per credential, not per model.+ expect(canUseKeyConcurrency('groq', 1)).toBe(false);+ });++ it('allows up to the cap and no further', () => {+ process.env.MAX_CONCURRENT_REQUESTS_PER_KEY_GROQ = '3';+ const ids = [0, 1, 2].map(() => acquireLease('groq', 'llama-3.3-70b', 1, 100));+ expect(inFlightForKey('groq', 1)).toBe(3);+ expect(canUseKeyConcurrency('groq', 1)).toBe(false);++ releaseLease(ids[0]!);+ expect(canUseKeyConcurrency('groq', 1)).toBe(true);+ });+});diff --git a/server/src/__tests__/services/provider-minute-cap.test.ts b/server/src/__tests__/services/provider-minute-cap.test.tsnew file mode 100644index 00000000..0bb1b4fb--- /dev/null+++ b/server/src/__tests__/services/provider-minute-cap.test.ts@@ -0,0 +1,92 @@+import { describe, it, expect, beforeEach, afterEach } from 'vitest';+import { initDb, getDb } from '../../db/index.js';+import {+ canUseProviderMinute,+ providerMinuteRequestCount,+ getProviderMinuteRequestCap,+} from '../../services/ratelimit.js';++/**+ * Account-wide per-minute caps. The per-model rpm_limit cannot express these:+ * NVIDIA NIM meters ~40 req/min across the whole account, so every model on it+ * draws from one bucket, and a model row with rpm_limit NULL isn't pre-throttled+ * at all. Without this gate the router keeps dispatching and eats real 429s.+ */+function seedRequests(platform: string, modelId: string, keyId: number, count: number, atMs: number) {+ const ins = getDb().prepare(`+ INSERT INTO rate_limit_usage (platform, model_id, key_id, kind, tokens, created_at_ms)+ VALUES (?, ?, ?, 'request', 0, ?)+ `);+ for (let i = 0; i < count; i++) ins.run(platform, modelId, keyId, atMs);+}++describe('provider-wide per-minute request cap', () => {+ let keyId: number;++ beforeEach(() => {+ initDb(':memory:');+ keyId = Math.floor(Math.random() * 1_000_000);+ });++ afterEach(() => {+ delete process.env.PROVIDER_MINUTE_REQUEST_CAP_NVIDIA;+ delete process.env.PROVIDER_MINUTE_REQUEST_CAP_GROQ;+ });++ it('ships a default cap for nvidia and none for a provider without one', () => {+ expect(getProviderMinuteRequestCap('nvidia')).toBe(40);+ expect(getProviderMinuteRequestCap('groq')).toBeNull();+ });++ it('honours an env override and treats 0 as disabled', () => {+ process.env.PROVIDER_MINUTE_REQUEST_CAP_NVIDIA = '5';+ expect(getProviderMinuteRequestCap('nvidia')).toBe(5);+ process.env.PROVIDER_MINUTE_REQUEST_CAP_NVIDIA = '0';+ expect(getProviderMinuteRequestCap('nvidia')).toBeNull();+ });++ it('sums usage across every model on the same account+key', () => {+ const now = Date.now();+ // The whole point: three different models, one shared budget.+ seedRequests('nvidia', 'glm-4.7', keyId, 10, now - 5_000);+ seedRequests('nvidia', 'minimax-m3', keyId, 10, now - 4_000);+ seedRequests('nvidia', 'deepseek-v4', keyId, 10, now - 3_000);++ expect(providerMinuteRequestCount('nvidia', keyId, now)).toBe(30);+ });++ it('blocks once the shared budget is spent, even for an untouched model', () => {+ process.env.PROVIDER_MINUTE_REQUEST_CAP_NVIDIA = '40';+ const now = Date.now();+ seedRequests('nvidia', 'glm-4.7', keyId, 40, now - 10_000);++ // deepseek-v4 has made zero requests of its own and may well have a NULL+ // rpm_limit, but the account budget is gone.+ expect(canUseProviderMinute('nvidia', keyId, now)).toBe(false);+ });++ it('allows again once the minute window rolls past', () => {+ process.env.PROVIDER_MINUTE_REQUEST_CAP_NVIDIA = '40';+ const now = Date.now();+ seedRequests('nvidia', 'glm-4.7', keyId, 40, now - 61_000);++ expect(providerMinuteRequestCount('nvidia', keyId, now)).toBe(0);+ expect(canUseProviderMinute('nvidia', keyId, now)).toBe(true);+ });++ it('does not gate a provider with no configured cap', () => {+ const now = Date.now();+ seedRequests('groq', 'llama-3.3-70b', keyId, 500, now - 1_000);+ expect(canUseProviderMinute('groq', keyId, now)).toBe(true);+ });++ it('keeps separate budgets per key on the same platform', () => {+ process.env.PROVIDER_MINUTE_REQUEST_CAP_NVIDIA = '40';+ const now = Date.now();+ const other = keyId + 1;+ seedRequests('nvidia', 'glm-4.7', keyId, 40, now - 5_000);++ expect(canUseProviderMinute('nvidia', keyId, now)).toBe(false);+ expect(canUseProviderMinute('nvidia', other, now)).toBe(true);+ });+});diff --git a/server/src/__tests__/services/provisional-usage.test.ts b/server/src/__tests__/services/provisional-usage.test.tsnew file mode 100644index 00000000..f6f70779--- /dev/null+++ b/server/src/__tests__/services/provisional-usage.test.ts@@ -0,0 +1,107 @@+import { describe, it, expect, beforeEach, afterEach } from 'vitest';+import { initDb, getDb } from '../../db/index.js';+import {+ canMakeRequest,+ canUseTokens,+ canUseProvider,+ canUseProviderMinute,+ acquireLease,+ releaseLease,+ resetLeases,+ recordRequest,+} from '../../services/ratelimit.js';++/**+ * The check-then-act race: routeRequest is synchronous, but usage is only written+ * after the awaited provider call. Without provisional accounting, N concurrent+ * requests all read the same pre-check, all pass, and collectively overshoot the+ * limit — which is exactly how a free tier gets 429ed by its own client.+ */+describe('in-flight leases count against the quota gates', () => {+ let keyId: number;++ beforeEach(() => {+ initDb(':memory:');+ resetLeases();+ keyId = Math.floor(Math.random() * 1_000_000);+ });++ afterEach(() => {+ resetLeases();+ delete process.env.PROVIDER_MINUTE_REQUEST_CAP_NVIDIA;+ });++ const limits = (rpm: number | null) => ({ rpm, rpd: null, tpm: null, tpd: null });++ it('a single in-flight request consumes an rpm of 1', () => {+ expect(canMakeRequest('groq', 'm1', keyId, limits(1))).toBe(true);++ // Selection happened; nothing recorded yet because the call is still in flight.+ acquireLease('groq', 'm1', keyId, 100);++ // Before this change the second caller saw an unspent budget and dispatched.+ expect(canMakeRequest('groq', 'm1', keyId, limits(1))).toBe(false);+ });++ it('frees the budget again once the lease is released', () => {+ const id = acquireLease('groq', 'm1', keyId, 100);+ expect(canMakeRequest('groq', 'm1', keyId, limits(1))).toBe(false);+ releaseLease(id);+ expect(canMakeRequest('groq', 'm1', keyId, limits(1))).toBe(true);+ });++ it('stacks provisional on top of recorded usage', () => {+ recordRequest('groq', 'm1', keyId); // 1 recorded+ acquireLease('groq', 'm1', keyId, 100); // 1 in flight+ expect(canMakeRequest('groq', 'm1', keyId, limits(3))).toBe(true); // 2 of 3+ acquireLease('groq', 'm1', keyId, 100); // 2 in flight → 3 of 3+ expect(canMakeRequest('groq', 'm1', keyId, limits(3))).toBe(false);+ });++ it('scopes provisional requests to the same model and key', () => {+ acquireLease('groq', 'other-model', keyId, 100);+ acquireLease('groq', 'm1', keyId + 1, 100);+ // Neither lease belongs to (m1, keyId), so its own budget is untouched.+ expect(canMakeRequest('groq', 'm1', keyId, limits(1))).toBe(true);+ });++ it('counts promised tokens against tpm', () => {+ const tokenLimits = { tpm: 1000, tpd: null };+ expect(canUseTokens('groq', 'm1', keyId, 600, tokenLimits)).toBe(true);++ acquireLease('groq', 'm1', keyId, 600); // 600 already promised+ // 600 in flight + 600 requested exceeds 1000.+ expect(canUseTokens('groq', 'm1', keyId, 600, tokenLimits)).toBe(false);+ // A smaller request still fits.+ expect(canUseTokens('groq', 'm1', keyId, 300, tokenLimits)).toBe(true);+ });++ it('counts in-flight requests against the provider-wide minute cap', () => {+ process.env.PROVIDER_MINUTE_REQUEST_CAP_NVIDIA = '2';+ // Different models, one shared account budget — the whole point of that gate.+ acquireLease('nvidia', 'glm-4.7', keyId, 100);+ acquireLease('nvidia', 'minimax-m3', keyId, 100);+ expect(canUseProviderMinute('nvidia', keyId)).toBe(false);+ });++ it('counts in-flight requests against the provider-wide daily cap', () => {+ const now = Date.now();+ const ins = getDb().prepare(`+ INSERT INTO rate_limit_usage (platform, model_id, key_id, kind, tokens, created_at_ms)+ VALUES ('openrouter', 'm1', ?, 'request', 0, ?)+ `);+ // OpenRouter ships a 1000/day default; sit one short of it.+ for (let i = 0; i < 999; i++) ins.run(keyId, now - 1000);++ expect(canUseProvider('openrouter', keyId, now)).toBe(true);+ acquireLease('openrouter', 'm1', keyId, 100, now);+ expect(canUseProvider('openrouter', keyId, now)).toBe(false);+ });++ it('ignores an aged-out lease so a leak cannot block the gate forever', () => {+ const now = Date.now();+ // Older than LEASE_MAX_AGE_MS (2 min).+ acquireLease('groq', 'm1', keyId, 100, now - 5 * 60 * 1000);+ expect(canMakeRequest('groq', 'm1', keyId, limits(1))).toBe(true);+ });+});diff --git a/server/src/lib/fallback-loop.ts b/server/src/lib/fallback-loop.tsindex c0fbc17b..39d3cc33 100644--- a/server/src/lib/fallback-loop.ts+++ b/server/src/lib/fallback-loop.ts@@ -38,7 +38,7 @@ import {isProviderDegradedError,} from './error-classify.js';import { sanitizeProviderErrorMessage } from './error-redaction.js';-import { checkKeyHealth } from '../services/health.js';+import { checkKeyHealth, markKeyHealthyFromRequest } from '../services/health.js';import { getSetting } from '../db/index.js';import { newBreaker, recordBreakerFailure } from './guardrails.js';@@ -213,6 +213,10 @@ export function recordUpstreamSuccess(route: RouteResult, rateLimitTokens: numberecordRequest(route.platform, route.modelId, route.keyId);recordTokens(route.platform, route.modelId, route.keyId, rateLimitTokens);recordSuccess(route.modelDbId);+ // A served request is the strongest possible evidence the key works, so clear+ // any stale 'error' status left by an earlier transport blip instead of waiting+ // for the next health pass to make the key routable again.+ markKeyHealthyFromRequest(route.keyId);}// ── Attempt trail ─────────────────────────────────────────────────────────────@@ -556,6 +560,13 @@ export async function runFallbackLoop(hooks: FallbackHooks): Promise<void> {return;}+ // Everything from here to the end of the iteration runs inside a finally that+ // frees the route's in-flight lease. Every exit — success, auth rotation,+ // retryable continue, fatal, breaker trip, contract violation — passes through+ // it, so no path can leak a lease and leave the key's concurrency budget short.+ // Success accounting happens inside dispatch, so the persisted counters are+ // already written by the time the provisional lease goes away.+ try {let outcome: DispatchOutcome;try {outcome = await hooks.dispatch(route, attempt);@@ -603,6 +614,9 @@ export async function runFallbackLoop(hooks: FallbackHooks): Promise<void> {hooks.onFatal(route, violation, attempt);}return;+ } finally {+ route.release?.();+ }}hooks.onExhausted(diff --git a/server/src/services/fusion.ts b/server/src/services/fusion.tsindex a4f601ee..d1f0f1a2 100644--- a/server/src/services/fusion.ts+++ b/server/src/services/fusion.ts@@ -270,6 +270,11 @@ async function runModelCall(}// Non-retryable (auth, validation) — this slot/judge is done.break;+ } finally {+ // Panel slots run concurrently, so a leaked lease here would starve the+ // rest of the panel of its own keys' concurrency budget. The route object+ // stays usable as a data carrier after release.+ route.release?.();}}@@ -353,6 +358,8 @@ async function runJudgeStreaming(continue;}break;+ } finally {+ route.release?.();}}return { ok: false, error: lastError ?? 'no available key for judge' };diff --git a/server/src/services/health.ts b/server/src/services/health.tsindex 39197486..79833b26 100644--- a/server/src/services/health.ts+++ b/server/src/services/health.ts@@ -8,6 +8,18 @@ import { sanitizeProviderErrorMessage } from '../lib/error-redaction.js';const CHECK_INTERVAL_MS = 5 * 60 * 1000; // 5 minutesconst CONSECUTIVE_FAILURES_TO_DISABLE = 3;+const DEFAULT_HEALTH_CHECK_CONCURRENCY = 8;++/** Parallel key probes per health pass. Tunable because the right number depends+ * on how many keys share one provider; 0 or a bad value falls back to the default. */+function getHealthCheckConcurrency(): number {+ const raw = process.env.HEALTH_CHECK_CONCURRENCY;+ if (raw !== undefined && raw.trim() !== '') {+ const n = Number(raw);+ if (Number.isInteger(n) && n > 0) return n;+ }+ return DEFAULT_HEALTH_CHECK_CONCURRENCY;+}// Track consecutive failures per keyconst failureCount = new Map<number, number>();@@ -75,11 +87,38 @@ export async function checkKeyHealth(keyId: number): Promise<KeyStatus> {const lastError = sanitizeProviderErrorMessage(err?.message ?? err);console.error(`[Health] Key ${keyId} (${row.platform}, base=${row.base_url ?? 'default'}) ` +- `transport error: ${lastError}`,+ `transport error: ${lastError} — status preserved as '${row.status}'`,);- db.prepare("UPDATE api_keys SET status = ?, last_health_error = ?, last_checked_at = datetime('now') WHERE id = ?")- .run('error', lastError, keyId);- return 'error';+ // Do NOT write status='error'. selectKeyForModel only considers keys with+ // status IN ('healthy','unknown'), so demoting here silently removes the+ // key's capacity for up to a full check interval — and a transport error is+ // evidence about the network, not about the key. One flaky DNS lookup, a+ // laptop suspended for thirty seconds, or a provider edge outage would take+ // every key on that provider out of rotation at once. Record the diagnostic+ // and the timestamp; leave the verdict to a probe that actually reached the+ // provider. Confirmed 401/403 (the isValid=false path above) still demotes.+ db.prepare("UPDATE api_keys SET last_health_error = ?, last_checked_at = datetime('now') WHERE id = ?")+ .run(lastError, keyId);+ return row.status as KeyStatus;+ }+}++/**+ * Promote a key out of 'error' after it successfully served a live request.+ *+ * Serving traffic is stronger evidence than any probe, so a key stuck at 'error'+ * from an earlier transport blip should not have to wait for the next health pass+ * to become routable again. Deliberately narrow: 'invalid' means a provider+ * confirmed the credential is bad, and only a real validateKey pass clears that.+ */+export function markKeyHealthyFromRequest(keyId: number): void {+ try {+ getDb()+ .prepare("UPDATE api_keys SET status = 'healthy', last_health_error = NULL WHERE id = ? AND status = 'error'")+ .run(keyId);+ failureCount.delete(keyId);+ } catch {+ // Never let health bookkeeping break a request that already succeeded.}}@@ -99,9 +138,26 @@ export function checkAllKeys(): Promise<void> {console.log(`[Health] Checking ${keys.length} keys...`);- for (const key of keys) {- await checkKeyHealth(key.id);- }+ // Bounded worker pool rather than a sequential await. validateKey allows up+ // to 30s per key (some provider /models endpoints are genuinely that slow),+ // so a serial pass over a large key fleet can outlast the 5-minute interval+ // it is scheduled on and leave the dashboard showing stale statuses. The cap+ // keeps us from opening one socket per key against the same provider.+ const concurrency = getHealthCheckConcurrency();+ let cursor = 0;+ const workers = Array.from({ length: Math.min(concurrency, keys.length) }, async () => {+ while (cursor < keys.length) {+ const key = keys[cursor++]!;+ try {+ await checkKeyHealth(key.id);+ } catch (err) {+ // checkKeyHealth handles its own errors; this is a backstop so one+ // rejection cannot abandon the rest of the pass.+ console.error(`[Health] Key ${key.id} check threw:`, err);+ }+ }+ });+ await Promise.all(workers);console.log(`[Health] Check complete.`);})().finally(() => {diff --git a/server/src/services/ratelimit.ts b/server/src/services/ratelimit.tsindex aa62e4db..b7690f26 100644--- a/server/src/services/ratelimit.ts+++ b/server/src/services/ratelimit.ts@@ -45,6 +45,152 @@ function withDb<T>(fn: (db: RateLimitDb) => T): T | undefined {}}+// ── In-flight leases ──────────────────────────────────────────────────────────+// Usage is only recorded *after* an attempt succeeds, so between key selection+// and that write the router has no idea a request is already in the air. A lease+// makes that window visible. Today it backs the per-key concurrency cap; it is+// also the hook for counting provisional usage against the quota gates, which is+// what closes the check-then-act race under parallel load.+//+// Leases live in memory only: they describe requests this process currently has+// open, so there is nothing meaningful to persist and a restart correctly starts+// from zero.++interface Lease {+ platform: string;+ modelId: string;+ keyId: number;+ tokens: number;+ createdAt: number;+}++const leases = new Map<number, Lease>();+let nextLeaseId = 1;++// A lease should always be released explicitly by the fallback loop's finally+// block. This bound is the backstop for a path that somehow doesn't: without it+// a single leaked lease would count against a key's concurrency budget forever.+// Comfortably longer than the per-attempt provider timeout.+const LEASE_MAX_AGE_MS = 2 * MINUTE;++function pruneLeases(now: number): void {+ if (leases.size === 0) return;+ for (const [id, lease] of leases) {+ if (now - lease.createdAt > LEASE_MAX_AGE_MS) leases.delete(id);+ }+}++/**+ * Concurrent in-flight requests allowed per key, or null for unlimited.+ *+ * Some free tiers meter concurrency per credential rather than per minute, so+ * parallel streams on one key mostly 429 each other. But most do not, and+ * capping by default would serialise every provider and cost real throughput —+ * so this is opt-in. There is deliberately no built-in per-platform table: the+ * providers that behave this way are not documented well enough to assert a+ * number for them here, and a wrong default is worse than none.+ *+ * `MAX_CONCURRENT_REQUESTS_PER_KEY_<PLATFORM>` sets it for one platform;+ * `MAX_CONCURRENT_REQUESTS_PER_KEY` sets a fallback for all of them.+ */+export function getKeyConcurrencyLimit(platform: string): number | null {+ const raw = process.env[`MAX_CONCURRENT_REQUESTS_PER_KEY_${platform.toUpperCase()}`]+ ?? process.env.MAX_CONCURRENT_REQUESTS_PER_KEY;+ if (raw !== undefined && raw.trim() !== '') {+ const n = Number(raw);+ if (Number.isInteger(n) && n > 0) return n;+ }+ return null;+}++/** Requests this process currently has in flight against one platform+key. */+export function inFlightForKey(platform: string, keyId: number, now = Date.now()): number {+ pruneLeases(now);+ let count = 0;+ for (const lease of leases.values()) {+ if (lease.platform === platform && lease.keyId === keyId) count++;+ }+ return count;+}++/** False when this key already has its allowed number of requests in the air.+ * Always true when no cap is configured, which is the default. */+export function canUseKeyConcurrency(platform: string, keyId: number, now = Date.now()): boolean {+ const limit = getKeyConcurrencyLimit(platform);+ if (limit === null) return true;+ return inFlightForKey(platform, keyId, now) < limit;+}++/** Take a lease for an attempt about to be dispatched. Returns the id to release. */+export function acquireLease(+ platform: string,+ modelId: string,+ keyId: number,+ tokens: number,+ now = Date.now(),+): number {+ pruneLeases(now);+ const id = nextLeaseId++;+ leases.set(id, { platform, modelId, keyId, tokens, createdAt: now });+ return id;+}++/** Release a lease. Idempotent, so a double release from overlapping cleanup+ * paths is harmless. */+export function releaseLease(leaseId: number): void {+ leases.delete(leaseId);+}++/** Test seam: drop all leases. */+export function resetLeases(): void {+ leases.clear();+}++// ── Provisional usage ─────────────────────────────────────────────────────────+// Every gate below adds in-flight leases on top of the recorded counters. This is+// what closes the check-then-act race: routeRequest is synchronous but usage is+// only written after the awaited provider call, so N concurrent requests all read+// the same pre-check and every one of them passes — then they collectively blow+// through the limit and collect real 429s. Counting leases makes the second+// caller see the first one's request.+//+// A lease is released just after the success write, so for a brief instant a+// completed request is counted twice. That errs toward under-dispatching by one,+// which is the safe direction for a free tier, and lasts microseconds.++function provisionalRequests(platform: string, modelId: string, keyId: number, now: number): number {+ pruneLeases(now);+ let count = 0;+ for (const lease of leases.values()) {+ if (lease.platform === platform && lease.modelId === modelId && lease.keyId === keyId) count++;+ }+ return count;+}++function provisionalTokens(platform: string, modelId: string, keyId: number, now: number): number {+ pruneLeases(now);+ let total = 0;+ for (const lease of leases.values()) {+ if (lease.platform === platform && lease.modelId === modelId && lease.keyId === keyId) total += lease.tokens;+ }+ return total;+}++/** In-flight requests for a provider account+key across every model — the+ * provider-wide analogue, for the account-level gates. */+function provisionalProviderRequests(platform: string, keyId: number, now: number): number {+ return inFlightForKey(platform, keyId, now);+}++function provisionalProviderTokens(platform: string, keyId: number, now: number): number {+ pruneLeases(now);+ let total = 0;+ for (const lease of leases.values()) {+ if (lease.platform === platform && lease.keyId === keyId) total += lease.tokens;+ }+ return total;+}+function recordUsage(platform: string,modelId: string,@@ -149,13 +295,16 @@ export function canMakeRequest(limits: { rpm: number | null; rpd: number | null; tpm: number | null; tpd: number | null },): boolean {const now = Date.now();+ // In-flight requests count against both windows: a lease means the request is+ // happening now, so it belongs to this minute and to today.+ const inFlight = provisionalRequests(platform, modelId, keyId, now);if (limits.rpm !== null) {- if (requestCount(platform, modelId, keyId, MINUTE, now) >= limits.rpm) return false;+ if (requestCount(platform, modelId, keyId, MINUTE, now) + inFlight >= limits.rpm) return false;}if (limits.rpd !== null) {- if (requestCount(platform, modelId, keyId, DAY, now) >= limits.rpd) return false;+ if (requestCount(platform, modelId, keyId, DAY, now) + inFlight >= limits.rpd) return false;}return true;@@ -169,15 +318,17 @@ export function canUseTokens(limits: { tpm: number | null; tpd: number | null },): boolean {const now = Date.now();+ // Tokens already promised to in-flight attempts on this model+key.+ const inFlight = provisionalTokens(platform, modelId, keyId, now);if (limits.tpm !== null) {const used = tokenCount(platform, modelId, keyId, MINUTE, now);- if (used + estimatedTokens > limits.tpm) return false;+ if (used + inFlight + estimatedTokens > limits.tpm) return false;}if (limits.tpd !== null) {const used = tokenCount(platform, modelId, keyId, DAY, now);- if (used + estimatedTokens > limits.tpd) return false;+ if (used + inFlight + estimatedTokens > limits.tpd) return false;}return true;@@ -204,6 +355,16 @@ const DEFAULT_PROVIDER_DAILY_TOKEN_CAPS: Record<string, number> = {navy: 150_000,};+// Per-minute caps that apply to the whole provider account rather than one model.+// The per-model rpm_limit cannot express these: NVIDIA NIM meters ~40 requests a+// minute across the entire account, so glm-4.7, minimax-m3 and deepseek all draw+// from one bucket. Without an account-level gate the router sees each model's own+// rpm as unspent — and a model row whose rpm_limit is NULL escapes pre-throttling+// entirely — so it keeps dispatching and eats real 429s.+const DEFAULT_PROVIDER_MINUTE_REQUEST_CAPS: Record<string, number> = {+ nvidia: 40,+};+export function getProviderDailyRequestCap(platform: string): number | null {const raw = process.env[`PROVIDER_DAILY_REQUEST_CAP_${platform.toUpperCase()}`];if (raw !== undefined && raw.trim() !== '') {@@ -213,6 +374,17 @@ export function getProviderDailyRequestCap(platform: string): number | null {return DEFAULT_PROVIDER_DAILY_REQUEST_CAPS[platform] ?? null;}+/** Account-wide requests-per-minute cap, or null when the provider has none.+ * `PROVIDER_MINUTE_REQUEST_CAP_<PLATFORM>=0` disables the gate for that platform. */+export function getProviderMinuteRequestCap(platform: string): number | null {+ const raw = process.env[`PROVIDER_MINUTE_REQUEST_CAP_${platform.toUpperCase()}`];+ if (raw !== undefined && raw.trim() !== '') {+ const n = Number(raw);+ if (Number.isFinite(n) && n >= 0) return n === 0 ? null : n;+ }+ return DEFAULT_PROVIDER_MINUTE_REQUEST_CAPS[platform] ?? null;+}+export function getProviderDailyTokenCap(platform: string): number | null {const raw = process.env[`PROVIDER_DAILY_TOKEN_CAP_${platform.toUpperCase()}`];if (raw !== undefined && raw.trim() !== '') {@@ -266,7 +438,32 @@ export function providerDailyRequestCount(platform: string, keyId: number, now =export function canUseProvider(platform: string, keyId: number, now = Date.now()): boolean {const cap = getProviderDailyRequestCap(platform);if (cap === null) return true;- return providerDailyRequestCount(platform, keyId, now) < cap;+ const used = providerDailyRequestCount(platform, keyId, now) + provisionalProviderRequests(platform, keyId, now);+ return used < cap;+}++/** Requests in the last minute for a provider account+key, across every model. */+export function providerMinuteRequestCount(platform: string, keyId: number, now = Date.now()): number {+ const persisted = countPersistedProviderRequests(platform, keyId, MINUTE, now);+ if (persisted !== undefined) return persisted;+ // DB-unavailable fallback: sum the per-model rpm windows for this platform+key.+ let total = 0;+ for (const [key, w] of windows) {+ if (key.startsWith(`${platform}:`) && key.endsWith(`:${keyId}:rpm`)) {+ total += pruneTimestamps(w.timestamps, MINUTE, now).length;+ }+ }+ return total;+}++// False when this provider account+key has spent its shared per-minute request+// budget, so the router skips every model on that provider rather than learning+// the limit again from a 429 on each one in turn.+export function canUseProviderMinute(platform: string, keyId: number, now = Date.now()): boolean {+ const cap = getProviderMinuteRequestCap(platform);+ if (cap === null) return true;+ const used = providerMinuteRequestCount(platform, keyId, now) + provisionalProviderRequests(platform, keyId, now);+ return used < cap;}type ModelQuotaRow = { tpd_limit: number | null; monthly_token_budget: string | null };@@ -365,7 +562,8 @@ export function canUseProviderTokens(): boolean {const cap = getProviderDailyTokenCap(platform);if (cap === null) return true;- const used = providerDailyTokenCount(platform, keyId, now);+ const used = providerDailyTokenCount(platform, keyId, now)+ + providerBilledTokens(platform, modelId, provisionalProviderTokens(platform, keyId, now));return used + providerBilledTokens(platform, modelId, estimatedTokens) <= cap;}diff --git a/server/src/services/router.ts b/server/src/services/router.tsindex e137171f..4a9b2cfa 100644--- a/server/src/services/router.ts+++ b/server/src/services/router.ts@@ -6,7 +6,11 @@ import {canUseTokens,isOnCooldown,canUseProvider,+ canUseProviderMinute,canUseProviderTokens,+ canUseKeyConcurrency,+ acquireLease,+ releaseLease,getSoonestCooldownExpiry,} from './ratelimit.js';import {@@ -145,6 +149,20 @@ export interface RouteResult {// exhaustion (escalate the cooldown) from a transient per-minute spike.rpdLimit: number | null;tpdLimit: number | null;+ /**+ * Frees the in-flight lease taken when this route was selected. Idempotent.+ *+ * Callers should invoke it once the attempt is finished, however it finished —+ * the shared fallback loop does so from a `finally` so no exit path can leak.+ *+ * Optional, and every call site uses `release?.()`, for a specific reason: the+ * invocation sits in a `finally`, and a TypeError thrown there would *replace*+ * the in-flight provider exception with a useless one, turning a diagnosable+ * 429 into a mystery 500. A route that arrives without it (a test double, a+ * future construction path) should quietly fall back to the lease ageing out+ * rather than destroy the error being propagated.+ */+ release?: () => void;}// ── Routing token estimate: cap the reserved OUTPUT, not the full max_tokens ──@@ -707,6 +725,14 @@ function selectKeyForModel(entry: ChainRow, estimatedTokens: number, skipKeys?:if (isOnCooldown(entry.platform, entry.model_id, key.id)) { note('cooldown'); continue; }if (!canUseProvider(entry.platform, key.id)) { note('provider-daily-cap'); continue; }+ // Account-wide per-minute budget, checked before the per-model gates: a model+ // with a NULL rpm_limit would otherwise sail past them and spend a budget its+ // siblings share.+ if (!canUseProviderMinute(entry.platform, key.id)) { note('provider-minute-cap'); continue; }+ // Skip a key that already has its allowed requests in the air. Without this,+ // parallel streams all pick the same key and 429 each other on providers that+ // meter concurrency per credential.+ if (!canUseKeyConcurrency(entry.platform, key.id)) { note('key-concurrency'); continue; }if (!canMakeRequest(entry.platform, entry.model_id, key.id, limits)) { note('rpm/rpd-limit'); continue; }if (!canUseTokens(entry.platform, entry.model_id, key.id, estimatedTokens, limits)) { note('tpm/tpd-limit'); continue; }if (!canUseProviderTokens(entry.platform, key.id, entry.model_id, estimatedTokens)) { note('provider-daily-token-cap'); continue; }@@ -727,6 +753,9 @@ function selectKeyForModel(entry: ChainRow, estimatedTokens: number, skipKeys?:if (!resolvedProvider) { note('no-resolved-provider'); continue; }roundRobinIndex.set(rrKey, idx);+ // Taken only once the key has cleared every gate and is definitely being+ // returned, so a rejected candidate never consumes concurrency budget.+ const leaseId = acquireLease(entry.platform, entry.model_id, key.id, estimatedTokens);return {provider: resolvedProvider,modelId: entry.model_id,@@ -737,6 +766,7 @@ function selectKeyForModel(entry: ChainRow, estimatedTokens: number, skipKeys?:displayName: entry.display_name,rpdLimit: limits.rpd,tpdLimit: limits.tpd,+ release: () => releaseLease(leaseId),};}@@ -788,6 +818,7 @@ export function hasOtherUsableKey(modelDbId: number, excludingKeyId: number, skiif (skipKeys?.has(`${m.platform}:${m.model_id}:${k.id}`)) continue;if (isOnCooldown(m.platform, m.model_id, k.id)) continue;if (!canUseProvider(m.platform, k.id)) continue;+ if (!canUseProviderMinute(m.platform, k.id)) continue;if (!canMakeRequest(m.platform, m.model_id, k.id, limits)) continue;// A per-minute token spike on the failed key doesn't mean a fresh key lacks// headroom; a nominal 1-token probe only rules out a key already at its@@ -932,6 +963,7 @@ export function getOrderedFusionChain(): FusionCandidate[] {(e.key_id == null || kid === e.key_id) &&!isOnCooldown(e.platform, e.model_id, kid) &&canUseProvider(e.platform, kid) &&+ canUseProviderMinute(e.platform, kid) &&canMakeRequest(e.platform, e.model_id, kid, limits) &&canUseProviderTokens(e.platform, kid, e.model_id, 1),);