Skip to content

7.4 billion tokens per month. 34 free LLM providers. 635 free model endpoints. All behind one /v1 endpoint, plus any custom OpenAI-compatible endpoint. Smart routing, automatic failover, encrypted keys. Personal experimentation only.

TypeScript

Quota discipline: in-flight leases, provider RPM gate, health false negatives (#598)

Tashfeenc911ded8a8fd
diff --git a/server/src/__tests__/lib/fallback-loop-lease.test.ts b/server/src/__tests__/lib/fallback-loop-lease.test.ts
new file mode 100644
index 00000000..dc1f67d6
--- /dev/null
+++ b/server/src/__tests__/lib/fallback-loop-lease.test.ts
@@ -0,0 +1,131 @@
+import { describe, it, expect, beforeEach, afterEach, vi } from 'vitest';
+import { initDb } from '../../db/index.js';
+import { runFallbackLoop, newFallbackState } from '../../lib/fallback-loop.js';
+import { acquireLease, releaseLease, resetLeases, inFlightForKey } from '../../services/ratelimit.js';
+import type { RouteResult } from '../../services/router.js';
+
+/**
+ * The loop must free a route's in-flight lease on every exit path. A leak would
+ * leave the key's concurrency budget permanently short (until the age-out
+ * backstop), so this covers each way an iteration can end.
+ */
+function leasedRoute(keyId: number): RouteResult {
+ const id = acquireLease('groq', 'llama-3.3-70b', keyId, 100);
+ return {
+ provider: {} as any,
+ modelId: 'llama-3.3-70b',
+ modelDbId: 1,
+ apiKey: 'test-key',
+ keyId,
+ platform: 'groq',
+ displayName: 'Llama 3.3 70B',
+ rpdLimit: null,
+ tpdLimit: null,
+ release: () => releaseLease(id),
+ };
+}
+
+function hooks(overrides: Record<string, unknown> = {}) {
+ return {
+ state: newFallbackState(),
+ attemptLog: [],
+ route: () => leasedRoute(1),
+ dispatch: async () => 'done' as const,
+ logFailure: () => undefined,
+ onFatal: () => undefined,
+ onRoutingExhausted: () => undefined,
+ onExhausted: () => undefined,
+ ...overrides,
+ } as any;
+}
+
+describe('fallback loop releases in-flight leases', () => {
+ beforeEach(() => {
+ initDb(':memory:');
+ resetLeases();
+ vi.spyOn(console, 'log').mockImplementation(() => {});
+ vi.spyOn(console, 'error').mockImplementation(() => {});
+ vi.spyOn(console, 'warn').mockImplementation(() => {});
+ });
+
+ afterEach(() => {
+ resetLeases();
+ vi.restoreAllMocks();
+ });
+
+ it('releases after a successful dispatch', async () => {
+ await runFallbackLoop(hooks());
+ expect(inFlightForKey('groq', 1)).toBe(0);
+ });
+
+ it('releases after a committed stream', async () => {
+ await runFallbackLoop(hooks({ dispatch: async () => 'committed' as const }));
+ expect(inFlightForKey('groq', 1)).toBe(0);
+ });
+
+ it('releases every attempt when retryable failures exhaust the budget', async () => {
+ let dispatched = 0;
+ await runFallbackLoop(hooks({
+ maxRetries: 4,
+ dispatch: async () => {
+ dispatched++;
+ throw Object.assign(new Error('Groq API error 429: rate limit'), { status: 429 });
+ },
+ }));
+
+ expect(dispatched).toBe(4);
+ // Four leases were taken across the four attempts; none may survive.
+ expect(inFlightForKey('groq', 1)).toBe(0);
+ });
+
+ it('releases on an auth failure that rotates the key', async () => {
+ await runFallbackLoop(hooks({
+ maxRetries: 2,
+ dispatch: async () => {
+ throw Object.assign(new Error('Groq API error 401: Invalid API Key'), { status: 401 });
+ },
+ }));
+ expect(inFlightForKey('groq', 1)).toBe(0);
+ });
+
+ it('releases on a non-retryable fatal error', async () => {
+ const onFatal = vi.fn();
+ await runFallbackLoop(hooks({
+ onFatal,
+ dispatch: async () => { throw Object.assign(new Error('boom'), { status: 418 }); },
+ }));
+ expect(onFatal).toHaveBeenCalled();
+ expect(inFlightForKey('groq', 1)).toBe(0);
+ });
+
+ it('releases when dispatch violates the outcome contract', async () => {
+ const onFatal = vi.fn();
+ await runFallbackLoop(hooks({ onFatal, dispatch: async () => 'nonsense' as any }));
+ expect(onFatal).toHaveBeenCalled();
+ expect(inFlightForKey('groq', 1)).toBe(0);
+ });
+
+ it('does not mask the provider error if release itself throws', async () => {
+ const onFatal = vi.fn();
+ const bad = {
+ ...leasedRoute(1),
+ release: () => { throw new Error('release exploded'); },
+ };
+ await runFallbackLoop(hooks({
+ route: () => bad,
+ onFatal,
+ dispatch: async () => { throw Object.assign(new Error('real provider failure'), { status: 418 }); },
+ })).catch(() => undefined);
+
+ // onFatal must have seen the provider error, not the release error.
+ expect(onFatal).toHaveBeenCalled();
+ const err = onFatal.mock.calls[0]![1] as Error;
+ expect(err.message).toContain('real provider failure');
+ });
+
+ it('tolerates a route with no release function', async () => {
+ const noRelease = { ...leasedRoute(2) } as RouteResult;
+ delete (noRelease as { release?: unknown }).release;
+ await expect(runFallbackLoop(hooks({ route: () => noRelease }))).resolves.toBeUndefined();
+ });
+});
diff --git a/server/src/__tests__/routes/proxy-auth-rotation.test.ts b/server/src/__tests__/routes/proxy-auth-rotation.test.ts
index a3900dd7..32e1d4e7 100644
--- a/server/src/__tests__/routes/proxy-auth-rotation.test.ts
+++ b/server/src/__tests__/routes/proxy-auth-rotation.test.ts
@@ -22,8 +22,17 @@ vi.mock('../../providers/index.js', async (importOriginal) => {
};
});
-const { mockCheckKeyHealth } = vi.hoisted(() => ({ mockCheckKeyHealth: vi.fn() }));
-vi.mock('../../services/health.js', () => ({ checkKeyHealth: mockCheckKeyHealth }));
+const { mockCheckKeyHealth, mockMarkKeyHealthy } = vi.hoisted(() => ({
+ mockCheckKeyHealth: vi.fn(),
+ mockMarkKeyHealthy: vi.fn(),
+}));
+// Both exports must be stubbed: recordUpstreamSuccess calls
+// markKeyHealthyFromRequest, and a missing stub would throw on the success path
+// and surface as a 502 rather than the 200 under test.
+vi.mock('../../services/health.js', () => ({
+ checkKeyHealth: mockCheckKeyHealth,
+ markKeyHealthyFromRequest: mockMarkKeyHealthy,
+}));
const { createApp } = await import('../../app.js');
const { initDb, getDb, getUnifiedApiKey } = await import('../../db/index.js');
diff --git a/server/src/__tests__/services/health-error.test.ts b/server/src/__tests__/services/health-error.test.ts
index 48413990..3bc3fc80 100644
--- a/server/src/__tests__/services/health-error.test.ts
+++ b/server/src/__tests__/services/health-error.test.ts
@@ -62,8 +62,13 @@ describe('persisted key health diagnostics', () => {
validateKey.mockRejectedValue(new Error('Bearer secret-token-value-1234567890 failed at https://api.example.test/models'));
const error = vi.spyOn(console, 'error').mockImplementation(() => {});
- expect(await checkKeyHealth(id)).toBe('error');
- expect(row(id).last_health_error).toBe('Bearer [redacted] failed at [redacted-url]');
+ // Seeded as 'unknown', and a transport error preserves that rather than
+ // demoting to 'error' — the diagnostic is still recorded either way.
+ expect(await checkKeyHealth(id)).toBe('unknown');
+ expect(row(id)).toMatchObject({
+ status: 'unknown',
+ last_health_error: 'Bearer [redacted] failed at [redacted-url]',
+ });
expect(String(error.mock.calls[0][0])).not.toContain('secret-token-value');
});
diff --git a/server/src/__tests__/services/health-log.test.ts b/server/src/__tests__/services/health-log.test.ts
index 56b25b7c..4c3ded55 100644
--- a/server/src/__tests__/services/health-log.test.ts
+++ b/server/src/__tests__/services/health-log.test.ts
@@ -19,7 +19,9 @@ const { checkKeyHealth } = await import('../../services/health.js');
// that drops the leading "[Health] Key N (" prefix or removes the platform/base
// context will silently break attribution. These tests pin the format.
-const ERROR_RE = /^\[Health\] Key (\d+) \(([^,]+), base=([^)]+)\) transport error: (.+)$/;
+// The trailing "— status preserved as '<status>'" records that a transport error
+// deliberately no longer demotes the key; the error message is captured up to it.
+const ERROR_RE = /^\[Health\] Key (\d+) \(([^,]+), base=([^)]+)\) transport error: (.+?)(?: — status preserved as '(\w+)')?$/;
beforeAll(() => {
process.env.ENCRYPTION_KEY = '0'.repeat(64);
@@ -61,7 +63,12 @@ describe('checkKeyHealth transport-error log format', () => {
it('emits a single line including platform and base_url', async () => {
const id = seedKey('openai-compat', 'https://api.example.com/v1');
const status = await checkKeyHealth(id);
- expect(status).toBe('error');
+ // A transport error is evidence about the network, not the key: the prior
+ // status is kept so the key stays routable (selectKeyForModel only accepts
+ // 'healthy'/'unknown', so demoting here would silently drop its capacity).
+ expect(status).toBe('healthy');
+ const persisted = getDb().prepare('SELECT status FROM api_keys WHERE id = ?').get(id) as { status: string };
+ expect(persisted.status).toBe('healthy');
expect(consoleSpy).toHaveBeenCalledTimes(1);
const line = firstLine();
@@ -78,7 +85,7 @@ describe('checkKeyHealth transport-error log format', () => {
it('falls back to base=default when base_url is null', async () => {
const id = seedKey('nvidia', null);
const status = await checkKeyHealth(id);
- expect(status).toBe('error');
+ expect(status).toBe('healthy');
const line = firstLine();
expect(line).toMatch(ERROR_RE);
diff --git a/server/src/__tests__/services/health-transport-preserve.test.ts b/server/src/__tests__/services/health-transport-preserve.test.ts
new file mode 100644
index 00000000..61018c2e
--- /dev/null
+++ b/server/src/__tests__/services/health-transport-preserve.test.ts
@@ -0,0 +1,135 @@
+import { describe, it, expect, beforeAll, beforeEach, vi } from 'vitest';
+
+const validateKey = vi.fn();
+vi.mock('../../providers/index.js', async (importOriginal) => {
+ const actual = await importOriginal() as any;
+ return {
+ ...actual,
+ resolveProvider: () => ({ name: 'fake', validateKey }),
+ };
+});
+
+const { initDb, getDb } = await import('../../db/index.js');
+const { encrypt } = await import('../../lib/crypto.js');
+const { checkKeyHealth, checkAllKeys, markKeyHealthyFromRequest } = await import('../../services/health.js');
+
+/**
+ * selectKeyForModel only accepts keys with status IN ('healthy','unknown'), so
+ * writing status='error' removes a key's capacity entirely. A transport error
+ * (DNS, TLS, timeout, suspended laptop) says nothing about the credential, so it
+ * must not demote — otherwise one blip benches every key on a provider at once.
+ */
+describe('transport errors do not demote a key', () => {
+ let nextId = 7000;
+
+ beforeAll(() => {
+ process.env.ENCRYPTION_KEY = '0'.repeat(64);
+ initDb(':memory:');
+ });
+
+ beforeEach(() => {
+ validateKey.mockReset();
+ vi.spyOn(console, 'error').mockImplementation(() => {});
+ vi.spyOn(console, 'warn').mockImplementation(() => {});
+ vi.spyOn(console, 'log').mockImplementation(() => {});
+ });
+
+ function seedKey(status: string): number {
+ const id = ++nextId;
+ const enc = encrypt(`health-preserve-${id}`);
+ getDb().prepare(`
+ INSERT INTO api_keys (id, platform, label, encrypted_key, iv, auth_tag, enabled, status)
+ VALUES (?, 'mistral', ?, ?, ?, ?, 1, ?)
+ `).run(id, `preserve-${id}`, enc.encrypted, enc.iv, enc.authTag, status);
+ return id;
+ }
+
+ function statusOf(id: number): string {
+ return (getDb().prepare('SELECT status FROM api_keys WHERE id = ?').get(id) as { status: string }).status;
+ }
+
+ it('keeps a healthy key healthy through a transport failure', async () => {
+ const id = seedKey('healthy');
+ validateKey.mockRejectedValue(new Error('getaddrinfo ENOTFOUND api.mistral.ai'));
+
+ expect(await checkKeyHealth(id)).toBe('healthy');
+ expect(statusOf(id)).toBe('healthy');
+ });
+
+ it('records the diagnostic and timestamp even though status is unchanged', async () => {
+ const id = seedKey('healthy');
+ validateKey.mockRejectedValue(new Error('socket hang up'));
+
+ await checkKeyHealth(id);
+ const row = getDb().prepare(
+ 'SELECT last_health_error, last_checked_at FROM api_keys WHERE id = ?',
+ ).get(id) as { last_health_error: string; last_checked_at: string | null };
+
+ expect(row.last_health_error).toContain('socket hang up');
+ expect(row.last_checked_at).not.toBeNull();
+ });
+
+ it('still demotes on a confirmed bad credential', async () => {
+ const id = seedKey('healthy');
+ validateKey.mockResolvedValue({ valid: false, error: 'HTTP 401 invalid key' });
+
+ expect(await checkKeyHealth(id)).toBe('invalid');
+ expect(statusOf(id)).toBe('invalid');
+ });
+
+ it('an offline burst leaves every key routable', async () => {
+ const ids = [seedKey('healthy'), seedKey('healthy'), seedKey('unknown')];
+ validateKey.mockRejectedValue(new Error('fetch failed'));
+
+ await checkAllKeys();
+
+ // Previously all three would have gone to 'error' and the router would have
+ // had zero usable keys until a later pass succeeded.
+ expect(ids.map(statusOf)).toEqual(['healthy', 'healthy', 'unknown']);
+ });
+});
+
+describe('markKeyHealthyFromRequest', () => {
+ let nextId = 8000;
+
+ beforeAll(() => {
+ process.env.ENCRYPTION_KEY = '0'.repeat(64);
+ initDb(':memory:');
+ });
+
+ function seedKey(status: string): number {
+ const id = ++nextId;
+ const enc = encrypt(`heal-${id}`);
+ getDb().prepare(`
+ INSERT INTO api_keys (id, platform, label, encrypted_key, iv, auth_tag, enabled, status, last_health_error)
+ VALUES (?, 'groq', ?, ?, ?, ?, 1, ?, 'stale reason')
+ `).run(id, `heal-${id}`, enc.encrypted, enc.iv, enc.authTag, status);
+ return id;
+ }
+
+ function row(id: number): { status: string; last_health_error: string | null } {
+ return getDb().prepare('SELECT status, last_health_error FROM api_keys WHERE id = ?').get(id) as any;
+ }
+
+ it("promotes an 'error' key back to healthy and clears the stale reason", () => {
+ const id = seedKey('error');
+ markKeyHealthyFromRequest(id);
+ expect(row(id)).toEqual({ status: 'healthy', last_health_error: null });
+ });
+
+ it("never resurrects an 'invalid' key — only a real probe clears that", () => {
+ const id = seedKey('invalid');
+ markKeyHealthyFromRequest(id);
+ expect(row(id).status).toBe('invalid');
+ });
+
+ it('leaves an already-healthy key untouched', () => {
+ const id = seedKey('healthy');
+ markKeyHealthyFromRequest(id);
+ expect(row(id).status).toBe('healthy');
+ });
+
+ it('is silent for an unknown key id', () => {
+ expect(() => markKeyHealthyFromRequest(999_999)).not.toThrow();
+ });
+});
diff --git a/server/src/__tests__/services/key-concurrency.test.ts b/server/src/__tests__/services/key-concurrency.test.ts
new file mode 100644
index 00000000..af931640
--- /dev/null
+++ b/server/src/__tests__/services/key-concurrency.test.ts
@@ -0,0 +1,135 @@
+import { describe, it, expect, beforeEach, afterEach, vi } from 'vitest';
+import { initDb } from '../../db/index.js';
+import {
+ acquireLease,
+ releaseLease,
+ resetLeases,
+ inFlightForKey,
+ canUseKeyConcurrency,
+ getKeyConcurrencyLimit,
+} from '../../services/ratelimit.js';
+
+/**
+ * In-flight leases make the window between key selection and usage recording
+ * visible. They back the per-key concurrency cap, which stops parallel streams
+ * from all picking one key and 429-ing each other on providers that meter
+ * concurrency per credential.
+ */
+describe('in-flight leases', () => {
+ beforeEach(() => {
+ initDb(':memory:');
+ resetLeases();
+ });
+
+ afterEach(() => {
+ resetLeases();
+ delete process.env.MAX_CONCURRENT_REQUESTS_PER_KEY;
+ delete process.env.MAX_CONCURRENT_REQUESTS_PER_KEY_GROQ;
+ vi.useRealTimers();
+ });
+
+ it('counts only the matching platform and key', () => {
+ acquireLease('groq', 'llama-3.3-70b', 1, 100);
+ acquireLease('groq', 'llama-3.1-8b', 1, 100);
+ acquireLease('groq', 'llama-3.3-70b', 2, 100);
+ acquireLease('cerebras', 'qwen-3-coder', 1, 100);
+
+ expect(inFlightForKey('groq', 1)).toBe(2);
+ expect(inFlightForKey('groq', 2)).toBe(1);
+ expect(inFlightForKey('cerebras', 1)).toBe(1);
+ expect(inFlightForKey('groq', 99)).toBe(0);
+ });
+
+ it('release frees the slot and is idempotent', () => {
+ const id = acquireLease('groq', 'llama-3.3-70b', 1, 100);
+ expect(inFlightForKey('groq', 1)).toBe(1);
+
+ releaseLease(id);
+ expect(inFlightForKey('groq', 1)).toBe(0);
+ // Overlapping cleanup paths must not double-decrement someone else's slot.
+ releaseLease(id);
+ releaseLease(id);
+ expect(inFlightForKey('groq', 1)).toBe(0);
+ });
+
+ it('ages out a leaked lease so a key cannot be blocked forever', () => {
+ vi.useFakeTimers();
+ acquireLease('groq', 'llama-3.3-70b', 1, 100);
+ expect(inFlightForKey('groq', 1)).toBe(1);
+
+ // Past LEASE_MAX_AGE_MS (2 min) — the backstop for a release that never came.
+ vi.advanceTimersByTime(3 * 60 * 1000);
+ expect(inFlightForKey('groq', 1)).toBe(0);
+ });
+});
+
+describe('per-key concurrency cap', () => {
+ beforeEach(() => {
+ initDb(':memory:');
+ resetLeases();
+ });
+
+ afterEach(() => {
+ resetLeases();
+ delete process.env.MAX_CONCURRENT_REQUESTS_PER_KEY;
+ delete process.env.MAX_CONCURRENT_REQUESTS_PER_KEY_GROQ;
+ });
+
+ it('is unlimited by default — capping every provider would cost throughput', () => {
+ expect(getKeyConcurrencyLimit('groq')).toBeNull();
+ for (let i = 0; i < 25; i++) acquireLease('groq', 'llama-3.3-70b', 1, 100);
+ expect(canUseKeyConcurrency('groq', 1)).toBe(true);
+ });
+
+ it('reads a per-platform env cap ahead of the global one', () => {
+ process.env.MAX_CONCURRENT_REQUESTS_PER_KEY = '5';
+ process.env.MAX_CONCURRENT_REQUESTS_PER_KEY_GROQ = '2';
+ expect(getKeyConcurrencyLimit('groq')).toBe(2);
+ expect(getKeyConcurrencyLimit('cerebras')).toBe(5);
+ });
+
+ it('ignores a malformed cap rather than blocking everything', () => {
+ process.env.MAX_CONCURRENT_REQUESTS_PER_KEY_GROQ = 'not-a-number';
+ expect(getKeyConcurrencyLimit('groq')).toBeNull();
+ process.env.MAX_CONCURRENT_REQUESTS_PER_KEY_GROQ = '0';
+ expect(getKeyConcurrencyLimit('groq')).toBeNull();
+ });
+
+ it('blocks a key at its cap and frees it on release', () => {
+ process.env.MAX_CONCURRENT_REQUESTS_PER_KEY_GROQ = '1';
+
+ expect(canUseKeyConcurrency('groq', 1)).toBe(true);
+ const id = acquireLease('groq', 'llama-3.3-70b', 1, 100);
+ expect(canUseKeyConcurrency('groq', 1)).toBe(false);
+
+ releaseLease(id);
+ expect(canUseKeyConcurrency('groq', 1)).toBe(true);
+ });
+
+ it('caps each key independently, so a sibling key stays available', () => {
+ process.env.MAX_CONCURRENT_REQUESTS_PER_KEY_GROQ = '1';
+ acquireLease('groq', 'llama-3.3-70b', 1, 100);
+
+ // This is the point: the request fails over to key 2 instead of queueing
+ // behind key 1 and collecting a 429.
+ expect(canUseKeyConcurrency('groq', 1)).toBe(false);
+ expect(canUseKeyConcurrency('groq', 2)).toBe(true);
+ });
+
+ it('counts a key busy on one model against a request for another', () => {
+ process.env.MAX_CONCURRENT_REQUESTS_PER_KEY_GROQ = '1';
+ acquireLease('groq', 'llama-3.3-70b', 1, 100);
+ // Concurrency is metered per credential, not per model.
+ expect(canUseKeyConcurrency('groq', 1)).toBe(false);
+ });
+
+ it('allows up to the cap and no further', () => {
+ process.env.MAX_CONCURRENT_REQUESTS_PER_KEY_GROQ = '3';
+ const ids = [0, 1, 2].map(() => acquireLease('groq', 'llama-3.3-70b', 1, 100));
+ expect(inFlightForKey('groq', 1)).toBe(3);
+ expect(canUseKeyConcurrency('groq', 1)).toBe(false);
+
+ releaseLease(ids[0]!);
+ expect(canUseKeyConcurrency('groq', 1)).toBe(true);
+ });
+});
diff --git a/server/src/__tests__/services/provider-minute-cap.test.ts b/server/src/__tests__/services/provider-minute-cap.test.ts
new file mode 100644
index 00000000..0bb1b4fb
--- /dev/null
+++ b/server/src/__tests__/services/provider-minute-cap.test.ts
@@ -0,0 +1,92 @@
+import { describe, it, expect, beforeEach, afterEach } from 'vitest';
+import { initDb, getDb } from '../../db/index.js';
+import {
+ canUseProviderMinute,
+ providerMinuteRequestCount,
+ getProviderMinuteRequestCap,
+} from '../../services/ratelimit.js';
+
+/**
+ * Account-wide per-minute caps. The per-model rpm_limit cannot express these:
+ * NVIDIA NIM meters ~40 req/min across the whole account, so every model on it
+ * draws from one bucket, and a model row with rpm_limit NULL isn't pre-throttled
+ * at all. Without this gate the router keeps dispatching and eats real 429s.
+ */
+function seedRequests(platform: string, modelId: string, keyId: number, count: number, atMs: number) {
+ const ins = getDb().prepare(`
+ INSERT INTO rate_limit_usage (platform, model_id, key_id, kind, tokens, created_at_ms)
+ VALUES (?, ?, ?, 'request', 0, ?)
+ `);
+ for (let i = 0; i < count; i++) ins.run(platform, modelId, keyId, atMs);
+}
+
+describe('provider-wide per-minute request cap', () => {
+ let keyId: number;
+
+ beforeEach(() => {
+ initDb(':memory:');
+ keyId = Math.floor(Math.random() * 1_000_000);
+ });
+
+ afterEach(() => {
+ delete process.env.PROVIDER_MINUTE_REQUEST_CAP_NVIDIA;
+ delete process.env.PROVIDER_MINUTE_REQUEST_CAP_GROQ;
+ });
+
+ it('ships a default cap for nvidia and none for a provider without one', () => {
+ expect(getProviderMinuteRequestCap('nvidia')).toBe(40);
+ expect(getProviderMinuteRequestCap('groq')).toBeNull();
+ });
+
+ it('honours an env override and treats 0 as disabled', () => {
+ process.env.PROVIDER_MINUTE_REQUEST_CAP_NVIDIA = '5';
+ expect(getProviderMinuteRequestCap('nvidia')).toBe(5);
+ process.env.PROVIDER_MINUTE_REQUEST_CAP_NVIDIA = '0';
+ expect(getProviderMinuteRequestCap('nvidia')).toBeNull();
+ });
+
+ it('sums usage across every model on the same account+key', () => {
+ const now = Date.now();
+ // The whole point: three different models, one shared budget.
+ seedRequests('nvidia', 'glm-4.7', keyId, 10, now - 5_000);
+ seedRequests('nvidia', 'minimax-m3', keyId, 10, now - 4_000);
+ seedRequests('nvidia', 'deepseek-v4', keyId, 10, now - 3_000);
+
+ expect(providerMinuteRequestCount('nvidia', keyId, now)).toBe(30);
+ });
+
+ it('blocks once the shared budget is spent, even for an untouched model', () => {
+ process.env.PROVIDER_MINUTE_REQUEST_CAP_NVIDIA = '40';
+ const now = Date.now();
+ seedRequests('nvidia', 'glm-4.7', keyId, 40, now - 10_000);
+
+ // deepseek-v4 has made zero requests of its own and may well have a NULL
+ // rpm_limit, but the account budget is gone.
+ expect(canUseProviderMinute('nvidia', keyId, now)).toBe(false);
+ });
+
+ it('allows again once the minute window rolls past', () => {
+ process.env.PROVIDER_MINUTE_REQUEST_CAP_NVIDIA = '40';
+ const now = Date.now();
+ seedRequests('nvidia', 'glm-4.7', keyId, 40, now - 61_000);
+
+ expect(providerMinuteRequestCount('nvidia', keyId, now)).toBe(0);
+ expect(canUseProviderMinute('nvidia', keyId, now)).toBe(true);
+ });
+
+ it('does not gate a provider with no configured cap', () => {
+ const now = Date.now();
+ seedRequests('groq', 'llama-3.3-70b', keyId, 500, now - 1_000);
+ expect(canUseProviderMinute('groq', keyId, now)).toBe(true);
+ });
+
+ it('keeps separate budgets per key on the same platform', () => {
+ process.env.PROVIDER_MINUTE_REQUEST_CAP_NVIDIA = '40';
+ const now = Date.now();
+ const other = keyId + 1;
+ seedRequests('nvidia', 'glm-4.7', keyId, 40, now - 5_000);
+
+ expect(canUseProviderMinute('nvidia', keyId, now)).toBe(false);
+ expect(canUseProviderMinute('nvidia', other, now)).toBe(true);
+ });
+});
diff --git a/server/src/__tests__/services/provisional-usage.test.ts b/server/src/__tests__/services/provisional-usage.test.ts
new file mode 100644
index 00000000..f6f70779
--- /dev/null
+++ b/server/src/__tests__/services/provisional-usage.test.ts
@@ -0,0 +1,107 @@
+import { describe, it, expect, beforeEach, afterEach } from 'vitest';
+import { initDb, getDb } from '../../db/index.js';
+import {
+ canMakeRequest,
+ canUseTokens,
+ canUseProvider,
+ canUseProviderMinute,
+ acquireLease,
+ releaseLease,
+ resetLeases,
+ recordRequest,
+} from '../../services/ratelimit.js';
+
+/**
+ * The check-then-act race: routeRequest is synchronous, but usage is only written
+ * after the awaited provider call. Without provisional accounting, N concurrent
+ * requests all read the same pre-check, all pass, and collectively overshoot the
+ * limit — which is exactly how a free tier gets 429ed by its own client.
+ */
+describe('in-flight leases count against the quota gates', () => {
+ let keyId: number;
+
+ beforeEach(() => {
+ initDb(':memory:');
+ resetLeases();
+ keyId = Math.floor(Math.random() * 1_000_000);
+ });
+
+ afterEach(() => {
+ resetLeases();
+ delete process.env.PROVIDER_MINUTE_REQUEST_CAP_NVIDIA;
+ });
+
+ const limits = (rpm: number | null) => ({ rpm, rpd: null, tpm: null, tpd: null });
+
+ it('a single in-flight request consumes an rpm of 1', () => {
+ expect(canMakeRequest('groq', 'm1', keyId, limits(1))).toBe(true);
+
+ // Selection happened; nothing recorded yet because the call is still in flight.
+ acquireLease('groq', 'm1', keyId, 100);
+
+ // Before this change the second caller saw an unspent budget and dispatched.
+ expect(canMakeRequest('groq', 'm1', keyId, limits(1))).toBe(false);
+ });
+
+ it('frees the budget again once the lease is released', () => {
+ const id = acquireLease('groq', 'm1', keyId, 100);
+ expect(canMakeRequest('groq', 'm1', keyId, limits(1))).toBe(false);
+ releaseLease(id);
+ expect(canMakeRequest('groq', 'm1', keyId, limits(1))).toBe(true);
+ });
+
+ it('stacks provisional on top of recorded usage', () => {
+ recordRequest('groq', 'm1', keyId); // 1 recorded
+ acquireLease('groq', 'm1', keyId, 100); // 1 in flight
+ expect(canMakeRequest('groq', 'm1', keyId, limits(3))).toBe(true); // 2 of 3
+ acquireLease('groq', 'm1', keyId, 100); // 2 in flight → 3 of 3
+ expect(canMakeRequest('groq', 'm1', keyId, limits(3))).toBe(false);
+ });
+
+ it('scopes provisional requests to the same model and key', () => {
+ acquireLease('groq', 'other-model', keyId, 100);
+ acquireLease('groq', 'm1', keyId + 1, 100);
+ // Neither lease belongs to (m1, keyId), so its own budget is untouched.
+ expect(canMakeRequest('groq', 'm1', keyId, limits(1))).toBe(true);
+ });
+
+ it('counts promised tokens against tpm', () => {
+ const tokenLimits = { tpm: 1000, tpd: null };
+ expect(canUseTokens('groq', 'm1', keyId, 600, tokenLimits)).toBe(true);
+
+ acquireLease('groq', 'm1', keyId, 600); // 600 already promised
+ // 600 in flight + 600 requested exceeds 1000.
+ expect(canUseTokens('groq', 'm1', keyId, 600, tokenLimits)).toBe(false);
+ // A smaller request still fits.
+ expect(canUseTokens('groq', 'm1', keyId, 300, tokenLimits)).toBe(true);
+ });
+
+ it('counts in-flight requests against the provider-wide minute cap', () => {
+ process.env.PROVIDER_MINUTE_REQUEST_CAP_NVIDIA = '2';
+ // Different models, one shared account budget — the whole point of that gate.
+ acquireLease('nvidia', 'glm-4.7', keyId, 100);
+ acquireLease('nvidia', 'minimax-m3', keyId, 100);
+ expect(canUseProviderMinute('nvidia', keyId)).toBe(false);
+ });
+
+ it('counts in-flight requests against the provider-wide daily cap', () => {
+ const now = Date.now();
+ const ins = getDb().prepare(`
+ INSERT INTO rate_limit_usage (platform, model_id, key_id, kind, tokens, created_at_ms)
+ VALUES ('openrouter', 'm1', ?, 'request', 0, ?)
+ `);
+ // OpenRouter ships a 1000/day default; sit one short of it.
+ for (let i = 0; i < 999; i++) ins.run(keyId, now - 1000);
+
+ expect(canUseProvider('openrouter', keyId, now)).toBe(true);
+ acquireLease('openrouter', 'm1', keyId, 100, now);
+ expect(canUseProvider('openrouter', keyId, now)).toBe(false);
+ });
+
+ it('ignores an aged-out lease so a leak cannot block the gate forever', () => {
+ const now = Date.now();
+ // Older than LEASE_MAX_AGE_MS (2 min).
+ acquireLease('groq', 'm1', keyId, 100, now - 5 * 60 * 1000);
+ expect(canMakeRequest('groq', 'm1', keyId, limits(1))).toBe(true);
+ });
+});
diff --git a/server/src/lib/fallback-loop.ts b/server/src/lib/fallback-loop.ts
index c0fbc17b..39d3cc33 100644
--- a/server/src/lib/fallback-loop.ts
+++ b/server/src/lib/fallback-loop.ts
@@ -38,7 +38,7 @@ import {
isProviderDegradedError,
} from './error-classify.js';
import { sanitizeProviderErrorMessage } from './error-redaction.js';
-import { checkKeyHealth } from '../services/health.js';
+import { checkKeyHealth, markKeyHealthyFromRequest } from '../services/health.js';
import { getSetting } from '../db/index.js';
import { newBreaker, recordBreakerFailure } from './guardrails.js';
@@ -213,6 +213,10 @@ export function recordUpstreamSuccess(route: RouteResult, rateLimitTokens: numbe
recordRequest(route.platform, route.modelId, route.keyId);
recordTokens(route.platform, route.modelId, route.keyId, rateLimitTokens);
recordSuccess(route.modelDbId);
+ // A served request is the strongest possible evidence the key works, so clear
+ // any stale 'error' status left by an earlier transport blip instead of waiting
+ // for the next health pass to make the key routable again.
+ markKeyHealthyFromRequest(route.keyId);
}
// ── Attempt trail ─────────────────────────────────────────────────────────────
@@ -556,6 +560,13 @@ export async function runFallbackLoop(hooks: FallbackHooks): Promise<void> {
return;
}
+ // Everything from here to the end of the iteration runs inside a finally that
+ // frees the route's in-flight lease. Every exit — success, auth rotation,
+ // retryable continue, fatal, breaker trip, contract violation — passes through
+ // it, so no path can leak a lease and leave the key's concurrency budget short.
+ // Success accounting happens inside dispatch, so the persisted counters are
+ // already written by the time the provisional lease goes away.
+ try {
let outcome: DispatchOutcome;
try {
outcome = await hooks.dispatch(route, attempt);
@@ -603,6 +614,9 @@ export async function runFallbackLoop(hooks: FallbackHooks): Promise<void> {
hooks.onFatal(route, violation, attempt);
}
return;
+ } finally {
+ route.release?.();
+ }
}
hooks.onExhausted(
diff --git a/server/src/services/fusion.ts b/server/src/services/fusion.ts
index a4f601ee..d1f0f1a2 100644
--- a/server/src/services/fusion.ts
+++ b/server/src/services/fusion.ts
@@ -270,6 +270,11 @@ async function runModelCall(
}
// Non-retryable (auth, validation) — this slot/judge is done.
break;
+ } finally {
+ // Panel slots run concurrently, so a leaked lease here would starve the
+ // rest of the panel of its own keys' concurrency budget. The route object
+ // stays usable as a data carrier after release.
+ route.release?.();
}
}
@@ -353,6 +358,8 @@ async function runJudgeStreaming(
continue;
}
break;
+ } finally {
+ route.release?.();
}
}
return { ok: false, error: lastError ?? 'no available key for judge' };
diff --git a/server/src/services/health.ts b/server/src/services/health.ts
index 39197486..79833b26 100644
--- a/server/src/services/health.ts
+++ b/server/src/services/health.ts
@@ -8,6 +8,18 @@ import { sanitizeProviderErrorMessage } from '../lib/error-redaction.js';
const CHECK_INTERVAL_MS = 5 * 60 * 1000; // 5 minutes
const CONSECUTIVE_FAILURES_TO_DISABLE = 3;
+const DEFAULT_HEALTH_CHECK_CONCURRENCY = 8;
+
+/** Parallel key probes per health pass. Tunable because the right number depends
+ * on how many keys share one provider; 0 or a bad value falls back to the default. */
+function getHealthCheckConcurrency(): number {
+ const raw = process.env.HEALTH_CHECK_CONCURRENCY;
+ if (raw !== undefined && raw.trim() !== '') {
+ const n = Number(raw);
+ if (Number.isInteger(n) && n > 0) return n;
+ }
+ return DEFAULT_HEALTH_CHECK_CONCURRENCY;
+}
// Track consecutive failures per key
const failureCount = new Map<number, number>();
@@ -75,11 +87,38 @@ export async function checkKeyHealth(keyId: number): Promise<KeyStatus> {
const lastError = sanitizeProviderErrorMessage(err?.message ?? err);
console.error(
`[Health] Key ${keyId} (${row.platform}, base=${row.base_url ?? 'default'}) ` +
- `transport error: ${lastError}`,
+ `transport error: ${lastError} — status preserved as '${row.status}'`,
);
- db.prepare("UPDATE api_keys SET status = ?, last_health_error = ?, last_checked_at = datetime('now') WHERE id = ?")
- .run('error', lastError, keyId);
- return 'error';
+ // Do NOT write status='error'. selectKeyForModel only considers keys with
+ // status IN ('healthy','unknown'), so demoting here silently removes the
+ // key's capacity for up to a full check interval — and a transport error is
+ // evidence about the network, not about the key. One flaky DNS lookup, a
+ // laptop suspended for thirty seconds, or a provider edge outage would take
+ // every key on that provider out of rotation at once. Record the diagnostic
+ // and the timestamp; leave the verdict to a probe that actually reached the
+ // provider. Confirmed 401/403 (the isValid=false path above) still demotes.
+ db.prepare("UPDATE api_keys SET last_health_error = ?, last_checked_at = datetime('now') WHERE id = ?")
+ .run(lastError, keyId);
+ return row.status as KeyStatus;
+ }
+}
+
+/**
+ * Promote a key out of 'error' after it successfully served a live request.
+ *
+ * Serving traffic is stronger evidence than any probe, so a key stuck at 'error'
+ * from an earlier transport blip should not have to wait for the next health pass
+ * to become routable again. Deliberately narrow: 'invalid' means a provider
+ * confirmed the credential is bad, and only a real validateKey pass clears that.
+ */
+export function markKeyHealthyFromRequest(keyId: number): void {
+ try {
+ getDb()
+ .prepare("UPDATE api_keys SET status = 'healthy', last_health_error = NULL WHERE id = ? AND status = 'error'")
+ .run(keyId);
+ failureCount.delete(keyId);
+ } catch {
+ // Never let health bookkeeping break a request that already succeeded.
}
}
@@ -99,9 +138,26 @@ export function checkAllKeys(): Promise<void> {
console.log(`[Health] Checking ${keys.length} keys...`);
- for (const key of keys) {
- await checkKeyHealth(key.id);
- }
+ // Bounded worker pool rather than a sequential await. validateKey allows up
+ // to 30s per key (some provider /models endpoints are genuinely that slow),
+ // so a serial pass over a large key fleet can outlast the 5-minute interval
+ // it is scheduled on and leave the dashboard showing stale statuses. The cap
+ // keeps us from opening one socket per key against the same provider.
+ const concurrency = getHealthCheckConcurrency();
+ let cursor = 0;
+ const workers = Array.from({ length: Math.min(concurrency, keys.length) }, async () => {
+ while (cursor < keys.length) {
+ const key = keys[cursor++]!;
+ try {
+ await checkKeyHealth(key.id);
+ } catch (err) {
+ // checkKeyHealth handles its own errors; this is a backstop so one
+ // rejection cannot abandon the rest of the pass.
+ console.error(`[Health] Key ${key.id} check threw:`, err);
+ }
+ }
+ });
+ await Promise.all(workers);
console.log(`[Health] Check complete.`);
})().finally(() => {
diff --git a/server/src/services/ratelimit.ts b/server/src/services/ratelimit.ts
index aa62e4db..b7690f26 100644
--- a/server/src/services/ratelimit.ts
+++ b/server/src/services/ratelimit.ts
@@ -45,6 +45,152 @@ function withDb<T>(fn: (db: RateLimitDb) => T): T | undefined {
}
}
+// ── In-flight leases ──────────────────────────────────────────────────────────
+// Usage is only recorded *after* an attempt succeeds, so between key selection
+// and that write the router has no idea a request is already in the air. A lease
+// makes that window visible. Today it backs the per-key concurrency cap; it is
+// also the hook for counting provisional usage against the quota gates, which is
+// what closes the check-then-act race under parallel load.
+//
+// Leases live in memory only: they describe requests this process currently has
+// open, so there is nothing meaningful to persist and a restart correctly starts
+// from zero.
+
+interface Lease {
+ platform: string;
+ modelId: string;
+ keyId: number;
+ tokens: number;
+ createdAt: number;
+}
+
+const leases = new Map<number, Lease>();
+let nextLeaseId = 1;
+
+// A lease should always be released explicitly by the fallback loop's finally
+// block. This bound is the backstop for a path that somehow doesn't: without it
+// a single leaked lease would count against a key's concurrency budget forever.
+// Comfortably longer than the per-attempt provider timeout.
+const LEASE_MAX_AGE_MS = 2 * MINUTE;
+
+function pruneLeases(now: number): void {
+ if (leases.size === 0) return;
+ for (const [id, lease] of leases) {
+ if (now - lease.createdAt > LEASE_MAX_AGE_MS) leases.delete(id);
+ }
+}
+
+/**
+ * Concurrent in-flight requests allowed per key, or null for unlimited.
+ *
+ * Some free tiers meter concurrency per credential rather than per minute, so
+ * parallel streams on one key mostly 429 each other. But most do not, and
+ * capping by default would serialise every provider and cost real throughput —
+ * so this is opt-in. There is deliberately no built-in per-platform table: the
+ * providers that behave this way are not documented well enough to assert a
+ * number for them here, and a wrong default is worse than none.
+ *
+ * `MAX_CONCURRENT_REQUESTS_PER_KEY_<PLATFORM>` sets it for one platform;
+ * `MAX_CONCURRENT_REQUESTS_PER_KEY` sets a fallback for all of them.
+ */
+export function getKeyConcurrencyLimit(platform: string): number | null {
+ const raw = process.env[`MAX_CONCURRENT_REQUESTS_PER_KEY_${platform.toUpperCase()}`]
+ ?? process.env.MAX_CONCURRENT_REQUESTS_PER_KEY;
+ if (raw !== undefined && raw.trim() !== '') {
+ const n = Number(raw);
+ if (Number.isInteger(n) && n > 0) return n;
+ }
+ return null;
+}
+
+/** Requests this process currently has in flight against one platform+key. */
+export function inFlightForKey(platform: string, keyId: number, now = Date.now()): number {
+ pruneLeases(now);
+ let count = 0;
+ for (const lease of leases.values()) {
+ if (lease.platform === platform && lease.keyId === keyId) count++;
+ }
+ return count;
+}
+
+/** False when this key already has its allowed number of requests in the air.
+ * Always true when no cap is configured, which is the default. */
+export function canUseKeyConcurrency(platform: string, keyId: number, now = Date.now()): boolean {
+ const limit = getKeyConcurrencyLimit(platform);
+ if (limit === null) return true;
+ return inFlightForKey(platform, keyId, now) < limit;
+}
+
+/** Take a lease for an attempt about to be dispatched. Returns the id to release. */
+export function acquireLease(
+ platform: string,
+ modelId: string,
+ keyId: number,
+ tokens: number,
+ now = Date.now(),
+): number {
+ pruneLeases(now);
+ const id = nextLeaseId++;
+ leases.set(id, { platform, modelId, keyId, tokens, createdAt: now });
+ return id;
+}
+
+/** Release a lease. Idempotent, so a double release from overlapping cleanup
+ * paths is harmless. */
+export function releaseLease(leaseId: number): void {
+ leases.delete(leaseId);
+}
+
+/** Test seam: drop all leases. */
+export function resetLeases(): void {
+ leases.clear();
+}
+
+// ── Provisional usage ─────────────────────────────────────────────────────────
+// Every gate below adds in-flight leases on top of the recorded counters. This is
+// what closes the check-then-act race: routeRequest is synchronous but usage is
+// only written after the awaited provider call, so N concurrent requests all read
+// the same pre-check and every one of them passes — then they collectively blow
+// through the limit and collect real 429s. Counting leases makes the second
+// caller see the first one's request.
+//
+// A lease is released just after the success write, so for a brief instant a
+// completed request is counted twice. That errs toward under-dispatching by one,
+// which is the safe direction for a free tier, and lasts microseconds.
+
+function provisionalRequests(platform: string, modelId: string, keyId: number, now: number): number {
+ pruneLeases(now);
+ let count = 0;
+ for (const lease of leases.values()) {
+ if (lease.platform === platform && lease.modelId === modelId && lease.keyId === keyId) count++;
+ }
+ return count;
+}
+
+function provisionalTokens(platform: string, modelId: string, keyId: number, now: number): number {
+ pruneLeases(now);
+ let total = 0;
+ for (const lease of leases.values()) {
+ if (lease.platform === platform && lease.modelId === modelId && lease.keyId === keyId) total += lease.tokens;
+ }
+ return total;
+}
+
+/** In-flight requests for a provider account+key across every model — the
+ * provider-wide analogue, for the account-level gates. */
+function provisionalProviderRequests(platform: string, keyId: number, now: number): number {
+ return inFlightForKey(platform, keyId, now);
+}
+
+function provisionalProviderTokens(platform: string, keyId: number, now: number): number {
+ pruneLeases(now);
+ let total = 0;
+ for (const lease of leases.values()) {
+ if (lease.platform === platform && lease.keyId === keyId) total += lease.tokens;
+ }
+ return total;
+}
+
function recordUsage(
platform: string,
modelId: string,
@@ -149,13 +295,16 @@ export function canMakeRequest(
limits: { rpm: number | null; rpd: number | null; tpm: number | null; tpd: number | null },
): boolean {
const now = Date.now();
+ // In-flight requests count against both windows: a lease means the request is
+ // happening now, so it belongs to this minute and to today.
+ const inFlight = provisionalRequests(platform, modelId, keyId, now);
if (limits.rpm !== null) {
- if (requestCount(platform, modelId, keyId, MINUTE, now) >= limits.rpm) return false;
+ if (requestCount(platform, modelId, keyId, MINUTE, now) + inFlight >= limits.rpm) return false;
}
if (limits.rpd !== null) {
- if (requestCount(platform, modelId, keyId, DAY, now) >= limits.rpd) return false;
+ if (requestCount(platform, modelId, keyId, DAY, now) + inFlight >= limits.rpd) return false;
}
return true;
@@ -169,15 +318,17 @@ export function canUseTokens(
limits: { tpm: number | null; tpd: number | null },
): boolean {
const now = Date.now();
+ // Tokens already promised to in-flight attempts on this model+key.
+ const inFlight = provisionalTokens(platform, modelId, keyId, now);
if (limits.tpm !== null) {
const used = tokenCount(platform, modelId, keyId, MINUTE, now);
- if (used + estimatedTokens > limits.tpm) return false;
+ if (used + inFlight + estimatedTokens > limits.tpm) return false;
}
if (limits.tpd !== null) {
const used = tokenCount(platform, modelId, keyId, DAY, now);
- if (used + estimatedTokens > limits.tpd) return false;
+ if (used + inFlight + estimatedTokens > limits.tpd) return false;
}
return true;
@@ -204,6 +355,16 @@ const DEFAULT_PROVIDER_DAILY_TOKEN_CAPS: Record<string, number> = {
navy: 150_000,
};
+// Per-minute caps that apply to the whole provider account rather than one model.
+// The per-model rpm_limit cannot express these: NVIDIA NIM meters ~40 requests a
+// minute across the entire account, so glm-4.7, minimax-m3 and deepseek all draw
+// from one bucket. Without an account-level gate the router sees each model's own
+// rpm as unspent — and a model row whose rpm_limit is NULL escapes pre-throttling
+// entirely — so it keeps dispatching and eats real 429s.
+const DEFAULT_PROVIDER_MINUTE_REQUEST_CAPS: Record<string, number> = {
+ nvidia: 40,
+};
+
export function getProviderDailyRequestCap(platform: string): number | null {
const raw = process.env[`PROVIDER_DAILY_REQUEST_CAP_${platform.toUpperCase()}`];
if (raw !== undefined && raw.trim() !== '') {
@@ -213,6 +374,17 @@ export function getProviderDailyRequestCap(platform: string): number | null {
return DEFAULT_PROVIDER_DAILY_REQUEST_CAPS[platform] ?? null;
}
+/** Account-wide requests-per-minute cap, or null when the provider has none.
+ * `PROVIDER_MINUTE_REQUEST_CAP_<PLATFORM>=0` disables the gate for that platform. */
+export function getProviderMinuteRequestCap(platform: string): number | null {
+ const raw = process.env[`PROVIDER_MINUTE_REQUEST_CAP_${platform.toUpperCase()}`];
+ if (raw !== undefined && raw.trim() !== '') {
+ const n = Number(raw);
+ if (Number.isFinite(n) && n >= 0) return n === 0 ? null : n;
+ }
+ return DEFAULT_PROVIDER_MINUTE_REQUEST_CAPS[platform] ?? null;
+}
+
export function getProviderDailyTokenCap(platform: string): number | null {
const raw = process.env[`PROVIDER_DAILY_TOKEN_CAP_${platform.toUpperCase()}`];
if (raw !== undefined && raw.trim() !== '') {
@@ -266,7 +438,32 @@ export function providerDailyRequestCount(platform: string, keyId: number, now =
export function canUseProvider(platform: string, keyId: number, now = Date.now()): boolean {
const cap = getProviderDailyRequestCap(platform);
if (cap === null) return true;
- return providerDailyRequestCount(platform, keyId, now) < cap;
+ const used = providerDailyRequestCount(platform, keyId, now) + provisionalProviderRequests(platform, keyId, now);
+ return used < cap;
+}
+
+/** Requests in the last minute for a provider account+key, across every model. */
+export function providerMinuteRequestCount(platform: string, keyId: number, now = Date.now()): number {
+ const persisted = countPersistedProviderRequests(platform, keyId, MINUTE, now);
+ if (persisted !== undefined) return persisted;
+ // DB-unavailable fallback: sum the per-model rpm windows for this platform+key.
+ let total = 0;
+ for (const [key, w] of windows) {
+ if (key.startsWith(`${platform}:`) && key.endsWith(`:${keyId}:rpm`)) {
+ total += pruneTimestamps(w.timestamps, MINUTE, now).length;
+ }
+ }
+ return total;
+}
+
+// False when this provider account+key has spent its shared per-minute request
+// budget, so the router skips every model on that provider rather than learning
+// the limit again from a 429 on each one in turn.
+export function canUseProviderMinute(platform: string, keyId: number, now = Date.now()): boolean {
+ const cap = getProviderMinuteRequestCap(platform);
+ if (cap === null) return true;
+ const used = providerMinuteRequestCount(platform, keyId, now) + provisionalProviderRequests(platform, keyId, now);
+ return used < cap;
}
type ModelQuotaRow = { tpd_limit: number | null; monthly_token_budget: string | null };
@@ -365,7 +562,8 @@ export function canUseProviderTokens(
): boolean {
const cap = getProviderDailyTokenCap(platform);
if (cap === null) return true;
- const used = providerDailyTokenCount(platform, keyId, now);
+ const used = providerDailyTokenCount(platform, keyId, now)
+ + providerBilledTokens(platform, modelId, provisionalProviderTokens(platform, keyId, now));
return used + providerBilledTokens(platform, modelId, estimatedTokens) <= cap;
}
diff --git a/server/src/services/router.ts b/server/src/services/router.ts
index e137171f..4a9b2cfa 100644
--- a/server/src/services/router.ts
+++ b/server/src/services/router.ts
@@ -6,7 +6,11 @@ import {
canUseTokens,
isOnCooldown,
canUseProvider,
+ canUseProviderMinute,
canUseProviderTokens,
+ canUseKeyConcurrency,
+ acquireLease,
+ releaseLease,
getSoonestCooldownExpiry,
} from './ratelimit.js';
import {
@@ -145,6 +149,20 @@ export interface RouteResult {
// exhaustion (escalate the cooldown) from a transient per-minute spike.
rpdLimit: number | null;
tpdLimit: number | null;
+ /**
+ * Frees the in-flight lease taken when this route was selected. Idempotent.
+ *
+ * Callers should invoke it once the attempt is finished, however it finished —
+ * the shared fallback loop does so from a `finally` so no exit path can leak.
+ *
+ * Optional, and every call site uses `release?.()`, for a specific reason: the
+ * invocation sits in a `finally`, and a TypeError thrown there would *replace*
+ * the in-flight provider exception with a useless one, turning a diagnosable
+ * 429 into a mystery 500. A route that arrives without it (a test double, a
+ * future construction path) should quietly fall back to the lease ageing out
+ * rather than destroy the error being propagated.
+ */
+ release?: () => void;
}
// ── Routing token estimate: cap the reserved OUTPUT, not the full max_tokens ──
@@ -707,6 +725,14 @@ function selectKeyForModel(entry: ChainRow, estimatedTokens: number, skipKeys?:
if (isOnCooldown(entry.platform, entry.model_id, key.id)) { note('cooldown'); continue; }
if (!canUseProvider(entry.platform, key.id)) { note('provider-daily-cap'); continue; }
+ // Account-wide per-minute budget, checked before the per-model gates: a model
+ // with a NULL rpm_limit would otherwise sail past them and spend a budget its
+ // siblings share.
+ if (!canUseProviderMinute(entry.platform, key.id)) { note('provider-minute-cap'); continue; }
+ // Skip a key that already has its allowed requests in the air. Without this,
+ // parallel streams all pick the same key and 429 each other on providers that
+ // meter concurrency per credential.
+ if (!canUseKeyConcurrency(entry.platform, key.id)) { note('key-concurrency'); continue; }
if (!canMakeRequest(entry.platform, entry.model_id, key.id, limits)) { note('rpm/rpd-limit'); continue; }
if (!canUseTokens(entry.platform, entry.model_id, key.id, estimatedTokens, limits)) { note('tpm/tpd-limit'); continue; }
if (!canUseProviderTokens(entry.platform, key.id, entry.model_id, estimatedTokens)) { note('provider-daily-token-cap'); continue; }
@@ -727,6 +753,9 @@ function selectKeyForModel(entry: ChainRow, estimatedTokens: number, skipKeys?:
if (!resolvedProvider) { note('no-resolved-provider'); continue; }
roundRobinIndex.set(rrKey, idx);
+ // Taken only once the key has cleared every gate and is definitely being
+ // returned, so a rejected candidate never consumes concurrency budget.
+ const leaseId = acquireLease(entry.platform, entry.model_id, key.id, estimatedTokens);
return {
provider: resolvedProvider,
modelId: entry.model_id,
@@ -737,6 +766,7 @@ function selectKeyForModel(entry: ChainRow, estimatedTokens: number, skipKeys?:
displayName: entry.display_name,
rpdLimit: limits.rpd,
tpdLimit: limits.tpd,
+ release: () => releaseLease(leaseId),
};
}
@@ -788,6 +818,7 @@ export function hasOtherUsableKey(modelDbId: number, excludingKeyId: number, ski
if (skipKeys?.has(`${m.platform}:${m.model_id}:${k.id}`)) continue;
if (isOnCooldown(m.platform, m.model_id, k.id)) continue;
if (!canUseProvider(m.platform, k.id)) continue;
+ if (!canUseProviderMinute(m.platform, k.id)) continue;
if (!canMakeRequest(m.platform, m.model_id, k.id, limits)) continue;
// A per-minute token spike on the failed key doesn't mean a fresh key lacks
// headroom; a nominal 1-token probe only rules out a key already at its
@@ -932,6 +963,7 @@ export function getOrderedFusionChain(): FusionCandidate[] {
(e.key_id == null || kid === e.key_id) &&
!isOnCooldown(e.platform, e.model_id, kid) &&
canUseProvider(e.platform, kid) &&
+ canUseProviderMinute(e.platform, kid) &&
canMakeRequest(e.platform, e.model_id, kid, limits) &&
canUseProviderTokens(e.platform, kid, e.model_id, 1),
);