7.4 billion tokens per month. 34 free LLM providers. 635 free model endpoints. All behind one /v1 endpoint, plus any custom OpenAI-compatible endpoint. Smart routing, automatic failover, encrypted keys. Personal experimentation only.
TypeScript
fix: rate ledger drift, token double-count, penalty decay, error sanitization
Fuad Hasanbfcef93e7c08
diff --git a/server/src/__tests__/services/router.test.ts b/server/src/__tests__/services/router.test.tsindex edee86b4..a5a72fcf 100644--- a/server/src/__tests__/services/router.test.ts+++ b/server/src/__tests__/services/router.test.ts@@ -1,7 +1,12 @@-import { describe, it, expect, beforeAll, beforeEach } from 'vitest';+import { describe, it, expect, beforeAll, beforeEach, afterEach, vi } from 'vitest';import { initDb, getDb } from '../../db/index.js';import { encrypt } from '../../lib/crypto.js';-import { routeRequest, setRoutingStrategy } from '../../services/router.js';+import {+ getAllPenalties,+ recordRateLimitHit,+ routeRequest,+ setRoutingStrategy,+} from '../../services/router.js';describe('Router', () => {beforeAll(() => {@@ -23,6 +28,10 @@ describe('Router', () => {}});+ afterEach(() => {+ vi.useRealTimers();+ });+it('should throw when no keys are configured', () => {expect(() => routeRequest()).toThrow(/exhausted/i);});@@ -158,4 +167,19 @@ describe('Router', () => {expect(result.apiKey).toBe('test-groq-key');expect(corruptKey.status).toBe('error');});++ it('applies elapsed decay before adding a new 429 penalty', () => {+ vi.useFakeTimers();+ const modelDbId = 987654321;++ recordRateLimitHit(modelDbId);+ vi.advanceTimersByTime(10 * 60 * 1000);+ recordRateLimitHit(modelDbId);++ expect(getAllPenalties()).toContainEqual({+ modelDbId,+ count: 2,+ penalty: 3,+ });+ });});diff --git a/server/src/routes/proxy.ts b/server/src/routes/proxy.tsindex 35264061..44b98cd9 100644--- a/server/src/routes/proxy.ts+++ b/server/src/routes/proxy.ts@@ -29,13 +29,14 @@ function isAutoModel(modelId: string | undefined): boolean {// length and per-character timing, which a network attacker could in principle// use to recover the key one byte at a time.export function timingSafeStringEqual(provided: string, expected: string): boolean {- const a = Buffer.from(provided);- const b = Buffer.from(expected);- // Compare against a same-length buffer regardless of input length so the- // comparison itself runs in constant time; the explicit length check at the- // end is what actually decides equality when lengths differ.- const compareA = a.length === b.length ? a : Buffer.alloc(b.length);- return crypto.timingSafeEqual(compareA, b) && a.length === b.length;+ // Use HMAC to produce fixed-length digests so timingSafeEqual always+ // receives same-length buffers regardless of input length. This eliminates+ // both the per-character timing leak and the length-branch timing leak that+ // the Buffer.alloc-on-mismatch approach had.+ const key = Buffer.alloc(32);+ const a = crypto.createHmac('sha256', key).update(provided).digest();+ const b = crypto.createHmac('sha256', key).update(expected).digest();+ return crypto.timingSafeEqual(a, b);}// Extract the unified API key from an incoming request. Accepts both the@@ -652,8 +653,6 @@ proxyRouter.post('/chat/completions', async (req: Request, res: Response) => {return;}- recordRequest(route.platform, route.modelId, route.keyId);-try {if (stream) {// — Stream turn-integrity (#231 audit) —@@ -783,7 +782,7 @@ proxyRouter.post('/chat/completions', async (req: Request, res: Response) => {const probe = heldText.trimStart();if (startsWithDialectMarker(probe)) {mode = 'dialect';- } else if (!couldBecomeDialectMarker(probe) || heldText.length > 256) {+ } else if (!couldBecomeDialectMarker(probe) || probe.length > 256) {mode = 'passthrough';flushHeaders();writeChunk(mkChunk({ content: heldText }, null));@@ -836,7 +835,6 @@ proxyRouter.post('/chat/completions', async (req: Request, res: Response) => {flushHeaders();if (heldText.length > 0) {writeChunk(mkChunk({ content: heldText }, null));- totalOutputTokens += Math.ceil(heldText.length / 4);}if (completedCalls.length > 0) {writeChunk(mkChunk({ tool_calls: completedCalls.map((c, i) => ({ index: i, ...c })) }, null));@@ -853,6 +851,7 @@ proxyRouter.post('/chat/completions', async (req: Request, res: Response) => {res.write('data: [DONE]\n\n');res.end();+ recordRequest(route.platform, route.modelId, route.keyId);recordTokens(route.platform, route.modelId, route.keyId, estimatedInputTokens + totalOutputTokens);recordSuccess(route.modelDbId);setStickyModel(messages, route.modelDbId, sessionIdHeader);@@ -920,6 +919,7 @@ proxyRouter.post('/chat/completions', async (req: Request, res: Response) => {}const totalTokens = result.usage?.total_tokens ?? 0;+ recordRequest(route.platform, route.modelId, route.keyId);recordTokens(route.platform, route.modelId, route.keyId, totalTokens);recordSuccess(route.modelDbId);setStickyModel(messages, route.modelDbId, sessionIdHeader);diff --git a/server/src/routes/responses.ts b/server/src/routes/responses.tsindex f337d82d..fd37dd61 100644--- a/server/src/routes/responses.ts+++ b/server/src/routes/responses.ts@@ -24,6 +24,7 @@ import {setStickyModel,logRequest,} from './proxy.js';+import { sanitizeProviderErrorMessage } from '../lib/error-redaction.js';export const responsesRouter = Router();@@ -359,7 +360,7 @@ responsesRouter.post('/responses', async (req: Request, res: Response) => {} catch (err: any) {const status = lastError ? 429 : (err.status ?? 503);const message = lastError- ? `All models rate-limited. Last error: ${lastError.message}`+ ? `All models rate-limited. Last error: ${sanitizeProviderErrorMessage(lastError.message)}`: err.message;const type = lastError ? 'rate_limit_error' : 'routing_error';if (streamStarted) {@@ -371,8 +372,6 @@ responsesRouter.post('/responses', async (req: Request, res: Response) => {return;}- recordRequest(route.platform, route.modelId, route.keyId);-try {if (stream) {let outputIndex = 0;@@ -586,6 +585,7 @@ responsesRouter.post('/responses', async (req: Request, res: Response) => {sse('response.completed', { response: finalResponse });res.end();+ recordRequest(route.platform, route.modelId, route.keyId);recordTokens(route.platform, route.modelId, route.keyId, estimatedInputTokens + totalOutputTokens);recordSuccess(route.modelDbId);setStickyModel(messages, route.modelDbId, sessionIdHeader);@@ -630,6 +630,7 @@ responsesRouter.post('/responses', async (req: Request, res: Response) => {continue;}+ recordRequest(route.platform, route.modelId, route.keyId);recordTokens(route.platform, route.modelId, route.keyId, result.usage?.total_tokens ?? 0);recordSuccess(route.modelDbId);setStickyModel(messages, route.modelDbId, sessionIdHeader);@@ -647,7 +648,8 @@ responsesRouter.post('/responses', async (req: Request, res: Response) => {}} catch (err: any) {const latency = Date.now() - start;- logRequest(route.platform, route.modelId, route.keyId, 'error', estimatedInputTokens, 0, latency, err.message);+ const safeError = sanitizeProviderErrorMessage(err.message);+ logRequest(route.platform, route.modelId, route.keyId, 'error', estimatedInputTokens, 0, latency, safeError);// Mid-stream failures can't be retried (bytes already sent) — close cleanly.if (stream && streamStarted) {@@ -669,7 +671,7 @@ responsesRouter.post('/responses', async (req: Request, res: Response) => {continue;}- res.status(502).json({ error: { message: `Provider error (${route.displayName}): ${err.message}`, type: 'provider_error' } });+ res.status(502).json({ error: { message: `Provider error (${route.displayName}): ${safeError}`, type: 'provider_error' } });return;}}diff --git a/server/src/services/router.ts b/server/src/services/router.tsindex 48b5cfdf..6e645e7a 100644--- a/server/src/services/router.ts+++ b/server/src/services/router.ts@@ -79,6 +79,8 @@ export function recordRateLimitHit(modelDbId: number) {const existing = rateLimitPenalties.get(modelDbId);const now = Date.now();if (existing) {+ const decaySteps = Math.floor((now - existing.lastHit) / DECAY_INTERVAL_MS);+ existing.penalty = Math.max(0, existing.penalty - decaySteps * DECAY_AMOUNT);existing.count++;existing.lastHit = now;existing.penalty = Math.min(existing.penalty + PENALTY_PER_429, MAX_PENALTY);@@ -102,25 +104,22 @@ export function recordSuccess(modelDbId: number) {/*** Get the current penalty for a model (with time-based decay).+ * Pure read — does not mutate the entry; decay is applied lazily only when+ * recording a new hit (recordRateLimitHit) so the clock isn't reset on every+ * routing call.*/function getPenalty(modelDbId: number): number {const entry = rateLimitPenalties.get(modelDbId);if (!entry) return 0;- // Apply time-based decay- const now = Date.now();- const elapsed = now - entry.lastHit;+ const elapsed = Date.now() - entry.lastHit;const decaySteps = Math.floor(elapsed / DECAY_INTERVAL_MS);- if (decaySteps > 0) {- entry.penalty = Math.max(0, entry.penalty - (decaySteps * DECAY_AMOUNT));- entry.lastHit = now; // reset so we don't double-decay- if (entry.penalty === 0) {- rateLimitPenalties.delete(modelDbId);- return 0;- }+ const decayed = Math.max(0, entry.penalty - decaySteps * DECAY_AMOUNT);+ if (decayed === 0) {+ rateLimitPenalties.delete(modelDbId);+ return 0;}-- return entry.penalty;+ return decayed;}/**