Skip to content

7.4 billion tokens per month. 34 free LLM providers. 635 free model endpoints. All behind one /v1 endpoint, plus any custom OpenAI-compatible endpoint. Smart routing, automatic failover, encrypted keys. Personal experimentation only.

TypeScript

fix: rate ledger drift, token double-count, penalty decay, error sanitization

Fuad Hasanbfcef93e7c08
diff --git a/server/src/__tests__/services/router.test.ts b/server/src/__tests__/services/router.test.ts
index edee86b4..a5a72fcf 100644
--- a/server/src/__tests__/services/router.test.ts
+++ b/server/src/__tests__/services/router.test.ts
@@ -1,7 +1,12 @@
-import { describe, it, expect, beforeAll, beforeEach } from 'vitest';
+import { describe, it, expect, beforeAll, beforeEach, afterEach, vi } from 'vitest';
import { initDb, getDb } from '../../db/index.js';
import { encrypt } from '../../lib/crypto.js';
-import { routeRequest, setRoutingStrategy } from '../../services/router.js';
+import {
+ getAllPenalties,
+ recordRateLimitHit,
+ routeRequest,
+ setRoutingStrategy,
+} from '../../services/router.js';
describe('Router', () => {
beforeAll(() => {
@@ -23,6 +28,10 @@ describe('Router', () => {
}
});
+ afterEach(() => {
+ vi.useRealTimers();
+ });
+
it('should throw when no keys are configured', () => {
expect(() => routeRequest()).toThrow(/exhausted/i);
});
@@ -158,4 +167,19 @@ describe('Router', () => {
expect(result.apiKey).toBe('test-groq-key');
expect(corruptKey.status).toBe('error');
});
+
+ it('applies elapsed decay before adding a new 429 penalty', () => {
+ vi.useFakeTimers();
+ const modelDbId = 987654321;
+
+ recordRateLimitHit(modelDbId);
+ vi.advanceTimersByTime(10 * 60 * 1000);
+ recordRateLimitHit(modelDbId);
+
+ expect(getAllPenalties()).toContainEqual({
+ modelDbId,
+ count: 2,
+ penalty: 3,
+ });
+ });
});
diff --git a/server/src/routes/proxy.ts b/server/src/routes/proxy.ts
index 35264061..44b98cd9 100644
--- a/server/src/routes/proxy.ts
+++ b/server/src/routes/proxy.ts
@@ -29,13 +29,14 @@ function isAutoModel(modelId: string | undefined): boolean {
// length and per-character timing, which a network attacker could in principle
// use to recover the key one byte at a time.
export function timingSafeStringEqual(provided: string, expected: string): boolean {
- const a = Buffer.from(provided);
- const b = Buffer.from(expected);
- // Compare against a same-length buffer regardless of input length so the
- // comparison itself runs in constant time; the explicit length check at the
- // end is what actually decides equality when lengths differ.
- const compareA = a.length === b.length ? a : Buffer.alloc(b.length);
- return crypto.timingSafeEqual(compareA, b) && a.length === b.length;
+ // Use HMAC to produce fixed-length digests so timingSafeEqual always
+ // receives same-length buffers regardless of input length. This eliminates
+ // both the per-character timing leak and the length-branch timing leak that
+ // the Buffer.alloc-on-mismatch approach had.
+ const key = Buffer.alloc(32);
+ const a = crypto.createHmac('sha256', key).update(provided).digest();
+ const b = crypto.createHmac('sha256', key).update(expected).digest();
+ return crypto.timingSafeEqual(a, b);
}
// Extract the unified API key from an incoming request. Accepts both the
@@ -652,8 +653,6 @@ proxyRouter.post('/chat/completions', async (req: Request, res: Response) => {
return;
}
- recordRequest(route.platform, route.modelId, route.keyId);
-
try {
if (stream) {
// — Stream turn-integrity (#231 audit) —
@@ -783,7 +782,7 @@ proxyRouter.post('/chat/completions', async (req: Request, res: Response) => {
const probe = heldText.trimStart();
if (startsWithDialectMarker(probe)) {
mode = 'dialect';
- } else if (!couldBecomeDialectMarker(probe) || heldText.length > 256) {
+ } else if (!couldBecomeDialectMarker(probe) || probe.length > 256) {
mode = 'passthrough';
flushHeaders();
writeChunk(mkChunk({ content: heldText }, null));
@@ -836,7 +835,6 @@ proxyRouter.post('/chat/completions', async (req: Request, res: Response) => {
flushHeaders();
if (heldText.length > 0) {
writeChunk(mkChunk({ content: heldText }, null));
- totalOutputTokens += Math.ceil(heldText.length / 4);
}
if (completedCalls.length > 0) {
writeChunk(mkChunk({ tool_calls: completedCalls.map((c, i) => ({ index: i, ...c })) }, null));
@@ -853,6 +851,7 @@ proxyRouter.post('/chat/completions', async (req: Request, res: Response) => {
res.write('data: [DONE]\n\n');
res.end();
+ recordRequest(route.platform, route.modelId, route.keyId);
recordTokens(route.platform, route.modelId, route.keyId, estimatedInputTokens + totalOutputTokens);
recordSuccess(route.modelDbId);
setStickyModel(messages, route.modelDbId, sessionIdHeader);
@@ -920,6 +919,7 @@ proxyRouter.post('/chat/completions', async (req: Request, res: Response) => {
}
const totalTokens = result.usage?.total_tokens ?? 0;
+ recordRequest(route.platform, route.modelId, route.keyId);
recordTokens(route.platform, route.modelId, route.keyId, totalTokens);
recordSuccess(route.modelDbId);
setStickyModel(messages, route.modelDbId, sessionIdHeader);
diff --git a/server/src/routes/responses.ts b/server/src/routes/responses.ts
index f337d82d..fd37dd61 100644
--- a/server/src/routes/responses.ts
+++ b/server/src/routes/responses.ts
@@ -24,6 +24,7 @@ import {
setStickyModel,
logRequest,
} from './proxy.js';
+import { sanitizeProviderErrorMessage } from '../lib/error-redaction.js';
export const responsesRouter = Router();
@@ -359,7 +360,7 @@ responsesRouter.post('/responses', async (req: Request, res: Response) => {
} catch (err: any) {
const status = lastError ? 429 : (err.status ?? 503);
const message = lastError
- ? `All models rate-limited. Last error: ${lastError.message}`
+ ? `All models rate-limited. Last error: ${sanitizeProviderErrorMessage(lastError.message)}`
: err.message;
const type = lastError ? 'rate_limit_error' : 'routing_error';
if (streamStarted) {
@@ -371,8 +372,6 @@ responsesRouter.post('/responses', async (req: Request, res: Response) => {
return;
}
- recordRequest(route.platform, route.modelId, route.keyId);
-
try {
if (stream) {
let outputIndex = 0;
@@ -586,6 +585,7 @@ responsesRouter.post('/responses', async (req: Request, res: Response) => {
sse('response.completed', { response: finalResponse });
res.end();
+ recordRequest(route.platform, route.modelId, route.keyId);
recordTokens(route.platform, route.modelId, route.keyId, estimatedInputTokens + totalOutputTokens);
recordSuccess(route.modelDbId);
setStickyModel(messages, route.modelDbId, sessionIdHeader);
@@ -630,6 +630,7 @@ responsesRouter.post('/responses', async (req: Request, res: Response) => {
continue;
}
+ recordRequest(route.platform, route.modelId, route.keyId);
recordTokens(route.platform, route.modelId, route.keyId, result.usage?.total_tokens ?? 0);
recordSuccess(route.modelDbId);
setStickyModel(messages, route.modelDbId, sessionIdHeader);
@@ -647,7 +648,8 @@ responsesRouter.post('/responses', async (req: Request, res: Response) => {
}
} catch (err: any) {
const latency = Date.now() - start;
- logRequest(route.platform, route.modelId, route.keyId, 'error', estimatedInputTokens, 0, latency, err.message);
+ const safeError = sanitizeProviderErrorMessage(err.message);
+ logRequest(route.platform, route.modelId, route.keyId, 'error', estimatedInputTokens, 0, latency, safeError);
// Mid-stream failures can't be retried (bytes already sent) — close cleanly.
if (stream && streamStarted) {
@@ -669,7 +671,7 @@ responsesRouter.post('/responses', async (req: Request, res: Response) => {
continue;
}
- res.status(502).json({ error: { message: `Provider error (${route.displayName}): ${err.message}`, type: 'provider_error' } });
+ res.status(502).json({ error: { message: `Provider error (${route.displayName}): ${safeError}`, type: 'provider_error' } });
return;
}
}
diff --git a/server/src/services/router.ts b/server/src/services/router.ts
index 48b5cfdf..6e645e7a 100644
--- a/server/src/services/router.ts
+++ b/server/src/services/router.ts
@@ -79,6 +79,8 @@ export function recordRateLimitHit(modelDbId: number) {
const existing = rateLimitPenalties.get(modelDbId);
const now = Date.now();
if (existing) {
+ const decaySteps = Math.floor((now - existing.lastHit) / DECAY_INTERVAL_MS);
+ existing.penalty = Math.max(0, existing.penalty - decaySteps * DECAY_AMOUNT);
existing.count++;
existing.lastHit = now;
existing.penalty = Math.min(existing.penalty + PENALTY_PER_429, MAX_PENALTY);
@@ -102,25 +104,22 @@ export function recordSuccess(modelDbId: number) {
/**
* Get the current penalty for a model (with time-based decay).
+ * Pure read — does not mutate the entry; decay is applied lazily only when
+ * recording a new hit (recordRateLimitHit) so the clock isn't reset on every
+ * routing call.
*/
function getPenalty(modelDbId: number): number {
const entry = rateLimitPenalties.get(modelDbId);
if (!entry) return 0;
- // Apply time-based decay
- const now = Date.now();
- const elapsed = now - entry.lastHit;
+ const elapsed = Date.now() - entry.lastHit;
const decaySteps = Math.floor(elapsed / DECAY_INTERVAL_MS);
- if (decaySteps > 0) {
- entry.penalty = Math.max(0, entry.penalty - (decaySteps * DECAY_AMOUNT));
- entry.lastHit = now; // reset so we don't double-decay
- if (entry.penalty === 0) {
- rateLimitPenalties.delete(modelDbId);
- return 0;
- }
+ const decayed = Math.max(0, entry.penalty - decaySteps * DECAY_AMOUNT);
+ if (decayed === 0) {
+ rateLimitPenalties.delete(modelDbId);
+ return 0;
}
-
- return entry.penalty;
+ return decayed;
}
/**