Skip to content

7.4 billion tokens per month. 34 free LLM providers. 635 free model endpoints. All behind one /v1 endpoint, plus any custom OpenAI-compatible endpoint. Smart routing, automatic failover, encrypted keys. Personal experimentation only.

TypeScript

fix: rate-limit and cooldown state is lost on restart (#88)

Ricardo7cc751a5458d
diff --git a/server/src/__tests__/services/ratelimit.test.ts b/server/src/__tests__/services/ratelimit.test.ts
index f77b9342..85062e36 100644
--- a/server/src/__tests__/services/ratelimit.test.ts
+++ b/server/src/__tests__/services/ratelimit.test.ts
@@ -1,4 +1,5 @@
-import { describe, it, expect, beforeEach } from 'vitest';
+import fs from 'fs';
+import { describe, it, expect, beforeEach, vi } from 'vitest';
import {
canMakeRequest,
canUseTokens,
@@ -7,6 +8,16 @@ import {
getRateLimitStatus,
} from '../../services/ratelimit.js';
+function removeDbFile(dbPath: string) {
+ for (const suffix of ['', '-shm', '-wal']) {
+ try {
+ fs.unlinkSync(`${dbPath}${suffix}`);
+ } catch {
+ // Best-effort cleanup for temp SQLite files.
+ }
+ }
+}
+
describe('Rate Limiter', () => {
// Use unique identifiers per test to avoid cross-contamination
let testId: number;
@@ -77,4 +88,42 @@ describe('Rate Limiter', () => {
expect(status.tpm.used).toBe(500);
});
});
+
+ describe('persistent state', () => {
+ it('preserves per-key usage and cooldowns after the limiter module reloads', async () => {
+ process.env.ENCRYPTION_KEY = '0'.repeat(64);
+ const dbPath = `/tmp/freeapi-ratelimit-${Date.now()}-${Math.random()}.db`;
+ const keyId = 4242;
+ let db: { close: () => void } | undefined;
+
+ try {
+ vi.resetModules();
+ const dbModule = await import('../../db/index.js');
+ db = dbModule.initDb(dbPath);
+ const limiter = await import('../../services/ratelimit.js');
+
+ limiter.recordRequest('groq', 'persistent-model', keyId);
+ limiter.recordTokens('groq', 'persistent-model', keyId, 950);
+ limiter.setCooldown('groq', 'persistent-model', keyId, 60_000);
+ db.close();
+ db = undefined;
+
+ vi.resetModules();
+ const dbModuleAfterReload = await import('../../db/index.js');
+ db = dbModuleAfterReload.initDb(dbPath);
+ const limiterAfterReload = await import('../../services/ratelimit.js');
+
+ expect(limiterAfterReload.canMakeRequest('groq', 'persistent-model', keyId, {
+ rpm: null, rpd: 1, tpm: null, tpd: null,
+ })).toBe(false);
+ expect(limiterAfterReload.canUseTokens('groq', 'persistent-model', keyId, 100, {
+ tpm: null, tpd: 1000,
+ })).toBe(false);
+ expect(limiterAfterReload.isOnCooldown('groq', 'persistent-model', keyId)).toBe(true);
+ } finally {
+ db?.close();
+ removeDbFile(dbPath);
+ }
+ });
+ });
});
diff --git a/server/src/db/index.ts b/server/src/db/index.ts
index 3e8e708f..af5a2bb6 100644
--- a/server/src/db/index.ts
+++ b/server/src/db/index.ts
@@ -92,6 +92,7 @@ function createTables(db: Database.Database) {
id INTEGER PRIMARY KEY AUTOINCREMENT,
platform TEXT NOT NULL,
model_id TEXT NOT NULL,
+ key_id INTEGER,
status TEXT NOT NULL,
input_tokens INTEGER NOT NULL DEFAULT 0,
output_tokens INTEGER NOT NULL DEFAULT 0,
@@ -100,6 +101,26 @@ function createTables(db: Database.Database) {
created_at TEXT NOT NULL DEFAULT (datetime('now'))
);
+ CREATE TABLE IF NOT EXISTS rate_limit_usage (
+ id INTEGER PRIMARY KEY AUTOINCREMENT,
+ platform TEXT NOT NULL,
+ model_id TEXT NOT NULL,
+ key_id INTEGER NOT NULL,
+ kind TEXT NOT NULL CHECK (kind IN ('request', 'tokens')),
+ tokens INTEGER NOT NULL DEFAULT 0,
+ created_at_ms INTEGER NOT NULL,
+ created_at TEXT NOT NULL DEFAULT (datetime('now'))
+ );
+
+ CREATE TABLE IF NOT EXISTS rate_limit_cooldowns (
+ platform TEXT NOT NULL,
+ model_id TEXT NOT NULL,
+ key_id INTEGER NOT NULL,
+ expires_at_ms INTEGER NOT NULL,
+ created_at TEXT NOT NULL DEFAULT (datetime('now')),
+ PRIMARY KEY (platform, model_id, key_id)
+ );
+
CREATE TABLE IF NOT EXISTS fallback_config (
id INTEGER PRIMARY KEY AUTOINCREMENT,
model_db_id INTEGER NOT NULL REFERENCES models(id),
@@ -115,8 +136,20 @@ function createTables(db: Database.Database) {
CREATE INDEX IF NOT EXISTS idx_requests_created_at ON requests(created_at);
CREATE INDEX IF NOT EXISTS idx_requests_platform ON requests(platform);
+ CREATE INDEX IF NOT EXISTS idx_rate_limit_usage_lookup ON rate_limit_usage(platform, model_id, key_id, kind, created_at_ms);
+ CREATE INDEX IF NOT EXISTS idx_rate_limit_cooldowns_expires ON rate_limit_cooldowns(expires_at_ms);
CREATE INDEX IF NOT EXISTS idx_api_keys_platform ON api_keys(platform);
`);
+
+ ensureRequestKeyIdColumn(db);
+}
+
+function ensureRequestKeyIdColumn(db: Database.Database) {
+ const columns = db.prepare('PRAGMA table_info(requests)').all() as { name: string }[];
+ if (!columns.some(col => col.name === 'key_id')) {
+ db.prepare('ALTER TABLE requests ADD COLUMN key_id INTEGER').run();
+ }
+ db.prepare('CREATE INDEX IF NOT EXISTS idx_requests_key_id ON requests(key_id)').run();
}
function seedModels(db: Database.Database) {
diff --git a/server/src/routes/proxy.ts b/server/src/routes/proxy.ts
index 5e7a55f2..892247fd 100644
--- a/server/src/routes/proxy.ts
+++ b/server/src/routes/proxy.ts
@@ -392,7 +392,7 @@ proxyRouter.post('/chat/completions', async (req: Request, res: Response) => {
recordTokens(route.platform, route.modelId, route.keyId, estimatedInputTokens + totalOutputTokens);
recordSuccess(route.modelDbId);
setStickyModel(messages, route.modelDbId);
- logRequest(route.platform, route.modelId, 'success', estimatedInputTokens, totalOutputTokens, Date.now() - start, null);
+ logRequest(route.platform, route.modelId, route.keyId, 'success', estimatedInputTokens, totalOutputTokens, Date.now() - start, null);
return;
} catch (streamErr: any) {
if (streamStarted) {
@@ -404,7 +404,7 @@ proxyRouter.post('/chat/completions', async (req: Request, res: Response) => {
const payload = { error: { message: `Provider error (${route.displayName}): stream interrupted`, type: 'stream_error' } };
try { res.write(`data: ${JSON.stringify(payload)}\n\n`); } catch { /* socket gone */ }
try { res.write('data: [DONE]\n\n'); res.end(); } catch { /* socket gone */ }
- logRequest(route.platform, route.modelId, 'error', estimatedInputTokens, totalOutputTokens, Date.now() - start, streamErr.message);
+ logRequest(route.platform, route.modelId, route.keyId, 'error', estimatedInputTokens, totalOutputTokens, Date.now() - start, streamErr.message);
return;
}
// Pre-stream error — bubble to outer retry/502 handler.
@@ -426,7 +426,7 @@ proxyRouter.post('/chat/completions', async (req: Request, res: Response) => {
res.json(result);
logRequest(
- route.platform, route.modelId, 'success',
+ route.platform, route.modelId, route.keyId, 'success',
result.usage?.prompt_tokens ?? 0,
result.usage?.completion_tokens ?? 0,
Date.now() - start, null,
@@ -435,7 +435,7 @@ proxyRouter.post('/chat/completions', async (req: Request, res: Response) => {
}
} catch (err: any) {
const latency = Date.now() - start;
- logRequest(route.platform, route.modelId, 'error', estimatedInputTokens, 0, latency, err.message);
+ logRequest(route.platform, route.modelId, route.keyId, 'error', estimatedInputTokens, 0, latency, err.message);
if (isRetryableError(err)) {
// Put this model+key on cooldown and try the next one
@@ -471,6 +471,7 @@ proxyRouter.post('/chat/completions', async (req: Request, res: Response) => {
function logRequest(
platform: string,
modelId: string,
+ keyId: number,
status: string,
inputTokens: number,
outputTokens: number,
@@ -480,9 +481,9 @@ function logRequest(
try {
const db = getDb();
db.prepare(`
- INSERT INTO requests (platform, model_id, status, input_tokens, output_tokens, latency_ms, error)
- VALUES (?, ?, ?, ?, ?, ?, ?)
- `).run(platform, modelId, status, inputTokens, outputTokens, latencyMs, error);
+ INSERT INTO requests (platform, model_id, key_id, status, input_tokens, output_tokens, latency_ms, error)
+ VALUES (?, ?, ?, ?, ?, ?, ?, ?)
+ `).run(platform, modelId, keyId, status, inputTokens, outputTokens, latencyMs, error);
} catch (e) {
console.error('Failed to log request:', e);
}
diff --git a/server/src/services/ratelimit.ts b/server/src/services/ratelimit.ts
index e495862c..0a8d9a4c 100644
--- a/server/src/services/ratelimit.ts
+++ b/server/src/services/ratelimit.ts
@@ -1,4 +1,6 @@
-// In-memory sliding window rate limit tracker
+// Sliding window rate limit tracker with SQLite persistence.
+
+import { getDb } from '../db/index.js';
interface Window {
timestamps: number[];
@@ -8,6 +10,8 @@ interface Window {
// Key format: "platform:modelId:keyId:type" where type is rpm|rpd|tpm|tpd
const windows = new Map<string, Window>();
+type RateLimitDb = ReturnType<typeof getDb>;
+type UsageKind = 'request' | 'tokens';
function getWindow(key: string): Window {
let w = windows.get(key);
@@ -26,6 +30,111 @@ function pruneTimestamps(timestamps: number[], windowMs: number, now: number): n
const MINUTE = 60 * 1000;
const DAY = 24 * 60 * MINUTE;
+function withDb<T>(fn: (db: RateLimitDb) => T): T | undefined {
+ try {
+ return fn(getDb());
+ } catch {
+ return undefined;
+ }
+}
+
+function recordUsage(
+ platform: string,
+ modelId: string,
+ keyId: number,
+ kind: UsageKind,
+ tokens: number,
+ now: number,
+) {
+ withDb(db => {
+ db.prepare(`
+ INSERT INTO rate_limit_usage (platform, model_id, key_id, kind, tokens, created_at_ms)
+ VALUES (?, ?, ?, ?, ?, ?)
+ `).run(platform, modelId, keyId, kind, tokens, now);
+ db.prepare('DELETE FROM rate_limit_usage WHERE created_at_ms <= ?').run(now - DAY);
+ });
+}
+
+function countPersistedRequests(
+ platform: string,
+ modelId: string,
+ keyId: number,
+ windowMs: number,
+ now: number,
+): number | undefined {
+ return withDb(db => {
+ const row = db.prepare(`
+ SELECT COUNT(*) AS used
+ FROM rate_limit_usage
+ WHERE platform = ?
+ AND model_id = ?
+ AND key_id = ?
+ AND kind = 'request'
+ AND created_at_ms > ?
+ `).get(platform, modelId, keyId, now - windowMs) as { used: number };
+ return row.used;
+ });
+}
+
+function sumPersistedTokens(
+ platform: string,
+ modelId: string,
+ keyId: number,
+ windowMs: number,
+ now: number,
+): number | undefined {
+ return withDb(db => {
+ const row = db.prepare(`
+ SELECT COALESCE(SUM(tokens), 0) AS used
+ FROM rate_limit_usage
+ WHERE platform = ?
+ AND model_id = ?
+ AND key_id = ?
+ AND kind = 'tokens'
+ AND created_at_ms > ?
+ `).get(platform, modelId, keyId, now - windowMs) as { used: number };
+ return row.used;
+ });
+}
+
+function memoryRequestCount(key: string, windowMs: number, now: number): number {
+ const w = getWindow(key);
+ w.timestamps = pruneTimestamps(w.timestamps, windowMs, now);
+ return w.timestamps.length;
+}
+
+function memoryTokenCount(key: string, windowMs: number, now: number): number {
+ const w = getWindow(key);
+ w.tokenTimestamps = w.tokenTimestamps.filter(t => t.ts > now - windowMs);
+ return w.tokenTimestamps.reduce((sum, t) => sum + t.tokens, 0);
+}
+
+function requestCount(
+ platform: string,
+ modelId: string,
+ keyId: number,
+ windowMs: number,
+ now: number,
+): number {
+ const persisted = countPersistedRequests(platform, modelId, keyId, windowMs, now);
+ if (persisted !== undefined) return persisted;
+ const type = windowMs === MINUTE ? 'rpm' : 'rpd';
+ return memoryRequestCount(`${platform}:${modelId}:${keyId}:${type}`, windowMs, now);
+}
+
+function tokenCount(
+ platform: string,
+ modelId: string,
+ keyId: number,
+ windowMs: number,
+ now: number,
+): number {
+ const persisted = sumPersistedTokens(platform, modelId, keyId, windowMs, now);
+ if (persisted !== undefined) return persisted;
+ const type = windowMs === MINUTE ? 'tpm' : 'tpd';
+ return memoryTokenCount(`${platform}:${modelId}:${keyId}:${type}`, windowMs, now);
+}
+
export function canMakeRequest(
platform: string,
modelId: string,
@@ -35,17 +144,11 @@ export function canMakeRequest(
const now = Date.now();
if (limits.rpm !== null) {
- const key = `${platform}:${modelId}:${keyId}:rpm`;
- const w = getWindow(key);
- w.timestamps = pruneTimestamps(w.timestamps, MINUTE, now);
- if (w.timestamps.length >= limits.rpm) return false;
+ if (requestCount(platform, modelId, keyId, MINUTE, now) >= limits.rpm) return false;
}
if (limits.rpd !== null) {
- const key = `${platform}:${modelId}:${keyId}:rpd`;
- const w = getWindow(key);
- w.timestamps = pruneTimestamps(w.timestamps, DAY, now);
- if (w.timestamps.length >= limits.rpd) return false;
+ if (requestCount(platform, modelId, keyId, DAY, now) >= limits.rpd) return false;
}
return true;
@@ -61,18 +164,12 @@ export function canUseTokens(
const now = Date.now();
if (limits.tpm !== null) {
- const key = `${platform}:${modelId}:${keyId}:tpm`;
- const w = getWindow(key);
- w.tokenTimestamps = w.tokenTimestamps.filter(t => t.ts > now - MINUTE);
- const used = w.tokenTimestamps.reduce((sum, t) => sum + t.tokens, 0);
+ const used = tokenCount(platform, modelId, keyId, MINUTE, now);
if (used + estimatedTokens > limits.tpm) return false;
}
if (limits.tpd !== null) {
- const key = `${platform}:${modelId}:${keyId}:tpd`;
- const w = getWindow(key);
- w.tokenTimestamps = w.tokenTimestamps.filter(t => t.ts > now - DAY);
- const used = w.tokenTimestamps.reduce((sum, t) => sum + t.tokens, 0);
+ const used = tokenCount(platform, modelId, keyId, DAY, now);
if (used + estimatedTokens > limits.tpd) return false;
}
@@ -87,6 +184,8 @@ export function recordRequest(platform: string, modelId: string, keyId: number)
const rpdKey = `${platform}:${modelId}:${keyId}:rpd`;
getWindow(rpdKey).timestamps.push(now);
+
+ recordUsage(platform, modelId, keyId, 'request', 0, now);
}
export function recordTokens(
@@ -102,21 +201,76 @@ export function recordTokens(
const tpdKey = `${platform}:${modelId}:${keyId}:tpd`;
getWindow(tpdKey).tokenTimestamps.push({ ts: now, tokens });
+
+ recordUsage(platform, modelId, keyId, 'tokens', tokens, now);
}
// Cooldown: when a provider returns 429, block that model+key for a period
const cooldowns = new Map<string, number>(); // key -> expiry timestamp
+function persistedCooldownExpiry(
+ platform: string,
+ modelId: string,
+ keyId: number,
+): number | null | undefined {
+ return withDb(db => {
+ const row = db.prepare(`
+ SELECT expires_at_ms
+ FROM rate_limit_cooldowns
+ WHERE platform = ?
+ AND model_id = ?
+ AND key_id = ?
+ `).get(platform, modelId, keyId) as { expires_at_ms: number } | undefined;
+ return row?.expires_at_ms ?? null;
+ });
+}
+
+function persistCooldown(platform: string, modelId: string, keyId: number, expiresAtMs: number) {
+ withDb(db => {
+ db.prepare(`
+ INSERT INTO rate_limit_cooldowns (platform, model_id, key_id, expires_at_ms)
+ VALUES (?, ?, ?, ?)
+ ON CONFLICT(platform, model_id, key_id)
+ DO UPDATE SET expires_at_ms = excluded.expires_at_ms
+ `).run(platform, modelId, keyId, expiresAtMs);
+ });
+}
+
+function clearPersistedCooldown(platform: string, modelId: string, keyId: number) {
+ withDb(db => {
+ db.prepare(`
+ DELETE FROM rate_limit_cooldowns
+ WHERE platform = ?
+ AND model_id = ?
+ AND key_id = ?
+ `).run(platform, modelId, keyId);
+ });
+}
+
export function setCooldown(platform: string, modelId: string, keyId: number, durationMs = 60_000) {
const key = `${platform}:${modelId}:${keyId}:cooldown`;
- cooldowns.set(key, Date.now() + durationMs);
+ const expiresAtMs = Date.now() + durationMs;
+ cooldowns.set(key, expiresAtMs);
+ persistCooldown(platform, modelId, keyId, expiresAtMs);
}
export function isOnCooldown(platform: string, modelId: string, keyId: number): boolean {
const key = `${platform}:${modelId}:${keyId}:cooldown`;
+ const now = Date.now();
+ const persistedExpiry = persistedCooldownExpiry(platform, modelId, keyId);
+ if (persistedExpiry !== undefined && persistedExpiry !== null) {
+ if (now > persistedExpiry) {
+ cooldowns.delete(key);
+ clearPersistedCooldown(platform, modelId, keyId);
+ return false;
+ }
+ cooldowns.set(key, persistedExpiry);
+ return true;
+ }
+
const expiry = cooldowns.get(key);
if (!expiry) return false;
- if (Date.now() > expiry) {
+ if (now > expiry) {
cooldowns.delete(key);
return false;
}
@@ -131,19 +285,9 @@ export function getRateLimitStatus(
) {
const now = Date.now();
- const rpmW = getWindow(`${platform}:${modelId}:${keyId}:rpm`);
- rpmW.timestamps = pruneTimestamps(rpmW.timestamps, MINUTE, now);
-
- const rpdW = getWindow(`${platform}:${modelId}:${keyId}:rpd`);
- rpdW.timestamps = pruneTimestamps(rpdW.timestamps, DAY, now);
-
- const tpmW = getWindow(`${platform}:${modelId}:${keyId}:tpm`);
- tpmW.tokenTimestamps = tpmW.tokenTimestamps.filter(t => t.ts > now - MINUTE);
- const tpmUsed = tpmW.tokenTimestamps.reduce((sum, t) => sum + t.tokens, 0);
-
return {
- rpm: { used: rpmW.timestamps.length, limit: limits.rpm },
- rpd: { used: rpdW.timestamps.length, limit: limits.rpd },
- tpm: { used: tpmUsed, limit: limits.tpm },
+ rpm: { used: requestCount(platform, modelId, keyId, MINUTE, now), limit: limits.rpm },
+ rpd: { used: requestCount(platform, modelId, keyId, DAY, now), limit: limits.rpd },
+ tpm: { used: tokenCount(platform, modelId, keyId, MINUTE, now), limit: limits.tpm },
};
}