7.4 billion tokens per month. 34 free LLM providers. 635 free model endpoints. All behind one /v1 endpoint, plus any custom OpenAI-compatible endpoint. Smart routing, automatic failover, encrypted keys. Personal experimentation only.
TypeScript
fix: rate-limit and cooldown state is lost on restart (#88)
Ricardo7cc751a5458d
diff --git a/server/src/__tests__/services/ratelimit.test.ts b/server/src/__tests__/services/ratelimit.test.tsindex f77b9342..85062e36 100644--- a/server/src/__tests__/services/ratelimit.test.ts+++ b/server/src/__tests__/services/ratelimit.test.ts@@ -1,4 +1,5 @@-import { describe, it, expect, beforeEach } from 'vitest';+import fs from 'fs';+import { describe, it, expect, beforeEach, vi } from 'vitest';import {canMakeRequest,canUseTokens,@@ -7,6 +8,16 @@ import {getRateLimitStatus,} from '../../services/ratelimit.js';+function removeDbFile(dbPath: string) {+ for (const suffix of ['', '-shm', '-wal']) {+ try {+ fs.unlinkSync(`${dbPath}${suffix}`);+ } catch {+ // Best-effort cleanup for temp SQLite files.+ }+ }+}+describe('Rate Limiter', () => {// Use unique identifiers per test to avoid cross-contaminationlet testId: number;@@ -77,4 +88,42 @@ describe('Rate Limiter', () => {expect(status.tpm.used).toBe(500);});});++ describe('persistent state', () => {+ it('preserves per-key usage and cooldowns after the limiter module reloads', async () => {+ process.env.ENCRYPTION_KEY = '0'.repeat(64);+ const dbPath = `/tmp/freeapi-ratelimit-${Date.now()}-${Math.random()}.db`;+ const keyId = 4242;+ let db: { close: () => void } | undefined;++ try {+ vi.resetModules();+ const dbModule = await import('../../db/index.js');+ db = dbModule.initDb(dbPath);+ const limiter = await import('../../services/ratelimit.js');++ limiter.recordRequest('groq', 'persistent-model', keyId);+ limiter.recordTokens('groq', 'persistent-model', keyId, 950);+ limiter.setCooldown('groq', 'persistent-model', keyId, 60_000);+ db.close();+ db = undefined;++ vi.resetModules();+ const dbModuleAfterReload = await import('../../db/index.js');+ db = dbModuleAfterReload.initDb(dbPath);+ const limiterAfterReload = await import('../../services/ratelimit.js');++ expect(limiterAfterReload.canMakeRequest('groq', 'persistent-model', keyId, {+ rpm: null, rpd: 1, tpm: null, tpd: null,+ })).toBe(false);+ expect(limiterAfterReload.canUseTokens('groq', 'persistent-model', keyId, 100, {+ tpm: null, tpd: 1000,+ })).toBe(false);+ expect(limiterAfterReload.isOnCooldown('groq', 'persistent-model', keyId)).toBe(true);+ } finally {+ db?.close();+ removeDbFile(dbPath);+ }+ });+ });});diff --git a/server/src/db/index.ts b/server/src/db/index.tsindex 3e8e708f..af5a2bb6 100644--- a/server/src/db/index.ts+++ b/server/src/db/index.ts@@ -92,6 +92,7 @@ function createTables(db: Database.Database) {id INTEGER PRIMARY KEY AUTOINCREMENT,platform TEXT NOT NULL,model_id TEXT NOT NULL,+ key_id INTEGER,status TEXT NOT NULL,input_tokens INTEGER NOT NULL DEFAULT 0,output_tokens INTEGER NOT NULL DEFAULT 0,@@ -100,6 +101,26 @@ function createTables(db: Database.Database) {created_at TEXT NOT NULL DEFAULT (datetime('now')));+ CREATE TABLE IF NOT EXISTS rate_limit_usage (+ id INTEGER PRIMARY KEY AUTOINCREMENT,+ platform TEXT NOT NULL,+ model_id TEXT NOT NULL,+ key_id INTEGER NOT NULL,+ kind TEXT NOT NULL CHECK (kind IN ('request', 'tokens')),+ tokens INTEGER NOT NULL DEFAULT 0,+ created_at_ms INTEGER NOT NULL,+ created_at TEXT NOT NULL DEFAULT (datetime('now'))+ );++ CREATE TABLE IF NOT EXISTS rate_limit_cooldowns (+ platform TEXT NOT NULL,+ model_id TEXT NOT NULL,+ key_id INTEGER NOT NULL,+ expires_at_ms INTEGER NOT NULL,+ created_at TEXT NOT NULL DEFAULT (datetime('now')),+ PRIMARY KEY (platform, model_id, key_id)+ );+CREATE TABLE IF NOT EXISTS fallback_config (id INTEGER PRIMARY KEY AUTOINCREMENT,model_db_id INTEGER NOT NULL REFERENCES models(id),@@ -115,8 +136,20 @@ function createTables(db: Database.Database) {CREATE INDEX IF NOT EXISTS idx_requests_created_at ON requests(created_at);CREATE INDEX IF NOT EXISTS idx_requests_platform ON requests(platform);+ CREATE INDEX IF NOT EXISTS idx_rate_limit_usage_lookup ON rate_limit_usage(platform, model_id, key_id, kind, created_at_ms);+ CREATE INDEX IF NOT EXISTS idx_rate_limit_cooldowns_expires ON rate_limit_cooldowns(expires_at_ms);CREATE INDEX IF NOT EXISTS idx_api_keys_platform ON api_keys(platform);`);++ ensureRequestKeyIdColumn(db);+}++function ensureRequestKeyIdColumn(db: Database.Database) {+ const columns = db.prepare('PRAGMA table_info(requests)').all() as { name: string }[];+ if (!columns.some(col => col.name === 'key_id')) {+ db.prepare('ALTER TABLE requests ADD COLUMN key_id INTEGER').run();+ }+ db.prepare('CREATE INDEX IF NOT EXISTS idx_requests_key_id ON requests(key_id)').run();}function seedModels(db: Database.Database) {diff --git a/server/src/routes/proxy.ts b/server/src/routes/proxy.tsindex 5e7a55f2..892247fd 100644--- a/server/src/routes/proxy.ts+++ b/server/src/routes/proxy.ts@@ -392,7 +392,7 @@ proxyRouter.post('/chat/completions', async (req: Request, res: Response) => {recordTokens(route.platform, route.modelId, route.keyId, estimatedInputTokens + totalOutputTokens);recordSuccess(route.modelDbId);setStickyModel(messages, route.modelDbId);- logRequest(route.platform, route.modelId, 'success', estimatedInputTokens, totalOutputTokens, Date.now() - start, null);+ logRequest(route.platform, route.modelId, route.keyId, 'success', estimatedInputTokens, totalOutputTokens, Date.now() - start, null);return;} catch (streamErr: any) {if (streamStarted) {@@ -404,7 +404,7 @@ proxyRouter.post('/chat/completions', async (req: Request, res: Response) => {const payload = { error: { message: `Provider error (${route.displayName}): stream interrupted`, type: 'stream_error' } };try { res.write(`data: ${JSON.stringify(payload)}\n\n`); } catch { /* socket gone */ }try { res.write('data: [DONE]\n\n'); res.end(); } catch { /* socket gone */ }- logRequest(route.platform, route.modelId, 'error', estimatedInputTokens, totalOutputTokens, Date.now() - start, streamErr.message);+ logRequest(route.platform, route.modelId, route.keyId, 'error', estimatedInputTokens, totalOutputTokens, Date.now() - start, streamErr.message);return;}// Pre-stream error — bubble to outer retry/502 handler.@@ -426,7 +426,7 @@ proxyRouter.post('/chat/completions', async (req: Request, res: Response) => {res.json(result);logRequest(- route.platform, route.modelId, 'success',+ route.platform, route.modelId, route.keyId, 'success',result.usage?.prompt_tokens ?? 0,result.usage?.completion_tokens ?? 0,Date.now() - start, null,@@ -435,7 +435,7 @@ proxyRouter.post('/chat/completions', async (req: Request, res: Response) => {}} catch (err: any) {const latency = Date.now() - start;- logRequest(route.platform, route.modelId, 'error', estimatedInputTokens, 0, latency, err.message);+ logRequest(route.platform, route.modelId, route.keyId, 'error', estimatedInputTokens, 0, latency, err.message);if (isRetryableError(err)) {// Put this model+key on cooldown and try the next one@@ -471,6 +471,7 @@ proxyRouter.post('/chat/completions', async (req: Request, res: Response) => {function logRequest(platform: string,modelId: string,+ keyId: number,status: string,inputTokens: number,outputTokens: number,@@ -480,9 +481,9 @@ function logRequest(try {const db = getDb();db.prepare(`- INSERT INTO requests (platform, model_id, status, input_tokens, output_tokens, latency_ms, error)- VALUES (?, ?, ?, ?, ?, ?, ?)- `).run(platform, modelId, status, inputTokens, outputTokens, latencyMs, error);+ INSERT INTO requests (platform, model_id, key_id, status, input_tokens, output_tokens, latency_ms, error)+ VALUES (?, ?, ?, ?, ?, ?, ?, ?)+ `).run(platform, modelId, keyId, status, inputTokens, outputTokens, latencyMs, error);} catch (e) {console.error('Failed to log request:', e);}diff --git a/server/src/services/ratelimit.ts b/server/src/services/ratelimit.tsindex e495862c..0a8d9a4c 100644--- a/server/src/services/ratelimit.ts+++ b/server/src/services/ratelimit.ts@@ -1,4 +1,6 @@-// In-memory sliding window rate limit tracker+// Sliding window rate limit tracker with SQLite persistence.++import { getDb } from '../db/index.js';interface Window {timestamps: number[];@@ -8,6 +10,8 @@ interface Window {// Key format: "platform:modelId:keyId:type" where type is rpm|rpd|tpm|tpdconst windows = new Map<string, Window>();+type RateLimitDb = ReturnType<typeof getDb>;+type UsageKind = 'request' | 'tokens';function getWindow(key: string): Window {let w = windows.get(key);@@ -26,6 +30,111 @@ function pruneTimestamps(timestamps: number[], windowMs: number, now: number): nconst MINUTE = 60 * 1000;const DAY = 24 * 60 * MINUTE;+function withDb<T>(fn: (db: RateLimitDb) => T): T | undefined {+ try {+ return fn(getDb());+ } catch {+ return undefined;+ }+}++function recordUsage(+ platform: string,+ modelId: string,+ keyId: number,+ kind: UsageKind,+ tokens: number,+ now: number,+) {+ withDb(db => {+ db.prepare(`+ INSERT INTO rate_limit_usage (platform, model_id, key_id, kind, tokens, created_at_ms)+ VALUES (?, ?, ?, ?, ?, ?)+ `).run(platform, modelId, keyId, kind, tokens, now);+ db.prepare('DELETE FROM rate_limit_usage WHERE created_at_ms <= ?').run(now - DAY);+ });+}++function countPersistedRequests(+ platform: string,+ modelId: string,+ keyId: number,+ windowMs: number,+ now: number,+): number | undefined {+ return withDb(db => {+ const row = db.prepare(`+ SELECT COUNT(*) AS used+ FROM rate_limit_usage+ WHERE platform = ?+ AND model_id = ?+ AND key_id = ?+ AND kind = 'request'+ AND created_at_ms > ?+ `).get(platform, modelId, keyId, now - windowMs) as { used: number };+ return row.used;+ });+}++function sumPersistedTokens(+ platform: string,+ modelId: string,+ keyId: number,+ windowMs: number,+ now: number,+): number | undefined {+ return withDb(db => {+ const row = db.prepare(`+ SELECT COALESCE(SUM(tokens), 0) AS used+ FROM rate_limit_usage+ WHERE platform = ?+ AND model_id = ?+ AND key_id = ?+ AND kind = 'tokens'+ AND created_at_ms > ?+ `).get(platform, modelId, keyId, now - windowMs) as { used: number };+ return row.used;+ });+}++function memoryRequestCount(key: string, windowMs: number, now: number): number {+ const w = getWindow(key);+ w.timestamps = pruneTimestamps(w.timestamps, windowMs, now);+ return w.timestamps.length;+}++function memoryTokenCount(key: string, windowMs: number, now: number): number {+ const w = getWindow(key);+ w.tokenTimestamps = w.tokenTimestamps.filter(t => t.ts > now - windowMs);+ return w.tokenTimestamps.reduce((sum, t) => sum + t.tokens, 0);+}++function requestCount(+ platform: string,+ modelId: string,+ keyId: number,+ windowMs: number,+ now: number,+): number {+ const persisted = countPersistedRequests(platform, modelId, keyId, windowMs, now);+ if (persisted !== undefined) return persisted;+ const type = windowMs === MINUTE ? 'rpm' : 'rpd';+ return memoryRequestCount(`${platform}:${modelId}:${keyId}:${type}`, windowMs, now);+}++function tokenCount(+ platform: string,+ modelId: string,+ keyId: number,+ windowMs: number,+ now: number,+): number {+ const persisted = sumPersistedTokens(platform, modelId, keyId, windowMs, now);+ if (persisted !== undefined) return persisted;+ const type = windowMs === MINUTE ? 'tpm' : 'tpd';+ return memoryTokenCount(`${platform}:${modelId}:${keyId}:${type}`, windowMs, now);+}+export function canMakeRequest(platform: string,modelId: string,@@ -35,17 +144,11 @@ export function canMakeRequest(const now = Date.now();if (limits.rpm !== null) {- const key = `${platform}:${modelId}:${keyId}:rpm`;- const w = getWindow(key);- w.timestamps = pruneTimestamps(w.timestamps, MINUTE, now);- if (w.timestamps.length >= limits.rpm) return false;+ if (requestCount(platform, modelId, keyId, MINUTE, now) >= limits.rpm) return false;}if (limits.rpd !== null) {- const key = `${platform}:${modelId}:${keyId}:rpd`;- const w = getWindow(key);- w.timestamps = pruneTimestamps(w.timestamps, DAY, now);- if (w.timestamps.length >= limits.rpd) return false;+ if (requestCount(platform, modelId, keyId, DAY, now) >= limits.rpd) return false;}return true;@@ -61,18 +164,12 @@ export function canUseTokens(const now = Date.now();if (limits.tpm !== null) {- const key = `${platform}:${modelId}:${keyId}:tpm`;- const w = getWindow(key);- w.tokenTimestamps = w.tokenTimestamps.filter(t => t.ts > now - MINUTE);- const used = w.tokenTimestamps.reduce((sum, t) => sum + t.tokens, 0);+ const used = tokenCount(platform, modelId, keyId, MINUTE, now);if (used + estimatedTokens > limits.tpm) return false;}if (limits.tpd !== null) {- const key = `${platform}:${modelId}:${keyId}:tpd`;- const w = getWindow(key);- w.tokenTimestamps = w.tokenTimestamps.filter(t => t.ts > now - DAY);- const used = w.tokenTimestamps.reduce((sum, t) => sum + t.tokens, 0);+ const used = tokenCount(platform, modelId, keyId, DAY, now);if (used + estimatedTokens > limits.tpd) return false;}@@ -87,6 +184,8 @@ export function recordRequest(platform: string, modelId: string, keyId: number)const rpdKey = `${platform}:${modelId}:${keyId}:rpd`;getWindow(rpdKey).timestamps.push(now);++ recordUsage(platform, modelId, keyId, 'request', 0, now);}export function recordTokens(@@ -102,21 +201,76 @@ export function recordTokens(const tpdKey = `${platform}:${modelId}:${keyId}:tpd`;getWindow(tpdKey).tokenTimestamps.push({ ts: now, tokens });++ recordUsage(platform, modelId, keyId, 'tokens', tokens, now);}// Cooldown: when a provider returns 429, block that model+key for a periodconst cooldowns = new Map<string, number>(); // key -> expiry timestamp+function persistedCooldownExpiry(+ platform: string,+ modelId: string,+ keyId: number,+): number | null | undefined {+ return withDb(db => {+ const row = db.prepare(`+ SELECT expires_at_ms+ FROM rate_limit_cooldowns+ WHERE platform = ?+ AND model_id = ?+ AND key_id = ?+ `).get(platform, modelId, keyId) as { expires_at_ms: number } | undefined;+ return row?.expires_at_ms ?? null;+ });+}++function persistCooldown(platform: string, modelId: string, keyId: number, expiresAtMs: number) {+ withDb(db => {+ db.prepare(`+ INSERT INTO rate_limit_cooldowns (platform, model_id, key_id, expires_at_ms)+ VALUES (?, ?, ?, ?)+ ON CONFLICT(platform, model_id, key_id)+ DO UPDATE SET expires_at_ms = excluded.expires_at_ms+ `).run(platform, modelId, keyId, expiresAtMs);+ });+}++function clearPersistedCooldown(platform: string, modelId: string, keyId: number) {+ withDb(db => {+ db.prepare(`+ DELETE FROM rate_limit_cooldowns+ WHERE platform = ?+ AND model_id = ?+ AND key_id = ?+ `).run(platform, modelId, keyId);+ });+}+export function setCooldown(platform: string, modelId: string, keyId: number, durationMs = 60_000) {const key = `${platform}:${modelId}:${keyId}:cooldown`;- cooldowns.set(key, Date.now() + durationMs);+ const expiresAtMs = Date.now() + durationMs;+ cooldowns.set(key, expiresAtMs);+ persistCooldown(platform, modelId, keyId, expiresAtMs);}export function isOnCooldown(platform: string, modelId: string, keyId: number): boolean {const key = `${platform}:${modelId}:${keyId}:cooldown`;+ const now = Date.now();+ const persistedExpiry = persistedCooldownExpiry(platform, modelId, keyId);+ if (persistedExpiry !== undefined && persistedExpiry !== null) {+ if (now > persistedExpiry) {+ cooldowns.delete(key);+ clearPersistedCooldown(platform, modelId, keyId);+ return false;+ }+ cooldowns.set(key, persistedExpiry);+ return true;+ }+const expiry = cooldowns.get(key);if (!expiry) return false;- if (Date.now() > expiry) {+ if (now > expiry) {cooldowns.delete(key);return false;}@@ -131,19 +285,9 @@ export function getRateLimitStatus() {const now = Date.now();- const rpmW = getWindow(`${platform}:${modelId}:${keyId}:rpm`);- rpmW.timestamps = pruneTimestamps(rpmW.timestamps, MINUTE, now);-- const rpdW = getWindow(`${platform}:${modelId}:${keyId}:rpd`);- rpdW.timestamps = pruneTimestamps(rpdW.timestamps, DAY, now);-- const tpmW = getWindow(`${platform}:${modelId}:${keyId}:tpm`);- tpmW.tokenTimestamps = tpmW.tokenTimestamps.filter(t => t.ts > now - MINUTE);- const tpmUsed = tpmW.tokenTimestamps.reduce((sum, t) => sum + t.tokens, 0);-return {- rpm: { used: rpmW.timestamps.length, limit: limits.rpm },- rpd: { used: rpdW.timestamps.length, limit: limits.rpd },- tpm: { used: tpmUsed, limit: limits.tpm },+ rpm: { used: requestCount(platform, modelId, keyId, MINUTE, now), limit: limits.rpm },+ rpd: { used: requestCount(platform, modelId, keyId, DAY, now), limit: limits.rpd },+ tpm: { used: tokenCount(platform, modelId, keyId, MINUTE, now), limit: limits.tpm },};}