7.4 billion tokens per month. 34 free LLM providers. 635 free model endpoints. All behind one /v1 endpoint, plus any custom OpenAI-compatible endpoint. Smart routing, automatic failover, encrypted keys. Personal experimentation only.
TypeScript
perf(routing): cache the active chain, platform key counts and key decryption (#1318)
suantea (asuan_suan)b264ffc6530a
diff --git a/server/src/services/router.ts b/server/src/services/router.tsindex 9f17506b..dac55b1a 100644--- a/server/src/services/router.ts+++ b/server/src/services/router.ts@@ -983,11 +983,22 @@ interface ScoredEntry {// must scale to match or the headroom guardrail damps a multi-key model to the// floor after just one account's worth of tokens.function usableKeyCountsByPlatform(db: Db): Map<string, number> {+ // Per-request GROUP BY on api_keys; only changes on key writes. Real-time+ // correctness is backed by selectKeyForModel's live per-key checks — a stale+ // count can at worst walk one extra candidate, never route to a dead key.+ // ponytail: 5s TTL instead of write-point invalidation; per-write hooks if that ever matters.+ if (cachesEnabled() && keyCountsCache && keyCountsCache.db === db && Date.now() - keyCountsCache.at < KEY_COUNTS_TTL_MS) {+ return keyCountsCache.map;+ }const rows = db.prepare("SELECT platform, COUNT(*) AS count FROM api_keys WHERE enabled = 1 AND status IN ('healthy', 'unknown') GROUP BY platform").all() as { platform: string; count: number }[];- return new Map(rows.map(r => [r.platform, r.count]));+ const map = new Map(rows.map(r => [r.platform, r.count]));+ keyCountsCache = { db, at: Date.now(), map };+ return map;}+const KEY_COUNTS_TTL_MS = 5_000;+let keyCountsCache: { db: Db; at: number; map: Map<string, number> } | null = null;function scoreChainEntry(entry: ChainRow,@@ -1178,10 +1189,29 @@ const GLOBAL_SORT_ALIASES: Record<string, string> = {* (`auto:<name>`) correctly refused. `fallback_config` is the chain only for an* install with no profile at all.*/+// Chain cache: the active chain only changes on admin writes (models/profiles/+// fallback_config), not per request — but routeRequest used to re-run the JOIN+// on every request (synchronously, on the event loop). Cache by db+profileId+// with a 1s TTL safety net; a profile switch changes profileId so it takes+// effect immediately. Copy on return: callers like routeRequest splice/unshift.+// Bypassed under NODE_ENV=test: suites write chain tables via raw SQL and route+// immediately afterwards.+let chainCache: { db: Db; profileId: number | null; rows: ChainRow[]; at: number } | null = null;+const CHAIN_CACHE_TTL_MS = 1_000;++function cachesEnabled(): boolean {+ return process.env.NODE_ENV !== 'test';+}+function getActiveChain(db: Db): ChainRow[] {const profileId = getActiveProfileId(db);+ if (cachesEnabled() && chainCache && chainCache.db === db && chainCache.profileId === profileId+ && Date.now() - chainCache.at < CHAIN_CACHE_TTL_MS) {+ return chainCache.rows.slice();+ }+ let rows: ChainRow[];if (profileId != null) {- return db.prepare(`+ rows = db.prepare(`SELECT pm.model_db_id, pm.priority, pm.enabled,m.platform, m.model_id, m.display_name, m.intelligence_rank,m.size_label, m.monthly_token_budget,@@ -1192,18 +1222,20 @@ function getActiveChain(db: Db): ChainRow[] {WHERE pm.profile_id = ?ORDER BY pm.priority ASC`).all(profileId) as ChainRow[];+ } else {+ rows = db.prepare(`+ SELECT fc.model_db_id, fc.priority, fc.enabled,+ m.platform, m.model_id, m.display_name, m.intelligence_rank,+ m.size_label, m.monthly_token_budget,+ m.rpm_limit, m.rpd_limit, m.tpm_limit, m.tpd_limit, m.supports_vision,+ m.supports_tools, m.context_window, m.key_id, m.endpoint_scope+ FROM fallback_config fc+ JOIN models m ON m.id = fc.model_db_id AND m.enabled = 1+ ORDER BY fc.priority ASC+ `).all() as ChainRow[];}-- return db.prepare(`- SELECT fc.model_db_id, fc.priority, fc.enabled,- m.platform, m.model_id, m.display_name, m.intelligence_rank,- m.size_label, m.monthly_token_budget,- m.rpm_limit, m.rpd_limit, m.tpm_limit, m.tpd_limit, m.supports_vision,- m.supports_tools, m.context_window, m.key_id, m.endpoint_scope- FROM fallback_config fc- JOIN models m ON m.id = fc.model_db_id AND m.enabled = 1- ORDER BY fc.priority ASC- `).all() as ChainRow[];+ chainCache = { db, profileId, rows, at: Date.now() };+ return rows.slice();}function getChainByProfileName(db: Db, name: string): ChainRow[] | null {@@ -1411,6 +1443,20 @@ function orderKeysByRemainingQuota(entry: ChainRow, ordered: KeyRow[]): KeyRow[]* Request-level filters (vision/tools/context window) stay in the caller; this* only does key selection + accounting pre-checks.*/+// Key decryption cache: AES-GCM decrypt of every candidate key used to run on+// each routing pass. Keyed by ciphertext — a rotated/updated key changes its+// encrypted_key, so stale entries can never be served and there is no hook to+// wire. ponytail: unbounded Map; a keys page refresh prunes nothing, but one+// entry is ~64 bytes and key counts are human-scale.+const decryptCache = new Map<number, { ct: string; plain: string }>();+function cachedDecrypt(keyId: number, ct: string, iv: string, authTag: string): string {+ const hit = decryptCache.get(keyId);+ if (hit && hit.ct === ct) return hit.plain;+ const plain = decrypt(ct, iv, authTag);+ decryptCache.set(keyId, { ct, plain });+ return plain;+}+function selectKeyForModel(entry: ChainRow, estimatedTokens: number, skipKeys?: Set<string>, diag?: string[]): RouteResult | null {const db = getDb();const label = `${entry.platform}/${entry.model_id}`;@@ -1510,7 +1556,7 @@ function selectKeyForModel(entry: ChainRow, estimatedTokens: number, skipKeys?:let decryptedKey: string;try {- decryptedKey = decrypt(key.encrypted_key, key.iv, key.auth_tag);+ decryptedKey = cachedDecrypt(key.id, key.encrypted_key, key.iv, key.auth_tag);} catch {db.prepare("UPDATE api_keys SET status = 'error', last_checked_at = datetime('now') WHERE id = ?").run(key.id);