7.4 billion tokens per month. 34 free LLM providers. 635 free model endpoints. All behind one /v1 endpoint, plus any custom OpenAI-compatible endpoint. Smart routing, automatic failover, encrypted keys. Personal experimentation only.
TypeScript
Respect NavyAI shared token cap (#545)
Tashfeene7b74a563d4e
diff --git a/server/src/__tests__/services/ratelimit.test.ts b/server/src/__tests__/services/ratelimit.test.tsindex 13f3392d..213059f9 100644--- a/server/src/__tests__/services/ratelimit.test.ts+++ b/server/src/__tests__/services/ratelimit.test.ts@@ -1,5 +1,6 @@import fs from 'fs';import { describe, it, expect, beforeEach, afterEach, vi } from 'vitest';+import { initDb, getDb } from '../../db/index.js';import {canMakeRequest,canUseTokens,@@ -10,8 +11,11 @@ import {getCooldownDurationForLimit,recentHitCount,canUseProvider,+ canUseProviderTokens,providerDailyRequestCount,getProviderDailyRequestCap,+ providerDailyTokenCount,+ getProviderDailyTokenCap,} from '../../services/ratelimit.js';import { parseRetryAfterMs } from '../../providers/base.js';@@ -285,6 +289,78 @@ describe('Rate Limiter', () => {expect(canUseProvider('openrouter', testId)).toBe(false); // 3 >= 3});});++ describe('provider-wide daily token cap (NavyAI)', () => {+ const ENV = 'PROVIDER_DAILY_TOKEN_CAP_NAVY';+ let original: string | undefined;+ let dbReady = false;++ function ensureDb() {+ if (dbReady) return;+ process.env.ENCRYPTION_KEY = '0'.repeat(64);+ initDb(':memory:');+ dbReady = true;+ }++ function seedNavyModel(modelId: string, tpdLimit: number, monthlyTokenBudget: string) {+ getDb().prepare(`+ INSERT INTO models (+ platform, model_id, display_name, intelligence_rank, speed_rank,+ size_label, rpm_limit, rpd_limit, tpm_limit, tpd_limit,+ monthly_token_budget, context_window, enabled, supports_vision,+ supports_tools+ ) VALUES (+ 'navy', ?, ?, 1, 1, 'Large', 20, NULL, NULL, ?,+ ?, NULL, 1, 0, 1+ )+ ON CONFLICT(platform, model_id) DO UPDATE SET+ tpd_limit = excluded.tpd_limit,+ monthly_token_budget = excluded.monthly_token_budget+ `).run(modelId, `${modelId} (NavyAI)`, tpdLimit, monthlyTokenBudget);+ }++ beforeEach(() => {+ ensureDb();+ original = process.env[ENV];+ getDb().prepare('DELETE FROM rate_limit_usage').run();+ getDb().prepare("DELETE FROM models WHERE platform = 'navy'").run();+ delete process.env[ENV];+ });++ afterEach(() => {+ if (original === undefined) delete process.env[ENV];+ else process.env[ENV] = original;+ });++ it('defaults to NavyAI 150K tokens/day and allows env override / disable', () => {+ expect(getProviderDailyTokenCap('navy')).toBe(150_000);+ expect(getProviderDailyTokenCap('groq')).toBeNull();+ process.env[ENV] = '5000';+ expect(getProviderDailyTokenCap('navy')).toBe(5000);+ process.env[ENV] = '0';+ expect(getProviderDailyTokenCap('navy')).toBeNull();+ });++ it('counts NavyAI tokens across all models using catalog multiplier hints', () => {+ seedNavyModel('gpt-5.4', 33_333, '150K/day shared \u00b7 4.5x');+ seedNavyModel('llama-3.3-70b-instruct', 150_000, '150K/day shared \u00b7 1x');++ recordTokens('navy', 'gpt-5.4', testId, 1000);+ recordTokens('navy', 'llama-3.3-70b-instruct', testId, 1000);++ expect(providerDailyTokenCount('navy', testId)).toBe(5500);+ });++ it('blocks the provider key when multiplier-adjusted tokens would exceed the shared cap', () => {+ process.env[ENV] = '5000';+ seedNavyModel('gemini-2.5-flash', 75_000, '150K/day shared \u00b7 2x');++ recordTokens('navy', 'gemini-2.5-flash', testId, 2000);++ expect(canUseProviderTokens('navy', testId, 'gemini-2.5-flash', 400)).toBe(true);+ expect(canUseProviderTokens('navy', testId, 'gemini-2.5-flash', 600)).toBe(false);+ });+ });});describe('Cooldown duration with upstream Retry-After', () => {diff --git a/server/src/services/ratelimit.ts b/server/src/services/ratelimit.tsindex ccdfac21..b05b6c8a 100644--- a/server/src/services/ratelimit.ts+++ b/server/src/services/ratelimit.ts@@ -190,6 +190,13 @@ const DEFAULT_PROVIDER_DAILY_REQUEST_CAPS: Record<string, number> = {openrouter: 1000,};+const DEFAULT_PROVIDER_DAILY_TOKEN_CAPS: Record<string, number> = {+ // NavyAI's free plan is one shared 150K-token/day pool per key. Individual+ // models carry token_multiplier values, so the provider-visible drain can be+ // higher than the OpenAI usage.total_tokens returned by /chat/completions.+ navy: 150_000,+};+export function getProviderDailyRequestCap(platform: string): number | null {const raw = process.env[`PROVIDER_DAILY_REQUEST_CAP_${platform.toUpperCase()}`];if (raw !== undefined && raw.trim() !== '') {@@ -199,6 +206,15 @@ export function getProviderDailyRequestCap(platform: string): number | null {return DEFAULT_PROVIDER_DAILY_REQUEST_CAPS[platform] ?? null;}+export function getProviderDailyTokenCap(platform: string): number | null {+ const raw = process.env[`PROVIDER_DAILY_TOKEN_CAP_${platform.toUpperCase()}`];+ if (raw !== undefined && raw.trim() !== '') {+ const n = Number(raw);+ if (Number.isFinite(n) && n >= 0) return n === 0 ? null : n;+ }+ return DEFAULT_PROVIDER_DAILY_TOKEN_CAPS[platform] ?? null;+}+function countPersistedProviderRequests(platform: string,keyId: number,@@ -241,6 +257,101 @@ export function canUseProvider(platform: string, keyId: number, now = Date.now()return providerDailyRequestCount(platform, keyId, now) < cap;}+type ModelQuotaRow = { tpd_limit: number | null; monthly_token_budget: string | null };++function multiplierFromQuotaRow(row: ModelQuotaRow | undefined, dailyCap: number): number {+ if (!row) return 1;++ const fromLabel = row.monthly_token_budget?.match(/(?:^|[\u00b7(\s])(\d+(?:\.\d+)?)x\b/i);+ if (fromLabel) {+ const n = Number(fromLabel[1]);+ if (Number.isFinite(n) && n > 0) return n;+ }++ const tpd = row.tpd_limit;+ if (tpd != null && tpd > 0 && tpd < dailyCap) return dailyCap / tpd;+ return 1;+}++function getProviderTokenMultiplier(platform: string, modelId: string, dailyCap: number): number {+ if (platform !== 'navy') return 1;+ return withDb(db => {+ const row = db.prepare(`+ SELECT tpd_limit, monthly_token_budget+ FROM models+ WHERE platform = ? AND model_id = ?+ LIMIT 1+ `).get(platform, modelId) as ModelQuotaRow | undefined;+ return multiplierFromQuotaRow(row, dailyCap);+ }) ?? 1;+}++function providerBilledTokens(platform: string, modelId: string, rawTokens: number): number {+ if (rawTokens <= 0) return 0;+ const dailyCap = getProviderDailyTokenCap(platform);+ if (dailyCap === null) return rawTokens;+ const multiplier = getProviderTokenMultiplier(platform, modelId, dailyCap);+ return Math.ceil(rawTokens * multiplier);+}++function sumPersistedProviderTokens(+ platform: string,+ keyId: number,+ windowMs: number,+ now: number,+): number | undefined {+ const dailyCap = getProviderDailyTokenCap(platform);+ if (dailyCap === null) return 0;++ return withDb(db => {+ const rows = db.prepare(`+ SELECT rlu.model_id, COALESCE(SUM(rlu.tokens), 0) AS used,+ m.tpd_limit, m.monthly_token_budget+ FROM rate_limit_usage rlu+ LEFT JOIN models m ON m.platform = rlu.platform AND m.model_id = rlu.model_id+ WHERE rlu.platform = ?+ AND rlu.key_id = ?+ AND rlu.kind = 'tokens'+ AND rlu.created_at_ms > ?+ GROUP BY rlu.model_id+ `).all(platform, keyId, now - windowMs) as (ModelQuotaRow & { model_id: string; used: number })[];++ return rows.reduce((sum, row) => {+ const multiplier = platform === 'navy' ? multiplierFromQuotaRow(row, dailyCap) : 1;+ return sum + Math.ceil(row.used * multiplier);+ }, 0);+ });+}++export function providerDailyTokenCount(platform: string, keyId: number, now = Date.now()): number {+ const persisted = sumPersistedProviderTokens(platform, keyId, DAY, now);+ if (persisted !== undefined) return persisted;++ let total = 0;+ const suffix = `:${keyId}:tpd`;+ for (const [key, w] of windows) {+ if (!key.startsWith(`${platform}:`) || !key.endsWith(suffix)) continue;+ const modelId = key.slice(platform.length + 1, -suffix.length);+ w.tokenTimestamps = w.tokenTimestamps.filter(t => t.ts > now - DAY);+ const raw = w.tokenTimestamps.reduce((sum, t) => sum + t.tokens, 0);+ total += providerBilledTokens(platform, modelId, raw);+ }+ return total;+}++export function canUseProviderTokens(+ platform: string,+ keyId: number,+ modelId: string,+ estimatedTokens: number,+ now = Date.now(),+): boolean {+ const cap = getProviderDailyTokenCap(platform);+ if (cap === null) return true;+ const used = providerDailyTokenCount(platform, keyId, now);+ return used + providerBilledTokens(platform, modelId, estimatedTokens) <= cap;+}+export function recordRequest(platform: string, modelId: string, keyId: number) {const now = Date.now();diff --git a/server/src/services/router.ts b/server/src/services/router.tsindex 915ec250..e137171f 100644--- a/server/src/services/router.ts+++ b/server/src/services/router.ts@@ -1,7 +1,14 @@import { getDb, getSetting, setSetting } from '../db/index.js';import { getProvider, hasProvider, resolveProvider } from '../providers/index.js';import { decrypt } from '../lib/crypto.js';-import { canMakeRequest, canUseTokens, isOnCooldown, canUseProvider, getSoonestCooldownExpiry } from './ratelimit.js';+import {+ canMakeRequest,+ canUseTokens,+ isOnCooldown,+ canUseProvider,+ canUseProviderTokens,+ getSoonestCooldownExpiry,+} from './ratelimit.js';import {BANDIT_PRESETS, DEFAULT_STRATEGY, type RoutingStrategy, type RoutingWeights,reliabilityPosterior, expectedReliability, sampleBeta,@@ -702,6 +709,7 @@ function selectKeyForModel(entry: ChainRow, estimatedTokens: number, skipKeys?:if (!canUseProvider(entry.platform, key.id)) { note('provider-daily-cap'); continue; }if (!canMakeRequest(entry.platform, entry.model_id, key.id, limits)) { note('rpm/rpd-limit'); continue; }if (!canUseTokens(entry.platform, entry.model_id, key.id, estimatedTokens, limits)) { note('tpm/tpd-limit'); continue; }+ if (!canUseProviderTokens(entry.platform, key.id, entry.model_id, estimatedTokens)) { note('provider-daily-token-cap'); continue; }let decryptedKey: string;try {@@ -785,6 +793,7 @@ export function hasOtherUsableKey(modelDbId: number, excludingKeyId: number, ski// headroom; a nominal 1-token probe only rules out a key already at its// TPM/TPD ceiling.if (!canUseTokens(m.platform, m.model_id, k.id, 1, limits)) continue;+ if (!canUseProviderTokens(m.platform, k.id, m.model_id, 1)) continue;return true;}return false;@@ -923,7 +932,8 @@ export function getOrderedFusionChain(): FusionCandidate[] {(e.key_id == null || kid === e.key_id) &&!isOnCooldown(e.platform, e.model_id, kid) &&canUseProvider(e.platform, kid) &&- canMakeRequest(e.platform, e.model_id, kid, limits),+ canMakeRequest(e.platform, e.model_id, kid, limits) &&+ canUseProviderTokens(e.platform, kid, e.model_id, 1),);});