Skip to content

7.4 billion tokens per month. 34 free LLM providers. 635 free model endpoints. All behind one /v1 endpoint, plus any custom OpenAI-compatible endpoint. Smart routing, automatic failover, encrypted keys. Personal experimentation only.

TypeScript

Respect NavyAI shared token cap (#545)

Tashfeene7b74a563d4e
diff --git a/server/src/__tests__/services/ratelimit.test.ts b/server/src/__tests__/services/ratelimit.test.ts
index 13f3392d..213059f9 100644
--- a/server/src/__tests__/services/ratelimit.test.ts
+++ b/server/src/__tests__/services/ratelimit.test.ts
@@ -1,5 +1,6 @@
import fs from 'fs';
import { describe, it, expect, beforeEach, afterEach, vi } from 'vitest';
+import { initDb, getDb } from '../../db/index.js';
import {
canMakeRequest,
canUseTokens,
@@ -10,8 +11,11 @@ import {
getCooldownDurationForLimit,
recentHitCount,
canUseProvider,
+ canUseProviderTokens,
providerDailyRequestCount,
getProviderDailyRequestCap,
+ providerDailyTokenCount,
+ getProviderDailyTokenCap,
} from '../../services/ratelimit.js';
import { parseRetryAfterMs } from '../../providers/base.js';
@@ -285,6 +289,78 @@ describe('Rate Limiter', () => {
expect(canUseProvider('openrouter', testId)).toBe(false); // 3 >= 3
});
});
+
+ describe('provider-wide daily token cap (NavyAI)', () => {
+ const ENV = 'PROVIDER_DAILY_TOKEN_CAP_NAVY';
+ let original: string | undefined;
+ let dbReady = false;
+
+ function ensureDb() {
+ if (dbReady) return;
+ process.env.ENCRYPTION_KEY = '0'.repeat(64);
+ initDb(':memory:');
+ dbReady = true;
+ }
+
+ function seedNavyModel(modelId: string, tpdLimit: number, monthlyTokenBudget: string) {
+ getDb().prepare(`
+ INSERT INTO models (
+ platform, model_id, display_name, intelligence_rank, speed_rank,
+ size_label, rpm_limit, rpd_limit, tpm_limit, tpd_limit,
+ monthly_token_budget, context_window, enabled, supports_vision,
+ supports_tools
+ ) VALUES (
+ 'navy', ?, ?, 1, 1, 'Large', 20, NULL, NULL, ?,
+ ?, NULL, 1, 0, 1
+ )
+ ON CONFLICT(platform, model_id) DO UPDATE SET
+ tpd_limit = excluded.tpd_limit,
+ monthly_token_budget = excluded.monthly_token_budget
+ `).run(modelId, `${modelId} (NavyAI)`, tpdLimit, monthlyTokenBudget);
+ }
+
+ beforeEach(() => {
+ ensureDb();
+ original = process.env[ENV];
+ getDb().prepare('DELETE FROM rate_limit_usage').run();
+ getDb().prepare("DELETE FROM models WHERE platform = 'navy'").run();
+ delete process.env[ENV];
+ });
+
+ afterEach(() => {
+ if (original === undefined) delete process.env[ENV];
+ else process.env[ENV] = original;
+ });
+
+ it('defaults to NavyAI 150K tokens/day and allows env override / disable', () => {
+ expect(getProviderDailyTokenCap('navy')).toBe(150_000);
+ expect(getProviderDailyTokenCap('groq')).toBeNull();
+ process.env[ENV] = '5000';
+ expect(getProviderDailyTokenCap('navy')).toBe(5000);
+ process.env[ENV] = '0';
+ expect(getProviderDailyTokenCap('navy')).toBeNull();
+ });
+
+ it('counts NavyAI tokens across all models using catalog multiplier hints', () => {
+ seedNavyModel('gpt-5.4', 33_333, '150K/day shared \u00b7 4.5x');
+ seedNavyModel('llama-3.3-70b-instruct', 150_000, '150K/day shared \u00b7 1x');
+
+ recordTokens('navy', 'gpt-5.4', testId, 1000);
+ recordTokens('navy', 'llama-3.3-70b-instruct', testId, 1000);
+
+ expect(providerDailyTokenCount('navy', testId)).toBe(5500);
+ });
+
+ it('blocks the provider key when multiplier-adjusted tokens would exceed the shared cap', () => {
+ process.env[ENV] = '5000';
+ seedNavyModel('gemini-2.5-flash', 75_000, '150K/day shared \u00b7 2x');
+
+ recordTokens('navy', 'gemini-2.5-flash', testId, 2000);
+
+ expect(canUseProviderTokens('navy', testId, 'gemini-2.5-flash', 400)).toBe(true);
+ expect(canUseProviderTokens('navy', testId, 'gemini-2.5-flash', 600)).toBe(false);
+ });
+ });
});
describe('Cooldown duration with upstream Retry-After', () => {
diff --git a/server/src/services/ratelimit.ts b/server/src/services/ratelimit.ts
index ccdfac21..b05b6c8a 100644
--- a/server/src/services/ratelimit.ts
+++ b/server/src/services/ratelimit.ts
@@ -190,6 +190,13 @@ const DEFAULT_PROVIDER_DAILY_REQUEST_CAPS: Record<string, number> = {
openrouter: 1000,
};
+const DEFAULT_PROVIDER_DAILY_TOKEN_CAPS: Record<string, number> = {
+ // NavyAI's free plan is one shared 150K-token/day pool per key. Individual
+ // models carry token_multiplier values, so the provider-visible drain can be
+ // higher than the OpenAI usage.total_tokens returned by /chat/completions.
+ navy: 150_000,
+};
+
export function getProviderDailyRequestCap(platform: string): number | null {
const raw = process.env[`PROVIDER_DAILY_REQUEST_CAP_${platform.toUpperCase()}`];
if (raw !== undefined && raw.trim() !== '') {
@@ -199,6 +206,15 @@ export function getProviderDailyRequestCap(platform: string): number | null {
return DEFAULT_PROVIDER_DAILY_REQUEST_CAPS[platform] ?? null;
}
+export function getProviderDailyTokenCap(platform: string): number | null {
+ const raw = process.env[`PROVIDER_DAILY_TOKEN_CAP_${platform.toUpperCase()}`];
+ if (raw !== undefined && raw.trim() !== '') {
+ const n = Number(raw);
+ if (Number.isFinite(n) && n >= 0) return n === 0 ? null : n;
+ }
+ return DEFAULT_PROVIDER_DAILY_TOKEN_CAPS[platform] ?? null;
+}
+
function countPersistedProviderRequests(
platform: string,
keyId: number,
@@ -241,6 +257,101 @@ export function canUseProvider(platform: string, keyId: number, now = Date.now()
return providerDailyRequestCount(platform, keyId, now) < cap;
}
+type ModelQuotaRow = { tpd_limit: number | null; monthly_token_budget: string | null };
+
+function multiplierFromQuotaRow(row: ModelQuotaRow | undefined, dailyCap: number): number {
+ if (!row) return 1;
+
+ const fromLabel = row.monthly_token_budget?.match(/(?:^|[\u00b7(\s])(\d+(?:\.\d+)?)x\b/i);
+ if (fromLabel) {
+ const n = Number(fromLabel[1]);
+ if (Number.isFinite(n) && n > 0) return n;
+ }
+
+ const tpd = row.tpd_limit;
+ if (tpd != null && tpd > 0 && tpd < dailyCap) return dailyCap / tpd;
+ return 1;
+}
+
+function getProviderTokenMultiplier(platform: string, modelId: string, dailyCap: number): number {
+ if (platform !== 'navy') return 1;
+ return withDb(db => {
+ const row = db.prepare(`
+ SELECT tpd_limit, monthly_token_budget
+ FROM models
+ WHERE platform = ? AND model_id = ?
+ LIMIT 1
+ `).get(platform, modelId) as ModelQuotaRow | undefined;
+ return multiplierFromQuotaRow(row, dailyCap);
+ }) ?? 1;
+}
+
+function providerBilledTokens(platform: string, modelId: string, rawTokens: number): number {
+ if (rawTokens <= 0) return 0;
+ const dailyCap = getProviderDailyTokenCap(platform);
+ if (dailyCap === null) return rawTokens;
+ const multiplier = getProviderTokenMultiplier(platform, modelId, dailyCap);
+ return Math.ceil(rawTokens * multiplier);
+}
+
+function sumPersistedProviderTokens(
+ platform: string,
+ keyId: number,
+ windowMs: number,
+ now: number,
+): number | undefined {
+ const dailyCap = getProviderDailyTokenCap(platform);
+ if (dailyCap === null) return 0;
+
+ return withDb(db => {
+ const rows = db.prepare(`
+ SELECT rlu.model_id, COALESCE(SUM(rlu.tokens), 0) AS used,
+ m.tpd_limit, m.monthly_token_budget
+ FROM rate_limit_usage rlu
+ LEFT JOIN models m ON m.platform = rlu.platform AND m.model_id = rlu.model_id
+ WHERE rlu.platform = ?
+ AND rlu.key_id = ?
+ AND rlu.kind = 'tokens'
+ AND rlu.created_at_ms > ?
+ GROUP BY rlu.model_id
+ `).all(platform, keyId, now - windowMs) as (ModelQuotaRow & { model_id: string; used: number })[];
+
+ return rows.reduce((sum, row) => {
+ const multiplier = platform === 'navy' ? multiplierFromQuotaRow(row, dailyCap) : 1;
+ return sum + Math.ceil(row.used * multiplier);
+ }, 0);
+ });
+}
+
+export function providerDailyTokenCount(platform: string, keyId: number, now = Date.now()): number {
+ const persisted = sumPersistedProviderTokens(platform, keyId, DAY, now);
+ if (persisted !== undefined) return persisted;
+
+ let total = 0;
+ const suffix = `:${keyId}:tpd`;
+ for (const [key, w] of windows) {
+ if (!key.startsWith(`${platform}:`) || !key.endsWith(suffix)) continue;
+ const modelId = key.slice(platform.length + 1, -suffix.length);
+ w.tokenTimestamps = w.tokenTimestamps.filter(t => t.ts > now - DAY);
+ const raw = w.tokenTimestamps.reduce((sum, t) => sum + t.tokens, 0);
+ total += providerBilledTokens(platform, modelId, raw);
+ }
+ return total;
+}
+
+export function canUseProviderTokens(
+ platform: string,
+ keyId: number,
+ modelId: string,
+ estimatedTokens: number,
+ now = Date.now(),
+): boolean {
+ const cap = getProviderDailyTokenCap(platform);
+ if (cap === null) return true;
+ const used = providerDailyTokenCount(platform, keyId, now);
+ return used + providerBilledTokens(platform, modelId, estimatedTokens) <= cap;
+}
+
export function recordRequest(platform: string, modelId: string, keyId: number) {
const now = Date.now();
diff --git a/server/src/services/router.ts b/server/src/services/router.ts
index 915ec250..e137171f 100644
--- a/server/src/services/router.ts
+++ b/server/src/services/router.ts
@@ -1,7 +1,14 @@
import { getDb, getSetting, setSetting } from '../db/index.js';
import { getProvider, hasProvider, resolveProvider } from '../providers/index.js';
import { decrypt } from '../lib/crypto.js';
-import { canMakeRequest, canUseTokens, isOnCooldown, canUseProvider, getSoonestCooldownExpiry } from './ratelimit.js';
+import {
+ canMakeRequest,
+ canUseTokens,
+ isOnCooldown,
+ canUseProvider,
+ canUseProviderTokens,
+ getSoonestCooldownExpiry,
+} from './ratelimit.js';
import {
BANDIT_PRESETS, DEFAULT_STRATEGY, type RoutingStrategy, type RoutingWeights,
reliabilityPosterior, expectedReliability, sampleBeta,
@@ -702,6 +709,7 @@ function selectKeyForModel(entry: ChainRow, estimatedTokens: number, skipKeys?:
if (!canUseProvider(entry.platform, key.id)) { note('provider-daily-cap'); continue; }
if (!canMakeRequest(entry.platform, entry.model_id, key.id, limits)) { note('rpm/rpd-limit'); continue; }
if (!canUseTokens(entry.platform, entry.model_id, key.id, estimatedTokens, limits)) { note('tpm/tpd-limit'); continue; }
+ if (!canUseProviderTokens(entry.platform, key.id, entry.model_id, estimatedTokens)) { note('provider-daily-token-cap'); continue; }
let decryptedKey: string;
try {
@@ -785,6 +793,7 @@ export function hasOtherUsableKey(modelDbId: number, excludingKeyId: number, ski
// headroom; a nominal 1-token probe only rules out a key already at its
// TPM/TPD ceiling.
if (!canUseTokens(m.platform, m.model_id, k.id, 1, limits)) continue;
+ if (!canUseProviderTokens(m.platform, k.id, m.model_id, 1)) continue;
return true;
}
return false;
@@ -923,7 +932,8 @@ export function getOrderedFusionChain(): FusionCandidate[] {
(e.key_id == null || kid === e.key_id) &&
!isOnCooldown(e.platform, e.model_id, kid) &&
canUseProvider(e.platform, kid) &&
- canMakeRequest(e.platform, e.model_id, kid, limits),
+ canMakeRequest(e.platform, e.model_id, kid, limits) &&
+ canUseProviderTokens(e.platform, kid, e.model_id, 1),
);
});