7.4 billion tokens per month. 34 free LLM providers. 635 free model endpoints. All behind one /v1 endpoint, plus any custom OpenAI-compatible endpoint. Smart routing, automatic failover, encrypted keys. Personal experimentation only.
TypeScript
fix(rerank): enforce the per-key monthly caps before dispatch (#1627)
tashdroidd9cccbb74b64
diff --git a/server/src/__tests__/services/rerank-monthly-budget.test.ts b/server/src/__tests__/services/rerank-monthly-budget.test.tsnew file mode 100644index 00000000..5a95604d--- /dev/null+++ b/server/src/__tests__/services/rerank-monthly-budget.test.ts@@ -0,0 +1,81 @@+import { describe, it, expect, beforeEach, afterEach, vi } from 'vitest';+import { initDb, getDb } from '../../db/index.js';+import { encrypt } from '../../lib/crypto.js';+import { runRerank, RerankError } from '../../services/rerank.js';+import { getMonthlyUsage } from '../../services/key-budget.js';++// POST /v1/rerank spends real provider quota, so a key at its monthly cap+// (#1158) must not serve one. Chat (router), embeddings, media, search and+// moderation all gate on the caps before dispatch; rerank read api_keys+// directly and skipped the gate — while rerankRetryAfterSec already answered+// a `quota_exceeded` code that nothing could ever set.++const realFetch = globalThis.fetch;++function addKey(platform: string, monthlyRequestCap: number, baseUrl: string | null = null): number {+ const { encrypted, iv, authTag } = encrypt(`${platform}-test-key`);+ const row = baseUrl+ ? getDb().prepare(`+ INSERT INTO api_keys (platform, label, encrypted_key, iv, auth_tag, status, enabled, base_url, monthly_request_cap)+ VALUES (?, 'test', ?, ?, ?, 'healthy', 1, ?, ?)+ `).run(platform, encrypted, iv, authTag, baseUrl, monthlyRequestCap)+ : getDb().prepare(`+ INSERT INTO api_keys (platform, label, encrypted_key, iv, auth_tag, status, enabled, monthly_request_cap)+ VALUES (?, 'test', ?, ?, ?, 'healthy', 1, ?)+ `).run(platform, encrypted, iv, authTag, monthlyRequestCap);+ return Number(row.lastInsertRowid);+}++function spendRequests(keyId: number, requests: number) {+ getDb().prepare(+ 'INSERT INTO key_monthly_usage (key_id, month, requests, tokens) VALUES (?, ?, ?, 10)',+ ).run(keyId, new Date().toISOString().slice(0, 7), requests);+}++const ok = (results: unknown) => new Response(JSON.stringify({ results }), { status: 200 });++describe('rerank honors the per-key monthly budget (#1158)', () => {+ beforeEach(() => {+ process.env.ENCRYPTION_KEY = '0'.repeat(64);+ initDb(':memory:');+ });+ afterEach(() => {+ globalThis.fetch = realFetch;+ vi.restoreAllMocks();+ });++ it('skips a capped key and serves from an uncapped one', async () => {+ addKey('cohere', 1);+ const capped = getDb().prepare("SELECT id FROM api_keys WHERE platform = 'cohere'").get() as { id: number };+ spendRequests(capped.id, 1);+ addKey('custom', 0, 'https://relay.example/v1');+ const fetch = vi.spyOn(global, 'fetch').mockResolvedValue(ok([{ index: 0, relevance_score: 0.5 }]));++ const out = await runRerank(undefined, 'q', ['a', 'b'], 1);+ expect(out.platform).toBe('custom');+ expect(fetch).toHaveBeenCalledTimes(1);+ expect(String(fetch.mock.calls[0][0])).toContain('relay.example');+ });++ it('a capped-only chain answers 429 quota_exceeded, not a 502', async () => {+ const key = addKey('cohere', 1);+ spendRequests(key, 1);+ const fetch = vi.spyOn(global, 'fetch').mockResolvedValue(ok([{ index: 0, relevance_score: 0.5 }]));++ const err = await runRerank(undefined, 'q', ['a'], 1).catch(e => e) as RerankError;+ expect(err).toBeInstanceOf(RerankError);+ expect(err.status).toBe(429);+ expect(err.code).toBe('quota_exceeded');+ // The capped key was never touched upstream.+ expect(fetch).not.toHaveBeenCalled();+ expect(getMonthlyUsage(key)).toEqual({ requests: 1, tokens: 10 });+ });++ it('an uncapped chain still serves normally (regression)', async () => {+ addKey('cohere', 0);+ vi.spyOn(global, 'fetch').mockResolvedValue(ok([{ index: 1, relevance_score: 0.9 }]));+ const out = await runRerank(undefined, 'q', ['a', 'b'], 1);+ expect(out.platform).toBe('cohere');+ expect(out.results).toEqual([{ index: 1, relevanceScore: 0.9, document: 'b' }]);+ });+});diff --git a/server/src/__tests__/services/rerank.test.ts b/server/src/__tests__/services/rerank.test.tsindex 44cc17cc..e6044df7 100644--- a/server/src/__tests__/services/rerank.test.ts+++ b/server/src/__tests__/services/rerank.test.ts@@ -12,6 +12,9 @@ vi.mock('../../db/index.js', () => ({// a custom row without a base_url is not a candidate{ id: 3, platform: 'custom', base_url: null, encrypted_key: 'e', iv: 'i', auth_tag: 'a' },],+ // The monthly-budget gate (#1158) reads api_keys caps and the ledger;+ // no rows here means every key is uncapped (0 = unlimited).+ get: () => undefined,}),}),}));diff --git a/server/src/services/rerank.ts b/server/src/services/rerank.tsindex 189841a4..fff755d0 100644--- a/server/src/services/rerank.ts+++ b/server/src/services/rerank.ts@@ -1,7 +1,7 @@import { getDb } from '../db/index.js';import { decrypt } from '../lib/crypto.js';import { parseRetryAfterMs } from '../providers/base.js';-import { secondsUntilNextMonth } from './key-budget.js';+import { checkMonthlyBudget, reserveMonthlyBudget, secondsUntilNextMonth } from './key-budget.js';import { markKeyHealthyFromRequest } from './health.js';import { getClientContext } from '../lib/client-context.js';@@ -118,12 +118,20 @@ export async function runRerank(model: string | undefined, query: string, documethrow new RerankError('No rerank-capable key configured. Add a Cohere key or a custom rerank endpoint.', 503);}const n = topN ?? documents.length;- // Rough token estimate for the row (query + documents, ~4 chars/token) —- // same heuristic embeddings log when upstream reports no usage.+ // Rough token estimate (query + documents, ~4 chars/token), the heuristic+ // embeddings log when upstream reports no usage. It sizes both the request+ // row and the monthly-budget admission check: rerank spends real provider+ // quota, so a key at its monthly cap (#1158) must not serve one.const estTokens = Math.ceil((query.length + documents.reduce((s, d) => s + d.length, 0)) / 4);+ let budgetBlocked = false;let lastError: RerankError | null = null;for (const cand of chain) {+ // Check-and-reserve (the search/embeddings semantics): an in-flight rerank+ // holds its estimate so two concurrent reranks can't both pass the gate on+ // a nearly-spent cap. Released the moment the attempt settles.+ const reservation = reserveMonthlyBudget(cand.id, estTokens);+ if (!reservation.allowed) { budgetBlocked = true; continue; }const started = Date.now();try {const out = await callProvider(cand, model, query, documents, n);@@ -138,8 +146,13 @@ export async function runRerank(model: string | undefined, query: string, documelastError = err instanceof RerankError ? err : new RerankError(String(err?.message ?? err), 502);logRerankRequest(cand, cand.id, model ?? (cand.platform === 'cohere' ? 'rerank-v3.5' : 'rerank'),'error', 0, Date.now() - started, lastError.message.slice(0, 300));+ } finally {+ reservation.release();}}+ if (budgetBlocked && chain.every(c => !checkMonthlyBudget(c.id, estTokens).allowed)) {+ throw new RerankError('All rerank keys are at their monthly budget.', 429, 'quota_exceeded');+ }throw new RerankError(`All rerank providers failed${lastError ? ` (last: ${lastError.message.slice(0, 160)})` : ''}.`,lastError?.status === 429 ? 429 : 502,