Skip to content

7.4 billion tokens per month. 34 free LLM providers. 635 free model endpoints. All behind one /v1 endpoint, plus any custom OpenAI-compatible endpoint. Smart routing, automatic failover, encrypted keys. Personal experimentation only.

TypeScript

fix(rerank): enforce the per-key monthly caps before dispatch (#1627)

tashdroidd9cccbb74b64
diff --git a/server/src/__tests__/services/rerank-monthly-budget.test.ts b/server/src/__tests__/services/rerank-monthly-budget.test.ts
new file mode 100644
index 00000000..5a95604d
--- /dev/null
+++ b/server/src/__tests__/services/rerank-monthly-budget.test.ts
@@ -0,0 +1,81 @@
+import { describe, it, expect, beforeEach, afterEach, vi } from 'vitest';
+import { initDb, getDb } from '../../db/index.js';
+import { encrypt } from '../../lib/crypto.js';
+import { runRerank, RerankError } from '../../services/rerank.js';
+import { getMonthlyUsage } from '../../services/key-budget.js';
+
+// POST /v1/rerank spends real provider quota, so a key at its monthly cap
+// (#1158) must not serve one. Chat (router), embeddings, media, search and
+// moderation all gate on the caps before dispatch; rerank read api_keys
+// directly and skipped the gate — while rerankRetryAfterSec already answered
+// a `quota_exceeded` code that nothing could ever set.
+
+const realFetch = globalThis.fetch;
+
+function addKey(platform: string, monthlyRequestCap: number, baseUrl: string | null = null): number {
+ const { encrypted, iv, authTag } = encrypt(`${platform}-test-key`);
+ const row = baseUrl
+ ? getDb().prepare(`
+ INSERT INTO api_keys (platform, label, encrypted_key, iv, auth_tag, status, enabled, base_url, monthly_request_cap)
+ VALUES (?, 'test', ?, ?, ?, 'healthy', 1, ?, ?)
+ `).run(platform, encrypted, iv, authTag, baseUrl, monthlyRequestCap)
+ : getDb().prepare(`
+ INSERT INTO api_keys (platform, label, encrypted_key, iv, auth_tag, status, enabled, monthly_request_cap)
+ VALUES (?, 'test', ?, ?, ?, 'healthy', 1, ?)
+ `).run(platform, encrypted, iv, authTag, monthlyRequestCap);
+ return Number(row.lastInsertRowid);
+}
+
+function spendRequests(keyId: number, requests: number) {
+ getDb().prepare(
+ 'INSERT INTO key_monthly_usage (key_id, month, requests, tokens) VALUES (?, ?, ?, 10)',
+ ).run(keyId, new Date().toISOString().slice(0, 7), requests);
+}
+
+const ok = (results: unknown) => new Response(JSON.stringify({ results }), { status: 200 });
+
+describe('rerank honors the per-key monthly budget (#1158)', () => {
+ beforeEach(() => {
+ process.env.ENCRYPTION_KEY = '0'.repeat(64);
+ initDb(':memory:');
+ });
+ afterEach(() => {
+ globalThis.fetch = realFetch;
+ vi.restoreAllMocks();
+ });
+
+ it('skips a capped key and serves from an uncapped one', async () => {
+ addKey('cohere', 1);
+ const capped = getDb().prepare("SELECT id FROM api_keys WHERE platform = 'cohere'").get() as { id: number };
+ spendRequests(capped.id, 1);
+ addKey('custom', 0, 'https://relay.example/v1');
+ const fetch = vi.spyOn(global, 'fetch').mockResolvedValue(ok([{ index: 0, relevance_score: 0.5 }]));
+
+ const out = await runRerank(undefined, 'q', ['a', 'b'], 1);
+ expect(out.platform).toBe('custom');
+ expect(fetch).toHaveBeenCalledTimes(1);
+ expect(String(fetch.mock.calls[0][0])).toContain('relay.example');
+ });
+
+ it('a capped-only chain answers 429 quota_exceeded, not a 502', async () => {
+ const key = addKey('cohere', 1);
+ spendRequests(key, 1);
+ const fetch = vi.spyOn(global, 'fetch').mockResolvedValue(ok([{ index: 0, relevance_score: 0.5 }]));
+
+ const err = await runRerank(undefined, 'q', ['a'], 1).catch(e => e) as RerankError;
+ expect(err).toBeInstanceOf(RerankError);
+ expect(err.status).toBe(429);
+ expect(err.code).toBe('quota_exceeded');
+ // The capped key was never touched upstream.
+ expect(fetch).not.toHaveBeenCalled();
+ expect(getMonthlyUsage(key)).toEqual({ requests: 1, tokens: 10 });
+ });
+
+ it('an uncapped chain still serves normally (regression)', async () => {
+ addKey('cohere', 0);
+ vi.spyOn(global, 'fetch').mockResolvedValue(ok([{ index: 1, relevance_score: 0.9 }]));
+ const out = await runRerank(undefined, 'q', ['a', 'b'], 1);
+ expect(out.platform).toBe('cohere');
+ expect(out.results).toEqual([{ index: 1, relevanceScore: 0.9, document: 'b' }]);
+ });
+});
diff --git a/server/src/__tests__/services/rerank.test.ts b/server/src/__tests__/services/rerank.test.ts
index 44cc17cc..e6044df7 100644
--- a/server/src/__tests__/services/rerank.test.ts
+++ b/server/src/__tests__/services/rerank.test.ts
@@ -12,6 +12,9 @@ vi.mock('../../db/index.js', () => ({
// a custom row without a base_url is not a candidate
{ id: 3, platform: 'custom', base_url: null, encrypted_key: 'e', iv: 'i', auth_tag: 'a' },
],
+ // The monthly-budget gate (#1158) reads api_keys caps and the ledger;
+ // no rows here means every key is uncapped (0 = unlimited).
+ get: () => undefined,
}),
}),
}));
diff --git a/server/src/services/rerank.ts b/server/src/services/rerank.ts
index 189841a4..fff755d0 100644
--- a/server/src/services/rerank.ts
+++ b/server/src/services/rerank.ts
@@ -1,7 +1,7 @@
import { getDb } from '../db/index.js';
import { decrypt } from '../lib/crypto.js';
import { parseRetryAfterMs } from '../providers/base.js';
-import { secondsUntilNextMonth } from './key-budget.js';
+import { checkMonthlyBudget, reserveMonthlyBudget, secondsUntilNextMonth } from './key-budget.js';
import { markKeyHealthyFromRequest } from './health.js';
import { getClientContext } from '../lib/client-context.js';
@@ -118,12 +118,20 @@ export async function runRerank(model: string | undefined, query: string, docume
throw new RerankError('No rerank-capable key configured. Add a Cohere key or a custom rerank endpoint.', 503);
}
const n = topN ?? documents.length;
- // Rough token estimate for the row (query + documents, ~4 chars/token) —
- // same heuristic embeddings log when upstream reports no usage.
+ // Rough token estimate (query + documents, ~4 chars/token), the heuristic
+ // embeddings log when upstream reports no usage. It sizes both the request
+ // row and the monthly-budget admission check: rerank spends real provider
+ // quota, so a key at its monthly cap (#1158) must not serve one.
const estTokens = Math.ceil((query.length + documents.reduce((s, d) => s + d.length, 0)) / 4);
+ let budgetBlocked = false;
let lastError: RerankError | null = null;
for (const cand of chain) {
+ // Check-and-reserve (the search/embeddings semantics): an in-flight rerank
+ // holds its estimate so two concurrent reranks can't both pass the gate on
+ // a nearly-spent cap. Released the moment the attempt settles.
+ const reservation = reserveMonthlyBudget(cand.id, estTokens);
+ if (!reservation.allowed) { budgetBlocked = true; continue; }
const started = Date.now();
try {
const out = await callProvider(cand, model, query, documents, n);
@@ -138,8 +146,13 @@ export async function runRerank(model: string | undefined, query: string, docume
lastError = err instanceof RerankError ? err : new RerankError(String(err?.message ?? err), 502);
logRerankRequest(cand, cand.id, model ?? (cand.platform === 'cohere' ? 'rerank-v3.5' : 'rerank'),
'error', 0, Date.now() - started, lastError.message.slice(0, 300));
+ } finally {
+ reservation.release();
}
}
+ if (budgetBlocked && chain.every(c => !checkMonthlyBudget(c.id, estTokens).allowed)) {
+ throw new RerankError('All rerank keys are at their monthly budget.', 429, 'quota_exceeded');
+ }
throw new RerankError(
`All rerank providers failed${lastError ? ` (last: ${lastError.message.slice(0, 160)})` : ''}.`,
lastError?.status === 429 ? 429 : 502,