7.4 billion tokens per month. 34 free LLM providers. 635 free model endpoints. All behind one /v1 endpoint, plus any custom OpenAI-compatible endpoint. Smart routing, automatic failover, encrypted keys. Personal experimentation only.
TypeScript
fix(rerank): log rerank traffic to the requests table (#1621)
tashdroideb890c4dd67a
diff --git a/server/src/__tests__/services/router-usable-key-monthly.test.ts b/server/src/__tests__/services/router-usable-key-monthly.test.tsindex de216584..a26ac97d 100644--- a/server/src/__tests__/services/router-usable-key-monthly.test.ts+++ b/server/src/__tests__/services/router-usable-key-monthly.test.ts@@ -12,8 +12,6 @@ import { hasOtherUsableKey, hasUsableKeyForModel, setRoutingStrategy } from '../// would immediately skip) and a 429 on a capped sibling failed to demote the// model, because the capped sibling counted as a "usable alternative".-let sequence = 90000;-function insertKey(overrides: { monthlyRequestCap?: number } = {}): number {const { encrypted, iv, authTag } = encrypt('sk-test');const result = getDb().prepare(`diff --git a/server/src/__tests__/services/served-nonchat-marks-key-healthy.test.ts b/server/src/__tests__/services/served-nonchat-marks-key-healthy.test.tsindex 0a25aa9a..b92ba4fa 100644--- a/server/src/__tests__/services/served-nonchat-marks-key-healthy.test.ts+++ b/server/src/__tests__/services/served-nonchat-marks-key-healthy.test.ts@@ -89,6 +89,22 @@ describe('a served non-chat request marks its key as working (#1493, #1612)', ()expect(keyState(keyId)).toMatchObject({ status: 'healthy', last_healthy_at: expect.any(String) });});+ it('rerank: success and failure both land in the requests table', async () => {+ addKey('cohere', 'healthy');+ const requests = () => getDb().prepare(+ "SELECT status, request_type, model_id, input_tokens FROM requests WHERE request_type = 'rerank' ORDER BY id",+ ).all() as Array<{ status: string; request_type: string; model_id: string; input_tokens: number }>;++ mockFetch(() => new Response(JSON.stringify({ results: [{ index: 0, relevance_score: 0.9 }] }), { status: 200 }));+ await runRerank(undefined, 'query text', ['doc a', 'doc b'], 1);+ expect(requests()).toEqual([{ status: 'success', request_type: 'rerank', model_id: 'rerank-v3.5', input_tokens: expect.any(Number) }]);++ mockFetch(() => new Response('boom', { status: 500 }));+ await expect(runRerank(undefined, 'q', ['a'])).rejects.toThrow();+ expect(requests()).toHaveLength(2);+ expect(requests()[1]).toMatchObject({ status: 'error', request_type: 'rerank', model_id: 'rerank-v3.5', input_tokens: 0 });+ });+it('decision: a served decision promotes an unknown key', async () => {seedDecisionModels();const keyId = addKey('cloudflare', 'unknown', undefined, 'acct123:cf-token');diff --git a/server/src/services/rerank.ts b/server/src/services/rerank.tsindex 89565c8b..189841a4 100644--- a/server/src/services/rerank.ts+++ b/server/src/services/rerank.ts@@ -3,6 +3,7 @@ import { decrypt } from '../lib/crypto.js';import { parseRetryAfterMs } from '../providers/base.js';import { secondsUntilNextMonth } from './key-budget.js';import { markKeyHealthyFromRequest } from './health.js';+import { getClientContext } from '../lib/client-context.js';// Cohere-style rerank over the existing free-tier key pool (#1029). Deliberately// narrow first slice: platform='cohere' keys and custom OpenAI-compatible@@ -58,6 +59,26 @@ async function upstreamError(r: Response): Promise<RerankError> {return new RerankError(`upstream ${r.status}: ${(await r.text()).slice(0, 200)}`, r.status, undefined, retryAfterMs);}+// Every other inference surface writes its attempts to `requests` (chat via+// logRequest, embeddings/media/decisions via their own loggers), which is+// where the dashboard history and retention sweeper read from. Rerank was the+// only one that served traffic without a row: invisible in analytics, outside+// retention, and with no key attribution when debugging a bad upstream.+// request_type='rerank' follows the media precedent of tagging non-chat+// traffic; the analytics type-count endpoint ignores types it doesn't map.+function logRerankRequest(cand: { platform: string }, keyId: number, modelId: string,+ status: 'success' | 'error', inputTokens: number, latencyMs: number, error: string | null): void {+ try {+ const client = getClientContext();+ getDb().prepare(`+ INSERT INTO requests (platform, model_id, key_id, status, input_tokens, output_tokens, latency_ms, error, request_type, client_ip, client_user_agent, client_agent)+ VALUES (?, ?, ?, ?, ?, 0, ?, ?, 'rerank', ?, ?, ?)+ `).run(cand.platform, modelId, keyId, status, inputTokens, latencyMs, error, client.ip, client.userAgent, client.agent);+ } catch (e) {+ console.error('Failed to log rerank request:', e);+ }+}+async function callProvider(cand: RerankCandidate, model: string | undefined, query: string,documents: string[], topN: number): Promise<{ modelId: string; results: RerankResult['results'] }> {// ponytail: cohere + jina-shaped custom only; per-provider adapters when a@@ -97,9 +118,13 @@ export async function runRerank(model: string | undefined, query: string, documethrow new RerankError('No rerank-capable key configured. Add a Cohere key or a custom rerank endpoint.', 503);}const n = topN ?? documents.length;+ // Rough token estimate for the row (query + documents, ~4 chars/token) —+ // same heuristic embeddings log when upstream reports no usage.+ const estTokens = Math.ceil((query.length + documents.reduce((s, d) => s + d.length, 0)) / 4);let lastError: RerankError | null = null;for (const cand of chain) {+ const started = Date.now();try {const out = await callProvider(cand, model, query, documents, n);// A served rerank is evidence the key works, the same as a served chat@@ -107,9 +132,12 @@ export async function runRerank(model: string | undefined, query: string, docume// from the same 'healthy'/'unknown' pool, so a key that only ever// serves reranks had no success signal to lift it out of 'error'.markKeyHealthyFromRequest(cand.id);+ logRerankRequest(cand, cand.id, out.modelId, 'success', estTokens, Date.now() - started, null);return { platform: cand.platform, modelId: out.modelId, results: out.results };} catch (err: any) {lastError = err instanceof RerankError ? err : new RerankError(String(err?.message ?? err), 502);+ logRerankRequest(cand, cand.id, model ?? (cand.platform === 'cohere' ? 'rerank-v3.5' : 'rerank'),+ 'error', 0, Date.now() - started, lastError.message.slice(0, 300));}}throw new RerankError(