Skip to content

7.4 billion tokens per month. 34 free LLM providers. 635 free model endpoints. All behind one /v1 endpoint, plus any custom OpenAI-compatible endpoint. Smart routing, automatic failover, encrypted keys. Personal experimentation only.

TypeScript

fix(rerank): log rerank traffic to the requests table (#1621)

tashdroideb890c4dd67a
diff --git a/server/src/__tests__/services/router-usable-key-monthly.test.ts b/server/src/__tests__/services/router-usable-key-monthly.test.ts
index de216584..a26ac97d 100644
--- a/server/src/__tests__/services/router-usable-key-monthly.test.ts
+++ b/server/src/__tests__/services/router-usable-key-monthly.test.ts
@@ -12,8 +12,6 @@ import { hasOtherUsableKey, hasUsableKeyForModel, setRoutingStrategy } from '../
// would immediately skip) and a 429 on a capped sibling failed to demote the
// model, because the capped sibling counted as a "usable alternative".
-let sequence = 90000;
-
function insertKey(overrides: { monthlyRequestCap?: number } = {}): number {
const { encrypted, iv, authTag } = encrypt('sk-test');
const result = getDb().prepare(`
diff --git a/server/src/__tests__/services/served-nonchat-marks-key-healthy.test.ts b/server/src/__tests__/services/served-nonchat-marks-key-healthy.test.ts
index 0a25aa9a..b92ba4fa 100644
--- a/server/src/__tests__/services/served-nonchat-marks-key-healthy.test.ts
+++ b/server/src/__tests__/services/served-nonchat-marks-key-healthy.test.ts
@@ -89,6 +89,22 @@ describe('a served non-chat request marks its key as working (#1493, #1612)', ()
expect(keyState(keyId)).toMatchObject({ status: 'healthy', last_healthy_at: expect.any(String) });
});
+ it('rerank: success and failure both land in the requests table', async () => {
+ addKey('cohere', 'healthy');
+ const requests = () => getDb().prepare(
+ "SELECT status, request_type, model_id, input_tokens FROM requests WHERE request_type = 'rerank' ORDER BY id",
+ ).all() as Array<{ status: string; request_type: string; model_id: string; input_tokens: number }>;
+
+ mockFetch(() => new Response(JSON.stringify({ results: [{ index: 0, relevance_score: 0.9 }] }), { status: 200 }));
+ await runRerank(undefined, 'query text', ['doc a', 'doc b'], 1);
+ expect(requests()).toEqual([{ status: 'success', request_type: 'rerank', model_id: 'rerank-v3.5', input_tokens: expect.any(Number) }]);
+
+ mockFetch(() => new Response('boom', { status: 500 }));
+ await expect(runRerank(undefined, 'q', ['a'])).rejects.toThrow();
+ expect(requests()).toHaveLength(2);
+ expect(requests()[1]).toMatchObject({ status: 'error', request_type: 'rerank', model_id: 'rerank-v3.5', input_tokens: 0 });
+ });
+
it('decision: a served decision promotes an unknown key', async () => {
seedDecisionModels();
const keyId = addKey('cloudflare', 'unknown', undefined, 'acct123:cf-token');
diff --git a/server/src/services/rerank.ts b/server/src/services/rerank.ts
index 89565c8b..189841a4 100644
--- a/server/src/services/rerank.ts
+++ b/server/src/services/rerank.ts
@@ -3,6 +3,7 @@ import { decrypt } from '../lib/crypto.js';
import { parseRetryAfterMs } from '../providers/base.js';
import { secondsUntilNextMonth } from './key-budget.js';
import { markKeyHealthyFromRequest } from './health.js';
+import { getClientContext } from '../lib/client-context.js';
// Cohere-style rerank over the existing free-tier key pool (#1029). Deliberately
// narrow first slice: platform='cohere' keys and custom OpenAI-compatible
@@ -58,6 +59,26 @@ async function upstreamError(r: Response): Promise<RerankError> {
return new RerankError(`upstream ${r.status}: ${(await r.text()).slice(0, 200)}`, r.status, undefined, retryAfterMs);
}
+// Every other inference surface writes its attempts to `requests` (chat via
+// logRequest, embeddings/media/decisions via their own loggers), which is
+// where the dashboard history and retention sweeper read from. Rerank was the
+// only one that served traffic without a row: invisible in analytics, outside
+// retention, and with no key attribution when debugging a bad upstream.
+// request_type='rerank' follows the media precedent of tagging non-chat
+// traffic; the analytics type-count endpoint ignores types it doesn't map.
+function logRerankRequest(cand: { platform: string }, keyId: number, modelId: string,
+ status: 'success' | 'error', inputTokens: number, latencyMs: number, error: string | null): void {
+ try {
+ const client = getClientContext();
+ getDb().prepare(`
+ INSERT INTO requests (platform, model_id, key_id, status, input_tokens, output_tokens, latency_ms, error, request_type, client_ip, client_user_agent, client_agent)
+ VALUES (?, ?, ?, ?, ?, 0, ?, ?, 'rerank', ?, ?, ?)
+ `).run(cand.platform, modelId, keyId, status, inputTokens, latencyMs, error, client.ip, client.userAgent, client.agent);
+ } catch (e) {
+ console.error('Failed to log rerank request:', e);
+ }
+}
+
async function callProvider(cand: RerankCandidate, model: string | undefined, query: string,
documents: string[], topN: number): Promise<{ modelId: string; results: RerankResult['results'] }> {
// ponytail: cohere + jina-shaped custom only; per-provider adapters when a
@@ -97,9 +118,13 @@ export async function runRerank(model: string | undefined, query: string, docume
throw new RerankError('No rerank-capable key configured. Add a Cohere key or a custom rerank endpoint.', 503);
}
const n = topN ?? documents.length;
+ // Rough token estimate for the row (query + documents, ~4 chars/token) —
+ // same heuristic embeddings log when upstream reports no usage.
+ const estTokens = Math.ceil((query.length + documents.reduce((s, d) => s + d.length, 0)) / 4);
let lastError: RerankError | null = null;
for (const cand of chain) {
+ const started = Date.now();
try {
const out = await callProvider(cand, model, query, documents, n);
// A served rerank is evidence the key works, the same as a served chat
@@ -107,9 +132,12 @@ export async function runRerank(model: string | undefined, query: string, docume
// from the same 'healthy'/'unknown' pool, so a key that only ever
// serves reranks had no success signal to lift it out of 'error'.
markKeyHealthyFromRequest(cand.id);
+ logRerankRequest(cand, cand.id, out.modelId, 'success', estTokens, Date.now() - started, null);
return { platform: cand.platform, modelId: out.modelId, results: out.results };
} catch (err: any) {
lastError = err instanceof RerankError ? err : new RerankError(String(err?.message ?? err), 502);
+ logRerankRequest(cand, cand.id, model ?? (cand.platform === 'cohere' ? 'rerank-v3.5' : 'rerank'),
+ 'error', 0, Date.now() - started, lastError.message.slice(0, 300));
}
}
throw new RerankError(