Skip to content

7.4 billion tokens per month. 34 free LLM providers. 635 free model endpoints. All behind one /v1 endpoint, plus any custom OpenAI-compatible endpoint. Smart routing, automatic failover, encrypted keys. Personal experimentation only.

TypeScript

feat(proxy): /v1/rerank endpoint over the existing key pool (#1029) (#1372)

suantea (asuan_suan)65b7310bbe59
diff --git a/server/src/__tests__/services/rerank.test.ts b/server/src/__tests__/services/rerank.test.ts
new file mode 100644
index 00000000..44cc17cc
--- /dev/null
+++ b/server/src/__tests__/services/rerank.test.ts
@@ -0,0 +1,41 @@
+import { describe, expect, it, vi, beforeEach } from 'vitest';
+import { runRerank, RerankError } from '../../services/rerank.js';
+
+vi.mock('../../db/index.js', () => ({
+ getDb: () => ({
+ prepare: () => ({
+ all: () => [
+ // cohere first in insertion order; the service must still try the
+ // custom endpoint first.
+ { id: 1, platform: 'cohere', base_url: null, encrypted_key: 'e', iv: 'i', auth_tag: 'a' },
+ { id: 2, platform: 'custom', base_url: 'https://relay.example/v1', encrypted_key: 'e', iv: 'i', auth_tag: 'a' },
+ // a custom row without a base_url is not a candidate
+ { id: 3, platform: 'custom', base_url: null, encrypted_key: 'e', iv: 'i', auth_tag: 'a' },
+ ],
+ }),
+ }),
+}));
+vi.mock('../../lib/crypto.js', () => ({
+ decrypt: () => 'test-key',
+}));
+
+const ok = (results: unknown) => new Response(JSON.stringify({ results }), { status: 200 });
+
+describe('runRerank', () => {
+ beforeEach(() => vi.restoreAllMocks());
+
+ it('walks candidates custom-first and takes the first success', async () => {
+ const fetch = vi.spyOn(global, 'fetch')
+ .mockResolvedValueOnce(new Response('boom', { status: 500 }))
+ .mockResolvedValueOnce(ok([{ index: 1, relevance_score: 0.9 }]));
+ const out = await runRerank(undefined, 'q', ['a', 'b'], 1);
+ expect((fetch.mock.calls[0][0] as string)).toBe('https://relay.example/v1/rerank');
+ expect(out.platform).toBe('cohere');
+ expect(out.results).toEqual([{ index: 1, relevanceScore: 0.9, document: 'b' }]);
+ });
+
+ it('rejects an out-of-range index and reports chain exhaustion', async () => {
+ vi.spyOn(global, 'fetch').mockResolvedValue(ok([{ index: 9, relevance_score: 1 }]));
+ await expect(runRerank(undefined, 'q', ['a'])).rejects.toBeInstanceOf(RerankError);
+ });
+});
diff --git a/server/src/routes/proxy.ts b/server/src/routes/proxy.ts
index 1ab4a015..bc5c04b4 100644
--- a/server/src/routes/proxy.ts
+++ b/server/src/routes/proxy.ts
@@ -6,6 +6,7 @@ import type { ChatMessage, ChatToolCall, TokenUsage } from '@freellmapi/shared/t
import { type RouteResult, type ResolvedChain, type ChainRow, routeRequest, resolveRoutingChain, resolveModelGroupCandidates, resolveStickyPreference, hasEnabledVisionModel, hasEnabledToolsModel, routingReserveTokens } from '../services/router.js';
import { secondsUntilNextMonth } from '../services/key-budget.js';
import { runEmbeddings, EmbeddingsError } from '../services/embeddings.js';
+import { runRerank, RerankError, rerankRetryAfterSec } from '../services/rerank.js';
import { retryAfterSeconds } from '../lib/retry-hint.js';
import { runImageGeneration, runVideoGeneration, runSpeech, runTranscription, MediaError, MAX_TRANSCRIPTION_BYTES } from '../services/media.js';
import multer from 'multer';
@@ -698,6 +699,45 @@ proxyRouter.post('/embeddings', async (req: Request, res: Response) => {
}
});
+// Cohere/Jina-style rerank over the key pool (#1029). No catalog: a Cohere key
+// or a custom OpenAI-compatible endpoint exposing POST {base_url}/rerank is a
+// provider; the first success wins, custom endpoints tried first.
+const RerankBody = z.object({
+ model: z.string().optional(),
+ query: z.string().min(1),
+ documents: z.array(z.string()).min(1),
+ top_n: z.number().int().positive().optional(),
+});
+
+proxyRouter.post('/rerank', async (req: Request, res: Response) => {
+ if (!requireInferenceAuth(req, res)) return;
+ const parsed = RerankBody.safeParse(req.body);
+ if (!parsed.success) {
+ res.status(400).json({ error: { message: 'Invalid request: `query` and `documents` are required', type: 'invalid_request_error' } });
+ return;
+ }
+ try {
+ const result = await runRerank(parsed.data.model, parsed.data.query, parsed.data.documents, parsed.data.top_n);
+ res.json({
+ results: result.results.map(r => ({
+ index: r.index,
+ relevance_score: r.relevanceScore,
+ document: { text: r.document },
+ })),
+ model: result.modelId,
+ provider: result.platform,
+ });
+ } catch (err: any) {
+ const status = err instanceof RerankError ? err.status : 502;
+ if (err instanceof RerankError) {
+ const retryAfter = rerankRetryAfterSec(err);
+ if (retryAfter !== undefined) res.setHeader('Retry-After', String(retryAfter));
+ }
+ const type = status === 400 ? 'invalid_request_error' : status === 429 ? 'rate_limit_error' : 'server_error';
+ res.status(status).json({ error: { message: `rerank error: ${err?.message ?? 'unknown'}`, type } });
+ }
+});
+
// OpenAI-compatible image generation. Routed through the media catalog (its own
// table, never the chat router): `model: "auto"` (or omitted) tries every enabled
// image provider in order; a provider model id pins to that one. Failover is
diff --git a/server/src/services/rerank.ts b/server/src/services/rerank.ts
new file mode 100644
index 00000000..bb3d5cdb
--- /dev/null
+++ b/server/src/services/rerank.ts
@@ -0,0 +1,123 @@
+import { getDb } from '../db/index.js';
+import { decrypt } from '../lib/crypto.js';
+import { parseRetryAfterMs } from '../providers/base.js';
+import { secondsUntilNextMonth } from './key-budget.js';
+
+// Cohere-style rerank over the existing free-tier key pool (#1029). Deliberately
+// narrow first slice: platform='cohere' keys and custom OpenAI-compatible
+// endpoints (POST {base_url}/rerank, Jina shape). No catalog table, no new
+// platforms — a rerank model id pins nothing; failover walks the candidate
+// keys in order and takes the first success.
+
+export interface RerankResult {
+ platform: string;
+ modelId: string;
+ results: { index: number; relevanceScore: number; document: string }[];
+}
+
+export class RerankError extends Error {
+ status: number;
+ code?: string;
+ retryAfterMs?: number;
+ constructor(message: string, status: number, code?: string, retryAfterMs?: number) {
+ super(message);
+ this.status = status;
+ this.code = code;
+ this.retryAfterMs = retryAfterMs;
+ }
+}
+
+interface RerankCandidate {
+ id: number;
+ platform: string;
+ key: string;
+ baseUrl: string | null;
+}
+
+const FETCH_TIMEOUT_MS = 30_000;
+const MAX_DOCUMENTS = 100;
+
+function candidates(): RerankCandidate[] {
+ const rows = getDb().prepare(
+ "SELECT id, platform, base_url, encrypted_key, iv, auth_tag FROM api_keys WHERE enabled = 1 AND status IN ('healthy', 'unknown') AND platform IN ('cohere', 'custom') ORDER BY platform, created_at",
+ ).all() as { id: number; platform: string; base_url: string | null; encrypted_key: string; iv: string; auth_tag: string }[];
+ const out: RerankCandidate[] = [];
+ for (const row of rows) {
+ if (row.platform === 'custom' && !row.base_url) continue;
+ try {
+ out.push({ id: row.id, platform: row.platform, key: decrypt(row.encrypted_key, row.iv, row.auth_tag), baseUrl: row.base_url });
+ } catch { /* undecryptable row is not a candidate */ }
+ }
+ // Custom endpoints (the operator's explicit rerank relays) go first.
+ return out.sort((a, b) => (a.platform === 'custom' ? -1 : 0) - (b.platform === 'custom' ? -1 : 0));
+}
+
+async function upstreamError(r: Response): Promise<RerankError> {
+ const retryAfterMs = parseRetryAfterMs(r.headers?.get('retry-after') ?? null);
+ return new RerankError(`upstream ${r.status}: ${(await r.text()).slice(0, 200)}`, r.status, undefined, retryAfterMs);
+}
+
+async function callProvider(cand: RerankCandidate, model: string | undefined, query: string,
+ documents: string[], topN: number): Promise<{ modelId: string; results: RerankResult['results'] }> {
+ // ponytail: cohere + jina-shaped custom only; per-provider adapters when a
+ // provider needs a different wire shape.
+ const url = cand.platform === 'cohere'
+ ? 'https://api.cohere.com/v2/rerank'
+ : `${cand.baseUrl!.replace(/\/$/, '')}/rerank`;
+ const modelId = model
+ || (cand.platform === 'cohere' ? 'rerank-v3.5' : 'rerank');
+ const res = await fetch(url, {
+ method: 'POST',
+ headers: { Authorization: `Bearer ${cand.key}`, 'Content-Type': 'application/json' },
+ body: JSON.stringify({ model: modelId, query, documents, top_n: topN }),
+ signal: AbortSignal.timeout(FETCH_TIMEOUT_MS),
+ });
+ if (!res.ok) throw await upstreamError(res);
+ const body = await res.json() as { results?: { index?: number; relevance_score?: number; document?: { text?: string } | string }[] };
+ if (!Array.isArray(body.results)) throw new RerankError('upstream returned malformed rerank results', 502);
+ const results = body.results.map(r => ({
+ index: r.index ?? -1,
+ relevanceScore: r.relevance_score ?? 0,
+ document: typeof r.document === 'string' ? r.document : r.document?.text ?? documents[r.index ?? -1] ?? '',
+ })).filter(r => r.index >= 0 && r.index < documents.length);
+ // An upstream answer with no in-range hit serves nobody — treat it as a
+ // malformed response so the chain moves on to the next provider.
+ if (results.length === 0) throw new RerankError('upstream returned no valid rerank hits', 502);
+ return { modelId, results };
+}
+
+export async function runRerank(model: string | undefined, query: string, documents: string[],
+ topN?: number): Promise<RerankResult> {
+ if (documents.length > MAX_DOCUMENTS) {
+ throw new RerankError(`Too many documents: ${documents.length} (max ${MAX_DOCUMENTS})`, 400);
+ }
+ const chain = candidates();
+ if (chain.length === 0) {
+ throw new RerankError('No rerank-capable key configured. Add a Cohere key or a custom rerank endpoint.', 503);
+ }
+ const n = topN ?? documents.length;
+
+ let lastError: RerankError | null = null;
+ for (const cand of chain) {
+ try {
+ const out = await callProvider(cand, model, query, documents, n);
+ return { platform: cand.platform, modelId: out.modelId, results: out.results };
+ } catch (err: any) {
+ lastError = err instanceof RerankError ? err : new RerankError(String(err?.message ?? err), 502);
+ }
+ }
+ throw new RerankError(
+ `All rerank providers failed${lastError ? ` (last: ${lastError.message.slice(0, 160)})` : ''}.`,
+ lastError?.status === 429 ? 429 : 502,
+ lastError?.code,
+ lastError?.status === 429 ? lastError?.retryAfterMs : undefined,
+ );
+}
+
+/** Whole seconds the client should wait before retrying: the upstream back-off
+ * when the chain was rate limited, else the monthly-budget reset. */
+export function rerankRetryAfterSec(err: RerankError): number | undefined {
+ if (err.retryAfterMs !== undefined) return Math.ceil(err.retryAfterMs / 1000);
+ if (err.code === 'quota_exceeded') return secondsUntilNextMonth();
+ return undefined;
+}