Skip to content

7.4 billion tokens per month. 34 free LLM providers. 635 free model endpoints. All behind one /v1 endpoint, plus any custom OpenAI-compatible endpoint. Smart routing, automatic failover, encrypted keys. Personal experimentation only.

TypeScript

feat(embeddings): family-based embeddings router with in-family provider failover (#195)

Tashfeenae47d45d0612
diff --git a/client/src/App.tsx b/client/src/App.tsx
index a9380da5..7568bc95 100644
--- a/client/src/App.tsx
+++ b/client/src/App.tsx
@@ -7,6 +7,7 @@ import { logout } from '@/lib/api'
import KeysPage from '@/pages/KeysPage'
import PlaygroundPage from '@/pages/PlaygroundPage'
import FallbackPage from '@/pages/FallbackPage'
+import EmbeddingsPage from '@/pages/EmbeddingsPage'
import AnalyticsPage from '@/pages/AnalyticsPage'
const queryClient = new QueryClient()
@@ -78,7 +79,7 @@ function App() {
<div className="max-w-6xl mx-auto px-6 flex items-center">
<Brand />
<nav className="flex items-center gap-6 ml-10">
- <NavItem to="/fallback">Fallback</NavItem>
+ <NavItem to="/models">Models</NavItem>
<NavItem to="/playground">Playground</NavItem>
<NavItem to="/keys">Keys</NavItem>
<NavItem to="/analytics">Analytics</NavItem>
@@ -91,10 +92,13 @@ function App() {
</header>
<main className="max-w-6xl mx-auto px-6 py-8">
<Routes>
- <Route path="/" element={<Navigate to="/fallback" replace />} />
+ <Route path="/" element={<Navigate to="/models/chat" replace />} />
+ <Route path="/models" element={<Navigate to="/models/chat" replace />} />
+ <Route path="/models/chat" element={<FallbackPage />} />
+ <Route path="/models/embeddings" element={<EmbeddingsPage />} />
<Route path="/playground" element={<PlaygroundPage />} />
<Route path="/keys" element={<KeysPage />} />
- <Route path="/fallback" element={<FallbackPage />} />
+ <Route path="/fallback" element={<Navigate to="/models/chat" replace />} />
<Route path="/analytics" element={<AnalyticsPage />} />
<Route path="/test" element={<Navigate to="/playground" replace />} />
<Route path="/health" element={<Navigate to="/keys" replace />} />
diff --git a/client/src/components/floating-bar.tsx b/client/src/components/floating-bar.tsx
new file mode 100644
index 00000000..9ba06e78
--- /dev/null
+++ b/client/src/components/floating-bar.tsx
@@ -0,0 +1,30 @@
+import { useEffect, useState, type ReactNode } from 'react'
+
+// Viewport-fixed action bar that slides up when shown and — instead of
+// vanishing — slides back down before unmounting (kept in the tree for the
+// duration of the exit animation).
+export function FloatingBar({ show, children }: { show: boolean; children: ReactNode }) {
+ const [render, setRender] = useState(show)
+ useEffect(() => {
+ if (show) {
+ setRender(true)
+ return
+ }
+ const t = setTimeout(() => setRender(false), 300) // match animation duration
+ return () => clearTimeout(t)
+ }, [show])
+ if (!render) return null
+ return (
+ <div
+ className={`fixed inset-x-0 bottom-6 z-50 flex justify-center px-6 pointer-events-none duration-300 ${
+ show
+ ? 'animate-in slide-in-from-bottom-4 fade-in'
+ : 'animate-out slide-out-to-bottom-4 fade-out fill-mode-forwards'
+ }`}
+ >
+ <div className="pointer-events-auto flex items-center gap-3 rounded-full border bg-card px-4 py-2 shadow-lg">
+ {children}
+ </div>
+ </div>
+ )
+}
diff --git a/client/src/components/models-tabs.tsx b/client/src/components/models-tabs.tsx
new file mode 100644
index 00000000..2d56801c
--- /dev/null
+++ b/client/src/components/models-tabs.tsx
@@ -0,0 +1,18 @@
+import { NavLink } from 'react-router-dom'
+
+// Segmented Chat | Embeddings switcher shared by the two Models pages.
+// Industry-standard layout: one "Models" section, modality as a tab — chat
+// routing (cross-model fallback) and embeddings routing (same-model,
+// cross-provider fallback) are different machines behind one roof.
+export function ModelsTabs() {
+ const tab = (isActive: boolean) =>
+ `px-3 py-1.5 text-xs rounded-lg transition-colors ${
+ isActive ? 'bg-foreground text-background font-medium' : 'text-muted-foreground hover:text-foreground hover:bg-muted'
+ }`
+ return (
+ <div className="inline-flex gap-1 rounded-xl border p-1">
+ <NavLink to="/models/chat" className={({ isActive }) => tab(isActive)}>Chat models</NavLink>
+ <NavLink to="/models/embeddings" className={({ isActive }) => tab(isActive)}>Embeddings</NavLink>
+ </div>
+ )
+}
diff --git a/client/src/pages/EmbeddingsPage.tsx b/client/src/pages/EmbeddingsPage.tsx
new file mode 100644
index 00000000..7daafc64
--- /dev/null
+++ b/client/src/pages/EmbeddingsPage.tsx
@@ -0,0 +1,221 @@
+import { useState } from 'react'
+import { useQuery, useMutation, useQueryClient } from '@tanstack/react-query'
+import { ArrowDown, ArrowUp } from 'lucide-react'
+import { apiFetch } from '@/lib/api'
+import { Button } from '@/components/ui/button'
+import { Switch } from '@/components/ui/switch'
+import { PageHeader } from '@/components/page-header'
+import { FloatingBar } from '@/components/floating-bar'
+import { ModelsTabs } from '@/components/models-tabs'
+
+interface ProviderEntry {
+ id: number
+ platform: string
+ modelId: string
+ displayName: string
+ priority: number
+ enabled: boolean
+ quotaLabel: string
+ keyCount: number
+}
+
+interface Family {
+ family: string
+ dimensions: number
+ maxInputTokens: number | null
+ isDefault: boolean
+ providers: ProviderEntry[]
+}
+
+interface EmbeddingsData {
+ defaultFamily: string
+ families: Family[]
+}
+
+interface UsageData {
+ families: { family: string; requestsToday: number; tokensMonth: number }[]
+}
+
+function formatTokens(n: number): string {
+ if (n >= 1_000_000) return `${(n / 1_000_000).toFixed(1)}M`
+ if (n >= 1_000) return `${(n / 1_000).toFixed(1)}K`
+ return String(n)
+}
+
+export default function EmbeddingsPage() {
+ const queryClient = useQueryClient()
+ // Local unsaved edits, same pattern as the chat fallback page.
+ const [localFamilies, setLocalFamilies] = useState<Family[] | null>(null)
+ const [localDefault, setLocalDefault] = useState<string | null>(null)
+
+ const { data, isLoading } = useQuery<EmbeddingsData>({
+ queryKey: ['embeddings'],
+ queryFn: () => apiFetch('/api/embeddings'),
+ })
+
+ const { data: usage } = useQuery<UsageData>({
+ queryKey: ['embeddings', 'usage'],
+ queryFn: () => apiFetch('/api/embeddings/usage'),
+ refetchInterval: 30_000,
+ })
+ const usageByFamily = new Map((usage?.families ?? []).map(u => [u.family, u]))
+
+ const saveMutation = useMutation({
+ mutationFn: (body: { defaultFamily?: string; providers?: { id: number; priority: number; enabled: boolean }[] }) =>
+ apiFetch('/api/embeddings', { method: 'PUT', body: JSON.stringify(body) }),
+ onSuccess: () => {
+ queryClient.invalidateQueries({ queryKey: ['embeddings'] })
+ setLocalFamilies(null)
+ setLocalDefault(null)
+ },
+ })
+
+ const families = localFamilies ?? data?.families ?? []
+ const defaultFamily = localDefault ?? data?.defaultFamily ?? ''
+ const hasChanges = localFamilies !== null || localDefault !== null
+
+ function updateProvider(familyName: string, id: number, patch: Partial<ProviderEntry>) {
+ setLocalFamilies(families.map(f =>
+ f.family === familyName
+ ? { ...f, providers: f.providers.map(p => (p.id === id ? { ...p, ...patch } : p)) }
+ : f,
+ ))
+ }
+
+ function moveProvider(familyName: string, index: number, dir: -1 | 1) {
+ setLocalFamilies(families.map(f => {
+ if (f.family !== familyName) return f
+ const list = [...f.providers]
+ const j = index + dir
+ if (j < 0 || j >= list.length) return f
+ ;[list[index], list[j]] = [list[j], list[index]]
+ return { ...f, providers: list.map((p, i) => ({ ...p, priority: i + 1 })) }
+ }))
+ }
+
+ function handleSave() {
+ saveMutation.mutate({
+ ...(localDefault !== null ? { defaultFamily: localDefault } : {}),
+ ...(localFamilies !== null
+ ? { providers: families.flatMap(f => f.providers.map(p => ({ id: p.id, priority: p.priority, enabled: p.enabled }))) }
+ : {}),
+ })
+ }
+
+ function discard() {
+ setLocalFamilies(null)
+ setLocalDefault(null)
+ }
+
+ return (
+ <div>
+ <PageHeader
+ title="Models"
+ description="Embeddings fail over within a family only — the same model served by another provider. Vectors from different models are incompatible, so the router never swaps models on you."
+ divider={false}
+ actions={<ModelsTabs />}
+ />
+
+ <div className="space-y-6">
+ <p className="text-xs text-muted-foreground">
+ <code className="rounded-md bg-muted px-1.5 py-0.5 font-mono">model: "auto"</code> on{' '}
+ <code className="rounded-md bg-muted px-1.5 py-0.5 font-mono">POST /v1/embeddings</code> routes to the
+ default family. Naming a family (or a provider model id) pins that family; providers inside it are tried in order.
+ </p>
+
+ {isLoading ? (
+ <p className="text-sm text-muted-foreground">Loading…</p>
+ ) : (
+ families.map(f => {
+ const u = usageByFamily.get(f.family)
+ const noKeys = f.providers.every(p => p.keyCount === 0)
+ return (
+ <section key={f.family} className={`rounded-3xl border bg-card p-5 ${noKeys ? 'opacity-60' : ''}`}>
+ <div className="flex items-baseline justify-between gap-4 mb-3 flex-wrap">
+ <div className="flex items-baseline gap-2.5 min-w-0">
+ <h2 className="text-sm font-medium font-mono truncate">{f.family}</h2>
+ <span className="text-[10px] rounded-full px-1.5 py-0.5 bg-muted text-muted-foreground tabular-nums">
+ {f.dimensions}d
+ </span>
+ {f.maxInputTokens && (
+ <span className="text-[11px] text-muted-foreground/70 tabular-nums">
+ {formatTokens(f.maxInputTokens)} tok max
+ </span>
+ )}
+ {f.family === defaultFamily ? (
+ <span className="text-[10px] rounded-full px-1.5 py-0.5 bg-foreground text-background font-medium">
+ Default · auto
+ </span>
+ ) : (
+ <button
+ onClick={() => setLocalDefault(f.family)}
+ className="text-[11px] text-muted-foreground hover:text-foreground underline decoration-dotted underline-offset-2 transition-colors"
+ >
+ Make default
+ </button>
+ )}
+ </div>
+ <span className="text-xs text-muted-foreground tabular-nums">
+ {u ? <>{u.requestsToday} req today · {formatTokens(u.tokensMonth)} tok this month</> : '—'}
+ </span>
+ </div>
+
+ <div className="divide-y">
+ {f.providers.map((p, i) => (
+ <div key={p.id} className={`flex items-center gap-3 py-2 ${p.enabled ? '' : 'opacity-50'}`}>
+ <span className="w-5 text-center font-mono text-xs text-muted-foreground tabular-nums">{i + 1}</span>
+ <div className="min-w-0 flex-1">
+ <div className="flex items-center gap-2">
+ <span className="text-sm font-medium">{p.platform}</span>
+ <span className="truncate font-mono text-[11px] text-muted-foreground">{p.modelId}</span>
+ {p.keyCount === 0 && (
+ <span className="text-[10px] rounded-full px-1.5 py-0.5 bg-amber-600/15 text-amber-700 dark:bg-amber-400/15 dark:text-amber-400">
+ no key
+ </span>
+ )}
+ </div>
+ <div className="text-[11px] text-muted-foreground/70">{p.quotaLabel}</div>
+ </div>
+ {f.providers.length > 1 && (
+ <div className="flex gap-0.5">
+ <button
+ onClick={() => moveProvider(f.family, i, -1)}
+ disabled={i === 0}
+ aria-label="Move up"
+ className="rounded-md p-1 text-muted-foreground/60 hover:text-foreground disabled:opacity-25 transition-colors"
+ >
+ <ArrowUp className="size-3.5" />
+ </button>
+ <button
+ onClick={() => moveProvider(f.family, i, 1)}
+ disabled={i === f.providers.length - 1}
+ aria-label="Move down"
+ className="rounded-md p-1 text-muted-foreground/60 hover:text-foreground disabled:opacity-25 transition-colors"
+ >
+ <ArrowDown className="size-3.5" />
+ </button>
+ </div>
+ )}
+ <Switch
+ checked={p.enabled}
+ onCheckedChange={(c) => updateProvider(f.family, p.id, { enabled: c })}
+ />
+ </div>
+ ))}
+ </div>
+ </section>
+ )
+ })
+ )}
+
+ <FloatingBar show={hasChanges}>
+ <span className="text-xs text-muted-foreground">Unsaved changes</span>
+ <Button variant="outline" size="sm" onClick={discard}>Discard</Button>
+ <Button size="sm" onClick={handleSave} disabled={saveMutation.isPending}>
+ {saveMutation.isPending ? 'Saving…' : 'Save changes'}
+ </Button>
+ </FloatingBar>
+ </div>
+ </div>
+ )
+}
diff --git a/client/src/pages/FallbackPage.tsx b/client/src/pages/FallbackPage.tsx
index 63b0c875..74acd98b 100644
--- a/client/src/pages/FallbackPage.tsx
+++ b/client/src/pages/FallbackPage.tsx
@@ -23,6 +23,8 @@ import { apiFetch } from '@/lib/api'
import { Button } from '@/components/ui/button'
import { Switch } from '@/components/ui/switch'
import { PageHeader } from '@/components/page-header'
+import { FloatingBar } from '@/components/floating-bar'
+import { ModelsTabs } from '@/components/models-tabs'
interface FallbackEntry {
modelDbId: number
@@ -258,35 +260,6 @@ function TokenUsageBar({ data }: { data: TokenUsageData }) {
)
}
-// Viewport-fixed action bar that slides up when shown and — instead of
-// vanishing — slides back down before unmounting (kept in the tree for the
-// duration of the exit animation).
-function FloatingBar({ show, children }: { show: boolean; children: ReactNode }) {
- const [render, setRender] = useState(show)
- useEffect(() => {
- if (show) {
- setRender(true)
- return
- }
- const t = setTimeout(() => setRender(false), 300) // match animation duration
- return () => clearTimeout(t)
- }, [show])
- if (!render) return null
- return (
- <div
- className={`fixed inset-x-0 bottom-6 z-50 flex justify-center px-6 pointer-events-none duration-300 ${
- show
- ? 'animate-in slide-in-from-bottom-4 fade-in'
- : 'animate-out slide-out-to-bottom-4 fade-out fill-mode-forwards'
- }`}
- >
- <div className="pointer-events-auto flex items-center gap-3 rounded-full border bg-card px-4 py-2 shadow-lg">
- {children}
- </div>
- </div>
- )
-}
-
// ── One row of the unified table ────────────────────────────────────────────
function RowContent({
row,
@@ -498,9 +471,10 @@ export default function FallbackPage() {
return (
<div>
<PageHeader
- title="Fallback chain"
+ title="Models"
description="Pick a routing strategy. In Manual mode you drag to set the order; the other strategies route by live score across reliability, speed and intelligence."
divider={false}
+ actions={<ModelsTabs />}
/>
<div className="space-y-6">
diff --git a/client/src/pages/KeysPage.tsx b/client/src/pages/KeysPage.tsx
index 506e892a..366d562b 100644
--- a/client/src/pages/KeysPage.tsx
+++ b/client/src/pages/KeysPage.tsx
@@ -161,8 +161,12 @@ function UnifiedKeySection() {
<div className="mt-4 grid grid-cols-[auto_1fr] gap-x-4 gap-y-1.5 text-xs">
<span className="text-muted-foreground">Base URL</span>
<code className="font-mono">{baseUrl}</code>
- <span className="text-muted-foreground">Endpoint</span>
+ <span className="text-muted-foreground">Chat</span>
<code className="font-mono">/v1/chat/completions</code>
+ <span className="text-muted-foreground">Responses</span>
+ <code className="font-mono">/v1/responses</code>
+ <span className="text-muted-foreground">Embeddings</span>
+ <code className="font-mono">/v1/embeddings <span className="text-muted-foreground">— model: "auto" or a family from the Embeddings tab</span></code>
</div>
</section>
)
diff --git a/server/src/__tests__/services/embeddings.test.ts b/server/src/__tests__/services/embeddings.test.ts
new file mode 100644
index 00000000..190b24bb
--- /dev/null
+++ b/server/src/__tests__/services/embeddings.test.ts
@@ -0,0 +1,196 @@
+import { describe, it, expect, beforeEach, afterEach, vi } from 'vitest';
+import { initDb, getDb } from '../../db/index.js';
+import { encrypt } from '../../lib/crypto.js';
+import { resolveFamily, getDefaultFamily, runEmbeddings, EmbeddingsError } from '../../services/embeddings.js';
+
+const realFetch = globalThis.fetch;
+
+function addKey(platform: string, raw = `${platform}-test-key`) {
+ const { encrypted, iv, authTag } = encrypt(raw);
+ getDb().prepare(`
+ INSERT INTO api_keys (platform, label, encrypted_key, iv, auth_tag, status, enabled)
+ VALUES (?, 'test', ?, ?, ?, 'healthy', 1)
+ `).run(platform, encrypted, iv, authTag);
+}
+
+function okEmbeddingResponse(dims: number, count = 1) {
+ return new Response(JSON.stringify({
+ data: Array.from({ length: count }, (_, i) => ({ index: i, embedding: Array(dims).fill(0.1) })),
+ usage: { prompt_tokens: 3 },
+ }), { status: 200, headers: { 'Content-Type': 'application/json' } });
+}
+
+describe('embeddings service', () => {
+ beforeEach(() => {
+ process.env.ENCRYPTION_KEY = '0'.repeat(64);
+ initDb(':memory:');
+ });
+
+ afterEach(() => {
+ globalThis.fetch = realFetch;
+ vi.restoreAllMocks();
+ });
+
+ describe('migration seed', () => {
+ it('seeds the embedding catalog with families and a default', () => {
+ const rows = getDb().prepare('SELECT DISTINCT family FROM embedding_models').all() as { family: string }[];
+ const families = rows.map(r => r.family);
+ expect(families).toContain('gemini-embedding-001');
+ expect(families).toContain('llama-nemotron-embed-vl-1b-v2');
+ expect(families).toContain('bge-m3');
+ expect(getDefaultFamily()).toBe('gemini-embedding-001');
+ });
+
+ it('cohere is seeded disabled (its quota is shared with chat)', () => {
+ const row = getDb().prepare("SELECT enabled FROM embedding_models WHERE platform = 'cohere'").get() as { enabled: number };
+ expect(row.enabled).toBe(0);
+ });
+
+ it('multi-provider families share one dimension', () => {
+ const dims = getDb().prepare(
+ "SELECT DISTINCT dimensions FROM embedding_models WHERE family = 'llama-nemotron-embed-vl-1b-v2'",
+ ).all();
+ expect(dims).toHaveLength(1);
+ });
+ });
+
+ describe('resolveFamily', () => {
+ it("maps 'auto', empty and undefined to the default family", () => {
+ expect(resolveFamily('auto')).toBe('gemini-embedding-001');
+ expect(resolveFamily('')).toBe('gemini-embedding-001');
+ expect(resolveFamily(undefined)).toBe('gemini-embedding-001');
+ });
+
+ it('accepts a family name directly', () => {
+ expect(resolveFamily('bge-m3')).toBe('bge-m3');
+ });
+
+ it('maps a provider-specific model id to its family', () => {
+ expect(resolveFamily('@cf/baai/bge-m3')).toBe('bge-m3');
+ expect(resolveFamily('nvidia/llama-nemotron-embed-vl-1b-v2')).toBe('llama-nemotron-embed-vl-1b-v2');
+ });
+
+ it('returns null for unknown models', () => {
+ expect(resolveFamily('text-embedding-ada-002')).toBeNull();
+ });
+ });
+
+ describe('runEmbeddings', () => {
+ it('rejects unknown models with a 400', async () => {
+ await expect(runEmbeddings('no-such-model', ['hi'])).rejects.toMatchObject({ status: 400 });
+ });
+
+ it('embeds via the first provider in the family chain', async () => {
+ addKey('nvidia');
+ addKey('openrouter');
+ const fetchMock = vi.fn(async () => okEmbeddingResponse(2048));
+ globalThis.fetch = fetchMock as any;
+
+ const result = await runEmbeddings('llama-nemotron-embed-vl-1b-v2', ['hello']);
+ expect(result.platform).toBe('nvidia');
+ expect(result.dimensions).toBe(2048);
+ expect(result.vectors).toHaveLength(1);
+ expect(fetchMock).toHaveBeenCalledTimes(1);
+ expect(String(fetchMock.mock.calls[0][0])).toContain('integrate.api.nvidia.com');
+ });
+
+ it('fails over WITHIN the family when the first provider errors', async () => {
+ addKey('nvidia');
+ addKey('openrouter');
+ const fetchMock = vi.fn()
+ .mockResolvedValueOnce(new Response('rate limited', { status: 429 }))
+ .mockResolvedValueOnce(okEmbeddingResponse(2048));
+ globalThis.fetch = fetchMock as any;
+
+ const result = await runEmbeddings('llama-nemotron-embed-vl-1b-v2', ['hello']);
+ expect(result.platform).toBe('openrouter');
+ expect(fetchMock).toHaveBeenCalledTimes(2);
+ expect(String(fetchMock.mock.calls[1][0])).toContain('openrouter.ai');
+ });
+
+ it('skips providers without a usable key instead of failing', async () => {
+ addKey('openrouter'); // no nvidia key
+ const fetchMock = vi.fn(async () => okEmbeddingResponse(2048));
+ globalThis.fetch = fetchMock as any;
+
+ const result = await runEmbeddings('llama-nemotron-embed-vl-1b-v2', ['hello']);
+ expect(result.platform).toBe('openrouter');
+ expect(fetchMock).toHaveBeenCalledTimes(1);
+ });
+
+ it('throws 429 when every provider is rate-limited', async () => {
+ addKey('nvidia');
+ addKey('openrouter');
+ globalThis.fetch = vi.fn(async () => new Response('slow down', { status: 429 })) as any;
+
+ await expect(runEmbeddings('llama-nemotron-embed-vl-1b-v2', ['hello'])).rejects.toMatchObject({ status: 429 });
+ });
+
+ it('throws 503 when the family has no enabled providers', async () => {
+ getDb().prepare("UPDATE embedding_models SET enabled = 0 WHERE family = 'bge-m3'").run();
+ await expect(runEmbeddings('bge-m3', ['hello'])).rejects.toMatchObject({ status: 503 });
+ });
+
+ it('splits cloudflare account_id:token keys', async () => {
+ addKey('cloudflare', 'acct-123:cf-token-xyz');
+ const fetchMock = vi.fn(async () => okEmbeddingResponse(1024));
+ globalThis.fetch = fetchMock as any;
+
+ const result = await runEmbeddings('embeddinggemma-300m', ['hello']);
+ expect(result.platform).toBe('cloudflare');
+ expect(String(fetchMock.mock.calls[0][0])).toContain('/accounts/acct-123/ai/v1/embeddings');
+ const headers = (fetchMock.mock.calls[0][1] as RequestInit).headers as Record<string, string>;
+ expect(headers.Authorization).toBe('Bearer cf-token-xyz');
+ });
+
+ it('normalizes hugging face feature-extraction output', async () => {
+ // bge-m3: cloudflare first (no key) → falls through to huggingface
+ addKey('huggingface');
+ const fetchMock = vi.fn(async () => new Response(JSON.stringify([[0.1, 0.2, 0.3]]), { status: 200 }));
+ globalThis.fetch = fetchMock as any;
+
+ const result = await runEmbeddings('bge-m3', ['hello']);
+ expect(result.platform).toBe('huggingface');
+ expect(result.dimensions).toBe(3);
+ expect(String(fetchMock.mock.calls[0][0])).toContain('feature-extraction');
+ });
+
+ it('rejects malformed upstream payloads and fails over', async () => {
+ addKey('nvidia');
+ addKey('openrouter');
+ const fetchMock = vi.fn()
+ .mockResolvedValueOnce(new Response(JSON.stringify({ data: [] }), { status: 200 })) // wrong count
+ .mockResolvedValueOnce(okEmbeddingResponse(2048));
+ globalThis.fetch = fetchMock as any;
+
+ const result = await runEmbeddings('llama-nemotron-embed-vl-1b-v2', ['hello']);
+ expect(result.platform).toBe('openrouter');
+ });
+
+ it("logs requests tagged request_type='embedding' so chat budgets ignore them", async () => {
+ addKey('nvidia');
+ addKey('openrouter');
+ const fetchMock = vi.fn()
+ .mockResolvedValueOnce(new Response('boom', { status: 500 }))
+ .mockResolvedValueOnce(okEmbeddingResponse(2048));
+ globalThis.fetch = fetchMock as any;
+
+ await runEmbeddings('llama-nemotron-embed-vl-1b-v2', ['hello']);
+ const rows = getDb().prepare(
+ "SELECT platform, status, request_type FROM requests ORDER BY id",
+ ).all() as { platform: string; status: string; request_type: string }[];
+ expect(rows).toEqual([
+ { platform: 'nvidia', status: 'error', request_type: 'embedding' },
+ { platform: 'openrouter', status: 'success', request_type: 'embedding' },
+ ]);
+
+ // and the chat-scoped monthly usage query sees none of it
+ const chatUsed = getDb().prepare(`
+ SELECT COALESCE(SUM(input_tokens + output_tokens), 0) AS used
+ FROM requests
+ WHERE created_at >= datetime('now', 'start of month') AND request_type = 'chat'
+ `).get() as { used: number };
+ expect(chatUsed.used).toBe(0);
+ });
+ });
+});
diff --git a/server/src/app.ts b/server/src/app.ts
index da29f4b3..355f649c 100644
--- a/server/src/app.ts
+++ b/server/src/app.ts
@@ -8,6 +8,7 @@ import { modelsRouter } from './routes/models.js';
import { proxyRouter } from './routes/proxy.js';
import { responsesRouter } from './routes/responses.js';
import { fallbackRouter } from './routes/fallback.js';
+import { embeddingsRouter } from './routes/embeddings.js';
import { analyticsRouter } from './routes/analytics.js';
import { healthRouter } from './routes/health.js';
import { settingsRouter } from './routes/settings.js';
@@ -60,6 +61,7 @@ export function createApp() {
app.use('/api/keys', requireAuth, keysRouter);
app.use('/api/models', requireAuth, modelsRouter);
app.use('/api/fallback', requireAuth, fallbackRouter);
+ app.use('/api/embeddings', requireAuth, embeddingsRouter);
app.use('/api/analytics', requireAuth, analyticsRouter);
app.use('/api/health', requireAuth, healthRouter);
app.use('/api/settings', requireAuth, settingsRouter);
diff --git a/server/src/db/index.ts b/server/src/db/index.ts
index 689ed5b9..81a5a699 100644
--- a/server/src/db/index.ts
+++ b/server/src/db/index.ts
@@ -57,6 +57,7 @@ export function initDb(dbPath?: string): Database.Database {
migrateModelsV20KiloFree(db);
migrateModelsV21PruneDead(db);
migrateModelsV22Tools(db);
+ migrateEmbeddingsV1(db);
ensureUnifiedKey(db);
console.log(`Database initialized at ${resolvedPath}`);
@@ -1690,6 +1691,66 @@ function migrateModelsV22Tools(db: Database.Database) {
apply();
}
+// Embeddings V1 (2026-06): per-family embedding catalog. A "family" is one
+// model identity + dimension — vectors from different families live in
+// incompatible spaces, so /v1/embeddings only ever fails over WITHIN a family
+// (same model served by another provider), never across families.
+// Every entry was live-verified against the provider on 2026-06-04.
+function migrateEmbeddingsV1(db: Database.Database) {
+ db.exec(`
+ CREATE TABLE IF NOT EXISTS embedding_models (
+ id INTEGER PRIMARY KEY AUTOINCREMENT,
+ family TEXT NOT NULL,
+ platform TEXT NOT NULL,
+ model_id TEXT NOT NULL,
+ display_name TEXT NOT NULL,
+ dimensions INTEGER NOT NULL,
+ max_input_tokens INTEGER,
+ priority INTEGER NOT NULL DEFAULT 0,
+ enabled INTEGER NOT NULL DEFAULT 1,
+ quota_label TEXT NOT NULL DEFAULT '',
+ UNIQUE(platform, model_id)
+ );
+ `);
+
+ // Tag request rows so embeddings traffic doesn't pollute the chat token
+ // budget / headroom math. Existing rows backfill to 'chat' via the default.
+ const columns = db.prepare('PRAGMA table_info(requests)').all() as { name: string }[];
+ if (!columns.some(col => col.name === 'request_type')) {
+ db.prepare("ALTER TABLE requests ADD COLUMN request_type TEXT NOT NULL DEFAULT 'chat'").run();
+ }
+
+ const seed = db.prepare(`
+ INSERT OR IGNORE INTO embedding_models
+ (family, platform, model_id, display_name, dimensions, max_input_tokens, priority, enabled, quota_label)
+ VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)
+ `);
+ const rows: Array<[string, string, string, string, number, number | null, number, number, string]> = [
+ // family, platform, provider model id, display name, dims, max input, priority, enabled, quota label
+ ['gemini-embedding-001', 'google', 'gemini-embedding-001', 'Gemini Embedding', 3072, 2048, 1, 1, '100 rpm · 1K req/day'],
+ ['llama-nemotron-embed-vl-1b-v2', 'nvidia', 'nvidia/llama-nemotron-embed-vl-1b-v2', 'Nemotron Embed VL 1B', 2048, 8192, 1, 1, '~40 rpm'],
+ ['llama-nemotron-embed-vl-1b-v2', 'openrouter', 'nvidia/llama-nemotron-embed-vl-1b-v2', 'Nemotron Embed VL 1B (OR)', 2048, 8192, 2, 1, '$0/M tok'],
+ ['llama-nemotron-embed-1b-v2', 'nvidia', 'nvidia/llama-nemotron-embed-1b-v2', 'Nemotron Embed 1B', 2048, 8192, 1, 1, '~40 rpm'],
+ ['nv-embedqa-e5-v5', 'nvidia', 'nvidia/nv-embedqa-e5-v5', 'NV-EmbedQA E5 v5', 1024, 512, 1, 1, '~40 rpm'],
+ ['text-embedding-3-small', 'github', 'openai/text-embedding-3-small', 'Text Embedding 3 Small', 1536, 8191, 1, 1, 'rate-limited free'],
+ ['text-embedding-3-large', 'github', 'openai/text-embedding-3-large', 'Text Embedding 3 Large', 3072, 8191, 1, 1, 'rate-limited free'],
+ ['bge-m3', 'cloudflare', '@cf/baai/bge-m3', 'BGE-M3', 1024, 8192, 1, 1, '10K neurons/day (shared)'],
+ ['bge-m3', 'huggingface', 'BAAI/bge-m3', 'BGE-M3 (HF)', 1024, 8192, 2, 1, '$0.10/mo credits'],
+ ['embeddinggemma-300m', 'cloudflare', '@cf/google/embeddinggemma-300m', 'EmbeddingGemma 300M', 768, 2048, 1, 1, '10K neurons/day (shared)'],
+ ['qwen3-embedding-0.6b', 'cloudflare', '@cf/qwen/qwen3-embedding-0.6b', 'Qwen3 Embedding 0.6B', 1024, 4096, 1, 1, '10K neurons/day (shared)'],
+ // Cohere trial keys allow 1,000 calls/month TOTAL shared with chat —
+ // disabled by default so embedding traffic can't silently eat chat quota.
+ ['embed-v4.0', 'cohere', 'embed-v4.0', 'Cohere Embed v4', 1536, 128000, 1, 0, '1K calls/mo (shared w/ chat)'],
+ ];
+ const apply = db.transaction(() => { for (const r of rows) seed.run(...r); });
+ apply();
+
+ const def = db.prepare("SELECT value FROM settings WHERE key = 'embeddings_default_family'").get();
+ if (!def) {
+ db.prepare("INSERT INTO settings (key, value) VALUES ('embeddings_default_family', 'gemini-embedding-001')").run();
+ }
+}
+
/** Append any models not yet in the fallback chain, lowest priority, ordered by
* intelligence_rank. Shared by the recent model migrations (V18–V20). */
function backfillFallback(db: Database.Database) {
diff --git a/server/src/routes/embeddings.ts b/server/src/routes/embeddings.ts
new file mode 100644
index 00000000..ddbb5280
--- /dev/null
+++ b/server/src/routes/embeddings.ts
@@ -0,0 +1,109 @@
+import { Router } from 'express';
+import type { Request, Response } from 'express';
+import { z } from 'zod';
+import { getDb, setSetting } from '../db/index.js';
+import { listEmbeddingModels, getDefaultFamily, type EmbeddingModelRow } from '../services/embeddings.js';
+
+export const embeddingsRouter = Router();
+
+// Families with their provider chains, for the dashboard Embeddings tab.
+embeddingsRouter.get('/', (_req: Request, res: Response) => {
+ const db = getDb();
+ const keyCounts = new Map(
+ (db.prepare(
+ "SELECT platform, COUNT(*) AS n FROM api_keys WHERE enabled = 1 AND status IN ('healthy', 'unknown') GROUP BY platform",
+ ).all() as { platform: string; n: number }[]).map(r => [r.platform, r.n]),
+ );
+
+ const byFamily = new Map<string, EmbeddingModelRow[]>();
+ for (const row of listEmbeddingModels()) {
+ const list = byFamily.get(row.family) ?? [];
+ list.push(row);
+ byFamily.set(row.family, list);
+ }
+
+ const defaultFamily = getDefaultFamily();
+ res.json({
+ defaultFamily,
+ families: [...byFamily.entries()].map(([family, rows]) => ({
+ family,
+ dimensions: rows[0].dimensions,
+ maxInputTokens: rows[0].max_input_tokens,
+ isDefault: family === defaultFamily,
+ providers: rows.map(r => ({
+ id: r.id,
+ platform: r.platform,
+ modelId: r.model_id,
+ displayName: r.display_name,
+ priority: r.priority,
+ enabled: r.enabled === 1,
+ quotaLabel: r.quota_label,
+ keyCount: keyCounts.get(r.platform) ?? 0,
+ })),
+ })),
+ });
+});
+
+const updateSchema = z.object({
+ defaultFamily: z.string().optional(),
+ providers: z.array(z.object({
+ id: z.number(),
+ priority: z.number(),
+ enabled: z.boolean(),
+ })).optional(),
+});
+
+embeddingsRouter.put('/', (req: Request, res: Response) => {
+ const parsed = updateSchema.safeParse(req.body);
+ if (!parsed.success) {
+ res.status(400).json({ error: { message: 'Invalid request body' } });
+ return;
+ }
+ const db = getDb();
+
+ if (parsed.data.defaultFamily) {
+ const exists = db.prepare('SELECT 1 FROM embedding_models WHERE family = ?').get(parsed.data.defaultFamily);
+ if (!exists) {
+ res.status(400).json({ error: { message: `Unknown family '${parsed.data.defaultFamily}'` } });
+ return;
+ }
+ setSetting('embeddings_default_family', parsed.data.defaultFamily);
+ }
+
+ if (parsed.data.providers) {
+ const update = db.prepare('UPDATE embedding_models SET priority = ?, enabled = ? WHERE id = ?');
+ const apply = db.transaction((rows: { id: number; priority: number; enabled: boolean }[]) => {
+ for (const r of rows) update.run(r.priority, r.enabled ? 1 : 0, r.id);
+ });
+ apply(parsed.data.providers);
+ }
+
+ res.json({ success: true });
+});
+
+// Per-family usage: requests today (most embedding quotas are daily/RPM) and
+// tokens this calendar month, from the tagged request log.
+embeddingsRouter.get('/usage', (_req: Request, res: Response) => {
+ const db = getDb();
+ const usage = db.prepare(`
+ SELECT em.family,
+ COALESCE(SUM(CASE WHEN r.created_at >= datetime('now', 'start of day') THEN 1 ELSE 0 END), 0) AS requests_today,
+ COALESCE(SUM(CASE WHEN r.created_at >= datetime('now', 'start of month') THEN r.input_tokens ELSE 0 END), 0) AS tokens_month
+ FROM embedding_models em
+ LEFT JOIN requests r
+ ON r.request_type = 'embedding'
+ AND r.status = 'success'
+ AND r.platform = em.platform
+ AND r.model_id = em.model_id
+ AND r.created_at >= datetime('now', 'start of month')
+ GROUP BY em.family
+ `).all() as { family: string; requests_today: number; tokens_month: number }[];
+
+ res.json({
+ families: usage.map(u => ({
+ family: u.family,
+ requestsToday: u.requests_today,
+ tokensMonth: u.tokens_month,
+ })),
+ });
+});
diff --git a/server/src/routes/fallback.ts b/server/src/routes/fallback.ts
index dbe162e5..224fed1b 100644
--- a/server/src/routes/fallback.ts
+++ b/server/src/routes/fallback.ts
@@ -187,6 +187,7 @@ fallbackRouter.get('/token-usage', (_req: Request, res: Response) => {
COALESCE(SUM(input_tokens + output_tokens), 0) as total_used
FROM requests
WHERE created_at >= datetime('now', 'start of month')
+ AND request_type = 'chat'
`).get() as { total_used: number };
res.json({
diff --git a/server/src/routes/proxy.ts b/server/src/routes/proxy.ts
index 69009f8c..e15d9fde 100644
--- a/server/src/routes/proxy.ts
+++ b/server/src/routes/proxy.ts
@@ -6,6 +6,7 @@ import type { ChatMessage } from '@freellmapi/shared/types.js';
import { routeRequest, recordRateLimitHit, recordSuccess, hasEnabledVisionModel, hasEnabledToolsModel, type RouteResult } from '../services/router.js';
import { recordRequest, recordTokens, setCooldown, getCooldownDurationForLimit } from '../services/ratelimit.js';
import { pruneRequestAnalytics } from '../services/request-retention.js';
+import { runEmbeddings, EmbeddingsError } from '../services/embeddings.js';
import { getDb, getUnifiedApiKey } from '../db/index.js';
import { contentToString, messageHasImage, normalizeOutboundContent } from '../lib/content.js';
import { repairToolArguments, toolSchemaMap } from '../lib/tool-args.js';
@@ -250,12 +251,11 @@ export function streamChunkText(chunk: any): string {
return chunk?.choices?.[0]?.delta?.content ?? '';
}
-// OpenAI-compatible embeddings endpoint. The chat key store is per-model and
-// chat-specific, so embeddings use their own dedicated key via the
-// EMBEDDINGS_GOOGLE_KEY env var, routed to Google's embedding API (Gemini
-// `text-embedding-004` by default). Same unified-key auth as the rest of /v1.
-// Lets downstream clients (e.g. RAG apps) get embeddings through the gateway
-// with no extra provider wiring. Returns 503 when no embeddings key is set.
+// OpenAI-compatible embeddings endpoint, routed through the embeddings family
+// catalog: `model: "auto"` (or omitted) → the configured default family; a
+// family name or provider model id → that family's provider chain. Failover
+// only happens WITHIN a family (same model on another provider) — never across
+// models, since vectors from different models are incompatible.
const EmbeddingsBody = z.object({
model: z.string().optional(),
input: z.union([z.string(), z.array(z.string())]),
@@ -268,49 +268,25 @@ proxyRouter.post('/embeddings', async (req: Request, res: Response) => {
res.status(401).json({ error: { message: 'Invalid API key', type: 'authentication_error' } });
return;
}
- const googleKey = process.env.EMBEDDINGS_GOOGLE_KEY;
- if (!googleKey) {
- res.status(503).json({
- error: { message: 'Embeddings not configured on this gateway (set EMBEDDINGS_GOOGLE_KEY)', type: 'server_error' },
- });
- return;
- }
const parsed = EmbeddingsBody.safeParse(req.body);
if (!parsed.success) {
res.status(400).json({ error: { message: 'Invalid request: `input` is required', type: 'invalid_request_error' } });
return;
}
const inputs = Array.isArray(parsed.data.input) ? parsed.data.input : [parsed.data.input];
- // gemini-embedding-001 is the current Generative Language embedding model
- // (text-embedding-004 isn't exposed on this API tier). Callers can override.
- const requested = parsed.data.model && parsed.data.model !== 'auto' ? parsed.data.model : 'gemini-embedding-001';
- const model = requested.startsWith('models/') ? requested : `models/${requested}`;
try {
- // These models support :embedContent (single), not :batchEmbedContents — so
- // fan out one call per input and reassemble in input order.
- const vectors = await Promise.all(
- inputs.map(async (text) => {
- const url = `https://generativelanguage.googleapis.com/v1beta/${model}:embedContent?key=${googleKey}`;
- const r = await fetch(url, {
- method: 'POST',
- headers: { 'Content-Type': 'application/json' },
- body: JSON.stringify({ content: { parts: [{ text }] } }),
- });
- if (!r.ok) {
- throw new Error(`upstream ${r.status}: ${(await r.text()).slice(0, 200)}`);
- }
- const j = (await r.json()) as { embedding?: { values: number[] } };
- return j.embedding?.values ?? [];
- }),
- );
+ const result = await runEmbeddings(parsed.data.model, inputs);
res.json({
object: 'list',
- data: vectors.map((values, i) => ({ object: 'embedding', index: i, embedding: values })),
- model: requested,
- usage: { prompt_tokens: 0, total_tokens: 0 },
+ data: result.vectors.map((values, i) => ({ object: 'embedding', index: i, embedding: values })),
+ model: result.family,
+ provider: result.platform,
+ usage: { prompt_tokens: result.inputTokens, total_tokens: result.inputTokens },
});
} catch (err: any) {
- res.status(502).json({ error: { message: `embedding error: ${err?.message ?? 'unknown'}`, type: 'server_error' } });
+ const status = err instanceof EmbeddingsError ? err.status : 502;
+ const type = status === 400 ? 'invalid_request_error' : status === 429 ? 'rate_limit_error' : 'server_error';
+ res.status(status).json({ error: { message: `embedding error: ${err?.message ?? 'unknown'}`, type } });
}
});
diff --git a/server/src/services/embeddings.ts b/server/src/services/embeddings.ts
new file mode 100644
index 00000000..8375d323
--- /dev/null
+++ b/server/src/services/embeddings.ts
@@ -0,0 +1,240 @@
+// Embeddings routing. Unlike chat, embeddings can NOT fail over across models:
+// vectors from different models live in incompatible spaces, and silently
+// switching models would corrupt any vector store built on top of us. So the
+// routing unit is a "family" (one model identity + dimension) and failover only
+// walks the providers serving that same family.
+//
+// `model: "auto"` (or empty) routes to the configured default family — so auto
+// always works: with one provider it just uses that one, with several it gets
+// cross-provider redundancy for free.
+import { getDb, getSetting } from '../db/index.js';
+import { decrypt } from '../lib/crypto.js';
+
+export interface EmbeddingModelRow {
+ id: number;
+ family: string;
+ platform: string;
+ model_id: string;
+ display_name: string;
+ dimensions: number;
+ max_input_tokens: number | null;
+ priority: number;
+ enabled: number;
+ quota_label: string;
+}
+
+export interface EmbeddingsResult {
+ family: string;
+ platform: string;
+ modelId: string;
+ dimensions: number;
+ vectors: number[][];
+ inputTokens: number;
+}
+
+export class EmbeddingsError extends Error {
+ status: number;
+ constructor(message: string, status: number) {
+ super(message);
+ this.status = status;
+ }
+}
+
+export function listEmbeddingModels(): EmbeddingModelRow[] {
+ return getDb().prepare(
+ 'SELECT * FROM embedding_models ORDER BY family, priority',
+ ).all() as EmbeddingModelRow[];
+}
+
+export function getDefaultFamily(): string {
+ return getSetting('embeddings_default_family') ?? 'gemini-embedding-001';
+}
+
+/** Map the request's `model` to a family: 'auto'/empty → default; a family
+ * name → itself; a provider-specific model id → its family. */
+export function resolveFamily(model: string | undefined): string | null {
+ if (!model || model === 'auto') return getDefaultFamily();
+ const rows = listEmbeddingModels();
+ if (rows.some(r => r.family === model)) return model;
+ const byModelId = rows.find(r => r.model_id === model);
+ return byModelId?.family ?? null;
+}
+
+function getPlatformKey(platform: string): string | null {
+ const row = getDb().prepare(
+ "SELECT encrypted_key, iv, auth_tag FROM api_keys WHERE platform = ? AND enabled = 1 AND status IN ('healthy', 'unknown') ORDER BY id LIMIT 1",
+ ).get(platform) as { encrypted_key: string; iv: string; auth_tag: string } | undefined;
+ if (!row) return null;
+ try {
+ return decrypt(row.encrypted_key, row.iv, row.auth_tag);
+ } catch {
+ return null;
+ }
+}
+
+// Rough token estimate when the provider doesn't report usage (~4 chars/token).
+function estimateTokens(inputs: string[]): number {
+ return Math.ceil(inputs.reduce((n, s) => n + s.length, 0) / 4);
+}
+
+const FETCH_TIMEOUT_MS = 30_000;
+
+interface ProviderCallResult {
+ vectors: number[][];
+ inputTokens: number | null; // provider-reported, when available
+}
+
+async function openAiStyleEmbed(
+ url: string,
+ key: string,
+ modelId: string,
+ inputs: string[],
+ extra: Record<string, unknown> = {},
+): Promise<ProviderCallResult> {
+ const r = await fetch(url, {
+ method: 'POST',
+ headers: { 'Content-Type': 'application/json', Authorization: `Bearer ${key}` },
+ body: JSON.stringify({ model: modelId, input: inputs, ...extra }),
+ signal: AbortSignal.timeout(FETCH_TIMEOUT_MS),
+ });
+ if (!r.ok) {
+ throw new EmbeddingsError(`upstream ${r.status}: ${(await r.text()).slice(0, 200)}`, r.status);
+ }
+ const j = (await r.json()) as {
+ data?: { index?: number; embedding: number[] }[];
+ usage?: { prompt_tokens?: number; total_tokens?: number };
+ };
+ const data = [...(j.data ?? [])].sort((a, b) => (a.index ?? 0) - (b.index ?? 0));
+ return {
+ vectors: data.map(d => d.embedding),
+ inputTokens: j.usage?.prompt_tokens ?? j.usage?.total_tokens ?? null,
+ };
+}
+
+async function callProvider(row: EmbeddingModelRow, key: string, inputs: string[]): Promise<ProviderCallResult> {
+ switch (row.platform) {
+ case 'google':
+ return openAiStyleEmbed('https://generativelanguage.googleapis.com/v1beta/openai/embeddings', key, row.model_id, inputs);
+ case 'nvidia':
+ // NeMo Retriever NIMs require input_type; 'query' is the symmetric-safe
+ // choice for a gateway that can't know whether this is index or query time.
+ return openAiStyleEmbed('https://integrate.api.nvidia.com/v1/embeddings', key, row.model_id, inputs, { input_type: 'query' });
+ case 'openrouter':
+ return openAiStyleEmbed('https://openrouter.ai/api/v1/embeddings', key, row.model_id, inputs);
+ case 'github':
+ return openAiStyleEmbed('https://models.github.ai/inference/embeddings', key, row.model_id, inputs);
+ case 'cloudflare': {
+ // Key is stored as "account_id:token".
+ const sep = key.indexOf(':');
+ if (sep === -1) throw new EmbeddingsError('cloudflare key is not in account_id:token form', 500);
+ const accountId = key.slice(0, sep);
+ const token = key.slice(sep + 1);
+ return openAiStyleEmbed(
+ `https://api.cloudflare.com/client/v4/accounts/${accountId}/ai/v1/embeddings`,
+ token, row.model_id, inputs,
+ );
+ }
+ case 'huggingface': {
+ // HF serves embeddings as the feature-extraction task, not /v1/embeddings.
+ const r = await fetch(
+ `https://router.huggingface.co/hf-inference/models/${row.model_id}/pipeline/feature-extraction`,
+ {
+ method: 'POST',
+ headers: { 'Content-Type': 'application/json', Authorization: `Bearer ${key}` },
+ body: JSON.stringify({ inputs }),
+ signal: AbortSignal.timeout(FETCH_TIMEOUT_MS),
+ },
+ );
+ if (!r.ok) throw new EmbeddingsError(`upstream ${r.status}: ${(await r.text()).slice(0, 200)}`, r.status);
+ const j = (await r.json()) as number[][] | number[];
+ const vectors = Array.isArray(j[0]) ? (j as number[][]) : [j as number[]];
+ return { vectors, inputTokens: null };
+ }
+ case 'cohere': {
+ const r = await fetch('https://api.cohere.com/v2/embed', {
+ method: 'POST',
+ headers: { 'Content-Type': 'application/json', Authorization: `Bearer ${key}` },
+ body: JSON.stringify({
+ model: row.model_id,
+ texts: inputs,
+ input_type: 'search_document',
+ embedding_types: ['float'],
+ }),
+ signal: AbortSignal.timeout(FETCH_TIMEOUT_MS),
+ });
+ if (!r.ok) throw new EmbeddingsError(`upstream ${r.status}: ${(await r.text()).slice(0, 200)}`, r.status);
+ const j = (await r.json()) as { embeddings?: { float?: number[][] }; meta?: { billed_units?: { input_tokens?: number } } };
+ return { vectors: j.embeddings?.float ?? [], inputTokens: j.meta?.billed_units?.input_tokens ?? null };
+ }
+ default:
+ throw new EmbeddingsError(`no embeddings adapter for platform '${row.platform}'`, 500);
+ }
+}
+
+function logEmbeddingRequest(
+ row: EmbeddingModelRow,
+ status: 'success' | 'error',
+ inputTokens: number,
+ latencyMs: number,
+ error: string | null,
+): void {
+ try {
+ getDb().prepare(`
+ INSERT INTO requests (platform, model_id, key_id, status, input_tokens, output_tokens, latency_ms, error, request_type)
+ VALUES (?, ?, NULL, ?, ?, 0, ?, ?, 'embedding')
+ `).run(row.platform, row.model_id, status, inputTokens, latencyMs, error);
+ } catch (e) {
+ console.error('Failed to log embedding request:', e);
+ }
+}
+
+/** Embed `inputs` via the family's provider chain, failing over within the
+ * family on any provider error. Throws EmbeddingsError when the chain is dry. */
+export async function runEmbeddings(model: string | undefined, inputs: string[]): Promise<EmbeddingsResult> {
+ const family = resolveFamily(model);
+ if (!family) {
+ throw new EmbeddingsError(
+ `Unknown embedding model '${model}'. Use 'auto', a family name, or a provider model id.`, 400,
+ );
+ }
+
+ const chain = (getDb().prepare(
+ 'SELECT * FROM embedding_models WHERE family = ? AND enabled = 1 ORDER BY priority',
+ ).all(family) as EmbeddingModelRow[]);
+ if (chain.length === 0) {
+ throw new EmbeddingsError(`No enabled providers for embedding family '${family}'.`, 503);
+ }
+
+ let lastError: EmbeddingsError | null = null;
+ for (const row of chain) {
+ const key = getPlatformKey(row.platform);
+ if (!key) continue; // no usable key for this provider — try the next one
+ const started = Date.now();
+ try {
+ const out = await callProvider(row, key, inputs);
+ if (out.vectors.length !== inputs.length || out.vectors.some(v => !Array.isArray(v) || v.length === 0)) {
+ throw new EmbeddingsError('upstream returned malformed embeddings', 502);
+ }
+ const tokens = out.inputTokens ?? estimateTokens(inputs);
+ logEmbeddingRequest(row, 'success', tokens, Date.now() - started, null);
+ return {
+ family,
+ platform: row.platform,
+ modelId: row.model_id,
+ dimensions: out.vectors[0].length,
+ vectors: out.vectors,
+ inputTokens: tokens,
+ };
+ } catch (err: any) {
+ const e = err instanceof EmbeddingsError ? err : new EmbeddingsError(String(err?.message ?? err), 502);
+ logEmbeddingRequest(row, 'error', 0, Date.now() - started, e.message.slice(0, 300));
+ lastError = e;
+ // fall through to the next provider in the family
+ }
+ }
+
+ throw new EmbeddingsError(
+ `All providers for embedding family '${family}' failed${lastError ? ` (last: ${lastError.message.slice(0, 160)})` : ' (no usable keys)'}.`,
+ lastError && lastError.status === 429 ? 429 : 502,
+ );
+}
diff --git a/server/src/services/router.ts b/server/src/services/router.ts
index d1334476..5a96ceb3 100644
--- a/server/src/services/router.ts
+++ b/server/src/services/router.ts
@@ -224,6 +224,7 @@ export function refreshStatsCache(db: Database, force = false): void {
SELECT platform, model_id, COALESCE(SUM(input_tokens + output_tokens), 0) AS used
FROM requests
WHERE created_at >= datetime('now', 'start of month')
+ AND request_type = 'chat'
GROUP BY platform, model_id
`).all() as Array<{ platform: string; model_id: string; used: number }>;
const usageMap = new Map(usageRows.map(r => [`${r.platform}:${r.model_id}`, r.used]));