7.4 billion tokens per month. 34 free LLM providers. 635 free model endpoints. All behind one /v1 endpoint, plus any custom OpenAI-compatible endpoint. Smart routing, automatic failover, encrypted keys. Personal experimentation only.
TypeScript
// Embeddings routing. Unlike chat, embeddings can NOT fail over across models:
// vectors from different models live in incompatible spaces, and silently
// switching models would corrupt any vector store built on top of us. So the
// routing unit is a "family" (one model identity + dimension) and failover only
// walks the providers serving that same family.
//
// `model: "auto"` (or empty) routes to the configured default family — so auto
// always works: with one provider it just uses that one, with several it gets
// cross-provider redundancy for free.
import { getDb, getSetting } from '../db/index.js';
import { secondsUntilNextMonth } from './key-budget.js';
import { noteBudgetAlerts } from './budget-alerts.js';
import { parseRetryAfterMs } from '../providers/base.js';
import { RetryHintTracker, retryAfterSeconds } from '../lib/retry-hint.js';
import { getClientContext } from '../lib/client-context.js';
import { reserveProviderCredential } from './provider-credential.js';
import { proxyFetch } from '../lib/proxy.js';
import { bearerAuthHeader } from '../lib/credential.js';
import { markKeyHealthyFromRequest } from './health.js';
import { customEndpointKeyIds } from './custom-endpoint.js';
import type { Db } from '../db/types.js';
import { SPEKA_BASE_URL } from '../providers/speka.js';
import { ELECTRONHUB_BASE_URL } from '../providers/electronhub.js';
import { INFERBASE_BASE_URL } from '../providers/inferbase.js';
import { LLMTR_BASE_URL } from '../providers/llmtr.js';
export interface EmbeddingModelRow {
id: number;
family: string;
platform: string;
model_id: string;
display_name: string;
dimensions: number;
max_input_tokens: number | null;
priority: number;
enabled: number;
quota_label: string;
key_id: number | null;
}
export interface EmbeddingsResult {
family: string;
platform: string;
modelId: string;
dimensions: number;
vectors: number[][];
inputTokens: number;
}
export class EmbeddingsError extends Error {
status: number;
code?: string;
/** Back-off the upstream provider stated (`Retry-After` header or a
* retry-delay field in the error body), in milliseconds. Kept separate from
* the message so the route can answer the client with a real Retry-After
* instead of a bare 429, the same way the chat router benches keys. */
retryAfterMs?: number;
constructor(message: string, status: number, code?: string, retryAfterMs?: number) {
super(message);
this.status = status;
this.code = code;
this.retryAfterMs = retryAfterMs;
}
}
/** EmbeddingsError for a non-OK upstream response: status, truncated body and
* any stated back-off read off the `Retry-After` header before the body is
* consumed. */
async function upstreamEmbeddingsError(r: Response): Promise<EmbeddingsError> {
const retryAfterMs = parseRetryAfterMs(r.headers?.get('retry-after') ?? null);
return new EmbeddingsError(
`upstream ${r.status}: ${(await r.text()).slice(0, 200)}`,
r.status,
undefined,
retryAfterMs,
);
}
export function listEmbeddingModels(): EmbeddingModelRow[] {
return getDb().prepare(
'SELECT * FROM embedding_models ORDER BY family, priority',
).all() as EmbeddingModelRow[];
}
export function getDefaultFamily(): string {
return getSetting('embeddings_default_family') ?? 'gemini-embedding-001';
}
/** Map the request's `model` to a family: 'auto'/empty → default; a family
* name → itself; a provider-specific model id → its family. */
export function resolveFamily(model: string | undefined): string | null {
if (!model || model === 'auto') return getDefaultFamily();
const rows = listEmbeddingModels();
if (rows.some(r => r.family === model)) return model;
const byModelId = rows.find(r => r.model_id === model);
return byModelId?.family ?? null;
}
interface ProviderCredential {
id: number;
key: string;
baseUrl: string | null;
}
// Rough token estimate when the provider doesn't report usage (~4 chars/token).
function estimateTokens(inputs: string[]): number {
return Math.ceil(inputs.reduce((n, s) => n + s.length, 0) / 4);
}
const FETCH_TIMEOUT_MS = 30_000;
/** Provider adapters that can safely receive catalog-managed embedding rows. */
export const EMBEDDING_PLATFORMS = new Set([
'google',
'nvidia',
'openrouter',
'cloudflare',
'huggingface',
'cohere',
'sealion',
'speka',
'electronhub',
'inferbase',
'llmtr',
]);
interface ProviderCallResult {
vectors: number[][];
inputTokens: number | null; // provider-reported, when available
}
async function openAiStyleEmbed(
url: string,
platform: string,
key: string,
modelId: string,
inputs: string[],
extra: Record<string, unknown> = {},
dimensions?: number,
expectedDimensions?: number,
timeoutMs = FETCH_TIMEOUT_MS,
): Promise<ProviderCallResult> {
const body: Record<string, unknown> = { model: modelId, input: inputs, ...extra };
// Some providers (NVIDIA NeMo NIM, Google Gemini Embedding, OpenAI v3) support
// Matryoshka Representation Learning (MRL) — a smaller output_dim is valid and
// truncates the vector rather than failing. Others (HuggingFace feature-extraction,
// Cloudflare BGE) ignore unknown fields silently. We only forward the param when
// the caller asked for an explicit override, so providers that don't accept it
// see a request body identical to today.
if (dimensions !== undefined) body.dimensions = dimensions;
const r = await proxyFetch(url, {
method: 'POST',
// A custom endpoint with auth off holds the `no-key` sentinel: omit the
// header instead of sending `Bearer no-key` (#1331).
headers: { 'Content-Type': 'application/json', ...bearerAuthHeader(key) },
body: JSON.stringify(body),
signal: AbortSignal.timeout(timeoutMs),
}, platform, 'embedding', timeoutMs);
if (!r.ok) {
throw await upstreamEmbeddingsError(r);
}
const j = (await r.json()) as {
model?: string;
data?: { index?: number; embedding: number[] }[];
usage?: { prompt_tokens?: number; total_tokens?: number };
};
const data = [...(j.data ?? [])].sort((a, b) => (a.index ?? 0) - (b.index ?? 0));
if (['electronhub', 'inferbase', 'llmtr'].includes(platform) && (
j.model !== modelId || data.length !== inputs.length ||
data.some((d, i) => d.index !== i || !Array.isArray(d.embedding) ||
d.embedding.length !== (dimensions ?? expectedDimensions) ||
d.embedding.some(value => typeof value !== 'number' || !Number.isFinite(value)))
)) {
// A successful HTTP response is not enough: wrong model/dimension or
// malformed vectors would corrupt the caller's embedding family.
const provider = platform === 'electronhub' ? 'ElectronHub' : platform === 'llmtr' ? 'LLMTR' : 'Inferbase';
throw new EmbeddingsError(`${provider} returned mismatched or malformed embeddings`, 502);
}
return {
vectors: data.map(d => d.embedding),
inputTokens: j.usage?.prompt_tokens ?? j.usage?.total_tokens ?? null,
};
}
export async function probeEmbeddingDimensions(baseUrl: string, key: string, modelId: string): Promise<number> {
const out = await openAiStyleEmbed(`${baseUrl.trim().replace(/\/+$/, '')}/embeddings`, 'custom', key, modelId, ['dimension probe']);
const vector = out.vectors[0];
if (!Array.isArray(vector) || vector.length === 0) {
throw new EmbeddingsError('upstream returned malformed embeddings', 502);
}
return vector.length;
}
export interface CustomEmbeddingRegistration {
keyId: number;
modelId: string;
displayName: string | null;
family: string;
dimensions: number;
maxInputTokens: number | null;
quotaLabel: string;
}
/**
* Upsert one custom embedding model bound to an endpoint credential — the
* shared write path behind POST /api/embeddings/custom and the bulk key
* importer (#382). Throws EmbeddingsError(400) when the family already exists
* at a different dimension: vectors from mismatched spaces must never mix, so
* the caller has to pick a new family name instead.
*/
export function registerCustomEmbeddingModel(db: Db, reg: CustomEmbeddingRegistration): { modelDbId: number; created: boolean } {
const sibling = db.prepare(`
SELECT dimensions
FROM embedding_models
WHERE family = ?
AND NOT (platform = 'custom' AND model_id = ?)
LIMIT 1
`).get(reg.family, reg.modelId) as { dimensions: number } | undefined;
if (sibling && sibling.dimensions !== reg.dimensions) {
throw new EmbeddingsError(
`Embedding family '${reg.family}' is ${sibling.dimensions} dimensions, but '${reg.modelId}' returned ${reg.dimensions}. Use a new family name.`,
400,
);
}
const endpointKeyIds = customEndpointKeyIds(db, reg.keyId);
const existingModel = db.prepare(`
SELECT id, priority, key_id
FROM embedding_models
WHERE platform = 'custom' AND model_id = ?
LIMIT 1
`).get(reg.modelId) as { id: number; priority: number; key_id: number | null } | undefined;
// A model already on this endpoint keeps the key it has; only a move to a
// different endpoint re-binds it.
const bindKeyId = existingModel?.key_id != null && endpointKeyIds.has(existingModel.key_id)
? existingModel.key_id
: reg.keyId;
const priority = existingModel?.priority ?? (
(db.prepare('SELECT COALESCE(MAX(priority), 0) AS maxPriority FROM embedding_models WHERE family = ?')
.get(reg.family) as { maxPriority: number }).maxPriority + 1
);
// `display_name` is optional: a new model takes its id, and a model already
// on record keeps the name it has instead of being reset by a submit that
// simply left the field blank (#704).
if (existingModel) {
db.prepare(`
UPDATE embedding_models
SET family = ?,
display_name = COALESCE(?, display_name),
dimensions = ?,
max_input_tokens = ?,
priority = ?,
enabled = 1,
quota_label = ?,
key_id = ?
WHERE id = ?
`).run(reg.family, reg.displayName, reg.dimensions, reg.maxInputTokens, priority, reg.quotaLabel, bindKeyId, existingModel.id);
return { modelDbId: existingModel.id, created: false };
}
const model = db.prepare(`
INSERT INTO embedding_models
(family, platform, model_id, display_name, dimensions, max_input_tokens, priority, enabled, quota_label, key_id)
VALUES (?, 'custom', ?, ?, ?, ?, ?, 1, ?, ?)
`).run(reg.family, reg.modelId, reg.displayName ?? reg.modelId, reg.dimensions, reg.maxInputTokens, priority, reg.quotaLabel, bindKeyId);
return { modelDbId: Number(model.lastInsertRowid), created: true };
}
async function callProvider(row: EmbeddingModelRow, credential: ProviderCredential, inputs: string[], dimensions?: number): Promise<ProviderCallResult> {
const { key } = credential;
switch (row.platform) {
case 'custom':
if (!credential.baseUrl) throw new EmbeddingsError('custom embedding provider is missing base_url', 500);
return openAiStyleEmbed(`${credential.baseUrl}/embeddings`, row.platform, key, row.model_id, inputs, {}, dimensions);
case 'google':
return openAiStyleEmbed('https://generativelanguage.googleapis.com/v1beta/openai/embeddings', row.platform, key, row.model_id, inputs, {}, dimensions);
case 'nvidia':
// NeMo Retriever NIMs require input_type; 'query' is the symmetric-safe
// choice for a gateway that can't know whether this is index or query time.
// MRL models (e.g. llama-nemotron-embed-1b-v2) accept dimensions and truncate.
return openAiStyleEmbed('https://integrate.api.nvidia.com/v1/embeddings', row.platform, key, row.model_id, inputs, { input_type: 'query' }, dimensions);
case 'openrouter':
return openAiStyleEmbed('https://openrouter.ai/api/v1/embeddings', row.platform, key, row.model_id, inputs, {}, dimensions);
case 'sealion':
return openAiStyleEmbed('https://api.sea-lion.ai/v1/embeddings', row.platform, key, row.model_id, inputs, {}, dimensions);
case 'speka':
return openAiStyleEmbed(`${SPEKA_BASE_URL}/embeddings`, row.platform, key, row.model_id, inputs, { encoding_format: 'float' }, dimensions);
case 'electronhub':
return openAiStyleEmbed(`${ELECTRONHUB_BASE_URL}/embeddings`, row.platform, key, row.model_id, inputs, { encoding_format: 'float' }, dimensions, row.dimensions);
case 'inferbase':
return openAiStyleEmbed(`${INFERBASE_BASE_URL}/embeddings`, row.platform, key, row.model_id, inputs, { encoding_format: 'float' }, dimensions, row.dimensions);
case 'llmtr':
// The catalog-approved models return fixed-width vectors. Do not send a
// dimension override or accept a substituted model/embedding space.
if (dimensions !== undefined && dimensions !== row.dimensions) {
throw new EmbeddingsError('LLMTR does not support an embedding dimension override', 400);
}
return openAiStyleEmbed(`${LLMTR_BASE_URL}/embeddings`, row.platform, key, row.model_id, inputs, {}, undefined, row.dimensions);
case 'cloudflare': {
// Key is stored as "account_id:token".
const sep = key.indexOf(':');
if (sep === -1) throw new EmbeddingsError('cloudflare key is not in account_id:token form', 500);
const accountId = key.slice(0, sep);
const token = key.slice(sep + 1);
return openAiStyleEmbed(
`https://api.cloudflare.com/client/v4/accounts/${accountId}/ai/v1/embeddings`,
row.platform, token, row.model_id, inputs, {},
);
}
case 'huggingface': {
// HF serves embeddings as the feature-extraction task, not /v1/embeddings.
const r = await proxyFetch(
`https://router.huggingface.co/hf-inference/models/${row.model_id}/pipeline/feature-extraction`,
{
method: 'POST',
headers: { 'Content-Type': 'application/json', Authorization: `Bearer ${key}` },
body: JSON.stringify({ inputs }),
signal: AbortSignal.timeout(FETCH_TIMEOUT_MS),
},
row.platform, 'embedding', FETCH_TIMEOUT_MS,
);
if (!r.ok) throw await upstreamEmbeddingsError(r);
const j = await r.json() as number[][] | number[];
const vectors = Array.isArray(j[0]) ? (j as number[][]) : [j as number[]];
return { vectors, inputTokens: null };
}
case 'cohere': {
const r = await proxyFetch('https://api.cohere.com/v2/embed', {
method: 'POST',
headers: { 'Content-Type': 'application/json', Authorization: `Bearer ${key}` },
body: JSON.stringify({
model: row.model_id,
texts: inputs,
input_type: 'search_document',
embedding_types: ['float'],
}),
signal: AbortSignal.timeout(FETCH_TIMEOUT_MS),
}, row.platform, 'embedding', FETCH_TIMEOUT_MS);
if (!r.ok) throw await upstreamEmbeddingsError(r);
const j = (await r.json()) as { embeddings?: { float?: number[][] }; meta?: { billed_units?: { input_tokens?: number } } };
return { vectors: j.embeddings?.float ?? [], inputTokens: j.meta?.billed_units?.input_tokens ?? null };
}
default:
throw new EmbeddingsError(`no embeddings adapter for platform '${row.platform}'`, 500);
}
}
function logEmbeddingRequest(
row: EmbeddingModelRow,
keyId: number | null,
status: 'success' | 'error',
inputTokens: number,
latencyMs: number,
error: string | null,
): void {
try {
const client = getClientContext();
getDb().prepare(`
INSERT INTO requests (platform, model_id, key_id, status, input_tokens, output_tokens, latency_ms, error, request_type, client_ip, client_user_agent, client_agent)
VALUES (?, ?, ?, ?, ?, 0, ?, ?, 'embedding', ?, ?, ?)
`).run(row.platform, row.model_id, keyId, status, inputTokens, latencyMs, error, client.ip, client.userAgent, client.agent);
// Soft budget alert (#1158 follow-on): the ledger trigger fired with this
// row, so compare the caps once. No-op for uncapped keys.
if (status === 'success' && keyId != null) noteBudgetAlerts(keyId);
} catch (e) {
console.error('Failed to log embedding request:', e);
}
}
/** Embed `inputs` via the family's provider chain, failing over within the
* family on any provider error. Throws EmbeddingsError when the chain is dry.
*
* `dimensions` (optional): client-supplied output-dimension override forwarded to
* providers that support MRL truncation (NVIDIA NeMo NIM, Google Gemini Embedding,
* OpenAI text-embedding-3-*). Providers that ignore the field see an identical
* request body. The override is independent of the model's native dimension — the
* family registry still pins the canonical dimension, this just lets callers ask
* for a smaller vector at the cost of some accuracy. */
export async function runEmbeddings(model: string | undefined, inputs: string[], dimensions?: number): Promise<EmbeddingsResult> {
const family = resolveFamily(model);
if (!family) {
throw new EmbeddingsError(
`Unknown embedding model '${model}'. Use 'auto', a family name, or a provider model id.`, 400,
);
}
const chain = (getDb().prepare(
'SELECT * FROM embedding_models WHERE family = ? AND enabled = 1 ORDER BY priority',
).all(family) as EmbeddingModelRow[]);
if (chain.length === 0) {
throw new EmbeddingsError(`No enabled providers for embedding family '${family}'.`, 503);
}
let lastError: EmbeddingsError | null = null;
// Upstream back-offs across the whole chain: only a chain that was rate
// limited end to end earns a Retry-After, and then the soonest one.
const hints = new RetryHintTracker();
for (const row of chain) {
const { credential, budgetBlocked } = reserveProviderCredential(row, estimateTokens(inputs));
if (!credential) {
if (budgetBlocked) {
lastError = new EmbeddingsError('Monthly key budget exhausted', 429, 'quota_exceeded');
hints.record(429, secondsUntilNextMonth() * 1000);
}
continue;
}
const started = Date.now();
try {
const out = await callProvider(row, credential, inputs, dimensions);
if (out.vectors.length !== inputs.length || out.vectors.some(v => !Array.isArray(v) || v.length === 0)) {
throw new EmbeddingsError('upstream returned malformed embeddings', 502);
}
const tokens = out.inputTokens ?? estimateTokens(inputs);
logEmbeddingRequest(row, credential.id, 'success', tokens, Date.now() - started, null);
// A served embedding is evidence the key works, the same as a served
// chat request (#1493) or a served search (#1612): stamp last_healthy_at
// and lift an 'unknown'/'error' key. Embedding-only keys (and #1459-era
// providers that refuse probes) had no success signal at all otherwise.
markKeyHealthyFromRequest(credential.id);
return {
family,
platform: row.platform,
modelId: row.model_id,
dimensions: out.vectors[0].length,
vectors: out.vectors,
inputTokens: tokens,
};
} catch (err: any) {
const e = err instanceof EmbeddingsError ? err : new EmbeddingsError(String(err?.message ?? err), 502);
logEmbeddingRequest(row, credential.id, 'error', 0, Date.now() - started, e.message.slice(0, 300));
lastError = e;
hints.record(e.status, e.retryAfterMs);
// fall through to the next provider in the family
} finally {
credential.release();
}
}
throw new EmbeddingsError(
`All providers for embedding family '${family}' failed${lastError ? ` (last: ${lastError.message.slice(0, 160)})` : ' (no usable keys)'}.`,
lastError && lastError.status === 429 ? 429 : 502,
lastError?.code,
lastError && lastError.status === 429 ? hints.retryAfterMs() : undefined,
);
}
/** Whole seconds the client should wait before retrying an embeddings request:
* the soonest upstream back-off when the whole chain was rate limited, else the
* monthly-budget reset for a local quota block. Undefined means the caller
* shouldn't set the header. */
export function embeddingsRetryAfterSec(err: EmbeddingsError): number | undefined {
if (err.retryAfterMs !== undefined) return retryAfterSeconds(err.retryAfterMs);
if (err.code === 'quota_exceeded') return secondsUntilNextMonth();
return undefined;
}