Skip to content

7.4 billion tokens per month. 34 free LLM providers. 635 free model endpoints. All behind one /v1 endpoint, plus any custom OpenAI-compatible endpoint. Smart routing, automatic failover, encrypted keys. Personal experimentation only.

TypeScript
19.8 KB
Raw Blame
// Embeddings routing. Unlike chat, embeddings can NOT fail over across models:
// vectors from different models live in incompatible spaces, and silently
// switching models would corrupt any vector store built on top of us. So the
// routing unit is a "family" (one model identity + dimension) and failover only
// walks the providers serving that same family.
//
// `model: "auto"` (or empty) routes to the configured default family — so auto
// always works: with one provider it just uses that one, with several it gets
// cross-provider redundancy for free.
import { getDb, getSetting } from '../db/index.js';
import { secondsUntilNextMonth } from './key-budget.js';
import { noteBudgetAlerts } from './budget-alerts.js';
import { parseRetryAfterMs } from '../providers/base.js';
import { RetryHintTracker, retryAfterSeconds } from '../lib/retry-hint.js';
import { getClientContext } from '../lib/client-context.js';
import { reserveProviderCredential } from './provider-credential.js';
import { proxyFetch } from '../lib/proxy.js';
import { bearerAuthHeader } from '../lib/credential.js';
import { markKeyHealthyFromRequest } from './health.js';
import { customEndpointKeyIds } from './custom-endpoint.js';
import type { Db } from '../db/types.js';
import { SPEKA_BASE_URL } from '../providers/speka.js';
import { ELECTRONHUB_BASE_URL } from '../providers/electronhub.js';
import { INFERBASE_BASE_URL } from '../providers/inferbase.js';
import { LLMTR_BASE_URL } from '../providers/llmtr.js';

export interface EmbeddingModelRow {
  id: number;
  family: string;
  platform: string;
  model_id: string;
  display_name: string;
  dimensions: number;
  max_input_tokens: number | null;
  priority: number;
  enabled: number;
  quota_label: string;
  key_id: number | null;
}

export interface EmbeddingsResult {
  family: string;
  platform: string;
  modelId: string;
  dimensions: number;
  vectors: number[][];
  inputTokens: number;
}

export class EmbeddingsError extends Error {
  status: number;
  code?: string;
  /** Back-off the upstream provider stated (`Retry-After` header or a
   *  retry-delay field in the error body), in milliseconds. Kept separate from
   *  the message so the route can answer the client with a real Retry-After
   *  instead of a bare 429, the same way the chat router benches keys. */
  retryAfterMs?: number;
  constructor(message: string, status: number, code?: string, retryAfterMs?: number) {
    super(message);
    this.status = status;
    this.code = code;
    this.retryAfterMs = retryAfterMs;
  }
}

/** EmbeddingsError for a non-OK upstream response: status, truncated body and
 *  any stated back-off read off the `Retry-After` header before the body is
 *  consumed. */
async function upstreamEmbeddingsError(r: Response): Promise<EmbeddingsError> {
  const retryAfterMs = parseRetryAfterMs(r.headers?.get('retry-after') ?? null);
  return new EmbeddingsError(
    `upstream ${r.status}: ${(await r.text()).slice(0, 200)}`,
    r.status,
    undefined,
    retryAfterMs,
  );
}

export function listEmbeddingModels(): EmbeddingModelRow[] {
  return getDb().prepare(
    'SELECT * FROM embedding_models ORDER BY family, priority',
  ).all() as EmbeddingModelRow[];
}

export function getDefaultFamily(): string {
  return getSetting('embeddings_default_family') ?? 'gemini-embedding-001';
}

/** Map the request's `model` to a family: 'auto'/empty → default; a family
 * name → itself; a provider-specific model id → its family. */
export function resolveFamily(model: string | undefined): string | null {
  if (!model || model === 'auto') return getDefaultFamily();
  const rows = listEmbeddingModels();
  if (rows.some(r => r.family === model)) return model;
  const byModelId = rows.find(r => r.model_id === model);
  return byModelId?.family ?? null;
}

interface ProviderCredential {
  id: number;
  key: string;
  baseUrl: string | null;
}

// Rough token estimate when the provider doesn't report usage (~4 chars/token).
function estimateTokens(inputs: string[]): number {
  return Math.ceil(inputs.reduce((n, s) => n + s.length, 0) / 4);
}

const FETCH_TIMEOUT_MS = 30_000;

/** Provider adapters that can safely receive catalog-managed embedding rows. */
export const EMBEDDING_PLATFORMS = new Set([
  'google',
  'nvidia',
  'openrouter',
  'cloudflare',
  'huggingface',
  'cohere',
  'sealion',
  'speka',
  'electronhub',
  'inferbase',
  'llmtr',
]);

interface ProviderCallResult {
  vectors: number[][];
  inputTokens: number | null; // provider-reported, when available
}

async function openAiStyleEmbed(
  url: string,
  platform: string,
  key: string,
  modelId: string,
  inputs: string[],
  extra: Record<string, unknown> = {},
  dimensions?: number,
  expectedDimensions?: number,
  timeoutMs = FETCH_TIMEOUT_MS,
): Promise<ProviderCallResult> {
  const body: Record<string, unknown> = { model: modelId, input: inputs, ...extra };
  // Some providers (NVIDIA NeMo NIM, Google Gemini Embedding, OpenAI v3) support
  // Matryoshka Representation Learning (MRL) — a smaller output_dim is valid and
  // truncates the vector rather than failing. Others (HuggingFace feature-extraction,
  // Cloudflare BGE) ignore unknown fields silently. We only forward the param when
  // the caller asked for an explicit override, so providers that don't accept it
  // see a request body identical to today.
  if (dimensions !== undefined) body.dimensions = dimensions;
  const r = await proxyFetch(url, {
    method: 'POST',
    // A custom endpoint with auth off holds the `no-key` sentinel: omit the
    // header instead of sending `Bearer no-key` (#1331).
    headers: { 'Content-Type': 'application/json', ...bearerAuthHeader(key) },
    body: JSON.stringify(body),
    signal: AbortSignal.timeout(timeoutMs),
  }, platform, 'embedding', timeoutMs);
  if (!r.ok) {
    throw await upstreamEmbeddingsError(r);
  }
  const j = (await r.json()) as {
    model?: string;
    data?: { index?: number; embedding: number[] }[];
    usage?: { prompt_tokens?: number; total_tokens?: number };
  };
  const data = [...(j.data ?? [])].sort((a, b) => (a.index ?? 0) - (b.index ?? 0));
  if (['electronhub', 'inferbase', 'llmtr'].includes(platform) && (
    j.model !== modelId || data.length !== inputs.length ||
    data.some((d, i) => d.index !== i || !Array.isArray(d.embedding) ||
      d.embedding.length !== (dimensions ?? expectedDimensions) ||
      d.embedding.some(value => typeof value !== 'number' || !Number.isFinite(value)))
  )) {
    // A successful HTTP response is not enough: wrong model/dimension or
    // malformed vectors would corrupt the caller's embedding family.
    const provider = platform === 'electronhub' ? 'ElectronHub' : platform === 'llmtr' ? 'LLMTR' : 'Inferbase';
    throw new EmbeddingsError(`${provider} returned mismatched or malformed embeddings`, 502);
  }
  return {
    vectors: data.map(d => d.embedding),
    inputTokens: j.usage?.prompt_tokens ?? j.usage?.total_tokens ?? null,
  };
}

export async function probeEmbeddingDimensions(baseUrl: string, key: string, modelId: string): Promise<number> {
  const out = await openAiStyleEmbed(`${baseUrl.trim().replace(/\/+$/, '')}/embeddings`, 'custom', key, modelId, ['dimension probe']);
  const vector = out.vectors[0];
  if (!Array.isArray(vector) || vector.length === 0) {
    throw new EmbeddingsError('upstream returned malformed embeddings', 502);
  }
  return vector.length;
}

export interface CustomEmbeddingRegistration {
  keyId: number;
  modelId: string;
  displayName: string | null;
  family: string;
  dimensions: number;
  maxInputTokens: number | null;
  quotaLabel: string;
}

/**
 * Upsert one custom embedding model bound to an endpoint credential — the
 * shared write path behind POST /api/embeddings/custom and the bulk key
 * importer (#382). Throws EmbeddingsError(400) when the family already exists
 * at a different dimension: vectors from mismatched spaces must never mix, so
 * the caller has to pick a new family name instead.
 */
export function registerCustomEmbeddingModel(db: Db, reg: CustomEmbeddingRegistration): { modelDbId: number; created: boolean } {
  const sibling = db.prepare(`
    SELECT dimensions
      FROM embedding_models
     WHERE family = ?
       AND NOT (platform = 'custom' AND model_id = ?)
     LIMIT 1
  `).get(reg.family, reg.modelId) as { dimensions: number } | undefined;
  if (sibling && sibling.dimensions !== reg.dimensions) {
    throw new EmbeddingsError(
      `Embedding family '${reg.family}' is ${sibling.dimensions} dimensions, but '${reg.modelId}' returned ${reg.dimensions}. Use a new family name.`,
      400,
    );
  }

  const endpointKeyIds = customEndpointKeyIds(db, reg.keyId);
  const existingModel = db.prepare(`
    SELECT id, priority, key_id
      FROM embedding_models
     WHERE platform = 'custom' AND model_id = ?
     LIMIT 1
  `).get(reg.modelId) as { id: number; priority: number; key_id: number | null } | undefined;
  // A model already on this endpoint keeps the key it has; only a move to a
  // different endpoint re-binds it.
  const bindKeyId = existingModel?.key_id != null && endpointKeyIds.has(existingModel.key_id)
    ? existingModel.key_id
    : reg.keyId;
  const priority = existingModel?.priority ?? (
    (db.prepare('SELECT COALESCE(MAX(priority), 0) AS maxPriority FROM embedding_models WHERE family = ?')
      .get(reg.family) as { maxPriority: number }).maxPriority + 1
  );

  // `display_name` is optional: a new model takes its id, and a model already
  // on record keeps the name it has instead of being reset by a submit that
  // simply left the field blank (#704).
  if (existingModel) {
    db.prepare(`
      UPDATE embedding_models
         SET family = ?,
             display_name = COALESCE(?, display_name),
             dimensions = ?,
             max_input_tokens = ?,
             priority = ?,
             enabled = 1,
             quota_label = ?,
             key_id = ?
       WHERE id = ?
    `).run(reg.family, reg.displayName, reg.dimensions, reg.maxInputTokens, priority, reg.quotaLabel, bindKeyId, existingModel.id);
    return { modelDbId: existingModel.id, created: false };
  }

  const model = db.prepare(`
    INSERT INTO embedding_models
      (family, platform, model_id, display_name, dimensions, max_input_tokens, priority, enabled, quota_label, key_id)
    VALUES (?, 'custom', ?, ?, ?, ?, ?, 1, ?, ?)
  `).run(reg.family, reg.modelId, reg.displayName ?? reg.modelId, reg.dimensions, reg.maxInputTokens, priority, reg.quotaLabel, bindKeyId);
  return { modelDbId: Number(model.lastInsertRowid), created: true };
}

async function callProvider(row: EmbeddingModelRow, credential: ProviderCredential, inputs: string[], dimensions?: number): Promise<ProviderCallResult> {
  const { key } = credential;
  switch (row.platform) {
    case 'custom':
      if (!credential.baseUrl) throw new EmbeddingsError('custom embedding provider is missing base_url', 500);
      return openAiStyleEmbed(`${credential.baseUrl}/embeddings`, row.platform, key, row.model_id, inputs, {}, dimensions);
    case 'google':
      return openAiStyleEmbed('https://generativelanguage.googleapis.com/v1beta/openai/embeddings', row.platform, key, row.model_id, inputs, {}, dimensions);
    case 'nvidia':
      // NeMo Retriever NIMs require input_type; 'query' is the symmetric-safe
      // choice for a gateway that can't know whether this is index or query time.
      // MRL models (e.g. llama-nemotron-embed-1b-v2) accept dimensions and truncate.
      return openAiStyleEmbed('https://integrate.api.nvidia.com/v1/embeddings', row.platform, key, row.model_id, inputs, { input_type: 'query' }, dimensions);
    case 'openrouter':
      return openAiStyleEmbed('https://openrouter.ai/api/v1/embeddings', row.platform, key, row.model_id, inputs, {}, dimensions);
    case 'sealion':
      return openAiStyleEmbed('https://api.sea-lion.ai/v1/embeddings', row.platform, key, row.model_id, inputs, {}, dimensions);
    case 'speka':
      return openAiStyleEmbed(`${SPEKA_BASE_URL}/embeddings`, row.platform, key, row.model_id, inputs, { encoding_format: 'float' }, dimensions);
    case 'electronhub':
      return openAiStyleEmbed(`${ELECTRONHUB_BASE_URL}/embeddings`, row.platform, key, row.model_id, inputs, { encoding_format: 'float' }, dimensions, row.dimensions);
    case 'inferbase':
      return openAiStyleEmbed(`${INFERBASE_BASE_URL}/embeddings`, row.platform, key, row.model_id, inputs, { encoding_format: 'float' }, dimensions, row.dimensions);
    case 'llmtr':
      // The catalog-approved models return fixed-width vectors. Do not send a
      // dimension override or accept a substituted model/embedding space.
      if (dimensions !== undefined && dimensions !== row.dimensions) {
        throw new EmbeddingsError('LLMTR does not support an embedding dimension override', 400);
      }
      return openAiStyleEmbed(`${LLMTR_BASE_URL}/embeddings`, row.platform, key, row.model_id, inputs, {}, undefined, row.dimensions);
    case 'cloudflare': {
      // Key is stored as "account_id:token".
      const sep = key.indexOf(':');
      if (sep === -1) throw new EmbeddingsError('cloudflare key is not in account_id:token form', 500);
      const accountId = key.slice(0, sep);
      const token = key.slice(sep + 1);
      return openAiStyleEmbed(
        `https://api.cloudflare.com/client/v4/accounts/${accountId}/ai/v1/embeddings`,
        row.platform, token, row.model_id, inputs, {},
      );
    }
    case 'huggingface': {
      // HF serves embeddings as the feature-extraction task, not /v1/embeddings.
      const r = await proxyFetch(
        `https://router.huggingface.co/hf-inference/models/${row.model_id}/pipeline/feature-extraction`,
        {
          method: 'POST',
          headers: { 'Content-Type': 'application/json', Authorization: `Bearer ${key}` },
          body: JSON.stringify({ inputs }),
          signal: AbortSignal.timeout(FETCH_TIMEOUT_MS),
        },
        row.platform, 'embedding', FETCH_TIMEOUT_MS,
      );
      if (!r.ok) throw await upstreamEmbeddingsError(r);
      const j = await r.json() as number[][] | number[];
      const vectors = Array.isArray(j[0]) ? (j as number[][]) : [j as number[]];
      return { vectors, inputTokens: null };
    }
    case 'cohere': {
      const r = await proxyFetch('https://api.cohere.com/v2/embed', {
        method: 'POST',
        headers: { 'Content-Type': 'application/json', Authorization: `Bearer ${key}` },
        body: JSON.stringify({
          model: row.model_id,
          texts: inputs,
          input_type: 'search_document',
          embedding_types: ['float'],
        }),
        signal: AbortSignal.timeout(FETCH_TIMEOUT_MS),
      }, row.platform, 'embedding', FETCH_TIMEOUT_MS);
      if (!r.ok) throw await upstreamEmbeddingsError(r);
      const j = (await r.json()) as { embeddings?: { float?: number[][] }; meta?: { billed_units?: { input_tokens?: number } } };
      return { vectors: j.embeddings?.float ?? [], inputTokens: j.meta?.billed_units?.input_tokens ?? null };
    }
    default:
      throw new EmbeddingsError(`no embeddings adapter for platform '${row.platform}'`, 500);
  }
}

function logEmbeddingRequest(
  row: EmbeddingModelRow,
  keyId: number | null,
  status: 'success' | 'error',
  inputTokens: number,
  latencyMs: number,
  error: string | null,
): void {
  try {
    const client = getClientContext();
    getDb().prepare(`
      INSERT INTO requests (platform, model_id, key_id, status, input_tokens, output_tokens, latency_ms, error, request_type, client_ip, client_user_agent, client_agent)
      VALUES (?, ?, ?, ?, ?, 0, ?, ?, 'embedding', ?, ?, ?)
    `).run(row.platform, row.model_id, keyId, status, inputTokens, latencyMs, error, client.ip, client.userAgent, client.agent);
    // Soft budget alert (#1158 follow-on): the ledger trigger fired with this
    // row, so compare the caps once. No-op for uncapped keys.
    if (status === 'success' && keyId != null) noteBudgetAlerts(keyId);
  } catch (e) {
    console.error('Failed to log embedding request:', e);
  }
}

/** Embed `inputs` via the family's provider chain, failing over within the
 * family on any provider error. Throws EmbeddingsError when the chain is dry.
 *
 * `dimensions` (optional): client-supplied output-dimension override forwarded to
 * providers that support MRL truncation (NVIDIA NeMo NIM, Google Gemini Embedding,
 * OpenAI text-embedding-3-*). Providers that ignore the field see an identical
 * request body. The override is independent of the model's native dimension — the
 * family registry still pins the canonical dimension, this just lets callers ask
 * for a smaller vector at the cost of some accuracy. */
export async function runEmbeddings(model: string | undefined, inputs: string[], dimensions?: number): Promise<EmbeddingsResult> {
  const family = resolveFamily(model);
  if (!family) {
    throw new EmbeddingsError(
      `Unknown embedding model '${model}'. Use 'auto', a family name, or a provider model id.`, 400,
    );
  }

  const chain = (getDb().prepare(
    'SELECT * FROM embedding_models WHERE family = ? AND enabled = 1 ORDER BY priority',
  ).all(family) as EmbeddingModelRow[]);
  if (chain.length === 0) {
    throw new EmbeddingsError(`No enabled providers for embedding family '${family}'.`, 503);
  }

  let lastError: EmbeddingsError | null = null;
  // Upstream back-offs across the whole chain: only a chain that was rate
  // limited end to end earns a Retry-After, and then the soonest one.
  const hints = new RetryHintTracker();
  for (const row of chain) {
    const { credential, budgetBlocked } = reserveProviderCredential(row, estimateTokens(inputs));
    if (!credential) {
      if (budgetBlocked) {
        lastError = new EmbeddingsError('Monthly key budget exhausted', 429, 'quota_exceeded');
        hints.record(429, secondsUntilNextMonth() * 1000);
      }
      continue;
    }
    const started = Date.now();
    try {
      const out = await callProvider(row, credential, inputs, dimensions);
      if (out.vectors.length !== inputs.length || out.vectors.some(v => !Array.isArray(v) || v.length === 0)) {
        throw new EmbeddingsError('upstream returned malformed embeddings', 502);
      }
      const tokens = out.inputTokens ?? estimateTokens(inputs);
      logEmbeddingRequest(row, credential.id, 'success', tokens, Date.now() - started, null);
      // A served embedding is evidence the key works, the same as a served
      // chat request (#1493) or a served search (#1612): stamp last_healthy_at
      // and lift an 'unknown'/'error' key. Embedding-only keys (and #1459-era
      // providers that refuse probes) had no success signal at all otherwise.
      markKeyHealthyFromRequest(credential.id);
      return {
        family,
        platform: row.platform,
        modelId: row.model_id,
        dimensions: out.vectors[0].length,
        vectors: out.vectors,
        inputTokens: tokens,
      };
    } catch (err: any) {
      const e = err instanceof EmbeddingsError ? err : new EmbeddingsError(String(err?.message ?? err), 502);
      logEmbeddingRequest(row, credential.id, 'error', 0, Date.now() - started, e.message.slice(0, 300));
      lastError = e;
      hints.record(e.status, e.retryAfterMs);
      // fall through to the next provider in the family
    } finally {
      credential.release();
    }
  }

  throw new EmbeddingsError(
    `All providers for embedding family '${family}' failed${lastError ? ` (last: ${lastError.message.slice(0, 160)})` : ' (no usable keys)'}.`,
    lastError && lastError.status === 429 ? 429 : 502,
    lastError?.code,
    lastError && lastError.status === 429 ? hints.retryAfterMs() : undefined,
  );
}

/** Whole seconds the client should wait before retrying an embeddings request:
 *  the soonest upstream back-off when the whole chain was rate limited, else the
 *  monthly-budget reset for a local quota block. Undefined means the caller
 *  shouldn't set the header. */
export function embeddingsRetryAfterSec(err: EmbeddingsError): number | undefined {
  if (err.retryAfterMs !== undefined) return retryAfterSeconds(err.retryAfterMs);
  if (err.code === 'quota_exceeded') return secondsUntilNextMonth();
  return undefined;
}