Skip to content

Never stop coding. Free MIT AI gateway: one endpoint, 375 providers (150+ free), 1,400+ models Kimi, Claude, GPT, Gemini, GLM, DeepSeek, MiniMax. Works with Claude Code, Codex, Cursor, OpenCode, Cline & Copilot. Quota-aware auto-fallback, RTK+Caveman compression saves 15-95% tokens, MCP/A2A, Desktop/PWA. Built by 900+ contributors

24.8 KB
Raw Blame
import { handleEmbedding } from "@omniroute/open-sse/handlers/embeddings.ts";
import {
  parseEmbeddingModel,
  getEmbeddingProvider,
  buildDynamicEmbeddingProvider,
  deriveEmbeddingProviderForChatProvider,
  type EmbeddingProviderNodeRow,
  type EmbeddingProvider,
} from "@omniroute/open-sse/config/embeddingRegistry.ts";
import { errorResponse, unavailableResponse } from "@omniroute/open-sse/utils/error.ts";
import { HTTP_STATUS } from "@omniroute/open-sse/config/constants.ts";
import * as log from "@/sse/utils/logger";
import { toJsonErrorPayload } from "@/shared/utils/upstreamError";
import {
  getProviderCredentials,
  clearRecoveredProviderState,
  markAccountUnavailable,
} from "@/sse/services/auth";
import {
  isKeyScopeBlockedCredentials,
  keyScopeBlockedResponse,
  keylessProviderRejection,
  narrowConnectionScope,
  withoutKeyScopeBlocked,
} from "@/sse/services/keyConnectionScope";
import { getCachedProviderNodes } from "@/lib/db/readCache";
import { getComboByName, getCombos } from "@/lib/db/combos";
import { getProviderConnections } from "@/lib/db/providers";
import { getUserDatabaseSettings } from "@/lib/db/databaseSettings";
import { resolveProxyForConnection } from "@/lib/db/settings";
import { runWithProxyContext } from "@omniroute/open-sse/utils/proxyFetch.ts";
import { handleComboChat } from "@omniroute/open-sse/services/combo.ts";
import { resolveBareModelToConnectionDefault } from "@omniroute/open-sse/services/model.ts";
import { findEmbeddingComboDimensionConflict } from "./familyGuard";
import {
  formatMissingEmbeddingCredentialsError,
  formatUnknownEmbeddingProviderError,
} from "./errors";
import { isPrivateHost, isCloudMetadataHost } from "@/shared/network/outboundUrlGuard";
import { calculateCost } from "@/lib/usage/costCalculator";
import { attachOmniRouteMetaHeaders } from "@/domain/omnirouteResponseMeta";
import { generateRequestId } from "@/shared/utils/requestId";
import { resolveLocalSyncedEndpointRoute } from "@/lib/providerModels/syncedEndpointRouting";
import { resolveAlibabaProviderEmbeddingUrl } from "@/shared/constants/alibabaProviderRegions";

/**
 * A local server (llama.cpp, LM Studio, …) embeds with whatever model it loaded and
 * ignores the request's `model`, so two connections of the same local provider are
 * NOT interchangeable accounts. Never pick a connection whose configured default model
 * names a different model: returns the ids of the remaining active connections, or
 * null (no restriction) when no connection declares a conflicting default model.
 */
async function localConnectionsServingModel(
  provider: string,
  model: string | null
): Promise<string[] | null> {
  if (!model) return null;
  const connections = await getProviderConnections({ provider, isActive: true });
  const normalize = (value: string) => value.replace(/^\/+/, "");
  const eligible = connections.filter((connection) => {
    const defaultModel = connection.defaultModel;
    return typeof defaultModel !== "string" || !defaultModel
      ? true
      : normalize(defaultModel) === normalize(model);
  });
  return eligible.length === connections.length ? null : eligible.map((c) => String(c.id));
}

type ValidatedEmbeddingBody = Record<string, unknown> & { model: string };
type ProviderCredentialsResult = Awaited<ReturnType<typeof getProviderCredentials>>;

// #6925: a private/LAN host (RFC1918 10/8, 192.168/16, 172.16/12, CGNAT 100.64/10,
// loopback, .local/.internal, ULA/link-local IPv6) is treated as a trusted no-auth
// local embedding provider — mirrors the outbound-URL guard's private-host
// classification instead of the old hand-rolled localhost/127.0.0.1/172.16-31-only
// regex, which excluded common LAN ranges (10.x, 192.168.x) and forced them through
// the apikey/bearer fallback even when no credentials exist. Cloud-metadata hosts
// (169.254.169.254 etc.) are never treated as no-auth local providers.
function isNoAuthLocalEmbeddingHost(hostname: string): boolean {
  return isPrivateHost(hostname) && !isCloudMetadataHost(hostname);
}

export interface EmbeddingHandlerOptions {
  clientRawRequest?: {
    endpoint: string;
    body: Record<string, unknown>;
    headers: Record<string, string>;
  };
  apiKeyId?: string | null;
  apiKeyName?: string | null;
  connectionId?: string | null;
  resolvedProvider?: EmbeddingProvider | null;
  resolvedModel?: string | null;
  /**
   * GHSA-3w2r: the caller API key's connection scope (null/undefined =
   * unscoped). Every credential lookup below — and every combo target — is
   * bounded by it, the same scope chat applies.
   */
  allowedConnections?: string[] | null;
  /** GHSA-3w2r: combo per-target model gate for the caller key (undefined = none). */
  isComboTargetAllowed?: (targetModelStr: string) => Promise<boolean>;
}

export async function createEmbeddingResponse(
  body: ValidatedEmbeddingBody,
  options: EmbeddingHandlerOptions = {}
): Promise<Response> {
  const modelStr = body.model;
  const startTime = Date.now();

  if (!modelStr.includes("/")) {
    try {
      const combo = await getComboByName(modelStr);
      if (combo) {
        let allCombos: Awaited<ReturnType<typeof getCombos>> = [];
        try {
          allCombos = await getCombos();
        } catch {}

        // Guard: an embedding combo whose targets span multiple vector
        // dimensions would corrupt any vector store on failover (vectors from
        // different models are not comparable). The generic combo engine has no
        // notion of embedding families, so reject loudly here before dispatch.
        // See _tasks/features-v3.8.12/01-embeddings-combo-family-guard.plan.md.
        const dimConflict = findEmbeddingComboDimensionConflict(combo as any, allCombos as any);
        if (dimConflict.conflict) {
          return errorResponse(
            HTTP_STATUS.BAD_REQUEST,
            `Embedding combo "${modelStr}" mixes models with incompatible vector ` +
              `dimensions (${dimConflict.distinct.join(", ")}). Failover between them ` +
              `would corrupt your vector store — use a single embedding dimension per combo.`
          );
        }

        let settings = {};
        try {
          settings = getUserDatabaseSettings();
        } catch {}

        // Inject the combo's configured dimensions into the request body so that
        // every upstream embedding call within this combo receives the same
        // dimensions override. The client's own dimensions value takes precedence
        // if already set. Ported from decolua/9router#1530.
        const comboRecord = combo as Record<string, unknown>;
        const comboDimensions =
          comboRecord.dimensions !== undefined && comboRecord.dimensions !== null
            ? String(comboRecord.dimensions)
            : undefined;
        const bodyWithDimensions =
          comboDimensions !== undefined && body.dimensions === undefined
            ? { ...body, dimensions: comboDimensions }
            : body;

        return handleComboChat({
          body: bodyWithDimensions,
          combo: combo as any,
          handleSingleModel: async (reqBody: any, targetModelStr: string, target?: any) => {
            const newBody = { ...reqBody, model: targetModelStr };
            return createEmbeddingResponse(newBody, {
              ...options,
              connectionId: target?.connectionId || options.connectionId,
            });
          },
          isModelAvailable: options.isComboTargetAllowed,
          log,
          settings,
          allCombos: allCombos as any,
          relayOptions: undefined,
          signal: undefined,
        });
      }
    } catch (err) {
      log.error("EMBED", `Combo resolution failed for ${modelStr}: ${err}`);
    }
  }
  let dynamicProviders: ReturnType<typeof buildDynamicEmbeddingProvider>[] = [];
  try {
    const nodes = (await getCachedProviderNodes()) as unknown as EmbeddingProviderNodeRow[];
    dynamicProviders = (Array.isArray(nodes) ? nodes : [])
      .filter((n) => {
        const validTypes = ["chat", "responses", "embeddings"];
        if (!validTypes.includes(n.apiType || "")) return false;
        try {
          const hostname = new URL(n.baseUrl).hostname;
          return isNoAuthLocalEmbeddingHost(hostname);
        } catch {
          return false;
        }
      })
      .map((n) => {
        try {
          return buildDynamicEmbeddingProvider(n);
        } catch (err) {
          log.error("EMBED", `Skipping invalid provider_node ${n.prefix}: ${err}`);
          return null;
        }
      })
      .filter((p): p is NonNullable<typeof p> => p !== null);
  } catch (err) {
    log.error("EMBED", `Failed to load provider_nodes for embeddings: ${err}`);
  }

  const parsedModel = options.resolvedProvider
    ? {
        provider: options.resolvedProvider.id,
        model: options.resolvedModel ?? body.model,
      }
    : parseEmbeddingModel(body.model, dynamicProviders);
  let { provider, model: resolvedModel } = parsedModel;
  // #11088: a bare local-model request routes through the connection that
  // advertises the requested endpoint — only when no explicit resolvedProvider
  // already won above (explicit resolution takes precedence).
  const syncedEndpointRoute = options.resolvedProvider
    ? null
    : await resolveLocalSyncedEndpointRoute(body.model, "embeddings");
  if (syncedEndpointRoute) {
    provider = syncedEndpointRoute.provider;
    resolvedModel = syncedEndpointRoute.model;
  }
  if (!provider) {
    return errorResponse(
      HTTP_STATUS.BAD_REQUEST,
      `Invalid embedding model: ${body.model}. Use format: provider/model`
    );
  }

  let credentials: ProviderCredentialsResult | null = null;
  let providerConfig: EmbeddingProvider | null =
    options.resolvedProvider ||
    dynamicProviders.find((dp) => dp.id === provider) ||
    getEmbeddingProvider(provider) ||
    null;
  let credentialsProviderId = provider;
  const keyConnections = options.allowedConnections ?? null;

  // #11088: synced-endpoint route — the connection advertising this endpoint
  // supplies credentials and its configured base URL directly.
  if (syncedEndpointRoute) {
    credentials = await getProviderCredentials(
      provider,
      null,
      narrowConnectionScope(keyConnections, syncedEndpointRoute.connectionIds),
      syncedEndpointRoute.model
    );
    if (!credentials) {
      return errorResponse(
        HTTP_STATUS.BAD_REQUEST,
        `No credentials for embedding provider: ${provider}`
      );
    }
    if (isKeyScopeBlockedCredentials(credentials)) {
      return keyScopeBlockedResponse(provider, credentials);
    }
    if ("allRateLimited" in credentials && credentials.allRateLimited) {
      return unavailableResponse(
        HTTP_STATUS.RATE_LIMITED,
        `[${provider}] All accounts rate limited`,
        credentials.retryAfter,
        credentials.retryAfterHuman
      );
    }

    const providerSpecificData = (credentials as { providerSpecificData?: Record<string, unknown> })
      .providerSpecificData;
    const configuredBaseUrl = providerSpecificData?.baseUrl;
    if (typeof configuredBaseUrl !== "string" || configuredBaseUrl.trim().length === 0) {
      return errorResponse(
        HTTP_STATUS.BAD_REQUEST,
        `No base URL configured for embedding provider: ${provider}`
      );
    }
    let baseUrl = configuredBaseUrl.trim();
    while (baseUrl.endsWith("/")) baseUrl = baseUrl.slice(0, -1);
    // Keep the registry provider's structured-input capabilities (e.g. llama.cpp
    // multimodal content parts); only the endpoint and auth come from the connection.
    const registryConfig = getEmbeddingProvider(provider);
    providerConfig = {
      id: provider,
      baseUrl: baseUrl.endsWith("/embeddings") ? baseUrl : `${baseUrl}/embeddings`,
      authType: "apikey",
      authHeader: "bearer",
      models: [],
      structuredInputProtocol: registryConfig?.structuredInputProtocol,
      passthroughModalities: registryConfig?.passthroughModalities,
    };
  }

  // Generic fallback: a configured OpenAI-compatible chat provider with no
  // curated embedding entry (groq, mistral, upstage, ...) still serves
  // embeddings via the standard <base>/embeddings endpoint. Curated registry
  // entries are checked first and keep their specialized configuration.
  if (!providerConfig && !options.resolvedProvider) {
    try {
      const { REGISTRY } = await import("@omniroute/open-sse/config/providerRegistry.ts");
      const chatEntry = (REGISTRY as Record<string, { baseUrl?: string } | undefined>)[provider];
      providerConfig = deriveEmbeddingProviderForChatProvider(provider, chatEntry);
      if (providerConfig) {
        log.info(
          "EMBED",
          `Derived generic embedding endpoint for configured provider ${provider}: ${providerConfig.baseUrl}`
        );
      }
    } catch (err) {
      log.error("EMBED", `Failed to derive generic embedding provider ${provider}: ${err}`);
    }
  }

  if (!providerConfig) {
    try {
      const allNodes = (await getCachedProviderNodes()) as unknown as EmbeddingProviderNodeRow[];
      const matchingNode = (Array.isArray(allNodes) ? allNodes : []).find(
        (n) =>
          n.prefix === provider &&
          (n.apiType === "chat" || n.apiType === "responses" || n.apiType === "embeddings") &&
          n.baseUrl
      );
      if (matchingNode) {
        const baseUrl = String(matchingNode.baseUrl).replace(/\/+$/, "");
        // #6925: a private/LAN node reaching this fallback (e.g. a matching
        // prefix that skipped the dynamicProviders pass above) must never be
        // forced through bearer-auth — only a non-private host falls back to
        // apikey/bearer credential resolution.
        let nodeHostname = "";
        try {
          nodeHostname = new URL(matchingNode.baseUrl).hostname;
        } catch {
          nodeHostname = "";
        }
        const isNoAuthLocal = nodeHostname !== "" && isNoAuthLocalEmbeddingHost(nodeHostname);
        providerConfig = {
          id: matchingNode.prefix,
          baseUrl: `${baseUrl}/embeddings`,
          authType: isNoAuthLocal ? "none" : "apikey",
          authHeader: isNoAuthLocal ? "none" : "bearer",
          models: [],
        };
        credentialsProviderId = matchingNode.id || provider;
        log.info(
          "EMBED",
          `Resolved custom embedding provider: ${provider} -> ${providerConfig.baseUrl}`
        );
      }
    } catch (err) {
      log.error("EMBED", `Failed to resolve custom embedding provider ${provider}: ${err}`);
    }
  }

  if (!providerConfig) {
    // Root cause is otherwise invisible: this 400 is returned before any
    // call_logs row is written, so a real embedding-provider misconfiguration
    // (e.g. a customModels override whose id prefix doesn't match its own
    // connection's provider id, or a stale synced-model cache -- both silent
    // for months in production) previously left no trace anywhere in
    // OmniRoute's own logs or dashboard, only in the calling client's log.
    log.warn(
      "EMBED",
      `Unknown embedding provider ${provider} for model "${body.model}" -- checked static ` +
        "registry, provider_nodes, chat-provider fallback, and the self-hosted synced-endpoint " +
        "route (customModels override + synced model cache) with no match"
    );
    return errorResponse(
      HTTP_STATUS.BAD_REQUEST,
      formatUnknownEmbeddingProviderError(provider, resolvedModel)
    );
  }

  if (!credentials && providerConfig.authType !== "none") {
    credentials = await getProviderCredentials(credentialsProviderId, null, keyConnections);
    if (!credentials) {
      return errorResponse(
        HTTP_STATUS.BAD_REQUEST,
        formatMissingEmbeddingCredentialsError(provider)
      );
    }
    if ("allRateLimited" in credentials && credentials.allRateLimited) {
      return unavailableResponse(
        HTTP_STATUS.RATE_LIMITED,
        `[${provider}] All accounts rate limited`,
        credentials.retryAfter,
        credentials.retryAfterHuman
      );
    }
    if ("allExpired" in credentials && credentials.allExpired) {
      const expiredStatus = (credentials as { expiredStatus?: string }).expiredStatus;
      const quota = expiredStatus === "credits_exhausted";
      const reason = quota ? "credits exhausted" : "authentication expired";
      return errorResponse(
        quota ? HTTP_STATUS.PAYMENT_REQUIRED : HTTP_STATUS.UNAUTHORIZED,
        `[${provider}] All ${credentials.expiredCount || 1} connection(s) ${reason} — please reconnect in the dashboard`
      );
    }
    // #13945: blockedByKeyPolicy is the third credential-diagnostic sentinel
    // (alongside allRateLimited/allExpired above) — without this check a
    // truthy sentinel would reach the embeddings executor below with no
    // apiKey/accessToken. It fires now that the key's connection scope is
    // wired in (GHSA-3w2r) and answers the same 403 as chat (#13832).
    if (isKeyScopeBlockedCredentials(credentials)) {
      return keyScopeBlockedResponse(provider, credentials);
    }
  } else if (
    provider === "ollama-local" ||
    provider === "lmstudio" ||
    provider === "llama-cpp" ||
    provider === "lemonade"
  ) {
    // Local providers are key-optional, but a configured connection can provide
    // a custom host or API key (e.g. Lemonade bearer auth). Hydrate that optional
    // connection without imposing an authentication requirement, then keep the
    // static localhost default when no connection exists.
    // An empty allowlist means "no filter" to getProviderCredentials, so a request
    // that no connection serves must skip connection selection entirely.
    const servingIds = await localConnectionsServingModel(credentialsProviderId, resolvedModel);
    const localCredentials =
      servingIds && servingIds.length === 0
        ? null
        : withoutKeyScopeBlocked(
            await getProviderCredentials(
              credentialsProviderId,
              null,
              narrowConnectionScope(keyConnections, servingIds)
            )
          );
    if (
      localCredentials &&
      !("allRateLimited" in localCredentials) &&
      !("allExpired" in localCredentials)
    ) {
      credentials = localCredentials;
    }
  } else if (!credentials && providerConfig.authType === "none") {
    // #13234: private-host nodes are classified no-auth so a keyless
    // LAN Ollama still works (#6925). A stored API key on that same
    // node must still ride outbound, matching dashboard Check.
    const keyedCredentials = withoutKeyScopeBlocked(
      await getProviderCredentials(credentialsProviderId, null, keyConnections)
    );
    if (
      keyedCredentials &&
      !("allRateLimited" in keyedCredentials) &&
      !("allExpired" in keyedCredentials)
    ) {
      const token =
        (typeof (keyedCredentials as { apiKey?: unknown }).apiKey === "string" &&
          (keyedCredentials as { apiKey?: string }).apiKey) ||
        (typeof (keyedCredentials as { accessToken?: unknown }).accessToken === "string" &&
          (keyedCredentials as { accessToken?: string }).accessToken) ||
        "";
      if (token) {
        credentials = keyedCredentials;
        providerConfig = {
          ...providerConfig,
          authType: "apikey",
          authHeader: "bearer",
        };
      }
    }
  }

  // GHSA-3w2r: a run without any connection (local default, private-host node)
  // is keyless. A connection-scoped key reaches it only when its allowlist
  // admits keyless providers or an in-scope connection serves the provider —
  // chat's #9057 rule. Unscoped callers are not affected.
  if (!credentials && providerConfig.authType === "none") {
    const keylessRejection = await keylessProviderRejection(provider, keyConnections, () =>
      getProviderCredentials(credentialsProviderId, null, keyConnections)
    );
    if (keylessRejection) return keylessRejection;
  }

  // Alibaba's embedding endpoint is connection-scoped: workspace and region
  // live in providerSpecificData, so the static chat registry cannot select it.
  if (
    credentials &&
    !options.resolvedProvider &&
    (provider === "alibaba" || provider === "alibaba-cn")
  ) {
    const providerSpecificData = (
      credentials as { providerSpecificData?: Record<string, unknown> | null }
    ).providerSpecificData;
    const connectionBaseUrl = resolveAlibabaProviderEmbeddingUrl(
      provider,
      providerSpecificData,
      providerConfig.baseUrl
    );
    if (connectionBaseUrl) providerConfig = { ...providerConfig, baseUrl: connectionBaseUrl };
  }

  // #474: when the request used a bare model name (no "/" — e.g. an alias that
  // resolved to "auto") and the selected connection declares a defaultModel,
  // resolve the bare name to that real model ID before the upstream call so the
  // provider receives a concrete model. A "/"-qualified name is left untouched.
  const connectionDefaultModel =
    credentials && typeof (credentials as { defaultModel?: unknown }).defaultModel === "string"
      ? ((credentials as { defaultModel?: string }).defaultModel as string)
      : null;
  const effectiveModel = resolveBareModelToConnectionDefault(
    modelStr,
    resolvedModel,
    connectionDefaultModel
  );

  // Resolve the connection-level proxy so the upstream embedding request honors
  // the same per-connection pinning as chat, image generation, and count_tokens
  // (#1904-style behavior). Without this, embeddings silently fall back to the
  // global/env proxy and ignore a connection's pinned proxy. Ported from
  // upstream decolua/9router#1701.
  let proxyInfo: Awaited<ReturnType<typeof resolveProxyForConnection>> | null = null;
  const connectionIdForProxy = (credentials as { connectionId?: string } | null)?.connectionId;
  if (connectionIdForProxy) {
    try {
      proxyInfo = await resolveProxyForConnection(connectionIdForProxy);
    } catch (err) {
      log.error("EMBED", `Failed to resolve proxy for connection ${connectionIdForProxy}: ${err}`);
    }
  }

  const runEmbedding = () =>
    handleEmbedding({
      body:
        effectiveModel !== resolvedModel
          ? { ...body, model: `${provider}/${effectiveModel}` }
          : body,
      // getProviderCredentials returns a richer connection object; handleEmbedding
      // reads auth plus the optional local baseUrl override. Bridge the wider
      // selection type to the handler's narrow credential shape.
      credentials: credentials as {
        apiKey?: string;
        accessToken?: string;
        providerSpecificData?: Record<string, unknown> | null;
      } | null,
      log,
      resolvedProvider: providerConfig,
      resolvedModel: effectiveModel,
      clientRawRequest: options.clientRawRequest || null,
      apiKeyId: options.apiKeyId || null,
      apiKeyName: options.apiKeyName || null,
      // #10347 — thread the selected connection id so handleEmbedding can cool the
      // account on a hard upstream failure (previously always null on /v1/embeddings).
      connectionId:
        (credentials as { connectionId?: string } | null)?.connectionId ||
        options.connectionId ||
        connectionIdForProxy ||
        null,
    });

  const result = connectionIdForProxy
    ? await runWithProxyContext(proxyInfo?.proxy || null, runEmbedding)
    : await runEmbedding();

  const responseHeaders = new Headers(result.headers);

  if (result.success) {
    if (credentials) await clearRecoveredProviderState(credentials as Record<string, unknown>);
    responseHeaders.set("Content-Type", "application/json");
    const usage = (result.data as { usage?: Record<string, number> })?.usage ?? null;
    const costUsd = usage ? await calculateCost(provider, effectiveModel ?? "", usage) : 0;
    attachOmniRouteMetaHeaders(responseHeaders, {
      provider,
      model: effectiveModel,
      usage,
      costUsd,
      latencyMs: Date.now() - startTime,
      requestId: generateRequestId(),
    });
    return new Response(JSON.stringify(result.data), {
      status: result.status,
      headers: responseHeaders,
    });
  }

  // #10347: cool down the account on hard errors (402 subscription expired,
  // 401 revoked, 403 forbidden, 404 model gone, 429 rate limit, 5xx server
  // errors) so the next embedding request skips this account. Mirrors chat.ts
  // behavior.
  // Skip for 400 (bad request) — the account is fine, the request was wrong.
  // Best-effort: don't block the error response on the DB write.
  const HARD_ERROR_STATUSES = new Set([401, 402, 403, 404, 429, 500, 502, 503, 504]);
  if (
    credentials &&
    "connectionId" in credentials &&
    typeof credentials.connectionId === "string" &&
    HARD_ERROR_STATUSES.has(result.status)
  ) {
    markAccountUnavailable(
      credentials.connectionId,
      result.status,
      result.error || "Embedding provider error",
      provider,
      resolvedModel || null
    ).catch((err) => {
      log.debug(
        "EMBED",
        `Cooldown write failed for ${provider}/${credentials.connectionId?.slice(0, 8)}: ${err}`
      );
    });
  }

  responseHeaders.set("Content-Type", "application/json");
  const errorPayload = toJsonErrorPayload(result.error, "Embedding provider error");
  return new Response(JSON.stringify(errorPayload), {
    status: result.status,
    headers: responseHeaders,
  });
}