Skip to content

Never stop coding. Free MIT AI gateway: one endpoint, 375 providers (150+ free), 1,400+ models Kimi, Claude, GPT, Gemini, GLM, DeepSeek, MiniMax. Works with Claude Code, Codex, Cursor, OpenCode, Cline & Copilot. Quota-aware auto-fallback, RTK+Caveman compression saves 15-95% tokens, MCP/A2A, Desktop/PWA. Built by 900+ contributors

15.6 KB
Raw Blame
import { handleRerank } from "@omniroute/open-sse/handlers/rerank.ts";
import { getRequestedModelEndpointMismatch } from "@/lib/providerModels/modelEndpointGuard";
import {
  getProviderCredentialsWithQuotaPreflight,
  clearRecoveredProviderState,
} from "@/sse/services/auth";
import { withInjectionGuard } from "@/middleware/promptInjectionGuard";
import { parseRerankModel, getRerankProvider } from "@omniroute/open-sse/config/rerankRegistry.ts";
import { errorResponse } from "@omniroute/open-sse/utils/error.ts";
import { HTTP_STATUS } from "@omniroute/open-sse/config/constants.ts";
import { enforceApiKeyPolicy } from "@/shared/utils/apiKeyPolicy";
import {
  createComboTargetKeyGate,
  isKeyScopeBlockedCredentials,
  keyScopeBlockedResponse,
  resolveKeyConnectionScope,
} from "@/sse/services/keyConnectionScope";
import { v1RerankSchema } from "@/shared/validation/schemas";
import { isValidationFailure, validateBody } from "@/shared/validation/helpers";
import { loadRerankProviderNodes } from "@/app/api/v1/_shared/rerankProviderNodes";
import {
  buildLocalRerankRequestBody,
  normalizeLocalRerankResponse,
} from "@/app/api/v1/_shared/rerankLocalNodeShapes";
import {
  isAllRateLimitedCredentials,
  rateLimitedProviderResponse,
} from "@/app/api/v1/_shared/rateLimit";
import { saveCallLog } from "@/lib/usageDb";
import { attachOmniRouteMetaHeaders } from "@/domain/omnirouteResponseMeta";
import { generateRequestId } from "@/shared/utils/requestId";
import { CORS_HEADERS } from "@omniroute/open-sse/utils/cors.ts";
import { deriveRerankProviderForChatProvider } from "@omniroute/open-sse/config/rerankRegistry.ts";
import { resolveAlibabaQwen3RerankUrl } from "@/shared/constants/alibabaProviderRegions";
import { getComboByName, getCombos } from "@/lib/db/combos";
import { getUserDatabaseSettings } from "@/lib/db/databaseSettings";
import { handleComboChat } from "@omniroute/open-sse/services/combo.ts";
import * as log from "@/sse/utils/logger";

/**
 * Handle CORS preflight
 */
export async function OPTIONS() {
  return new Response(null, {
    headers: {
      "Access-Control-Allow-Methods": "POST, OPTIONS",
      "Access-Control-Allow-Headers": "*",
    },
  });
}

/**
 * POST /v1/rerank - Cohere-compatible rerank endpoint
 *
 * Supports cloud providers (Cohere, Together, NVIDIA, Fireworks)
 * and OpenAI-compatible provider_nodes (oMLX, vLLM, Infinity, TEI behind a gateway, …)
 * via dynamic routing. Loopback nodes are always eligible; remote nodes require the
 * `RERANK_REMOTE_PROVIDER_NODES` opt-in and must pass the provider outbound URL policy
 * (see `_shared/rerankProviderNodes.ts`).
 */
async function postHandler(request, context) {
  let rawBody;
  try {
    rawBody = await request.json();
  } catch {
    return errorResponse(HTTP_STATUS.BAD_REQUEST, "Invalid JSON body");
  }

  const validation = validateBody(v1RerankSchema, rawBody);
  if (isValidationFailure(validation)) {
    return errorResponse(HTTP_STATUS.BAD_REQUEST, validation.error.message);
  }
  const body = validation.data;

  // Enforce API key policies (model restrictions + budget limits)
  const policy = await enforceApiKeyPolicy(request, body.model);
  if (policy.rejection) return policy.rejection;
  // GHSA-3w2r: credential selection (direct and per combo target) stays inside
  // the key's connection scope, the same scope chat applies.
  const scope = await resolveKeyConnectionScope(policy.apiKeyInfo);
  if (scope.rejection) return scope.rejection;
  // Respect the authenticated API key's credential scope before checking model capabilities.
  const endpointMismatch = await getRequestedModelEndpointMismatch(body.model, "rerank");
  if (endpointMismatch) return errorResponse(HTTP_STATUS.BAD_REQUEST, endpointMismatch);

  return handleValidatedRerankRequestBody(
    {
      ...body,
      documents: body.documents,
      top_n: typeof body.top_n === "number" ? body.top_n : undefined,
      return_documents:
        typeof body.return_documents === "boolean" ? body.return_documents : undefined,
    },
    {
      apiKeyId: policy.apiKeyInfo?.id || null,
      apiKeyName: policy.apiKeyInfo?.name || null,
      allowedConnections: scope.allowedConnections,
      isComboTargetAllowed: createComboTargetKeyGate(policy, body.model),
    }
  );
}

type ValidatedRerankBody = {
  model: string;
  query: string;
  documents: unknown[];
  top_n?: number;
  return_documents?: boolean;
};

type RerankRequestMeta = {
  apiKeyId?: string | null;
  apiKeyName?: string | null;
  /** GHSA-3w2r: the caller key's connection scope (null/undefined = unscoped). */
  allowedConnections?: string[] | null;
  /** GHSA-3w2r: combo per-target model gate for the caller key (undefined = none). */
  isComboTargetAllowed?: (targetModelStr: string) => Promise<boolean>;
};

/**
 * Dispatch a validated rerank request, including named-combo expansion.
 *
 * `/v1/models` advertises stored combos and chat, embeddings, and audio already resolve
 * those names. Keeping rerank on the same contract lets a memory client use one stable
 * route while the combo fails over between a local reranker and hosted providers.
 */
export async function handleValidatedRerankRequestBody(
  body: ValidatedRerankBody,
  meta: RerankRequestMeta = {}
): Promise<Response> {
  const modelStr = body.model;

  if (!modelStr.includes("/")) {
    try {
      const combo = await getComboByName(modelStr);
      if (combo) {
        let allCombos: Awaited<ReturnType<typeof getCombos>> = [];
        try {
          allCombos = await getCombos();
        } catch {}

        let settings = {};
        try {
          settings = getUserDatabaseSettings();
        } catch {}

        return handleComboChat({
          body: body as any,
          combo: combo as any,
          handleSingleModel: async (reqBody: any, targetModelStr: string) =>
            handleValidatedRerankRequestBody(
              { ...reqBody, model: targetModelStr } as ValidatedRerankBody,
              meta
            ),
          isModelAvailable: meta.isComboTargetAllowed,
          log,
          settings,
          allCombos: allCombos as any,
          relayOptions: undefined,
          signal: undefined,
        });
      }
    } catch (err) {
      log.error("RERANK", `Combo resolution failed for ${modelStr}: ${err}`);
    }
  }

  // Load eligible provider_nodes for rerank routing (loopback always; remote when
  // RERANK_REMOTE_PROVIDER_NODES is on and the URL passes the outbound policy).
  const localProviders = await loadRerankProviderNodes();

  // Try cloud registry first
  const { provider, model: modelId } = parseRerankModel(body.model);
  const prefixSeparator = body.model.indexOf("/");
  const resolvedModelId =
    provider || prefixSeparator < 0 ? modelId : body.model.slice(prefixSeparator + 1);

  // Generic fallback: a configured OpenAI-compatible chat provider with no
  // curated rerank entry (groq, mistral, ...) still exposes a Cohere-compatible
  // <base>/rerank endpoint. Only used when the prefix matches a chat provider
  // that can actually derive an endpoint — otherwise fall through to local nodes.
  let derivedProvider: ReturnType<typeof deriveRerankProviderForChatProvider> = null;
  if (!provider) {
    const prefix = body.model.split("/")[0];
    if (prefix && prefix !== body.model) {
      try {
        const { REGISTRY } = await import("@omniroute/open-sse/config/providerRegistry.ts");
        const chatEntry = (REGISTRY as Record<string, { baseUrl?: string } | undefined>)[prefix];
        derivedProvider = deriveRerankProviderForChatProvider(prefix, chatEntry);
      } catch {
        derivedProvider = null;
      }
    }
  }

  if (provider || derivedProvider) {
    // Cloud provider matched (or a generic Cohere-compatible endpoint was derived)
    const effectiveProviderId = provider || derivedProvider!.id;
    const credentials = await getProviderCredentialsWithQuotaPreflight(
      effectiveProviderId,
      null,
      meta.allowedConnections ?? null,
      resolvedModelId
    );
    if (!credentials) {
      return errorResponse(
        HTTP_STATUS.BAD_REQUEST,
        `No credentials for provider: ${effectiveProviderId}`
      );
    }
    if (isKeyScopeBlockedCredentials(credentials)) {
      return keyScopeBlockedResponse(effectiveProviderId, credentials);
    }
    if (isAllRateLimitedCredentials(credentials)) {
      return rateLimitedProviderResponse(effectiveProviderId, credentials);
    }

    let runtimeProvider = derivedProvider as
      | (NonNullable<ReturnType<typeof deriveRerankProviderForChatProvider>> & {
          format?: string;
        })
      | null;
    if (
      (effectiveProviderId === "alibaba" || effectiveProviderId === "alibaba-cn") &&
      resolvedModelId === "qwen3-rerank"
    ) {
      const providerSpecificData = (
        credentials as { providerSpecificData?: Record<string, unknown> | null }
      ).providerSpecificData;
      const baseUrl = resolveAlibabaQwen3RerankUrl(
        effectiveProviderId,
        providerSpecificData,
        derivedProvider?.baseUrl || ""
      );
      if (!baseUrl) {
        return errorResponse(
          HTTP_STATUS.BAD_REQUEST,
          `No rerank endpoint configured for provider: ${effectiveProviderId}`
        );
      }
      runtimeProvider = {
        id: effectiveProviderId,
        baseUrl,
        authType: "apikey",
        authHeader: "bearer",
        models: [],
        format: "alibaba-qwen3",
      };
    }

    const response = await handleRerank({
      model: body.model,
      query: body.query,
      documents: body.documents,
      top_n: body.top_n,
      return_documents: body.return_documents,
      credentials,
      resolvedProvider: runtimeProvider,
      resolvedModel: resolvedModelId,
      connectionId: (credentials as { connectionId?: string } | null)?.connectionId || null,
      apiKeyId: meta.apiKeyId || null,
      apiKeyName: meta.apiKeyName || null,
    });
    if (response?.ok) {
      await clearRecoveredProviderState(credentials);
    }
    return response;
  }

  // Try local provider_nodes (model format: prefix/model-name)
  const parts = body.model.split("/");
  if (parts.length >= 2) {
    const prefix = parts[0];
    const localModel = parts.slice(1).join("/");
    const localProvider = localProviders.find((p) => p.id === prefix);

    if (localProvider) {
      const credentials = await getProviderCredentialsWithQuotaPreflight(
        localProvider.providerId,
        null,
        meta.allowedConnections ?? null
      );
      if (!credentials) {
        return errorResponse(
          HTTP_STATUS.BAD_REQUEST,
          `No credentials for local provider: ${prefix}`
        );
      }
      if (isKeyScopeBlockedCredentials(credentials)) {
        return keyScopeBlockedResponse(prefix, credentials);
      }
      if (isAllRateLimitedCredentials(credentials)) {
        return rateLimitedProviderResponse(prefix, credentials);
      }

      const token = credentials?.apiKey || credentials?.accessToken;
      const startTime = Date.now();
      // One body serves every known local server: Cohere/OpenAI spelling (`documents`,
      // `return_documents`) plus the TEI spelling (`texts`, `return_text`). See
      // `_shared/rerankLocalNodeShapes.ts`.
      const upstreamBody = JSON.stringify(
        buildLocalRerankRequestBody({
          model: localModel,
          query: body.query,
          documents: body.documents,
          top_n: body.top_n as number | undefined,
          return_documents: body.return_documents as boolean | undefined,
        })
      );
      const upstreamInit: RequestInit = {
        method: "POST",
        headers: {
          "Content-Type": "application/json",
          Authorization: `Bearer ${token}`,
        },
        body: upstreamBody,
      };
      try {
        let res = await fetch(localProvider.baseUrl, upstreamInit);

        // Some local providers (e.g. Infinity, TEI) mount at /rerank rather than /v1/rerank
        if (res.status === 404 && localProvider.baseUrl.endsWith("/v1/rerank")) {
          const fallbackUrl = localProvider.baseUrl.replace(/\/v1\/rerank$/, "/rerank");
          try {
            const fallbackRes = await fetch(fallbackUrl, upstreamInit);
            if (fallbackRes.ok || fallbackRes.status !== 404) {
              res = fallbackRes;
            }
          } catch {
            // retain original 404 response if fallback fetch fails
          }
        }

        if (!res.ok) {
          const errData = await res.json().catch(() => ({}));
          const errorMessage =
            errData.message || errData.detail || `Provider returned HTTP ${res.status}`;
          saveCallLog({
            method: "POST",
            path: "/v1/rerank",
            status: res.status,
            model: body.model,
            provider: prefix,
            connectionId:
              (credentials as { connectionId?: string } | null)?.connectionId || undefined,
            duration: Date.now() - startTime,
            requestBody: {
              model: body.model,
              query: body.query,
              documents: body.documents,
              top_n: body.top_n,
              return_documents: body.return_documents,
            },
            responseBody: errData,
            error: errorMessage,
            apiKeyId: meta.apiKeyId || undefined,
            apiKeyName: meta.apiKeyName || undefined,
          }).catch(() => {});
          return errorResponse(res.status, errorMessage);
        }

        // Fold TEI's bare `[{index, score, text}]`, `score`-only gateways, and
        // Voyage-style `{data: [...]}` into the Cohere envelope clients (and the
        // memory engine, which reads `relevance_score`) expect.
        const data = normalizeLocalRerankResponse(await res.json(), body.documents, {
          top_n: body.top_n as number | undefined,
          return_documents: body.return_documents as boolean | undefined,
        });
        const latencyMs = Date.now() - startTime;
        saveCallLog({
          method: "POST",
          path: "/v1/rerank",
          status: 200,
          model: body.model,
          provider: prefix,
          connectionId:
            (credentials as { connectionId?: string } | null)?.connectionId || undefined,
          duration: latencyMs,
          tokens: { prompt_tokens: 0, completion_tokens: 0 },
          requestBody: {
            model: body.model,
            query: body.query,
            documents: body.documents,
            top_n: body.top_n,
            return_documents: body.return_documents,
          },
          responseBody: data,
          apiKeyId: meta.apiKeyId || undefined,
          apiKeyName: meta.apiKeyName || undefined,
        }).catch(() => {});

        const headers = new Headers({ ...CORS_HEADERS, "Content-Type": "application/json" });
        attachOmniRouteMetaHeaders(headers, {
          provider: prefix,
          model: localModel,
          costUsd: 0,
          latencyMs,
          requestId: generateRequestId(),
        });
        return new Response(JSON.stringify(data), {
          status: 200,
          headers,
        });
      } catch (err: any) {
        saveCallLog({
          method: "POST",
          path: "/v1/rerank",
          status: 500,
          model: body.model,
          provider: prefix,
          connectionId:
            (credentials as { connectionId?: string } | null)?.connectionId || undefined,
          duration: Date.now() - startTime,
          error: err.message,
          apiKeyId: meta.apiKeyId || undefined,
          apiKeyName: meta.apiKeyName || undefined,
        }).catch(() => {});
        return errorResponse(500, `Rerank request failed: ${err.message}`);
      }
    }
  }

  return errorResponse(
    HTTP_STATUS.BAD_REQUEST,
    `Invalid rerank model: ${body.model}. Use format: provider/model`
  );
}

export const POST = withInjectionGuard(postHandler);