Skip to content

Enhanced ChatGPT Clone: Features Agents, MCP, Skills, DeepSeek, Anthropic, AWS, OpenAI, Responses API, Azure, Groq, o1, GPT-5, Mistral, OpenRouter, Vertex AI, Gemini, Artifacts, AI model switching, message search, Code Interpreter, langchain, DALL-E-3, OpenAPI Actions, Functions, Secure Multi-User Auth, Presets, open-source for self-hosting. Active

19.6 KB
Raw Blame
import { z } from 'zod';
import axios from 'axios';
import crypto from 'crypto';
import { logger } from '@librechat/data-schemas';
import {
  Time,
  CacheKeys,
  ReasoningEffort,
  EModelEndpoint,
  normalizeEndpointName,
  getModelReasoning,
  effectiveModelReasoning,
  isOpenRouterEffortSupported,
} from 'librechat-data-provider';
import type { TEndpoint, TModelReasoning } from 'librechat-data-provider';
import type { AppConfig } from '@librechat/data-schemas';
import type {
  ReasoningCapabilityDeps,
  ReasoningCapabilityCache,
  ReasoningCapabilityResult,
} from '~/types';
import { isUserProvided, applyAxiosProxyConfig, resolveHeaders } from '~/utils';
import { usesOpenRouterCatalog } from '~/endpoints/custom/config';
import { resolveConfigSecret } from '~/admin/secrets';
import { standardCache } from '~/cache';

/** Applies when an endpoint does not set `customParams.reasoningCatalogTimeoutMs`. */
const DEFAULT_CATALOG_TIMEOUT_MS = 5000;

/** Applies when an endpoint does not set `customParams.reasoningCatalogFailureTtlMs`. */
const DEFAULT_CATALOG_FAILURE_TTL_MS = 30000;

/** Applies when an endpoint does not set `customParams.reasoningCatalogTtlMs`. */
const DEFAULT_CATALOG_TTL_MS = Time.ONE_HOUR;

/** Applies when an endpoint does not set `customParams.reasoningCatalogMaxPages`. */
const DEFAULT_CATALOG_MAX_PAGES = 20;

const reasoningSchema = z.object({
  supported_efforts: z.array(z.string()).nullable().optional(),
  mandatory: z.boolean().optional(),
});

const pageSchema = z.object({
  data: z.array(z.object({ id: z.string(), reasoning: z.unknown().optional() })),
  /** Absent, `{}`, `{ next: null }` and `{ next: '' }` all end the catalog; anything else that is
   *  not a string `next` is malformed and makes the whole catalog unreadable. */
  links: z.object({ next: z.string().nullable().optional() }).optional(),
});

/** Models whose reasoning depends on the route a request takes, so no fixed efforts exist. */
const isDynamicRouterModel = (id: string): boolean => id.startsWith('openrouter/');

type CatalogModels = Record<string, TModelReasoning>;

/** A read catalog and the moment the cache drops it, kept with it so a cached read reports the same. */
type CatalogEntry = { models: CatalogModels; expiresAt: number };

/**
 * OpenRouter's `supported_efforts: null` means no allowlist applies and every gateway effort
 * is accepted; an omitted field, or an omitted `reasoning` object, means the model exposes no
 * effort selection at all, which is recorded as an empty list. A `reasoning` object that does not
 * parse leaves the model unknown instead.
 */
const UNRESTRICTED_EFFORTS: string[] = Object.values(ReasoningEffort).filter(
  (effort) => effort !== ReasoningEffort.unset,
);

/**
 * Identity of a catalog lookup: the endpoint, the headers it sends and the limits that decide its
 * outcome. Two endpoints sharing a base URL and key but configuring different headers or limits
 * must not share a result, or the first one's configuration would govern both.
 */
function cacheKey({
  baseURL,
  apiKey,
  timeoutMs,
  maxPages,
  failureTtlMs,
  ttlMs,
  headers,
}: ResolvedTarget): string {
  const sortedHeaders = Object.entries(headers).sort(([a], [b]) => a.localeCompare(b));
  const identity = JSON.stringify([
    baseURL,
    apiKey,
    timeoutMs,
    maxPages,
    failureTtlMs,
    ttlMs,
    sortedHeaders,
  ]);
  return crypto.createHash('sha256').update(identity).digest('hex').slice(0, 32);
}

/** The scheme and host of a configured URL: never its credentials, path or query. */
function safeOrigin(baseURL: string): string {
  try {
    return new URL(baseURL).origin;
  } catch {
    return 'invalid-url';
  }
}

/** A header value that is filled in per user or per request, which a shared catalog cannot honor. */
const PER_REQUEST_PLACEHOLDER = /\{\{[^}]*\}\}/;

/** Placeholders that carry the request's own ids (`{{LIBRECHAT_BODY_PARENTMESSAGEID}}`), not who is
 *  asking. They do not change which models a user is offered, so they do not scope the model list;
 *  the documented OpenRouter example sets one. Any other placeholder (user fields, OpenID tokens, an
 *  unknown name) is treated as identity-dependent. */
const REQUEST_METADATA_PLACEHOLDER = /\{\{LIBRECHAT_BODY_[A-Z_]+\}\}/g;

/** An environment variable reference left in a value, which means the variable is not set. */
const UNRESOLVED_ENV_VARIABLE = /\$\{[^}]*\}/;

/**
 * The configured headers a catalog request may carry. The catalog is read once with the
 * administrator's credentials and shared by every user, so a header that resolves per user or per
 * request (`{{LIBRECHAT_USER_EMAIL}}`, an OpenID token) is never forwarded, and neither is one that
 * resolves to nothing: an unset environment variable stays a literal `${NAME}` after resolution,
 * which a proxy would receive as a credential. Static values and environment
 * variables are forwarded, as inference and the model list forward them, because a proxy that
 * requires one answers `/models` with an authorization failure otherwise.
 */
function staticCatalogHeaders(headers: Record<string, string> | undefined): Record<string, string> {
  const candidates = Object.fromEntries(
    Object.entries(headers ?? {}).filter(([, value]) => !PER_REQUEST_PLACEHOLDER.test(value)),
  );
  const resolved = resolveHeaders({ headers: candidates, stripUnresolved: true });
  return Object.fromEntries(
    Object.entries(resolved).filter(
      ([, value]) => value !== '' && !UNRESOLVED_ENV_VARIABLE.test(value),
    ),
  );
}

/**
 * The headers of one catalog request: the configured ones, plus the API key as a Bearer token
 * unless a configured `Authorization` header already carries the credential, as the model list does.
 */
export function catalogRequestHeaders(
  apiKey: string,
  headers: Record<string, string>,
): Record<string, string> {
  const hasAuthorization = Object.keys(headers).some(
    (key) => key.toLowerCase() === 'authorization',
  );
  return hasAuthorization ? { ...headers } : { ...headers, Authorization: `Bearer ${apiKey}` };
}

/** The `/models` URL of an API base, with `/models` on the path and any query kept in place. */
function catalogURL(baseURL: string): string {
  const url = new URL(baseURL);
  url.pathname = `${url.pathname.replace(/\/+$/, '')}/models`;
  return url.toString();
}

/**
 * Reads every page of a model catalog, following `links.next` on the provider's own host.
 * A page that cannot be read, a link that leaves the host or revisits a page, or a catalog
 * longer than the endpoint's page limit makes the whole catalog unavailable: serving part of
 * it would hide the controls of every model on the pages not read.
 */
async function readCatalog(
  target: ResolvedTarget,
  deps: ReasoningCapabilityDeps,
): Promise<CatalogModels | undefined> {
  const { baseURL, apiKey, timeoutMs, maxPages, headers } = target;
  /** Null-prototype, because model ids are unrestricted strings: assigning `__proto__` into an
   *  ordinary object would set its prototype instead of creating an entry. */
  const models: CatalogModels = Object.create(null);
  const origin = new URL(baseURL).origin;
  const configuredQuery = new URL(baseURL).searchParams;
  const seen = new Set<string>();
  let url: string | null = catalogURL(baseURL);

  while (url != null) {
    if (seen.has(url) || seen.size >= maxPages) {
      return undefined;
    }
    seen.add(url);
    const parsed = pageSchema.safeParse(await deps.fetchPage({ url, apiKey, timeoutMs, headers }));
    if (!parsed.success) {
      return undefined;
    }
    for (const { id, reasoning } of parsed.data.data) {
      if (isDynamicRouterModel(id)) {
        continue;
      }
      if (reasoning == null) {
        models[id] = { efforts: [] };
        continue;
      }
      const details = reasoningSchema.safeParse(reasoning);
      if (!details.success) {
        continue;
      }
      const reported = details.data.supported_efforts;
      const efforts = reported === null ? UNRESTRICTED_EFFORTS : (reported ?? []);
      models[id] =
        efforts.length === 0
          ? { efforts: [] }
          : { efforts, mandatory: details.data.mandatory === true };
    }
    const next: string | null | undefined = parsed.data.links?.next;
    if (next == null || next === '') {
      return models;
    }
    /** Relative to the page just read, so `?offset=100` and `models?offset=100` keep its path. */
    const nextURL: URL = new URL(next, url);
    if (nextURL.origin !== origin) {
      return undefined;
    }
    /** A next link carries pagination, not the query the administrator configured on the base URL
     *  (a signature, a key): each page needs it, and a parameter the link sets itself wins. */
    for (const [name, value] of configuredQuery) {
      if (!nextURL.searchParams.has(name)) {
        nextURL.searchParams.append(name, value);
      }
    }
    url = nextURL.toString();
  }
  return models;
}

type ResolvedTarget = {
  name: string;
  baseURL: string;
  apiKey: string;
  timeoutMs: number;
  maxPages: number;
  failureTtlMs: number;
  ttlMs: number;
  headers: Record<string, string>;
  /** The base names of the models the endpoint exposes, or `undefined` when it fetches every one. */
  exposedModels: Set<string> | undefined;
};

/** The part of a model name before a routing variant such as `:nitro` or `:free`. */
const baseModelName = (model: string): string => model.split(':')[0];

/**
 * The models an endpoint exposes to its users. An endpoint that fetches its model list exposes
 * every catalog model; one that does not exposes only its configured models, and the capabilities
 * route must not disclose the rest of what a proxy's catalog lists to its users.
 */
function exposedModelNames(endpoint: TEndpoint): Set<string> | undefined {
  if (endpoint.models?.fetch === true) {
    return undefined;
  }
  const names = new Set<string>();
  for (const model of endpoint.models?.default ?? []) {
    const configured = typeof model === 'string' ? model : model.name;
    names.add(configured);
    /** A variant (`vendor/model:free`) is catalogued under its base, so the base entry is exposed as
     *  a fallback for it. Sibling variants (`vendor/model:nitro`) are not: they are not configured. */
    names.add(baseModelName(configured));
  }
  return names;
}

/** The catalog's models narrowed to those the endpoint exposes. */
function exposedCapabilities(
  models: CatalogModels,
  exposed: Set<string> | undefined,
): CatalogModels {
  if (exposed == null) {
    return models;
  }
  return Object.fromEntries(Object.entries(models).filter(([id]) => exposed.has(id)));
}

/**
 * Whether this endpoint's requests depend on who is asking: its model list is fetched with
 * `models.userIdQuery`, or a configured header resolves to the user's identity or credentials. The
 * shared catalog is read with the administrator's key and no user, so for such an endpoint it could
 * be refused, or describe a different user's models, whether or not the list is fetched. Headers
 * that only carry request ids do not count.
 */
function hasUserScopedModelList(endpoint: TEndpoint): boolean {
  if (endpoint.models?.userIdQuery === true) {
    return true;
  }
  return Object.values(endpoint.headers ?? {}).some((value) =>
    PER_REQUEST_PLACEHOLDER.test(value.replace(REQUEST_METADATA_PLACEHOLDER, '')),
  );
}

/**
 * The endpoint's catalog target, or nothing when it does not use the OpenRouter catalog (see
 * {@link usesOpenRouterCatalog}) or has no administrator credentials. A `directEndpoint` is skipped:
 * its base URL is the exact inference URL, so `/models` cannot be derived from it. So is an endpoint
 * that pins its model through `addParams.model`: every request goes to that one model, so the efforts
 * of the model a user selected say nothing about what is sent, and narrowing by them would be wrong.
 * And so is an endpoint whose model list is filtered per user, by `models.userIdQuery` or by headers
 * that resolve per request: this catalog is read once with the administrator's key and shared, so it
 * could expose models the requesting user is not offered.
 */
function resolveTarget(endpoint: TEndpoint): ResolvedTarget | undefined {
  if (
    endpoint.directEndpoint === true ||
    typeof endpoint.addParams?.model === 'string' ||
    hasUserScopedModelList(endpoint)
  ) {
    return undefined;
  }
  const name = normalizeEndpointName(endpoint.name);
  const baseURL = resolveConfigSecret(endpoint.baseURL) ?? '';
  const apiKey = resolveConfigSecret(endpoint.apiKey) ?? '';
  if (
    !name ||
    isUserProvided(baseURL) ||
    isUserProvided(apiKey) ||
    !usesOpenRouterCatalog(endpoint, baseURL)
  ) {
    return undefined;
  }
  return {
    name,
    baseURL,
    apiKey,
    timeoutMs: endpoint.customParams?.reasoningCatalogTimeoutMs ?? DEFAULT_CATALOG_TIMEOUT_MS,
    maxPages: endpoint.customParams?.reasoningCatalogMaxPages ?? DEFAULT_CATALOG_MAX_PAGES,
    failureTtlMs:
      endpoint.customParams?.reasoningCatalogFailureTtlMs ?? DEFAULT_CATALOG_FAILURE_TTL_MS,
    ttlMs: endpoint.customParams?.reasoningCatalogTtlMs ?? DEFAULT_CATALOG_TTL_MS,
    headers: staticCatalogHeaders(endpoint.headers),
    exposedModels: exposedModelNames(endpoint),
  };
}

/** Where a recent failed lookup is remembered, beside the entry a successful one fills. */
const failureKey = (key: string): string => `${key}:failed`;

/**
 * Lookups in progress, by base URL and key. Held at module scope so concurrent requests,
 * from any caller and any user, share one catalog walk instead of each issuing their own
 * when the cache is cold or has just expired. An entry is removed once it settles; a failure is
 * then remembered for `reasoningCatalogFailureTtlMs` so it is retried after that, not per call.
 */
const inFlight = new Map<string, Promise<CatalogEntry | undefined>>();

/** One cached, shared lookup per base URL and key; `undefined` when the catalog is unavailable. */
function resolveCatalog(
  target: ResolvedTarget,
  deps: ReasoningCapabilityDeps,
): Promise<CatalogEntry | undefined> {
  const key = cacheKey(target);
  const existing = inFlight.get(key);
  if (existing != null) {
    return existing;
  }
  const lookup = (async () => {
    const cached = (await deps.cache.get(key)) as CatalogEntry | undefined;
    if (cached != null && cached.expiresAt > Date.now()) {
      return cached;
    }
    /** A recent failure is not retried: the middleware and the endpoint initializer of one
     *  request, and the requests that follow it, would otherwise each wait out the timeout. */
    if (target.failureTtlMs > 0 && (await deps.cache.get(failureKey(key))) != null) {
      return undefined;
    }
    let models: CatalogModels | undefined;
    try {
      models = await readCatalog(target, deps);
    } catch (error) {
      logger.warn('[reasoning] Failed to load the OpenRouter model catalog', {
        origin: safeOrigin(target.baseURL),
        error: error instanceof Error ? error.name : 'unknown',
      });
    }
    if (models != null) {
      const entry: CatalogEntry = { models, expiresAt: Date.now() + target.ttlMs };
      await deps.cache.set(key, entry, target.ttlMs);
      return entry;
    }
    if (target.failureTtlMs > 0) {
      await deps.cache.set(failureKey(key), true, target.failureTtlMs);
    }
    return undefined;
  })().finally(() => inFlight.delete(key));
  inFlight.set(key, lookup);
  return lookup;
}

/**
 * Resolves which reasoning efforts each OpenRouter model accepts, for the
 * endpoints configured against OpenRouter. The catalog is fetched with the
 * administrator's key, never a user's, and is the same for every caller, so it
 * is cached by base URL and key and shared across users. A catalog that cannot
 * be fetched or read is reported in `unavailable`, never as an endpoint without
 * reasoning, and is not cached, so the next request retries. Each endpoint's result holds only the
 * models it exposes. `only` limits the result to one endpoint, so one endpoint's outage does not hide
 * another's data. `expiresAt` is when the earliest catalog read expires.
 */
export async function loadReasoningCapabilities(
  customEndpoints: TEndpoint[] | undefined,
  deps: ReasoningCapabilityDeps,
  only?: string,
): Promise<ReasoningCapabilityResult> {
  const wanted = only == null ? undefined : normalizeEndpointName(only);
  const targets = (customEndpoints ?? []).flatMap((endpoint) => {
    const target = resolveTarget(endpoint);
    return target != null && (wanted == null || target.name === wanted) ? [target] : [];
  });
  const catalogs = await Promise.all(targets.map((target) => resolveCatalog(target, deps)));

  const result: ReasoningCapabilityResult = {
    capabilities: Object.create(null) as ReasoningCapabilityResult['capabilities'],
    unavailable: [],
  };
  targets.forEach(({ name, exposedModels }, index) => {
    const entry = catalogs[index];
    if (entry == null) {
      result.unavailable.push(name);
      return;
    }
    result.capabilities[name] = exposedCapabilities(entry.models, exposedModels);
    result.expiresAt = Math.min(result.expiresAt ?? entry.expiresAt, entry.expiresAt);
  });
  return result;
}

/**
 * Drops a stored `reasoning_effort` the selected OpenRouter model does not accept, so a
 * conversation or agent saved on another model cannot send a request the provider rejects.
 * Auto sends no effort and is kept, as is everything while the catalog is unavailable, and
 * an effort the administrator defined for the endpoint, including one set through
 * `addParams`, which replaces the stored value outright. An endpoint that pins its model
 * through `addParams.model` is not checked (see {@link resolveTarget}). Only a request that
 * stores an effort on an eligible OpenRouter endpoint reads the catalog.
 */
export async function withSupportedEffort<T extends object>(
  modelOptions: T,
  endpoint: TEndpoint,
  deps: ReasoningCapabilityDeps,
): Promise<T> {
  const { model, reasoning_effort: effort } = modelOptions as {
    model?: unknown;
    reasoning_effort?: unknown;
  };
  if (
    typeof effort !== 'string' ||
    effort === '' ||
    typeof model !== 'string' ||
    endpoint.addParams?.reasoning_effort !== undefined
  ) {
    return modelOptions;
  }
  const target = resolveTarget(endpoint);
  if (target == null) {
    return modelOptions;
  }
  const entry = await resolveCatalog(target, deps);
  const modelReasoning = effectiveModelReasoning(
    getModelReasoning(
      entry == null ? undefined : { [target.name]: entry.models },
      target.name,
      model,
    ),
    endpoint.customParams?.paramDefinitions,
  );
  if (isOpenRouterEffortSupported(effort, modelReasoning)) {
    return modelOptions;
  }
  const rest = { ...modelOptions } as T & { reasoning_effort?: unknown };
  delete rest.reasoning_effort;
  return rest;
}

const reasoningCache = (): ReasoningCapabilityCache =>
  standardCache(CacheKeys.REASONING_CAPABILITIES, Time.ONE_HOUR);

async function fetchPage({
  url,
  apiKey,
  timeoutMs,
  headers,
}: {
  url: string;
  apiKey: string;
  timeoutMs: number;
  headers: Record<string, string>;
}): Promise<unknown> {
  const options = applyAxiosProxyConfig(
    { headers: catalogRequestHeaders(apiKey, headers), timeout: timeoutMs },
    url,
  );
  return (await axios.get(url, options)).data;
}

/** The dependencies the server uses; callers in `/api` and the endpoint initializers share them. */
export function getReasoningCapabilityDeps(): ReasoningCapabilityDeps {
  return { fetchPage, cache: reasoningCache() };
}

/**
 * Per-model reasoning efforts for the OpenRouter endpoints in an app config, or for the one
 * named by `endpoint`, wired to the real catalog fetch and cache.
 */
export function getReasoningCapabilities(
  appConfig?: AppConfig,
  endpoint?: string,
): Promise<ReasoningCapabilityResult> {
  return loadReasoningCapabilities(
    (appConfig?.endpoints?.[EModelEndpoint.custom] ?? []) as TEndpoint[],
    getReasoningCapabilityDeps(),
    endpoint,
  );
}