Skip to content

🪢 Open source agent evals & observability: Trace, evaluate, and improve LLM applications with one open platform.

19.5 KB
Raw Blame
/* eslint-disable @typescript-eslint/no-use-before-define */
import { type Redis, type Cluster } from "ioredis";
import { type z } from "zod";
import { RateLimiterRedis, RateLimiterRes } from "rate-limiter-flexible";
import { env } from "@/src/env.mjs";
import {
  InvalidRequestError,
  type RateLimitResult,
  type RateLimitResource,
  type RateLimitConfig,
  type Plan,
} from "@langfuse/shared";
import {
  ClickHouseResourceError,
  recordIncrement,
  type ApiAccessScopeWithOptionalApiKeyId,
  logger,
  createNewRedisInstance,
  redisQueueRetryOptions,
} from "@langfuse/shared/src/server";
import { env as sharedEnv } from "@langfuse/shared/src/env";
import { type NextApiResponse } from "next";
import {
  createStructuredPublicApiRateLimitError,
  sendStructuredPublicApiErrorResponse,
  type PublicApiErrorContract,
} from "./structuredPublicApiErrorContract";
import { type RateLimitUpgradePath } from "@/src/features/public-api/server/rateLimitUpgradePaths";

export const RATE_LIMIT_REDIS_KEY_PREFIX = "rate-limit";

export type RateLimitResponseOptions = {
  errorContract?: PublicApiErrorContract | undefined;
  upgradePath?: RateLimitUpgradePath | undefined;
};

export type RateLimitResponseOptionsInput =
  | PublicApiErrorContract
  | RateLimitResponseOptions;

const normalizeRateLimitResponseOptions = (
  options?: RateLimitResponseOptionsInput,
): RateLimitResponseOptions => {
  return typeof options === "string"
    ? { errorContract: options }
    : (options ?? {});
};

// Business Logic
// - rate limit strategy is based on org-id, org plan, and resources. Rate limits are applied in buckets of minutes.
// - rate limits are not applied for self hosters and are also not applied when Redis is not available
// - infos for rate-limits are taken from the API access scope. Info for this scope is stored alongside API Keys in Redis for efficient access.
// - isRateLimited returns false for self-hosters
// - sendRestResponseIfLimited sends a 429 response with headers if the rate limit is exceeded. Return this from the route handler.
export class RateLimitService {
  private static redis: Redis | Cluster | null;
  private static instance: RateLimitService | null = null;

  public static getInstance(redis: Redis | null = null) {
    if (!RateLimitService.instance) {
      RateLimitService.redis =
        redis ??
        createNewRedisInstance({
          keyPrefix: sharedEnv.REDIS_KEY_PREFIX ?? undefined, // For multi-tenant Redis isolation
          enableAutoPipelining: false, // This may help avoid https://github.com/redis/ioredis/issues/1931
          enableOfflineQueue: false,
          lazyConnect: true, // Connect when first command is sent
          ...redisQueueRetryOptions,
        });
      RateLimitService.instance = new RateLimitService();
    }
    return RateLimitService.instance;
  }

  public static shutdown() {
    if (RateLimitService.redis && RateLimitService.redis.status !== "end") {
      RateLimitService.redis.disconnect();
    }
    RateLimitService.redis = null;
    RateLimitService.instance = null;
  }

  /** `points` is what this request costs, e.g. its byte size for byte-based resources. */
  async rateLimitRequest(
    scope: ApiAccessScopeWithOptionalApiKeyId,
    resource: z.infer<typeof RateLimitResource>,
    points = 1,
  ) {
    // if cloud config is not present, we don't apply rate limits and just return
    if (!env.NEXT_PUBLIC_LANGFUSE_CLOUD_REGION) {
      return new RateLimitHelper(undefined);
    }

    if (env.LANGFUSE_RATE_LIMITS_ENABLED === "false") {
      return new RateLimitHelper(undefined);
    }

    if (!RateLimitService.redis) {
      logger.warn("Rate limiting not available without Redis");
      return new RateLimitHelper(undefined);
    }

    return new RateLimitHelper(
      await this.checkRateLimit(scope, resource, points),
    );
  }

  async checkRateLimit(
    scope: ApiAccessScopeWithOptionalApiKeyId,
    resource: z.infer<typeof RateLimitResource>,
    points = 1,
  ) {
    const effectiveConfig = getRateLimitConfig(scope, resource);

    // returning early if no rate limit is set
    if (
      !effectiveConfig ||
      !effectiveConfig.points ||
      !effectiveConfig.durationInSec
    ) {
      return;
    }

    // Connect Redis if not initialized
    if (RateLimitService?.redis?.status !== "ready") {
      try {
        await RateLimitService?.redis?.connect();
      } catch (_err) {
        // Do nothing here. We will fail open if Redis is not available.
      }
    }

    const rateLimiter = new RateLimiterRedis({
      // Basic options
      points: effectiveConfig.points, // Number of points
      duration: effectiveConfig.durationInSec, // Per second(s)

      keyPrefix: this.rateLimitPrefix(resource), // must be unique for limiters with different purpose
      storeClient: RateLimitService.redis,
      rejectIfRedisNotReady: true,
    });

    let res: RateLimitResult | undefined = undefined;
    try {
      // orgId used as key for different resources
      const libRes = await rateLimiter.consume(scope.orgId, points);
      res = {
        resource,
        scope,
        points: effectiveConfig.points,
        remainingPoints: libRes.remainingPoints,
        msBeforeNext: libRes.msBeforeNext,
        consumedPoints: libRes.consumedPoints,
        isFirstInDuration: libRes.isFirstInDuration,
      };
    } catch (err) {
      if (err instanceof RateLimiterRes) {
        // No points available or key is blocked
        res = {
          resource,
          scope,
          points: effectiveConfig.points,
          remainingPoints: err.remainingPoints,
          msBeforeNext: err.msBeforeNext,
          consumedPoints: err.consumedPoints,
          isFirstInDuration: err.isFirstInDuration,
        };
      } else {
        // Some other error occurred, return undefined to fail open
        logger.error("Internal Rate limit error", err);
        return undefined;
      }
    }

    // Library floors remainingPoints at 0; reject only when consumedPoints > points.
    if (res.consumedPoints > res.points) {
      recordIncrement("langfuse.rate_limit.exceeded", 1, {
        orgId: scope.orgId,
        plan: scope.plan,
        resource: resource,
      });
    }

    return res;
  }

  rateLimitPrefix(resource: string) {
    return `${RATE_LIMIT_REDIS_KEY_PREFIX}:${resource}`;
  }

  async withMcpQueryCooldown<T>(projectId: string, query: () => Promise<T>) {
    const redis = RateLimitService.redis;
    if (
      !env.NEXT_PUBLIC_LANGFUSE_CLOUD_REGION ||
      env.LANGFUSE_RATE_LIMITS_ENABLED === "false" ||
      !redis
    ) {
      return query();
    }

    const key = `${this.rateLimitPrefix("mcp-query-cooldown")}:${projectId}`;
    const cooldownError = (seconds: number) =>
      new InvalidRequestError(
        `MCP observation searches for this project are paused after a query exceeded resource limits. Retry in ${Math.max(1, seconds)}s with a shorter date range or more specific filters, and run searches sequentially.`,
      );
    const onRedisError = (error: unknown) => {
      logger.warn("MCP query cooldown Redis operation failed", error);
      return null;
    };
    if (redis.status === "wait") await redis.connect().catch(onRedisError);
    if (redis.status !== "ready") return query();
    const ttl = await redis.ttl(key).catch(onRedisError);
    if (ttl !== null && ttl >= 0) throw cooldownError(ttl);

    try {
      return await query();
    } catch (error) {
      if (ClickHouseResourceError.is(error)) {
        const blocked = await redis.set(key, "1", "EX", 30).catch(onRedisError);
        if (blocked) {
          logger.warn("MCP ClickHouse resource limit exceeded", {
            errorType: error.errorType,
            tags: error.tags,
          });
          throw cooldownError(30);
        }
      }
      throw error;
    }
  }
}

class RateLimitHelper {
  res: RateLimitResult | undefined;

  constructor(res: RateLimitResult | undefined) {
    this.res = res;
  }

  isRateLimited() {
    return this.res ? this.res.consumedPoints > this.res.points : false;
  }

  sendRestResponseIfLimited(
    nextResponse: NextApiResponse,
    options?: RateLimitResponseOptionsInput,
  ) {
    if (!this.res || !this.isRateLimited()) {
      logger.error("Trying to send rate limit response without being limited.");
      throw new Error(
        "Trying to send rate limit response without being limited.",
      );
    }
    return sendRateLimitResponse(nextResponse, this.res, options);
  }
}

export const sendRateLimitResponse = (
  res: NextApiResponse,
  rateLimitRes: RateLimitResult,
  options?: RateLimitResponseOptionsInput,
) => {
  const responseOptions = normalizeRateLimitResponseOptions(options);
  const httpHeader = createHttpHeaderFromRateLimit(rateLimitRes);

  for (const [header, value] of Object.entries(httpHeader)) {
    res.setHeader(header, value);
  }

  return sendStructuredPublicApiErrorResponse(
    res,
    createStructuredPublicApiRateLimitError(rateLimitRes, responseOptions),
  );
};

export const createHttpHeaderFromRateLimit = (res: RateLimitResult) => {
  return {
    "Retry-After": Math.ceil(res.msBeforeNext / 1000),
    "X-RateLimit-Limit": res.points,
    "X-RateLimit-Remaining": res.remainingPoints,
    "X-RateLimit-Reset": Math.ceil((Date.now() + res.msBeforeNext) / 1000),
  };
};

const getRateLimitConfig = (
  scope: ApiAccessScopeWithOptionalApiKeyId,
  resource: z.infer<typeof RateLimitResource>,
) => {
  const planBasedConfig = getPlanBasedRateLimitConfig(scope.plan, resource);
  const customConfig = scope.rateLimitOverrides?.find(
    (config) => config.resource === resource,
  );

  return customConfig || planBasedConfig;
};

const MiB = 1024 * 1024;
const GiB = 1024 * MiB;

const getPlanBasedRateLimitConfig = (
  plan: Plan,
  resource: z.infer<typeof RateLimitResource>,
): z.infer<typeof RateLimitConfig> => {
  switch (plan) {
    case "oss":
    case "self-hosted:pro":
    case "self-hosted:enterprise":
    case "self-hosted:gov":
      return {
        resource,
        points: null,
        durationInSec: null,
      };
    case "cloud:hobby":
      switch (resource) {
        case "ingestion":
          return {
            resource: "ingestion",
            points: 1000,
            durationInSec: 60,
          };
        case "media-upload":
          return {
            resource: "media-upload",
            points: 1000,
            durationInSec: 60,
          };
        case "legacy-ingestion":
          return {
            resource: "legacy-ingestion",
            points: 100,
            durationInSec: 60,
          };
        case "prompts":
          return {
            resource: "prompts",
            points: null,
            durationInSec: null,
          };
        case "public-api":
          return {
            resource: "public-api",
            points: 30,
            durationInSec: 60,
          };
        case "public-api-legacy":
          return {
            resource: "public-api-legacy",
            points: 2,
            durationInSec: 60, // 2 requests per minute (nominal 120/hour)
          };
        case "datasets":
          return {
            resource: "datasets",
            points: 100,
            durationInSec: 60,
          };
        case "annotation-queues":
          return {
            resource: "annotation-queues",
            points: 100,
            durationInSec: 60,
          };
        case "public-api-metrics":
          return {
            resource: "public-api-metrics",
            points: 100,
            durationInSec: 86400, // 100 requests per day
          };
        case "public-api-v2-metrics":
          return {
            resource,
            points: 100,
            durationInSec: 86400, // 100 requests per day
          };
        case "public-api-daily-metrics-legacy":
          return {
            resource: "public-api-daily-metrics-legacy",
            points: 10,
            durationInSec: 86400, // 10 requests per day
          };
        case "trace-delete":
          return {
            resource: "trace-delete",
            points: 50,
            durationInSec: 86400, // 50 requests per day
          };
        case "score-delete":
          return {
            resource: "score-delete",
            points: 50,
            durationInSec: 86400, // 50 requests per day
          };
        case "in-app-agent-run":
          return {
            resource: "in-app-agent-run",
            points: 100,
            durationInSec: 86400,
          };
        case "feedback":
          return {
            resource: "feedback",
            points: 20,
            durationInSec: 86400, // 20 submissions per day
          };
        case "ai-gateway-requests":
          return { resource, points: 100, durationInSec: 60 };
        case "ai-gateway-request-bytes":
          return { resource, points: 64 * MiB, durationInSec: 60 };
        default:
          const exhaustiveCheck: never = resource;
          throw new Error(`Unhandled resource case: ${exhaustiveCheck}`);
      }
    case "cloud:core":
      // TEMPORARY: Expanded selected core plan rate limits to pro limits to enable legacy pro -> core migration
      // Original core limits (commented out):
      // ingestion: 4000, public-api: 100, datasets: 200, public-api-metrics: 200, public-api-daily-metrics-legacy: 20
      switch (resource) {
        case "ingestion":
          return {
            resource: "ingestion",
            // points: 4000, // original core limit
            points: 20_000, // temporary: using pro limit
            durationInSec: 60,
          };
        case "media-upload":
          return {
            resource: "media-upload",
            points: 4_000,
            durationInSec: 60,
          };
        case "legacy-ingestion":
          return {
            resource: "legacy-ingestion",
            points: 400,
            durationInSec: 60,
          };
        case "prompts":
          return {
            resource: "prompts",
            points: null,
            durationInSec: null,
          };
        case "public-api":
          return {
            resource: "public-api",
            // points: 100, // original core limit
            points: 1000, // temporary: using pro limit
            durationInSec: 60,
          };
        case "public-api-legacy":
          return {
            resource: "public-api-legacy",
            points: 7,
            durationInSec: 60, // 7 requests per minute (nominal 420/hour)
          };
        case "datasets":
          return {
            resource: "datasets",
            // points: 200, // original core limit
            points: 1000, // temporary: using pro limit
            durationInSec: 60,
          };
        case "annotation-queues":
          return {
            resource: "annotation-queues",
            points: 1000,
            durationInSec: 60,
          };
        case "public-api-metrics":
          return {
            resource: "public-api-metrics",
            // points: 200, // original core limit
            points: 2000, // temporary: using pro limit
            durationInSec: 86400, // 2000 requests per day
          };
        case "public-api-v2-metrics":
          return {
            resource,
            points: 100,
            durationInSec: 3600, // 100 requests per hour
          };
        case "public-api-daily-metrics-legacy":
          return {
            resource: "public-api-daily-metrics-legacy",
            // points: 20, // original core limit
            points: 200, // temporary: using pro limit
            durationInSec: 86400, // 200 requests per day
          };
        case "trace-delete":
          return {
            resource: "trace-delete",
            points: 200,
            durationInSec: 86400, // 200 requests per day
          };
        case "score-delete":
          return {
            resource: "score-delete",
            points: 200,
            durationInSec: 86400, // 200 requests per day
          };
        case "in-app-agent-run":
          return {
            resource: "in-app-agent-run",
            points: 1000,
            durationInSec: 86400,
          };
        case "feedback":
          return {
            resource: "feedback",
            points: 20,
            durationInSec: 86400, // 20 submissions per day
          };
        case "ai-gateway-requests":
          return { resource, points: 500, durationInSec: 60 };
        case "ai-gateway-request-bytes":
          return { resource, points: 256 * MiB, durationInSec: 60 };
        default:
          const exhaustiveCheck: never = resource;
          throw new Error(`Unhandled resource case: ${exhaustiveCheck}`);
      }
    case "cloud:pro":
    case "cloud:team":
    case "cloud:enterprise":
      switch (resource) {
        case "ingestion":
          return {
            resource: "ingestion",
            points: 20_000,
            durationInSec: 60,
          };
        case "media-upload":
          return {
            resource: "media-upload",
            points: 20_000,
            durationInSec: 60,
          };
        case "legacy-ingestion":
          return {
            resource: "legacy-ingestion",
            points: 400,
            durationInSec: 60,
          };
        case "prompts":
          return {
            resource: "prompts",
            points: null,
            durationInSec: null,
          };
        case "public-api":
          return {
            resource: "public-api",
            points: 1000,
            durationInSec: 60,
          };
        case "public-api-legacy":
          return {
            resource: "public-api-legacy",
            points: 20,
            durationInSec: 60, // 20 requests per minute (nominal 1,200/hour)
          };
        case "datasets":
          return {
            resource: "datasets",
            points: 1000,
            durationInSec: 60,
          };
        case "annotation-queues":
          return {
            resource: "annotation-queues",
            points: 1000,
            durationInSec: 60,
          };
        case "public-api-metrics":
          return {
            resource: "public-api-metrics",
            points: 2000,
            durationInSec: 86400, // 2000 requests per day
          };
        case "public-api-v2-metrics":
          return {
            resource,
            points: 500,
            durationInSec: 3600, // 500 requests per hour
          };
        case "public-api-daily-metrics-legacy":
          return {
            resource: "public-api-daily-metrics-legacy",
            points: 200,
            durationInSec: 86400, // 200 requests per day
          };
        case "trace-delete":
          return {
            resource: "trace-delete",
            points: 1000,
            durationInSec: 86400, // 1000 requests per day
          };
        case "score-delete":
          return {
            resource: "score-delete",
            points: 1000,
            durationInSec: 86400, // 1000 requests per day
          };
        case "in-app-agent-run":
          return {
            resource: "in-app-agent-run",
            points: 1000,
            durationInSec: 86400,
          };
        case "feedback":
          return {
            resource: "feedback",
            points: 20,
            durationInSec: 86400, // 20 submissions per day
          };
        case "ai-gateway-requests":
          return { resource, points: 10_000, durationInSec: 60 };
        case "ai-gateway-request-bytes":
          return { resource, points: 5 * GiB, durationInSec: 60 };
        default:
          const exhaustiveCheck: never = resource;
          throw new Error(`Unhandled resource case: ${exhaustiveCheck}`);
      }
    default:
      const exhaustiveCheck: never = plan;
      throw new Error(`Unhandled plan case: ${exhaustiveCheck}`);
  }
};