🪢 Open source agent evals & observability: Trace, evaluate, and improve LLM applications with one open platform.
TypeScript analyticsautogenevaluationlangchainlarge-language-modelsllama-indexllmllm-evaluationllm-observabilityllmopsmonitoringobservabilityopen-sourceopenaiplaygroundprompt-engineeringprompt-managementself-hostedycombinator
/* eslint-disable @typescript-eslint/no-use-before-define */
import { type Redis, type Cluster } from "ioredis";
import { type z } from "zod";
import { RateLimiterRedis, RateLimiterRes } from "rate-limiter-flexible";
import { env } from "@/src/env.mjs";
import {
InvalidRequestError,
type RateLimitResult,
type RateLimitResource,
type RateLimitConfig,
type Plan,
} from "@langfuse/shared";
import {
ClickHouseResourceError,
recordIncrement,
type ApiAccessScopeWithOptionalApiKeyId,
logger,
createNewRedisInstance,
redisQueueRetryOptions,
} from "@langfuse/shared/src/server";
import { env as sharedEnv } from "@langfuse/shared/src/env";
import { type NextApiResponse } from "next";
import {
createStructuredPublicApiRateLimitError,
sendStructuredPublicApiErrorResponse,
type PublicApiErrorContract,
} from "./structuredPublicApiErrorContract";
import { type RateLimitUpgradePath } from "@/src/features/public-api/server/rateLimitUpgradePaths";
export const RATE_LIMIT_REDIS_KEY_PREFIX = "rate-limit";
export type RateLimitResponseOptions = {
errorContract?: PublicApiErrorContract | undefined;
upgradePath?: RateLimitUpgradePath | undefined;
};
export type RateLimitResponseOptionsInput =
| PublicApiErrorContract
| RateLimitResponseOptions;
const normalizeRateLimitResponseOptions = (
options?: RateLimitResponseOptionsInput,
): RateLimitResponseOptions => {
return typeof options === "string"
? { errorContract: options }
: (options ?? {});
};
// Business Logic
// - rate limit strategy is based on org-id, org plan, and resources. Rate limits are applied in buckets of minutes.
// - rate limits are not applied for self hosters and are also not applied when Redis is not available
// - infos for rate-limits are taken from the API access scope. Info for this scope is stored alongside API Keys in Redis for efficient access.
// - isRateLimited returns false for self-hosters
// - sendRestResponseIfLimited sends a 429 response with headers if the rate limit is exceeded. Return this from the route handler.
export class RateLimitService {
private static redis: Redis | Cluster | null;
private static instance: RateLimitService | null = null;
public static getInstance(redis: Redis | null = null) {
if (!RateLimitService.instance) {
RateLimitService.redis =
redis ??
createNewRedisInstance({
keyPrefix: sharedEnv.REDIS_KEY_PREFIX ?? undefined, // For multi-tenant Redis isolation
enableAutoPipelining: false, // This may help avoid https://github.com/redis/ioredis/issues/1931
enableOfflineQueue: false,
lazyConnect: true, // Connect when first command is sent
...redisQueueRetryOptions,
});
RateLimitService.instance = new RateLimitService();
}
return RateLimitService.instance;
}
public static shutdown() {
if (RateLimitService.redis && RateLimitService.redis.status !== "end") {
RateLimitService.redis.disconnect();
}
RateLimitService.redis = null;
RateLimitService.instance = null;
}
/** `points` is what this request costs, e.g. its byte size for byte-based resources. */
async rateLimitRequest(
scope: ApiAccessScopeWithOptionalApiKeyId,
resource: z.infer<typeof RateLimitResource>,
points = 1,
) {
// if cloud config is not present, we don't apply rate limits and just return
if (!env.NEXT_PUBLIC_LANGFUSE_CLOUD_REGION) {
return new RateLimitHelper(undefined);
}
if (env.LANGFUSE_RATE_LIMITS_ENABLED === "false") {
return new RateLimitHelper(undefined);
}
if (!RateLimitService.redis) {
logger.warn("Rate limiting not available without Redis");
return new RateLimitHelper(undefined);
}
return new RateLimitHelper(
await this.checkRateLimit(scope, resource, points),
);
}
async checkRateLimit(
scope: ApiAccessScopeWithOptionalApiKeyId,
resource: z.infer<typeof RateLimitResource>,
points = 1,
) {
const effectiveConfig = getRateLimitConfig(scope, resource);
// returning early if no rate limit is set
if (
!effectiveConfig ||
!effectiveConfig.points ||
!effectiveConfig.durationInSec
) {
return;
}
// Connect Redis if not initialized
if (RateLimitService?.redis?.status !== "ready") {
try {
await RateLimitService?.redis?.connect();
} catch (_err) {
// Do nothing here. We will fail open if Redis is not available.
}
}
const rateLimiter = new RateLimiterRedis({
// Basic options
points: effectiveConfig.points, // Number of points
duration: effectiveConfig.durationInSec, // Per second(s)
keyPrefix: this.rateLimitPrefix(resource), // must be unique for limiters with different purpose
storeClient: RateLimitService.redis,
rejectIfRedisNotReady: true,
});
let res: RateLimitResult | undefined = undefined;
try {
// orgId used as key for different resources
const libRes = await rateLimiter.consume(scope.orgId, points);
res = {
resource,
scope,
points: effectiveConfig.points,
remainingPoints: libRes.remainingPoints,
msBeforeNext: libRes.msBeforeNext,
consumedPoints: libRes.consumedPoints,
isFirstInDuration: libRes.isFirstInDuration,
};
} catch (err) {
if (err instanceof RateLimiterRes) {
// No points available or key is blocked
res = {
resource,
scope,
points: effectiveConfig.points,
remainingPoints: err.remainingPoints,
msBeforeNext: err.msBeforeNext,
consumedPoints: err.consumedPoints,
isFirstInDuration: err.isFirstInDuration,
};
} else {
// Some other error occurred, return undefined to fail open
logger.error("Internal Rate limit error", err);
return undefined;
}
}
// Library floors remainingPoints at 0; reject only when consumedPoints > points.
if (res.consumedPoints > res.points) {
recordIncrement("langfuse.rate_limit.exceeded", 1, {
orgId: scope.orgId,
plan: scope.plan,
resource: resource,
});
}
return res;
}
rateLimitPrefix(resource: string) {
return `${RATE_LIMIT_REDIS_KEY_PREFIX}:${resource}`;
}
async withMcpQueryCooldown<T>(projectId: string, query: () => Promise<T>) {
const redis = RateLimitService.redis;
if (
!env.NEXT_PUBLIC_LANGFUSE_CLOUD_REGION ||
env.LANGFUSE_RATE_LIMITS_ENABLED === "false" ||
!redis
) {
return query();
}
const key = `${this.rateLimitPrefix("mcp-query-cooldown")}:${projectId}`;
const cooldownError = (seconds: number) =>
new InvalidRequestError(
`MCP observation searches for this project are paused after a query exceeded resource limits. Retry in ${Math.max(1, seconds)}s with a shorter date range or more specific filters, and run searches sequentially.`,
);
const onRedisError = (error: unknown) => {
logger.warn("MCP query cooldown Redis operation failed", error);
return null;
};
if (redis.status === "wait") await redis.connect().catch(onRedisError);
if (redis.status !== "ready") return query();
const ttl = await redis.ttl(key).catch(onRedisError);
if (ttl !== null && ttl >= 0) throw cooldownError(ttl);
try {
return await query();
} catch (error) {
if (ClickHouseResourceError.is(error)) {
const blocked = await redis.set(key, "1", "EX", 30).catch(onRedisError);
if (blocked) {
logger.warn("MCP ClickHouse resource limit exceeded", {
errorType: error.errorType,
tags: error.tags,
});
throw cooldownError(30);
}
}
throw error;
}
}
}
class RateLimitHelper {
res: RateLimitResult | undefined;
constructor(res: RateLimitResult | undefined) {
this.res = res;
}
isRateLimited() {
return this.res ? this.res.consumedPoints > this.res.points : false;
}
sendRestResponseIfLimited(
nextResponse: NextApiResponse,
options?: RateLimitResponseOptionsInput,
) {
if (!this.res || !this.isRateLimited()) {
logger.error("Trying to send rate limit response without being limited.");
throw new Error(
"Trying to send rate limit response without being limited.",
);
}
return sendRateLimitResponse(nextResponse, this.res, options);
}
}
export const sendRateLimitResponse = (
res: NextApiResponse,
rateLimitRes: RateLimitResult,
options?: RateLimitResponseOptionsInput,
) => {
const responseOptions = normalizeRateLimitResponseOptions(options);
const httpHeader = createHttpHeaderFromRateLimit(rateLimitRes);
for (const [header, value] of Object.entries(httpHeader)) {
res.setHeader(header, value);
}
return sendStructuredPublicApiErrorResponse(
res,
createStructuredPublicApiRateLimitError(rateLimitRes, responseOptions),
);
};
export const createHttpHeaderFromRateLimit = (res: RateLimitResult) => {
return {
"Retry-After": Math.ceil(res.msBeforeNext / 1000),
"X-RateLimit-Limit": res.points,
"X-RateLimit-Remaining": res.remainingPoints,
"X-RateLimit-Reset": Math.ceil((Date.now() + res.msBeforeNext) / 1000),
};
};
const getRateLimitConfig = (
scope: ApiAccessScopeWithOptionalApiKeyId,
resource: z.infer<typeof RateLimitResource>,
) => {
const planBasedConfig = getPlanBasedRateLimitConfig(scope.plan, resource);
const customConfig = scope.rateLimitOverrides?.find(
(config) => config.resource === resource,
);
return customConfig || planBasedConfig;
};
const MiB = 1024 * 1024;
const GiB = 1024 * MiB;
const getPlanBasedRateLimitConfig = (
plan: Plan,
resource: z.infer<typeof RateLimitResource>,
): z.infer<typeof RateLimitConfig> => {
switch (plan) {
case "oss":
case "self-hosted:pro":
case "self-hosted:enterprise":
case "self-hosted:gov":
return {
resource,
points: null,
durationInSec: null,
};
case "cloud:hobby":
switch (resource) {
case "ingestion":
return {
resource: "ingestion",
points: 1000,
durationInSec: 60,
};
case "media-upload":
return {
resource: "media-upload",
points: 1000,
durationInSec: 60,
};
case "legacy-ingestion":
return {
resource: "legacy-ingestion",
points: 100,
durationInSec: 60,
};
case "prompts":
return {
resource: "prompts",
points: null,
durationInSec: null,
};
case "public-api":
return {
resource: "public-api",
points: 30,
durationInSec: 60,
};
case "public-api-legacy":
return {
resource: "public-api-legacy",
points: 2,
durationInSec: 60, // 2 requests per minute (nominal 120/hour)
};
case "datasets":
return {
resource: "datasets",
points: 100,
durationInSec: 60,
};
case "annotation-queues":
return {
resource: "annotation-queues",
points: 100,
durationInSec: 60,
};
case "public-api-metrics":
return {
resource: "public-api-metrics",
points: 100,
durationInSec: 86400, // 100 requests per day
};
case "public-api-v2-metrics":
return {
resource,
points: 100,
durationInSec: 86400, // 100 requests per day
};
case "public-api-daily-metrics-legacy":
return {
resource: "public-api-daily-metrics-legacy",
points: 10,
durationInSec: 86400, // 10 requests per day
};
case "trace-delete":
return {
resource: "trace-delete",
points: 50,
durationInSec: 86400, // 50 requests per day
};
case "score-delete":
return {
resource: "score-delete",
points: 50,
durationInSec: 86400, // 50 requests per day
};
case "in-app-agent-run":
return {
resource: "in-app-agent-run",
points: 100,
durationInSec: 86400,
};
case "feedback":
return {
resource: "feedback",
points: 20,
durationInSec: 86400, // 20 submissions per day
};
case "ai-gateway-requests":
return { resource, points: 100, durationInSec: 60 };
case "ai-gateway-request-bytes":
return { resource, points: 64 * MiB, durationInSec: 60 };
default:
const exhaustiveCheck: never = resource;
throw new Error(`Unhandled resource case: ${exhaustiveCheck}`);
}
case "cloud:core":
// TEMPORARY: Expanded selected core plan rate limits to pro limits to enable legacy pro -> core migration
// Original core limits (commented out):
// ingestion: 4000, public-api: 100, datasets: 200, public-api-metrics: 200, public-api-daily-metrics-legacy: 20
switch (resource) {
case "ingestion":
return {
resource: "ingestion",
// points: 4000, // original core limit
points: 20_000, // temporary: using pro limit
durationInSec: 60,
};
case "media-upload":
return {
resource: "media-upload",
points: 4_000,
durationInSec: 60,
};
case "legacy-ingestion":
return {
resource: "legacy-ingestion",
points: 400,
durationInSec: 60,
};
case "prompts":
return {
resource: "prompts",
points: null,
durationInSec: null,
};
case "public-api":
return {
resource: "public-api",
// points: 100, // original core limit
points: 1000, // temporary: using pro limit
durationInSec: 60,
};
case "public-api-legacy":
return {
resource: "public-api-legacy",
points: 7,
durationInSec: 60, // 7 requests per minute (nominal 420/hour)
};
case "datasets":
return {
resource: "datasets",
// points: 200, // original core limit
points: 1000, // temporary: using pro limit
durationInSec: 60,
};
case "annotation-queues":
return {
resource: "annotation-queues",
points: 1000,
durationInSec: 60,
};
case "public-api-metrics":
return {
resource: "public-api-metrics",
// points: 200, // original core limit
points: 2000, // temporary: using pro limit
durationInSec: 86400, // 2000 requests per day
};
case "public-api-v2-metrics":
return {
resource,
points: 100,
durationInSec: 3600, // 100 requests per hour
};
case "public-api-daily-metrics-legacy":
return {
resource: "public-api-daily-metrics-legacy",
// points: 20, // original core limit
points: 200, // temporary: using pro limit
durationInSec: 86400, // 200 requests per day
};
case "trace-delete":
return {
resource: "trace-delete",
points: 200,
durationInSec: 86400, // 200 requests per day
};
case "score-delete":
return {
resource: "score-delete",
points: 200,
durationInSec: 86400, // 200 requests per day
};
case "in-app-agent-run":
return {
resource: "in-app-agent-run",
points: 1000,
durationInSec: 86400,
};
case "feedback":
return {
resource: "feedback",
points: 20,
durationInSec: 86400, // 20 submissions per day
};
case "ai-gateway-requests":
return { resource, points: 500, durationInSec: 60 };
case "ai-gateway-request-bytes":
return { resource, points: 256 * MiB, durationInSec: 60 };
default:
const exhaustiveCheck: never = resource;
throw new Error(`Unhandled resource case: ${exhaustiveCheck}`);
}
case "cloud:pro":
case "cloud:team":
case "cloud:enterprise":
switch (resource) {
case "ingestion":
return {
resource: "ingestion",
points: 20_000,
durationInSec: 60,
};
case "media-upload":
return {
resource: "media-upload",
points: 20_000,
durationInSec: 60,
};
case "legacy-ingestion":
return {
resource: "legacy-ingestion",
points: 400,
durationInSec: 60,
};
case "prompts":
return {
resource: "prompts",
points: null,
durationInSec: null,
};
case "public-api":
return {
resource: "public-api",
points: 1000,
durationInSec: 60,
};
case "public-api-legacy":
return {
resource: "public-api-legacy",
points: 20,
durationInSec: 60, // 20 requests per minute (nominal 1,200/hour)
};
case "datasets":
return {
resource: "datasets",
points: 1000,
durationInSec: 60,
};
case "annotation-queues":
return {
resource: "annotation-queues",
points: 1000,
durationInSec: 60,
};
case "public-api-metrics":
return {
resource: "public-api-metrics",
points: 2000,
durationInSec: 86400, // 2000 requests per day
};
case "public-api-v2-metrics":
return {
resource,
points: 500,
durationInSec: 3600, // 500 requests per hour
};
case "public-api-daily-metrics-legacy":
return {
resource: "public-api-daily-metrics-legacy",
points: 200,
durationInSec: 86400, // 200 requests per day
};
case "trace-delete":
return {
resource: "trace-delete",
points: 1000,
durationInSec: 86400, // 1000 requests per day
};
case "score-delete":
return {
resource: "score-delete",
points: 1000,
durationInSec: 86400, // 1000 requests per day
};
case "in-app-agent-run":
return {
resource: "in-app-agent-run",
points: 1000,
durationInSec: 86400,
};
case "feedback":
return {
resource: "feedback",
points: 20,
durationInSec: 86400, // 20 submissions per day
};
case "ai-gateway-requests":
return { resource, points: 10_000, durationInSec: 60 };
case "ai-gateway-request-bytes":
return { resource, points: 5 * GiB, durationInSec: 60 };
default:
const exhaustiveCheck: never = resource;
throw new Error(`Unhandled resource case: ${exhaustiveCheck}`);
}
default:
const exhaustiveCheck: never = plan;
throw new Error(`Unhandled plan case: ${exhaustiveCheck}`);
}
};