🪢 Open source agent evals & observability: Trace, evaluate, and improve LLM applications with one open platform.
TypeScript analyticsautogenevaluationlangchainlarge-language-modelsllama-indexllmllm-evaluationllm-observabilityllmopsmonitoringobservabilityopen-sourceopenaiplaygroundprompt-engineeringprompt-managementself-hostedycombinator
fix(api): restore per-minute throttling for legacy public API reads (#19562)
Max Deichmann0f8555a42bcf
diff --git a/packages/shared/src/server/test-utils/org-factory.ts b/packages/shared/src/server/test-utils/org-factory.tsindex 3f8d615089..36050bc462 100644--- a/packages/shared/src/server/test-utils/org-factory.ts+++ b/packages/shared/src/server/test-utils/org-factory.ts@@ -33,6 +33,17 @@ export const createOrgProjectAndApiKey = async (name: v4(),cloudConfig: CloudConfigSchema.parse({plan: props?.plan ?? "Team",+ // Team's public-api-legacy cap is 20 requests per minute, shared+ // across legacy trace, observation, and session reads. A single test+ // org issues more than that while exercising those routes, so lift+ // the cap here. Production plans are unchanged.+ rateLimitOverrides: [+ {+ resource: "public-api-legacy",+ points: null,+ durationInSec: null,+ },+ ],}),},});diff --git a/web/src/__tests__/server/api-auth.servertest.ts b/web/src/__tests__/server/api-auth.servertest.tsindex 2e00adff41..0acdb42c4e 100644--- a/web/src/__tests__/server/api-auth.servertest.ts+++ b/web/src/__tests__/server/api-auth.servertest.ts@@ -870,6 +870,13 @@ describe("Authenticate API calls", () => {orgId: testApiKey.orgId,plan: "cloud:hobby",scope: "PROJECT",+ rateLimitOverrides: [+ {+ resource: "public-api-legacy",+ points: null,+ durationInSec: null,+ },+ ],organizationCreatedAt: expect.any(String),});});@@ -1000,6 +1007,13 @@ describe("Authenticate API calls", () => {createdAt: apiKey?.createdAt.toISOString(),organizationCreatedAt: expect.any(String),scope: "PROJECT",+ rateLimitOverrides: [+ {+ resource: "public-api-legacy",+ points: null,+ durationInSec: null,+ },+ ],isIngestionSuspended: expect.anything(),});diff --git a/web/src/__tests__/server/rate-limit.servertest.ts b/web/src/__tests__/server/rate-limit.servertest.tsindex b7fe6d742a..481774432c 100644--- a/web/src/__tests__/server/rate-limit.servertest.ts+++ b/web/src/__tests__/server/rate-limit.servertest.ts@@ -218,8 +218,17 @@ describe("RateLimitService", () => {const rateLimitService = RateLimitService.getInstance(redis as Redis);- for (let i = 0; i < 5; i++) {- await rateLimitService.rateLimitRequest(asScope(scope), "public-api");+ // Every one of the N allowed requests, including the Nth, must resolve+ // as not rate-limited. rate-limiter-flexible floors remainingPoints at+ // 0 for the Nth request too, so asserting only on the final (N+1)th+ // call would miss a regression that flags the Nth request itself.+ for (let i = 1; i <= 5; i++) {+ const allowedResult = await rateLimitService.rateLimitRequest(+ asScope(scope),+ "public-api",+ );+ expect(allowedResult?.res?.consumedPoints).toBe(i);+ expect(allowedResult?.isRateLimited()).toBe(false);}const result = await rateLimitService.rateLimitRequest(@@ -421,22 +430,23 @@ describe("RateLimitService", () => {});it("should apply public-api-legacy rate limits by cloud plan", async () => {+ // Hobby and Core round their nominal hourly equivalent down (120 and+ // 420, versus the previous release's 150 and 450); Pro/Team/Enterprise+ // keep the same nominal 1,200/hour, now enforced over a 60s window.const cases = [- { plan: "cloud:hobby" as const, points: 150, durationInSec: 3600 },- { plan: "cloud:core" as const, points: 450, durationInSec: 3600 },- { plan: "cloud:pro" as const, points: 1_200, durationInSec: 3600 },- { plan: "cloud:team" as const, points: 1_200, durationInSec: 3600 },- { plan: "cloud:enterprise" as const, points: 1_200, durationInSec: 3600 },+ { plan: "cloud:hobby" as const, points: 2, durationInSec: 60 },+ { plan: "cloud:core" as const, points: 7, durationInSec: 60 },+ { plan: "cloud:pro" as const, points: 20, durationInSec: 60 },+ { plan: "cloud:team" as const, points: 20, durationInSec: 60 },+ { plan: "cloud:enterprise" as const, points: 20, durationInSec: 60 },];const rateLimitService = RateLimitService.getInstance(redis as Redis);for (const testCase of cases) {- const activeKey = `${RATE_LIMIT_REDIS_KEY_PREFIX}:public-api-legacy:${testCase.durationInSec}:${orgId}`;- const previousWindowKey = `${RATE_LIMIT_REDIS_KEY_PREFIX}:public-api-legacy:${orgId}`;+ const activeKey = `${RATE_LIMIT_REDIS_KEY_PREFIX}:public-api-legacy:${orgId}`;await redis.del(activeKey);- await redis.set(previousWindowKey, "1", "EX", 60);const scope = {orgId: orgId,@@ -466,14 +476,273 @@ describe("RateLimitService", () => {expect(ttlInSec).toBeGreaterThan(0);expect(ttlInSec).toBeLessThanOrEqual(testCase.durationInSec);- expect(ttlInSec).toBeGreaterThan(testCase.durationInSec - 60);+ expect(ttlInSec).toBeGreaterThan(testCase.durationInSec - 10);+ }+ });++ it("should admit exactly N requests and reject only the (N+1)th request (off-by-one regression)", async () => {+ // rate-limiter-flexible floors remainingPoints at 0 for both the Nth+ // (allowed) and (N+1)th (rejected) request, so remainingPoints alone+ // cannot decide rejection. Cover public-api-legacy plus other resources+ // to confirm the fix in checkRateLimit/isRateLimited is not limited to+ // one resource.+ const rateLimitService = RateLimitService.getInstance(redis as Redis);++ const cases: Array<{+ resource: "public-api" | "public-api-legacy" | "score-delete";+ points: number;+ }> = [+ { resource: "public-api-legacy", points: 3 },+ { resource: "public-api", points: 4 },+ { resource: "score-delete", points: 2 },+ ];++ for (const testCase of cases) {+ const scope = {+ orgId: orgId,+ plan: "cloud:hobby" as const,+ projectId,+ accessLevel: "project" as const,+ rateLimitOverrides: [+ {+ resource: testCase.resource,+ points: testCase.points,+ durationInSec: 60,+ },+ ],+ };++ for (let i = 1; i <= testCase.points; i++) {+ const result = await rateLimitService.rateLimitRequest(+ asScope(scope),+ testCase.resource,+ );+ expect(result?.res?.consumedPoints).toBe(i);+ expect(result?.res?.remainingPoints).toBe(testCase.points - i);+ expect(result?.isRateLimited()).toBe(false);+ }++ const rejected = await rateLimitService.rateLimitRequest(+ asScope(scope),+ testCase.resource,+ );- const previousWindowTtl = await redis.ttl(previousWindowKey);- expect(previousWindowTtl).toBeGreaterThan(0);- expect(previousWindowTtl).toBeLessThanOrEqual(60);+ expect(rejected?.res?.consumedPoints).toBe(testCase.points + 1);+ expect(rejected?.res?.remainingPoints).toBe(0);+ expect(rejected?.isRateLimited()).toBe(true);}});+ it("should admit exactly N concurrent requests and reject the rest", async () => {+ const points = 5;+ const extraRequests = 4;+ const scope = {+ orgId: orgId,+ plan: "cloud:hobby" as const,+ projectId,+ accessLevel: "project" as const,+ rateLimitOverrides: [+ { resource: "public-api-legacy" as const, points, durationInSec: 60 },+ ],+ };++ const rateLimitService = RateLimitService.getInstance(redis as Redis);++ const results = await Promise.all(+ Array.from({ length: points + extraRequests }, () =>+ rateLimitService.rateLimitRequest(asScope(scope), "public-api-legacy"),+ ),+ );++ const allowed = results.filter(+ (result) => result?.isRateLimited() === false,+ );+ const rejected = results.filter(+ (result) => result?.isRateLimited() === true,+ );++ expect(allowed).toHaveLength(points);+ expect(rejected).toHaveLength(extraRequests);++ // Redis's atomic increment must still hand out unique, sequential+ // consumedPoints values under concurrency.+ const consumedPointsSeen = results+ .map((result) => result?.res?.consumedPoints)+ .filter((value): value is number => typeof value === "number")+ .toSorted((a, b) => a - b);+ expect(consumedPointsSeen).toEqual(+ Array.from({ length: points + extraRequests }, (_, i) => i + 1),+ );+ });++ it("should admit requests again once the window has actually expired", async () => {+ const points = 2;+ const durationInSec = 1;+ const scope = {+ orgId: orgId,+ plan: "cloud:hobby" as const,+ projectId,+ accessLevel: "project" as const,+ rateLimitOverrides: [+ { resource: "public-api-legacy" as const, points, durationInSec },+ ],+ };++ const rateLimitService = RateLimitService.getInstance(redis as Redis);++ for (let i = 1; i <= points; i++) {+ const result = await rateLimitService.rateLimitRequest(+ asScope(scope),+ "public-api-legacy",+ );+ expect(result?.isRateLimited()).toBe(false);+ }++ const rejected = await rateLimitService.rateLimitRequest(+ asScope(scope),+ "public-api-legacy",+ );+ expect(rejected?.isRateLimited()).toBe(true);++ // Wait for the real Redis TTL to expire rather than deleting the key,+ // so this exercises the actual window-expiry behavior.+ await new Promise((resolve) =>+ setTimeout(resolve, (durationInSec + 1) * 1000),+ );++ const afterExpiry = await rateLimitService.rateLimitRequest(+ asScope(scope),+ "public-api-legacy",+ );++ expect(afterExpiry?.res).toEqual({+ scope: scope,+ resource: "public-api-legacy",+ points,+ remainingPoints: points - 1,+ msBeforeNext: expect.any(Number),+ consumedPoints: 1,+ isFirstInDuration: true,+ });+ expect(afterExpiry?.isRateLimited()).toBe(false);+ }, 10_000);++ it("should share the public-api-legacy bucket across legacy routes within an org, and isolate it between organizations", async () => {+ const otherOrgId = `rate-limit-test-org-${randomUUID()}`;+ const otherOrgPattern = `${RATE_LIMIT_REDIS_KEY_PREFIX}:*:${otherOrgId}`;++ try {+ const points = 3;+ const makeScope = (org: string) => ({+ orgId: org,+ plan: "cloud:hobby" as const,+ projectId,+ accessLevel: "project" as const,+ rateLimitOverrides: [+ {+ resource: "public-api-legacy" as const,+ points,+ durationInSec: 60,+ },+ ],+ });++ const rateLimitService = RateLimitService.getInstance(redis as Redis);++ // Legacy trace, observation, and session reads all call+ // checkRateLimit with the same "public-api-legacy" resource, so they+ // share one organization-level bucket regardless of which legacy+ // route issued the call. There is no per-route key to isolate here.+ const traceRead = await rateLimitService.rateLimitRequest(+ asScope(makeScope(orgId)),+ "public-api-legacy",+ );+ const observationRead = await rateLimitService.rateLimitRequest(+ asScope(makeScope(orgId)),+ "public-api-legacy",+ );+ const sessionRead = await rateLimitService.rateLimitRequest(+ asScope(makeScope(orgId)),+ "public-api-legacy",+ );++ expect(traceRead?.res?.consumedPoints).toBe(1);+ expect(observationRead?.res?.consumedPoints).toBe(2);+ expect(sessionRead?.res?.consumedPoints).toBe(3);+ expect(sessionRead?.isRateLimited()).toBe(false);++ // The organization's shared bucket is now exhausted...+ const exhausted = await rateLimitService.rateLimitRequest(+ asScope(makeScope(orgId)),+ "public-api-legacy",+ );+ expect(exhausted?.isRateLimited()).toBe(true);++ // ...but a different organization has its own, unaffected bucket.+ const otherOrgRead = await rateLimitService.rateLimitRequest(+ asScope(makeScope(otherOrgId)),+ "public-api-legacy",+ );+ expect(otherOrgRead?.res?.consumedPoints).toBe(1);+ expect(otherOrgRead?.isRateLimited()).toBe(false);+ } finally {+ await clearRedisKeysByPatternSafely(redis, otherOrgPattern);+ }+ });++ it("should apply a custom public-api-legacy override on the shared org key", async () => {+ const overridePoints = 2;+ const overrideDurationInSec = 5;+ const activeKey = `${RATE_LIMIT_REDIS_KEY_PREFIX}:public-api-legacy:${orgId}`;++ const overrideScope = {+ orgId: orgId,+ plan: "cloud:hobby" as const,+ projectId,+ accessLevel: "project" as const,+ rateLimitOverrides: [+ {+ resource: "public-api-legacy" as const,+ points: overridePoints,+ durationInSec: overrideDurationInSec,+ },+ ],+ };++ const rateLimitService = RateLimitService.getInstance(redis as Redis);++ const first = await rateLimitService.rateLimitRequest(+ asScope(overrideScope),+ "public-api-legacy",+ );+ expect(first?.res).toEqual({+ scope: overrideScope,+ resource: "public-api-legacy",+ points: overridePoints,+ remainingPoints: overridePoints - 1,+ msBeforeNext: expect.any(Number),+ consumedPoints: 1,+ isFirstInDuration: true,+ });+ expect(first?.isRateLimited()).toBe(false);++ const second = await rateLimitService.rateLimitRequest(+ asScope(overrideScope),+ "public-api-legacy",+ );+ expect(second?.isRateLimited()).toBe(false);++ const rejected = await rateLimitService.rateLimitRequest(+ asScope(overrideScope),+ "public-api-legacy",+ );+ expect(rejected?.isRateLimited()).toBe(true);++ const ttlInSec = await redis.ttl(activeKey);+ expect(ttlInSec).toBeGreaterThan(0);+ expect(ttlInSec).toBeLessThanOrEqual(overrideDurationInSec);+ });+it("should apply public-api-v2-metrics rate limits by cloud plan", async () => {const cases = [{ plan: "cloud:hobby" as const, points: 100, durationInSec: 86400 },diff --git a/web/src/features/public-api/server/RateLimitService.ts b/web/src/features/public-api/server/RateLimitService.tsindex b1ec4ef926..51fd49c32b 100644--- a/web/src/features/public-api/server/RateLimitService.ts+++ b/web/src/features/public-api/server/RateLimitService.ts@@ -130,12 +130,7 @@ export class RateLimitService {points: effectiveConfig.points, // Number of pointsduration: effectiveConfig.durationInSec, // Per second(s)- // Legacy reads include the window so a 60s config and a 3600s config- // do not increment the same counter.- keyPrefix:- resource === "public-api-legacy"- ? `${this.rateLimitPrefix(resource)}:${effectiveConfig.durationInSec}`- : this.rateLimitPrefix(resource),+ keyPrefix: this.rateLimitPrefix(resource), // must be unique for limiters with different purposestoreClient: RateLimitService.redis,rejectIfRedisNotReady: true,});@@ -172,7 +167,8 @@ export class RateLimitService {}}- if (res.remainingPoints < 1) {+ // Library floors remainingPoints at 0; reject only when consumedPoints > points.+ if (res.consumedPoints > res.points) {recordIncrement("langfuse.rate_limit.exceeded", 1, {orgId: scope.orgId,plan: scope.plan,@@ -237,7 +233,7 @@ class RateLimitHelper {}isRateLimited() {- return this.res ? this.res.remainingPoints < 1 : false;+ return this.res ? this.res.consumedPoints > this.res.points : false;}sendRestResponseIfLimited(@@ -342,8 +338,8 @@ const getPlanBasedRateLimitConfig = (case "public-api-legacy":return {resource: "public-api-legacy",- points: 150,- durationInSec: 3600, // 150 requests per hour+ points: 2,+ durationInSec: 60, // 2 requests per minute (nominal 120/hour)};case "datasets":return {@@ -443,8 +439,8 @@ const getPlanBasedRateLimitConfig = (case "public-api-legacy":return {resource: "public-api-legacy",- points: 450,- durationInSec: 3600, // 450 requests per hour+ points: 7,+ durationInSec: 60, // 7 requests per minute (nominal 420/hour)};case "datasets":return {@@ -544,8 +540,8 @@ const getPlanBasedRateLimitConfig = (case "public-api-legacy":return {resource: "public-api-legacy",- points: 1_200,- durationInSec: 3600, // 1,200 requests per hour+ points: 20,+ durationInSec: 60, // 20 requests per minute (nominal 1,200/hour)};case "datasets":return {