Skip to content

🪢 Open source agent evals & observability: Trace, evaluate, and improve LLM applications with one open platform.

fix(api): restore per-minute throttling for legacy public API reads (#19562)

Max Deichmann0f8555a42bcf
diff --git a/packages/shared/src/server/test-utils/org-factory.ts b/packages/shared/src/server/test-utils/org-factory.ts
index 3f8d615089..36050bc462 100644
--- a/packages/shared/src/server/test-utils/org-factory.ts
+++ b/packages/shared/src/server/test-utils/org-factory.ts
@@ -33,6 +33,17 @@ export const createOrgProjectAndApiKey = async (
name: v4(),
cloudConfig: CloudConfigSchema.parse({
plan: props?.plan ?? "Team",
+ // Team's public-api-legacy cap is 20 requests per minute, shared
+ // across legacy trace, observation, and session reads. A single test
+ // org issues more than that while exercising those routes, so lift
+ // the cap here. Production plans are unchanged.
+ rateLimitOverrides: [
+ {
+ resource: "public-api-legacy",
+ points: null,
+ durationInSec: null,
+ },
+ ],
}),
},
});
diff --git a/web/src/__tests__/server/api-auth.servertest.ts b/web/src/__tests__/server/api-auth.servertest.ts
index 2e00adff41..0acdb42c4e 100644
--- a/web/src/__tests__/server/api-auth.servertest.ts
+++ b/web/src/__tests__/server/api-auth.servertest.ts
@@ -870,6 +870,13 @@ describe("Authenticate API calls", () => {
orgId: testApiKey.orgId,
plan: "cloud:hobby",
scope: "PROJECT",
+ rateLimitOverrides: [
+ {
+ resource: "public-api-legacy",
+ points: null,
+ durationInSec: null,
+ },
+ ],
organizationCreatedAt: expect.any(String),
});
});
@@ -1000,6 +1007,13 @@ describe("Authenticate API calls", () => {
createdAt: apiKey?.createdAt.toISOString(),
organizationCreatedAt: expect.any(String),
scope: "PROJECT",
+ rateLimitOverrides: [
+ {
+ resource: "public-api-legacy",
+ points: null,
+ durationInSec: null,
+ },
+ ],
isIngestionSuspended: expect.anything(),
});
diff --git a/web/src/__tests__/server/rate-limit.servertest.ts b/web/src/__tests__/server/rate-limit.servertest.ts
index b7fe6d742a..481774432c 100644
--- a/web/src/__tests__/server/rate-limit.servertest.ts
+++ b/web/src/__tests__/server/rate-limit.servertest.ts
@@ -218,8 +218,17 @@ describe("RateLimitService", () => {
const rateLimitService = RateLimitService.getInstance(redis as Redis);
- for (let i = 0; i < 5; i++) {
- await rateLimitService.rateLimitRequest(asScope(scope), "public-api");
+ // Every one of the N allowed requests, including the Nth, must resolve
+ // as not rate-limited. rate-limiter-flexible floors remainingPoints at
+ // 0 for the Nth request too, so asserting only on the final (N+1)th
+ // call would miss a regression that flags the Nth request itself.
+ for (let i = 1; i <= 5; i++) {
+ const allowedResult = await rateLimitService.rateLimitRequest(
+ asScope(scope),
+ "public-api",
+ );
+ expect(allowedResult?.res?.consumedPoints).toBe(i);
+ expect(allowedResult?.isRateLimited()).toBe(false);
}
const result = await rateLimitService.rateLimitRequest(
@@ -421,22 +430,23 @@ describe("RateLimitService", () => {
});
it("should apply public-api-legacy rate limits by cloud plan", async () => {
+ // Hobby and Core round their nominal hourly equivalent down (120 and
+ // 420, versus the previous release's 150 and 450); Pro/Team/Enterprise
+ // keep the same nominal 1,200/hour, now enforced over a 60s window.
const cases = [
- { plan: "cloud:hobby" as const, points: 150, durationInSec: 3600 },
- { plan: "cloud:core" as const, points: 450, durationInSec: 3600 },
- { plan: "cloud:pro" as const, points: 1_200, durationInSec: 3600 },
- { plan: "cloud:team" as const, points: 1_200, durationInSec: 3600 },
- { plan: "cloud:enterprise" as const, points: 1_200, durationInSec: 3600 },
+ { plan: "cloud:hobby" as const, points: 2, durationInSec: 60 },
+ { plan: "cloud:core" as const, points: 7, durationInSec: 60 },
+ { plan: "cloud:pro" as const, points: 20, durationInSec: 60 },
+ { plan: "cloud:team" as const, points: 20, durationInSec: 60 },
+ { plan: "cloud:enterprise" as const, points: 20, durationInSec: 60 },
];
const rateLimitService = RateLimitService.getInstance(redis as Redis);
for (const testCase of cases) {
- const activeKey = `${RATE_LIMIT_REDIS_KEY_PREFIX}:public-api-legacy:${testCase.durationInSec}:${orgId}`;
- const previousWindowKey = `${RATE_LIMIT_REDIS_KEY_PREFIX}:public-api-legacy:${orgId}`;
+ const activeKey = `${RATE_LIMIT_REDIS_KEY_PREFIX}:public-api-legacy:${orgId}`;
await redis.del(activeKey);
- await redis.set(previousWindowKey, "1", "EX", 60);
const scope = {
orgId: orgId,
@@ -466,14 +476,273 @@ describe("RateLimitService", () => {
expect(ttlInSec).toBeGreaterThan(0);
expect(ttlInSec).toBeLessThanOrEqual(testCase.durationInSec);
- expect(ttlInSec).toBeGreaterThan(testCase.durationInSec - 60);
+ expect(ttlInSec).toBeGreaterThan(testCase.durationInSec - 10);
+ }
+ });
+
+ it("should admit exactly N requests and reject only the (N+1)th request (off-by-one regression)", async () => {
+ // rate-limiter-flexible floors remainingPoints at 0 for both the Nth
+ // (allowed) and (N+1)th (rejected) request, so remainingPoints alone
+ // cannot decide rejection. Cover public-api-legacy plus other resources
+ // to confirm the fix in checkRateLimit/isRateLimited is not limited to
+ // one resource.
+ const rateLimitService = RateLimitService.getInstance(redis as Redis);
+
+ const cases: Array<{
+ resource: "public-api" | "public-api-legacy" | "score-delete";
+ points: number;
+ }> = [
+ { resource: "public-api-legacy", points: 3 },
+ { resource: "public-api", points: 4 },
+ { resource: "score-delete", points: 2 },
+ ];
+
+ for (const testCase of cases) {
+ const scope = {
+ orgId: orgId,
+ plan: "cloud:hobby" as const,
+ projectId,
+ accessLevel: "project" as const,
+ rateLimitOverrides: [
+ {
+ resource: testCase.resource,
+ points: testCase.points,
+ durationInSec: 60,
+ },
+ ],
+ };
+
+ for (let i = 1; i <= testCase.points; i++) {
+ const result = await rateLimitService.rateLimitRequest(
+ asScope(scope),
+ testCase.resource,
+ );
+ expect(result?.res?.consumedPoints).toBe(i);
+ expect(result?.res?.remainingPoints).toBe(testCase.points - i);
+ expect(result?.isRateLimited()).toBe(false);
+ }
+
+ const rejected = await rateLimitService.rateLimitRequest(
+ asScope(scope),
+ testCase.resource,
+ );
- const previousWindowTtl = await redis.ttl(previousWindowKey);
- expect(previousWindowTtl).toBeGreaterThan(0);
- expect(previousWindowTtl).toBeLessThanOrEqual(60);
+ expect(rejected?.res?.consumedPoints).toBe(testCase.points + 1);
+ expect(rejected?.res?.remainingPoints).toBe(0);
+ expect(rejected?.isRateLimited()).toBe(true);
}
});
+ it("should admit exactly N concurrent requests and reject the rest", async () => {
+ const points = 5;
+ const extraRequests = 4;
+ const scope = {
+ orgId: orgId,
+ plan: "cloud:hobby" as const,
+ projectId,
+ accessLevel: "project" as const,
+ rateLimitOverrides: [
+ { resource: "public-api-legacy" as const, points, durationInSec: 60 },
+ ],
+ };
+
+ const rateLimitService = RateLimitService.getInstance(redis as Redis);
+
+ const results = await Promise.all(
+ Array.from({ length: points + extraRequests }, () =>
+ rateLimitService.rateLimitRequest(asScope(scope), "public-api-legacy"),
+ ),
+ );
+
+ const allowed = results.filter(
+ (result) => result?.isRateLimited() === false,
+ );
+ const rejected = results.filter(
+ (result) => result?.isRateLimited() === true,
+ );
+
+ expect(allowed).toHaveLength(points);
+ expect(rejected).toHaveLength(extraRequests);
+
+ // Redis's atomic increment must still hand out unique, sequential
+ // consumedPoints values under concurrency.
+ const consumedPointsSeen = results
+ .map((result) => result?.res?.consumedPoints)
+ .filter((value): value is number => typeof value === "number")
+ .toSorted((a, b) => a - b);
+ expect(consumedPointsSeen).toEqual(
+ Array.from({ length: points + extraRequests }, (_, i) => i + 1),
+ );
+ });
+
+ it("should admit requests again once the window has actually expired", async () => {
+ const points = 2;
+ const durationInSec = 1;
+ const scope = {
+ orgId: orgId,
+ plan: "cloud:hobby" as const,
+ projectId,
+ accessLevel: "project" as const,
+ rateLimitOverrides: [
+ { resource: "public-api-legacy" as const, points, durationInSec },
+ ],
+ };
+
+ const rateLimitService = RateLimitService.getInstance(redis as Redis);
+
+ for (let i = 1; i <= points; i++) {
+ const result = await rateLimitService.rateLimitRequest(
+ asScope(scope),
+ "public-api-legacy",
+ );
+ expect(result?.isRateLimited()).toBe(false);
+ }
+
+ const rejected = await rateLimitService.rateLimitRequest(
+ asScope(scope),
+ "public-api-legacy",
+ );
+ expect(rejected?.isRateLimited()).toBe(true);
+
+ // Wait for the real Redis TTL to expire rather than deleting the key,
+ // so this exercises the actual window-expiry behavior.
+ await new Promise((resolve) =>
+ setTimeout(resolve, (durationInSec + 1) * 1000),
+ );
+
+ const afterExpiry = await rateLimitService.rateLimitRequest(
+ asScope(scope),
+ "public-api-legacy",
+ );
+
+ expect(afterExpiry?.res).toEqual({
+ scope: scope,
+ resource: "public-api-legacy",
+ points,
+ remainingPoints: points - 1,
+ msBeforeNext: expect.any(Number),
+ consumedPoints: 1,
+ isFirstInDuration: true,
+ });
+ expect(afterExpiry?.isRateLimited()).toBe(false);
+ }, 10_000);
+
+ it("should share the public-api-legacy bucket across legacy routes within an org, and isolate it between organizations", async () => {
+ const otherOrgId = `rate-limit-test-org-${randomUUID()}`;
+ const otherOrgPattern = `${RATE_LIMIT_REDIS_KEY_PREFIX}:*:${otherOrgId}`;
+
+ try {
+ const points = 3;
+ const makeScope = (org: string) => ({
+ orgId: org,
+ plan: "cloud:hobby" as const,
+ projectId,
+ accessLevel: "project" as const,
+ rateLimitOverrides: [
+ {
+ resource: "public-api-legacy" as const,
+ points,
+ durationInSec: 60,
+ },
+ ],
+ });
+
+ const rateLimitService = RateLimitService.getInstance(redis as Redis);
+
+ // Legacy trace, observation, and session reads all call
+ // checkRateLimit with the same "public-api-legacy" resource, so they
+ // share one organization-level bucket regardless of which legacy
+ // route issued the call. There is no per-route key to isolate here.
+ const traceRead = await rateLimitService.rateLimitRequest(
+ asScope(makeScope(orgId)),
+ "public-api-legacy",
+ );
+ const observationRead = await rateLimitService.rateLimitRequest(
+ asScope(makeScope(orgId)),
+ "public-api-legacy",
+ );
+ const sessionRead = await rateLimitService.rateLimitRequest(
+ asScope(makeScope(orgId)),
+ "public-api-legacy",
+ );
+
+ expect(traceRead?.res?.consumedPoints).toBe(1);
+ expect(observationRead?.res?.consumedPoints).toBe(2);
+ expect(sessionRead?.res?.consumedPoints).toBe(3);
+ expect(sessionRead?.isRateLimited()).toBe(false);
+
+ // The organization's shared bucket is now exhausted...
+ const exhausted = await rateLimitService.rateLimitRequest(
+ asScope(makeScope(orgId)),
+ "public-api-legacy",
+ );
+ expect(exhausted?.isRateLimited()).toBe(true);
+
+ // ...but a different organization has its own, unaffected bucket.
+ const otherOrgRead = await rateLimitService.rateLimitRequest(
+ asScope(makeScope(otherOrgId)),
+ "public-api-legacy",
+ );
+ expect(otherOrgRead?.res?.consumedPoints).toBe(1);
+ expect(otherOrgRead?.isRateLimited()).toBe(false);
+ } finally {
+ await clearRedisKeysByPatternSafely(redis, otherOrgPattern);
+ }
+ });
+
+ it("should apply a custom public-api-legacy override on the shared org key", async () => {
+ const overridePoints = 2;
+ const overrideDurationInSec = 5;
+ const activeKey = `${RATE_LIMIT_REDIS_KEY_PREFIX}:public-api-legacy:${orgId}`;
+
+ const overrideScope = {
+ orgId: orgId,
+ plan: "cloud:hobby" as const,
+ projectId,
+ accessLevel: "project" as const,
+ rateLimitOverrides: [
+ {
+ resource: "public-api-legacy" as const,
+ points: overridePoints,
+ durationInSec: overrideDurationInSec,
+ },
+ ],
+ };
+
+ const rateLimitService = RateLimitService.getInstance(redis as Redis);
+
+ const first = await rateLimitService.rateLimitRequest(
+ asScope(overrideScope),
+ "public-api-legacy",
+ );
+ expect(first?.res).toEqual({
+ scope: overrideScope,
+ resource: "public-api-legacy",
+ points: overridePoints,
+ remainingPoints: overridePoints - 1,
+ msBeforeNext: expect.any(Number),
+ consumedPoints: 1,
+ isFirstInDuration: true,
+ });
+ expect(first?.isRateLimited()).toBe(false);
+
+ const second = await rateLimitService.rateLimitRequest(
+ asScope(overrideScope),
+ "public-api-legacy",
+ );
+ expect(second?.isRateLimited()).toBe(false);
+
+ const rejected = await rateLimitService.rateLimitRequest(
+ asScope(overrideScope),
+ "public-api-legacy",
+ );
+ expect(rejected?.isRateLimited()).toBe(true);
+
+ const ttlInSec = await redis.ttl(activeKey);
+ expect(ttlInSec).toBeGreaterThan(0);
+ expect(ttlInSec).toBeLessThanOrEqual(overrideDurationInSec);
+ });
+
it("should apply public-api-v2-metrics rate limits by cloud plan", async () => {
const cases = [
{ plan: "cloud:hobby" as const, points: 100, durationInSec: 86400 },
diff --git a/web/src/features/public-api/server/RateLimitService.ts b/web/src/features/public-api/server/RateLimitService.ts
index b1ec4ef926..51fd49c32b 100644
--- a/web/src/features/public-api/server/RateLimitService.ts
+++ b/web/src/features/public-api/server/RateLimitService.ts
@@ -130,12 +130,7 @@ export class RateLimitService {
points: effectiveConfig.points, // Number of points
duration: effectiveConfig.durationInSec, // Per second(s)
- // Legacy reads include the window so a 60s config and a 3600s config
- // do not increment the same counter.
- keyPrefix:
- resource === "public-api-legacy"
- ? `${this.rateLimitPrefix(resource)}:${effectiveConfig.durationInSec}`
- : this.rateLimitPrefix(resource),
+ keyPrefix: this.rateLimitPrefix(resource), // must be unique for limiters with different purpose
storeClient: RateLimitService.redis,
rejectIfRedisNotReady: true,
});
@@ -172,7 +167,8 @@ export class RateLimitService {
}
}
- if (res.remainingPoints < 1) {
+ // Library floors remainingPoints at 0; reject only when consumedPoints > points.
+ if (res.consumedPoints > res.points) {
recordIncrement("langfuse.rate_limit.exceeded", 1, {
orgId: scope.orgId,
plan: scope.plan,
@@ -237,7 +233,7 @@ class RateLimitHelper {
}
isRateLimited() {
- return this.res ? this.res.remainingPoints < 1 : false;
+ return this.res ? this.res.consumedPoints > this.res.points : false;
}
sendRestResponseIfLimited(
@@ -342,8 +338,8 @@ const getPlanBasedRateLimitConfig = (
case "public-api-legacy":
return {
resource: "public-api-legacy",
- points: 150,
- durationInSec: 3600, // 150 requests per hour
+ points: 2,
+ durationInSec: 60, // 2 requests per minute (nominal 120/hour)
};
case "datasets":
return {
@@ -443,8 +439,8 @@ const getPlanBasedRateLimitConfig = (
case "public-api-legacy":
return {
resource: "public-api-legacy",
- points: 450,
- durationInSec: 3600, // 450 requests per hour
+ points: 7,
+ durationInSec: 60, // 7 requests per minute (nominal 420/hour)
};
case "datasets":
return {
@@ -544,8 +540,8 @@ const getPlanBasedRateLimitConfig = (
case "public-api-legacy":
return {
resource: "public-api-legacy",
- points: 1_200,
- durationInSec: 3600, // 1,200 requests per hour
+ points: 20,
+ durationInSec: 60, // 20 requests per minute (nominal 1,200/hour)
};
case "datasets":
return {