Skip to content

🪢 Open source agent evals & observability: Trace, evaluate, and improve LLM applications with one open platform.

fix(mcp): back off observation searches after resource errors (#19175)

Valery Meleshkinb914c2a58139
diff --git a/web/src/__tests__/server/unit/run-mcp-tool.servertest.ts b/web/src/__tests__/server/unit/run-mcp-tool.servertest.ts
index 4099e196a1..bc11fb59a5 100644
--- a/web/src/__tests__/server/unit/run-mcp-tool.servertest.ts
+++ b/web/src/__tests__/server/unit/run-mcp-tool.servertest.ts
@@ -5,26 +5,39 @@ import { ErrorCode } from "@modelcontextprotocol/sdk/types.js";
import { formatErrorForUser } from "@/src/features/mcp/core/error-formatting";
import { z } from "zod";
import type { ServerContext } from "@/src/features/mcp/types";
+import type { Redis } from "ioredis";
+import { RateLimitService } from "@/src/features/public-api/server/RateLimitService";
+import { handleListObservations } from "@/src/features/mcp/server/observations/tools/listObservations";
-const { addUserToSpanMock, fakeSpan, instrumentAsyncMock } = vi.hoisted(() => {
- const fakeSpan = {
- setAttribute: vi.fn(),
- setAttributes: vi.fn(),
- };
+const { addUserToSpanMock, fakeSpan, instrumentAsyncMock, queryMock } =
+ vi.hoisted(() => {
+ const fakeSpan = {
+ setAttribute: vi.fn(),
+ setAttributes: vi.fn(),
+ };
- return {
- addUserToSpanMock: vi.fn(),
- fakeSpan,
- instrumentAsyncMock: vi.fn(async (_options, callback) =>
- callback(fakeSpan),
- ),
- };
-});
+ return {
+ addUserToSpanMock: vi.fn(),
+ queryMock: vi.fn(),
+ fakeSpan,
+ instrumentAsyncMock: vi.fn(async (_options, callback) =>
+ callback(fakeSpan),
+ ),
+ };
+ });
vi.mock("@langfuse/shared/src/server", async (importOriginal) => ({
...(await importOriginal()),
addUserToSpan: addUserToSpanMock,
instrumentAsync: instrumentAsyncMock,
+ getObservationsV2FromEventsTableForPublicApi: queryMock,
+}));
+
+vi.mock("@/src/env.mjs", () => ({
+ env: {
+ NEXT_PUBLIC_LANGFUSE_CLOUD_REGION: "us",
+ LANGFUSE_RATE_LIMITS_ENABLED: "true",
+ },
}));
import { runMcpTool } from "@/src/features/mcp/core/run-mcp-tool";
@@ -114,3 +127,110 @@ describe("runMcpTool outcome telemetry", () => {
},
);
});
+
+describe("MCP observation search cooldown", () => {
+ const input = {
+ fromStartTime: "2026-10-01T00:00:00Z",
+ toStartTime: "2026-10-02T00:00:00Z",
+ limit: 10,
+ };
+ const cooldowns = new Map<string, number>();
+ const redis = {
+ status: "wait",
+ connect: vi.fn(async () => {
+ redis.status = "ready";
+ }),
+ ttl: vi.fn(async (key: string) => cooldowns.get(key) ?? -2),
+ set: vi.fn(
+ async (key: string, _value: string, _ex: string, seconds: number) => {
+ cooldowns.set(key, seconds);
+ return "OK";
+ },
+ ),
+ disconnect: vi.fn(),
+ };
+
+ beforeEach(() => {
+ vi.clearAllMocks();
+ cooldowns.clear();
+ redis.status = "wait";
+ queryMock.mockReset().mockResolvedValue([]);
+ RateLimitService.shutdown();
+ RateLimitService.getInstance(redis as unknown as Redis);
+ });
+
+ afterEach(() => RateLimitService.shutdown());
+
+ it.each(["TIMEOUT", "MEMORY_LIMIT", "OVERCOMMIT"] as const)(
+ "backs off after %s, isolates projects, and resumes after expiry",
+ async (errorType) => {
+ await expect(
+ handleListObservations(input, context),
+ ).resolves.toBeDefined();
+ queryMock.mockRejectedValueOnce(
+ new ClickHouseResourceError(
+ errorType,
+ new Error("private query details"),
+ ),
+ );
+ await expect(
+ handleListObservations(input, context),
+ ).rejects.toMatchObject({
+ code: ErrorCode.InvalidRequest,
+ });
+
+ await expect(handleListObservations(input, context)).rejects.toThrow(
+ /retry in 30s/i,
+ );
+ expect(queryMock).toHaveBeenCalledTimes(2);
+ expect(redis.set).toHaveBeenCalledWith(
+ "rate-limit:mcp-query-cooldown:project-id",
+ "1",
+ "EX",
+ 30,
+ );
+ await expect(
+ handleListObservations(input, {
+ ...context,
+ projectId: "other-project",
+ }),
+ ).resolves.toBeDefined();
+
+ // Redis TTL 0 still means the key exists for less than a second.
+ cooldowns.set("rate-limit:mcp-query-cooldown:project-id", 0);
+ await expect(handleListObservations(input, context)).rejects.toThrow(
+ /retry in 1s/i,
+ );
+ expect(redis.set).toHaveBeenCalledTimes(1);
+ cooldowns.clear();
+ await expect(
+ handleListObservations(input, context),
+ ).resolves.toBeDefined();
+ expect(queryMock).toHaveBeenCalledTimes(4);
+ },
+ );
+
+ it("fails open on Redis errors and preserves normal query error handling", async () => {
+ redis.ttl.mockRejectedValueOnce(new Error("Redis unavailable"));
+ await expect(handleListObservations(input, context)).resolves.toBeDefined();
+
+ redis.set.mockRejectedValueOnce(new Error("Redis unavailable"));
+ queryMock.mockRejectedValueOnce(
+ new ClickHouseResourceError(
+ "TIMEOUT",
+ new Error("private query details"),
+ ),
+ );
+ await expect(handleListObservations(input, context)).rejects.toThrow(
+ /query timed out/i,
+ );
+ await expect(handleListObservations(input, context)).resolves.toBeDefined();
+
+ redis.set.mockClear();
+ queryMock.mockRejectedValueOnce(new InvalidRequestError("invalid filter"));
+ await expect(handleListObservations(input, context)).rejects.toThrow(
+ "invalid filter",
+ );
+ expect(redis.set).not.toHaveBeenCalled();
+ });
+});
diff --git a/web/src/features/mcp/server/observations/tools/listObservations.ts b/web/src/features/mcp/server/observations/tools/listObservations.ts
index 8b19e38a6c..23a1846027 100644
--- a/web/src/features/mcp/server/observations/tools/listObservations.ts
+++ b/web/src/features/mcp/server/observations/tools/listObservations.ts
@@ -33,6 +33,7 @@ import { defineTool } from "../../../core/define-tool";
import { runMcpTool } from "../../../core/run-mcp-tool";
import { MCP_CLICKHOUSE_SETTINGS } from "../../../core/clickhouse-settings";
import { clampToDataAccessDays } from "@/src/features/entitlements/server";
+import { RateLimitService } from "@/src/features/public-api/server/RateLimitService";
import {
ExpandMetadataKeysSchema,
getMetadataExpansionForProjection,
@@ -393,36 +394,41 @@ export const [listObservationsTool, handleListObservations] = defineTool({
"mcp.field_groups": fieldGroups.join(","),
});
- const items = await getObservationsV2FromEventsTableForPublicApi(
- {
- projectId: context.projectId,
- page: 0,
- limit: input.limit,
- traceId: input.traceId,
- userId: input.userId,
- level: input.level,
- name: input.name,
- type: input.type,
- environment: input.environment,
- parentObservationId: input.parentObservationId,
- isRootObservation: input.isRootObservation,
- fromStartTime:
- dataAccessWindow.effectiveFromTimestamp?.toISOString(),
- toStartTime: input.toStartTime,
- version: input.version,
- advancedFilters,
- cursor: input.cursor
- ? EncodedObservationsCursorV2.parse(input.cursor)
- : undefined,
- fields: fieldGroups,
- expandMetadataKeys: getMetadataExpansionForProjection(
- projectionFields,
- input.expandMetadataKeys,
- ),
- },
- {
- clickhouseSettings: MCP_CLICKHOUSE_SETTINGS,
- },
+ const query = () =>
+ getObservationsV2FromEventsTableForPublicApi(
+ {
+ projectId: context.projectId,
+ page: 0,
+ limit: input.limit,
+ traceId: input.traceId,
+ userId: input.userId,
+ level: input.level,
+ name: input.name,
+ type: input.type,
+ environment: input.environment,
+ parentObservationId: input.parentObservationId,
+ isRootObservation: input.isRootObservation,
+ fromStartTime:
+ dataAccessWindow.effectiveFromTimestamp?.toISOString(),
+ toStartTime: input.toStartTime,
+ version: input.version,
+ advancedFilters,
+ cursor: input.cursor
+ ? EncodedObservationsCursorV2.parse(input.cursor)
+ : undefined,
+ fields: fieldGroups,
+ expandMetadataKeys: getMetadataExpansionForProjection(
+ projectionFields,
+ input.expandMetadataKeys,
+ ),
+ },
+ {
+ clickhouseSettings: MCP_CLICKHOUSE_SETTINGS,
+ },
+ );
+ const items = await RateLimitService.getInstance().withMcpQueryCooldown(
+ context.projectId,
+ query,
);
const hasMore = items.length > input.limit;
diff --git a/web/src/features/public-api/server/RateLimitService.ts b/web/src/features/public-api/server/RateLimitService.ts
index 6717d0d240..34da5b182e 100644
--- a/web/src/features/public-api/server/RateLimitService.ts
+++ b/web/src/features/public-api/server/RateLimitService.ts
@@ -3,12 +3,14 @@ import { type z } from "zod";
import { RateLimiterRedis, RateLimiterRes } from "rate-limiter-flexible";
import { env } from "@/src/env.mjs";
import {
+ InvalidRequestError,
type RateLimitResult,
type RateLimitResource,
type RateLimitConfig,
type Plan,
} from "@langfuse/shared";
import {
+ ClickHouseResourceError,
recordIncrement,
type ApiAccessScopeWithOptionalApiKeyId,
logger,
@@ -178,6 +180,47 @@ export class RateLimitService {
rateLimitPrefix(resource: string) {
return `${RATE_LIMIT_REDIS_KEY_PREFIX}:${resource}`;
}
+
+ async withMcpQueryCooldown<T>(projectId: string, query: () => Promise<T>) {
+ const redis = RateLimitService.redis;
+ if (
+ !env.NEXT_PUBLIC_LANGFUSE_CLOUD_REGION ||
+ env.LANGFUSE_RATE_LIMITS_ENABLED === "false" ||
+ !redis
+ ) {
+ return query();
+ }
+
+ const key = `${this.rateLimitPrefix("mcp-query-cooldown")}:${projectId}`;
+ const cooldownError = (seconds: number) =>
+ new InvalidRequestError(
+ `MCP observation searches for this project are paused after a query exceeded resource limits. Retry in ${Math.max(1, seconds)}s with a shorter date range or more specific filters, and run searches sequentially.`,
+ );
+ const onRedisError = (error: unknown) => {
+ logger.warn("MCP query cooldown Redis operation failed", error);
+ return null;
+ };
+ if (redis.status === "wait") await redis.connect().catch(onRedisError);
+ if (redis.status !== "ready") return query();
+ const ttl = await redis.ttl(key).catch(onRedisError);
+ if (ttl !== null && ttl >= 0) throw cooldownError(ttl);
+
+ try {
+ return await query();
+ } catch (error) {
+ if (ClickHouseResourceError.is(error)) {
+ const blocked = await redis.set(key, "1", "EX", 30).catch(onRedisError);
+ if (blocked) {
+ logger.warn("MCP ClickHouse resource limit exceeded", {
+ errorType: error.errorType,
+ tags: error.tags,
+ });
+ throw cooldownError(30);
+ }
+ }
+ throw error;
+ }
+ }
}
class RateLimitHelper {