🪢 Open source agent evals & observability: Trace, evaluate, and improve LLM applications with one open platform.
TypeScript analyticsautogenevaluationlangchainlarge-language-modelsllama-indexllmllm-evaluationllm-observabilityllmopsmonitoringobservabilityopen-sourceopenaiplaygroundprompt-engineeringprompt-managementself-hostedycombinator
fix(in-app-agent): wait out MCP rate limits and retry LLM calls (#16058)
Lysander Kieself9e1108f91bb
diff --git a/packages/shared/src/in-app-agent/server/agent.ts b/packages/shared/src/in-app-agent/server/agent.tsindex a7e9ed8e21..43c4f2ca37 100644--- a/packages/shared/src/in-app-agent/server/agent.ts+++ b/packages/shared/src/in-app-agent/server/agent.ts@@ -22,6 +22,10 @@ import type {InAppAgentTracingConfig,} from "./instrumentation";import { createInAppAgentInstrumentation } from "./instrumentation";+import {+ parseMcpRateLimitError,+ withMcpRateLimitWait,+} from "./mcpRateLimitWait";import { getToolFailureMessage } from "./toolErrors";import {createSandboxTools,@@ -29,6 +33,7 @@ import {filterInAppAgentAvailableLangfuseMcpTools,getInAppAgentMcpAllowedToolNames,getInAppAgentRegistryToolName,+ hasCallableExecute,type CompletedInAppAgentMcpToolCall,type InAppAgentToolPolicy,withOptionalSilentMcpOutput,@@ -785,7 +790,31 @@ async function createMastraAdapter(params: {});try {- const { toolsets, errors } = await mcpClient.listToolsetsWithErrors();+ // Discovery costs one `public-api` rate limit point like any other MCP+ // request, so a busy org can be rate limited before the run has a chance to+ // call a single tool.+ const { toolsets, errors } = await withMcpRateLimitWait({+ signal: params.signal,+ logContext: {+ operation: "listToolsets",+ runId: params.input.runId,+ threadId: params.input.threadId,+ },+ fn: async () => {+ const result = await mcpClient.listToolsetsWithErrors();++ if (+ result.errors.langfuse &&+ parseMcpRateLimitError(result.errors.langfuse)+ ) {+ throw new Error(+ `Failed to initialize Langfuse MCP: ${result.errors.langfuse}`,+ );+ }++ return result;+ },+ });if (errors.langfuse) {throw new Error(`Failed to initialize Langfuse MCP: ${errors.langfuse}`);@@ -806,13 +835,18 @@ async function createMastraAdapter(params: {const tools = withInAppAgentToolApproval({...withOptionalSilentMcpOutput({- tools: prefixToolsetTools(- "langfuse",- filterInAppAgentAvailableLangfuseMcpTools({- tools: toolsets.langfuse,- policy: params.options.langfuseMcp.toolPolicy,- }),- ),+ tools: withLangfuseMcpRateLimitWait({+ tools: prefixToolsetTools(+ "langfuse",+ filterInAppAgentAvailableLangfuseMcpTools({+ tools: toolsets.langfuse,+ policy: params.options.langfuseMcp.toolPolicy,+ }),+ ),+ signal: params.signal,+ runId: params.input.runId,+ threadId: params.input.threadId,+ }),sandbox: params.options.sandbox,onToolCallCompleted: params.options.onMcpToolCallCompleted,}),@@ -853,6 +887,7 @@ async function createMastraAdapter(params: {),skills: LANGFUSE_IN_APP_AGENT_SKILLS,tools,+ maxRetries: 2,defaultOptions: {abortSignal: params.signal,maxSteps: MAX_AGENT_STEPS,@@ -943,6 +978,44 @@ function prefixToolsetTools<TTool>();}+function withLangfuseMcpRateLimitWait<TTool>(params: {+ tools: Record<string, TTool>;+ signal: AbortSignal;+ runId: string;+ threadId: string;+}): Record<string, TTool> {+ return Object.fromEntries(+ Object.entries(params.tools).map(([toolName, tool]) => {+ if (!hasCallableExecute(tool)) {+ return [toolName, tool];+ }++ const execute = tool.execute.bind(tool) as (+ inputData: unknown,+ context: unknown,+ ) => Promise<unknown>;++ return [+ toolName,+ {+ ...tool,+ execute: (inputData: unknown, context: unknown) =>+ withMcpRateLimitWait({+ signal: params.signal,+ logContext: {+ operation: "tools/call",+ toolName,+ runId: params.runId,+ threadId: params.threadId,+ },+ fn: () => execute(inputData, context),+ }),+ } as TTool,+ ];+ }),+ );+}+type MastraChunkProcessor = {handleChunk: (chunk: unknown) => boolean;flush: () => void;diff --git a/packages/shared/src/in-app-agent/server/mcpRateLimitWait.test.ts b/packages/shared/src/in-app-agent/server/mcpRateLimitWait.test.tsnew file mode 100644index 0000000000..2e5fe4af92--- /dev/null+++ b/packages/shared/src/in-app-agent/server/mcpRateLimitWait.test.ts@@ -0,0 +1,159 @@+import { afterEach, describe, expect, it, vi } from "vitest";++import {+ parseMcpRateLimitError,+ withMcpRateLimitWait,+} from "./mcpRateLimitWait";++function rateLimitPayload(retryAfterSeconds: number) {+ return {+ message: "Rate limit exceeded",+ code: "rate_limited" as const,+ details: {+ retryAfterSeconds,+ limit: 30,+ remaining: 0,+ },+ };+}++function rateLimitError(retryAfterSeconds: number) {+ return new Error(JSON.stringify(rateLimitPayload(retryAfterSeconds)));+}++describe("parseMcpRateLimitError", () => {+ it("parses a thrown Error whose message is the rate-limit JSON", () => {+ expect(parseMcpRateLimitError(rateLimitError(12))).toEqual({+ retryAfterSeconds: 12,+ });+ });++ it("parses the embedded Langfuse MCP init error shape", () => {+ const payload = JSON.stringify(rateLimitPayload(32));++ expect(+ parseMcpRateLimitError(`Failed to initialize Langfuse MCP: ${payload}`),+ ).toEqual({ retryAfterSeconds: 32 });+ });++ it("returns null for MCP invalid-argument and schema errors", () => {+ expect(+ parseMcpRateLimitError(+ new Error(+ "MCP error -32602: Invalid params: missing required parameter 'name'",+ ),+ ),+ ).toBeNull();+ expect(+ parseMcpRateLimitError(+ new Error(+ JSON.stringify({+ jsonrpc: "2.0",+ error: { code: -32602, message: "Invalid params" },+ id: 1,+ }),+ ),+ ),+ ).toBeNull();+ });+});++describe("withMcpRateLimitWait", () => {+ afterEach(() => {+ vi.useRealTimers();+ });++ it("sleeps the first retryAfterSeconds then succeeds", async () => {+ vi.useFakeTimers();+ const fn = vi+ .fn()+ .mockRejectedValueOnce(rateLimitError(3))+ .mockResolvedValueOnce("ok");++ const resultPromise = withMcpRateLimitWait({+ fn,+ logContext: { runId: "run-1" },+ });++ await vi.advanceTimersByTimeAsync(2_999);+ expect(fn).toHaveBeenCalledTimes(1);++ await vi.advanceTimersByTimeAsync(1);+ await expect(resultPromise).resolves.toBe("ok");+ expect(fn).toHaveBeenCalledTimes(2);+ });++ it("uses the later retryAfterSeconds on a second 429", async () => {+ vi.useFakeTimers();+ const fn = vi+ .fn()+ .mockRejectedValueOnce(rateLimitError(1))+ .mockRejectedValueOnce(rateLimitError(4))+ .mockResolvedValueOnce("ok");++ const resultPromise = withMcpRateLimitWait({+ fn,+ logContext: { runId: "run-1" },+ });++ await vi.advanceTimersByTimeAsync(1_000);+ expect(fn).toHaveBeenCalledTimes(2);++ await vi.advanceTimersByTimeAsync(3_999);+ expect(fn).toHaveBeenCalledTimes(2);++ await vi.advanceTimersByTimeAsync(1);+ await expect(resultPromise).resolves.toBe("ok");+ expect(fn).toHaveBeenCalledTimes(3);+ });++ it("does not retry an MCP invalid-argument error with the same arguments", async () => {+ const error = new Error(+ "MCP error -32602: Invalid params: missing required parameter 'name'",+ );+ const fn = vi.fn().mockRejectedValue(error);++ await expect(+ withMcpRateLimitWait({+ fn,+ logContext: { runId: "run-1" },+ }),+ ).rejects.toBe(error);++ expect(fn).toHaveBeenCalledTimes(1);+ });++ it("fails immediately when retryAfterSeconds exceeds the remaining budget", async () => {+ const error = rateLimitError(60);+ const fn = vi.fn().mockRejectedValue(error);++ await expect(+ withMcpRateLimitWait({+ fn,+ budgetMs: 5_000,+ logContext: { runId: "run-1" },+ }),+ ).rejects.toBe(error);++ expect(fn).toHaveBeenCalledTimes(1);+ });++ it("does not hang when aborted during the wait", async () => {+ vi.useFakeTimers();+ const controller = new AbortController();+ const abortError = new Error("aborted");+ const fn = vi.fn().mockRejectedValue(rateLimitError(3));++ const resultPromise = withMcpRateLimitWait({+ fn,+ signal: controller.signal,+ logContext: { runId: "run-1" },+ });++ await vi.advanceTimersByTimeAsync(0);+ controller.abort(abortError);++ await expect(resultPromise).rejects.toBe(abortError);+ expect(fn).toHaveBeenCalledTimes(1);+ });+});diff --git a/packages/shared/src/in-app-agent/server/mcpRateLimitWait.ts b/packages/shared/src/in-app-agent/server/mcpRateLimitWait.tsnew file mode 100644index 0000000000..76fa78a2ff--- /dev/null+++ b/packages/shared/src/in-app-agent/server/mcpRateLimitWait.ts@@ -0,0 +1,127 @@+import { logger } from "../../server";+import { InAppAgentRateLimitErrorResponseSchema } from "../schema";++/**+ * Total time a single MCP operation may spend waiting out `rate_limited`+ * responses. Sized so a full Hobby `public-api` window (`retryAfterSeconds: 60`)+ * can still be waited out within one operation.+ */+export const IN_APP_AGENT_MCP_RATE_LIMIT_WAIT_BUDGET_MS = 60_000;++type McpRateLimitError = {+ retryAfterSeconds: number;+};++/**+ * Recognizes the public API `rate_limited` payload wherever the MCP transport+ * surfaces it: as a structured value, as a thrown `Error`, or embedded as JSON+ * in an error string (`Failed to initialize Langfuse MCP: ...`).+ */+export function parseMcpRateLimitError(+ error: unknown,+): McpRateLimitError | null {+ const parsed = InAppAgentRateLimitErrorResponseSchema.safeParse(error);++ if (parsed.success) {+ return { retryAfterSeconds: parsed.data.details.retryAfterSeconds };+ }++ const message =+ typeof error === "string"+ ? error+ : error instanceof Error+ ? error.message+ : null;++ return message === null ? null : parseEmbeddedRateLimitError(message);+}++function parseEmbeddedRateLimitError(message: string) {+ const endIndex = message.lastIndexOf("}");+ let startIndex = message.indexOf("{");++ while (startIndex !== -1 && startIndex < endIndex) {+ try {+ const parsed = InAppAgentRateLimitErrorResponseSchema.safeParse(+ JSON.parse(message.slice(startIndex, endIndex + 1)),+ );++ if (parsed.success) {+ return {+ retryAfterSeconds: parsed.data.details.retryAfterSeconds,+ };+ }+ } catch {+ // The transport prefixes JSON with error context, so try the next object.+ }++ startIndex = message.indexOf("{", startIndex + 1);+ }++ return null;+}++/**+ * Retries a single MCP operation while the public API answers `rate_limited`,+ * sleeping exactly as long as the response asks for. The server escalates+ * `retryAfterSeconds` as its window empties, so there is no local backoff+ * ladder. A wait longer than the remaining budget fails immediately instead of+ * sleeping and hoping.+ */+export async function withMcpRateLimitWait<T>(params: {+ fn: () => Promise<T>;+ signal?: AbortSignal;+ budgetMs?: number;+ logContext: Record<string, unknown>;+}): Promise<T> {+ let remainingBudgetMs =+ params.budgetMs ?? IN_APP_AGENT_MCP_RATE_LIMIT_WAIT_BUDGET_MS;++ for (;;) {+ try {+ return await params.fn();+ } catch (error: unknown) {+ const rateLimit = parseMcpRateLimitError(error);++ if (!rateLimit) {+ throw error;+ }++ const waitMs = rateLimit.retryAfterSeconds * 1_000;++ if (waitMs > remainingBudgetMs) {+ throw error;+ }++ logger.warn("Waiting out in-app agent MCP rate limit", {+ ...params.logContext,+ retryAfterSeconds: rateLimit.retryAfterSeconds,+ remainingBudgetMs,+ });++ await sleep(waitMs, params.signal);+ remainingBudgetMs -= waitMs;+ }+ }+}++function sleep(ms: number, signal?: AbortSignal) {+ return new Promise<void>((resolve, reject) => {+ if (signal?.aborted) {+ reject(signal.reason);+ return;+ }++ const onAbort = () => {+ clearTimeout(timeout);+ reject(signal?.reason);+ };++ const timeout = setTimeout(() => {+ signal?.removeEventListener("abort", onAbort);+ resolve();+ }, ms);++ signal?.addEventListener("abort", onAbort, { once: true });+ });+}diff --git a/packages/shared/src/in-app-agent/server/tools.ts b/packages/shared/src/in-app-agent/server/tools.tsindex 43c420a6a9..1aa8c5aa1e 100644--- a/packages/shared/src/in-app-agent/server/tools.ts+++ b/packages/shared/src/in-app-agent/server/tools.ts@@ -632,18 +632,30 @@ export type CompletedInAppAgentMcpToolCall = {createdAt: Date;};+type ToolWithCallableExecute = {+ execute: (...args: never[]) => unknown;+};+type WrappableMcpTool = Pick<Tool,"execute" | "inputSchema" | "toModelOutput"> &Required<Pick<Tool, "execute" | "inputSchema">>;-function isWrappableMcpTool(tool: unknown): tool is WrappableMcpTool {+export function hasCallableExecute(+ tool: unknown,+): tool is ToolWithCallableExecute {return (typeof tool === "object" &&tool !== null &&"execute" in tool &&- typeof tool.execute === "function" &&+ typeof tool.execute === "function"+ );+}++function isWrappableMcpTool(tool: unknown): tool is WrappableMcpTool {+ return (+ hasCallableExecute(tool) &&"inputSchema" in tool &&tool.inputSchema !== undefined &&(!("toModelOutput" in tool) ||diff --git a/web/src/__tests__/server/in-app-agent-stream.servertest.ts b/web/src/__tests__/server/in-app-agent-stream.servertest.tsindex 185d6c16b1..63517e87a5 100644--- a/web/src/__tests__/server/in-app-agent-stream.servertest.ts+++ b/web/src/__tests__/server/in-app-agent-stream.servertest.ts@@ -1255,10 +1255,7 @@ describe("createAgUiStream", () => {status: "approved",});- const agentConfig = vi.mocked(Agent).mock.calls[0]?.[0];- const createScoreConfigTool =- getAgentTools(agentConfig)?.langfuse_createScoreConfig;- expect(createScoreConfigTool?.execute).toHaveBeenCalledWith(+ expect(adapterEvents.createScoreConfigExecute).toHaveBeenCalledWith({name: "readiness",dataType: "NUMERIC",