Skip to content

🪢 Open source agent evals & observability: Trace, evaluate, and improve LLM applications with one open platform.

fix(in-app-agent): wait out MCP rate limits and retry LLM calls (#16058)

Lysander Kieself9e1108f91bb
diff --git a/packages/shared/src/in-app-agent/server/agent.ts b/packages/shared/src/in-app-agent/server/agent.ts
index a7e9ed8e21..43c4f2ca37 100644
--- a/packages/shared/src/in-app-agent/server/agent.ts
+++ b/packages/shared/src/in-app-agent/server/agent.ts
@@ -22,6 +22,10 @@ import type {
InAppAgentTracingConfig,
} from "./instrumentation";
import { createInAppAgentInstrumentation } from "./instrumentation";
+import {
+ parseMcpRateLimitError,
+ withMcpRateLimitWait,
+} from "./mcpRateLimitWait";
import { getToolFailureMessage } from "./toolErrors";
import {
createSandboxTools,
@@ -29,6 +33,7 @@ import {
filterInAppAgentAvailableLangfuseMcpTools,
getInAppAgentMcpAllowedToolNames,
getInAppAgentRegistryToolName,
+ hasCallableExecute,
type CompletedInAppAgentMcpToolCall,
type InAppAgentToolPolicy,
withOptionalSilentMcpOutput,
@@ -785,7 +790,31 @@ async function createMastraAdapter(params: {
});
try {
- const { toolsets, errors } = await mcpClient.listToolsetsWithErrors();
+ // Discovery costs one `public-api` rate limit point like any other MCP
+ // request, so a busy org can be rate limited before the run has a chance to
+ // call a single tool.
+ const { toolsets, errors } = await withMcpRateLimitWait({
+ signal: params.signal,
+ logContext: {
+ operation: "listToolsets",
+ runId: params.input.runId,
+ threadId: params.input.threadId,
+ },
+ fn: async () => {
+ const result = await mcpClient.listToolsetsWithErrors();
+
+ if (
+ result.errors.langfuse &&
+ parseMcpRateLimitError(result.errors.langfuse)
+ ) {
+ throw new Error(
+ `Failed to initialize Langfuse MCP: ${result.errors.langfuse}`,
+ );
+ }
+
+ return result;
+ },
+ });
if (errors.langfuse) {
throw new Error(`Failed to initialize Langfuse MCP: ${errors.langfuse}`);
@@ -806,13 +835,18 @@ async function createMastraAdapter(params: {
const tools = withInAppAgentToolApproval(
{
...withOptionalSilentMcpOutput({
- tools: prefixToolsetTools(
- "langfuse",
- filterInAppAgentAvailableLangfuseMcpTools({
- tools: toolsets.langfuse,
- policy: params.options.langfuseMcp.toolPolicy,
- }),
- ),
+ tools: withLangfuseMcpRateLimitWait({
+ tools: prefixToolsetTools(
+ "langfuse",
+ filterInAppAgentAvailableLangfuseMcpTools({
+ tools: toolsets.langfuse,
+ policy: params.options.langfuseMcp.toolPolicy,
+ }),
+ ),
+ signal: params.signal,
+ runId: params.input.runId,
+ threadId: params.input.threadId,
+ }),
sandbox: params.options.sandbox,
onToolCallCompleted: params.options.onMcpToolCallCompleted,
}),
@@ -853,6 +887,7 @@ async function createMastraAdapter(params: {
),
skills: LANGFUSE_IN_APP_AGENT_SKILLS,
tools,
+ maxRetries: 2,
defaultOptions: {
abortSignal: params.signal,
maxSteps: MAX_AGENT_STEPS,
@@ -943,6 +978,44 @@ function prefixToolsetTools<TTool>(
);
}
+function withLangfuseMcpRateLimitWait<TTool>(params: {
+ tools: Record<string, TTool>;
+ signal: AbortSignal;
+ runId: string;
+ threadId: string;
+}): Record<string, TTool> {
+ return Object.fromEntries(
+ Object.entries(params.tools).map(([toolName, tool]) => {
+ if (!hasCallableExecute(tool)) {
+ return [toolName, tool];
+ }
+
+ const execute = tool.execute.bind(tool) as (
+ inputData: unknown,
+ context: unknown,
+ ) => Promise<unknown>;
+
+ return [
+ toolName,
+ {
+ ...tool,
+ execute: (inputData: unknown, context: unknown) =>
+ withMcpRateLimitWait({
+ signal: params.signal,
+ logContext: {
+ operation: "tools/call",
+ toolName,
+ runId: params.runId,
+ threadId: params.threadId,
+ },
+ fn: () => execute(inputData, context),
+ }),
+ } as TTool,
+ ];
+ }),
+ );
+}
+
type MastraChunkProcessor = {
handleChunk: (chunk: unknown) => boolean;
flush: () => void;
diff --git a/packages/shared/src/in-app-agent/server/mcpRateLimitWait.test.ts b/packages/shared/src/in-app-agent/server/mcpRateLimitWait.test.ts
new file mode 100644
index 0000000000..2e5fe4af92
--- /dev/null
+++ b/packages/shared/src/in-app-agent/server/mcpRateLimitWait.test.ts
@@ -0,0 +1,159 @@
+import { afterEach, describe, expect, it, vi } from "vitest";
+
+import {
+ parseMcpRateLimitError,
+ withMcpRateLimitWait,
+} from "./mcpRateLimitWait";
+
+function rateLimitPayload(retryAfterSeconds: number) {
+ return {
+ message: "Rate limit exceeded",
+ code: "rate_limited" as const,
+ details: {
+ retryAfterSeconds,
+ limit: 30,
+ remaining: 0,
+ },
+ };
+}
+
+function rateLimitError(retryAfterSeconds: number) {
+ return new Error(JSON.stringify(rateLimitPayload(retryAfterSeconds)));
+}
+
+describe("parseMcpRateLimitError", () => {
+ it("parses a thrown Error whose message is the rate-limit JSON", () => {
+ expect(parseMcpRateLimitError(rateLimitError(12))).toEqual({
+ retryAfterSeconds: 12,
+ });
+ });
+
+ it("parses the embedded Langfuse MCP init error shape", () => {
+ const payload = JSON.stringify(rateLimitPayload(32));
+
+ expect(
+ parseMcpRateLimitError(`Failed to initialize Langfuse MCP: ${payload}`),
+ ).toEqual({ retryAfterSeconds: 32 });
+ });
+
+ it("returns null for MCP invalid-argument and schema errors", () => {
+ expect(
+ parseMcpRateLimitError(
+ new Error(
+ "MCP error -32602: Invalid params: missing required parameter 'name'",
+ ),
+ ),
+ ).toBeNull();
+ expect(
+ parseMcpRateLimitError(
+ new Error(
+ JSON.stringify({
+ jsonrpc: "2.0",
+ error: { code: -32602, message: "Invalid params" },
+ id: 1,
+ }),
+ ),
+ ),
+ ).toBeNull();
+ });
+});
+
+describe("withMcpRateLimitWait", () => {
+ afterEach(() => {
+ vi.useRealTimers();
+ });
+
+ it("sleeps the first retryAfterSeconds then succeeds", async () => {
+ vi.useFakeTimers();
+ const fn = vi
+ .fn()
+ .mockRejectedValueOnce(rateLimitError(3))
+ .mockResolvedValueOnce("ok");
+
+ const resultPromise = withMcpRateLimitWait({
+ fn,
+ logContext: { runId: "run-1" },
+ });
+
+ await vi.advanceTimersByTimeAsync(2_999);
+ expect(fn).toHaveBeenCalledTimes(1);
+
+ await vi.advanceTimersByTimeAsync(1);
+ await expect(resultPromise).resolves.toBe("ok");
+ expect(fn).toHaveBeenCalledTimes(2);
+ });
+
+ it("uses the later retryAfterSeconds on a second 429", async () => {
+ vi.useFakeTimers();
+ const fn = vi
+ .fn()
+ .mockRejectedValueOnce(rateLimitError(1))
+ .mockRejectedValueOnce(rateLimitError(4))
+ .mockResolvedValueOnce("ok");
+
+ const resultPromise = withMcpRateLimitWait({
+ fn,
+ logContext: { runId: "run-1" },
+ });
+
+ await vi.advanceTimersByTimeAsync(1_000);
+ expect(fn).toHaveBeenCalledTimes(2);
+
+ await vi.advanceTimersByTimeAsync(3_999);
+ expect(fn).toHaveBeenCalledTimes(2);
+
+ await vi.advanceTimersByTimeAsync(1);
+ await expect(resultPromise).resolves.toBe("ok");
+ expect(fn).toHaveBeenCalledTimes(3);
+ });
+
+ it("does not retry an MCP invalid-argument error with the same arguments", async () => {
+ const error = new Error(
+ "MCP error -32602: Invalid params: missing required parameter 'name'",
+ );
+ const fn = vi.fn().mockRejectedValue(error);
+
+ await expect(
+ withMcpRateLimitWait({
+ fn,
+ logContext: { runId: "run-1" },
+ }),
+ ).rejects.toBe(error);
+
+ expect(fn).toHaveBeenCalledTimes(1);
+ });
+
+ it("fails immediately when retryAfterSeconds exceeds the remaining budget", async () => {
+ const error = rateLimitError(60);
+ const fn = vi.fn().mockRejectedValue(error);
+
+ await expect(
+ withMcpRateLimitWait({
+ fn,
+ budgetMs: 5_000,
+ logContext: { runId: "run-1" },
+ }),
+ ).rejects.toBe(error);
+
+ expect(fn).toHaveBeenCalledTimes(1);
+ });
+
+ it("does not hang when aborted during the wait", async () => {
+ vi.useFakeTimers();
+ const controller = new AbortController();
+ const abortError = new Error("aborted");
+ const fn = vi.fn().mockRejectedValue(rateLimitError(3));
+
+ const resultPromise = withMcpRateLimitWait({
+ fn,
+ signal: controller.signal,
+ logContext: { runId: "run-1" },
+ });
+
+ await vi.advanceTimersByTimeAsync(0);
+ controller.abort(abortError);
+
+ await expect(resultPromise).rejects.toBe(abortError);
+ expect(fn).toHaveBeenCalledTimes(1);
+ });
+});
diff --git a/packages/shared/src/in-app-agent/server/mcpRateLimitWait.ts b/packages/shared/src/in-app-agent/server/mcpRateLimitWait.ts
new file mode 100644
index 0000000000..76fa78a2ff
--- /dev/null
+++ b/packages/shared/src/in-app-agent/server/mcpRateLimitWait.ts
@@ -0,0 +1,127 @@
+import { logger } from "../../server";
+import { InAppAgentRateLimitErrorResponseSchema } from "../schema";
+
+/**
+ * Total time a single MCP operation may spend waiting out `rate_limited`
+ * responses. Sized so a full Hobby `public-api` window (`retryAfterSeconds: 60`)
+ * can still be waited out within one operation.
+ */
+export const IN_APP_AGENT_MCP_RATE_LIMIT_WAIT_BUDGET_MS = 60_000;
+
+type McpRateLimitError = {
+ retryAfterSeconds: number;
+};
+
+/**
+ * Recognizes the public API `rate_limited` payload wherever the MCP transport
+ * surfaces it: as a structured value, as a thrown `Error`, or embedded as JSON
+ * in an error string (`Failed to initialize Langfuse MCP: ...`).
+ */
+export function parseMcpRateLimitError(
+ error: unknown,
+): McpRateLimitError | null {
+ const parsed = InAppAgentRateLimitErrorResponseSchema.safeParse(error);
+
+ if (parsed.success) {
+ return { retryAfterSeconds: parsed.data.details.retryAfterSeconds };
+ }
+
+ const message =
+ typeof error === "string"
+ ? error
+ : error instanceof Error
+ ? error.message
+ : null;
+
+ return message === null ? null : parseEmbeddedRateLimitError(message);
+}
+
+function parseEmbeddedRateLimitError(message: string) {
+ const endIndex = message.lastIndexOf("}");
+ let startIndex = message.indexOf("{");
+
+ while (startIndex !== -1 && startIndex < endIndex) {
+ try {
+ const parsed = InAppAgentRateLimitErrorResponseSchema.safeParse(
+ JSON.parse(message.slice(startIndex, endIndex + 1)),
+ );
+
+ if (parsed.success) {
+ return {
+ retryAfterSeconds: parsed.data.details.retryAfterSeconds,
+ };
+ }
+ } catch {
+ // The transport prefixes JSON with error context, so try the next object.
+ }
+
+ startIndex = message.indexOf("{", startIndex + 1);
+ }
+
+ return null;
+}
+
+/**
+ * Retries a single MCP operation while the public API answers `rate_limited`,
+ * sleeping exactly as long as the response asks for. The server escalates
+ * `retryAfterSeconds` as its window empties, so there is no local backoff
+ * ladder. A wait longer than the remaining budget fails immediately instead of
+ * sleeping and hoping.
+ */
+export async function withMcpRateLimitWait<T>(params: {
+ fn: () => Promise<T>;
+ signal?: AbortSignal;
+ budgetMs?: number;
+ logContext: Record<string, unknown>;
+}): Promise<T> {
+ let remainingBudgetMs =
+ params.budgetMs ?? IN_APP_AGENT_MCP_RATE_LIMIT_WAIT_BUDGET_MS;
+
+ for (;;) {
+ try {
+ return await params.fn();
+ } catch (error: unknown) {
+ const rateLimit = parseMcpRateLimitError(error);
+
+ if (!rateLimit) {
+ throw error;
+ }
+
+ const waitMs = rateLimit.retryAfterSeconds * 1_000;
+
+ if (waitMs > remainingBudgetMs) {
+ throw error;
+ }
+
+ logger.warn("Waiting out in-app agent MCP rate limit", {
+ ...params.logContext,
+ retryAfterSeconds: rateLimit.retryAfterSeconds,
+ remainingBudgetMs,
+ });
+
+ await sleep(waitMs, params.signal);
+ remainingBudgetMs -= waitMs;
+ }
+ }
+}
+
+function sleep(ms: number, signal?: AbortSignal) {
+ return new Promise<void>((resolve, reject) => {
+ if (signal?.aborted) {
+ reject(signal.reason);
+ return;
+ }
+
+ const onAbort = () => {
+ clearTimeout(timeout);
+ reject(signal?.reason);
+ };
+
+ const timeout = setTimeout(() => {
+ signal?.removeEventListener("abort", onAbort);
+ resolve();
+ }, ms);
+
+ signal?.addEventListener("abort", onAbort, { once: true });
+ });
+}
diff --git a/packages/shared/src/in-app-agent/server/tools.ts b/packages/shared/src/in-app-agent/server/tools.ts
index 43c420a6a9..1aa8c5aa1e 100644
--- a/packages/shared/src/in-app-agent/server/tools.ts
+++ b/packages/shared/src/in-app-agent/server/tools.ts
@@ -632,18 +632,30 @@ export type CompletedInAppAgentMcpToolCall = {
createdAt: Date;
};
+type ToolWithCallableExecute = {
+ execute: (...args: never[]) => unknown;
+};
+
type WrappableMcpTool = Pick<
Tool,
"execute" | "inputSchema" | "toModelOutput"
> &
Required<Pick<Tool, "execute" | "inputSchema">>;
-function isWrappableMcpTool(tool: unknown): tool is WrappableMcpTool {
+export function hasCallableExecute(
+ tool: unknown,
+): tool is ToolWithCallableExecute {
return (
typeof tool === "object" &&
tool !== null &&
"execute" in tool &&
- typeof tool.execute === "function" &&
+ typeof tool.execute === "function"
+ );
+}
+
+function isWrappableMcpTool(tool: unknown): tool is WrappableMcpTool {
+ return (
+ hasCallableExecute(tool) &&
"inputSchema" in tool &&
tool.inputSchema !== undefined &&
(!("toModelOutput" in tool) ||
diff --git a/web/src/__tests__/server/in-app-agent-stream.servertest.ts b/web/src/__tests__/server/in-app-agent-stream.servertest.ts
index 185d6c16b1..63517e87a5 100644
--- a/web/src/__tests__/server/in-app-agent-stream.servertest.ts
+++ b/web/src/__tests__/server/in-app-agent-stream.servertest.ts
@@ -1255,10 +1255,7 @@ describe("createAgUiStream", () => {
status: "approved",
});
- const agentConfig = vi.mocked(Agent).mock.calls[0]?.[0];
- const createScoreConfigTool =
- getAgentTools(agentConfig)?.langfuse_createScoreConfig;
- expect(createScoreConfigTool?.execute).toHaveBeenCalledWith(
+ expect(adapterEvents.createScoreConfigExecute).toHaveBeenCalledWith(
{
name: "readiness",
dataType: "NUMERIC",