Skip to content

🪢 Open source agent evals & observability: Trace, evaluate, and improve LLM applications with one open platform.

fix(in-app-agent): keep the redirect tool schema within OpenAI's accepted nesting (#16809)

David Traina5ef2c0392591
diff --git a/worker/src/features/in-app-agent/runtime/agent.test.ts b/worker/src/features/in-app-agent/runtime/agent.test.ts
index 911eae8e1a..73f786cf7f 100644
--- a/worker/src/features/in-app-agent/runtime/agent.test.ts
+++ b/worker/src/features/in-app-agent/runtime/agent.test.ts
@@ -1185,6 +1185,27 @@ describe("createAgUiStream", () => {
href: "/project/project-1/widgets/widget-1",
});
+ // Destinations without params receive {}; required params still reject it.
+ await expect(
+ redirectTool?.execute?.({
+ label: "Open alerts",
+ destination: "alerts",
+ params: {},
+ }),
+ ).resolves.toEqual({
+ type: "redirectAction",
+ label: "Open alerts",
+ href: "/project/project-1/alerts",
+ });
+
+ await expect(
+ redirectTool?.execute?.({
+ label: "Open session",
+ destination: "session",
+ params: {},
+ }),
+ ).resolves.toMatchObject({ error: true });
+
expect(promptMocks.getPrompt).toHaveBeenCalledWith(
"in-app-agent-system-prompt",
undefined,
diff --git a/worker/src/features/in-app-agent/runtime/tools.openaiSchema.test.ts b/worker/src/features/in-app-agent/runtime/tools.openaiSchema.test.ts
new file mode 100644
index 0000000000..65cb05739b
--- /dev/null
+++ b/worker/src/features/in-app-agent/runtime/tools.openaiSchema.test.ts
@@ -0,0 +1,116 @@
+import { Agent } from "@mastra/core/agent";
+import type { MastraModelConfig } from "@mastra/core/llm";
+import { describe, expect, it } from "vitest";
+
+import { IN_APP_AGENT_REDIRECT_TOOL_NAME } from "@langfuse/shared/in-app-agent";
+import { createRedirectActionTool } from "./tools";
+
+type OpenAIResponsesModel = Extract<
+ MastraModelConfig,
+ { specificationVersion: "v3" }
+>;
+type CallOptions = Parameters<OpenAIResponsesModel["doStream"]>[0];
+
+// Records the tool definitions Mastra sends and ends the loop after one step.
+function createRecordingOpenAIModel() {
+ const calls: CallOptions[] = [];
+ const respond = async (options: CallOptions) => {
+ calls.push(options);
+
+ return {
+ stream: new ReadableStream({
+ start(controller) {
+ controller.enqueue({ type: "stream-start", warnings: [] });
+ controller.enqueue({
+ type: "finish",
+ finishReason: { unified: "stop", raw: "stop" },
+ usage: {
+ inputTokens: {
+ total: 0,
+ noCache: 0,
+ cacheRead: 0,
+ cacheWrite: 0,
+ },
+ outputTokens: { total: 0, text: 0, reasoning: 0 },
+ },
+ });
+ controller.close();
+ },
+ }),
+ };
+ };
+ const model = {
+ specificationVersion: "v3",
+ provider: "openai.responses",
+ modelId: "gpt-4.1",
+ supportedUrls: {},
+ doGenerate: respond,
+ doStream: respond,
+ } satisfies OpenAIResponsesModel;
+
+ return { model, calls };
+}
+
+// Nesting depth of optional wrappers (`anyOf: [..., { type: "null" }]`).
+function maxOptionalNesting(node: unknown): number {
+ if (typeof node !== "object" || node === null) {
+ return 0;
+ }
+
+ const self =
+ Array.isArray((node as { anyOf?: unknown[] }).anyOf) &&
+ (node as { anyOf: unknown[] }).anyOf.some(
+ (branch) =>
+ typeof branch === "object" &&
+ branch !== null &&
+ (branch as { type?: unknown }).type === "null",
+ )
+ ? 1
+ : 0;
+
+ return (
+ self +
+ Math.max(
+ 0,
+ ...Object.values(node).map((child) => maxOptionalNesting(child)),
+ )
+ );
+}
+
+describe("in-app agent redirect tool schema on OpenAI", () => {
+ // Nesting 3 (~23 KB) makes OpenAI Responses return `incomplete` /
+ // `max_output_tokens` with zero usage; nesting 2 (~12 KB) is accepted.
+ it("stays within the optional nesting OpenAI accepts", async () => {
+ const { model, calls } = createRecordingOpenAIModel();
+ const agent = new Agent({
+ id: "redirect-schema-probe",
+ name: "redirect-schema-probe",
+ instructions: "probe",
+ model,
+ tools: {
+ [IN_APP_AGENT_REDIRECT_TOOL_NAME]: createRedirectActionTool({
+ projectId: "project-id",
+ isV4Enabled: true,
+ }),
+ },
+ });
+
+ const result = await agent.stream("hi");
+ await result.text;
+
+ const redirectTool = calls
+ .flatMap((call) => call.tools ?? [])
+ .find(
+ (tool) =>
+ tool.type === "function" &&
+ tool.name === IN_APP_AGENT_REDIRECT_TOOL_NAME,
+ );
+ if (redirectTool?.type !== "function") {
+ throw new Error("redirect tool was not sent to the model");
+ }
+
+ const schema = redirectTool.inputSchema;
+ expect(maxOptionalNesting(schema)).toBeLessThanOrEqual(2);
+ expect(JSON.stringify(schema).length).toBeLessThan(16_000);
+ });
+});
diff --git a/worker/src/features/in-app-agent/runtime/tools.ts b/worker/src/features/in-app-agent/runtime/tools.ts
index bf821b9598..4a4dd6a3bc 100644
--- a/worker/src/features/in-app-agent/runtime/tools.ts
+++ b/worker/src/features/in-app-agent/runtime/tools.ts
@@ -424,9 +424,11 @@ const InAppAgentRedirectParamsSchema = z.object({
timeRange: InAppAgentTableTimeRangeSchema.optional(),
});
+// Mastra's OpenAI compat layer duplicates each optional level; at three nested
+// optionals OpenAI silently rejects the tool set. Keep `params` required.
const InAppAgentRedirectToolInputSchema = InAppAgentRedirectBaseSchema.extend({
destination: InAppAgentRedirectDestinationSchema,
- params: InAppAgentRedirectParamsSchema.optional(),
+ params: InAppAgentRedirectParamsSchema,
}).superRefine((value, ctx) => {
const result = InAppAgentRedirectToolInputStrictSchema.safeParse(value);
@@ -452,7 +454,7 @@ export function createRedirectActionTool({
return createTool({
id: IN_APP_AGENT_REDIRECT_TOOL_NAME,
description:
- "Propose a user-confirmed navigation action to a known Langfuse page. This does not navigate automatically.",
+ "Propose a user-confirmed navigation action to a known Langfuse page. This does not navigate automatically. Always include params; pass {} when the destination takes none.",
inputSchema: InAppAgentRedirectToolInputSchema,
execute: async (input) => {
return getRedirectActionToolResult({