The Open Framework for Building Agents
fix(eve): send only MCP content blocks to the model (#3924)
Ruie574dd323d66
diff --git a/.changeset/mcp-tool-results-content-only.md b/.changeset/mcp-tool-results-content-only.mdnew file mode 100644index 000000000..973a190f5--- /dev/null+++ b/.changeset/mcp-tool-results-content-only.md@@ -0,0 +1,5 @@+---+"eve": patch+---++MCP tools found through `connection_search` now pass the model only their result's `content`, not the full MCP result with its duplicated `structuredContent`, so history grows about half as fast on MCP-heavy sessions.diff --git a/.github/scripts/resolve-benchmark-request.test.mjs b/.github/scripts/resolve-benchmark-request.test.mjsindex 7814e9e2c..823725f77 100644--- a/.github/scripts/resolve-benchmark-request.test.mjs+++ b/.github/scripts/resolve-benchmark-request.test.mjs@@ -232,8 +232,9 @@ test("workflow names and concurrency preserve independent authorized harness run);assert.match(workflow, /artifact-name: swe-lean-\$\{\{ needs\.request\.outputs\.slug \}\}/u);assert.match(workflow, /push:\n\s+branches: \[main\]/u);- // One sandbox per task and harness: all eight SWE-lean trials start at once.- assert.match(workflow, /concurrency: "8"/u);+ // Five attempts per task, run in waves of the action's maximum of 16 sandboxes.+ assert.match(workflow, /attempts: "5"/u);+ assert.match(workflow, /concurrency: "16"/u);assert.match(workflow, /agent-ref: \$\{\{ needs\.request\.outputs\.sha \}\}/u);});@@ -246,7 +247,7 @@ test("the consumer tracks eve-bench main and owns its model selection", async ()assert.match(workflow, /uses: \.\/\.eve-bench-action/u);assert.match(workflow,- /model: \$\{\{ vars\.EVE_CODE_BENCH_MODEL \|\| 'openai\/gpt-6-luna' \}\}/u,+ /model: \$\{\{ vars\.EVE_CODE_BENCH_MODEL \|\| 'anthropic\/claude-sonnet-5\.5' \}\}/u,);assert.match(workflow, /reasoning: low\n/u);assert.doesNotMatch(workflow, /runner-revision:/u);diff --git a/.github/workflows/eve-code-benchmark.yml b/.github/workflows/eve-code-benchmark.ymlindex f17b17761..1cae8b964 100644--- a/.github/workflows/eve-code-benchmark.yml+++ b/.github/workflows/eve-code-benchmark.yml@@ -67,8 +67,8 @@ jobs:group: eve-code-${{ needs.request.outputs.pr || github.ref }}-${{ needs.request.outputs.slug }}cancel-in-progress: trueruns-on: ubuntu-latest- # Every trial runs in parallel, so a run lasts about as long as its slowest- # trial (15-minute agent timeout plus preparation and verification).+ # Trials run in waves of up to 16 sandboxes; each trial can take up to the+ # 15-minute agent timeout plus preparation and verification.timeout-minutes: 90steps:# vercel/eve is public and eve-bench is private; GitHub never lets a public@@ -92,13 +92,16 @@ jobs:dataset: swe-lean# The PR comment's ranking tables show these plus each dimension's winner.contenders: eve-code@baseline,eve-code@head- model: ${{ vars.EVE_CODE_BENCH_MODEL || 'openai/gpt-6-luna' }}- # opencode drops Gemini's tool-call thought signatures, so Gemini stops reasoning- # after its first steps there; gpt-6-luna needs no signature to keep reasoning.+ # Sonnet keeps Anthropic's prompt-cache behavior (tools first, cache writes)+ # visible at half Opus's price. Avoid Gemini: opencode drops its tool-call+ # thought signatures, so it stops reasoning after the first steps there.+ model: ${{ vars.EVE_CODE_BENCH_MODEL || 'anthropic/claude-sonnet-5.5' }}reasoning: low- attempts: "1"- # Per harness: one sandbox per task, so all trials start at once.- concurrency: "8"+ # One attempt per task could not separate a 10-15% token change from+ # trial noise; five gives the paired bootstrap enough samples.+ attempts: "5"+ # The action's maximum. 7 tasks x 5 attempts per contender run in waves.+ concurrency: "16"native-build: "false"# Baselines and cross-harness references resolve automatically from the# durable result store; only dispatch/push/schedule runs publish to it.diff --git a/packages/eve/src/execution/tools/connection-search.test.ts b/packages/eve/src/execution/tools/connection-search.test.tsindex 2ea8b31b8..5e6563651 100644--- a/packages/eve/src/execution/tools/connection-search.test.ts+++ b/packages/eve/src/execution/tools/connection-search.test.ts@@ -241,6 +241,66 @@ describe("connection dynamic tools", () => {});});+ it("delegates MCP model output to the connection client and leaves OpenAPI results whole", async () => {+ const projected = { type: "content", value: [{ type: "text", text: '{"count":2}' }] };+ const toModelOutput = vi.fn(async (_toolName: string, _output: unknown) => projected);+ const base = registry({+ connections: [+ connection("linear"),+ { ...connection("billing"), protocol: "openapi", url: "https://billing.example.com" },+ ],+ loadTools: {+ billing: async () => [+ { description: "List invoices", inputSchema: { type: "object" }, name: "list_invoices" },+ ],+ linear: async () => [+ { description: "List issues", inputSchema: { type: "object" }, name: "list_issues" },+ ],+ },+ });+ const ctx = new ContextContainer();+ ctx.set(ConnectionRegistryKey, {+ ...base,+ getClient: (name) => ({ ...base.getClient(name), toModelOutput }),+ });+ const result = {+ content: [{ text: '{"count":2}', type: "text" }],+ structuredContent: { count: 2 },+ };++ await contextStorage.run(ctx, async () => {+ const resolve = getConnectionSearchResolver().events["step.started"]!;+ const context = {+ channel: {},+ model: null,+ messages: [],+ session: { auth: { current: null, initiator: null }, id: "projection-test" },+ } satisfies DynamicResolveContext;+ const initial = (await resolve({}, context)) as DynamicToolSet;+ await initial["connection_search"]!.execute({ keywords: "list" }, {} as ToolContext);+ const tools = (await resolve({}, context)) as DynamicToolSet;++ const linear = tools["linear__list_issues"] as {+ toModelOutput?: (output: unknown) => Promise<unknown>;+ };+ await expect(linear.toModelOutput?.(result)).resolves.toBe(projected);+ const durable = readDurableDynamicToolCallbacks(+ tools["linear__list_issues"]!,+ )!.toModelOutput!;+ await expect(durable.callback(durable.closure, result as never)).resolves.toBe(projected);+ expect(toModelOutput.mock.calls).toEqual([+ ["list_issues", result],+ ["list_issues", result],+ ]);++ const billing = tools["billing__list_invoices"] as { toModelOutput?: unknown };+ expect(billing.toModelOutput).toBeUndefined();+ expect(+ readDurableDynamicToolCallbacks(tools["billing__list_invoices"]!)?.toModelOutput,+ ).toBeUndefined();+ });+ });+it("forwards the authored tool call ID to connection execution", async () => {const linear = connection("linear");const executeTool = vi.fn(diff --git a/packages/eve/src/execution/tools/connection-search.ts b/packages/eve/src/execution/tools/connection-search.tsindex 7dcbd84e6..8f9bd1886 100644--- a/packages/eve/src/execution/tools/connection-search.ts+++ b/packages/eve/src/execution/tools/connection-search.ts@@ -12,6 +12,7 @@ import {} from "#connections/errors.js";import { defineTool } from "#tools/definition.js";import type { ToolContext } from "#tools/definition.js";+import type { ToolModelOutput } from "#tools/model-output.js";import {resolveApprovalPolicy,type ApprovalContext,@@ -382,6 +383,18 @@ async function executeDiscoveredConnectionTool(}}+/** Delegates to the connection client's projection, falling back to the raw result as JSON. */+async function discoveredConnectionToolModelOutput(+ closure: JsonObject,+ output: unknown,+): Promise<ToolModelOutput> {+ const { connectionName, toolName } = readDiscoveredToolClosure(closure);+ const client = loadContext().get(ConnectionRegistryKey)?.getClient(connectionName);+ const modelOutput = await client?.toModelOutput?.(toolName, output);+ // The harness validates the shape in `normalizeToolModelOutput`.+ return (modelOutput ?? { type: "json", value: output ?? null }) as ToolModelOutput;+}+function assertPendingConnectionAuthorizationInstances(registry: ConnectionRegistry): void {const connections = new Map(registry.getConnections().map((connection) => [connection.connectionName, connection]),@@ -459,9 +472,11 @@ export async function resolveConnectionSearchDynamicTools() {const toolName = result.tool!;const approval = registry.getConnectionApproval(connectionName);- const instanceId = connections.find(+ const resolvedConnection = connections.find((connection) => connection.connectionName === connectionName,- )?.instanceId;+ );+ const instanceId = resolvedConnection?.instanceId;+ const projectsMcpResult = resolvedConnection?.protocol === "mcp";const closure: { connectionName: string; toolName: string; instanceId?: string } = {connectionName,toolName,@@ -477,9 +492,15 @@ export async function resolveConnectionSearchDynamicTools() {async execute(input: Record<string, unknown>, executeCtx) {return await executeDiscoveredConnectionTool(closure, input, executeCtx);},+ toModelOutput: projectsMcpResult+ ? (output: unknown) => discoveredConnectionToolModelOutput(closure, output)+ : undefined,});stampDurableDynamicToolCallbacks(discoveredTool, {execute: { callback: executeDiscoveredConnectionTool, closure },+ toModelOutput: projectsMcpResult+ ? { callback: discoveredConnectionToolModelOutput, closure }+ : undefined,...(approval === undefined? {}: {diff --git a/packages/eve/src/runtime/connections/mcp-client.integration.test.ts b/packages/eve/src/runtime/connections/mcp-client.integration.test.tsindex fcb6f0179..2bc3c2d4f 100644--- a/packages/eve/src/runtime/connections/mcp-client.integration.test.ts+++ b/packages/eve/src/runtime/connections/mcp-client.integration.test.ts@@ -19,6 +19,7 @@ describe.each(["static", "dynamic"] as const)(let client: McpConnectionClient;let discoveryError: { code: number; message: string; data?: unknown } | undefined;let initializeError: { code: number; message: string } | undefined;+ let callResult: Record<string, unknown>;async function createClient(protocolVersionDiscovery?: boolean) {const authored = defineMcpClientConnection({@@ -67,6 +68,7 @@ describe.each(["static", "dynamic"] as const)(message: "Unsupported protocol version",};initializeError = undefined;+ callResult = { content: [{ type: "text", text: "fixture-user" }] };// Replace only HTTP I/O: eve's connection, bundled SDK, and protocol parsing stay real.vi.stubGlobal("fetch", async (input: string | URL | Request, init?: RequestInit) => {@@ -112,7 +114,7 @@ describe.each(["static", "dynamic"] as const)(break;case "tools/call":expect(message.params?.name).toBe("getMyUser");- result = { content: [{ type: "text", text: "fixture-user" }] };+ result = callResult;break;default:throw new Error(`Unexpected MCP method: ${message.method}`);@@ -147,6 +149,42 @@ describe.each(["static", "dynamic"] as const)(]);});+ it("projects call results for the model with the SDK's MCP conversion", async () => {+ await createClient(false);+ const project = async (result: Record<string, unknown>) => {+ callResult = result;+ const output = await client.executeTool("getMyUser", {}, { callId: "read-user" });+ return client.toModelOutput("getMyUser", output);+ };++ await expect(+ project({+ _meta: { "io.modelcontextprotocol/serverInfo": { name: "fixture", version: "1" } },+ content: [{ type: "text", text: '{"seats":132}' }],+ structuredContent: { seats: 132 },+ }),+ ).resolves.toEqual({ type: "content", value: [{ type: "text", text: '{"seats":132}' }] });+ await expect(+ project({ content: [{ type: "text", text: "no such user" }], isError: true }),+ ).resolves.toEqual({+ type: "content",+ value: [+ { type: "text", text: "Tool call failed:" },+ { type: "text", text: "no such user" },+ ],+ });+ await expect(+ project({ isError: true, structuredContent: { code: "NOT_FOUND" } }),+ ).resolves.toEqual({+ type: "content",+ value: [+ { type: "text", text: "Tool call failed:" },+ { type: "text", text: '{"code":"NOT_FOUND"}' },+ ],+ });+ await expect(client.toModelOutput("missing", {})).resolves.toBeUndefined();+ });+it.each([undefined, true])("keeps discovery enabled when the option is %s", async (option) => {await createClient(option);discoveryError = undefined;diff --git a/packages/eve/src/runtime/connections/mcp-client.ts b/packages/eve/src/runtime/connections/mcp-client.tsindex 0cd887421..e2e1af3dd 100644--- a/packages/eve/src/runtime/connections/mcp-client.ts+++ b/packages/eve/src/runtime/connections/mcp-client.ts@@ -158,6 +158,27 @@ export class McpConnectionClient implements ConnectionClient {}}+ /**+ * Projects a `tools/call` result with the AI SDK tool's own+ * `toModelOutput`: the result's `content` blocks as text and file parts,+ * without the `_meta` envelope or the `structuredContent` copy. That+ * conversion drops `isError`, so failed calls get a leading marker.+ */+ async toModelOutput(toolName: string, output: unknown): Promise<unknown> {+ const { tools } = await this.#ensureTools();+ const convert = tools[toolName]?.toModelOutput;+ if (convert === undefined) return undefined;+ // The SDK conversion reads only `output`.+ const modelOutput = await convert({ input: undefined, output, toolCallId: "" });+ if (!isObject(output) || output.isError !== true || modelOutput.type !== "content") {+ return modelOutput;+ }+ return {+ type: "content",+ value: [{ type: "text", text: "Tool call failed:" }, ...modelOutput.value],+ };+ }+async #ensureTools(): Promise<McpToolCache> {if (this.#tools !== undefined) {return this.#tools;diff --git a/packages/eve/src/shared/connection-types.ts b/packages/eve/src/shared/connection-types.tsindex bd6d2c41a..4c6d86829 100644--- a/packages/eve/src/shared/connection-types.ts+++ b/packages/eve/src/shared/connection-types.ts@@ -467,4 +467,9 @@ export interface ConnectionClient {options: ConnectionToolExecuteOptions,): Promise<unknown>;getToolMetadata(): Promise<readonly ConnectionToolMetadata[]>;+ /**+ * Projects an {@link executeTool} result to what the model sees, when the+ * protocol defines a projection. Resolves `undefined` when it does not.+ */+ toModelOutput?(toolName: string, output: unknown): Promise<unknown>;}