Skip to content

armgit / eve

Public Imported from GitHub
0

The Open Framework for Building Agents

fix(eve): send only MCP content blocks to the model (#3924)

Ruie574dd323d66
diff --git a/.changeset/mcp-tool-results-content-only.md b/.changeset/mcp-tool-results-content-only.md
new file mode 100644
index 000000000..973a190f5
--- /dev/null
+++ b/.changeset/mcp-tool-results-content-only.md
@@ -0,0 +1,5 @@
+---
+"eve": patch
+---
+
+MCP tools found through `connection_search` now pass the model only their result's `content`, not the full MCP result with its duplicated `structuredContent`, so history grows about half as fast on MCP-heavy sessions.
diff --git a/.github/scripts/resolve-benchmark-request.test.mjs b/.github/scripts/resolve-benchmark-request.test.mjs
index 7814e9e2c..823725f77 100644
--- a/.github/scripts/resolve-benchmark-request.test.mjs
+++ b/.github/scripts/resolve-benchmark-request.test.mjs
@@ -232,8 +232,9 @@ test("workflow names and concurrency preserve independent authorized harness run
);
assert.match(workflow, /artifact-name: swe-lean-\$\{\{ needs\.request\.outputs\.slug \}\}/u);
assert.match(workflow, /push:\n\s+branches: \[main\]/u);
- // One sandbox per task and harness: all eight SWE-lean trials start at once.
- assert.match(workflow, /concurrency: "8"/u);
+ // Five attempts per task, run in waves of the action's maximum of 16 sandboxes.
+ assert.match(workflow, /attempts: "5"/u);
+ assert.match(workflow, /concurrency: "16"/u);
assert.match(workflow, /agent-ref: \$\{\{ needs\.request\.outputs\.sha \}\}/u);
});
@@ -246,7 +247,7 @@ test("the consumer tracks eve-bench main and owns its model selection", async ()
assert.match(workflow, /uses: \.\/\.eve-bench-action/u);
assert.match(
workflow,
- /model: \$\{\{ vars\.EVE_CODE_BENCH_MODEL \|\| 'openai\/gpt-6-luna' \}\}/u,
+ /model: \$\{\{ vars\.EVE_CODE_BENCH_MODEL \|\| 'anthropic\/claude-sonnet-5\.5' \}\}/u,
);
assert.match(workflow, /reasoning: low\n/u);
assert.doesNotMatch(workflow, /runner-revision:/u);
diff --git a/.github/workflows/eve-code-benchmark.yml b/.github/workflows/eve-code-benchmark.yml
index f17b17761..1cae8b964 100644
--- a/.github/workflows/eve-code-benchmark.yml
+++ b/.github/workflows/eve-code-benchmark.yml
@@ -67,8 +67,8 @@ jobs:
group: eve-code-${{ needs.request.outputs.pr || github.ref }}-${{ needs.request.outputs.slug }}
cancel-in-progress: true
runs-on: ubuntu-latest
- # Every trial runs in parallel, so a run lasts about as long as its slowest
- # trial (15-minute agent timeout plus preparation and verification).
+ # Trials run in waves of up to 16 sandboxes; each trial can take up to the
+ # 15-minute agent timeout plus preparation and verification.
timeout-minutes: 90
steps:
# vercel/eve is public and eve-bench is private; GitHub never lets a public
@@ -92,13 +92,16 @@ jobs:
dataset: swe-lean
# The PR comment's ranking tables show these plus each dimension's winner.
contenders: eve-code@baseline,eve-code@head
- model: ${{ vars.EVE_CODE_BENCH_MODEL || 'openai/gpt-6-luna' }}
- # opencode drops Gemini's tool-call thought signatures, so Gemini stops reasoning
- # after its first steps there; gpt-6-luna needs no signature to keep reasoning.
+ # Sonnet keeps Anthropic's prompt-cache behavior (tools first, cache writes)
+ # visible at half Opus's price. Avoid Gemini: opencode drops its tool-call
+ # thought signatures, so it stops reasoning after the first steps there.
+ model: ${{ vars.EVE_CODE_BENCH_MODEL || 'anthropic/claude-sonnet-5.5' }}
reasoning: low
- attempts: "1"
- # Per harness: one sandbox per task, so all trials start at once.
- concurrency: "8"
+ # One attempt per task could not separate a 10-15% token change from
+ # trial noise; five gives the paired bootstrap enough samples.
+ attempts: "5"
+ # The action's maximum. 7 tasks x 5 attempts per contender run in waves.
+ concurrency: "16"
native-build: "false"
# Baselines and cross-harness references resolve automatically from the
# durable result store; only dispatch/push/schedule runs publish to it.
diff --git a/packages/eve/src/execution/tools/connection-search.test.ts b/packages/eve/src/execution/tools/connection-search.test.ts
index 2ea8b31b8..5e6563651 100644
--- a/packages/eve/src/execution/tools/connection-search.test.ts
+++ b/packages/eve/src/execution/tools/connection-search.test.ts
@@ -241,6 +241,66 @@ describe("connection dynamic tools", () => {
});
});
+ it("delegates MCP model output to the connection client and leaves OpenAPI results whole", async () => {
+ const projected = { type: "content", value: [{ type: "text", text: '{"count":2}' }] };
+ const toModelOutput = vi.fn(async (_toolName: string, _output: unknown) => projected);
+ const base = registry({
+ connections: [
+ connection("linear"),
+ { ...connection("billing"), protocol: "openapi", url: "https://billing.example.com" },
+ ],
+ loadTools: {
+ billing: async () => [
+ { description: "List invoices", inputSchema: { type: "object" }, name: "list_invoices" },
+ ],
+ linear: async () => [
+ { description: "List issues", inputSchema: { type: "object" }, name: "list_issues" },
+ ],
+ },
+ });
+ const ctx = new ContextContainer();
+ ctx.set(ConnectionRegistryKey, {
+ ...base,
+ getClient: (name) => ({ ...base.getClient(name), toModelOutput }),
+ });
+ const result = {
+ content: [{ text: '{"count":2}', type: "text" }],
+ structuredContent: { count: 2 },
+ };
+
+ await contextStorage.run(ctx, async () => {
+ const resolve = getConnectionSearchResolver().events["step.started"]!;
+ const context = {
+ channel: {},
+ model: null,
+ messages: [],
+ session: { auth: { current: null, initiator: null }, id: "projection-test" },
+ } satisfies DynamicResolveContext;
+ const initial = (await resolve({}, context)) as DynamicToolSet;
+ await initial["connection_search"]!.execute({ keywords: "list" }, {} as ToolContext);
+ const tools = (await resolve({}, context)) as DynamicToolSet;
+
+ const linear = tools["linear__list_issues"] as {
+ toModelOutput?: (output: unknown) => Promise<unknown>;
+ };
+ await expect(linear.toModelOutput?.(result)).resolves.toBe(projected);
+ const durable = readDurableDynamicToolCallbacks(
+ tools["linear__list_issues"]!,
+ )!.toModelOutput!;
+ await expect(durable.callback(durable.closure, result as never)).resolves.toBe(projected);
+ expect(toModelOutput.mock.calls).toEqual([
+ ["list_issues", result],
+ ["list_issues", result],
+ ]);
+
+ const billing = tools["billing__list_invoices"] as { toModelOutput?: unknown };
+ expect(billing.toModelOutput).toBeUndefined();
+ expect(
+ readDurableDynamicToolCallbacks(tools["billing__list_invoices"]!)?.toModelOutput,
+ ).toBeUndefined();
+ });
+ });
+
it("forwards the authored tool call ID to connection execution", async () => {
const linear = connection("linear");
const executeTool = vi.fn(
diff --git a/packages/eve/src/execution/tools/connection-search.ts b/packages/eve/src/execution/tools/connection-search.ts
index 7dcbd84e6..8f9bd1886 100644
--- a/packages/eve/src/execution/tools/connection-search.ts
+++ b/packages/eve/src/execution/tools/connection-search.ts
@@ -12,6 +12,7 @@ import {
} from "#connections/errors.js";
import { defineTool } from "#tools/definition.js";
import type { ToolContext } from "#tools/definition.js";
+import type { ToolModelOutput } from "#tools/model-output.js";
import {
resolveApprovalPolicy,
type ApprovalContext,
@@ -382,6 +383,18 @@ async function executeDiscoveredConnectionTool(
}
}
+/** Delegates to the connection client's projection, falling back to the raw result as JSON. */
+async function discoveredConnectionToolModelOutput(
+ closure: JsonObject,
+ output: unknown,
+): Promise<ToolModelOutput> {
+ const { connectionName, toolName } = readDiscoveredToolClosure(closure);
+ const client = loadContext().get(ConnectionRegistryKey)?.getClient(connectionName);
+ const modelOutput = await client?.toModelOutput?.(toolName, output);
+ // The harness validates the shape in `normalizeToolModelOutput`.
+ return (modelOutput ?? { type: "json", value: output ?? null }) as ToolModelOutput;
+}
+
function assertPendingConnectionAuthorizationInstances(registry: ConnectionRegistry): void {
const connections = new Map(
registry.getConnections().map((connection) => [connection.connectionName, connection]),
@@ -459,9 +472,11 @@ export async function resolveConnectionSearchDynamicTools() {
const toolName = result.tool!;
const approval = registry.getConnectionApproval(connectionName);
- const instanceId = connections.find(
+ const resolvedConnection = connections.find(
(connection) => connection.connectionName === connectionName,
- )?.instanceId;
+ );
+ const instanceId = resolvedConnection?.instanceId;
+ const projectsMcpResult = resolvedConnection?.protocol === "mcp";
const closure: { connectionName: string; toolName: string; instanceId?: string } = {
connectionName,
toolName,
@@ -477,9 +492,15 @@ export async function resolveConnectionSearchDynamicTools() {
async execute(input: Record<string, unknown>, executeCtx) {
return await executeDiscoveredConnectionTool(closure, input, executeCtx);
},
+ toModelOutput: projectsMcpResult
+ ? (output: unknown) => discoveredConnectionToolModelOutput(closure, output)
+ : undefined,
});
stampDurableDynamicToolCallbacks(discoveredTool, {
execute: { callback: executeDiscoveredConnectionTool, closure },
+ toModelOutput: projectsMcpResult
+ ? { callback: discoveredConnectionToolModelOutput, closure }
+ : undefined,
...(approval === undefined
? {}
: {
diff --git a/packages/eve/src/runtime/connections/mcp-client.integration.test.ts b/packages/eve/src/runtime/connections/mcp-client.integration.test.ts
index fcb6f0179..2bc3c2d4f 100644
--- a/packages/eve/src/runtime/connections/mcp-client.integration.test.ts
+++ b/packages/eve/src/runtime/connections/mcp-client.integration.test.ts
@@ -19,6 +19,7 @@ describe.each(["static", "dynamic"] as const)(
let client: McpConnectionClient;
let discoveryError: { code: number; message: string; data?: unknown } | undefined;
let initializeError: { code: number; message: string } | undefined;
+ let callResult: Record<string, unknown>;
async function createClient(protocolVersionDiscovery?: boolean) {
const authored = defineMcpClientConnection({
@@ -67,6 +68,7 @@ describe.each(["static", "dynamic"] as const)(
message: "Unsupported protocol version",
};
initializeError = undefined;
+ callResult = { content: [{ type: "text", text: "fixture-user" }] };
// Replace only HTTP I/O: eve's connection, bundled SDK, and protocol parsing stay real.
vi.stubGlobal("fetch", async (input: string | URL | Request, init?: RequestInit) => {
@@ -112,7 +114,7 @@ describe.each(["static", "dynamic"] as const)(
break;
case "tools/call":
expect(message.params?.name).toBe("getMyUser");
- result = { content: [{ type: "text", text: "fixture-user" }] };
+ result = callResult;
break;
default:
throw new Error(`Unexpected MCP method: ${message.method}`);
@@ -147,6 +149,42 @@ describe.each(["static", "dynamic"] as const)(
]);
});
+ it("projects call results for the model with the SDK's MCP conversion", async () => {
+ await createClient(false);
+ const project = async (result: Record<string, unknown>) => {
+ callResult = result;
+ const output = await client.executeTool("getMyUser", {}, { callId: "read-user" });
+ return client.toModelOutput("getMyUser", output);
+ };
+
+ await expect(
+ project({
+ _meta: { "io.modelcontextprotocol/serverInfo": { name: "fixture", version: "1" } },
+ content: [{ type: "text", text: '{"seats":132}' }],
+ structuredContent: { seats: 132 },
+ }),
+ ).resolves.toEqual({ type: "content", value: [{ type: "text", text: '{"seats":132}' }] });
+ await expect(
+ project({ content: [{ type: "text", text: "no such user" }], isError: true }),
+ ).resolves.toEqual({
+ type: "content",
+ value: [
+ { type: "text", text: "Tool call failed:" },
+ { type: "text", text: "no such user" },
+ ],
+ });
+ await expect(
+ project({ isError: true, structuredContent: { code: "NOT_FOUND" } }),
+ ).resolves.toEqual({
+ type: "content",
+ value: [
+ { type: "text", text: "Tool call failed:" },
+ { type: "text", text: '{"code":"NOT_FOUND"}' },
+ ],
+ });
+ await expect(client.toModelOutput("missing", {})).resolves.toBeUndefined();
+ });
+
it.each([undefined, true])("keeps discovery enabled when the option is %s", async (option) => {
await createClient(option);
discoveryError = undefined;
diff --git a/packages/eve/src/runtime/connections/mcp-client.ts b/packages/eve/src/runtime/connections/mcp-client.ts
index 0cd887421..e2e1af3dd 100644
--- a/packages/eve/src/runtime/connections/mcp-client.ts
+++ b/packages/eve/src/runtime/connections/mcp-client.ts
@@ -158,6 +158,27 @@ export class McpConnectionClient implements ConnectionClient {
}
}
+ /**
+ * Projects a `tools/call` result with the AI SDK tool's own
+ * `toModelOutput`: the result's `content` blocks as text and file parts,
+ * without the `_meta` envelope or the `structuredContent` copy. That
+ * conversion drops `isError`, so failed calls get a leading marker.
+ */
+ async toModelOutput(toolName: string, output: unknown): Promise<unknown> {
+ const { tools } = await this.#ensureTools();
+ const convert = tools[toolName]?.toModelOutput;
+ if (convert === undefined) return undefined;
+ // The SDK conversion reads only `output`.
+ const modelOutput = await convert({ input: undefined, output, toolCallId: "" });
+ if (!isObject(output) || output.isError !== true || modelOutput.type !== "content") {
+ return modelOutput;
+ }
+ return {
+ type: "content",
+ value: [{ type: "text", text: "Tool call failed:" }, ...modelOutput.value],
+ };
+ }
+
async #ensureTools(): Promise<McpToolCache> {
if (this.#tools !== undefined) {
return this.#tools;
diff --git a/packages/eve/src/shared/connection-types.ts b/packages/eve/src/shared/connection-types.ts
index bd6d2c41a..4c6d86829 100644
--- a/packages/eve/src/shared/connection-types.ts
+++ b/packages/eve/src/shared/connection-types.ts
@@ -467,4 +467,9 @@ export interface ConnectionClient {
options: ConnectionToolExecuteOptions,
): Promise<unknown>;
getToolMetadata(): Promise<readonly ConnectionToolMetadata[]>;
+ /**
+ * Projects an {@link executeTool} result to what the model sees, when the
+ * protocol defines a projection. Resolves `undefined` when it does not.
+ */
+ toModelOutput?(toolName: string, output: unknown): Promise<unknown>;
}