The Open Framework for Building Agents
fix(eve): web search - preserve provider result ordering (#334)
Andrew Barbad8449cf60312
diff --git a/.changeset/quiet-taxis-search.md b/.changeset/quiet-taxis-search.mdnew file mode 100644index 000000000..e7e1d038e--- /dev/null+++ b/.changeset/quiet-taxis-search.md@@ -0,0 +1,5 @@+---+"eve": patch+---++Keep provider-managed web search calls replayable when the model emits narration before results or when the provider returns an error.diff --git a/e2e/fixtures/agent-tools/evals/static-tools/web-search-parallel.eval.ts b/e2e/fixtures/agent-tools/evals/static-tools/web-search-parallel.eval.tsnew file mode 100644index 000000000..2a64aad9d--- /dev/null+++ b/e2e/fixtures/agent-tools/evals/static-tools/web-search-parallel.eval.ts@@ -0,0 +1,21 @@+import { defineEval } from "eve/evals";++const EXPECTED_WINNERS =+ "2026 New York Knicks; 2025 Oklahoma City Thunder; 2024 Boston Celtics; " ++ "2023 Denver Nuggets; 2022 Golden State Warriors; 2021 Milwaukee Bucks; " ++ "2020 Los Angeles Lakers; 2019 Toronto Raptors; 2018 Golden State Warriors; " ++ "2017 Golden State Warriors.";++export default defineEval({+ description: "Provider tools smoke: ten parallel gateway web searches complete successfully.",+ async test(t) {+ const turn = await t.send(+ "Using 10 parallel web_search calls: lookup the nba finals winner from 2026 back to 2017",+ );++ t.succeeded();+ t.calledTool("web_search", { count: 10 });+ t.noFailedActions();+ t.judge.autoevals.factuality(EXPECTED_WINNERS, { on: turn.message }).atLeast(0.5);+ },+});diff --git a/packages/eve/src/harness/provider-tool-history.test.ts b/packages/eve/src/harness/provider-tool-history.test.tsnew file mode 100644index 000000000..94a517378--- /dev/null+++ b/packages/eve/src/harness/provider-tool-history.test.ts@@ -0,0 +1,144 @@+import type { ModelMessage } from "ai";+import { describe, expect, it } from "vitest";++import { normalizeProviderToolHistory } from "#harness/provider-tool-history.js";++describe("normalizeProviderToolHistory", () => {+ it("preserves text ordering around provider-executed tool results", () => {+ const messages: ModelMessage[] = [+ {+ role: "assistant",+ content: [+ { type: "text", text: "I will look that up." },+ {+ type: "tool-call",+ toolCallId: "search-1",+ toolName: "web_search",+ input: { objective: "Current result" },+ },+ {+ type: "tool-result",+ toolCallId: "search-1",+ toolName: "web_search",+ output: { type: "json", value: { results: [] } },+ },+ { type: "text", text: "The search returned no results." },+ ],+ },+ ];++ const normalized = normalizeProviderToolHistory({+ messages,+ providerExecutedOutcomeIds: new Set(["search-1"]),+ });++ expect(normalized.messages).toEqual([+ {+ role: "assistant",+ content: [+ { type: "text", text: "I will look that up." },+ {+ type: "tool-call",+ toolCallId: "search-1",+ toolName: "web_search",+ input: { objective: "Current result" },+ providerExecuted: false,+ },+ ],+ },+ {+ role: "tool",+ content: [+ {+ type: "tool-result",+ toolCallId: "search-1",+ toolName: "web_search",+ output: { type: "json", value: { results: [] } },+ },+ ],+ },+ {+ role: "assistant",+ content: [{ type: "text", text: "The search returned no results." }],+ },+ ]);+ expect(normalized.outcomeEndsResponse).toBe(false);+ });++ it("groups consecutive provider results without changing their order", () => {+ const messages: ModelMessage[] = [+ {+ role: "assistant",+ content: [+ {+ type: "tool-call",+ toolCallId: "search-1",+ toolName: "web_search",+ input: { objective: "First" },+ },+ {+ type: "tool-call",+ toolCallId: "search-2",+ toolName: "web_search",+ input: { objective: "Second" },+ },+ {+ type: "tool-result",+ toolCallId: "search-2",+ toolName: "web_search",+ output: { type: "json", value: { results: [2] } },+ },+ {+ type: "tool-result",+ toolCallId: "search-1",+ toolName: "web_search",+ output: { type: "json", value: { results: [1] } },+ },+ ],+ },+ ];++ const normalized = normalizeProviderToolHistory({+ messages,+ providerExecutedOutcomeIds: new Set(["search-1", "search-2"]),+ });++ expect(normalized.messages.map((message) => message.role)).toEqual(["assistant", "tool"]);+ expect(normalized.messages[1]?.content).toEqual([+ expect.objectContaining({ toolCallId: "search-2" }),+ expect.objectContaining({ toolCallId: "search-1" }),+ ]);+ expect(normalized.outcomeEndsResponse).toBe(true);+ });++ it("preserves native provider-owned tool history", () => {+ const messages: ModelMessage[] = [+ {+ role: "assistant",+ content: [+ {+ type: "tool-call",+ toolCallId: "search-1",+ toolName: "web_search",+ input: { query: "Current result" },+ providerExecuted: true,+ },+ {+ type: "tool-result",+ toolCallId: "search-1",+ toolName: "web_search",+ output: { type: "json", value: { results: [] } },+ },+ ],+ },+ ];++ const normalized = normalizeProviderToolHistory({+ messages,+ providerExecutedOutcomeIds: new Set(["search-1"]),+ });++ expect(normalized.messages).toEqual(messages);+ expect(normalized.outcomeEndsResponse).toBe(true);+ });+});diff --git a/packages/eve/src/harness/provider-tool-history.ts b/packages/eve/src/harness/provider-tool-history.tsnew file mode 100644index 000000000..b95d70039--- /dev/null+++ b/packages/eve/src/harness/provider-tool-history.ts@@ -0,0 +1,107 @@+import type { ModelMessage, ToolModelMessage } from "ai";++/**+ * Converts provider-executed tool outcomes into replay-safe model messages.+ *+ * Provider SDKs can return a tool call and its result inside one assistant+ * message. When the result is provider-executed but the call lacks the matching+ * marker, each matching call is rewritten as a normal call and consecutive+ * results are moved into a tool message at their original position. Native+ * provider-owned call/result pairs remain untouched. Text before a result+ * remains before it; text after a result remains after it.+ */+export function normalizeProviderToolHistory(input: {+ readonly messages: readonly ModelMessage[];+ readonly providerExecutedOutcomeIds: ReadonlySet<string>;+}): { readonly messages: ModelMessage[]; readonly outcomeEndsResponse: boolean } {+ const toolCallIdsToNormalize = findUnmarkedProviderToolCalls(input);+ if (input.providerExecutedOutcomeIds.size === 0) {+ return { messages: [...input.messages], outcomeEndsResponse: false };+ }++ const normalized: ModelMessage[] = [];+ let lastOutcomePosition = -1;+ let lastTextPosition = -1;+ let position = 0;++ for (const message of input.messages) {+ if (message.role !== "assistant" || !Array.isArray(message.content)) {+ normalized.push(message);+ if (+ message.role === "assistant" &&+ typeof message.content === "string" &&+ message.content.trim().length > 0+ ) {+ lastTextPosition = position;+ }+ position += 1;+ continue;+ }++ let assistantContent: typeof message.content = [];+ let toolContent: ToolModelMessage["content"] = [];++ const flushAssistant = (): void => {+ if (assistantContent.length === 0) return;+ normalized.push({ ...message, content: assistantContent });+ assistantContent = [];+ };+ const flushTool = (): void => {+ if (toolContent.length === 0) return;+ normalized.push({ role: "tool", content: toolContent });+ toolContent = [];+ };++ for (const part of message.content) {+ if (part.type === "tool-result" && toolCallIdsToNormalize.has(part.toolCallId)) {+ flushAssistant();+ toolContent.push(part);+ } else {+ flushTool();+ assistantContent.push(+ part.type === "tool-call" && toolCallIdsToNormalize.has(part.toolCallId)+ ? { ...part, providerExecuted: false }+ : part,+ );+ }++ if (part.type === "tool-result" && input.providerExecutedOutcomeIds.has(part.toolCallId)) {+ lastOutcomePosition = position;+ } else if (part.type === "text" && part.text.trim().length > 0) {+ lastTextPosition = position;+ }+ position += 1;+ }++ flushAssistant();+ flushTool();+ }++ return {+ messages: normalized,+ outcomeEndsResponse: lastOutcomePosition >= 0 && lastOutcomePosition > lastTextPosition,+ };+}++function findUnmarkedProviderToolCalls(input: {+ readonly messages: readonly ModelMessage[];+ readonly providerExecutedOutcomeIds: ReadonlySet<string>;+}): ReadonlySet<string> {+ const result = new Set<string>();++ for (const message of input.messages) {+ if (message.role !== "assistant" || !Array.isArray(message.content)) continue;++ for (const part of message.content) {+ if (+ part.type === "tool-call" &&+ part.providerExecuted !== true &&+ input.providerExecutedOutcomeIds.has(part.toolCallId)+ ) {+ result.add(part.toolCallId);+ }+ }+ }++ return result;+}diff --git a/packages/eve/src/harness/tool-loop.test.ts b/packages/eve/src/harness/tool-loop.test.tsindex 28a114835..e70e77524 100644--- a/packages/eve/src/harness/tool-loop.test.ts+++ b/packages/eve/src/harness/tool-loop.test.ts@@ -1,5 +1,12 @@import { context as otelContext, trace } from "#compiled/@opentelemetry/api/index.js";-import { type FilePart, jsonSchema, type LanguageModel, ToolLoopAgent, type UserContent } from "ai";+import {+ type FilePart,+ jsonSchema,+ type LanguageModel,+ type ModelMessage,+ ToolLoopAgent,+ type UserContent,+} from "ai";import { afterEach, describe, expect, it, vi } from "vitest";import { ContextContainer, contextStorage } from "#context/container.js";@@ -4058,7 +4065,7 @@ describe("createToolLoopHarness", () => {createTestConfig("conversation", emit, { tools: new Map() }),);- await harness(session, { message: "Use web_search." });+ const result = await harness(session, { message: "Use web_search." });expect(events.filter((event) => event.type === "actions.requested")).toEqual([{@@ -4104,6 +4111,272 @@ describe("createToolLoopHarness", () => {.slice(finalMessageIndex + 1).filter((event) => event.type === "actions.requested" || event.type === "action.result"),).toEqual([]);+ expect(result.next).toBeNull();+ expect(result.session.history.at(-1)?.role).toBe("assistant");+ });++ it("continues after provider-executed web_search follows assistant narration", async () => {+ const toolCallId = "parallel_search_narrated";+ const webSearchOutput = {+ search_id: "search-1",+ results: [+ { excerpts: ["Current result"], title: "Current result", url: "https://example.com" },+ ],+ usage: [{ count: 1, name: "sku_search" }],+ };+ setupMockAgent({+ content: [+ { text: "I will look that up.", type: "text" },+ {+ input: { objective: "Search the web." },+ toolCallId,+ toolName: "web_search",+ type: "tool-call",+ },+ {+ output: webSearchOutput,+ providerExecuted: true,+ toolCallId,+ toolName: "web_search",+ type: "tool-result",+ },+ ],+ finishReason: "stop",+ response: {+ messages: [+ {+ content: [+ { text: "I will look that up.", type: "text" },+ {+ input: { objective: "Search the web." },+ toolCallId,+ toolName: "web_search",+ type: "tool-call",+ },+ {+ output: { type: "json", value: webSearchOutput },+ toolCallId,+ toolName: "web_search",+ type: "tool-result",+ },+ ],+ role: "assistant",+ },+ ],+ },+ text: "I will look that up.",+ toolCalls: [+ {+ input: { objective: "Search the web." },+ toolCallId,+ toolName: "web_search",+ type: "tool-call",+ },+ ],+ toolResults: [+ {+ output: webSearchOutput,+ providerExecuted: true,+ toolCallId,+ toolName: "web_search",+ type: "tool-result",+ },+ ],+ });++ const harness = createToolLoopHarness(+ createTestConfig("conversation", undefined, { tools: new Map() }),+ );+ const result = await harness(+ createTestSession({+ agent: {+ modelReference: { id: "anthropic/claude-sonnet-4.6" },+ system: "You are a test assistant.",+ tools: [{ description: "Search the web", name: "web_search", inputSchema: null }],+ },+ }),+ { message: "Search the web." },+ );++ expect(typeof result.next).toBe("function");+ expect(result.session.history.slice(-2)).toEqual([+ {+ content: [+ { text: "I will look that up.", type: "text" },+ {+ input: { objective: "Search the web." },+ providerExecuted: false,+ toolCallId,+ toolName: "web_search",+ type: "tool-call",+ },+ ],+ role: "assistant",+ },+ {+ content: [+ {+ output: { type: "json", value: webSearchOutput },+ toolCallId,+ toolName: "web_search",+ type: "tool-result",+ },+ ],+ role: "tool",+ },+ ]);+ });++ it("normalizes provider history before parking on an input request", async () => {+ const searchCallId = "parallel_search_before_question";+ const questionCallId = "question-after-search";+ const webSearchOutput = { results: [], searchId: "search-1" };+ const questionInput = {+ options: [{ id: "one", label: "One" }],+ prompt: "Choose one.",+ };+ setupMockAgent({+ content: [+ {+ input: { objective: "Search the web." },+ toolCallId: searchCallId,+ toolName: "web_search",+ type: "tool-call",+ },+ {+ output: webSearchOutput,+ providerExecuted: true,+ toolCallId: searchCallId,+ toolName: "web_search",+ type: "tool-result",+ },+ {+ input: questionInput,+ toolCallId: questionCallId,+ toolName: "ask_question",+ type: "tool-call",+ },+ ],+ finishReason: "tool-calls",+ response: {+ messages: [+ {+ content: [+ { text: "I looked that up and need a choice.", type: "text" },+ {+ input: { objective: "Search the web." },+ toolCallId: searchCallId,+ toolName: "web_search",+ type: "tool-call",+ },+ {+ output: { type: "json", value: webSearchOutput },+ toolCallId: searchCallId,+ toolName: "web_search",+ type: "tool-result",+ },+ {+ input: questionInput,+ toolCallId: questionCallId,+ toolName: "ask_question",+ type: "tool-call",+ },+ ],+ role: "assistant",+ },+ ],+ },+ text: "I looked that up and need a choice.",+ toolCalls: [+ {+ input: { objective: "Search the web." },+ toolCallId: searchCallId,+ toolName: "web_search",+ type: "tool-call",+ },+ {+ input: questionInput,+ toolCallId: questionCallId,+ toolName: "ask_question",+ type: "tool-call",+ },+ ],+ toolResults: [+ {+ output: webSearchOutput,+ providerExecuted: true,+ toolCallId: searchCallId,+ toolName: "web_search",+ type: "tool-result",+ },+ ],+ });++ const harness = createToolLoopHarness(+ createTestConfig("conversation", undefined, { tools: new Map() }),+ );+ const result = await harness(+ createTestSession({+ agent: {+ modelReference: { id: "anthropic/claude-sonnet-4.6" },+ system: "You are a test assistant.",+ tools: [+ { description: "Search the web", name: "web_search", inputSchema: null },+ {+ description: "Ask the user a question.",+ name: "ask_question",+ inputSchema: { type: "object" },+ },+ ],+ },+ }),+ { message: "Search, then ask me a question." },+ );++ const pendingResponseMessages = (+ result.session.state?.["eve.runtime.pendingInputBatch"] as+ | { responseMessages?: readonly ModelMessage[] }+ | undefined+ )?.responseMessages;++ expect(result.next).toBeNull();+ expect(pendingResponseMessages).toEqual([+ {+ content: [+ { text: "I looked that up and need a choice.", type: "text" },+ {+ input: { objective: "Search the web." },+ providerExecuted: false,+ toolCallId: searchCallId,+ toolName: "web_search",+ type: "tool-call",+ },+ ],+ role: "assistant",+ },+ {+ content: [+ {+ output: { type: "json", value: webSearchOutput },+ toolCallId: searchCallId,+ toolName: "web_search",+ type: "tool-result",+ },+ ],+ role: "tool",+ },+ {+ content: [+ {+ input: questionInput,+ toolCallId: questionCallId,+ toolName: "ask_question",+ type: "tool-call",+ },+ ],+ role: "assistant",+ },+ ]);});it("continues after a provider-executed web_search returns without assistant text", async () => {@@ -4168,6 +4441,88 @@ describe("createToolLoopHarness", () => {{ message: "Search the web." },);+ expect(typeof result.next).toBe("function");+ expect(result.session.history.at(-1)).toEqual({+ content: [+ {+ input: { objective: "Search the web." },+ providerExecuted: true,+ toolCallId,+ toolName: "web_search",+ type: "tool-call",+ },+ {+ output: { type: "json", value: { results: [], searchId: "search-1" } },+ toolCallId,+ toolName: "web_search",+ type: "tool-result",+ },+ ],+ role: "assistant",+ });+ });++ it("normalizes an unmarked provider call without assistant text", async () => {+ const toolCallId = "parallel_search_unmarked";+ setupMockAgent({+ content: [+ {+ output: { results: [], searchId: "search-1" },+ providerExecuted: true,+ toolCallId,+ toolName: "web_search",+ type: "tool-result",+ },+ ],+ finishReason: "stop",+ response: {+ messages: [+ {+ content: [+ {+ input: { objective: "Search the web." },+ toolCallId,+ toolName: "web_search",+ type: "tool-call",+ },+ {+ output: { type: "json", value: { results: [], searchId: "search-1" } },+ toolCallId,+ toolName: "web_search",+ type: "tool-result",+ },+ ],+ role: "assistant",+ },+ ],+ },+ text: "",+ toolCalls: [],+ toolResults: [+ {+ output: { results: [], searchId: "search-1" },+ providerExecuted: true,+ toolCallId,+ toolName: "web_search",+ type: "tool-result",+ },+ ],+ });++ const harness = createToolLoopHarness(+ createTestConfig("conversation", undefined, { tools: new Map() }),+ );+ const result = await harness(+ createTestSession({+ agent: {+ modelReference: { id: "openai/gpt-5.5" },+ system: "You are a test assistant.",+ tools: [{ description: "Search the web", name: "web_search", inputSchema: null }],+ },+ }),+ { message: "Search the web." },+ );+expect(typeof result.next).toBe("function");expect(result.session.history.slice(-2)).toEqual([{@@ -4202,11 +4557,18 @@ describe("createToolLoopHarness", () => {content: [{input: { query: "eve release note current" },- providerExecuted: true,toolCallId,toolName: "web_search",type: "tool-call",},+ {+ error: new Error("Search failed"),+ input: { query: "eve release note current" },+ providerExecuted: true,+ toolCallId,+ toolName: "web_search",+ type: "tool-error",+ },],finishReason: "stop",fullStreamParts: [@@ -4233,11 +4595,16 @@ describe("createToolLoopHarness", () => {content: [{input: { query: "eve release note current" },- providerExecuted: true,toolCallId,toolName: "web_search",type: "tool-call",},+ {+ output: { type: "error-text", value: "Search failed" },+ toolCallId,+ toolName: "web_search",+ type: "tool-result",+ },],role: "assistant",},@@ -4247,7 +4614,6 @@ describe("createToolLoopHarness", () => {toolCalls: [{input: { query: "eve release note current" },- providerExecuted: true,toolCallId,toolName: "web_search",type: "tool-call",@@ -4268,7 +4634,7 @@ describe("createToolLoopHarness", () => {createTestConfig("conversation", emit, { tools: new Map() }),);- await harness(session, { message: "Use web_search." });+ const result = await harness(session, { message: "Use web_search." });expect(events.filter((event) => event.type === "action.result")).toEqual([{@@ -4292,6 +4658,11 @@ describe("createToolLoopHarness", () => {},},]);+ expect(typeof result.next).toBe("function");+ expect(result.session.history.slice(-2).map((message) => message.role)).toEqual([+ "assistant",+ "tool",+ ]);});it("preserves AI-SDK StepResult getter properties when synthesizing the inline tool-result message", async () => {diff --git a/packages/eve/src/harness/tool-loop.ts b/packages/eve/src/harness/tool-loop.tsindex eb7e2114b..cddb5844e 100644--- a/packages/eve/src/harness/tool-loop.ts+++ b/packages/eve/src/harness/tool-loop.ts@@ -10,7 +10,6 @@ import {type ModelMessage,type SystemModelMessage,type TelemetryOptions,- type ToolModelMessage,ToolLoopAgent,type ToolSet,type TypedToolCall,@@ -99,6 +98,7 @@ import {} from "#harness/input-requests.js";import { getInstrumentationConfig } from "#harness/instrumentation-config.js";import { resolveAssistantStepText } from "#harness/messages.js";+import { normalizeProviderToolHistory } from "#harness/provider-tool-history.js";import {type AuthorizationSignal,isAuthorizationSignal,@@ -1398,9 +1398,24 @@ async function handleStepResult(input: {result.finishReason !== "tool-calls" &&result.toolCalls.length === 0 &&hasEmptyDeliverySentinel(resolvedStepOutput);- const responseMessages = emptyDelivery ? [] : result.response.messages;+ const rawResponseMessages = emptyDelivery ? [] : result.response.messages;const stepOutput = emptyDelivery ? null : resolvedStepOutput;+ const providerExecutedOutcomeIds = new Set<string>();+ for (const part of [...(result.content ?? []), ...(result.toolResults ?? [])]) {+ if (+ (part.type === "tool-result" || part.type === "tool-error") &&+ part.providerExecuted === true+ ) {+ providerExecutedOutcomeIds.add(part.toolCallId);+ }+ }+ const normalizedProviderHistory = normalizeProviderToolHistory({+ messages: rawResponseMessages,+ providerExecutedOutcomeIds,+ });+ const responseMessages = normalizedProviderHistory.messages;+const baseSession: HarnessSession = {...session,compaction: createNextCompactionConfig(session.compaction, promptMessages, result),@@ -1539,33 +1554,7 @@ async function handleStepResult(input: {// so the prompt prefix stays stable and the provider's prompt cache keeps// hitting across steps. Compaction is the sole mechanism that ever rewrites// history, and it runs before the model call (see `maybeCompact`).- const providerExecutedToolCallIds = new Set(- (stepOutput === null ? (result.toolResults ?? []) : [])- .filter((toolResult) => toolResult.providerExecuted === true)- .map((toolResult) => toolResult.toolCallId),- );- const providerExecutedToolResults: ToolModelMessage["content"] = [];- const continuationMessages: ModelMessage[] = responseMessages.map((message) =>- message.role === "assistant" && Array.isArray(message.content)- ? {- ...message,- content: message.content.flatMap((part) => {- if (part.type === "tool-result" && providerExecutedToolCallIds.has(part.toolCallId)) {- providerExecutedToolResults.push(part);- return [];- }- return [- part.type === "tool-call" && providerExecutedToolCallIds.has(part.toolCallId)- ? { ...part, providerExecuted: false }- : part,- ];- }),- }- : message,- );- if (providerExecutedToolResults.length > 0) {- continuationMessages.push({ role: "tool", content: providerExecutedToolResults });- }+ const continuationMessages = responseMessages;const updatedHistory: ModelMessage[] = [...promptMessages, ...continuationMessages];let nextSession: HarnessSession = { ...baseSession, history: updatedHistory };@@ -1577,7 +1566,9 @@ async function handleStepResult(input: {const continueLoop =!calledFinalOutput &&- (continuationMessages.at(-1)?.role === "tool" || hasDeferredStepInput(nextSession));+ (continuationMessages.at(-1)?.role === "tool" ||+ normalizedProviderHistory.outcomeEndsResponse ||+ hasDeferredStepInput(nextSession));if (continueLoop) {if (emit) {emissionState = advanceStep(emissionState);