The Open Framework for Building Agents
test(e2e): split web search ordering evals (#1052)
prha0dd620ab140b
diff --git a/e2e/fixtures/agent-tools/evals/static-tools/web-search-narrated-ordering.eval.ts b/e2e/fixtures/agent-tools/evals/static-tools/web-search-narrated-ordering.eval.tsnew file mode 100644index 000000000..0963d47a9--- /dev/null+++ b/e2e/fixtures/agent-tools/evals/static-tools/web-search-narrated-ordering.eval.ts@@ -0,0 +1,23 @@+import { defineEval } from "eve/evals";+import { narratedWebSearchOrder, WEB_SEARCH_TOOL_NAME } from "./web-search-ordering";++export default defineEval({+ description: "Provider tools: narrated web searches preserve event order.",+ async test(t) {+ const turn = await t.send(+ [+ "Important date context: the 2026 NBA Finals have absolutely already been played, and a champion has been crowned. Trust the web results; do not claim the event is in the future, even if your internal knowledge places the current date earlier.",+ `Before calling \`${WEB_SEARCH_TOOL_NAME}\`, write one short visible sentence explaining that you will search.`,+ `Then call \`${WEB_SEARCH_TOOL_NAME}\` exactly once to answer: Who won the 2026 NBA Finals?`,+ "After the result returns, reply with only the winning team name. Do not call another tool.",+ ].join("\n"),+ );++ turn.expectOk();+ turn.calledTool(WEB_SEARCH_TOOL_NAME, { count: 1 });+ turn.noFailedActions();+ turn.eventsSatisfy("narration completes before the provider request and result", (events) =>+ narratedWebSearchOrder(events),+ );+ },+});diff --git a/e2e/fixtures/agent-tools/evals/static-tools/web-search-narration-ordering.eval.ts b/e2e/fixtures/agent-tools/evals/static-tools/web-search-narration-ordering.eval.tsdeleted file mode 100644index 2c4bd94cf..000000000--- a/e2e/fixtures/agent-tools/evals/static-tools/web-search-narration-ordering.eval.ts+++ /dev/null@@ -1,126 +0,0 @@-import type { HandleMessageStreamEvent } from "eve/client";-import { defineEval } from "eve/evals";--const TOOL_NAME = "web_search";--export default defineEval({- description: "Provider tools: narrated and un-narrated web searches preserve event order.",- async test(t) {- const narrated = await t.send(- [- "Important date context: the 2026 NBA Finals have absolutely already been played, and a champion has been crowned. Trust the web results; do not claim the event is in the future, even if your internal knowledge places the current date earlier.",- `Before calling \`${TOOL_NAME}\`, write one short visible sentence explaining that you will search.`,- `Then call \`${TOOL_NAME}\` exactly once to answer: Who won the 2026 NBA Finals?`,- "After the result returns, reply with only the winning team name. Do not call another tool.",- ].join("\n"),- );- narrated.expectOk();- narrated.calledTool(TOOL_NAME, { count: 1 });- narrated.noFailedActions();- narrated.eventsSatisfy("narration completes before the provider request and result", (events) =>- narratedWebSearchOrder(events),- );-- const withoutNarration = await t- .newSession()- .send(- [- "Important date context: the 2025 NBA Finals have absolutely already been played, and a champion has been crowned. Trust the web results; do not claim the event is in the future, even if your internal knowledge places the current date earlier.",- `Call \`${TOOL_NAME}\` exactly once to answer: Who won the 2025 NBA Finals?`,- "Do not write any visible text before the tool call.",- "After the result returns, reply with only the winning team name. Do not call another tool.",- ].join("\n"),- );- withoutNarration.expectOk();- withoutNarration.calledTool(TOOL_NAME, { count: 1 });- withoutNarration.noFailedActions();- withoutNarration.eventsSatisfy(- "provider request and result stay ordered without narration",- (events) => unNarratedWebSearchOrder(events),- );- },-});--interface WebSearchEventOrder {- readonly requestIndex: number;- readonly resultIndex: number;-}--function narratedWebSearchOrder(events: readonly HandleMessageStreamEvent[]): boolean {- const order = webSearchEventOrder(events);- return (- order !== undefined &&- preToolNarrationExists(events, order.requestIndex) &&- finalMessageFollowsResult(events, order.resultIndex)- );-}--function unNarratedWebSearchOrder(events: readonly HandleMessageStreamEvent[]): boolean {- const order = webSearchEventOrder(events);- return (- order !== undefined &&- !preToolNarrationExists(events, order.requestIndex) &&- finalMessageFollowsResult(events, order.resultIndex)- );-}--function webSearchEventOrder(- events: readonly HandleMessageStreamEvent[],-): WebSearchEventOrder | undefined {- const requests = events.flatMap((event, eventIndex) => {- if (event.type !== "actions.requested") return [];-- return event.data.actions.flatMap((action) => {- if (action.kind !== "tool-call" || action.toolName !== TOOL_NAME) return [];- return [{ callId: action.callId, eventIndex }];- });- });- const results = events.flatMap((event, eventIndex) => {- if (event.type !== "action.result" || event.data.result.kind !== "tool-result") return [];- if (event.data.result.toolName !== TOOL_NAME) return [];- return [{ callId: event.data.result.callId, eventIndex }];- });-- const [request] = requests;- const [result] = results;- if (- request === undefined ||- result === undefined ||- requests.length !== 1 ||- results.length !== 1 ||- request.callId !== result.callId ||- request.eventIndex >= result.eventIndex- ) {- return undefined;- }- return { requestIndex: request.eventIndex, resultIndex: result.eventIndex };-}--function preToolNarrationExists(- events: readonly HandleMessageStreamEvent[],- requestIndex: number,-): boolean {- return events- .slice(0, requestIndex)- .some(- (event) =>- event.type === "message.completed" &&- event.data.finishReason === "tool-calls" &&- event.data.message !== null &&- event.data.message.trim().length > 0,- );-}--function finalMessageFollowsResult(- events: readonly HandleMessageStreamEvent[],- resultIndex: number,-): boolean {- return events- .slice(resultIndex + 1)- .some(- (event) =>- event.type === "message.completed" &&- event.data.finishReason !== "tool-calls" &&- event.data.message !== null,- );-}diff --git a/e2e/fixtures/agent-tools/evals/static-tools/web-search-ordering.ts b/e2e/fixtures/agent-tools/evals/static-tools/web-search-ordering.tsnew file mode 100644index 000000000..a4cf0bdc4--- /dev/null+++ b/e2e/fixtures/agent-tools/evals/static-tools/web-search-ordering.ts@@ -0,0 +1,87 @@+import type { HandleMessageStreamEvent } from "eve/client";++export const WEB_SEARCH_TOOL_NAME = "web_search";++interface WebSearchEventOrder {+ readonly requestIndex: number;+ readonly resultIndex: number;+}++export function narratedWebSearchOrder(events: readonly HandleMessageStreamEvent[]): boolean {+ const order = webSearchEventOrder(events);+ return (+ order !== undefined &&+ preToolNarrationExists(events, order.requestIndex) &&+ finalMessageFollowsResult(events, order.resultIndex)+ );+}++export function unNarratedWebSearchOrder(events: readonly HandleMessageStreamEvent[]): boolean {+ const order = webSearchEventOrder(events);+ return (+ order !== undefined &&+ !preToolNarrationExists(events, order.requestIndex) &&+ finalMessageFollowsResult(events, order.resultIndex)+ );+}++function webSearchEventOrder(+ events: readonly HandleMessageStreamEvent[],+): WebSearchEventOrder | undefined {+ const requests = events.flatMap((event, eventIndex) => {+ if (event.type !== "actions.requested") return [];++ return event.data.actions.flatMap((action) => {+ if (action.kind !== "tool-call" || action.toolName !== WEB_SEARCH_TOOL_NAME) return [];+ return [{ callId: action.callId, eventIndex }];+ });+ });+ const results = events.flatMap((event, eventIndex) => {+ if (event.type !== "action.result" || event.data.result.kind !== "tool-result") return [];+ if (event.data.result.toolName !== WEB_SEARCH_TOOL_NAME) return [];+ return [{ callId: event.data.result.callId, eventIndex }];+ });++ const [request] = requests;+ const [result] = results;+ if (+ request === undefined ||+ result === undefined ||+ requests.length !== 1 ||+ results.length !== 1 ||+ request.callId !== result.callId ||+ request.eventIndex >= result.eventIndex+ ) {+ return undefined;+ }+ return { requestIndex: request.eventIndex, resultIndex: result.eventIndex };+}++function preToolNarrationExists(+ events: readonly HandleMessageStreamEvent[],+ requestIndex: number,+): boolean {+ return events+ .slice(0, requestIndex)+ .some(+ (event) =>+ event.type === "message.completed" &&+ event.data.finishReason === "tool-calls" &&+ event.data.message !== null &&+ event.data.message.trim().length > 0,+ );+}++function finalMessageFollowsResult(+ events: readonly HandleMessageStreamEvent[],+ resultIndex: number,+): boolean {+ return events+ .slice(resultIndex + 1)+ .some(+ (event) =>+ event.type === "message.completed" &&+ event.data.finishReason !== "tool-calls" &&+ event.data.message !== null,+ );+}diff --git a/e2e/fixtures/agent-tools/evals/static-tools/web-search-unnarrated-ordering.eval.ts b/e2e/fixtures/agent-tools/evals/static-tools/web-search-unnarrated-ordering.eval.tsnew file mode 100644index 000000000..06f57bedd--- /dev/null+++ b/e2e/fixtures/agent-tools/evals/static-tools/web-search-unnarrated-ordering.eval.ts@@ -0,0 +1,23 @@+import { defineEval } from "eve/evals";+import { unNarratedWebSearchOrder, WEB_SEARCH_TOOL_NAME } from "./web-search-ordering";++export default defineEval({+ description: "Provider tools: un-narrated web searches preserve event order.",+ async test(t) {+ const turn = await t.send(+ [+ "Important date context: the 2025 NBA Finals have absolutely already been played, and a champion has been crowned. Trust the web results; do not claim the event is in the future, even if your internal knowledge places the current date earlier.",+ `Call \`${WEB_SEARCH_TOOL_NAME}\` exactly once to answer: Who won the 2025 NBA Finals?`,+ "Do not write any visible text before the tool call.",+ "After the result returns, reply with only the winning team name. Do not call another tool.",+ ].join("\n"),+ );++ turn.expectOk();+ turn.calledTool(WEB_SEARCH_TOOL_NAME, { count: 1 });+ turn.noFailedActions();+ turn.eventsSatisfy("provider request and result stay ordered without narration", (events) =>+ unNarratedWebSearchOrder(events),+ );+ },+});