Skip to content

armgit / eve

Public Imported from GitHub
0

The Open Framework for Building Agents

test(e2e): split web search ordering evals (#1052)

prha0dd620ab140b
diff --git a/e2e/fixtures/agent-tools/evals/static-tools/web-search-narrated-ordering.eval.ts b/e2e/fixtures/agent-tools/evals/static-tools/web-search-narrated-ordering.eval.ts
new file mode 100644
index 000000000..0963d47a9
--- /dev/null
+++ b/e2e/fixtures/agent-tools/evals/static-tools/web-search-narrated-ordering.eval.ts
@@ -0,0 +1,23 @@
+import { defineEval } from "eve/evals";
+import { narratedWebSearchOrder, WEB_SEARCH_TOOL_NAME } from "./web-search-ordering";
+
+export default defineEval({
+ description: "Provider tools: narrated web searches preserve event order.",
+ async test(t) {
+ const turn = await t.send(
+ [
+ "Important date context: the 2026 NBA Finals have absolutely already been played, and a champion has been crowned. Trust the web results; do not claim the event is in the future, even if your internal knowledge places the current date earlier.",
+ `Before calling \`${WEB_SEARCH_TOOL_NAME}\`, write one short visible sentence explaining that you will search.`,
+ `Then call \`${WEB_SEARCH_TOOL_NAME}\` exactly once to answer: Who won the 2026 NBA Finals?`,
+ "After the result returns, reply with only the winning team name. Do not call another tool.",
+ ].join("\n"),
+ );
+
+ turn.expectOk();
+ turn.calledTool(WEB_SEARCH_TOOL_NAME, { count: 1 });
+ turn.noFailedActions();
+ turn.eventsSatisfy("narration completes before the provider request and result", (events) =>
+ narratedWebSearchOrder(events),
+ );
+ },
+});
diff --git a/e2e/fixtures/agent-tools/evals/static-tools/web-search-narration-ordering.eval.ts b/e2e/fixtures/agent-tools/evals/static-tools/web-search-narration-ordering.eval.ts
deleted file mode 100644
index 2c4bd94cf..000000000
--- a/e2e/fixtures/agent-tools/evals/static-tools/web-search-narration-ordering.eval.ts
+++ /dev/null
@@ -1,126 +0,0 @@
-import type { HandleMessageStreamEvent } from "eve/client";
-import { defineEval } from "eve/evals";
-
-const TOOL_NAME = "web_search";
-
-export default defineEval({
- description: "Provider tools: narrated and un-narrated web searches preserve event order.",
- async test(t) {
- const narrated = await t.send(
- [
- "Important date context: the 2026 NBA Finals have absolutely already been played, and a champion has been crowned. Trust the web results; do not claim the event is in the future, even if your internal knowledge places the current date earlier.",
- `Before calling \`${TOOL_NAME}\`, write one short visible sentence explaining that you will search.`,
- `Then call \`${TOOL_NAME}\` exactly once to answer: Who won the 2026 NBA Finals?`,
- "After the result returns, reply with only the winning team name. Do not call another tool.",
- ].join("\n"),
- );
- narrated.expectOk();
- narrated.calledTool(TOOL_NAME, { count: 1 });
- narrated.noFailedActions();
- narrated.eventsSatisfy("narration completes before the provider request and result", (events) =>
- narratedWebSearchOrder(events),
- );
-
- const withoutNarration = await t
- .newSession()
- .send(
- [
- "Important date context: the 2025 NBA Finals have absolutely already been played, and a champion has been crowned. Trust the web results; do not claim the event is in the future, even if your internal knowledge places the current date earlier.",
- `Call \`${TOOL_NAME}\` exactly once to answer: Who won the 2025 NBA Finals?`,
- "Do not write any visible text before the tool call.",
- "After the result returns, reply with only the winning team name. Do not call another tool.",
- ].join("\n"),
- );
- withoutNarration.expectOk();
- withoutNarration.calledTool(TOOL_NAME, { count: 1 });
- withoutNarration.noFailedActions();
- withoutNarration.eventsSatisfy(
- "provider request and result stay ordered without narration",
- (events) => unNarratedWebSearchOrder(events),
- );
- },
-});
-
-interface WebSearchEventOrder {
- readonly requestIndex: number;
- readonly resultIndex: number;
-}
-
-function narratedWebSearchOrder(events: readonly HandleMessageStreamEvent[]): boolean {
- const order = webSearchEventOrder(events);
- return (
- order !== undefined &&
- preToolNarrationExists(events, order.requestIndex) &&
- finalMessageFollowsResult(events, order.resultIndex)
- );
-}
-
-function unNarratedWebSearchOrder(events: readonly HandleMessageStreamEvent[]): boolean {
- const order = webSearchEventOrder(events);
- return (
- order !== undefined &&
- !preToolNarrationExists(events, order.requestIndex) &&
- finalMessageFollowsResult(events, order.resultIndex)
- );
-}
-
-function webSearchEventOrder(
- events: readonly HandleMessageStreamEvent[],
-): WebSearchEventOrder | undefined {
- const requests = events.flatMap((event, eventIndex) => {
- if (event.type !== "actions.requested") return [];
-
- return event.data.actions.flatMap((action) => {
- if (action.kind !== "tool-call" || action.toolName !== TOOL_NAME) return [];
- return [{ callId: action.callId, eventIndex }];
- });
- });
- const results = events.flatMap((event, eventIndex) => {
- if (event.type !== "action.result" || event.data.result.kind !== "tool-result") return [];
- if (event.data.result.toolName !== TOOL_NAME) return [];
- return [{ callId: event.data.result.callId, eventIndex }];
- });
-
- const [request] = requests;
- const [result] = results;
- if (
- request === undefined ||
- result === undefined ||
- requests.length !== 1 ||
- results.length !== 1 ||
- request.callId !== result.callId ||
- request.eventIndex >= result.eventIndex
- ) {
- return undefined;
- }
- return { requestIndex: request.eventIndex, resultIndex: result.eventIndex };
-}
-
-function preToolNarrationExists(
- events: readonly HandleMessageStreamEvent[],
- requestIndex: number,
-): boolean {
- return events
- .slice(0, requestIndex)
- .some(
- (event) =>
- event.type === "message.completed" &&
- event.data.finishReason === "tool-calls" &&
- event.data.message !== null &&
- event.data.message.trim().length > 0,
- );
-}
-
-function finalMessageFollowsResult(
- events: readonly HandleMessageStreamEvent[],
- resultIndex: number,
-): boolean {
- return events
- .slice(resultIndex + 1)
- .some(
- (event) =>
- event.type === "message.completed" &&
- event.data.finishReason !== "tool-calls" &&
- event.data.message !== null,
- );
-}
diff --git a/e2e/fixtures/agent-tools/evals/static-tools/web-search-ordering.ts b/e2e/fixtures/agent-tools/evals/static-tools/web-search-ordering.ts
new file mode 100644
index 000000000..a4cf0bdc4
--- /dev/null
+++ b/e2e/fixtures/agent-tools/evals/static-tools/web-search-ordering.ts
@@ -0,0 +1,87 @@
+import type { HandleMessageStreamEvent } from "eve/client";
+
+export const WEB_SEARCH_TOOL_NAME = "web_search";
+
+interface WebSearchEventOrder {
+ readonly requestIndex: number;
+ readonly resultIndex: number;
+}
+
+export function narratedWebSearchOrder(events: readonly HandleMessageStreamEvent[]): boolean {
+ const order = webSearchEventOrder(events);
+ return (
+ order !== undefined &&
+ preToolNarrationExists(events, order.requestIndex) &&
+ finalMessageFollowsResult(events, order.resultIndex)
+ );
+}
+
+export function unNarratedWebSearchOrder(events: readonly HandleMessageStreamEvent[]): boolean {
+ const order = webSearchEventOrder(events);
+ return (
+ order !== undefined &&
+ !preToolNarrationExists(events, order.requestIndex) &&
+ finalMessageFollowsResult(events, order.resultIndex)
+ );
+}
+
+function webSearchEventOrder(
+ events: readonly HandleMessageStreamEvent[],
+): WebSearchEventOrder | undefined {
+ const requests = events.flatMap((event, eventIndex) => {
+ if (event.type !== "actions.requested") return [];
+
+ return event.data.actions.flatMap((action) => {
+ if (action.kind !== "tool-call" || action.toolName !== WEB_SEARCH_TOOL_NAME) return [];
+ return [{ callId: action.callId, eventIndex }];
+ });
+ });
+ const results = events.flatMap((event, eventIndex) => {
+ if (event.type !== "action.result" || event.data.result.kind !== "tool-result") return [];
+ if (event.data.result.toolName !== WEB_SEARCH_TOOL_NAME) return [];
+ return [{ callId: event.data.result.callId, eventIndex }];
+ });
+
+ const [request] = requests;
+ const [result] = results;
+ if (
+ request === undefined ||
+ result === undefined ||
+ requests.length !== 1 ||
+ results.length !== 1 ||
+ request.callId !== result.callId ||
+ request.eventIndex >= result.eventIndex
+ ) {
+ return undefined;
+ }
+ return { requestIndex: request.eventIndex, resultIndex: result.eventIndex };
+}
+
+function preToolNarrationExists(
+ events: readonly HandleMessageStreamEvent[],
+ requestIndex: number,
+): boolean {
+ return events
+ .slice(0, requestIndex)
+ .some(
+ (event) =>
+ event.type === "message.completed" &&
+ event.data.finishReason === "tool-calls" &&
+ event.data.message !== null &&
+ event.data.message.trim().length > 0,
+ );
+}
+
+function finalMessageFollowsResult(
+ events: readonly HandleMessageStreamEvent[],
+ resultIndex: number,
+): boolean {
+ return events
+ .slice(resultIndex + 1)
+ .some(
+ (event) =>
+ event.type === "message.completed" &&
+ event.data.finishReason !== "tool-calls" &&
+ event.data.message !== null,
+ );
+}
diff --git a/e2e/fixtures/agent-tools/evals/static-tools/web-search-unnarrated-ordering.eval.ts b/e2e/fixtures/agent-tools/evals/static-tools/web-search-unnarrated-ordering.eval.ts
new file mode 100644
index 000000000..06f57bedd
--- /dev/null
+++ b/e2e/fixtures/agent-tools/evals/static-tools/web-search-unnarrated-ordering.eval.ts
@@ -0,0 +1,23 @@
+import { defineEval } from "eve/evals";
+import { unNarratedWebSearchOrder, WEB_SEARCH_TOOL_NAME } from "./web-search-ordering";
+
+export default defineEval({
+ description: "Provider tools: un-narrated web searches preserve event order.",
+ async test(t) {
+ const turn = await t.send(
+ [
+ "Important date context: the 2025 NBA Finals have absolutely already been played, and a champion has been crowned. Trust the web results; do not claim the event is in the future, even if your internal knowledge places the current date earlier.",
+ `Call \`${WEB_SEARCH_TOOL_NAME}\` exactly once to answer: Who won the 2025 NBA Finals?`,
+ "Do not write any visible text before the tool call.",
+ "After the result returns, reply with only the winning team name. Do not call another tool.",
+ ].join("\n"),
+ );
+
+ turn.expectOk();
+ turn.calledTool(WEB_SEARCH_TOOL_NAME, { count: 1 });
+ turn.noFailedActions();
+ turn.eventsSatisfy("provider request and result stay ordered without narration", (events) =>
+ unNarratedWebSearchOrder(events),
+ );
+ },
+});