The Open Framework for Building Agents
fix(eve): accept extra parallel web_search calls (#437)
Rui96baec9e0c2a
diff --git a/e2e/fixtures/agent-tools/evals/static-tools/web-search-parallel.eval.ts b/e2e/fixtures/agent-tools/evals/static-tools/web-search-parallel.eval.tsindex 2a64aad9d..060556b85 100644--- a/e2e/fixtures/agent-tools/evals/static-tools/web-search-parallel.eval.ts+++ b/e2e/fixtures/agent-tools/evals/static-tools/web-search-parallel.eval.ts@@ -1,11 +1,30 @@+import type { HandleMessageStreamEvent } from "eve/client";import { defineEval } from "eve/evals";+const TOOL_NAME = "web_search";+const MIN_COMPLETED_SEARCHES = 10;+const EXPECTED_WINNERS ="2026 New York Knicks; 2025 Oklahoma City Thunder; 2024 Boston Celtics; " +"2023 Denver Nuggets; 2022 Golden State Warriors; 2021 Milwaukee Bucks; " +"2020 Los Angeles Lakers; 2019 Toronto Raptors; 2018 Golden State Warriors; " +"2017 Golden State Warriors.";+function completedToolResultCount(events: readonly HandleMessageStreamEvent[], toolName: string) {+ const callIds = new Set<string>();+ for (const event of events) {+ if (+ event.type === "action.result" &&+ event.data.status === "completed" &&+ event.data.result.kind === "tool-result" &&+ event.data.result.toolName === toolName+ ) {+ callIds.add(event.data.result.callId);+ }+ }+ return callIds.size;+}+export default defineEval({description: "Provider tools smoke: ten parallel gateway web searches complete successfully.",async test(t) {@@ -14,7 +33,10 @@ export default defineEval({);t.succeeded();- t.calledTool("web_search", { count: 10 });+ turn.eventsSatisfy(+ "at least ten completed web_search calls",+ (events) => completedToolResultCount(events, TOOL_NAME) >= MIN_COMPLETED_SEARCHES,+ );t.noFailedActions();t.judge.autoevals.factuality(EXPECTED_WINNERS, { on: turn.message }).atLeast(0.5);},