The Open Framework for Building Agents
test(eve): reduce web-search-parallel fan-out from 10 to 8 (#924)
prha71d57185233d
diff --git a/e2e/fixtures/agent-tools/evals/static-tools/web-search-parallel.eval.ts b/e2e/fixtures/agent-tools/evals/static-tools/web-search-parallel.eval.tsindex 0e51a70b8..5a9bc7a15 100644--- a/e2e/fixtures/agent-tools/evals/static-tools/web-search-parallel.eval.ts+++ b/e2e/fixtures/agent-tools/evals/static-tools/web-search-parallel.eval.ts@@ -2,13 +2,12 @@ import type { HandleMessageStreamEvent } from "eve/client";import { defineEval } from "eve/evals";const TOOL_NAME = "web_search";-const MIN_COMPLETED_SEARCHES = 10;+const MIN_COMPLETED_SEARCHES = 8;const EXPECTED_WINNERS ="2026 New York Knicks; 2025 Oklahoma City Thunder; 2024 Boston Celtics; " +"2023 Denver Nuggets; 2022 Golden State Warriors; 2021 Milwaukee Bucks; " +- "2020 Los Angeles Lakers; 2019 Toronto Raptors; 2018 Golden State Warriors; " +- "2017 Golden State Warriors.";+ "2020 Los Angeles Lakers; 2019 Toronto Raptors.";function completedToolResultCount(events: readonly HandleMessageStreamEvent[], toolName: string) {const callIds = new Set<string>();@@ -26,19 +25,19 @@ function completedToolResultCount(events: readonly HandleMessageStreamEvent[], t}export default defineEval({- description: "Provider tools smoke: ten parallel gateway web searches complete successfully.",+ description: "Provider tools smoke: eight parallel gateway web searches complete successfully.",async test(t) {const turn = await t.send(["Important date context: the 2026 NBA Finals have absolutely already been played, and a champion has been crowned.","Do not claim any of these seasons are in the future or unresolved, even if your internal knowledge incorrectly places the current date earlier; trust the web results.",- "Using 10 parallel web_search calls: lookup the nba finals winner from 2026 back to 2017.",+ "Using 8 parallel web_search calls: lookup the nba finals winner from 2026 back to 2019.",].join("\n"),);t.succeeded();turn.eventsSatisfy(- "at least ten completed web_search calls",+ "at least eight completed web_search calls",(events) => completedToolResultCount(events, TOOL_NAME) >= MIN_COMPLETED_SEARCHES,);t.noFailedActions();