Skip to content

armgit / eve

Public Imported from GitHub
0

The Open Framework for Building Agents

test(eve): reduce web-search-parallel fan-out from 10 to 8 (#924)

prha71d57185233d
diff --git a/e2e/fixtures/agent-tools/evals/static-tools/web-search-parallel.eval.ts b/e2e/fixtures/agent-tools/evals/static-tools/web-search-parallel.eval.ts
index 0e51a70b8..5a9bc7a15 100644
--- a/e2e/fixtures/agent-tools/evals/static-tools/web-search-parallel.eval.ts
+++ b/e2e/fixtures/agent-tools/evals/static-tools/web-search-parallel.eval.ts
@@ -2,13 +2,12 @@ import type { HandleMessageStreamEvent } from "eve/client";
import { defineEval } from "eve/evals";
const TOOL_NAME = "web_search";
-const MIN_COMPLETED_SEARCHES = 10;
+const MIN_COMPLETED_SEARCHES = 8;
const EXPECTED_WINNERS =
"2026 New York Knicks; 2025 Oklahoma City Thunder; 2024 Boston Celtics; " +
"2023 Denver Nuggets; 2022 Golden State Warriors; 2021 Milwaukee Bucks; " +
- "2020 Los Angeles Lakers; 2019 Toronto Raptors; 2018 Golden State Warriors; " +
- "2017 Golden State Warriors.";
+ "2020 Los Angeles Lakers; 2019 Toronto Raptors.";
function completedToolResultCount(events: readonly HandleMessageStreamEvent[], toolName: string) {
const callIds = new Set<string>();
@@ -26,19 +25,19 @@ function completedToolResultCount(events: readonly HandleMessageStreamEvent[], t
}
export default defineEval({
- description: "Provider tools smoke: ten parallel gateway web searches complete successfully.",
+ description: "Provider tools smoke: eight parallel gateway web searches complete successfully.",
async test(t) {
const turn = await t.send(
[
"Important date context: the 2026 NBA Finals have absolutely already been played, and a champion has been crowned.",
"Do not claim any of these seasons are in the future or unresolved, even if your internal knowledge incorrectly places the current date earlier; trust the web results.",
- "Using 10 parallel web_search calls: lookup the nba finals winner from 2026 back to 2017.",
+ "Using 8 parallel web_search calls: lookup the nba finals winner from 2026 back to 2019.",
].join("\n"),
);
t.succeeded();
turn.eventsSatisfy(
- "at least ten completed web_search calls",
+ "at least eight completed web_search calls",
(events) => completedToolResultCount(events, TOOL_NAME) >= MIN_COMPLETED_SEARCHES,
);
t.noFailedActions();