Skip to content

armgit / eve

Public Imported from GitHub
0

The Open Framework for Building Agents

fix(eve): accept extra parallel web_search calls (#437)

Rui96baec9e0c2a
diff --git a/e2e/fixtures/agent-tools/evals/static-tools/web-search-parallel.eval.ts b/e2e/fixtures/agent-tools/evals/static-tools/web-search-parallel.eval.ts
index 2a64aad9d..060556b85 100644
--- a/e2e/fixtures/agent-tools/evals/static-tools/web-search-parallel.eval.ts
+++ b/e2e/fixtures/agent-tools/evals/static-tools/web-search-parallel.eval.ts
@@ -1,11 +1,30 @@
+import type { HandleMessageStreamEvent } from "eve/client";
import { defineEval } from "eve/evals";
+const TOOL_NAME = "web_search";
+const MIN_COMPLETED_SEARCHES = 10;
+
const EXPECTED_WINNERS =
"2026 New York Knicks; 2025 Oklahoma City Thunder; 2024 Boston Celtics; " +
"2023 Denver Nuggets; 2022 Golden State Warriors; 2021 Milwaukee Bucks; " +
"2020 Los Angeles Lakers; 2019 Toronto Raptors; 2018 Golden State Warriors; " +
"2017 Golden State Warriors.";
+function completedToolResultCount(events: readonly HandleMessageStreamEvent[], toolName: string) {
+ const callIds = new Set<string>();
+ for (const event of events) {
+ if (
+ event.type === "action.result" &&
+ event.data.status === "completed" &&
+ event.data.result.kind === "tool-result" &&
+ event.data.result.toolName === toolName
+ ) {
+ callIds.add(event.data.result.callId);
+ }
+ }
+ return callIds.size;
+}
+
export default defineEval({
description: "Provider tools smoke: ten parallel gateway web searches complete successfully.",
async test(t) {
@@ -14,7 +33,10 @@ export default defineEval({
);
t.succeeded();
- t.calledTool("web_search", { count: 10 });
+ turn.eventsSatisfy(
+ "at least ten completed web_search calls",
+ (events) => completedToolResultCount(events, TOOL_NAME) >= MIN_COMPLETED_SEARCHES,
+ );
t.noFailedActions();
t.judge.autoevals.factuality(EXPECTED_WINNERS, { on: turn.message }).atLeast(0.5);
},