Skip to content

armgit / eve

Public Imported from GitHub
0

The Open Framework for Building Agents

fix(e2e): stabilize model-sensitive evals (#2925)

Ruicf9fba473ce0
diff --git a/e2e/fixtures/agent-tools/evals/static-tools/web-search-parallel.eval.ts b/e2e/fixtures/agent-tools/evals/static-tools/web-search-parallel.eval.ts
index d351e8e77..982f94979 100644
--- a/e2e/fixtures/agent-tools/evals/static-tools/web-search-parallel.eval.ts
+++ b/e2e/fixtures/agent-tools/evals/static-tools/web-search-parallel.eval.ts
@@ -4,10 +4,16 @@ import { defineEval } from "eve/evals";
const TOOL_NAME = "web_search";
const MIN_COMPLETED_SEARCHES = 8;
-const EXPECTED_WINNERS =
- "2026 New York Knicks; 2025 Oklahoma City Thunder; 2024 Boston Celtics; " +
- "2023 Denver Nuggets; 2022 Golden State Warriors; 2021 Milwaukee Bucks; " +
- "2020 Los Angeles Lakers; 2019 Toronto Raptors.";
+const EXPECTED_WINNERS = [
+ /Knicks/iu,
+ /Thunder/iu,
+ /Celtics/iu,
+ /Nuggets/iu,
+ /Warriors/iu,
+ /Bucks/iu,
+ /Lakers/iu,
+ /Raptors/iu,
+];
function completedToolResultCount(events: readonly MessageStreamEvent[], toolName: string) {
const callIds = new Set<string>();
@@ -42,6 +48,8 @@ export default defineEval({
(events) => completedToolResultCount(events, TOOL_NAME) >= MIN_COMPLETED_SEARCHES,
);
t.noFailedActions();
- t.judge.autoevals.factuality(EXPECTED_WINNERS, { on: turn.message }).atLeast(0.5);
+ for (const winner of EXPECTED_WINNERS) {
+ turn.messageIncludes(winner);
+ }
},
});
diff --git a/e2e/fixtures/agent-tools/evals/static-tools/web-search.eval.ts b/e2e/fixtures/agent-tools/evals/static-tools/web-search.eval.ts
index 50e8507a5..5adf7e32d 100644
--- a/e2e/fixtures/agent-tools/evals/static-tools/web-search.eval.ts
+++ b/e2e/fixtures/agent-tools/evals/static-tools/web-search.eval.ts
@@ -61,10 +61,5 @@ export default defineEval({
(events) => providerRequestsPrecedeResults(events),
);
t.messageIncludes(/New York Knicks/iu);
- t.judge.autoevals
- .factuality("The New York Knicks won the 2026 NBA Finals.", {
- on: turn.message,
- })
- .atLeast(0.5);
},
});