Skip to content

armgit / eve

Public Imported from GitHub
0

The Open Framework for Building Agents

test(eve): de-flake web-search cutoff-refusal evals with date-context guard (#893)

prha49b101e6fbaf
diff --git a/e2e/fixtures/agent-tools/evals/static-tools/web-search-narration-ordering.eval.ts b/e2e/fixtures/agent-tools/evals/static-tools/web-search-narration-ordering.eval.ts
index 9221becbc..2c4bd94cf 100644
--- a/e2e/fixtures/agent-tools/evals/static-tools/web-search-narration-ordering.eval.ts
+++ b/e2e/fixtures/agent-tools/evals/static-tools/web-search-narration-ordering.eval.ts
@@ -8,6 +8,7 @@ export default defineEval({
async test(t) {
const narrated = await t.send(
[
+ "Important date context: the 2026 NBA Finals have absolutely already been played, and a champion has been crowned. Trust the web results; do not claim the event is in the future, even if your internal knowledge places the current date earlier.",
`Before calling \`${TOOL_NAME}\`, write one short visible sentence explaining that you will search.`,
`Then call \`${TOOL_NAME}\` exactly once to answer: Who won the 2026 NBA Finals?`,
"After the result returns, reply with only the winning team name. Do not call another tool.",
@@ -24,6 +25,7 @@ export default defineEval({
.newSession()
.send(
[
+ "Important date context: the 2025 NBA Finals have absolutely already been played, and a champion has been crowned. Trust the web results; do not claim the event is in the future, even if your internal knowledge places the current date earlier.",
`Call \`${TOOL_NAME}\` exactly once to answer: Who won the 2025 NBA Finals?`,
"Do not write any visible text before the tool call.",
"After the result returns, reply with only the winning team name. Do not call another tool.",
diff --git a/e2e/fixtures/agent-tools/evals/static-tools/web-search-parallel.eval.ts b/e2e/fixtures/agent-tools/evals/static-tools/web-search-parallel.eval.ts
index 060556b85..0e51a70b8 100644
--- a/e2e/fixtures/agent-tools/evals/static-tools/web-search-parallel.eval.ts
+++ b/e2e/fixtures/agent-tools/evals/static-tools/web-search-parallel.eval.ts
@@ -29,7 +29,11 @@ export default defineEval({
description: "Provider tools smoke: ten parallel gateway web searches complete successfully.",
async test(t) {
const turn = await t.send(
- "Using 10 parallel web_search calls: lookup the nba finals winner from 2026 back to 2017",
+ [
+ "Important date context: the 2026 NBA Finals have absolutely already been played, and a champion has been crowned.",
+ "Do not claim any of these seasons are in the future or unresolved, even if your internal knowledge incorrectly places the current date earlier; trust the web results.",
+ "Using 10 parallel web_search calls: lookup the nba finals winner from 2026 back to 2017.",
+ ].join("\n"),
);
t.succeeded();