The Open Framework for Building Agents
test(eve): de-flake web-search cutoff-refusal evals with date-context guard (#893)
prha49b101e6fbaf
diff --git a/e2e/fixtures/agent-tools/evals/static-tools/web-search-narration-ordering.eval.ts b/e2e/fixtures/agent-tools/evals/static-tools/web-search-narration-ordering.eval.tsindex 9221becbc..2c4bd94cf 100644--- a/e2e/fixtures/agent-tools/evals/static-tools/web-search-narration-ordering.eval.ts+++ b/e2e/fixtures/agent-tools/evals/static-tools/web-search-narration-ordering.eval.ts@@ -8,6 +8,7 @@ export default defineEval({async test(t) {const narrated = await t.send([+ "Important date context: the 2026 NBA Finals have absolutely already been played, and a champion has been crowned. Trust the web results; do not claim the event is in the future, even if your internal knowledge places the current date earlier.",`Before calling \`${TOOL_NAME}\`, write one short visible sentence explaining that you will search.`,`Then call \`${TOOL_NAME}\` exactly once to answer: Who won the 2026 NBA Finals?`,"After the result returns, reply with only the winning team name. Do not call another tool.",@@ -24,6 +25,7 @@ export default defineEval({.newSession().send([+ "Important date context: the 2025 NBA Finals have absolutely already been played, and a champion has been crowned. Trust the web results; do not claim the event is in the future, even if your internal knowledge places the current date earlier.",`Call \`${TOOL_NAME}\` exactly once to answer: Who won the 2025 NBA Finals?`,"Do not write any visible text before the tool call.","After the result returns, reply with only the winning team name. Do not call another tool.",diff --git a/e2e/fixtures/agent-tools/evals/static-tools/web-search-parallel.eval.ts b/e2e/fixtures/agent-tools/evals/static-tools/web-search-parallel.eval.tsindex 060556b85..0e51a70b8 100644--- a/e2e/fixtures/agent-tools/evals/static-tools/web-search-parallel.eval.ts+++ b/e2e/fixtures/agent-tools/evals/static-tools/web-search-parallel.eval.ts@@ -29,7 +29,11 @@ export default defineEval({description: "Provider tools smoke: ten parallel gateway web searches complete successfully.",async test(t) {const turn = await t.send(- "Using 10 parallel web_search calls: lookup the nba finals winner from 2026 back to 2017",+ [+ "Important date context: the 2026 NBA Finals have absolutely already been played, and a champion has been crowned.",+ "Do not claim any of these seasons are in the future or unresolved, even if your internal knowledge incorrectly places the current date earlier; trust the web results.",+ "Using 10 parallel web_search calls: lookup the nba finals winner from 2026 back to 2017.",+ ].join("\n"),);t.succeeded();