The Open Framework for Building Agents
test(eve): clarify stale-ask-question-selection judge criterion (both variants) (#923)
prha477762e2ee58
diff --git a/e2e/fixtures/agent-tools-hitl/evals/hitl/stale-ask-question-selection-idle.eval.ts b/e2e/fixtures/agent-tools-hitl/evals/hitl/stale-ask-question-selection-idle.eval.tsindex e91c43316..3f0fa2094 100644--- a/e2e/fixtures/agent-tools-hitl/evals/hitl/stale-ask-question-selection-idle.eval.ts+++ b/e2e/fixtures/agent-tools-hitl/evals/hitl/stale-ask-question-selection-idle.eval.ts@@ -49,7 +49,7 @@ export default defineEval({// a literal echo the model can rightly decline.t.judge.autoevals.closedQA(- "The reply acknowledges a late selection of the STALE-CANDIDATE-7Q4M option, either by applying it or by explaining that the question was already answered and the selection is no longer relevant.",+ "The reply treats the message as a late response to an EARLIER question — either applying that earlier selection, or explaining that the earlier question was already answered or is no longer relevant (stale). Answer yes unless the reply ignores the message entirely or is unrelated to it.",{ on: staleSelection.message },).atLeast(0.5);diff --git a/e2e/fixtures/agent-tools-hitl/evals/hitl/stale-ask-question-selection.eval.ts b/e2e/fixtures/agent-tools-hitl/evals/hitl/stale-ask-question-selection.eval.tsindex 452e4cd98..56185cb3c 100644--- a/e2e/fixtures/agent-tools-hitl/evals/hitl/stale-ask-question-selection.eval.ts+++ b/e2e/fixtures/agent-tools-hitl/evals/hitl/stale-ask-question-selection.eval.ts@@ -69,7 +69,7 @@ export default defineEval({// a literal echo the model can rightly decline.t.judge.autoevals.closedQA(- "The reply acknowledges a late selection of the STALE-CANDIDATE-7Q4M option, either by applying it or by explaining that the question was already answered and the selection is no longer relevant.",+ "The reply treats the message as a late response to an EARLIER question — either applying that earlier selection, or explaining that the earlier question was already answered or is no longer relevant (stale). Answer yes unless the reply instead treats the message as the answer to the current pending question or ignores it entirely.",{ on: staleSelection.message },).atLeast(0.5);