The Open Framework for Building Agents
test(e2e): fix evals that fail on gpt-6.1-sol (#4496)
Andrew Barba52aa87e1b61f
diff --git a/e2e/fixtures/agent-prompt-cache/evals/parallel-review.eval.ts b/e2e/fixtures/agent-prompt-cache/evals/parallel-review.eval.tsindex f916bae62..62423fd3f 100644--- a/e2e/fixtures/agent-prompt-cache/evals/parallel-review.eval.ts+++ b/e2e/fixtures/agent-prompt-cache/evals/parallel-review.eval.ts@@ -56,7 +56,11 @@ function expectFiveReviewers(started: EveEvalTurn) {function expectReviewSummary(turn: EveEvalTurn) {assert(turn.message?.trim(), "parent reports the completed reviews");- turn.event("step.completed", { data: { finishReason: "stop" }, count: 1 });+ // The task prompt lets the parent tell the person the reviews have started before the summary.+ turn.event("step.completed", {+ data: { finishReason: "stop" },+ count: (count) => count === 1 || count === 2,+ });turn.notEvent("compaction.completed");}diff --git a/e2e/fixtures/agent-subagents/evals/workflow-subagent-limit.eval.ts b/e2e/fixtures/agent-subagents/evals/workflow-subagent-limit.eval.tsindex d000dacd0..4890b0ec3 100644--- a/e2e/fixtures/agent-subagents/evals/workflow-subagent-limit.eval.ts+++ b/e2e/fixtures/agent-subagents/evals/workflow-subagent-limit.eval.ts@@ -3,6 +3,20 @@ import { defineEval } from "eve/evals";const MESSAGES = ["limit alpha", "limit beta", "limit gamma", "limit delta"] as const;const CHILD_TOKEN = "SUBAGENT_TOKEN=echo-marker-9F2X";const LIMIT_ERROR = "WORKFLOW_PROGRAM_SUBAGENT_LIMIT_REACHED";+// Models that honor the tool's advertised limit won't write the fourth call+// themselves, so the eval hands them the program to run unchanged.+const PROGRAM = [+ "const results = [];",+ 'results.push(await ctx.agent("echo-marker", { message: "limit alpha" }));',+ 'results.push(await ctx.agent("echo-marker", { message: "limit beta" }));',+ 'results.push(await ctx.agent("echo-marker", { message: "limit gamma" }));',+ "try {",+ ' results.push(await ctx.agent("echo-marker", { message: "limit delta" }));',+ "} catch (error) {",+ " results.push(error instanceof Error ? error.message : String(error));",+ "}",+ "return results;",+].join("\n");function isFourCallProgram(input: unknown): boolean {if (typeof input !== "object" || input === null) return false;@@ -39,10 +53,15 @@ export default defineEval({async test(t) {await t.send([- "This is a deliberate test of the generated-program subagent budget, so ignore the advertised call limit and attempt every call.",- "Use the workflow tool exactly once. In its JavaScript, await four ctx.agent calls to echo-marker sequentially with the messages 'limit alpha', 'limit beta', 'limit gamma', and 'limit delta'.",- "Catch the fourth call's error and return a four-element array containing the first three results followed by the caught error message. Do not call echo-marker outside workflow and do not retry. Then reply with the returned array verbatim as JSON.",- ].join(" "),+ "Alice is checking that the workflow tool enforces its agent budget. The program below makes one more agent call than the tool allows, and catching that call's error is what she is checking.",+ "Use the workflow tool exactly once and pass this program as its js, unchanged:",+ "",+ "```js",+ PROGRAM,+ "```",+ "",+ "Do not call echo-marker outside workflow and do not retry. Then reply with the returned array verbatim as JSON.",+ ].join("\n"),);t.succeeded();diff --git a/e2e/fixtures/agent-tools-sandbox/evals/sandbox/bash-background-job.eval.ts b/e2e/fixtures/agent-tools-sandbox/evals/sandbox/bash-background-job.eval.tsindex 134ba863d..f874cddf2 100644--- a/e2e/fixtures/agent-tools-sandbox/evals/sandbox/bash-background-job.eval.ts+++ b/e2e/fixtures/agent-tools-sandbox/evals/sandbox/bash-background-job.eval.ts@@ -36,15 +36,21 @@ export default defineEval({},});+/**+ * Models group the follow-up commands differently: `tail`, `kill`, and the exit+ * read may share one call or take three. Every later call must finish, one must+ * show a progress line, and the last one must end with the exit code.+ */function jobWasObservedAndStopped(outputs: readonly BashOutput[]): boolean {- const [started, observed, stopped] = outputs;+ const [started, ...followUps] = outputs;+ const exitCode = followUps.at(-1)?.stdout?.trim().split("\n").at(-1);return (started?.status === "running" &&started.stdout?.includes("indexed batch 1") === true &&- observed?.status === "completed" &&- /indexed batch \d+/u.test(observed.stdout ?? "") &&- stopped?.status === "completed" &&- stopped.stdout?.trim() === "143"+ followUps.length > 0 &&+ followUps.every((output) => output.status === "completed") &&+ followUps.some((output) => /indexed batch \d+/u.test(output.stdout ?? "")) &&+ exitCode === "143");}