The Open Framework for Building Agents
fix: isolate authoring benchmark artifacts to reduce distraction for benchmark agents (#2240)
Colton Padden30ec0f8b148f
diff --git a/apps/benchmarks/lib/harness-agent.ts b/apps/benchmarks/lib/harness-agent.tsindex b15b036c7..3eb1beb56 100644--- a/apps/benchmarks/lib/harness-agent.ts+++ b/apps/benchmarks/lib/harness-agent.ts@@ -27,6 +27,7 @@ import type { AuthoringTokenUsage, AuthoringTranscriptEntry } from "./protocol.jimport { BenchmarkTimings } from "./timing.js";const HARNESS_BRIDGE_PORT = 4172;+const POST_RUN_GRADER_DIRECTORY = ".eve-grader";const BOOTSTRAP_VERSION = "v8";// A turn that stops producing output should end the turn, not the eval: the// remaining turns still run and the graders still see what the agent did.@@ -224,6 +225,7 @@ export function createAuthoringAgent(subject: {? workspace: `${workspace}/${authoringCase.projectDirectory}`;const context = setupContext(activeSandbox, workspace);+ const graderContext = setupContext(activeSandbox, projectWorkspace);await prepareGraderDirectory(context);await context.write(`${AGENT_EVAL_DIRECTORY}/results.json`,@@ -234,20 +236,20 @@ export function createAuthoringAgent(subject: {JSON.stringify(transcript),);await Promise.all([- context.write(- `${AGENT_EVAL_DIRECTORY}/EVAL.test.ts`,+ graderContext.write(+ `${POST_RUN_GRADER_DIRECTORY}/EVAL.test.ts`,readFileSync(`${fixturePath}/EVAL.ts`, "utf8"),),- context.write(- `${AGENT_EVAL_DIRECTORY}/grader.ts`,+ graderContext.write(+ `${POST_RUN_GRADER_DIRECTORY}/grader.ts`,readFileSync(new URL("./grader.ts", import.meta.url), "utf8"),),- context.write(- `${AGENT_EVAL_DIRECTORY}/paths.ts`,+ graderContext.write(+ `${POST_RUN_GRADER_DIRECTORY}/paths.ts`,readFileSync(new URL("./paths.ts", import.meta.url), "utf8"),),- context.write(- `${AGENT_EVAL_DIRECTORY}/protocol.ts`,+ graderContext.write(+ `${POST_RUN_GRADER_DIRECTORY}/protocol.ts`,readFileSync(new URL("./protocol.ts", import.meta.url), "utf8"),),]);@@ -256,7 +258,7 @@ export function createAuthoringAgent(subject: {const test = await timings.measure("validation.grader", () =>resultOf(activeSandbox!,- `ln -s ${workspace}/${AGENT_EVAL_DIRECTORY} .eve-grader && trap 'rm -f .eve-grader' EXIT && vitest run .eve-grader/EVAL.test.ts`,+ `vitest run ${POST_RUN_GRADER_DIRECTORY}/EVAL.test.ts`,projectWorkspace,),);@@ -544,7 +546,12 @@ async function bootstrapSubject(if (workspaceKind === "scaffolded") {workspaceCommands.push(`cd ${shellQuote(workspace)} && AI_AGENT=benchmark eve init .`);}- workspaceCommands.push("command -v vitest >/dev/null");+ // Grader files must not change what subject commands observe in the project directory.+ workspaceCommands.push(+ `rm -rf ${AGENT_EVAL_DIRECTORY} && mkdir -p ${AGENT_EVAL_DIRECTORY}`,+ `printf '{"private":true,"type":"module"}\\n' >${AGENT_EVAL_DIRECTORY}/package.json`,+ "command -v vitest >/dev/null",+ );if (workspaceKind === "scaffolded") {workspaceCommands.push(`test -f ${workspace}/package.json`);}