Skip to content

armgit / eve

Public Imported from GitHub
0

The Open Framework for Building Agents

fix: isolate authoring benchmark artifacts to reduce distraction for benchmark agents (#2240)

Colton Padden30ec0f8b148f
diff --git a/apps/benchmarks/lib/harness-agent.ts b/apps/benchmarks/lib/harness-agent.ts
index b15b036c7..3eb1beb56 100644
--- a/apps/benchmarks/lib/harness-agent.ts
+++ b/apps/benchmarks/lib/harness-agent.ts
@@ -27,6 +27,7 @@ import type { AuthoringTokenUsage, AuthoringTranscriptEntry } from "./protocol.j
import { BenchmarkTimings } from "./timing.js";
const HARNESS_BRIDGE_PORT = 4172;
+const POST_RUN_GRADER_DIRECTORY = ".eve-grader";
const BOOTSTRAP_VERSION = "v8";
// A turn that stops producing output should end the turn, not the eval: the
// remaining turns still run and the graders still see what the agent did.
@@ -224,6 +225,7 @@ export function createAuthoringAgent(subject: {
? workspace
: `${workspace}/${authoringCase.projectDirectory}`;
const context = setupContext(activeSandbox, workspace);
+ const graderContext = setupContext(activeSandbox, projectWorkspace);
await prepareGraderDirectory(context);
await context.write(
`${AGENT_EVAL_DIRECTORY}/results.json`,
@@ -234,20 +236,20 @@ export function createAuthoringAgent(subject: {
JSON.stringify(transcript),
);
await Promise.all([
- context.write(
- `${AGENT_EVAL_DIRECTORY}/EVAL.test.ts`,
+ graderContext.write(
+ `${POST_RUN_GRADER_DIRECTORY}/EVAL.test.ts`,
readFileSync(`${fixturePath}/EVAL.ts`, "utf8"),
),
- context.write(
- `${AGENT_EVAL_DIRECTORY}/grader.ts`,
+ graderContext.write(
+ `${POST_RUN_GRADER_DIRECTORY}/grader.ts`,
readFileSync(new URL("./grader.ts", import.meta.url), "utf8"),
),
- context.write(
- `${AGENT_EVAL_DIRECTORY}/paths.ts`,
+ graderContext.write(
+ `${POST_RUN_GRADER_DIRECTORY}/paths.ts`,
readFileSync(new URL("./paths.ts", import.meta.url), "utf8"),
),
- context.write(
- `${AGENT_EVAL_DIRECTORY}/protocol.ts`,
+ graderContext.write(
+ `${POST_RUN_GRADER_DIRECTORY}/protocol.ts`,
readFileSync(new URL("./protocol.ts", import.meta.url), "utf8"),
),
]);
@@ -256,7 +258,7 @@ export function createAuthoringAgent(subject: {
const test = await timings.measure("validation.grader", () =>
resultOf(
activeSandbox!,
- `ln -s ${workspace}/${AGENT_EVAL_DIRECTORY} .eve-grader && trap 'rm -f .eve-grader' EXIT && vitest run .eve-grader/EVAL.test.ts`,
+ `vitest run ${POST_RUN_GRADER_DIRECTORY}/EVAL.test.ts`,
projectWorkspace,
),
);
@@ -544,7 +546,12 @@ async function bootstrapSubject(
if (workspaceKind === "scaffolded") {
workspaceCommands.push(`cd ${shellQuote(workspace)} && AI_AGENT=benchmark eve init .`);
}
- workspaceCommands.push("command -v vitest >/dev/null");
+ // Grader files must not change what subject commands observe in the project directory.
+ workspaceCommands.push(
+ `rm -rf ${AGENT_EVAL_DIRECTORY} && mkdir -p ${AGENT_EVAL_DIRECTORY}`,
+ `printf '{"private":true,"type":"module"}\\n' >${AGENT_EVAL_DIRECTORY}/package.json`,
+ "command -v vitest >/dev/null",
+ );
if (workspaceKind === "scaffolded") {
workspaceCommands.push(`test -f ${workspace}/package.json`);
}