Skip to content

armgit / eve

Public Imported from GitHub
0

The Open Framework for Building Agents

test: allow `eve init <agent-name>` outside of workspace in project creation benchmark (#2241)

Colton Padden2d5306f8a9ba
diff --git a/apps/benchmarks/evals/author-002-new-project/CASE.ts b/apps/benchmarks/evals/author-002-new-project/CASE.ts
index 0418635f0..c65de8618 100644
--- a/apps/benchmarks/evals/author-002-new-project/CASE.ts
+++ b/apps/benchmarks/evals/author-002-new-project/CASE.ts
@@ -2,9 +2,10 @@ import { defineAuthoringCase, emptyProject } from "../../lib/authoring-case.js";
export default defineAuthoringCase({
startingPoint: emptyProject,
+ projectDirectory: "wayfinder",
async interact({ send }) {
await send(
- "Create a new eve project directly in the current working directory for a concise travel assistant called Wayfinder. Do not create a subdirectory: initialize the current directory (for example, with `eve init .`). It should use the default model and be ready to build.",
+ "Create a new eve project named `wayfinder` for a concise travel assistant called Wayfinder. It should use the default model and be ready to build.",
);
},
});
diff --git a/apps/benchmarks/evals/author-002-new-project/EVAL.ts b/apps/benchmarks/evals/author-002-new-project/EVAL.ts
index cc5d10799..fb45b91c4 100644
--- a/apps/benchmarks/evals/author-002-new-project/EVAL.ts
+++ b/apps/benchmarks/evals/author-002-new-project/EVAL.ts
@@ -4,7 +4,7 @@ import { expect, test } from "vitest";
import { subjectDefaultAgentModel } from "./grader.js";
-test("creates a complete eve project in place", () => {
+test("creates a complete eve project", () => {
expect(existsSync("agent/agent.ts")).toBe(true);
expect(existsSync("agent/channels/eve.ts")).toBe(true);
expect(existsSync("agent/instructions.md")).toBe(true);
diff --git a/apps/benchmarks/lib/authoring-case.ts b/apps/benchmarks/lib/authoring-case.ts
index bd9481e85..d708b24f3 100644
--- a/apps/benchmarks/lib/authoring-case.ts
+++ b/apps/benchmarks/lib/authoring-case.ts
@@ -41,6 +41,8 @@ export interface AuthoringInteractionContext {
export interface AuthoringCase {
readonly startingPoint: AuthoringStartingPoint;
+ /** Directory created by the agent, relative to the starting workspace. */
+ readonly projectDirectory?: string;
readonly setup?: AuthoringSetup;
readonly interact: (context: AuthoringInteractionContext) => Promise<void>;
}
diff --git a/apps/benchmarks/lib/harness-agent.ts b/apps/benchmarks/lib/harness-agent.ts
index 9c82df7e1..9fed52f71 100644
--- a/apps/benchmarks/lib/harness-agent.ts
+++ b/apps/benchmarks/lib/harness-agent.ts
@@ -145,6 +145,10 @@ export function createAuthoringAgent(subject: {
},
});
+ const projectWorkspace =
+ authoringCase.projectDirectory === undefined
+ ? workspace
+ : `${workspace}/${authoringCase.projectDirectory}`;
const context = setupContext(activeSandbox, workspace);
await context.write(
`${AGENT_EVAL_DIRECTORY}/results.json`,
@@ -176,8 +180,8 @@ export function createAuthoringAgent(subject: {
log("[grade] running deterministic assertions");
const test = await resultOf(
activeSandbox,
- `vitest run ${workspace}/${AGENT_EVAL_DIRECTORY}/EVAL.test.ts`,
- workspace,
+ `ln -s ${workspace}/${AGENT_EVAL_DIRECTORY} .eve-grader && trap 'rm -f .eve-grader' EXIT && vitest run .eve-grader/EVAL.test.ts`,
+ projectWorkspace,
);
const scriptsResults = Object.fromEntries(
await Promise.all(
@@ -186,7 +190,7 @@ export function createAuthoringAgent(subject: {
const result = await resultOf(
activeSandbox!,
`npm run ${shellQuote(script)}`,
- workspace,
+ projectWorkspace,
);
return [
script,
diff --git a/apps/benchmarks/lib/load-authoring-case.test.mjs b/apps/benchmarks/lib/load-authoring-case.test.mjs
index a51154f5e..4d9659e13 100644
--- a/apps/benchmarks/lib/load-authoring-case.test.mjs
+++ b/apps/benchmarks/lib/load-authoring-case.test.mjs
@@ -21,6 +21,11 @@ for (const entry of await readdir(evalsRoot, { withFileTypes: true })) {
});
}
+test("loads the named project output directory", async () => {
+ const authoringCase = await loadAuthoringCase(resolve(evalsRoot, "author-002-new-project"));
+ assert.equal(authoringCase.projectDirectory, "wayfinder");
+});
+
test("requires an iMessage phone-number request before supplying the number", async () => {
const authoringCase = await loadAuthoringCase(resolve(evalsRoot, "author-000-imessage"));
const prompts = [];