The Open Framework for Building Agents
test: allow `eve init <agent-name>` outside of workspace in project creation benchmark (#2241)
Colton Padden2d5306f8a9ba
diff --git a/apps/benchmarks/evals/author-002-new-project/CASE.ts b/apps/benchmarks/evals/author-002-new-project/CASE.tsindex 0418635f0..c65de8618 100644--- a/apps/benchmarks/evals/author-002-new-project/CASE.ts+++ b/apps/benchmarks/evals/author-002-new-project/CASE.ts@@ -2,9 +2,10 @@ import { defineAuthoringCase, emptyProject } from "../../lib/authoring-case.js";export default defineAuthoringCase({startingPoint: emptyProject,+ projectDirectory: "wayfinder",async interact({ send }) {await send(- "Create a new eve project directly in the current working directory for a concise travel assistant called Wayfinder. Do not create a subdirectory: initialize the current directory (for example, with `eve init .`). It should use the default model and be ready to build.",+ "Create a new eve project named `wayfinder` for a concise travel assistant called Wayfinder. It should use the default model and be ready to build.",);},});diff --git a/apps/benchmarks/evals/author-002-new-project/EVAL.ts b/apps/benchmarks/evals/author-002-new-project/EVAL.tsindex cc5d10799..fb45b91c4 100644--- a/apps/benchmarks/evals/author-002-new-project/EVAL.ts+++ b/apps/benchmarks/evals/author-002-new-project/EVAL.ts@@ -4,7 +4,7 @@ import { expect, test } from "vitest";import { subjectDefaultAgentModel } from "./grader.js";-test("creates a complete eve project in place", () => {+test("creates a complete eve project", () => {expect(existsSync("agent/agent.ts")).toBe(true);expect(existsSync("agent/channels/eve.ts")).toBe(true);expect(existsSync("agent/instructions.md")).toBe(true);diff --git a/apps/benchmarks/lib/authoring-case.ts b/apps/benchmarks/lib/authoring-case.tsindex bd9481e85..d708b24f3 100644--- a/apps/benchmarks/lib/authoring-case.ts+++ b/apps/benchmarks/lib/authoring-case.ts@@ -41,6 +41,8 @@ export interface AuthoringInteractionContext {export interface AuthoringCase {readonly startingPoint: AuthoringStartingPoint;+ /** Directory created by the agent, relative to the starting workspace. */+ readonly projectDirectory?: string;readonly setup?: AuthoringSetup;readonly interact: (context: AuthoringInteractionContext) => Promise<void>;}diff --git a/apps/benchmarks/lib/harness-agent.ts b/apps/benchmarks/lib/harness-agent.tsindex 9c82df7e1..9fed52f71 100644--- a/apps/benchmarks/lib/harness-agent.ts+++ b/apps/benchmarks/lib/harness-agent.ts@@ -145,6 +145,10 @@ export function createAuthoringAgent(subject: {},});+ const projectWorkspace =+ authoringCase.projectDirectory === undefined+ ? workspace+ : `${workspace}/${authoringCase.projectDirectory}`;const context = setupContext(activeSandbox, workspace);await context.write(`${AGENT_EVAL_DIRECTORY}/results.json`,@@ -176,8 +180,8 @@ export function createAuthoringAgent(subject: {log("[grade] running deterministic assertions");const test = await resultOf(activeSandbox,- `vitest run ${workspace}/${AGENT_EVAL_DIRECTORY}/EVAL.test.ts`,- workspace,+ `ln -s ${workspace}/${AGENT_EVAL_DIRECTORY} .eve-grader && trap 'rm -f .eve-grader' EXIT && vitest run .eve-grader/EVAL.test.ts`,+ projectWorkspace,);const scriptsResults = Object.fromEntries(await Promise.all(@@ -186,7 +190,7 @@ export function createAuthoringAgent(subject: {const result = await resultOf(activeSandbox!,`npm run ${shellQuote(script)}`,- workspace,+ projectWorkspace,);return [script,diff --git a/apps/benchmarks/lib/load-authoring-case.test.mjs b/apps/benchmarks/lib/load-authoring-case.test.mjsindex a51154f5e..4d9659e13 100644--- a/apps/benchmarks/lib/load-authoring-case.test.mjs+++ b/apps/benchmarks/lib/load-authoring-case.test.mjs@@ -21,6 +21,11 @@ for (const entry of await readdir(evalsRoot, { withFileTypes: true })) {});}+test("loads the named project output directory", async () => {+ const authoringCase = await loadAuthoringCase(resolve(evalsRoot, "author-002-new-project"));+ assert.equal(authoringCase.projectDirectory, "wayfinder");+});+test("requires an iMessage phone-number request before supplying the number", async () => {const authoringCase = await loadAuthoringCase(resolve(evalsRoot, "author-000-imessage"));const prompts = [];