The Open Framework for Building Agents
feat(cli): classify init target failures (#3292)
OwenKephart3576e0af2b5c
diff --git a/.changeset/classify-init-target-failures.md b/.changeset/classify-init-target-failures.mdnew file mode 100644index 000000000..d6fc36431--- /dev/null+++ b/.changeset/classify-init-target-failures.md@@ -0,0 +1,5 @@+---+"eve": patch+---++Classify known `eve init` target and workspace-input failures into bounded telemetry categories. Telemetry continues to exclude target paths, directory contents, and error messages.diff --git a/docs/reference/telemetry.md b/docs/reference/telemetry.mdindex ec9b9312e..a32434464 100644--- a/docs/reference/telemetry.md+++ b/docs/reference/telemetry.md@@ -12,7 +12,7 @@ eve collects usage data from its CLI to help improve its commands and developmeneve sends the following information to Vercel:- The eve version, operating system, CPU architecture, and whether stdin is a terminal.-- The command you ran, its outcome, and setup or onboarding steps when applicable. When setup or onboarding fails, eve sends a bounded failure category such as target resolution, scaffolding, package-manager startup, workspace probing, dependency installation, Git initialization, handoff, or onboarding. It does not send the underlying error.+- The command you ran, its outcome, and setup or onboarding steps when applicable. When setup or onboarding fails, eve sends a bounded category describing the failed step. It does not send the underlying error.- For `eve dev`, whether you connected to a local or remote agent and whether the UI was interactive or headless.- Random identifiers for the CLI session, installation, and project, plus whether the installation and project identifiers are ephemeral or persistent.diff --git a/packages/eve/src/cli/commands/init-agent-workspace.ts b/packages/eve/src/cli/commands/init-agent-workspace.tsindex 829d96702..a6fefba31 100644--- a/packages/eve/src/cli/commands/init-agent-workspace.ts+++ b/packages/eve/src/cli/commands/init-agent-workspace.ts@@ -15,6 +15,8 @@ import { createPrompter } from "#setup/prompter.js";import { agentTemplateFiles } from "#setup/scaffold/create/project.js";import { writeTextFile } from "#setup/scaffold/files.js";+import { InitTargetError } from "./init-telemetry.js";+export interface InitCommandOptions {agents?: readonly string[];channelWebNextjs?: boolean;@@ -93,25 +95,48 @@ export async function addAgentsToWorkspace(const context = await findEveProjectContext(workspaceRoot);if (context?.kind !== "workspace") return false;if (options.channelWebNextjs === true) {- throw new Error("--channel-web-nextjs is not supported when adding a workspace agent.");+ throw new InitTargetError(+ "workspace_input",+ "--channel-web-nextjs is not supported when adding a workspace agent.",+ );}let names = options.agents;if (target !== undefined && names !== undefined) {- throw new Error("Pass either an agent name or --agents, not both.");+ throw new InitTargetError(+ "workspace_input",+ "Pass either an agent name or --agents, not both.",+ );}if (names === undefined && target !== undefined) names = [target];if (names === undefined) {if (!(process.stdin.isTTY && process.stdout.isTTY)) {- throw new Error(+ throw new InitTargetError(+ "workspace_input","This directory is an eve agent workspace. Pass an agent name, for example: eve init billing.",);}names = [await createPrompter().text({ message: "Name the new agent" })];}- validateAgentNames(names);- if (options.model !== undefined) {- const rejection = await validateModel(workspaceRoot, options.model);- if (rejection !== null) throw new Error(rejection);+ try {+ validateAgentNames(names);+ if (options.model !== undefined) {+ const rejection = await validateModel(workspaceRoot, options.model);+ if (rejection !== null) throw new Error(rejection);+ }+ } catch (error) {+ throw new InitTargetError(+ "workspace_input",+ error instanceof Error ? error.message : String(error),+ );+ }+ for (const name of names) {+ const appRoot = join(workspaceRoot, "agents", name);+ if (await pathExists(appRoot)) {+ throw new InitTargetError(+ "target_conflict",+ `Cannot create agent ${JSON.stringify(name)} because ${appRoot} already exists.`,+ );+ }}for (const name of names) await writeWorkspaceAgent(workspaceRoot, name, options);logger.log(diff --git a/packages/eve/src/cli/commands/init-target.ts b/packages/eve/src/cli/commands/init-target.tsindex 5a1e330cb..69556afb5 100644--- a/packages/eve/src/cli/commands/init-target.ts+++ b/packages/eve/src/cli/commands/init-target.ts@@ -5,6 +5,7 @@ import { classifyAgentRootEntry, getDirectoryEntryType } from "#discover/filesysimport { parseProjectName, PROJECT_NAME_ERROR } from "#setup/project-name.js";import type { InitFailurePolicy } from "./init-recovery.js";+import { InitTargetError } from "./init-telemetry.js";const ENVIRONMENT_ONLY_ENTRIES = new Set([".DS_Store",@@ -80,8 +81,9 @@ function assertTargetStaysWithinParent(): void {if (target === undefined || isAbsolute(target)) return;const relativeTarget = relative(parentPath, projectPath);- if (relativeTarget === ".." || relativeTarget.startsWith(`..${sep}`))- throw new Error(PROJECT_NAME_ERROR);+ if (relativeTarget === ".." || relativeTarget.startsWith(`..${sep}`)) {+ throw new InitTargetError("target_invalid", PROJECT_NAME_ERROR);+ }}/** Classifies the target itself without walking ancestor projects. */@@ -91,14 +93,33 @@ export async function resolveInitTarget(input: ResolveInitTargetInput): Promise<const projectPath = resolve(parentPath, input.target ?? ".");assertTargetStaysWithinParent(parentPath, input.target, projectPath);const createInPlace = projectPath === parentPath;- const kind = await pathKind(projectPath);+ let kind: Awaited<ReturnType<typeof pathKind>>;+ try {+ kind = await pathKind(projectPath);+ } catch (error) {+ throw new InitTargetError(+ "target_filesystem",+ `Could not access the initialization target: ${error instanceof Error ? error.message : String(error)}`,+ );+ }if (kind === "other") {- throw new Error(`Cannot initialize an agent because "${projectPath}" is not a directory.`);+ throw new InitTargetError(+ "target_invalid",+ `Cannot initialize an agent because "${projectPath}" is not a directory.`,+ );}if (kind === "missing") {- const projectName = parseProjectName(basename(projectPath));+ let projectName: string;+ try {+ projectName = parseProjectName(basename(projectPath));+ } catch (error) {+ throw new InitTargetError(+ "target_invalid",+ error instanceof Error ? error.message : String(error),+ );+ }return {createInPlace: false,failurePolicy: "remove",@@ -110,9 +131,25 @@ export async function resolveInitTarget(input: ResolveInitTargetInput): Promise<};}- const entries = (await readdir(projectPath)).sort();+ let entries: string[];+ try {+ entries = (await readdir(projectPath)).sort();+ } catch (error) {+ throw new InitTargetError(+ "target_filesystem",+ `Could not read the initialization target: ${error instanceof Error ? error.message : String(error)}`,+ );+ }if (entries.length === 0 || isEnvironmentOnly(entries)) {- const projectName = createInPlace ? "." : parseProjectName(basename(projectPath));+ let projectName: string;+ try {+ projectName = createInPlace ? "." : parseProjectName(basename(projectPath));+ } catch (error) {+ throw new InitTargetError(+ "target_invalid",+ error instanceof Error ? error.message : String(error),+ );+ }return {createInPlace: createInPlace || entries.length > 0,failurePolicy: "clear",@@ -125,21 +162,24 @@ export async function resolveInitTarget(input: ResolveInitTargetInput): Promise<}if (await isExistingEveProject(projectPath, entries)) {- throw new Error(+ throw new InitTargetError(+ "target_conflict",`An eve project already exists at "${projectPath}". Run an existing-project command from that directory instead.`,);}if (entries.includes("package.json")) {if (!targetProvided || input.target !== ".") {- throw new Error(+ throw new InitTargetError(+ "target_conflict",`Adding eve to an existing package requires an explicit \`eve init .\` from "${projectPath}".`,);}return { kind: "existing", projectPath };}- throw new Error(+ throw new InitTargetError(+ "target_conflict",`Cannot initialize an agent in the non-empty directory "${projectPath}". Move or remove these entries, or choose an empty target:\n${listEntries(entries)}`,);}diff --git a/packages/eve/src/cli/commands/init-telemetry.ts b/packages/eve/src/cli/commands/init-telemetry.tsnew file mode 100644index 000000000..d98748df8--- /dev/null+++ b/packages/eve/src/cli/commands/init-telemetry.ts@@ -0,0 +1,15 @@+import type { EveCliSetupFailureCode } from "#cli/telemetry/index.js";++export type InitTargetFailureCode = Extract<+ EveCliSetupFailureCode,+ "target_conflict" | "target_filesystem" | "target_invalid" | "workspace_input"+>;++export class InitTargetError extends Error {+ readonly failureCode: InitTargetFailureCode;++ constructor(failureCode: InitTargetFailureCode, message: string) {+ super(message);+ this.failureCode = failureCode;+ }+}diff --git a/packages/eve/src/cli/commands/init.integration.test.ts b/packages/eve/src/cli/commands/init.integration.test.tsindex 2a0072ea5..ae7dcbd99 100644--- a/packages/eve/src/cli/commands/init.integration.test.ts+++ b/packages/eve/src/cli/commands/init.integration.test.ts@@ -194,6 +194,37 @@ describe("runInitCommand", () => {expect(deps.tryInitializeGit).not.toHaveBeenCalled();});+ it("reports a target conflict when a workspace agent already exists", async () => {+ const workspaceRoot = await mkdtemp(join(tmpdir(), "eve-init-workspace-conflict-"));+ await mkdir(join(workspaceRoot, "agents", "support", "agent"), { recursive: true });+ await writeFile(+ join(workspaceRoot, "package.json"),+ '{"name":"workspace","dependencies":{"eve":"*"}}\n',+ );+ const output = logger();+ const deps = dependencies();+ const terminalEvents: Array<{ failureCode?: string; result: string; step: string }> = [];++ await expect(+ runInitCommand(+ output,+ workspaceRoot,+ "support",+ {},+ deps,+ undefined,+ (step, result, failureCode) => {+ terminalEvents.push({ failureCode, result, step });+ },+ ),+ ).rejects.toThrow('Cannot create agent "support"');++ expect(terminalEvents).toEqual([+ { step: "resolve_target", result: "error", failureCode: "target_conflict" },+ ]);+ expect(deps.runPackageManagerInstall).not.toHaveBeenCalled();+ });+it("creates the base agent with the runtime default model and invoking eve dependency", async () => {const parentDirectory = await mkdtemp(join(tmpdir(), "eve-init-base-"));const output = logger();@@ -430,20 +461,26 @@ describe("runInitCommand", () => {},);- it("refuses arbitrary non-empty current directories without prompting or writing", async () => {+ it("reports a target conflict for arbitrary non-empty current directories", async () => {const projectPath = await mkdtemp(join(tmpdir(), "eve-init-nonempty-"));await writeFile(join(projectPath, "notes.md"), "keep me\n", "utf8");const output = logger();const deps = dependencies();+ const terminalEvents: Array<{ failureCode?: string; result: string; step: string }> = [];- await expect(runInitCommand(output, projectPath, ".", {}, deps)).rejects.toThrow(- "Cannot initialize an agent in the non-empty directory",- );+ await expect(+ runInitCommand(output, projectPath, ".", {}, deps, undefined, (step, result, failureCode) => {+ terminalEvents.push({ failureCode, result, step });+ }),+ ).rejects.toThrow("Cannot initialize an agent in the non-empty directory");await expect(readFile(join(projectPath, "notes.md"), "utf8")).resolves.toBe("keep me\n");await expect(pathExists(join(projectPath, "package.json"))).resolves.toBe(false);await expect(pathExists(join(projectPath, "agent"))).resolves.toBe(false);expect(deps.runPackageManagerInstall).not.toHaveBeenCalled();+ expect(terminalEvents).toEqual([+ { step: "resolve_target", result: "error", failureCode: "target_conflict" },+ ]);});it.each([diff --git a/packages/eve/src/cli/commands/init.ts b/packages/eve/src/cli/commands/init.tsindex 77f91c598..41282dd23 100644--- a/packages/eve/src/cli/commands/init.ts+++ b/packages/eve/src/cli/commands/init.ts@@ -58,6 +58,7 @@ import {type InitCommandOptions,} from "./init-agent-workspace.js";import { initAgentReadySummary } from "./agent-instructions.js";+import { InitTargetError } from "./init-telemetry.js";import {cleanupFreshInitTarget,workspaceFailureNote,@@ -552,22 +553,22 @@ export async function runInitCommand(trackTerminal?: InitTerminalTracker,): Promise<void> {trackStep?.("resolve_target");- if (- await addAgentsToWorkspace(- logger,- parentDirectory,- target,- options,- dependencies.validateModelSlug,- )- ) {- trackStep?.("handoff");- trackTerminal?.("handoff", "completed");- return;- }-let result: InitResult;try {+ if (+ await addAgentsToWorkspace(+ logger,+ parentDirectory,+ target,+ options,+ dependencies.validateModelSlug,+ )+ ) {+ trackStep?.("handoff");+ trackTerminal?.("handoff", "completed");+ return;+ }+result = await runInitSteps({dependencies,logger,@@ -582,6 +583,9 @@ export async function runInitCommand(trackTerminal?.("resolve_target", "cancelled");return;}+ if (error instanceof InitTargetError) {+ trackTerminal?.("resolve_target", "error", error.failureCode);+ }throw error;}diff --git a/packages/eve/src/cli/telemetry/index.ts b/packages/eve/src/cli/telemetry/index.tsindex c52a3d3e5..788d1f3e5 100644--- a/packages/eve/src/cli/telemetry/index.ts+++ b/packages/eve/src/cli/telemetry/index.ts@@ -38,6 +38,10 @@ export type EveCliSetupTerminalResult = "completed" | "cancelled" | "error";/** A bounded, non-sensitive reason for a failed setup terminal event. */export type EveCliSetupFailureCode =| "target_resolution"+ | "target_conflict"+ | "target_invalid"+ | "target_filesystem"+ | "workspace_input"| "scaffolding"| "package_manager_not_found"| "package_manager_start_failed"