The Open Framework for Building Agents
feat(cli): classify setup telemetry failures (#3182)
OwenKephart71fbdb832740
diff --git a/.changeset/telemetry-setup-failure-taxonomy.md b/.changeset/telemetry-setup-failure-taxonomy.mdnew file mode 100644index 000000000..4c496197b--- /dev/null+++ b/.changeset/telemetry-setup-failure-taxonomy.md@@ -0,0 +1,5 @@+---+"eve": patch+---++Classify CLI setup and onboarding failures into bounded, privacy-preserving categories, including package-manager startup and pnpm workspace-probe failures. Telemetry continues to exclude error messages and command output.diff --git a/docs/reference/telemetry.md b/docs/reference/telemetry.mdindex a9b509742..ec9b9312e 100644--- a/docs/reference/telemetry.md+++ b/docs/reference/telemetry.md@@ -12,7 +12,7 @@ eve collects usage data from its CLI to help improve its commands and developmeneve sends the following information to Vercel:- The eve version, operating system, CPU architecture, and whether stdin is a terminal.-- The command you ran, its outcome, and setup or onboarding steps when applicable.+- The command you ran, its outcome, and setup or onboarding steps when applicable. When setup or onboarding fails, eve sends a bounded failure category such as target resolution, scaffolding, package-manager startup, workspace probing, dependency installation, Git initialization, handoff, or onboarding. It does not send the underlying error.- For `eve dev`, whether you connected to a local or remote agent and whether the UI was interactive or headless.- Random identifiers for the CLI session, installation, and project, plus whether the installation and project identifiers are ephemeral or persistent.diff --git a/packages/eve/src/cli/commands/init-agent-workspace.ts b/packages/eve/src/cli/commands/init-agent-workspace.tsindex 2652937a8..829d96702 100644--- a/packages/eve/src/cli/commands/init-agent-workspace.ts+++ b/packages/eve/src/cli/commands/init-agent-workspace.ts@@ -7,6 +7,8 @@ import { assertValidPublicAgentName } from "#internal/agent-name.js";import { findEveProjectContext } from "#internal/project-context.js";import { DEFAULT_AGENT_MODEL_ID } from "#shared/default-agent-model.js";import type { AgentReasoningDefinition } from "#shared/agent-definition.js";+import { formatNodeEngineOverrideWarning } from "#setup/node-engine.js";+import type { WorkspaceRootMutation } from "#setup/scaffold/workspace-root.js";import { validateModelSlug } from "#setup/flows/model-source-change.js";import { pathExists } from "#setup/path-exists.js";import { createPrompter } from "#setup/prompter.js";@@ -25,6 +27,15 @@ export interface InitCliLogger {log(message: string): void;}+export function formatWorkspaceRootMutationWarning(mutation: WorkspaceRootMutation): string {+ const target = mutation.kind === "package-json" ? "package.json" : "configuration";+ const suffix =+ mutation.nodeEngineOverride === undefined+ ? ""+ : ` (${formatNodeEngineOverrideWarning(mutation.nodeEngineOverride)})`;+ return `Updated workspace root ${target} at ${mutation.path}${suffix}`;+}+function validateAgentNames(names: readonly string[]): void {if (names.length === 0) throw new Error("--agents requires at least one agent name.");const unique = new Set<string>();diff --git a/packages/eve/src/cli/commands/init-install.ts b/packages/eve/src/cli/commands/init-install.tsnew file mode 100644index 000000000..90636f675--- /dev/null+++ b/packages/eve/src/cli/commands/init-install.ts@@ -0,0 +1,37 @@+import type { EveCliSetupFailureCode } from "#cli/telemetry/index.js";+import type { PackageManagerKind } from "#setup/package-manager.js";+import { type PackageManagerInstallResult } from "#setup/primitives/index.js";+import type { ProcessOutputLine } from "#setup/primitives/process-output.js";++export function installProgressDetail(+ packageManager: PackageManagerKind,+ line: ProcessOutputLine,+): string | undefined {+ const text = line.text.trim();+ if (text === "" || packageManager !== "npm") return text || undefined;++ const manifest = /^npm silly fetch manifest (.+)$/u.exec(text);+ if (manifest !== null) return `Resolving ${manifest[1]}`;++ const failedRequest = /^npm http fetch \S+ \S+ attempt (\d+) failed with (\S+)$/u.exec(text);+ if (failedRequest !== null) {+ return `npm registry · attempt ${failedRequest[1]} failed: ${failedRequest[2]}`;+ }++ if (line.stream === "stdout" || /^npm (?:error|warn)\b/u.test(text)) return text;+ return undefined;+}++export const NPM_NOISE_LINE = /^\s*npm (?:silly|verbose|http|timing)\b/u;+export const INSTALL_OUTPUT_FALLBACK_LINES = 20;++export function packageManagerInstallFailureCode(+ result: PackageManagerInstallResult,+): EveCliSetupFailureCode {+ if (result.kind === "workspace-probe-failed") return "workspace_probe_failed";+ if (result.kind === "workspace-probe-unrecognized") return "workspace_probe_unrecognized";+ if (result.result.termination.kind !== "spawn-error") return "dependency_installation";+ return result.result.termination.code === "ENOENT"+ ? "package_manager_not_found"+ : "package_manager_start_failed";+}diff --git a/packages/eve/src/cli/commands/init.integration.test.ts b/packages/eve/src/cli/commands/init.integration.test.tsindex 9dc158b83..2a0072ea5 100644--- a/packages/eve/src/cli/commands/init.integration.test.ts+++ b/packages/eve/src/cli/commands/init.integration.test.ts@@ -63,9 +63,6 @@ function logger(): InitCliLogger & { messages: string[]; errors: string[] } {function dependencies(gitResult: GitInitResult = { kind: "initialized" },): InitCommandDependencies & {- confirmInitInNonEmptyDirectory: ReturnType<- typeof vi.fn<InitCommandDependencies["confirmInitInNonEmptyDirectory"]>- >;detectInvokingPackageManager: ReturnType<typeof vi.fn<InitCommandDependencies["detectInvokingPackageManager"]>>;@@ -88,7 +85,6 @@ function dependencies(}return addAgentToProject(merged);},- confirmInitInNonEmptyDirectory: vi.fn(async () => ({ kind: "current-directory" })),// Stubbed to "no visible manager" so assertions do not depend on which// manager launched the test runner itself.detectInvokingPackageManager: vi.fn(() => undefined),@@ -447,7 +443,6 @@ describe("runInitCommand", () => {await expect(readFile(join(projectPath, "notes.md"), "utf8")).resolves.toBe("keep me\n");await expect(pathExists(join(projectPath, "package.json"))).resolves.toBe(false);await expect(pathExists(join(projectPath, "agent"))).resolves.toBe(false);- expect(deps.confirmInitInNonEmptyDirectory).not.toHaveBeenCalled();expect(deps.runPackageManagerInstall).not.toHaveBeenCalled();});@@ -889,7 +884,6 @@ describe("runInitCommand", () => {);// An existing project's history is its own; only fresh scaffolds get git init.expect(deps.tryInitializeGit).not.toHaveBeenCalled();- expect(deps.confirmInitInNonEmptyDirectory).not.toHaveBeenCalled();expect(deps.spawnPackageManager).toHaveBeenCalledWith("pnpm", projectRoot, ["exec","eve",@@ -1262,10 +1256,11 @@ describe("runInitCommand", () => {);});- it("prints a spawn failure when installation produces no child output", async () => {+ it("categorizes a missing package manager without collecting process output", async () => {const parentDirectory = await mkdtemp(join(tmpdir(), "eve-init-spawn-fail-"));const output = logger();const deps = dependencies();+ const terminalEvents: Array<{ failureCode?: string; result: string; step: string }> = [];deps.runPackageManagerInstall.mockResolvedValue({kind: "installed",result: {@@ -1275,11 +1270,28 @@ describe("runInitCommand", () => {},});- await expect(runInitCommand(output, parentDirectory, "my-agent", {}, deps)).rejects.toThrow(- "Failed to install dependencies",- );+ await expect(+ runInitCommand(+ output,+ parentDirectory,+ "my-agent",+ {},+ deps,+ undefined,+ (step, result, failureCode) => {+ terminalEvents.push({ failureCode, result, step });+ },+ ),+ ).rejects.toThrow("Failed to install dependencies");expect(output.errors).toEqual(["pnpm was not found. Install it before running this step."]);+ expect(terminalEvents).toEqual([+ {+ step: "install_dependencies",+ result: "error",+ failureCode: "package_manager_not_found",+ },+ ]);});it("preserves an existing host after install failure and prints the retry command", async () => {diff --git a/packages/eve/src/cli/commands/init.ts b/packages/eve/src/cli/commands/init.tsindex bef13c2b0..77f91c598 100644--- a/packages/eve/src/cli/commands/init.ts+++ b/packages/eve/src/cli/commands/init.ts@@ -5,7 +5,11 @@ import { performance } from "node:perf_hooks";import pc from "#compiled/picocolors/index.js";import { isCodingAgentLaunch } from "#cli/agent-detection.js";-import type { EveCliSetupStep, EveCliSetupTerminalResult } from "#cli/telemetry/index.js";+import type {+ EveCliSetupFailureCode,+ EveCliSetupStep,+ EveCliSetupTerminalResult,+} from "#cli/telemetry/index.js";import { EVE_WORDMARK } from "#cli/banner.js";import { formatElapsed } from "#cli/format-elapsed.js";import { startCliLiveRow } from "#cli/ui/live-row.js";@@ -26,7 +30,6 @@ import {runPackageManagerInstall,spawnPackageManager,} from "#setup/primitives/index.js";-import type { ProcessOutputLine } from "#setup/primitives/process-output.js";import { addAgentToProject } from "#setup/scaffold/create/add-to-project.js";import { ensureChannel, scaffoldBaseProject } from "#setup/scaffold/index.js";import { WizardCancelledError } from "#setup/step.js";@@ -41,14 +44,20 @@ import {} from "#setup/scaffold/create/project.js";import { initAgentDevHandoff, initAgentReplPrompt } from "./agent-instructions.js";+import {+ installProgressDetail,+ INSTALL_OUTPUT_FALLBACK_LINES,+ NPM_NOISE_LINE,+ packageManagerInstallFailureCode,+} from "./init-install.js";import {addAgentsToWorkspace,convertScaffoldToAgentWorkspace,+ formatWorkspaceRootMutationWarning,type InitCliLogger,type InitCommandOptions,} from "./init-agent-workspace.js";import { initAgentReadySummary } from "./agent-instructions.js";-import { confirmInitInNonEmptyDirectory } from "./init-confirm.js";import {cleanupFreshInitTarget,workspaceFailureNote,@@ -62,7 +71,6 @@ export type { InitCliLogger, InitCommandOptions } from "./init-agent-workspace.jexport interface InitCommandDependencies {addAgentToProject: typeof addAgentToProject;- confirmInitInNonEmptyDirectory: typeof confirmInitInNonEmptyDirectory;detectInvokingPackageManager: typeof detectInvokingPackageManager;detectPackageManager: typeof detectPackageManager;ensureChannel: typeof ensureChannel;@@ -79,7 +87,6 @@ export interface InitCommandDependencies {const defaultDependencies: InitCommandDependencies = {addAgentToProject,- confirmInitInNonEmptyDirectory,detectInvokingPackageManager,detectPackageManager,ensureChannel,@@ -120,15 +127,6 @@ function uniqueWorkspaceRootMutations(return [...byKey.values()];}-function formatWorkspaceRootMutationWarning(mutation: WorkspaceRootMutation): string {- const target = mutation.kind === "package-json" ? "package.json" : "configuration";- const suffix =- mutation.nodeEngineOverride === undefined- ? ""- : ` (${formatNodeEngineOverrideWarning(mutation.nodeEngineOverride)})`;- return `Updated workspace root ${target} at ${mutation.path}${suffix}`;-}-async function addToExistingProject(targetPath: string,options: InitCommandOptions,@@ -285,6 +283,12 @@ type PreparedInitProject =workspaceRootMutations: WorkspaceRootMutation[];};+type InitTerminalTracker = (+ step: EveCliSetupStep,+ result: EveCliSetupTerminalResult,+ failureCode?: EveCliSetupFailureCode,+) => void;+type InitResult = {agentElapsedMs: number;agentLaunched: boolean;@@ -306,28 +310,6 @@ type InitResult = {});-function installProgressDetail(- packageManager: PackageManagerKind,- line: ProcessOutputLine,-): string | undefined {- const text = line.text.trim();- if (text === "" || packageManager !== "npm") return text || undefined;-- const manifest = /^npm silly fetch manifest (.+)$/u.exec(text);- if (manifest !== null) return `Resolving ${manifest[1]}`;-- const failedRequest = /^npm http fetch \S+ \S+ attempt (\d+) failed with (\S+)$/u.exec(text);- if (failedRequest !== null) {- return `npm registry · attempt ${failedRequest[1]} failed: ${failedRequest[2]}`;- }-- if (line.stream === "stdout" || /^npm (?:error|warn)\b/u.test(text)) return text;- return undefined;-}--const NPM_NOISE_LINE = /^\s*npm (?:silly|verbose|http|timing)\b/u;-const INSTALL_OUTPUT_FALLBACK_LINES = 20;-function reportExistingProjectChanges(logger: InitCliLogger,project: Extract<PreparedInitProject, { kind: "added" }>,@@ -354,18 +336,22 @@ async function runInitSteps(input: {parentDirectory: string;target: string | undefined;trackStep?: (step: EveCliSetupStep) => void;+ trackTerminal?: InitTerminalTracker;}): Promise<InitResult> {- const { dependencies, logger, options, parentDirectory, target, trackStep } = input;+ const { dependencies, logger, options, parentDirectory, target, trackStep, trackTerminal } =+ input;const debug = isLogLevelEnabled("debug");const agentLaunched = await dependencies.isCodingAgentLaunch();const initTarget = await resolveInitTarget({ parentDirectory, target });const evePackage = resolveInitEvePackageOverride();let progress = startCliLiveRow(logger);+ let activeInitStep: EveCliSetupStep = "scaffold";+ let installFailureCode: EveCliSetupFailureCode | undefined;progress.update("Preparing project");try {const scaffoldPhase = initTarget.kind === "fresh" ? "creating agent" : "adding agent";- trackStep?.("scaffold");+ trackStep?.(activeInitStep);progress.update(initTarget.kind === "fresh" ? "Creating agent" : "Adding agent");initLog.debug(scaffoldPhase);const agentStartedAt = dependencies.now();@@ -457,7 +443,8 @@ async function runInitSteps(input: {progress = startCliLiveRow(logger);}- trackStep?.("install_dependencies");+ activeInitStep = "install_dependencies";+ trackStep?.(activeInitStep);progress.update("Installing dependencies", `${project.packageManager} install`);initLog.debug(`installing dependencies with ${project.packageManager}`);const installStartedAt = dependencies.now();@@ -486,6 +473,7 @@ async function runInitSteps(input: {);const installElapsedMs = dependencies.now() - installStartedAt;if (!packageManagerInstallSucceeded(installResult)) {+ installFailureCode = packageManagerInstallFailureCode(installResult);initLog.debug("dependency installation failed", { ms: installElapsedMs });progress.stop();const failureOutput =@@ -528,7 +516,8 @@ async function runInitSteps(input: {initLog.debug("dependencies installed", { ms: installElapsedMs });if (project.kind === "created") {- trackStep?.("initialize_git");+ activeInitStep = "initialize_git";+ trackStep?.(activeInitStep);progress.update("Initializing Git repository");initLog.debug("initializing git repository");return {@@ -541,6 +530,13 @@ async function runInitSteps(input: {}return { ...project, agentElapsedMs, agentLaunched, installElapsedMs };+ } catch (error) {+ trackTerminal?.(+ activeInitStep,+ "error",+ activeInitStep === "install_dependencies" ? installFailureCode : undefined,+ );+ throw error;} finally {progress.stop();}@@ -553,7 +549,7 @@ export async function runInitCommand(options: InitCommandOptions,dependencies: InitCommandDependencies = defaultDependencies,trackStep?: (step: EveCliSetupStep) => void,- trackTerminal?: (step: EveCliSetupStep, result: EveCliSetupTerminalResult) => void,+ trackTerminal?: InitTerminalTracker,): Promise<void> {trackStep?.("resolve_target");if (@@ -579,6 +575,7 @@ export async function runInitCommand(parentDirectory,target,trackStep,+ trackTerminal,});} catch (error) {if (error instanceof WizardCancelledError) {diff --git a/packages/eve/src/cli/run.ts b/packages/eve/src/cli/run.tsindex 2f64c6ab1..1fa072ccd 100644--- a/packages/eve/src/cli/run.ts+++ b/packages/eve/src/cli/run.ts@@ -258,8 +258,8 @@ export function createCliProgram((step) => {telemetry.trackSetupStep({ flow: "init", step });},- (step, result) => {- telemetry.trackSetupTerminal({ flow: "init", step, result });+ (step, result, failureCode) => {+ telemetry.trackSetupTerminal({ flow: "init", step, result, failureCode });},);},diff --git a/packages/eve/src/cli/telemetry/index.test.ts b/packages/eve/src/cli/telemetry/index.test.tsindex 438fec754..8e271d177 100644--- a/packages/eve/src/cli/telemetry/index.test.ts+++ b/packages/eve/src/cli/telemetry/index.test.ts@@ -190,6 +190,33 @@ describe("createEveCliTelemetry", () => {expect(events).toContainEqual(expect.objectContaining({ key: "setup_step", value: "model_provider" }),);+ expect(events).toContainEqual(+ expect.objectContaining({ key: "setup_failure_code", value: "dependency_installation" }),+ );+ });++ it("records an explicit bounded setup failure category", async () => {+ vi.stubEnv("NODE_ENV", "production");+ vi.stubEnv("EVE_TELEMETRY_DEBUG", "1");+ const write = vi.spyOn(process.stderr, "write").mockImplementation(() => true);+ const telemetry = createEveCliTelemetry("1.0.0");+ telemetry.trackCommand("init");+ telemetry.trackSetupStep({ flow: "init", step: "resolve_target" });+ telemetry.trackSetupTerminal({+ flow: "init",+ step: "resolve_target",+ result: "error",+ failureCode: "target_resolution",+ });++ await telemetry.flush();++ const events = JSON.parse(+ String(write.mock.calls[0]?.[0]).replace("[eve telemetry] ", ""),+ ) as Array<{ key: string; value: string }>;+ expect(events).toContainEqual(+ expect.objectContaining({ key: "setup_failure_code", value: "target_resolution" }),+ );});it("skips telemetry when identity initialization fails", async () => {diff --git a/packages/eve/src/cli/telemetry/index.ts b/packages/eve/src/cli/telemetry/index.tsindex 19f6a2b17..c52a3d3e5 100644--- a/packages/eve/src/cli/telemetry/index.ts+++ b/packages/eve/src/cli/telemetry/index.ts@@ -35,6 +35,19 @@ export type EveCliSetupStep =| "registry_install";export type EveCliSetupTerminalResult = "completed" | "cancelled" | "error";+/** A bounded, non-sensitive reason for a failed setup terminal event. */+export type EveCliSetupFailureCode =+ | "target_resolution"+ | "scaffolding"+ | "package_manager_not_found"+ | "package_manager_start_failed"+ | "workspace_probe_failed"+ | "workspace_probe_unrecognized"+ | "dependency_installation"+ | "git_initialization"+ | "handoff"+ | "onboarding";+export type EveCliSetupStepEvent = {flow: EveCliSetupFlow;step: EveCliSetupStep;@@ -45,6 +58,7 @@ export type EveCliSetupTerminalEvent = {flow: EveCliSetupFlow;step: EveCliSetupStep;result: EveCliSetupTerminalResult;+ failureCode?: EveCliSetupFailureCode;};export type EveCliTelemetry = {@@ -69,6 +83,28 @@ function event(key: string, value: string): EveCliTelemetryEvent {return { id: randomUUID(), event_time: Date.now(), key, value };}+function setupFailureCode(step: EveCliSetupStep): EveCliSetupFailureCode {+ switch (step) {+ case "resolve_target":+ return "target_resolution";+ case "scaffold":+ return "scaffolding";+ case "install_dependencies":+ case "registry_install":+ return "dependency_installation";+ case "initialize_git":+ return "git_initialization";+ case "handoff":+ return "handoff";+ case "model_provider":+ case "model_settings":+ case "registry_channels":+ case "registry_integrations":+ case "registry_review":+ return "onboarding";+ }+}+const CLI_TELEMETRY_COMMANDS = new Map<string, string>([["acp", "acp"],["add", "add"],@@ -163,6 +199,11 @@ export function createEveCliTelemetry(version: string): EveCliTelemetry {event("setup_terminal_step", input.step),event("setup_terminal_result", input.result),);+ if (input.result === "error") {+ setupEvents.push(+ event("setup_failure_code", input.failureCode ?? setupFailureCode(input.step)),+ );+ }},trackOutcome(outcome) {if (activeSetup !== undefined && !setupTerminalRecorded) {