Skip to content

armgit / eve

Public Imported from GitHub
0

The Open Framework for Building Agents

feat(cli): classify setup telemetry failures (#3182)

OwenKephart71fbdb832740
diff --git a/.changeset/telemetry-setup-failure-taxonomy.md b/.changeset/telemetry-setup-failure-taxonomy.md
new file mode 100644
index 000000000..4c496197b
--- /dev/null
+++ b/.changeset/telemetry-setup-failure-taxonomy.md
@@ -0,0 +1,5 @@
+---
+"eve": patch
+---
+
+Classify CLI setup and onboarding failures into bounded, privacy-preserving categories, including package-manager startup and pnpm workspace-probe failures. Telemetry continues to exclude error messages and command output.
diff --git a/docs/reference/telemetry.md b/docs/reference/telemetry.md
index a9b509742..ec9b9312e 100644
--- a/docs/reference/telemetry.md
+++ b/docs/reference/telemetry.md
@@ -12,7 +12,7 @@ eve collects usage data from its CLI to help improve its commands and developmen
eve sends the following information to Vercel:
- The eve version, operating system, CPU architecture, and whether stdin is a terminal.
-- The command you ran, its outcome, and setup or onboarding steps when applicable.
+- The command you ran, its outcome, and setup or onboarding steps when applicable. When setup or onboarding fails, eve sends a bounded failure category such as target resolution, scaffolding, package-manager startup, workspace probing, dependency installation, Git initialization, handoff, or onboarding. It does not send the underlying error.
- For `eve dev`, whether you connected to a local or remote agent and whether the UI was interactive or headless.
- Random identifiers for the CLI session, installation, and project, plus whether the installation and project identifiers are ephemeral or persistent.
diff --git a/packages/eve/src/cli/commands/init-agent-workspace.ts b/packages/eve/src/cli/commands/init-agent-workspace.ts
index 2652937a8..829d96702 100644
--- a/packages/eve/src/cli/commands/init-agent-workspace.ts
+++ b/packages/eve/src/cli/commands/init-agent-workspace.ts
@@ -7,6 +7,8 @@ import { assertValidPublicAgentName } from "#internal/agent-name.js";
import { findEveProjectContext } from "#internal/project-context.js";
import { DEFAULT_AGENT_MODEL_ID } from "#shared/default-agent-model.js";
import type { AgentReasoningDefinition } from "#shared/agent-definition.js";
+import { formatNodeEngineOverrideWarning } from "#setup/node-engine.js";
+import type { WorkspaceRootMutation } from "#setup/scaffold/workspace-root.js";
import { validateModelSlug } from "#setup/flows/model-source-change.js";
import { pathExists } from "#setup/path-exists.js";
import { createPrompter } from "#setup/prompter.js";
@@ -25,6 +27,15 @@ export interface InitCliLogger {
log(message: string): void;
}
+export function formatWorkspaceRootMutationWarning(mutation: WorkspaceRootMutation): string {
+ const target = mutation.kind === "package-json" ? "package.json" : "configuration";
+ const suffix =
+ mutation.nodeEngineOverride === undefined
+ ? ""
+ : ` (${formatNodeEngineOverrideWarning(mutation.nodeEngineOverride)})`;
+ return `Updated workspace root ${target} at ${mutation.path}${suffix}`;
+}
+
function validateAgentNames(names: readonly string[]): void {
if (names.length === 0) throw new Error("--agents requires at least one agent name.");
const unique = new Set<string>();
diff --git a/packages/eve/src/cli/commands/init-install.ts b/packages/eve/src/cli/commands/init-install.ts
new file mode 100644
index 000000000..90636f675
--- /dev/null
+++ b/packages/eve/src/cli/commands/init-install.ts
@@ -0,0 +1,37 @@
+import type { EveCliSetupFailureCode } from "#cli/telemetry/index.js";
+import type { PackageManagerKind } from "#setup/package-manager.js";
+import { type PackageManagerInstallResult } from "#setup/primitives/index.js";
+import type { ProcessOutputLine } from "#setup/primitives/process-output.js";
+
+export function installProgressDetail(
+ packageManager: PackageManagerKind,
+ line: ProcessOutputLine,
+): string | undefined {
+ const text = line.text.trim();
+ if (text === "" || packageManager !== "npm") return text || undefined;
+
+ const manifest = /^npm silly fetch manifest (.+)$/u.exec(text);
+ if (manifest !== null) return `Resolving ${manifest[1]}`;
+
+ const failedRequest = /^npm http fetch \S+ \S+ attempt (\d+) failed with (\S+)$/u.exec(text);
+ if (failedRequest !== null) {
+ return `npm registry · attempt ${failedRequest[1]} failed: ${failedRequest[2]}`;
+ }
+
+ if (line.stream === "stdout" || /^npm (?:error|warn)\b/u.test(text)) return text;
+ return undefined;
+}
+
+export const NPM_NOISE_LINE = /^\s*npm (?:silly|verbose|http|timing)\b/u;
+export const INSTALL_OUTPUT_FALLBACK_LINES = 20;
+
+export function packageManagerInstallFailureCode(
+ result: PackageManagerInstallResult,
+): EveCliSetupFailureCode {
+ if (result.kind === "workspace-probe-failed") return "workspace_probe_failed";
+ if (result.kind === "workspace-probe-unrecognized") return "workspace_probe_unrecognized";
+ if (result.result.termination.kind !== "spawn-error") return "dependency_installation";
+ return result.result.termination.code === "ENOENT"
+ ? "package_manager_not_found"
+ : "package_manager_start_failed";
+}
diff --git a/packages/eve/src/cli/commands/init.integration.test.ts b/packages/eve/src/cli/commands/init.integration.test.ts
index 9dc158b83..2a0072ea5 100644
--- a/packages/eve/src/cli/commands/init.integration.test.ts
+++ b/packages/eve/src/cli/commands/init.integration.test.ts
@@ -63,9 +63,6 @@ function logger(): InitCliLogger & { messages: string[]; errors: string[] } {
function dependencies(
gitResult: GitInitResult = { kind: "initialized" },
): InitCommandDependencies & {
- confirmInitInNonEmptyDirectory: ReturnType<
- typeof vi.fn<InitCommandDependencies["confirmInitInNonEmptyDirectory"]>
- >;
detectInvokingPackageManager: ReturnType<
typeof vi.fn<InitCommandDependencies["detectInvokingPackageManager"]>
>;
@@ -88,7 +85,6 @@ function dependencies(
}
return addAgentToProject(merged);
},
- confirmInitInNonEmptyDirectory: vi.fn(async () => ({ kind: "current-directory" })),
// Stubbed to "no visible manager" so assertions do not depend on which
// manager launched the test runner itself.
detectInvokingPackageManager: vi.fn(() => undefined),
@@ -447,7 +443,6 @@ describe("runInitCommand", () => {
await expect(readFile(join(projectPath, "notes.md"), "utf8")).resolves.toBe("keep me\n");
await expect(pathExists(join(projectPath, "package.json"))).resolves.toBe(false);
await expect(pathExists(join(projectPath, "agent"))).resolves.toBe(false);
- expect(deps.confirmInitInNonEmptyDirectory).not.toHaveBeenCalled();
expect(deps.runPackageManagerInstall).not.toHaveBeenCalled();
});
@@ -889,7 +884,6 @@ describe("runInitCommand", () => {
);
// An existing project's history is its own; only fresh scaffolds get git init.
expect(deps.tryInitializeGit).not.toHaveBeenCalled();
- expect(deps.confirmInitInNonEmptyDirectory).not.toHaveBeenCalled();
expect(deps.spawnPackageManager).toHaveBeenCalledWith("pnpm", projectRoot, [
"exec",
"eve",
@@ -1262,10 +1256,11 @@ describe("runInitCommand", () => {
);
});
- it("prints a spawn failure when installation produces no child output", async () => {
+ it("categorizes a missing package manager without collecting process output", async () => {
const parentDirectory = await mkdtemp(join(tmpdir(), "eve-init-spawn-fail-"));
const output = logger();
const deps = dependencies();
+ const terminalEvents: Array<{ failureCode?: string; result: string; step: string }> = [];
deps.runPackageManagerInstall.mockResolvedValue({
kind: "installed",
result: {
@@ -1275,11 +1270,28 @@ describe("runInitCommand", () => {
},
});
- await expect(runInitCommand(output, parentDirectory, "my-agent", {}, deps)).rejects.toThrow(
- "Failed to install dependencies",
- );
+ await expect(
+ runInitCommand(
+ output,
+ parentDirectory,
+ "my-agent",
+ {},
+ deps,
+ undefined,
+ (step, result, failureCode) => {
+ terminalEvents.push({ failureCode, result, step });
+ },
+ ),
+ ).rejects.toThrow("Failed to install dependencies");
expect(output.errors).toEqual(["pnpm was not found. Install it before running this step."]);
+ expect(terminalEvents).toEqual([
+ {
+ step: "install_dependencies",
+ result: "error",
+ failureCode: "package_manager_not_found",
+ },
+ ]);
});
it("preserves an existing host after install failure and prints the retry command", async () => {
diff --git a/packages/eve/src/cli/commands/init.ts b/packages/eve/src/cli/commands/init.ts
index bef13c2b0..77f91c598 100644
--- a/packages/eve/src/cli/commands/init.ts
+++ b/packages/eve/src/cli/commands/init.ts
@@ -5,7 +5,11 @@ import { performance } from "node:perf_hooks";
import pc from "#compiled/picocolors/index.js";
import { isCodingAgentLaunch } from "#cli/agent-detection.js";
-import type { EveCliSetupStep, EveCliSetupTerminalResult } from "#cli/telemetry/index.js";
+import type {
+ EveCliSetupFailureCode,
+ EveCliSetupStep,
+ EveCliSetupTerminalResult,
+} from "#cli/telemetry/index.js";
import { EVE_WORDMARK } from "#cli/banner.js";
import { formatElapsed } from "#cli/format-elapsed.js";
import { startCliLiveRow } from "#cli/ui/live-row.js";
@@ -26,7 +30,6 @@ import {
runPackageManagerInstall,
spawnPackageManager,
} from "#setup/primitives/index.js";
-import type { ProcessOutputLine } from "#setup/primitives/process-output.js";
import { addAgentToProject } from "#setup/scaffold/create/add-to-project.js";
import { ensureChannel, scaffoldBaseProject } from "#setup/scaffold/index.js";
import { WizardCancelledError } from "#setup/step.js";
@@ -41,14 +44,20 @@ import {
} from "#setup/scaffold/create/project.js";
import { initAgentDevHandoff, initAgentReplPrompt } from "./agent-instructions.js";
+import {
+ installProgressDetail,
+ INSTALL_OUTPUT_FALLBACK_LINES,
+ NPM_NOISE_LINE,
+ packageManagerInstallFailureCode,
+} from "./init-install.js";
import {
addAgentsToWorkspace,
convertScaffoldToAgentWorkspace,
+ formatWorkspaceRootMutationWarning,
type InitCliLogger,
type InitCommandOptions,
} from "./init-agent-workspace.js";
import { initAgentReadySummary } from "./agent-instructions.js";
-import { confirmInitInNonEmptyDirectory } from "./init-confirm.js";
import {
cleanupFreshInitTarget,
workspaceFailureNote,
@@ -62,7 +71,6 @@ export type { InitCliLogger, InitCommandOptions } from "./init-agent-workspace.j
export interface InitCommandDependencies {
addAgentToProject: typeof addAgentToProject;
- confirmInitInNonEmptyDirectory: typeof confirmInitInNonEmptyDirectory;
detectInvokingPackageManager: typeof detectInvokingPackageManager;
detectPackageManager: typeof detectPackageManager;
ensureChannel: typeof ensureChannel;
@@ -79,7 +87,6 @@ export interface InitCommandDependencies {
const defaultDependencies: InitCommandDependencies = {
addAgentToProject,
- confirmInitInNonEmptyDirectory,
detectInvokingPackageManager,
detectPackageManager,
ensureChannel,
@@ -120,15 +127,6 @@ function uniqueWorkspaceRootMutations(
return [...byKey.values()];
}
-function formatWorkspaceRootMutationWarning(mutation: WorkspaceRootMutation): string {
- const target = mutation.kind === "package-json" ? "package.json" : "configuration";
- const suffix =
- mutation.nodeEngineOverride === undefined
- ? ""
- : ` (${formatNodeEngineOverrideWarning(mutation.nodeEngineOverride)})`;
- return `Updated workspace root ${target} at ${mutation.path}${suffix}`;
-}
-
async function addToExistingProject(
targetPath: string,
options: InitCommandOptions,
@@ -285,6 +283,12 @@ type PreparedInitProject =
workspaceRootMutations: WorkspaceRootMutation[];
};
+type InitTerminalTracker = (
+ step: EveCliSetupStep,
+ result: EveCliSetupTerminalResult,
+ failureCode?: EveCliSetupFailureCode,
+) => void;
+
type InitResult = {
agentElapsedMs: number;
agentLaunched: boolean;
@@ -306,28 +310,6 @@ type InitResult = {
}
);
-function installProgressDetail(
- packageManager: PackageManagerKind,
- line: ProcessOutputLine,
-): string | undefined {
- const text = line.text.trim();
- if (text === "" || packageManager !== "npm") return text || undefined;
-
- const manifest = /^npm silly fetch manifest (.+)$/u.exec(text);
- if (manifest !== null) return `Resolving ${manifest[1]}`;
-
- const failedRequest = /^npm http fetch \S+ \S+ attempt (\d+) failed with (\S+)$/u.exec(text);
- if (failedRequest !== null) {
- return `npm registry · attempt ${failedRequest[1]} failed: ${failedRequest[2]}`;
- }
-
- if (line.stream === "stdout" || /^npm (?:error|warn)\b/u.test(text)) return text;
- return undefined;
-}
-
-const NPM_NOISE_LINE = /^\s*npm (?:silly|verbose|http|timing)\b/u;
-const INSTALL_OUTPUT_FALLBACK_LINES = 20;
-
function reportExistingProjectChanges(
logger: InitCliLogger,
project: Extract<PreparedInitProject, { kind: "added" }>,
@@ -354,18 +336,22 @@ async function runInitSteps(input: {
parentDirectory: string;
target: string | undefined;
trackStep?: (step: EveCliSetupStep) => void;
+ trackTerminal?: InitTerminalTracker;
}): Promise<InitResult> {
- const { dependencies, logger, options, parentDirectory, target, trackStep } = input;
+ const { dependencies, logger, options, parentDirectory, target, trackStep, trackTerminal } =
+ input;
const debug = isLogLevelEnabled("debug");
const agentLaunched = await dependencies.isCodingAgentLaunch();
const initTarget = await resolveInitTarget({ parentDirectory, target });
const evePackage = resolveInitEvePackageOverride();
let progress = startCliLiveRow(logger);
+ let activeInitStep: EveCliSetupStep = "scaffold";
+ let installFailureCode: EveCliSetupFailureCode | undefined;
progress.update("Preparing project");
try {
const scaffoldPhase = initTarget.kind === "fresh" ? "creating agent" : "adding agent";
- trackStep?.("scaffold");
+ trackStep?.(activeInitStep);
progress.update(initTarget.kind === "fresh" ? "Creating agent" : "Adding agent");
initLog.debug(scaffoldPhase);
const agentStartedAt = dependencies.now();
@@ -457,7 +443,8 @@ async function runInitSteps(input: {
progress = startCliLiveRow(logger);
}
- trackStep?.("install_dependencies");
+ activeInitStep = "install_dependencies";
+ trackStep?.(activeInitStep);
progress.update("Installing dependencies", `${project.packageManager} install`);
initLog.debug(`installing dependencies with ${project.packageManager}`);
const installStartedAt = dependencies.now();
@@ -486,6 +473,7 @@ async function runInitSteps(input: {
);
const installElapsedMs = dependencies.now() - installStartedAt;
if (!packageManagerInstallSucceeded(installResult)) {
+ installFailureCode = packageManagerInstallFailureCode(installResult);
initLog.debug("dependency installation failed", { ms: installElapsedMs });
progress.stop();
const failureOutput =
@@ -528,7 +516,8 @@ async function runInitSteps(input: {
initLog.debug("dependencies installed", { ms: installElapsedMs });
if (project.kind === "created") {
- trackStep?.("initialize_git");
+ activeInitStep = "initialize_git";
+ trackStep?.(activeInitStep);
progress.update("Initializing Git repository");
initLog.debug("initializing git repository");
return {
@@ -541,6 +530,13 @@ async function runInitSteps(input: {
}
return { ...project, agentElapsedMs, agentLaunched, installElapsedMs };
+ } catch (error) {
+ trackTerminal?.(
+ activeInitStep,
+ "error",
+ activeInitStep === "install_dependencies" ? installFailureCode : undefined,
+ );
+ throw error;
} finally {
progress.stop();
}
@@ -553,7 +549,7 @@ export async function runInitCommand(
options: InitCommandOptions,
dependencies: InitCommandDependencies = defaultDependencies,
trackStep?: (step: EveCliSetupStep) => void,
- trackTerminal?: (step: EveCliSetupStep, result: EveCliSetupTerminalResult) => void,
+ trackTerminal?: InitTerminalTracker,
): Promise<void> {
trackStep?.("resolve_target");
if (
@@ -579,6 +575,7 @@ export async function runInitCommand(
parentDirectory,
target,
trackStep,
+ trackTerminal,
});
} catch (error) {
if (error instanceof WizardCancelledError) {
diff --git a/packages/eve/src/cli/run.ts b/packages/eve/src/cli/run.ts
index 2f64c6ab1..1fa072ccd 100644
--- a/packages/eve/src/cli/run.ts
+++ b/packages/eve/src/cli/run.ts
@@ -258,8 +258,8 @@ export function createCliProgram(
(step) => {
telemetry.trackSetupStep({ flow: "init", step });
},
- (step, result) => {
- telemetry.trackSetupTerminal({ flow: "init", step, result });
+ (step, result, failureCode) => {
+ telemetry.trackSetupTerminal({ flow: "init", step, result, failureCode });
},
);
},
diff --git a/packages/eve/src/cli/telemetry/index.test.ts b/packages/eve/src/cli/telemetry/index.test.ts
index 438fec754..8e271d177 100644
--- a/packages/eve/src/cli/telemetry/index.test.ts
+++ b/packages/eve/src/cli/telemetry/index.test.ts
@@ -190,6 +190,33 @@ describe("createEveCliTelemetry", () => {
expect(events).toContainEqual(
expect.objectContaining({ key: "setup_step", value: "model_provider" }),
);
+ expect(events).toContainEqual(
+ expect.objectContaining({ key: "setup_failure_code", value: "dependency_installation" }),
+ );
+ });
+
+ it("records an explicit bounded setup failure category", async () => {
+ vi.stubEnv("NODE_ENV", "production");
+ vi.stubEnv("EVE_TELEMETRY_DEBUG", "1");
+ const write = vi.spyOn(process.stderr, "write").mockImplementation(() => true);
+ const telemetry = createEveCliTelemetry("1.0.0");
+ telemetry.trackCommand("init");
+ telemetry.trackSetupStep({ flow: "init", step: "resolve_target" });
+ telemetry.trackSetupTerminal({
+ flow: "init",
+ step: "resolve_target",
+ result: "error",
+ failureCode: "target_resolution",
+ });
+
+ await telemetry.flush();
+
+ const events = JSON.parse(
+ String(write.mock.calls[0]?.[0]).replace("[eve telemetry] ", ""),
+ ) as Array<{ key: string; value: string }>;
+ expect(events).toContainEqual(
+ expect.objectContaining({ key: "setup_failure_code", value: "target_resolution" }),
+ );
});
it("skips telemetry when identity initialization fails", async () => {
diff --git a/packages/eve/src/cli/telemetry/index.ts b/packages/eve/src/cli/telemetry/index.ts
index 19f6a2b17..c52a3d3e5 100644
--- a/packages/eve/src/cli/telemetry/index.ts
+++ b/packages/eve/src/cli/telemetry/index.ts
@@ -35,6 +35,19 @@ export type EveCliSetupStep =
| "registry_install";
export type EveCliSetupTerminalResult = "completed" | "cancelled" | "error";
+/** A bounded, non-sensitive reason for a failed setup terminal event. */
+export type EveCliSetupFailureCode =
+ | "target_resolution"
+ | "scaffolding"
+ | "package_manager_not_found"
+ | "package_manager_start_failed"
+ | "workspace_probe_failed"
+ | "workspace_probe_unrecognized"
+ | "dependency_installation"
+ | "git_initialization"
+ | "handoff"
+ | "onboarding";
+
export type EveCliSetupStepEvent = {
flow: EveCliSetupFlow;
step: EveCliSetupStep;
@@ -45,6 +58,7 @@ export type EveCliSetupTerminalEvent = {
flow: EveCliSetupFlow;
step: EveCliSetupStep;
result: EveCliSetupTerminalResult;
+ failureCode?: EveCliSetupFailureCode;
};
export type EveCliTelemetry = {
@@ -69,6 +83,28 @@ function event(key: string, value: string): EveCliTelemetryEvent {
return { id: randomUUID(), event_time: Date.now(), key, value };
}
+function setupFailureCode(step: EveCliSetupStep): EveCliSetupFailureCode {
+ switch (step) {
+ case "resolve_target":
+ return "target_resolution";
+ case "scaffold":
+ return "scaffolding";
+ case "install_dependencies":
+ case "registry_install":
+ return "dependency_installation";
+ case "initialize_git":
+ return "git_initialization";
+ case "handoff":
+ return "handoff";
+ case "model_provider":
+ case "model_settings":
+ case "registry_channels":
+ case "registry_integrations":
+ case "registry_review":
+ return "onboarding";
+ }
+}
+
const CLI_TELEMETRY_COMMANDS = new Map<string, string>([
["acp", "acp"],
["add", "add"],
@@ -163,6 +199,11 @@ export function createEveCliTelemetry(version: string): EveCliTelemetry {
event("setup_terminal_step", input.step),
event("setup_terminal_result", input.result),
);
+ if (input.result === "error") {
+ setupEvents.push(
+ event("setup_failure_code", input.failureCode ?? setupFailureCode(input.step)),
+ );
+ }
},
trackOutcome(outcome) {
if (activeSetup !== undefined && !setupTerminalRecorded) {