Skip to content

Composio powers 1000+ toolkits, tool search, context management, authentication, and a sandboxed workbench to help you build AI agents that turn intent into action.

feat(cli): track plugin setup lifecycle and runtime setup telemetry

Kshitij Jhunjhunwalaf8ad9538f44b
diff --git a/install.sh b/install.sh
index 4ac559085..6a2488de5 100755
--- a/install.sh
+++ b/install.sh
@@ -407,7 +407,7 @@ rm -f "$install_err"
if [[ $install_plugins = true ]]; then
echo
info "Checking for supported agent hosts..."
- if ! "$exe" setup --target auto --yes --if-present; then
+ if ! COMPOSIO_CLI_INVOCATION_ORIGIN=installer "$exe" setup --target auto --yes --if-present; then
error 'Composio CLI was installed, but agent plugin setup failed. Retry with `composio setup --target auto --yes`.'
fi
fi
diff --git a/test/install-sh-release-resolution.test.sh b/test/install-sh-release-resolution.test.sh
index f492e4a5c..1fcc5fe2d 100755
--- a/test/install-sh-release-resolution.test.sh
+++ b/test/install-sh-release-resolution.test.sh
@@ -154,6 +154,10 @@ install)
exit 0
;;
setup)
+ if [[ "${COMPOSIO_CLI_INVOCATION_ORIGIN:-}" != 'installer' ]]; then
+ echo 'Expected automatic plugin setup telemetry origin to be installer.' >&2
+ exit 97
+ fi
exit "${TEST_SETUP_EXIT:-0}"
;;
--version|version)
diff --git a/ts/packages/cli/src/analytics/events.ts b/ts/packages/cli/src/analytics/events.ts
index 3dadf1d65..7bf73c077 100644
--- a/ts/packages/cli/src/analytics/events.ts
+++ b/ts/packages/cli/src/analytics/events.ts
@@ -27,6 +27,18 @@ export const CLI_ANALYTICS_EVENTS = {
CLI_RUN_INVOKED: 'CLI_RUN_INVOKED',
CLI_RUN_SUCCEEDED: 'CLI_RUN_SUCCEEDED',
CLI_RUN_FAILED: 'CLI_RUN_FAILED',
+ CLI_INSTALL_INVOKED: 'CLI_INSTALL_INVOKED',
+ CLI_INSTALL_SUCCEEDED: 'CLI_INSTALL_SUCCEEDED',
+ CLI_INSTALL_FAILED: 'CLI_INSTALL_FAILED',
+ CLI_SETUP_INVOKED: 'CLI_SETUP_INVOKED',
+ CLI_SETUP_SUCCEEDED: 'CLI_SETUP_SUCCEEDED',
+ CLI_SETUP_FAILED: 'CLI_SETUP_FAILED',
+ CLI_SETUP_HOST_DETECTED: 'CLI_SETUP_HOST_DETECTED',
+ CLI_SETUP_CANCELLED: 'CLI_SETUP_CANCELLED',
+ CLI_SETUP_SKIPPED: 'CLI_SETUP_SKIPPED',
+ CLI_PLUGIN_SETUP_SUCCEEDED: 'CLI_PLUGIN_SETUP_SUCCEEDED',
+ CLI_PLUGIN_SETUP_FAILED: 'CLI_PLUGIN_SETUP_FAILED',
+ CLI_PLUGIN_UNINSTALL_SUCCEEDED: 'CLI_PLUGIN_UNINSTALL_SUCCEEDED',
CLI_TOOL_INVOCATION_VALIDATION_FAILED: 'CLI_TOOL_INVOCATION_VALIDATION_FAILED',
CLI_TOOL_INVOCATION_TOOL_NOT_FOUND: 'CLI_TOOL_INVOCATION_TOOL_NOT_FOUND',
CLI_TOOL_INVOCATION_FAILED: 'CLI_TOOL_INVOCATION_FAILED',
@@ -40,6 +52,7 @@ const KNOWN_COMMAND_TOKENS = new Set([
'logout',
'run',
'install',
+ 'setup',
'dev',
'generate',
'tools',
@@ -286,6 +299,29 @@ const getRunCommandProperties = (context: CliCommandTelemetryContext) => ({
arg_count: Math.max(0, context.argv.length - 3),
});
+const getInstallCommandProperties = (context: CliCommandTelemetryContext) => ({
+ source: 'cli',
+ invocation_origin: getInvocationOrigin(),
+ cli_version: context.cliVersion,
+ command_path: context.commandPath,
+ duration_ms: Date.now() - context.startedAt,
+ completions: isFlagPresent(context.argv, '--completions'),
+ no_completions: isFlagPresent(context.argv, '--no-completions'),
+});
+
+const getSetupCommandProperties = (context: CliCommandTelemetryContext) => ({
+ source: 'cli',
+ invocation_origin: getInvocationOrigin(),
+ cli_version: context.cliVersion,
+ command_path: context.commandPath,
+ duration_ms: Date.now() - context.startedAt,
+ operation: isFlagPresent(context.argv, '--uninstall') ? 'uninstall' : 'setup',
+ target: getFlagValue(context.argv, '--target') ?? 'auto',
+ yes: isFlagPresent(context.argv, '--yes', '-y'),
+ if_present: isFlagPresent(context.argv, '--if-present'),
+ stdout_is_tty: context.stdoutIsTTY,
+});
+
const getLoginCommandProperties = (context: CliCommandTelemetryContext) => ({
source: 'cli',
invocation_origin: getInvocationOrigin(),
@@ -338,6 +374,10 @@ const isProxyCommand = (commandPath: string): boolean => commandPath === 'proxy'
const isRunCommand = (commandPath: string): boolean => commandPath === 'run';
+const isInstallCommand = (commandPath: string): boolean => commandPath === 'install';
+
+const isSetupCommand = (commandPath: string): boolean => commandPath === 'setup';
+
const isGenericOnlyCommand = (commandPath: string): boolean =>
commandPath === 'composio' || commandPath.startsWith('dev');
@@ -461,6 +501,20 @@ const SPECIAL_LIFECYCLE_FAMILIES: ReadonlyArray<SpecialLifecycleFamily> = [
failedEventName: CLI_ANALYTICS_EVENTS.CLI_RUN_FAILED,
getProperties: getRunCommandProperties,
},
+ {
+ match: isInstallCommand,
+ invokedEventName: CLI_ANALYTICS_EVENTS.CLI_INSTALL_INVOKED,
+ succeededEventName: CLI_ANALYTICS_EVENTS.CLI_INSTALL_SUCCEEDED,
+ failedEventName: CLI_ANALYTICS_EVENTS.CLI_INSTALL_FAILED,
+ getProperties: getInstallCommandProperties,
+ },
+ {
+ match: isSetupCommand,
+ invokedEventName: CLI_ANALYTICS_EVENTS.CLI_SETUP_INVOKED,
+ succeededEventName: CLI_ANALYTICS_EVENTS.CLI_SETUP_SUCCEEDED,
+ failedEventName: CLI_ANALYTICS_EVENTS.CLI_SETUP_FAILED,
+ getProperties: getSetupCommandProperties,
+ },
];
const getSpecialLifecycleFamily = (commandPath: string): SpecialLifecycleFamily | undefined =>
@@ -510,6 +564,109 @@ export const getPrimaryLifecycleFailedEvent = (
};
};
+export const getPluginLifecycleSucceededEvent = (params: {
+ readonly operation: 'setup' | 'uninstall';
+ readonly target: 'claude' | 'codex';
+ readonly action: 'installed' | 'enabled' | 'configured' | 'uninstalled';
+ readonly cliVersion: string;
+}): TrackEvent => ({
+ name:
+ params.operation === 'uninstall'
+ ? CLI_ANALYTICS_EVENTS.CLI_PLUGIN_UNINSTALL_SUCCEEDED
+ : CLI_ANALYTICS_EVENTS.CLI_PLUGIN_SETUP_SUCCEEDED,
+ properties: {
+ source: 'cli',
+ invocation_origin: getInvocationOrigin(),
+ cli_version: params.cliVersion,
+ command_path: 'setup',
+ operation: params.operation,
+ agent_host: params.target,
+ action: params.action,
+ },
+});
+
+export const getPluginLifecycleFailedEvent = (params: {
+ readonly operation: 'setup' | 'uninstall';
+ readonly target: 'claude' | 'codex';
+ readonly phase: 'install' | 'uninstall';
+ readonly error: unknown;
+ readonly cliVersion: string;
+}): TrackEvent => ({
+ name: CLI_ANALYTICS_EVENTS.CLI_PLUGIN_SETUP_FAILED,
+ properties: {
+ source: 'cli',
+ invocation_origin: getInvocationOrigin(),
+ cli_version: params.cliVersion,
+ command_path: 'setup',
+ operation: params.operation,
+ agent_host: params.target,
+ phase: params.phase,
+ error_name: errorNameOf(params.error),
+ error_message: errorMessageOf(params.error),
+ },
+});
+
+export const getSetupHostDetectedEvent = (params: {
+ readonly operation: 'setup' | 'uninstall';
+ readonly requestedTarget: 'auto' | 'claude' | 'codex' | 'all';
+ readonly target: 'claude' | 'codex';
+ readonly available: boolean;
+ readonly supported: boolean;
+ readonly hostVersion?: string;
+ readonly unsupportedReasonCode?:
+ 'codex_too_old' | 'no_json_inspection' | 'host_command_failed' | 'unknown';
+ readonly cliVersion: string;
+}): TrackEvent => ({
+ name: CLI_ANALYTICS_EVENTS.CLI_SETUP_HOST_DETECTED,
+ properties: {
+ source: 'cli',
+ invocation_origin: getInvocationOrigin(),
+ cli_version: params.cliVersion,
+ command_path: 'setup',
+ operation: params.operation,
+ requested_target: params.requestedTarget,
+ agent_host: params.target,
+ available: params.available,
+ supported: params.supported,
+ host_version: params.hostVersion,
+ unsupported_reason_code: params.unsupportedReasonCode,
+ },
+});
+
+export const getSetupCancelledEvent = (params: {
+ readonly operation: 'setup' | 'uninstall';
+ readonly requestedTarget: 'auto' | 'claude' | 'codex' | 'all';
+ readonly cliVersion: string;
+}): TrackEvent => ({
+ name: CLI_ANALYTICS_EVENTS.CLI_SETUP_CANCELLED,
+ properties: {
+ source: 'cli',
+ invocation_origin: getInvocationOrigin(),
+ cli_version: params.cliVersion,
+ command_path: 'setup',
+ operation: params.operation,
+ requested_target: params.requestedTarget,
+ reason: 'user_declined',
+ },
+});
+
+export const getSetupSkippedEvent = (params: {
+ readonly operation: 'setup' | 'uninstall';
+ readonly requestedTarget: 'auto' | 'claude' | 'codex' | 'all';
+ readonly cliVersion: string;
+}): TrackEvent => ({
+ name: CLI_ANALYTICS_EVENTS.CLI_SETUP_SKIPPED,
+ properties: {
+ source: 'cli',
+ invocation_origin: getInvocationOrigin(),
+ cli_version: params.cliVersion,
+ command_path: 'setup',
+ operation: params.operation,
+ requested_target: params.requestedTarget,
+ reason: 'no_host_detected',
+ },
+});
+
export const getToolExecuteValidationFailedEvent = (params: {
readonly toolSlug: string;
readonly args: Record<string, unknown>;
diff --git a/ts/packages/cli/src/commands/setup.cmd.ts b/ts/packages/cli/src/commands/setup.cmd.ts
index 90bbe9a14..e4e7b2bff 100644
--- a/ts/packages/cli/src/commands/setup.cmd.ts
+++ b/ts/packages/cli/src/commands/setup.cmd.ts
@@ -1,5 +1,12 @@
import { Command, Options } from '@effect/cli';
import { Effect, Predicate } from 'effect';
+import { trackCliEventEffect } from 'src/analytics/dispatch';
+import {
+ getSetupCancelledEvent,
+ getSetupHostDetectedEvent,
+ getSetupSkippedEvent,
+} from 'src/analytics/events';
+import { APP_VERSION } from 'src/constants';
import {
detectSetupTargets,
inspectSetupTargets,
@@ -64,6 +71,23 @@ const setupBaseCmd = Command.make(
yield* ui.intro(uninstall ? 'composio setup --uninstall' : 'composio setup');
const detections = yield* detectSetupTargets(target);
+ yield* Effect.forEach(detections, detection =>
+ trackCliEventEffect(
+ getSetupHostDetectedEvent({
+ operation,
+ requestedTarget: target,
+ target: detection.target,
+ available: detection.available,
+ supported: detection.supported,
+ hostVersion: detection.version,
+ unsupportedReasonCode:
+ detection.available && !detection.supported
+ ? (detection.unsupportedReasonCode ?? 'unknown')
+ : undefined,
+ cliVersion: APP_VERSION,
+ })
+ )
+ );
const detected = detections.filter(result => result.available).map(result => result.target);
const supported = detections.filter(result => result.available && result.supported);
const unsupported = detections.filter(result => result.available && !result.supported);
@@ -96,6 +120,9 @@ const setupBaseCmd = Command.make(
);
if (supported.length === 0) {
if (ifPresent) {
+ yield* trackCliEventEffect(
+ getSetupSkippedEvent({ operation, requestedTarget: target, cliVersion: APP_VERSION })
+ );
yield* ui.outro(
`No supported agent host detected; plugin ${uninstall ? 'uninstall' : 'setup'} skipped.`
);
@@ -117,6 +144,9 @@ const setupBaseCmd = Command.make(
operation
);
}
+ yield* trackCliEventEffect(
+ getSetupSkippedEvent({ operation, requestedTarget: target, cliVersion: APP_VERSION })
+ );
yield* ui.outro(
`No supported agent host detected; plugin ${uninstall ? 'uninstall' : 'setup'} skipped.`
);
@@ -161,6 +191,13 @@ const setupBaseCmd = Command.make(
{ defaultValue: false }
);
if (!confirmed) {
+ yield* trackCliEventEffect(
+ getSetupCancelledEvent({
+ operation,
+ requestedTarget: target,
+ cliVersion: APP_VERSION,
+ })
+ );
yield* ui.outro('Uninstall cancelled.');
return;
}
@@ -204,6 +241,13 @@ const setupBaseCmd = Command.make(
: `Finish Composio setup for ${formatTargets(pending.map(status => status.target))}?`;
const confirmed = yield* ui.confirm(prompt, { defaultValue: true });
if (!confirmed) {
+ yield* trackCliEventEffect(
+ getSetupCancelledEvent({
+ operation,
+ requestedTarget: target,
+ cliVersion: APP_VERSION,
+ })
+ );
yield* ui.outro('Setup cancelled.');
return;
}
diff --git a/ts/packages/cli/src/services/setup.ts b/ts/packages/cli/src/services/setup.ts
index 22725333b..73523433e 100644
--- a/ts/packages/cli/src/services/setup.ts
+++ b/ts/packages/cli/src/services/setup.ts
@@ -1,6 +1,12 @@
import { Command, Error as PlatformError } from '@effect/platform';
import { Data, Effect, Either, Option, Predicate, Schema } from 'effect';
import semver from 'semver';
+import { trackCliEventEffect } from 'src/analytics/dispatch';
+import {
+ getPluginLifecycleFailedEvent,
+ getPluginLifecycleSucceededEvent,
+} from 'src/analytics/events';
+import { APP_VERSION } from 'src/constants';
import { CommandRunner, type CommandResult } from './command-runner';
import { SetupSkillInstaller } from './setup-skill-installer';
@@ -330,7 +336,11 @@ const supportsInspection = (adapter: SetupTargetAdapter, versionOutput?: string)
if (adapter.target === 'codex') {
const version = versionOutput?.match(/\b\d+\.\d+\.\d+(?:-[0-9A-Za-z.-]+)?\b/)?.[0];
if (!version || !semver.valid(version) || semver.lt(version, MINIMUM_CODEX_SETUP_VERSION)) {
- return { supported: false, reason: unsupportedInspectionMessage(adapter) };
+ return {
+ supported: false,
+ reason: unsupportedInspectionMessage(adapter),
+ reasonCode: 'codex_too_old',
+ } as const;
}
}
@@ -350,14 +360,19 @@ const supportsInspection = (adapter: SetupTargetAdapter, versionOutput?: string)
return {
supported: false,
reason: capabilityCheckFailureMessage(adapter, command, commandResultDetail(result)),
- };
+ reasonCode: 'host_command_failed',
+ } as const;
}
const help = `${result.stdout}\n${result.stderr}`;
if (!/^\s*--json\b/m.test(help)) {
- return { supported: false, reason: unsupportedInspectionMessage(adapter) };
+ return {
+ supported: false,
+ reason: unsupportedInspectionMessage(adapter),
+ reasonCode: 'no_json_inspection',
+ } as const;
}
}
- return { supported: true };
+ return { supported: true } as const;
});
const detectAdapter = (adapter: SetupTargetAdapter) =>
@@ -375,6 +390,7 @@ const detectAdapter = (adapter: SetupTargetAdapter) =>
versionCommand,
errorMessage(cause)
),
+ unsupportedReasonCode: 'host_command_failed' as const,
}),
onSuccess: result => {
if (!result || result.exitCode === 127) {
@@ -389,13 +405,18 @@ const detectAdapter = (adapter: SetupTargetAdapter) =>
versionCommand,
commandResultDetail(result)
),
+ unsupportedReasonCode: 'host_command_failed' as const,
});
}
const version = result.stdout.trim() || result.stderr.trim();
return supportsInspection(adapter, version).pipe(
Effect.match({
- onFailure: cause => ({ supported: false, reason: errorMessage(cause) }),
+ onFailure: cause => ({
+ supported: false as const,
+ reason: errorMessage(cause),
+ reasonCode: 'host_command_failed' as const,
+ }),
onSuccess: support => support,
}),
Effect.map(support =>
@@ -404,12 +425,22 @@ const detectAdapter = (adapter: SetupTargetAdapter) =>
available: true,
supported: support.supported,
version,
- ...(support.supported ? {} : { unsupportedReason: support.reason }),
+ ...(support.supported
+ ? {}
+ : {
+ unsupportedReason: support.reason,
+ unsupportedReasonCode: support.reasonCode,
+ }),
}
: {
available: true,
supported: support.supported,
- ...(support.supported ? {} : { unsupportedReason: support.reason }),
+ ...(support.supported
+ ? {}
+ : {
+ unsupportedReason: support.reason,
+ unsupportedReasonCode: support.reasonCode,
+ }),
}
)
);
@@ -736,12 +767,16 @@ const FIXED_TARGETS: Readonly<Partial<Record<SetupTarget, ReadonlyArray<AgentHos
all: ['claude', 'codex'],
};
+export type SetupUnsupportedReasonCode =
+ 'codex_too_old' | 'no_json_inspection' | 'host_command_failed' | 'unknown';
+
export interface SetupTargetDetection {
readonly target: AgentHost;
readonly available: boolean;
readonly supported: boolean;
readonly version?: string;
readonly unsupportedReason?: string;
+ readonly unsupportedReasonCode?: SetupUnsupportedReasonCode;
}
export const detectSetupTargets = (target: SetupTarget) =>
@@ -786,9 +821,22 @@ const runSetupTargets = <E, R>(
verb: 'Setup' | 'Uninstall'
) =>
Effect.gen(function* () {
+ const operation = verb === 'Uninstall' ? 'uninstall' : 'setup';
+ const phase = verb === 'Uninstall' ? 'uninstall' : 'install';
const completed: SetupTargetResult[] = [];
for (const status of inspected) {
const result = yield* runAdapter(ADAPTERS[status.target], status).pipe(
+ Effect.tapError(error =>
+ trackCliEventEffect(
+ getPluginLifecycleFailedEvent({
+ operation,
+ target: status.target,
+ phase,
+ error,
+ cliVersion: APP_VERSION,
+ })
+ )
+ ),
Effect.mapError(error => {
if (completed.length === 0) return error;
const targets = completed.map(item => item.target).join(', ');
@@ -800,6 +848,24 @@ const runSetupTargets = <E, R>(
});
})
);
+ if (result.plugin_changed) {
+ const action =
+ operation === 'uninstall'
+ ? 'uninstalled'
+ : !status.plugin_installed
+ ? 'installed'
+ : !status.plugin_enabled
+ ? 'enabled'
+ : 'configured';
+ yield* trackCliEventEffect(
+ getPluginLifecycleSucceededEvent({
+ operation,
+ target: result.target,
+ action,
+ cliVersion: APP_VERSION,
+ })
+ );
+ }
completed.push(result);
}
return completed;
diff --git a/ts/packages/cli/test/__utils__/services/terminal-ui-test.ts b/ts/packages/cli/test/__utils__/services/terminal-ui-test.ts
index 3945988d7..9681951bc 100644
--- a/ts/packages/cli/test/__utils__/services/terminal-ui-test.ts
+++ b/ts/packages/cli/test/__utils__/services/terminal-ui-test.ts
@@ -7,69 +7,68 @@ import { TerminalUI } from 'src/services/terminal-ui';
* all terminal UI output is captured deterministically — no animations,
* no ANSI codes, no spinners.
*/
-export const TerminalUITest = Layer.succeed(
- TerminalUI,
- TerminalUI.of({
- capabilities: Effect.succeed({
- stdinIsTTY: false,
- stdoutIsTTY: false,
- stderrIsTTY: false,
- isInteractive: false,
- canDecorate: false,
- }),
- output: data => Console.log(data),
- error: data => Console.error(data),
+export const terminalUITestImpl = TerminalUI.of({
+ capabilities: Effect.succeed({
+ stdinIsTTY: false,
+ stdoutIsTTY: false,
+ stderrIsTTY: false,
+ isInteractive: false,
+ canDecorate: false,
+ }),
+ output: data => Console.log(data),
+ error: data => Console.error(data),
+
+ intro: title => Console.log(`-- ${title} --`),
+ outro: message => Console.log(`-- ${message} --`),
- intro: title => Console.log(`-- ${title} --`),
- outro: message => Console.log(`-- ${message} --`),
+ log: {
+ info: message => Console.log(message),
+ success: message => Console.log(message),
+ warn: message => Console.warn(message),
+ error: message => Console.error(message),
+ step: message => Console.log(message),
+ message: message => Console.log(message),
+ },
- log: {
- info: message => Console.log(message),
- success: message => Console.log(message),
- warn: message => Console.warn(message),
- error: message => Console.error(message),
- step: message => Console.log(message),
- message: message => Console.log(message),
- },
+ note: (message, title) => Console.log(title ? `[${title}] ${message}` : message),
- note: (message, title) => Console.log(title ? `[${title}] ${message}` : message),
+ select: (_message, options) => Effect.succeed(options[0].value),
- select: (_message, options) => Effect.succeed(options[0].value),
+ confirm: (_message, options) => Effect.succeed(options?.defaultValue ?? true),
- confirm: (_message, options) => Effect.succeed(options?.defaultValue ?? true),
+ withSpinner: (message, effect, options) =>
+ Effect.gen(function* () {
+ const result = yield* effect;
+ const successMsg =
+ typeof options?.successMessage === 'function'
+ ? options.successMessage(result)
+ : (options?.successMessage ?? message);
+ yield* Console.log(successMsg);
+ return result;
+ }),
- withSpinner: (message, effect, options) =>
- Effect.gen(function* () {
- const result = yield* effect;
- const successMsg =
- typeof options?.successMessage === 'function'
- ? options.successMessage(result)
- : (options?.successMessage ?? message);
- yield* Console.log(successMsg);
- return result;
- }),
+ useMakeSpinner: (message, use) =>
+ Effect.gen(function* () {
+ let stopped = false;
+ const handle = {
+ message: (_msg: string) => Effect.void,
+ stop: (msg?: string) =>
+ Effect.gen(function* () {
+ stopped = true;
+ if (msg) yield* Console.log(msg);
+ }),
+ error: (msg?: string) =>
+ Effect.gen(function* () {
+ stopped = true;
+ if (msg) yield* Console.error(msg);
+ }),
+ };
+ const exit = yield* Effect.exit(use(handle));
+ if (Exit.isFailure(exit) && !stopped) {
+ yield* Console.error(message);
+ }
+ return yield* exit;
+ }),
+});
- useMakeSpinner: (message, use) =>
- Effect.gen(function* () {
- let stopped = false;
- const handle = {
- message: (_msg: string) => Effect.void,
- stop: (msg?: string) =>
- Effect.gen(function* () {
- stopped = true;
- if (msg) yield* Console.log(msg);
- }),
- error: (msg?: string) =>
- Effect.gen(function* () {
- stopped = true;
- if (msg) yield* Console.error(msg);
- }),
- };
- const exit = yield* Effect.exit(use(handle));
- if (Exit.isFailure(exit) && !stopped) {
- yield* Console.error(message);
- }
- return yield* exit;
- }),
- })
-);
+export const TerminalUITest = Layer.succeed(TerminalUI, terminalUITestImpl);
diff --git a/ts/packages/cli/test/src/analytics.events.test.ts b/ts/packages/cli/test/src/analytics.events.test.ts
index 8c2d0c27f..5586f7b34 100644
--- a/ts/packages/cli/test/src/analytics.events.test.ts
+++ b/ts/packages/cli/test/src/analytics.events.test.ts
@@ -1,14 +1,23 @@
import { describe, expect, it } from 'vitest';
import { createCliCodactFailureBody } from 'src/analytics/dispatch';
import {
+ CLI_ANALYTICS_EVENTS,
createCliCommandTelemetryContext,
+ getPluginLifecycleFailedEvent,
+ getPluginLifecycleSucceededEvent,
+ getPrimaryLifecycleFailedEvent,
getPrimaryLifecycleInvokedEvent,
+ getPrimaryLifecycleSucceededEvent,
+ getSetupCancelledEvent,
+ getSetupHostDetectedEvent,
+ getSetupSkippedEvent,
getToolExecuteFailedEvent,
getToolExecuteToolNotFoundEvent,
getToolExecuteValidationFailedEvent,
isMaybeToolNotFoundError,
isMaybeToolValidationError,
} from 'src/analytics/events';
+import { APP_VERSION } from 'src/constants';
import { ToolInputValidationError } from 'src/services/tool-input-validation';
describe('CLI analytics execute failure events', () => {
@@ -145,3 +154,175 @@ describe('CLI analytics execute failure events', () => {
});
});
});
+
+describe('CLI analytics setup and install lifecycle events', () => {
+ it('tracks setup as a dedicated lifecycle with safe option properties', () => {
+ const context = createCliCommandTelemetryContext(
+ ['bun', 'composio', 'setup', '--target', 'codex', '--uninstall', '--yes'],
+ APP_VERSION,
+ { stdoutIsTTY: false, stderrIsTTY: false }
+ );
+
+ const invoked = getPrimaryLifecycleInvokedEvent(context);
+ const succeeded = getPrimaryLifecycleSucceededEvent(context);
+ const failed = getPrimaryLifecycleFailedEvent(context, new Error('native failure'));
+
+ expect(invoked?.name).toBe(CLI_ANALYTICS_EVENTS.CLI_SETUP_INVOKED);
+ expect(succeeded?.name).toBe(CLI_ANALYTICS_EVENTS.CLI_SETUP_SUCCEEDED);
+ expect(failed?.name).toBe(CLI_ANALYTICS_EVENTS.CLI_SETUP_FAILED);
+ expect(invoked?.properties).toMatchObject({
+ command_path: 'setup',
+ operation: 'uninstall',
+ target: 'codex',
+ yes: true,
+ if_present: false,
+ stdout_is_tty: expect.any(Boolean),
+ });
+ });
+
+ it('tracks shell installation separately from plugin setup', () => {
+ const context = createCliCommandTelemetryContext(
+ ['bun', 'composio', 'install', '--completions'],
+ APP_VERSION,
+ { stdoutIsTTY: false, stderrIsTTY: false }
+ );
+
+ expect(getPrimaryLifecycleInvokedEvent(context)).toMatchObject({
+ name: CLI_ANALYTICS_EVENTS.CLI_INSTALL_INVOKED,
+ properties: {
+ command_path: 'install',
+ completions: true,
+ no_completions: false,
+ },
+ });
+ });
+
+ it('tracks a verified per-host plugin change', () => {
+ expect(
+ getPluginLifecycleSucceededEvent({
+ operation: 'setup',
+ target: 'claude',
+ action: 'installed',
+ cliVersion: APP_VERSION,
+ })
+ ).toMatchObject({
+ name: CLI_ANALYTICS_EVENTS.CLI_PLUGIN_SETUP_SUCCEEDED,
+ properties: {
+ command_path: 'setup',
+ operation: 'setup',
+ agent_host: 'claude',
+ action: 'installed',
+ },
+ });
+ });
+});
+
+describe('CLI analytics setup runtime-context events', () => {
+ it('tracks a detected and supported host with its version', () => {
+ expect(
+ getSetupHostDetectedEvent({
+ operation: 'setup',
+ requestedTarget: 'auto',
+ target: 'claude',
+ available: true,
+ supported: true,
+ hostVersion: '2.1.0',
+ cliVersion: APP_VERSION,
+ })
+ ).toMatchObject({
+ name: CLI_ANALYTICS_EVENTS.CLI_SETUP_HOST_DETECTED,
+ properties: {
+ source: 'cli',
+ command_path: 'setup',
+ operation: 'setup',
+ requested_target: 'auto',
+ agent_host: 'claude',
+ available: true,
+ supported: true,
+ host_version: '2.1.0',
+ unsupported_reason_code: undefined,
+ },
+ });
+ });
+
+ it('tracks an unsupported host with a normalized reason code', () => {
+ expect(
+ getSetupHostDetectedEvent({
+ operation: 'setup',
+ requestedTarget: 'auto',
+ target: 'codex',
+ available: true,
+ supported: false,
+ hostVersion: 'codex-cli 0.137.0',
+ unsupportedReasonCode: 'codex_too_old',
+ cliVersion: APP_VERSION,
+ })
+ ).toMatchObject({
+ name: CLI_ANALYTICS_EVENTS.CLI_SETUP_HOST_DETECTED,
+ properties: {
+ agent_host: 'codex',
+ available: true,
+ supported: false,
+ unsupported_reason_code: 'codex_too_old',
+ },
+ });
+ });
+
+ it('tracks a per-host failure with truncated error details', () => {
+ const error = new Error(`Adding the claude marketplace failed${'x'.repeat(600)}`);
+
+ const event = getPluginLifecycleFailedEvent({
+ operation: 'setup',
+ target: 'claude',
+ phase: 'install',
+ error,
+ cliVersion: APP_VERSION,
+ });
+
+ expect(event).toMatchObject({
+ name: CLI_ANALYTICS_EVENTS.CLI_PLUGIN_SETUP_FAILED,
+ properties: {
+ command_path: 'setup',
+ operation: 'setup',
+ agent_host: 'claude',
+ phase: 'install',
+ error_name: 'Error',
+ },
+ });
+ expect(String(event?.properties?.error_message).length).toBeLessThanOrEqual(500);
+ });
+
+ it('tracks user cancellation and installer skips with normalized reasons', () => {
+ expect(
+ getSetupCancelledEvent({
+ operation: 'setup',
+ requestedTarget: 'claude',
+ cliVersion: APP_VERSION,
+ })
+ ).toMatchObject({
+ name: CLI_ANALYTICS_EVENTS.CLI_SETUP_CANCELLED,
+ properties: {
+ command_path: 'setup',
+ operation: 'setup',
+ requested_target: 'claude',
+ reason: 'user_declined',
+ },
+ });
+
+ expect(
+ getSetupSkippedEvent({
+ operation: 'uninstall',
+ requestedTarget: 'auto',
+ cliVersion: APP_VERSION,
+ })
+ ).toMatchObject({
+ name: CLI_ANALYTICS_EVENTS.CLI_SETUP_SKIPPED,
+ properties: {
+ command_path: 'setup',
+ operation: 'uninstall',
+ requested_target: 'auto',
+ reason: 'no_host_detected',
+ },
+ });
+ });
+});
diff --git a/ts/packages/cli/test/src/commands/setup.telemetry.cmd.test.ts b/ts/packages/cli/test/src/commands/setup.telemetry.cmd.test.ts
new file mode 100644
index 000000000..cbc486bc0
--- /dev/null
+++ b/ts/packages/cli/test/src/commands/setup.telemetry.cmd.test.ts
@@ -0,0 +1,373 @@
+import { Command, CommandExecutor } from '@effect/platform';
+import { describe, expect, layer } from '@effect/vitest';
+import { Effect, Exit } from 'effect';
+import { afterEach, beforeEach, vi } from 'vitest';
+import { CommandRunner } from 'src/services/command-runner';
+import { SetupSkillInstaller } from 'src/services/setup-skill-installer';
+import { TerminalUI } from 'src/services/terminal-ui';
+import { cli, TestLive } from 'test/__utils__';
+import { terminalUITestImpl } from 'test/__utils__/services/terminal-ui-test';
+
+const tracked = vi.hoisted(() => ({
+ events: [] as Array<{ readonly name: string; readonly properties?: Record<string, unknown> }>,
+}));
+
+vi.mock('src/analytics/dispatch', async importOriginal => {
+ const actual = await importOriginal<typeof import('src/analytics/dispatch')>();
+ const { Effect } = await import('effect');
+ return {
+ ...actual,
+ trackCliEventEffect: (
+ event: { readonly name: string; readonly properties?: Record<string, unknown> } | null
+ ) =>
+ Effect.sync(() => {
+ if (event) tracked.events.push(event);
+ }),
+ };
+});
+
+const eventsNamed = (name: string) => tracked.events.filter(event => event.name === name);
+
+type AgentHost = 'claude' | 'codex';
+
+interface FakeHostState {
+ available: boolean;
+ marketplace: boolean;
+ plugin: boolean;
+}
+
+const makeFakeHosts = (
+ initial: Partial<Record<AgentHost, Partial<FakeHostState>>>,
+ options: {
+ readonly failOn?: string;
+ readonly codexVersion?: string;
+ } = {}
+) => {
+ const state: Record<AgentHost, FakeHostState> = {
+ claude: { available: false, marketplace: false, plugin: false, ...initial.claude },
+ codex: { available: false, marketplace: false, plugin: false, ...initial.codex },
+ };
+
+ const marketplaceOutput = (host: AgentHost): string => {
+ if (!state[host].marketplace) return host === 'claude' ? '[]' : '{"marketplaces":[]}';
+ if (host === 'claude') {
+ return JSON.stringify([
+ {
+ name: 'composio',
+ source: {
+ source: 'github',
+ repo: 'https://github.com/ComposioHQ/composio-plugin-cc.git',
+ },
+ },
+ ]);
+ }
+ return JSON.stringify({
+ marketplaces: [
+ {
+ name: 'composio',
+ marketplaceSource: { sourceType: 'git', source: 'ComposioHQ/composio-plugin-openai' },
+ },
+ ],
+ });
+ };
+
+ const pluginOutput = (host: AgentHost): string => {
+ if (host === 'claude') {
+ if (!state.claude.plugin) return '[]';
+ return JSON.stringify([{ id: 'composio@composio', scope: 'user', enabled: true }]);
+ }
+ if (!state.codex.plugin) return '{"installed":[],"available":[]}';
+ return JSON.stringify({
+ installed: [{ pluginId: 'composio@composio', installed: true, enabled: true }],
+ available: [],
+ });
+ };
+
+ const runner = new CommandRunner({
+ run: () => Effect.succeed(CommandExecutor.ExitCode(0)),
+ capture: rawCommand => {
+ const flattened = Command.flatten(rawCommand)[0];
+ const parts = [flattened.command, ...flattened.args];
+ const host = parts[0] as AgentHost;
+ const command = parts.join(' ');
+ const respond = (result: { exitCode?: number; stdout?: string; stderr?: string }) =>
+ Effect.succeed({
+ exitCode: result.exitCode ?? 0,
+ stdout: result.stdout ?? '',
+ stderr: result.stderr ?? '',
+ });
+
+ if (parts[1] === '--version') {
+ if (!state[host].available) return respond({ exitCode: 127, stderr: 'not found' });
+ return respond({
+ stdout: host === 'claude' ? '2.1.0' : (options.codexVersion ?? 'codex-cli 0.144.1'),
+ });
+ }
+ if (command.endsWith('--help')) {
+ return respond({ stdout: `Usage: ${host} plugin\n\n --json` });
+ }
+ if (options.failOn && command.includes(options.failOn)) {
+ return respond({ exitCode: 2, stderr: 'native operation failed' });
+ }
+ if (command === `${host} plugin marketplace list --json`) {
+ return respond({ stdout: marketplaceOutput(host) });
+ }
+ if (command === `${host} plugin list --json`) {
+ return respond({ stdout: pluginOutput(host) });
+ }
+ if (command.includes('plugin marketplace add')) {
+ state[host].marketplace = true;
+ return respond({});
+ }
+ if (
+ command === 'claude plugin uninstall composio@composio --scope user --yes' ||
+ command === 'codex plugin remove composio@composio --json'
+ ) {
+ state[host].plugin = false;
+ return respond({});
+ }
+ if (
+ command.includes('plugin install') ||
+ command.includes('plugin enable') ||
+ command === 'codex plugin add composio@composio --json'
+ ) {
+ state[host].plugin = true;
+ return respond({});
+ }
+ return respond({});
+ },
+ });
+
+ return { runner, state };
+};
+
+const makeSkillInstaller = (initiallyReady = false) => {
+ let ready = initiallyReady;
+ return new SetupSkillInstaller({
+ isClaudeSkillReady: Effect.sync(() => ready),
+ hasManagedClaudeSkill: Effect.sync(() => ready),
+ ensureClaudeSkill: Effect.sync(() => {
+ const changed = !ready;
+ ready = true;
+ return changed;
+ }),
+ removeClaudeSkill: Effect.sync(() => {
+ const changed = ready;
+ ready = false;
+ return changed;
+ }),
+ });
+};
+
+// The cancellation path requires an interactive terminal to reach the confirm prompt.
+const decliningUI = TerminalUI.of({
+ ...terminalUITestImpl,
+ capabilities: Effect.succeed({
+ stdinIsTTY: false,
+ stdoutIsTTY: false,
+ stderrIsTTY: false,
+ isInteractive: true,
+ canDecorate: false,
+ }),
+ confirm: () => Effect.succeed(false),
+});
+
+describe('CLI: composio setup telemetry', () => {
+ beforeEach(() => {
+ tracked.events.length = 0;
+ });
+
+ afterEach(() => {
+ process.exitCode = undefined;
+ });
+
+ const freshClaude = makeFakeHosts({ claude: { available: true } });
+ layer(
+ TestLive({
+ commandRunner: freshClaude.runner,
+ setupSkillInstaller: makeSkillInstaller(),
+ })
+ )('fresh Claude install', it => {
+ it.scoped('tracks per-host detection and a verified plugin install', () =>
+ Effect.gen(function* () {
+ yield* cli(['setup', '--target', 'auto', '--yes']);
+
+ const detectedEvents = eventsNamed('CLI_SETUP_HOST_DETECTED');
+ expect(detectedEvents).toHaveLength(2);
+ expect(detectedEvents.map(event => event.properties)).toEqual(
+ expect.arrayContaining([
+ expect.objectContaining({
+ agent_host: 'claude',
+ available: true,
+ supported: true,
+ host_version: '2.1.0',
+ unsupported_reason_code: undefined,
+ operation: 'setup',
+ requested_target: 'auto',
+ source: 'cli',
+ }),
+ expect.objectContaining({
+ agent_host: 'codex',
+ available: false,
+ supported: false,
+ unsupported_reason_code: undefined,
+ }),
+ ])
+ );
+
+ expect(eventsNamed('CLI_PLUGIN_SETUP_SUCCEEDED')).toEqual([
+ expect.objectContaining({
+ properties: expect.objectContaining({
+ agent_host: 'claude',
+ operation: 'setup',
+ action: 'installed',
+ }),
+ }),
+ ]);
+ expect(eventsNamed('CLI_PLUGIN_SETUP_FAILED')).toHaveLength(0);
+ })
+ );
+ });
+
+ const legacyCodex = makeFakeHosts(
+ { codex: { available: true } },
+ { codexVersion: 'codex-cli 0.137.0' }
+ );
+ layer(TestLive({ commandRunner: legacyCodex.runner }))('unsupported Codex only', it => {
+ it.scoped('tracks the normalized reason code and the installer skip', () =>
+ Effect.gen(function* () {
+ const exit = yield* Effect.exit(
+ cli(['setup', '--target', 'auto', '--yes', '--if-present'])
+ );
+ expect(Exit.isSuccess(exit)).toBe(true);
+
+ expect(eventsNamed('CLI_SETUP_HOST_DETECTED')).toEqual(
+ expect.arrayContaining([
+ expect.objectContaining({
+ properties: expect.objectContaining({
+ agent_host: 'codex',
+ available: true,
+ supported: false,
+ unsupported_reason_code: 'codex_too_old',
+ }),
+ }),
+ ])
+ );
+ expect(eventsNamed('CLI_SETUP_SKIPPED')).toEqual([
+ expect.objectContaining({
+ properties: expect.objectContaining({
+ operation: 'setup',
+ requested_target: 'auto',
+ reason: 'no_host_detected',
+ }),
+ }),
+ ]);
+ })
+ );
+ });
+
+ const noHosts = makeFakeHosts({});
+ layer(TestLive({ commandRunner: noHosts.runner }))('no detected host', it => {
+ it.scoped('tracks the installer skip when nothing is detected', () =>
+ Effect.gen(function* () {
+ const exit = yield* Effect.exit(
+ cli(['setup', '--target', 'auto', '--yes', '--if-present'])
+ );
+ expect(Exit.isSuccess(exit)).toBe(true);
+
+ expect(eventsNamed('CLI_SETUP_SKIPPED')).toEqual([
+ expect.objectContaining({
+ properties: expect.objectContaining({ reason: 'no_host_detected' }),
+ }),
+ ]);
+ expect(eventsNamed('CLI_SETUP_CANCELLED')).toHaveLength(0);
+ })
+ );
+ });
+
+ const declinedSetup = makeFakeHosts({ claude: { available: true } });
+ layer(
+ TestLive({
+ commandRunner: declinedSetup.runner,
+ setupSkillInstaller: makeSkillInstaller(),
+ terminalUI: decliningUI,
+ })
+ )('declined interactive setup', it => {
+ it.scoped('tracks user cancellation without mutating the host', () =>
+ Effect.gen(function* () {
+ yield* cli(['setup', '--target', 'claude']);
+
+ expect(eventsNamed('CLI_SETUP_CANCELLED')).toEqual([
+ expect.objectContaining({
+ properties: expect.objectContaining({
+ operation: 'setup',
+ requested_target: 'claude',
+ reason: 'user_declined',
+ }),
+ }),
+ ]);
+ expect(declinedSetup.state.claude.plugin).toBe(false);
+ expect(eventsNamed('CLI_PLUGIN_SETUP_SUCCEEDED')).toHaveLength(0);
+ })
+ );
+ });
+
+ const declinedUninstall = makeFakeHosts({
+ claude: { available: true, marketplace: true, plugin: true },
+ });
+ layer(
+ TestLive({
+ commandRunner: declinedUninstall.runner,
+ setupSkillInstaller: makeSkillInstaller(true),
+ terminalUI: decliningUI,
+ })
+ )('declined interactive uninstall', it => {
+ it.scoped('tracks user cancellation for the uninstall operation', () =>
+ Effect.gen(function* () {
+ yield* cli(['setup', '--uninstall', '--target', 'claude']);
+
+ expect(eventsNamed('CLI_SETUP_CANCELLED')).toEqual([
+ expect.objectContaining({
+ properties: expect.objectContaining({
+ operation: 'uninstall',
+ requested_target: 'claude',
+ reason: 'user_declined',
+ }),
+ }),
+ ]);
+ expect(declinedUninstall.state.claude.plugin).toBe(true);
+ })
+ );
+ });
+
+ const failedInstall = makeFakeHosts(
+ { claude: { available: true } },
+ { failOn: 'plugin install' }
+ );
+ layer(
+ TestLive({
+ commandRunner: failedInstall.runner,
+ setupSkillInstaller: makeSkillInstaller(),
+ })
+ )('native install failure', it => {
+ it.scoped('tracks the per-host failure with its phase', () =>
+ Effect.gen(function* () {
+ const exit = yield* Effect.exit(cli(['setup', '--target', 'claude', '--yes']));
+ expect(Exit.isFailure(exit)).toBe(true);
+
+ expect(eventsNamed('CLI_PLUGIN_SETUP_FAILED')).toEqual([
+ expect.objectContaining({
+ properties: expect.objectContaining({
+ agent_host: 'claude',
+ operation: 'setup',
+ phase: 'install',
+ error_name: 'services/SetupProcessError',
+ error_message: expect.stringContaining('native operation failed'),
+ }),
+ }),
+ ]);
+ expect(eventsNamed('CLI_PLUGIN_SETUP_SUCCEEDED')).toHaveLength(0);
+ })
+ );
+ });
+});