Composio powers 1000+ toolkits, tool search, context management, authentication, and a sandboxed workbench to help you build AI agents that turn intent into action.
TypeScript agentic-aiagentsaiai-agentsaiagentsdeveloper-toolsfunction-callinggpt-4javascriptjsllmllmopsmcppythonremote-mcp-serverssetypescript
import { readFileSync, writeFileSync } from 'node:fs';
import path from 'node:path';
import * as tempy from 'tempy';
import { it } from '@effect/vitest';
import { afterEach, describe, expect, vi } from 'vitest';
import * as BunFileSystem from '@effect/platform-bun/BunFileSystem';
import * as BunPath from '@effect/platform-bun/BunPath';
import { Effect, Layer } from 'effect';
import { createCliCodactFailureBody } from 'src/analytics/dispatch';
import {
CLI_ANALYTICS_EVENTS,
CLI_EVENT_JOURNEY_STAGES,
CLI_JOURNEY_STAGES,
configureCliAnalyticsReleaseVersion,
createCliCommandTelemetryContext,
getPluginLifecycleFailedEvent,
getPluginLifecycleSucceededEvent,
getPrimaryLifecycleFailedEvent,
getPrimaryLifecycleInvokedEvent,
getPrimaryLifecycleSucceededEvent,
getSetupCancelledEvent,
getSetupHostDetectedEvent,
getSetupSkippedEvent,
getToolExecuteFailedEvent,
getToolExecuteToolNotFoundEvent,
getToolExecuteValidationFailedEvent,
isMaybeToolNotFoundError,
isMaybeToolValidationError,
} from 'src/analytics/events';
import { APP_VERSION } from 'src/constants';
import { inferSkillReleaseChannel } from 'src/effects/install-skill';
import { CLI_RELEASE_CHANNELS } from 'src/experimental-features';
import {
DEFAULT_CLI_INVOCATION_ORIGIN,
type CliInvocationContext,
} from 'src/services/runtime-cli-context';
import { ToolInputValidationError } from 'src/services/tool-input-validation';
import { resolveInstalledCliVersion } from 'src/services/run-companion-modules';
const CLI_INVOCATION = {
invocationOrigin: DEFAULT_CLI_INVOCATION_ORIGIN,
parentRunId: undefined,
} as const;
describe('CLI analytics execute failure events', () => {
it('records terminal capabilities supplied by the terminal service', () => {
const context = createCliCommandTelemetryContext(
['bun', 'composio'],
'0.3.0',
{ stdoutIsTTY: true, stderrIsTTY: false },
CLI_INVOCATION
);
const event = getPrimaryLifecycleInvokedEvent(context);
expect(event?.properties).toMatchObject({
stdout_is_tty: true,
stderr_is_tty: false,
});
});
it('marks cached-schema validation failures as fast_fail', () => {
const error = new ToolInputValidationError({
toolSlug: 'GMAIL_SEND_EMAIL',
schemaPath: '/tmp/schema.json',
issues: ['Unknown key "recipient"'],
});
const event = getToolExecuteValidationFailedEvent({
toolSlug: 'GMAIL_SEND_EMAIL',
args: { recipient: '[email protected]' },
error,
invocationOrigin: DEFAULT_CLI_INVOCATION_ORIGIN,
surface: 'root',
projectMode: 'consumer',
stage: 'validation',
failureOrigin: 'fast_fail',
});
expect(event).not.toBeNull();
expect(event!.properties?.failure_origin).toBe('fast_fail');
expect(event!.properties?.tool_log_id).toBeUndefined();
});
it('marks endpoint tool-not-found failures as main_endpoint and keeps log id', () => {
const event = getToolExecuteToolNotFoundEvent({
toolSlug: 'FAKE_TOOL',
args: {},
invocationOrigin: DEFAULT_CLI_INVOCATION_ORIGIN,
surface: 'root',
projectMode: 'consumer',
stage: 'execution',
failureOrigin: 'main_endpoint',
logId: 'log_123',
message: 'Tool not found',
status: 404,
});
expect(event).not.toBeNull();
expect(event!.properties?.failure_origin).toBe('main_endpoint');
expect(event!.properties?.tool_log_id).toBe('log_123');
});
it('marks endpoint execution failures as main_endpoint and keeps log id', () => {
const event = getToolExecuteFailedEvent({
toolSlug: 'GMAIL_SEND_EMAIL',
args: { to: '[email protected]' },
invocationOrigin: DEFAULT_CLI_INVOCATION_ORIGIN,
surface: 'root',
projectMode: 'consumer',
stage: 'execution',
failureOrigin: 'main_endpoint',
logId: 'log_456',
message: 'Invalid tool arguments',
});
expect(event).not.toBeNull();
expect(event!.properties?.failure_origin).toBe('main_endpoint');
expect(event!.properties?.tool_log_id).toBe('log_456');
});
it('classifies known Hermes tool-not-found codes', () => {
expect(
isMaybeToolNotFoundError({
apiCode: 2306,
message: 'random',
})
).toBe(true);
expect(
isMaybeToolNotFoundError({
apiCode: 3703,
message: 'random',
})
).toBe(true);
});
it('classifies known Hermes validation codes', () => {
expect(
isMaybeToolValidationError({
apiCode: 3702,
message: 'random',
})
).toBe(true);
expect(
isMaybeToolValidationError({
apiCode: 1149,
message: 'random',
})
).toBe(true);
});
it('builds the CLI codact failure body using backend field names', () => {
const body = createCliCodactFailureBody({
failureType: 'wrong_tool_slug',
toolInfo: {
toolkit: 'github',
},
ctx: {
invalid_tool_slug: 'GITHUB_MAKE_ISSUE',
},
session: {
command_path: 'execute',
},
requestId: 'req_123',
});
expect(body).toMatchObject({
failure_type: 'wrong_tool_slug',
tool_info: {
toolkit: 'github',
},
ctx: {
invalid_tool_slug: 'GITHUB_MAKE_ISSUE',
},
session: {
source: 'cli',
command_path: 'execute',
cli_version: expect.any(String),
},
request_id: 'req_123',
});
});
});
describe('CLI analytics setup and install lifecycle events', () => {
it('tracks setup as a dedicated lifecycle with safe option properties', () => {
const context = createCliCommandTelemetryContext(
['bun', 'composio', 'setup', '--target', 'codex', '--uninstall', '--yes'],
APP_VERSION,
{ stdoutIsTTY: false, stderrIsTTY: false },
CLI_INVOCATION
);
const invoked = getPrimaryLifecycleInvokedEvent(context);
const succeeded = getPrimaryLifecycleSucceededEvent(context);
const failed = getPrimaryLifecycleFailedEvent(context, new Error('native failure'));
expect(invoked?.name).toBe(CLI_ANALYTICS_EVENTS.CLI_SETUP_INVOKED);
expect(succeeded?.name).toBe(CLI_ANALYTICS_EVENTS.CLI_SETUP_SUCCEEDED);
expect(failed?.name).toBe(CLI_ANALYTICS_EVENTS.CLI_SETUP_FAILED);
expect(invoked?.properties).toMatchObject({
command_path: 'setup',
operation: 'uninstall',
target: 'codex',
yes: true,
if_present: false,
stdout_is_tty: expect.any(Boolean),
});
});
it('tracks shell installation separately from plugin setup', () => {
const context = createCliCommandTelemetryContext(
['bun', 'composio', 'install', '--completions'],
APP_VERSION,
{ stdoutIsTTY: false, stderrIsTTY: false },
CLI_INVOCATION
);
expect(getPrimaryLifecycleInvokedEvent(context)).toMatchObject({
name: CLI_ANALYTICS_EVENTS.CLI_INSTALL_INVOKED,
properties: {
command_path: 'install',
completions: true,
no_completions: false,
},
});
});
it('summarizes user-controlled search and proxy input without recording it', () => {
const terminal = { stdoutIsTTY: false, stderrIsTTY: false };
const search = getPrimaryLifecycleInvokedEvent(
createCliCommandTelemetryContext(
['bun', 'composio', 'search', '[email protected] secret'],
APP_VERSION,
terminal,
CLI_INVOCATION
)
);
const proxy = getPrimaryLifecycleInvokedEvent(
createCliCommandTelemetryContext(
['bun', 'composio', 'proxy', 'https://user:[email protected]/private?token=small'],
APP_VERSION,
terminal,
CLI_INVOCATION
)
);
expect(search?.properties).toMatchObject({
query_length: '[email protected] secret'.length,
query_term_count: 2,
});
expect(search?.properties).not.toHaveProperty('query');
expect(search?.properties).not.toHaveProperty('search_query');
expect(proxy?.properties).toMatchObject({ has_endpoint: true });
expect(proxy?.properties).not.toHaveProperty('endpoint');
});
it('tracks a verified per-host plugin change', () => {
expect(
getPluginLifecycleSucceededEvent({
operation: 'setup',
target: 'claude',
action: 'installed',
invocationOrigin: DEFAULT_CLI_INVOCATION_ORIGIN,
cliVersion: APP_VERSION,
})
).toMatchObject({
name: CLI_ANALYTICS_EVENTS.CLI_PLUGIN_SETUP_SUCCEEDED,
properties: {
command_path: 'setup',
operation: 'setup',
agent_host: 'claude',
action: 'installed',
},
});
});
});
describe('CLI analytics setup runtime-context events', () => {
it('tracks a detected and supported host with its version', () => {
expect(
getSetupHostDetectedEvent({
operation: 'setup',
requestedTarget: 'auto',
target: 'claude',
available: true,
supported: true,
hostVersion: '2.1.0',
invocationOrigin: DEFAULT_CLI_INVOCATION_ORIGIN,
cliVersion: APP_VERSION,
})
).toMatchObject({
name: CLI_ANALYTICS_EVENTS.CLI_SETUP_HOST_DETECTED,
properties: {
source: 'cli',
command_path: 'setup',
operation: 'setup',
requested_target: 'auto',
agent_host: 'claude',
available: true,
supported: true,
host_version: '2.1.0',
unsupported_reason_code: undefined,
},
});
});
it('tracks an unsupported host with a normalized reason code', () => {
expect(
getSetupHostDetectedEvent({
operation: 'setup',
requestedTarget: 'auto',
target: 'codex',
available: true,
supported: false,
hostVersion: 'codex-cli 0.137.0',
unsupportedReasonCode: 'codex_too_old',
invocationOrigin: DEFAULT_CLI_INVOCATION_ORIGIN,
cliVersion: APP_VERSION,
})
).toMatchObject({
name: CLI_ANALYTICS_EVENTS.CLI_SETUP_HOST_DETECTED,
properties: {
agent_host: 'codex',
available: true,
supported: false,
unsupported_reason_code: 'codex_too_old',
},
});
});
it('tracks a per-host failure without recording its free-form message', () => {
const error = new Error(`Adding the claude marketplace failed${'x'.repeat(600)}`);
const event = getPluginLifecycleFailedEvent({
operation: 'setup',
target: 'claude',
phase: 'install',
error,
invocationOrigin: DEFAULT_CLI_INVOCATION_ORIGIN,
cliVersion: APP_VERSION,
});
expect(event).toMatchObject({
name: CLI_ANALYTICS_EVENTS.CLI_PLUGIN_SETUP_FAILED,
properties: {
command_path: 'setup',
operation: 'setup',
agent_host: 'claude',
phase: 'install',
error_name: 'Error',
},
});
expect(event?.properties).not.toHaveProperty('error_message');
});
it('tracks user cancellation and installer skips with normalized reasons', () => {
expect(
getSetupCancelledEvent({
operation: 'setup',
requestedTarget: 'claude',
invocationOrigin: DEFAULT_CLI_INVOCATION_ORIGIN,
cliVersion: APP_VERSION,
})
).toMatchObject({
name: CLI_ANALYTICS_EVENTS.CLI_SETUP_CANCELLED,
properties: {
command_path: 'setup',
operation: 'setup',
requested_target: 'claude',
reason: 'user_declined',
},
});
expect(
getSetupSkippedEvent({
operation: 'uninstall',
requestedTarget: 'auto',
invocationOrigin: DEFAULT_CLI_INVOCATION_ORIGIN,
cliVersion: APP_VERSION,
})
).toMatchObject({
name: CLI_ANALYTICS_EVENTS.CLI_SETUP_SKIPPED,
properties: {
command_path: 'setup',
operation: 'uninstall',
requested_target: 'auto',
reason: 'no_host_detected',
},
});
});
});
describe('CLI analytics journey taxonomy', () => {
afterEach(() => {
configureCliAnalyticsReleaseVersion(APP_VERSION);
vi.unstubAllEnvs();
});
const contextFor = (
argv: ReadonlyArray<string>,
invocation: CliInvocationContext = CLI_INVOCATION
) =>
createCliCommandTelemetryContext(
['bun', 'composio', ...argv],
APP_VERSION,
{ stdoutIsTTY: false, stderrIsTTY: false },
invocation
);
const lifecycleCases: ReadonlyArray<[ReadonlyArray<string>, string]> = [
[['execute', 'GMAIL_SEND_EMAIL'], 'execute'],
[['search', 'send email'], 'other'],
[['link', 'github'], 'connect'],
[['login'], 'login'],
[['logout'], 'other'],
[['proxy', '/api/v3/toolkits'], 'other'],
[['run', 'echo hi'], 'other'],
[['install'], 'install'],
[['setup'], 'setup'],
[['version'], 'other'],
];
it.each(lifecycleCases)('stamps %j lifecycle events with journey_stage %s', (argv, stage) => {
const context = contextFor(argv);
expect(getPrimaryLifecycleInvokedEvent(context)?.properties?.journey_stage).toBe(stage);
expect(getPrimaryLifecycleSucceededEvent(context)?.properties?.journey_stage).toBe(stage);
expect(
getPrimaryLifecycleFailedEvent(context, new Error('boom'))?.properties?.journey_stage
).toBe(stage);
});
it('maps every analytics event name to a declared journey stage', () => {
for (const name of Object.values(CLI_ANALYTICS_EVENTS)) {
expect(CLI_JOURNEY_STAGES).toContain(CLI_EVENT_JOURNEY_STAGES[name]);
}
});
it('stamps standalone setup and tool-invocation events with their stages', () => {
expect(
getPluginLifecycleSucceededEvent({
operation: 'setup',
target: 'claude',
action: 'installed',
invocationOrigin: DEFAULT_CLI_INVOCATION_ORIGIN,
cliVersion: APP_VERSION,
})?.properties?.journey_stage
).toBe('setup');
expect(
getSetupSkippedEvent({
operation: 'setup',
requestedTarget: 'auto',
invocationOrigin: DEFAULT_CLI_INVOCATION_ORIGIN,
cliVersion: APP_VERSION,
})?.properties?.journey_stage
).toBe('setup');
expect(
getToolExecuteFailedEvent({
toolSlug: 'GMAIL_SEND_EMAIL',
args: {},
invocationOrigin: DEFAULT_CLI_INVOCATION_ORIGIN,
surface: 'root',
projectMode: 'consumer',
stage: 'execution',
failureOrigin: 'main_endpoint',
})?.properties?.journey_stage
).toBe('execute');
});
it('stamps events with the release channel of the running build', () => {
const properties = getPrimaryLifecycleInvokedEvent(contextFor(['login']))?.properties;
expect(properties?.cli_channel).toBe(inferSkillReleaseChannel(APP_VERSION));
expect(CLI_RELEASE_CHANNELS).toContain(properties?.cli_channel);
});
it.effect('uses beta release metadata even when the package version is stable', () => {
const installDir = tempy.temporaryDirectory();
const execPath = path.join(installDir, 'composio');
writeFileSync(execPath, 'fake binary');
writeFileSync(path.join(installDir, 'release-tag.txt'), '@composio/[email protected]\n');
return Effect.gen(function* () {
const resolvedVersion = yield* resolveInstalledCliVersion(execPath, APP_VERSION);
configureCliAnalyticsReleaseVersion(resolvedVersion);
const context = createCliCommandTelemetryContext(
['bun', 'composio', 'login'],
resolvedVersion,
{ stdoutIsTTY: false, stderrIsTTY: false },
CLI_INVOCATION
);
const properties = getPrimaryLifecycleInvokedEvent(context)?.properties;
const pluginProperties = getPluginLifecycleSucceededEvent({
operation: 'setup',
target: 'codex',
action: 'installed',
invocationOrigin: DEFAULT_CLI_INVOCATION_ORIGIN,
cliVersion: APP_VERSION,
})?.properties;
expect(inferSkillReleaseChannel(APP_VERSION)).toBe('stable');
expect(properties?.cli_channel).toBe('beta');
expect(pluginProperties?.cli_channel).toBe('beta');
}).pipe(Effect.provide(Layer.merge(BunFileSystem.layer, BunPath.layer)));
});
it('propagates the configured installer invocation origin', () => {
const installerInvocation = { invocationOrigin: 'installer', parentRunId: undefined };
expect(
getPrimaryLifecycleInvokedEvent(contextFor(['install'], installerInvocation))?.properties
).toMatchObject({
invocation_origin: 'installer',
journey_stage: 'install',
});
});
it('uses the configured parent run id for nested run telemetry', () => {
const context = contextFor(['run', 'console.log("hi")'], {
invocationOrigin: 'run',
parentRunId: 'run_parent',
});
expect(context.runId).toBe('run_parent');
expect(getPrimaryLifecycleInvokedEvent(context)?.properties).toMatchObject({
invocation_origin: 'run',
run_id: 'run_parent',
});
});
it('keeps the base installer install-only and marks shell-setup delegation as installer-origin', () => {
const installScript = readFileSync(
new URL('../../../../../install.sh', import.meta.url),
'utf8'
);
// Join backslash-continued lines so multi-line invocations match as one logical line.
const logicalLines = installScript.replace(/\\\n\s*/g, ' ').split('\n');
const installInvocations = logicalLines.filter(line => line.includes('"$exe" install'));
// The installer may invoke `composio install` only for the `--shell` capability probe
// and the shell-setup delegation; any other invocation would emit install analytics
// events without the installer origin attached.
const helpProbes = installInvocations.filter(line => line.includes('"$exe" install --help'));
const shellDelegations = installInvocations.filter(line =>
line.includes('"$exe" install --shell')
);
expect(helpProbes).toHaveLength(1);
expect(shellDelegations).toHaveLength(1);
expect(installInvocations).toHaveLength(2);
expect(shellDelegations[0]).toContain('COMPOSIO_CLI_INVOCATION_ORIGIN=installer');
for (const shell of ['zsh', 'bash', 'fish']) {
const variantScript = readFileSync(
new URL(`../../../../../install/${shell}.sh`, import.meta.url),
'utf8'
);
// Variants never invoke the CLI themselves: they re-exec the base installer with the
// shell pinned, so the delegation asserted above stays the only CLI install call.
expect(variantScript).not.toContain('"$exe"');
expect(variantScript).not.toContain(' install --shell');
expect(variantScript).toContain('COMPOSIO_INSTALL_SHELL="$variant_shell"');
}
});
});