Skip to content

Composio powers 1000+ toolkits, tool search, context management, authentication, and a sandboxed workbench to help you build AI agents that turn intent into action.

fix: codact failures were not logging properly (#3056)

Rahul Tarak6e7d6953c1a5
diff --git a/ts/packages/cli/src/analytics/dispatch.ts b/ts/packages/cli/src/analytics/dispatch.ts
index 11ac34955..822844b8f 100644
--- a/ts/packages/cli/src/analytics/dispatch.ts
+++ b/ts/packages/cli/src/analytics/dispatch.ts
@@ -8,12 +8,32 @@ import type { AnalyticsEnvelope, TrackEvent } from './types';
import * as constants from 'src/constants';
const INTERNAL_ANALYTICS_WORKER_FLAG = '__analytics-worker';
+const INTERNAL_CODACT_FAILURE_WORKER_FLAG = '__codact-failure-worker';
const COMPOSIO_DIR = '.composio';
const ANALYTICS_STATE_FILE_NAME = 'analytics.json';
const CONSUMER_SHORT_TERM_CACHE_FILE_NAME = 'consumer-short-term-cache.json';
const CLI_ANALYTICS_PATH = '/api/v3/cli/analytics';
+const CLI_CODACT_FAILURES_PATH = '/api/v3/cli/codact_failures';
const TELEMETRY_DEBUG_ENV_VAR = 'COMPOSIO_CLI_TELEMETRY_DEBUG';
+export type CliCodactFailureType = 'wrong_tool_slug' | 'wrong_tool_input_param';
+
+export type CliCodactFailureToolInfo = {
+ readonly toolkit?: string;
+ readonly tool?: {
+ readonly slug: string;
+ readonly version: string;
+ };
+};
+
+export type CliCodactFailure = {
+ readonly failureType: CliCodactFailureType;
+ readonly toolInfo?: CliCodactFailureToolInfo;
+ readonly ctx: Record<string, unknown>;
+ readonly session?: Record<string, unknown>;
+ readonly requestId?: string;
+};
+
const truthy = (value: string | undefined): boolean =>
value === '1' || value === 'true' || value === 'yes' || value === 'on';
@@ -165,7 +185,7 @@ const getUserApiKey = (): string | null => {
const cwdHash = (cwd: string): string => djb2Hash(cwd).toString(36);
-const getCurrentCwdSessionId = (): string | undefined => {
+export const getCurrentCwdSessionId = (): string | undefined => {
try {
const raw = fs.readFileSync(consumerShortTermCachePath(), 'utf8');
const parsed = JSON.parse(raw) as ConsumerShortTermCacheState;
@@ -202,7 +222,7 @@ const withCliSessionId = (event: TrackEvent): TrackEvent => {
};
};
-const readApiBaseUrl = (): string | null => {
+export const readApiBaseUrl = (): string | null => {
const envBaseUrl = process.env.COMPOSIO_BASE_URL?.trim();
if (envBaseUrl) {
return envBaseUrl.replace(/\/+$/u, '');
@@ -226,7 +246,12 @@ const getAnalyticsEndpoint = (): string | null => {
return baseUrl ? `${baseUrl}${CLI_ANALYTICS_PATH}` : null;
};
-const shouldDisableAnalytics = (): boolean =>
+const getCliCodactFailuresEndpoint = (): string | null => {
+ const baseUrl = readApiBaseUrl();
+ return baseUrl ? `${baseUrl}${CLI_CODACT_FAILURES_PATH}` : null;
+};
+
+export const shouldDisableAnalytics = (): boolean =>
truthy(process.env.COMPOSIO_CLI_TELEMETRY_DISABLED) ||
truthy(process.env.TELEMETRY_DISABLED) ||
truthy(process.env.COMPOSIO_DISABLE_TELEMETRY) ||
@@ -285,6 +310,70 @@ const captureToComposioAnalytics = async (envelope: AnalyticsEnvelope): Promise<
});
};
+export const createCliCodactFailureBody = (
+ failure: CliCodactFailure
+): {
+ failure_type: CliCodactFailureType;
+ tool_info?: CliCodactFailureToolInfo;
+ ctx: Record<string, unknown>;
+ session: Record<string, unknown>;
+ request_id?: string;
+} => ({
+ failure_type: failure.failureType,
+ ...(failure.toolInfo ? { tool_info: failure.toolInfo } : {}),
+ ctx: failure.ctx,
+ session: {
+ source: 'cli',
+ id: getCurrentCwdSessionId(),
+ cli_version: constants.APP_VERSION,
+ invocation_origin: process.env.COMPOSIO_CLI_INVOCATION_ORIGIN ?? 'cli',
+ parent_run_id: process.env.COMPOSIO_CLI_PARENT_RUN_ID,
+ ...(failure.session ?? {}),
+ },
+ ...(failure.requestId ? { request_id: failure.requestId } : {}),
+});
+
+const captureToComposioCodactFailures = async (failure: CliCodactFailure): Promise<void> => {
+ const endpoint = getCliCodactFailuresEndpoint();
+ if (!endpoint || shouldDisableAnalytics()) {
+ telemetryDebugLog('codact_delivery_skipped', {
+ reason: shouldDisableAnalytics() ? 'disabled' : 'missing_endpoint',
+ endpoint,
+ failureType: failure.failureType,
+ });
+ return;
+ }
+
+ const userApiKey = getUserApiKey();
+ if (!userApiKey) {
+ telemetryDebugLog('codact_delivery_skipped', {
+ reason: 'missing_user_api_key',
+ endpoint,
+ failureType: failure.failureType,
+ });
+ return;
+ }
+
+ const response = await fetch(endpoint, {
+ method: 'POST',
+ headers: {
+ 'Content-Type': 'application/json',
+ 'x-user-api-key': userApiKey,
+ },
+ body: JSON.stringify(createCliCodactFailureBody(failure)),
+ });
+ const responseBody =
+ !response.ok && isTelemetryDebugEnabled() ? await response.text() : undefined;
+
+ telemetryDebugLog(response.ok ? 'codact_delivery_succeeded' : 'codact_delivery_failed', {
+ endpoint,
+ failureType: failure.failureType,
+ status: response.status,
+ ok: response.ok,
+ responseBody: responseBody?.slice(0, 1000),
+ });
+};
+
export const trackCliEvent = (event: TrackEvent): void => {
const endpoint = getAnalyticsEndpoint();
if (!event) {
@@ -337,33 +426,130 @@ export const trackCliEvent = (event: TrackEvent): void => {
export const trackCliEventEffect = (event: TrackEvent) => Effect.sync(() => trackCliEvent(event));
-const getAnalyticsWorkerFlagIndex = (argv: ReadonlyArray<string>): number =>
- argv.findIndex(token => token === INTERNAL_ANALYTICS_WORKER_FLAG);
+export const trackCliCodactFailure = (failure: CliCodactFailure): void => {
+ const endpoint = getCliCodactFailuresEndpoint();
+ const userApiKey = getUserApiKey();
+ if (shouldDisableAnalytics() || !endpoint || !userApiKey) {
+ telemetryDebugLog('codact_skip', {
+ reason: shouldDisableAnalytics()
+ ? 'disabled'
+ : !endpoint
+ ? 'missing_endpoint'
+ : 'missing_user_api_key',
+ failureType: failure.failureType,
+ endpoint,
+ });
+ return;
+ }
+
+ try {
+ const encodedPayload = encodeBase64Url(JSON.stringify(createCliCodactFailureBody(failure)));
+ const { command, args } = getWorkerSpawnArgsForFlag(
+ INTERNAL_CODACT_FAILURE_WORKER_FLAG,
+ encodedPayload
+ );
+ const child = spawn(command, args, {
+ detached: true,
+ stdio: isTelemetryDebugEnabled() ? ['ignore', 'ignore', 'inherit'] : 'ignore',
+ env: {
+ ...process.env,
+ COMPOSIO_CLI_ANALYTICS_WORKER: '1',
+ },
+ });
+ child.unref();
+ } catch {
+ // Best-effort — codact failure tracking must never break CLI execution.
+ }
+};
+
+export const trackCliCodactFailureEffect = (failure: CliCodactFailure) =>
+ Effect.sync(() => trackCliCodactFailure(failure));
-export const isAnalyticsWorkerInvocation = (argv: ReadonlyArray<string>): boolean =>
- getAnalyticsWorkerFlagIndex(argv) >= 0;
+const getWorkerSpawnArgsForFlag = (
+ workerFlag: string,
+ encodedPayload: string
+): { command: string; args: string[] } => {
+ const maybeScriptPath = process.argv[1];
+ const scriptPathLooksReal =
+ typeof maybeScriptPath === 'string' &&
+ maybeScriptPath.length > 0 &&
+ fs.existsSync(maybeScriptPath) &&
+ /\.(?:[cm]?[jt]s|mjs|mts|cts)$/u.test(maybeScriptPath);
-export const runAnalyticsWorkerFromArgv = async (argv: ReadonlyArray<string>): Promise<void> => {
- const flagIndex = getAnalyticsWorkerFlagIndex(argv);
- const encodedPayload = flagIndex >= 0 ? argv[flagIndex + 1] : undefined;
+ return scriptPathLooksReal
+ ? {
+ command: process.execPath,
+ args: [maybeScriptPath, workerFlag, encodedPayload],
+ }
+ : {
+ command: process.execPath,
+ args: [workerFlag, encodedPayload],
+ };
+};
+
+const getWorkerFlagIndex = (argv: ReadonlyArray<string>, flag: string): number =>
+ argv.findIndex(token => token === flag);
+
+export const isBackgroundWorkerInvocation = (argv: ReadonlyArray<string>): boolean =>
+ getWorkerFlagIndex(argv, INTERNAL_ANALYTICS_WORKER_FLAG) >= 0 ||
+ getWorkerFlagIndex(argv, INTERNAL_CODACT_FAILURE_WORKER_FLAG) >= 0;
+
+export const runBackgroundWorkerFromArgv = async (argv: ReadonlyArray<string>): Promise<void> => {
+ const analyticsFlagIndex = getWorkerFlagIndex(argv, INTERNAL_ANALYTICS_WORKER_FLAG);
+ if (analyticsFlagIndex >= 0) {
+ const encodedPayload = argv[analyticsFlagIndex + 1];
+ if (!encodedPayload) {
+ return;
+ }
+
+ try {
+ const decoded = decodeBase64Url(encodedPayload);
+ const envelope = JSON.parse(decoded) as AnalyticsEnvelope;
+ if (typeof envelope?.event !== 'string' || envelope.event.length === 0) {
+ return;
+ }
+ await captureToComposioAnalytics(envelope);
+ } catch (error) {
+ telemetryDebugLog('delivery_error', {
+ error:
+ error instanceof Error
+ ? { name: error.name, message: error.message }
+ : { message: String(error) },
+ });
+ }
+ return;
+ }
+
+ const codactFlagIndex = getWorkerFlagIndex(argv, INTERNAL_CODACT_FAILURE_WORKER_FLAG);
+ if (codactFlagIndex < 0) {
+ return;
+ }
+
+ const encodedPayload = argv[codactFlagIndex + 1];
if (!encodedPayload) {
return;
}
try {
const decoded = decodeBase64Url(encodedPayload);
- const envelope = JSON.parse(decoded) as AnalyticsEnvelope;
- if (typeof envelope?.event !== 'string' || envelope.event.length === 0) {
+ const body = JSON.parse(decoded) as ReturnType<typeof createCliCodactFailureBody>;
+ if (body?.failure_type !== 'wrong_tool_slug' && body?.failure_type !== 'wrong_tool_input_param') {
return;
}
- await captureToComposioAnalytics(envelope);
+
+ await captureToComposioCodactFailures({
+ failureType: body.failure_type,
+ toolInfo: body.tool_info,
+ ctx: body.ctx,
+ session: body.session,
+ requestId: body.request_id,
+ });
} catch (error) {
- telemetryDebugLog('delivery_error', {
+ telemetryDebugLog('codact_delivery_error', {
error:
error instanceof Error
? { name: error.name, message: error.message }
: { message: String(error) },
});
- // Analytics must never break CLI execution.
}
};
diff --git a/ts/packages/cli/src/bin.ts b/ts/packages/cli/src/bin.ts
index 0517d764b..13dc24ddc 100644
--- a/ts/packages/cli/src/bin.ts
+++ b/ts/packages/cli/src/bin.ts
@@ -1,5 +1,5 @@
import process from 'node:process';
-import { isAnalyticsWorkerInvocation, runAnalyticsWorkerFromArgv } from 'src/analytics/dispatch';
+import { isBackgroundWorkerInvocation, runBackgroundWorkerFromArgv } from 'src/analytics/dispatch';
const TELEMETRY_DEBUG_FLAG = '--telemetry-debug';
const CLI_TELEMETRY_DEBUG_ENV_VAR = 'COMPOSIO_CLI_TELEMETRY_DEBUG';
@@ -16,8 +16,8 @@ const stripTelemetryDebugFlag = (argv: ReadonlyArray<string>): string[] => {
return normalizedArgv;
};
-if (isAnalyticsWorkerInvocation(process.argv)) {
- void runAnalyticsWorkerFromArgv(process.argv).finally(() => {
+if (isBackgroundWorkerInvocation(process.argv)) {
+ void runBackgroundWorkerFromArgv(process.argv).finally(() => {
process.exit(0);
});
} else {
diff --git a/ts/packages/cli/src/commands/tools/commands/tools.execute.cmd.ts b/ts/packages/cli/src/commands/tools/commands/tools.execute.cmd.ts
index 7f7c9286b..669e18868 100644
--- a/ts/packages/cli/src/commands/tools/commands/tools.execute.cmd.ts
+++ b/ts/packages/cli/src/commands/tools/commands/tools.execute.cmd.ts
@@ -22,7 +22,7 @@ import type { ToolExecuteParams, ToolExecuteResponse } from 'src/services/tools-
import { ComposioToolkitsRepository } from 'src/services/composio-clients';
import { ComposioUserContext } from 'src/services/user-context';
import { ProjectContext } from 'src/services/project-context';
-import { trackCliEventEffect } from 'src/analytics/dispatch';
+import { trackCliCodactFailureEffect, trackCliEventEffect } from 'src/analytics/dispatch';
import {
getToolExecuteFailedEvent,
getToolExecuteToolNotFoundEvent,
@@ -53,6 +53,7 @@ import {
mapComposioError,
normalizeCliError,
} from 'src/services/composio-error-overrides';
+import * as constants from 'src/constants';
const slug = Args.text({ name: 'slug' }).pipe(
Args.withDescription('Tool slug (e.g. "GITHUB_CREATE_ISSUE")')
@@ -285,6 +286,27 @@ const emitExecuteFailureTelemetry = (params: {
logId: params.logId,
})
);
+ yield* trackCliCodactFailureEffect({
+ failureType: 'wrong_tool_input_param',
+ toolInfo: {
+ ...(toolkitFromToolSlug(params.toolSlug)
+ ? { toolkit: toolkitFromToolSlug(params.toolSlug) }
+ : {}),
+ },
+ ctx: {
+ tool_slug: params.toolSlug,
+ issues: [...normalized.issues].slice(0, 50),
+ schema_path: normalized.schemaPath,
+ stage: params.stage === 'dry_run' ? 'dry_run' : 'validation',
+ },
+ session: {
+ source: 'cli',
+ command_path: params.surface === 'root' ? 'execute' : 'dev playground-execute',
+ project_mode: params.projectMode,
+ surface: params.surface,
+ cli_version: constants.APP_VERSION,
+ },
+ });
return;
}
@@ -358,6 +380,78 @@ const emitExecuteFailureTelemetry = (params: {
});
yield* trackCliEventEffect(event);
+
+ if (isNoConnectionError) {
+ return;
+ }
+
+ if (
+ isMaybeToolValidationError({
+ message,
+ errorSlug,
+ apiCode,
+ })
+ ) {
+ yield* trackCliCodactFailureEffect({
+ failureType: 'wrong_tool_input_param',
+ toolInfo: {
+ ...(toolkitFromToolSlug(params.toolSlug)
+ ? { toolkit: toolkitFromToolSlug(params.toolSlug) }
+ : {}),
+ },
+ ctx: {
+ tool_slug: params.toolSlug,
+ error_slug: errorSlug,
+ api_error_code: apiCode,
+ http_status: status,
+ error_message: message,
+ stage: params.stage === 'dry_run' ? 'dry_run' : 'execution',
+ },
+ session: {
+ source: 'cli',
+ command_path: params.surface === 'root' ? 'execute' : 'dev playground-execute',
+ project_mode: params.projectMode,
+ surface: params.surface,
+ cli_version: constants.APP_VERSION,
+ },
+ requestId: apiDetails?.request_id,
+ });
+ return;
+ }
+
+ if (
+ isMaybeToolNotFoundError({
+ message,
+ errorSlug,
+ status,
+ apiCode,
+ })
+ ) {
+ yield* trackCliCodactFailureEffect({
+ failureType: 'wrong_tool_slug',
+ toolInfo: {
+ ...(toolkitFromToolSlug(params.toolSlug)
+ ? { toolkit: toolkitFromToolSlug(params.toolSlug) }
+ : {}),
+ },
+ ctx: {
+ invalid_tool_slug: params.toolSlug,
+ error_slug: errorSlug,
+ api_error_code: apiCode,
+ http_status: status,
+ error_message: message,
+ stage: params.stage === 'validation' ? 'execution' : params.stage,
+ },
+ session: {
+ source: 'cli',
+ command_path: params.surface === 'root' ? 'execute' : 'dev playground-execute',
+ project_mode: params.projectMode,
+ surface: params.surface,
+ cli_version: constants.APP_VERSION,
+ },
+ requestId: apiDetails?.request_id,
+ });
+ }
});
export const showToolsExecuteInputHelp = (toolSlug: string) =>
diff --git a/ts/packages/cli/test/src/analytics.events.test.ts b/ts/packages/cli/test/src/analytics.events.test.ts
index eecfe16fb..a959d20c1 100644
--- a/ts/packages/cli/test/src/analytics.events.test.ts
+++ b/ts/packages/cli/test/src/analytics.events.test.ts
@@ -1,4 +1,5 @@
import { describe, expect, it } from 'vitest';
+import { createCliCodactFailureBody } from 'src/analytics/dispatch';
import {
getToolExecuteFailedEvent,
getToolExecuteToolNotFoundEvent,
@@ -93,4 +94,35 @@ describe('CLI analytics execute failure events', () => {
})
).toBe(true);
});
+
+ it('builds the CLI codact failure body using backend field names', () => {
+ const body = createCliCodactFailureBody({
+ failureType: 'wrong_tool_slug',
+ toolInfo: {
+ toolkit: 'github',
+ },
+ ctx: {
+ invalid_tool_slug: 'GITHUB_MAKE_ISSUE',
+ },
+ session: {
+ command_path: 'execute',
+ },
+ requestId: 'req_123',
+ });
+
+ expect(body).toMatchObject({
+ failure_type: 'wrong_tool_slug',
+ tool_info: {
+ toolkit: 'github',
+ },
+ ctx: {
+ invalid_tool_slug: 'GITHUB_MAKE_ISSUE',
+ },
+ session: {
+ source: 'cli',
+ command_path: 'execute',
+ },
+ request_id: 'req_123',
+ });
+ });
});