Skip to content

The open alternative to Salesforce, designed for AI.

Sample AI traces at 100% in Sentry and attribute AI calls (#24207)

Raphaël Bosi66e03060aa8d
diff --git a/packages/twenty-server/src/engine/metadata-modules/ai/ai-agent-execution/services/agent-async-executor.service.ts b/packages/twenty-server/src/engine/metadata-modules/ai/ai-agent-execution/services/agent-async-executor.service.ts
index fdfa2a65c68..fc444d54310 100644
--- a/packages/twenty-server/src/engine/metadata-modules/ai/ai-agent-execution/services/agent-async-executor.service.ts
+++ b/packages/twenty-server/src/engine/metadata-modules/ai/ai-agent-execution/services/agent-async-executor.service.ts
@@ -62,7 +62,7 @@ import {
} from 'src/engine/metadata-modules/ai/ai-billing/utils/extract-cache-creation-tokens.util';
import { mergeLanguageModelUsage } from 'src/engine/metadata-modules/ai/ai-billing/utils/merge-language-model-usage.util';
import { getCallLevelProviderOptions } from 'src/engine/metadata-modules/ai/ai-chat/utils/provider-options.util';
-import { AI_TELEMETRY_CONFIG } from 'src/engine/metadata-modules/ai/ai-models/constants/ai-telemetry.const';
+import { buildAiTelemetry } from 'src/engine/metadata-modules/ai/ai-models/utils/build-ai-telemetry.util';
import { AiModelConfigService } from 'src/engine/metadata-modules/ai/ai-models/services/ai-model-config.service';
import { AiModelRegistryService } from 'src/engine/metadata-modules/ai/ai-models/services/ai-model-registry.service';
import { NativeToolBinderService } from 'src/engine/metadata-modules/ai/ai-models/services/native-tool-binder.service';
@@ -388,7 +388,12 @@ export class AgentAsyncExecutorService {
stepCountIs(AGENT_CONFIG.MAX_STEPS)(step) ||
hasNoMoreAvailableCredits,
providerOptions,
- experimental_telemetry: AI_TELEMETRY_CONFIG,
+ experimental_telemetry: buildAiTelemetry({
+ functionId: 'agent-execution',
+ workspaceId,
+ userWorkspaceId,
+ agentId: agent?.id,
+ }),
experimental_onToolCallFinish: (event) => {
this.metricsService.recordHistogram({
key: MetricsKeys.WorkflowAgentToolExecutionDurationMs,
@@ -498,7 +503,12 @@ export class AgentAsyncExecutorService {
providerOptions: undefined,
promptCacheKey: agent?.id,
}),
- experimental_telemetry: AI_TELEMETRY_CONFIG,
+ experimental_telemetry: buildAiTelemetry({
+ functionId: 'agent-structured-output',
+ workspaceId,
+ userWorkspaceId,
+ agentId: agent?.id,
+ }),
onStepFinish: async (step) => {
const { hasNoMoreAvailableCredits: stepHasNoMoreAvailableCredits } =
await this.aiBillingService.decrementAndCheckAvailableCredits(
diff --git a/packages/twenty-server/src/engine/metadata-modules/ai/ai-agent-monitor/services/agent-turn-grader.service.ts b/packages/twenty-server/src/engine/metadata-modules/ai/ai-agent-monitor/services/agent-turn-grader.service.ts
index 60657b9d62a..d1a3e76ea48 100644
--- a/packages/twenty-server/src/engine/metadata-modules/ai/ai-agent-monitor/services/agent-turn-grader.service.ts
+++ b/packages/twenty-server/src/engine/metadata-modules/ai/ai-agent-monitor/services/agent-turn-grader.service.ts
@@ -7,7 +7,7 @@ import { NotFoundError } from 'src/engine/core-modules/graphql/utils/graphql-err
import { AgentMessageEntity } from 'src/engine/metadata-modules/ai/ai-agent-execution/entities/agent-message.entity';
import { AgentTurnEntity } from 'src/engine/metadata-modules/ai/ai-agent-execution/entities/agent-turn.entity';
import { AgentTurnEvaluationEntity } from 'src/engine/metadata-modules/ai/ai-agent-monitor/entities/agent-turn-evaluation.entity';
-import { AI_TELEMETRY_CONFIG } from 'src/engine/metadata-modules/ai/ai-models/constants/ai-telemetry.const';
+import { buildAiTelemetry } from 'src/engine/metadata-modules/ai/ai-models/utils/build-ai-telemetry.util';
import { AiModelRegistryService } from 'src/engine/metadata-modules/ai/ai-models/services/ai-model-registry.service';
import { InjectWorkspaceScopedRepository } from 'src/engine/twenty-orm/workspace-scoped-repository/inject-workspace-scoped-repository.decorator';
import { WorkspaceScopedRepository } from 'src/engine/twenty-orm/workspace-scoped-repository/workspace-scoped-repository';
@@ -85,7 +85,13 @@ Respond ONLY with valid JSON in this exact format:
model: defaultModel.model,
prompt,
temperature: 0.3,
- experimental_telemetry: AI_TELEMETRY_CONFIG,
+ experimental_telemetry: buildAiTelemetry({
+ functionId: 'agent-turn-grading',
+ workspaceId: turn.workspaceId,
+ agentId: turn.agentId,
+ threadId: turn.threadId,
+ turnId: turn.id,
+ }),
});
const parsed = JSON.parse(result.text);
diff --git a/packages/twenty-server/src/engine/metadata-modules/ai/ai-agent/utils/repair-tool-call.util.ts b/packages/twenty-server/src/engine/metadata-modules/ai/ai-agent/utils/repair-tool-call.util.ts
index 84d43a88e00..762b0b80bfe 100644
--- a/packages/twenty-server/src/engine/metadata-modules/ai/ai-agent/utils/repair-tool-call.util.ts
+++ b/packages/twenty-server/src/engine/metadata-modules/ai/ai-agent/utils/repair-tool-call.util.ts
@@ -12,7 +12,7 @@ import { type z } from 'zod';
import { UsageOperationType } from 'src/engine/core-modules/usage/enums/usage-operation-type.enum';
import { AiBillingService } from 'src/engine/metadata-modules/ai/ai-billing/services/ai-billing.service';
import { extractCacheCreationTokensFromSteps } from 'src/engine/metadata-modules/ai/ai-billing/utils/extract-cache-creation-tokens.util';
-import { AI_TELEMETRY_CONFIG } from 'src/engine/metadata-modules/ai/ai-models/constants/ai-telemetry.const';
+import { buildAiTelemetry } from 'src/engine/metadata-modules/ai/ai-models/utils/build-ai-telemetry.util';
type ToolCall = {
type: 'tool-call';
@@ -83,7 +83,11 @@ export const repairToolCall = async ({
`- Object structures must match the schema shape`,
`- Array items must follow the specified format`,
].join('\n'),
- experimental_telemetry: AI_TELEMETRY_CONFIG,
+ experimental_telemetry: buildAiTelemetry({
+ functionId: 'repair-tool-call',
+ workspaceId: billingContext?.workspaceId,
+ userWorkspaceId: billingContext?.userWorkspaceId,
+ }),
});
usage = result.usage;
diff --git a/packages/twenty-server/src/engine/metadata-modules/ai/ai-chat/services/agent-title-generation.service.ts b/packages/twenty-server/src/engine/metadata-modules/ai/ai-chat/services/agent-title-generation.service.ts
index 100efe760a8..3a6bbad6d42 100644
--- a/packages/twenty-server/src/engine/metadata-modules/ai/ai-chat/services/agent-title-generation.service.ts
+++ b/packages/twenty-server/src/engine/metadata-modules/ai/ai-chat/services/agent-title-generation.service.ts
@@ -11,7 +11,7 @@ import { BillingUsageService } from 'src/engine/core-modules/billing/services/bi
import { UsageOperationType } from 'src/engine/core-modules/usage/enums/usage-operation-type.enum';
import { AiBillingService } from 'src/engine/metadata-modules/ai/ai-billing/services/ai-billing.service';
import { extractCacheCreationTokensFromSteps } from 'src/engine/metadata-modules/ai/ai-billing/utils/extract-cache-creation-tokens.util';
-import { AI_TELEMETRY_CONFIG } from 'src/engine/metadata-modules/ai/ai-models/constants/ai-telemetry.const';
+import { buildAiTelemetry } from 'src/engine/metadata-modules/ai/ai-models/utils/build-ai-telemetry.util';
import { AiModelRegistryService } from 'src/engine/metadata-modules/ai/ai-models/services/ai-model-registry.service';
@Injectable()
@@ -46,7 +46,11 @@ export class AgentTitleGenerationService {
const result = await generateText({
model: defaultModel.model,
prompt: `Generate a concise, descriptive title (maximum 60 characters) for a chat thread based on the following message. The title should capture the main topic or purpose of the conversation. Return only the title, nothing else. Message: "${messageContent}"`,
- experimental_telemetry: AI_TELEMETRY_CONFIG,
+ experimental_telemetry: buildAiTelemetry({
+ functionId: 'agent-title-generation',
+ workspaceId,
+ userWorkspaceId,
+ }),
});
usage = result.usage;
diff --git a/packages/twenty-server/src/engine/metadata-modules/ai/ai-chat/services/chat-execution.service.ts b/packages/twenty-server/src/engine/metadata-modules/ai/ai-chat/services/chat-execution.service.ts
index 0cd785c7dcd..091df179e96 100644
--- a/packages/twenty-server/src/engine/metadata-modules/ai/ai-chat/services/chat-execution.service.ts
+++ b/packages/twenty-server/src/engine/metadata-modules/ai/ai-chat/services/chat-execution.service.ts
@@ -77,7 +77,7 @@ import {
injectCacheBreakpoint,
} from 'src/engine/metadata-modules/ai/ai-chat/utils/provider-options.util';
import { replaceUnsupportedFileParts } from 'src/engine/metadata-modules/ai/ai-chat/utils/replace-unsupported-file-parts.util';
-import { AI_TELEMETRY_CONFIG } from 'src/engine/metadata-modules/ai/ai-models/constants/ai-telemetry.const';
+import { buildAiTelemetry } from 'src/engine/metadata-modules/ai/ai-models/utils/build-ai-telemetry.util';
import { AiModelRegistryService } from 'src/engine/metadata-modules/ai/ai-models/services/ai-model-registry.service';
import { NativeToolBinderService } from 'src/engine/metadata-modules/ai/ai-models/services/native-tool-binder.service';
import { type AiModelConfig } from 'src/engine/metadata-modules/ai/ai-models/types/ai-model-config.type';
@@ -470,16 +470,14 @@ export class ChatExecutionService {
hasToolCall(ASK_QUESTIONS_TOOL_NAME)(step) ||
hasToolCall(COMPLETE_WORKSPACE_SETUP_TOOL_NAME)(step) ||
hasNoMoreAvailableCredits,
- experimental_telemetry: {
- ...AI_TELEMETRY_CONFIG,
+ experimental_telemetry: buildAiTelemetry({
functionId: 'ai-chat-stream',
- metadata: {
- streamId: streamId ?? '',
- turnId: turnId ?? '',
- threadId: threadId ?? '',
- workspaceId: workspace.id,
- },
- },
+ workspaceId: workspace.id,
+ userWorkspaceId,
+ threadId,
+ turnId,
+ streamId,
+ }),
providerOptions: getCallLevelProviderOptions({
sdkPackage: registeredModel.sdkPackage,
providerOptions: undefined,
diff --git a/packages/twenty-server/src/engine/metadata-modules/ai/ai-generate-text/controllers/ai-generate-text.controller.ts b/packages/twenty-server/src/engine/metadata-modules/ai/ai-generate-text/controllers/ai-generate-text.controller.ts
index 27fe4e0100d..5874a6f3511 100644
--- a/packages/twenty-server/src/engine/metadata-modules/ai/ai-generate-text/controllers/ai-generate-text.controller.ts
+++ b/packages/twenty-server/src/engine/metadata-modules/ai/ai-generate-text/controllers/ai-generate-text.controller.ts
@@ -21,6 +21,7 @@ import { AiBillingService } from 'src/engine/metadata-modules/ai/ai-billing/serv
import { AiRestApiExceptionFilter } from 'src/engine/metadata-modules/ai/filters/ai-api-exception.filter';
import { GenerateTextInput } from 'src/engine/metadata-modules/ai/ai-generate-text/dtos/generate-text.input';
import { AiModelRegistryService } from 'src/engine/metadata-modules/ai/ai-models/services/ai-model-registry.service';
+import { buildAiTelemetry } from 'src/engine/metadata-modules/ai/ai-models/utils/build-ai-telemetry.util';
import { PermissionsRestApiExceptionFilter } from 'src/engine/metadata-modules/permissions/utils/permissions-rest-api-exception.filter';
@Controller(`${ApiPath.Rest}/ai`)
@@ -72,6 +73,11 @@ export class AiGenerateTextController {
model: registeredModel.model,
system: body.systemPrompt,
prompt: body.userPrompt,
+ experimental_telemetry: buildAiTelemetry({
+ functionId: 'ai-generate-text',
+ workspaceId: workspace.id,
+ userWorkspaceId,
+ }),
});
return {
diff --git a/packages/twenty-server/src/engine/metadata-modules/ai/ai-models/utils/__tests__/build-ai-telemetry.util.spec.ts b/packages/twenty-server/src/engine/metadata-modules/ai/ai-models/utils/__tests__/build-ai-telemetry.util.spec.ts
new file mode 100644
index 00000000000..2c18688a66d
--- /dev/null
+++ b/packages/twenty-server/src/engine/metadata-modules/ai/ai-models/utils/__tests__/build-ai-telemetry.util.spec.ts
@@ -0,0 +1,44 @@
+import { AI_TELEMETRY_CONFIG } from 'src/engine/metadata-modules/ai/ai-models/constants/ai-telemetry.const';
+import { buildAiTelemetry } from 'src/engine/metadata-modules/ai/ai-models/utils/build-ai-telemetry.util';
+
+describe('buildAiTelemetry', () => {
+ it('should spread the shared telemetry config and set the functionId', () => {
+ const telemetry = buildAiTelemetry({ functionId: 'ai-chat-stream' });
+
+ expect(telemetry).toMatchObject(AI_TELEMETRY_CONFIG);
+ expect(telemetry.functionId).toBe('ai-chat-stream');
+ });
+
+ it('should include every provided identifier in metadata', () => {
+ const telemetry = buildAiTelemetry({
+ functionId: 'agent-execution',
+ workspaceId: 'workspace-id',
+ userWorkspaceId: 'user-workspace-id',
+ agentId: 'agent-id',
+ threadId: 'thread-id',
+ turnId: 'turn-id',
+ streamId: 'stream-id',
+ });
+
+ expect(telemetry.metadata).toEqual({
+ workspaceId: 'workspace-id',
+ userWorkspaceId: 'user-workspace-id',
+ agentId: 'agent-id',
+ threadId: 'thread-id',
+ turnId: 'turn-id',
+ streamId: 'stream-id',
+ });
+ });
+
+ it('should omit undefined, null, and empty identifiers', () => {
+ const telemetry = buildAiTelemetry({
+ functionId: 'agent-title-generation',
+ workspaceId: 'workspace-id',
+ userWorkspaceId: null,
+ agentId: undefined,
+ threadId: '',
+ });
+
+ expect(telemetry.metadata).toEqual({ workspaceId: 'workspace-id' });
+ });
+});
diff --git a/packages/twenty-server/src/engine/metadata-modules/ai/ai-models/utils/build-ai-telemetry.util.ts b/packages/twenty-server/src/engine/metadata-modules/ai/ai-models/utils/build-ai-telemetry.util.ts
new file mode 100644
index 00000000000..b00ed3584d8
--- /dev/null
+++ b/packages/twenty-server/src/engine/metadata-modules/ai/ai-models/utils/build-ai-telemetry.util.ts
@@ -0,0 +1,35 @@
+import { isNonEmptyString } from '@sniptt/guards';
+import { type TelemetrySettings } from 'ai';
+
+import { AI_TELEMETRY_CONFIG } from 'src/engine/metadata-modules/ai/ai-models/constants/ai-telemetry.const';
+
+type BuildAiTelemetryArgs = {
+ functionId: string;
+ workspaceId?: string;
+ userWorkspaceId?: string | null;
+ agentId?: string | null;
+ threadId?: string;
+ turnId?: string;
+ streamId?: string;
+};
+
+export const buildAiTelemetry = ({
+ functionId,
+ workspaceId,
+ userWorkspaceId,
+ agentId,
+ threadId,
+ turnId,
+ streamId,
+}: BuildAiTelemetryArgs): TelemetrySettings => ({
+ ...AI_TELEMETRY_CONFIG,
+ functionId,
+ metadata: {
+ ...(isNonEmptyString(workspaceId) && { workspaceId }),
+ ...(isNonEmptyString(userWorkspaceId) && { userWorkspaceId }),
+ ...(isNonEmptyString(agentId) && { agentId }),
+ ...(isNonEmptyString(threadId) && { threadId }),
+ ...(isNonEmptyString(turnId) && { turnId }),
+ ...(isNonEmptyString(streamId) && { streamId }),
+ },
+});
diff --git a/packages/twenty-server/src/instrument.ts b/packages/twenty-server/src/instrument.ts
index 7d9e0db1521..02cad6c022c 100644
--- a/packages/twenty-server/src/instrument.ts
+++ b/packages/twenty-server/src/instrument.ts
@@ -83,10 +83,13 @@ if (process.env.EXCEPTION_HANDLER_DRIVER === ExceptionHandlerDriver.SENTRY) {
nodeProfilingIntegration(),
],
tracesSampleRate,
+ tracesSampler: ({ name, inheritOrSampleWith }) =>
+ name.startsWith('ai.') ? 1 : inheritOrSampleWith(tracesSampleRate),
profilesSampleRate: parseSampleRate({
value: process.env.SENTRY_PROFILES_SAMPLE_RATE,
fallback: 0.01,
}),
+ maxValueLength: 8192,
sendDefaultPii: true,
debug: process.env.NODE_ENV === NodeEnvironment.DEVELOPMENT,
beforeSendSpan: (span) => {