Skip to content

The open alternative to Salesforce, designed for AI.

Account for every AI chat turn outcome exactly once (#25019)

Félix Malfaitc51b6c5ef758
diff --git a/packages/twenty-server/src/engine/core-modules/metrics/types/metrics-keys.type.ts b/packages/twenty-server/src/engine/core-modules/metrics/types/metrics-keys.type.ts
index d567098f00f..97a6ed35809 100644
--- a/packages/twenty-server/src/engine/core-modules/metrics/types/metrics-keys.type.ts
+++ b/packages/twenty-server/src/engine/core-modules/metrics/types/metrics-keys.type.ts
@@ -70,6 +70,7 @@ export enum MetricsKeys {
AiChatTtftMs = 'ai-chat/ttft-ms',
AiChatTurnStarted = 'ai-chat/turn-started',
AiChatTurnCompleted = 'ai-chat/turn-completed',
+ AiChatTurnCancelled = 'ai-chat/turn-cancelled',
AiChatTurnFailed = 'ai-chat/turn-failed',
WorkspaceMetadataCacheLocalEviction = 'workspace-metadata-cache/local-eviction',
WorkspaceMetadataCachePacked = 'workspace-metadata-cache/packed',
diff --git a/packages/twenty-server/src/engine/metadata-modules/ai/ai-chat/jobs/__tests__/stream-agent-chat.job.spec.ts b/packages/twenty-server/src/engine/metadata-modules/ai/ai-chat/jobs/__tests__/stream-agent-chat.job.spec.ts
index bcdac582f1b..2e4e6f35d8c 100644
--- a/packages/twenty-server/src/engine/metadata-modules/ai/ai-chat/jobs/__tests__/stream-agent-chat.job.spec.ts
+++ b/packages/twenty-server/src/engine/metadata-modules/ai/ai-chat/jobs/__tests__/stream-agent-chat.job.spec.ts
@@ -1,4 +1,5 @@
import { type UIMessageChunk } from 'ai';
+import { isDefined } from 'twenty-shared/utils';
import { type WorkspaceEntity } from 'src/engine/core-modules/workspace/workspace.entity';
import { StreamAgentChatJob } from 'src/engine/metadata-modules/ai/ai-chat/jobs/stream-agent-chat.job';
@@ -15,9 +16,26 @@ const TEXT_CHUNKS: UIMessageChunk[] = [
{ type: 'text-end', id: 'text-1' },
];
-const RESPONSE_MESSAGE = {
- role: 'assistant' as const,
- parts: [{ type: 'text' as const, text: 'Hello' }],
+type FakeResponseMessage = {
+ role: 'assistant';
+ parts: Array<Record<string, unknown> & { type: string }>;
+};
+
+const RESPONSE_MESSAGE: FakeResponseMessage = {
+ role: 'assistant',
+ parts: [{ type: 'text', text: 'Hello' }],
+};
+
+const PENDING_QUESTION_RESPONSE_MESSAGE: FakeResponseMessage = {
+ role: 'assistant',
+ parts: [
+ {
+ type: 'tool-ask_questions',
+ toolCallId: 'tool-call-id',
+ state: 'output-available',
+ output: { result: { status: 'pending' } },
+ },
+ ],
};
const createFakeChatStream = ({
@@ -25,16 +43,18 @@ const createFakeChatStream = ({
responseMessage = RESPONSE_MESSAGE,
midStreamError,
onFirstChunk,
+ isAborted = false,
}: {
chunks?: UIMessageChunk[];
- responseMessage?: typeof RESPONSE_MESSAGE;
+ responseMessage?: FakeResponseMessage;
midStreamError?: Error;
onFirstChunk?: () => void;
+ isAborted?: boolean;
} = {}) => ({
toUIMessageStream: (options: {
onError?: (error: unknown) => string;
onFinish?: (event: {
- responseMessage: typeof RESPONSE_MESSAGE;
+ responseMessage: FakeResponseMessage;
isAborted: boolean;
}) => Promise<void> | void;
}) =>
@@ -57,7 +77,7 @@ const createFakeChatStream = ({
controller.enqueue({ type: 'error', errorText });
}
- await options.onFinish?.({ responseMessage, isAborted: false });
+ await options.onFinish?.({ responseMessage, isAborted });
controller.close();
},
@@ -65,7 +85,10 @@ const createFakeChatStream = ({
});
describe('StreamAgentChatJob', () => {
- const workspace = { id: 'workspace-id' } as WorkspaceEntity;
+ const workspace = {
+ id: 'workspace-id',
+ smartModel: 'default-smart-model',
+ } as WorkspaceEntity;
const jobData: StreamAgentChatJobData = {
threadId: 'thread-id',
@@ -88,6 +111,7 @@ describe('StreamAgentChatJob', () => {
addMessageRejection,
assistantPersistRejection,
totalsUpdateAffected = 1,
+ finalPublishRejection,
}: {
workspaceFound?: boolean;
chatStream?: ReturnType<typeof createFakeChatStream>;
@@ -95,6 +119,7 @@ describe('StreamAgentChatJob', () => {
addMessageRejection?: Error;
assistantPersistRejection?: Error;
totalsUpdateAffected?: number;
+ finalPublishRejection?: Error;
} = {}) => {
const publishedEvents: PublishedEvent[] = [];
@@ -140,6 +165,13 @@ describe('StreamAgentChatJob', () => {
publish: jest
.fn()
.mockImplementation(({ event }: { event: PublishedEvent }) => {
+ if (
+ isDefined(finalPublishRejection) &&
+ event.type === 'message-persisted'
+ ) {
+ return Promise.reject(finalPublishRejection);
+ }
+
publishedEvents.push(event);
return Promise.resolve();
@@ -164,6 +196,12 @@ describe('StreamAgentChatJob', () => {
markClaimed: jest.fn().mockResolvedValue(undefined),
clear: jest.fn().mockResolvedValue(undefined),
};
+ const metricsService = { incrementCounterBy: jest.fn() };
+ const aiModelRegistryService = {
+ getEffectiveModelConfig: jest
+ .fn()
+ .mockReturnValue({ modelId: 'openai/gpt-5.6-luna' }),
+ };
const job = new StreamAgentChatJob(
threadRepository as never,
workspaceRepository as never,
@@ -173,9 +211,15 @@ describe('StreamAgentChatJob', () => {
cancelSubscriberService as never,
agentChatStreamingService as never,
streamHeartbeatService as never,
- { incrementCounterBy: jest.fn() } as never,
+ metricsService as never,
+ aiModelRegistryService as never,
);
+ const turnCounts = (key: string) =>
+ metricsService.incrementCounterBy.mock.calls
+ .map(([call]) => call)
+ .filter((call: { key: string }) => call.key === key);
+
return {
job,
publishedEvents,
@@ -184,6 +228,9 @@ describe('StreamAgentChatJob', () => {
eventPublisherService,
agentChatStreamingService,
cancelCallbacks,
+ metricsService,
+ aiModelRegistryService,
+ turnCounts,
};
};
@@ -531,4 +578,173 @@ describe('StreamAgentChatJob', () => {
).not.toHaveBeenCalled();
expect(agentChatService.notifyThreadUsageUpdated).toHaveBeenCalled();
});
+ it('labels turn-started with the resolved model, not the auto-select id', async () => {
+ const { job, aiModelRegistryService, turnCounts } = buildJob();
+
+ await job.handle({ ...jobData, modelId: 'default-fast-model' });
+
+ expect(aiModelRegistryService.getEffectiveModelConfig).toHaveBeenCalledWith(
+ 'default-fast-model',
+ );
+ expect(turnCounts('ai-chat/turn-started')).toEqual([
+ expect.objectContaining({
+ attributes: { model: 'openai/gpt-5.6-luna' },
+ }),
+ ]);
+ });
+
+ it('falls back to the workspace default model when the turn did not pick one', async () => {
+ const { job, aiModelRegistryService } = buildJob();
+
+ await job.handle(jobData);
+
+ expect(aiModelRegistryService.getEffectiveModelConfig).toHaveBeenCalledWith(
+ 'default-smart-model',
+ );
+ });
+
+ it('counts a text reply once, as an answered completion', async () => {
+ const { job, turnCounts } = buildJob();
+
+ await job.handle(jobData);
+
+ expect(turnCounts('ai-chat/turn-completed')).toEqual([
+ expect.objectContaining({
+ attributes: { model: 'openai/gpt-5.6-luna', outcome: 'answered' },
+ }),
+ ]);
+ expect(turnCounts('ai-chat/turn-failed')).toEqual([]);
+ expect(turnCounts('ai-chat/turn-cancelled')).toEqual([]);
+ });
+
+ it('counts a turn that ended on a question as completed and awaiting the user', async () => {
+ const { job, turnCounts } = buildJob({
+ chatStream: createFakeChatStream({
+ responseMessage: PENDING_QUESTION_RESPONSE_MESSAGE,
+ }),
+ });
+
+ await job.handle(jobData);
+
+ expect(turnCounts('ai-chat/turn-completed')).toEqual([
+ expect.objectContaining({
+ attributes: { model: 'openai/gpt-5.6-luna', outcome: 'awaiting_user' },
+ }),
+ ]);
+ expect(turnCounts('ai-chat/turn-failed')).toEqual([]);
+ });
+
+ it('counts an aborted turn as cancelled rather than leaving it unaccounted', async () => {
+ const { job, turnCounts } = buildJob({
+ chatStream: createFakeChatStream({
+ responseMessage: { role: 'assistant', parts: [] },
+ isAborted: true,
+ }),
+ });
+
+ await job.handle(jobData);
+
+ expect(turnCounts('ai-chat/turn-cancelled')).toEqual([
+ expect.objectContaining({
+ attributes: {
+ model: 'openai/gpt-5.6-luna',
+ reason: 'user_cancelled',
+ },
+ }),
+ ]);
+ expect(turnCounts('ai-chat/turn-completed')).toEqual([]);
+ });
+
+ it('counts a turn whose claim moved on as superseded', async () => {
+ const { job, turnCounts } = buildJob({ totalsUpdateAffected: 0 });
+
+ await job.handle(jobData);
+
+ expect(turnCounts('ai-chat/turn-cancelled')).toEqual([
+ expect.objectContaining({
+ attributes: { model: 'openai/gpt-5.6-luna', reason: 'superseded' },
+ }),
+ ]);
+ expect(turnCounts('ai-chat/turn-completed')).toEqual([]);
+ });
+
+ it('counts an empty reply as a no_text failure exactly once', async () => {
+ const { job, turnCounts } = buildJob({
+ chatStream: createFakeChatStream({
+ responseMessage: { role: 'assistant', parts: [] },
+ }),
+ });
+
+ await job.handle(jobData);
+
+ expect(turnCounts('ai-chat/turn-failed')).toEqual([
+ expect.objectContaining({
+ attributes: { model: 'openai/gpt-5.6-luna', failure_phase: 'no_text' },
+ }),
+ ]);
+ });
+
+ it('leaves a stream error to the execution counter so it is not counted twice', async () => {
+ const { job, turnCounts } = buildJob({
+ streamChatRejection: new Error('provider exploded'),
+ });
+
+ await job.handle(jobData).catch(() => {});
+
+ expect(turnCounts('ai-chat/turn-failed')).toEqual([
+ expect.objectContaining({
+ attributes: expect.objectContaining({
+ model: 'openai/gpt-5.6-luna',
+ failure_phase: 'execution',
+ }),
+ }),
+ ]);
+ expect(turnCounts('ai-chat/turn-completed')).toEqual([]);
+ });
+
+ it('records exactly one outcome for every started turn', async () => {
+ const scenarios = [
+ buildJob(),
+ buildJob({ totalsUpdateAffected: 0 }),
+ buildJob({
+ chatStream: createFakeChatStream({
+ responseMessage: PENDING_QUESTION_RESPONSE_MESSAGE,
+ }),
+ }),
+ buildJob({
+ chatStream: createFakeChatStream({
+ responseMessage: { role: 'assistant', parts: [] },
+ isAborted: true,
+ }),
+ }),
+ buildJob({ streamChatRejection: new Error('provider exploded') }),
+ ];
+
+ for (const scenario of scenarios) {
+ await scenario.job.handle(jobData).catch(() => {});
+
+ const started = scenario.turnCounts('ai-chat/turn-started').length;
+ const terminal =
+ scenario.turnCounts('ai-chat/turn-completed').length +
+ scenario.turnCounts('ai-chat/turn-cancelled').length +
+ scenario.turnCounts('ai-chat/turn-failed').length;
+
+ expect({ started, terminal }).toEqual({ started: 1, terminal: 1 });
+ }
+ });
+ it('does not count a turn twice when the final publish fails after the outcome was recorded', async () => {
+ const { job, turnCounts } = buildJob({
+ finalPublishRejection: new Error('redis is down'),
+ });
+
+ await job.handle(jobData).catch(() => {});
+
+ expect(turnCounts('ai-chat/turn-completed')).toEqual([
+ expect.objectContaining({
+ attributes: { model: 'openai/gpt-5.6-luna', outcome: 'answered' },
+ }),
+ ]);
+ expect(turnCounts('ai-chat/turn-failed')).toEqual([]);
+ expect(turnCounts('ai-chat/turn-started')).toHaveLength(1);
+ });
});
diff --git a/packages/twenty-server/src/engine/metadata-modules/ai/ai-chat/jobs/stream-agent-chat.job.ts b/packages/twenty-server/src/engine/metadata-modules/ai/ai-chat/jobs/stream-agent-chat.job.ts
index b8215f9f777..f7f5f905729 100644
--- a/packages/twenty-server/src/engine/metadata-modules/ai/ai-chat/jobs/stream-agent-chat.job.ts
+++ b/packages/twenty-server/src/engine/metadata-modules/ai/ai-chat/jobs/stream-agent-chat.job.ts
@@ -8,7 +8,7 @@ import type {
ExtendedUIMessage,
ExtendedUIMessagePart,
} from 'twenty-shared/ai';
-import { isDefined } from 'twenty-shared/utils';
+import { assertUnreachable, isDefined } from 'twenty-shared/utils';
import { Repository } from 'typeorm';
import { v5 as uuidv5 } from 'uuid';
@@ -36,11 +36,17 @@ import { AgentChatStreamHeartbeatService } from 'src/engine/metadata-modules/ai/
import { AgentChatStreamingService } from 'src/engine/metadata-modules/ai/ai-chat/services/agent-chat-streaming.service';
import { AgentChatService } from 'src/engine/metadata-modules/ai/ai-chat/services/agent-chat.service';
import { ChatExecutionService } from 'src/engine/metadata-modules/ai/ai-chat/services/chat-execution.service';
+import { type AgentChatTurnOutcome } from 'src/engine/metadata-modules/ai/ai-chat/types/agent-chat-turn-outcome.type';
+import {
+ classifyAgentChatTurnOutcome,
+ resolveSupersededTurnOutcome,
+} from 'src/engine/metadata-modules/ai/ai-chat/utils/classify-agent-chat-turn-outcome.util';
import { findPendingQuestionPart } from 'src/engine/metadata-modules/ai/ai-chat/utils/find-pending-question-part.util';
import { AGENT_CHAT_CHECKPOINT_INTERVAL_MS } from 'src/engine/metadata-modules/ai/ai-chat/constants/agent-chat-checkpoint-interval-ms.constant';
import { getCancelChannel } from 'src/engine/metadata-modules/ai/ai-chat/utils/get-cancel-channel.util';
import { mapErrorToStreamError } from 'src/engine/metadata-modules/ai/ai-chat/utils/map-error-to-stream-error.util';
import { tagAiChatStreamScope } from 'src/engine/metadata-modules/ai/ai-chat/utils/tag-ai-chat-stream-scope.util';
+import { AiModelRegistryService } from 'src/engine/metadata-modules/ai/ai-models/services/ai-model-registry.service';
import type { AiModelConfig } from 'src/engine/metadata-modules/ai/ai-models/types/ai-model-config.type';
import { InjectWorkspaceScopedRepository } from 'src/engine/twenty-orm/workspace-scoped-repository/inject-workspace-scoped-repository.decorator';
import { WorkspaceScopedRepository } from 'src/engine/twenty-orm/workspace-scoped-repository/workspace-scoped-repository';
@@ -59,6 +65,11 @@ const ASSISTANT_MESSAGE_ID_NAMESPACE = '0b9c2a3d-4e5f-4a1b-8c2d-3e4f5a6b7c8d';
export class StreamAgentChatJob {
private readonly logger = new Logger(StreamAgentChatJob.name);
+ // The processor is REQUEST-scoped, so this is per job. A publish failure
+ // after the stream already classified its outcome reaches the catch in
+ // handle(), which would otherwise count the same turn a second time.
+ private hasRecordedTurnOutcome = false;
+
constructor(
@InjectWorkspaceScopedRepository(AgentChatThreadEntity)
private readonly threadRepository: WorkspaceScopedRepository<AgentChatThreadEntity>,
@@ -71,6 +82,7 @@ export class StreamAgentChatJob {
private readonly agentChatStreamingService: AgentChatStreamingService,
private readonly streamHeartbeatService: AgentChatStreamHeartbeatService,
private readonly metricsService: MetricsService,
+ private readonly aiModelRegistryService: AiModelRegistryService,
) {}
@Process(STREAM_AGENT_CHAT_JOB_NAME)
@@ -98,10 +110,16 @@ export class StreamAgentChatJob {
return;
}
+ const workspace = await this.workspaceRepository.findOne({
+ where: { id: data.workspaceId },
+ });
+
+ const turnModelId = this.resolveTurnModelId(data.modelId, workspace);
+
this.metricsService.incrementCounterBy({
key: MetricsKeys.AiChatTurnStarted,
amount: 1,
- attributes: { model: data.modelId ?? 'unknown' },
+ attributes: { model: turnModelId },
});
await this.eventPublisherService.resetStreamState(data.threadId);
@@ -131,10 +149,6 @@ export class StreamAgentChatJob {
);
try {
- const workspace = await this.workspaceRepository.findOne({
- where: { id: data.workspaceId },
- });
-
if (!workspace) {
throw new AiException(
`Workspace ${data.workspaceId} not found`,
@@ -142,22 +156,26 @@ export class StreamAgentChatJob {
);
}
- await this.executeStream(data, workspace, abortController.signal);
+ await this.executeStream(
+ data,
+ workspace,
+ abortController.signal,
+ turnModelId,
+ );
} catch (error) {
this.logger.error(
`Stream ${data.streamId} failed: ${error instanceof Error ? error.message : String(error)}`,
);
const streamError = mapErrorToStreamError(error);
- this.metricsService.incrementCounterBy({
- key: MetricsKeys.AiChatTurnFailed,
- amount: 1,
- attributes: {
- model: data.modelId ?? 'unknown',
- failure_phase: 'execution',
- error_code: streamError.code,
+ this.recordTurnOutcome(
+ {
+ kind: 'failed',
+ failurePhase: 'execution',
+ errorCode: streamError.code,
},
- });
+ turnModelId,
+ );
await this.threadRepository
.update(
@@ -225,10 +243,79 @@ export class StreamAgentChatJob {
}
}
+ // The turn-started counter fires before the model is resolved downstream, so
+ // resolve it here too: auto-select ids such as `default-fast-model` would
+ // otherwise label the start of a turn differently from its outcome, and every
+ // per-model rate is computed across the two.
+ private resolveTurnModelId(
+ requestedModelId: string | undefined,
+ workspace: WorkspaceEntity | null,
+ ): string {
+ const modelId = requestedModelId ?? workspace?.smartModel;
+
+ if (!isNonEmptyString(modelId)) {
+ return 'unknown';
+ }
+
+ try {
+ return this.aiModelRegistryService.getEffectiveModelConfig(modelId)
+ .modelId;
+ } catch {
+ return modelId;
+ }
+ }
+
+ private recordTurnOutcome(
+ outcome: AgentChatTurnOutcome,
+ turnModelId: string,
+ ): void {
+ if (this.hasRecordedTurnOutcome) {
+ return;
+ }
+
+ this.hasRecordedTurnOutcome = true;
+
+ switch (outcome.kind) {
+ case 'completed':
+ this.metricsService.incrementCounterBy({
+ key: MetricsKeys.AiChatTurnCompleted,
+ amount: 1,
+ attributes: { model: turnModelId, outcome: outcome.outcome },
+ });
+
+ return;
+ case 'cancelled':
+ this.metricsService.incrementCounterBy({
+ key: MetricsKeys.AiChatTurnCancelled,
+ amount: 1,
+ attributes: { model: turnModelId, reason: outcome.reason },
+ });
+
+ return;
+ case 'failed':
+ this.metricsService.incrementCounterBy({
+ key: MetricsKeys.AiChatTurnFailed,
+ amount: 1,
+ attributes: {
+ model: turnModelId,
+ failure_phase: outcome.failurePhase,
+ ...(isDefined(outcome.errorCode) && {
+ error_code: outcome.errorCode,
+ }),
+ },
+ });
+
+ return;
+ default:
+ return assertUnreachable(outcome);
+ }
+ }
+
private async executeStream(
data: StreamAgentChatJobData,
workspace: WorkspaceEntity,
abortSignal: AbortSignal,
+ turnModelId: string,
): Promise<void> {
// When processing a promoted queued message, the user message already
// exists in the DB with a turn — skip persisting it again.
@@ -264,6 +351,7 @@ export class StreamAgentChatJob {
userMessagePromise,
titlePromise,
abortSignal,
+ turnModelId,
});
}
@@ -273,12 +361,14 @@ export class StreamAgentChatJob {
userMessagePromise,
titlePromise,
abortSignal,
+ turnModelId,
}: {
workspace: WorkspaceEntity;
data: StreamAgentChatJobData;
userMessagePromise: Promise<{ turnId: string | null }>;
titlePromise: Promise<string | null>;
abortSignal: AbortSignal;
+ turnModelId: string;
}): Promise<void> {
const assistantMessageId = uuidv5(
data.streamId,
@@ -432,6 +522,7 @@ export class StreamAgentChatJob {
lastStepConversationSize,
totalCacheCreationTokens,
modelConfig,
+ turnModelId,
userMessagePromise,
});
await titleWritePromise;
@@ -638,7 +729,39 @@ export class StreamAgentChatJob {
return undefined;
}
- private async handleStreamFinish({
+ private async handleStreamFinish(args: {
+ assistantMessageId: string;
+ streamId: string;
+ responseMessage: Omit<ExtendedUIMessage, 'id'>;
+ isAborted: boolean;
+ streamError: unknown;
+ outOfCredits: boolean;
+ threadId: string;
+ workspaceId: string;
+ userWorkspaceId: string;
+ streamUsage: {
+ inputTokens: number;
+ outputTokens: number;
+ inputCredits: number;
+ outputCredits: number;
+ cacheReadTokens: number;
+ };
+ lastStepConversationSize: number;
+ totalCacheCreationTokens: number;
+ modelConfig: AiModelConfig;
+ turnModelId: string;
+ userMessagePromise: Promise<{ turnId: string | null }>;
+ }): Promise<void> {
+ const outcome = await this.persistStreamFinish(args);
+
+ if (isDefined(outcome)) {
+ this.recordTurnOutcome(outcome, args.turnModelId);
+ }
+ }
+
+ // Returns null when the stream errored: that turn is accounted for by the
+ // catch in handle(), and counting it here too would double it.
+ private async persistStreamFinish({
assistantMessageId,
streamId,
responseMessage,
@@ -652,6 +775,7 @@ export class StreamAgentChatJob {
lastStepConversationSize,
totalCacheCreationTokens,
modelConfig,
+ turnModelId,
userMessagePromise,
}: {
assistantMessageId: string;
@@ -673,8 +797,9 @@ export class StreamAgentChatJob {
lastStepConversationSize: number;
totalCacheCreationTokens: number;
modelConfig: AiModelConfig;
+ turnModelId: string;
userMessagePromise: Promise<{ turnId: string | null }>;
- }): Promise<void> {
+ }): Promise<AgentChatTurnOutcome | null> {
const hasText = responseMessage.parts.some(
(part) => part.type === 'text' && isNonEmptyString(part.text),
);
@@ -691,12 +816,23 @@ export class StreamAgentChatJob {
threadId,
workspaceId,
streamUsage,
- modelId: modelConfig.modelId,
+ modelId: turnModelId,
});
}
+ if (isDefined(streamError)) {
+ return null;
+ }
+
+ const outcome = classifyAgentChatTurnOutcome({
+ hasText,
+ isAborted,
+ isAwaitingUserAnswer: isDefined(pendingQuestionPart),
+ outOfCredits,
+ });
+
if (responseMessage.parts.length === 0) {
- return;
+ return outcome;
}
const threadStatus = await this.threadRepository.findOne(workspaceId, {
@@ -705,7 +841,7 @@ export class StreamAgentChatJob {
});
if (!threadStatus || threadStatus.deletedAt) {
- return;
+ return resolveSupersededTurnOutcome(outcome);
}
const userMessage = await userMessagePromise;
@@ -753,15 +889,7 @@ export class StreamAgentChatJob {
);
if (!totalsUpdate.affected) {
- return;
- }
-
- if (hasText && !isAborted && !isDefined(pendingQuestionPart)) {
- this.metricsService.incrementCounterBy({
- key: MetricsKeys.AiChatTurnCompleted,
- amount: 1,
- attributes: { model: modelConfig.modelId },
- });
+ return resolveSupersededTurnOutcome(outcome);
}
await this.agentChatService.notifyThreadUsageUpdated({
@@ -769,6 +897,8 @@ export class StreamAgentChatJob {
userWorkspaceId,
workspaceId,
});
+
+ return outcome;
}
private logAssistantTurnWithoutText({
@@ -803,14 +933,6 @@ export class StreamAgentChatJob {
? 'credits-exhausted'
: 'empty-completion';
- if (reason === 'empty-completion') {
- this.metricsService.incrementCounterBy({
- key: MetricsKeys.AiChatTurnFailed,
- amount: 1,
- attributes: { model: modelId, failure_phase: 'no_text' },
- });
- }
-
const errorDetail =
streamError instanceof Error
? `${streamError.name}: ${streamError.message}`
@@ -820,7 +942,8 @@ export class StreamAgentChatJob {
this.logger.warn(
`[AI_CHAT_NO_TEXT] Assistant turn ended without a text reply — ` +
- `reason=${reason}, threadId=${threadId}, workspaceId=${workspaceId}, ` +
+ `reason=${reason}, model=${modelId}, ` +
+ `threadId=${threadId}, workspaceId=${workspaceId}, ` +
`isAborted=${isAborted}, outOfCredits=${outOfCredits}, hasText=${hasText}, ` +
`streamError=${errorDetail}, ` +
`inputTokens=${streamUsage.inputTokens},` +
diff --git a/packages/twenty-server/src/engine/metadata-modules/ai/ai-chat/types/agent-chat-turn-outcome.type.ts b/packages/twenty-server/src/engine/metadata-modules/ai/ai-chat/types/agent-chat-turn-outcome.type.ts
new file mode 100644
index 00000000000..169e1987853
--- /dev/null
+++ b/packages/twenty-server/src/engine/metadata-modules/ai/ai-chat/types/agent-chat-turn-outcome.type.ts
@@ -0,0 +1,10 @@
+// Every started turn resolves to exactly one of these so that
+// started = completed + cancelled + failed holds on the dashboards.
+export type AgentChatTurnOutcome =
+ | { kind: 'completed'; outcome: 'answered' | 'awaiting_user' }
+ | { kind: 'cancelled'; reason: 'user_cancelled' | 'superseded' }
+ | {
+ kind: 'failed';
+ failurePhase: 'no_text' | 'credits_exhausted' | 'execution';
+ errorCode?: string;
+ };
diff --git a/packages/twenty-server/src/engine/metadata-modules/ai/ai-chat/utils/__tests__/classify-agent-chat-turn-outcome.util.spec.ts b/packages/twenty-server/src/engine/metadata-modules/ai/ai-chat/utils/__tests__/classify-agent-chat-turn-outcome.util.spec.ts
new file mode 100644
index 00000000000..a82c7c2c400
--- /dev/null
+++ b/packages/twenty-server/src/engine/metadata-modules/ai/ai-chat/utils/__tests__/classify-agent-chat-turn-outcome.util.spec.ts
@@ -0,0 +1,113 @@
+import {
+ classifyAgentChatTurnOutcome,
+ resolveSupersededTurnOutcome,
+} from 'src/engine/metadata-modules/ai/ai-chat/utils/classify-agent-chat-turn-outcome.util';
+
+const baseTurn = {
+ hasText: true,
+ isAborted: false,
+ isAwaitingUserAnswer: false,
+ outOfCredits: false,
+};
+
+describe('classifyAgentChatTurnOutcome', () => {
+ it('counts a turn that replied with text as answered', () => {
+ expect(classifyAgentChatTurnOutcome(baseTurn)).toEqual({
+ kind: 'completed',
+ outcome: 'answered',
+ });
+ });
+
+ it('counts a turn that ended on a question as completed, not abandoned', () => {
+ expect(
+ classifyAgentChatTurnOutcome({
+ ...baseTurn,
+ hasText: false,
+ isAwaitingUserAnswer: true,
+ }),
+ ).toEqual({ kind: 'completed', outcome: 'awaiting_user' });
+ });
+
+ it('keeps a question asked mid-cancellation as awaiting the user', () => {
+ expect(
+ classifyAgentChatTurnOutcome({
+ ...baseTurn,
+ isAborted: true,
+ isAwaitingUserAnswer: true,
+ }),
+ ).toEqual({ kind: 'completed', outcome: 'awaiting_user' });
+ });
+
+ it('counts a cancelled turn as cancelled rather than failed', () => {
+ expect(
+ classifyAgentChatTurnOutcome({
+ ...baseTurn,
+ hasText: false,
+ isAborted: true,
+ }),
+ ).toEqual({ kind: 'cancelled', reason: 'user_cancelled' });
+ });
+
+ it('counts an empty reply as a no_text failure', () => {
+ expect(
+ classifyAgentChatTurnOutcome({ ...baseTurn, hasText: false }),
+ ).toEqual({ kind: 'failed', failurePhase: 'no_text' });
+ });
+
+ it('separates running out of credits from an empty reply', () => {
+ expect(
+ classifyAgentChatTurnOutcome({
+ ...baseTurn,
+ hasText: false,
+ outOfCredits: true,
+ }),
+ ).toEqual({ kind: 'failed', failurePhase: 'credits_exhausted' });
+ });
+
+ it('still counts a turn that produced text before credits ran out as answered', () => {
+ expect(
+ classifyAgentChatTurnOutcome({ ...baseTurn, outOfCredits: true }),
+ ).toEqual({ kind: 'completed', outcome: 'answered' });
+ });
+});
+
+describe('resolveSupersededTurnOutcome', () => {
+ it('keeps a credits failure rather than filing it as a cancellation', () => {
+ expect(
+ resolveSupersededTurnOutcome({
+ kind: 'failed',
+ failurePhase: 'credits_exhausted',
+ }),
+ ).toEqual({ kind: 'failed', failurePhase: 'credits_exhausted' });
+ });
+
+ it('keeps a no_text failure', () => {
+ expect(
+ resolveSupersededTurnOutcome({ kind: 'failed', failurePhase: 'no_text' }),
+ ).toEqual({ kind: 'failed', failurePhase: 'no_text' });
+ });
+
+ it('supersedes a completed turn, whose answer may never have landed', () => {
+ expect(
+ resolveSupersededTurnOutcome({ kind: 'completed', outcome: 'answered' }),
+ ).toEqual({ kind: 'cancelled', reason: 'superseded' });
+ });
+
+ it('supersedes a turn that was awaiting a user answer', () => {
+ expect(
+ resolveSupersededTurnOutcome({
+ kind: 'completed',
+ outcome: 'awaiting_user',
+ }),
+ ).toEqual({ kind: 'cancelled', reason: 'superseded' });
+ });
+
+ it('supersedes a user cancellation', () => {
+ expect(
+ resolveSupersededTurnOutcome({
+ kind: 'cancelled',
+ reason: 'user_cancelled',
+ }),
+ ).toEqual({ kind: 'cancelled', reason: 'superseded' });
+ });
+});
diff --git a/packages/twenty-server/src/engine/metadata-modules/ai/ai-chat/utils/classify-agent-chat-turn-outcome.util.ts b/packages/twenty-server/src/engine/metadata-modules/ai/ai-chat/utils/classify-agent-chat-turn-outcome.util.ts
new file mode 100644
index 00000000000..eeb4a705f85
--- /dev/null
+++ b/packages/twenty-server/src/engine/metadata-modules/ai/ai-chat/utils/classify-agent-chat-turn-outcome.util.ts
@@ -0,0 +1,45 @@
+import { type AgentChatTurnOutcome } from 'src/engine/metadata-modules/ai/ai-chat/types/agent-chat-turn-outcome.type';
+
+export const classifyAgentChatTurnOutcome = ({
+ hasText,
+ isAborted,
+ isAwaitingUserAnswer,
+ outOfCredits,
+}: {
+ hasText: boolean;
+ isAborted: boolean;
+ isAwaitingUserAnswer: boolean;
+ outOfCredits: boolean;
+}): AgentChatTurnOutcome => {
+ // Asking the user a question is how a turn is meant to end when the request is
+ // ambiguous, and stopWhen ends the stream on it — so it is a completed turn
+ // waiting on an answer, not an abandoned one.
+ if (isAwaitingUserAnswer) {
+ return { kind: 'completed', outcome: 'awaiting_user' };
+ }
+
+ if (isAborted) {
+ return { kind: 'cancelled', reason: 'user_cancelled' };
+ }
+
+ if (hasText) {
+ return { kind: 'completed', outcome: 'answered' };
+ }
+
+ return {
+ kind: 'failed',
+ failurePhase: outOfCredits ? 'credits_exhausted' : 'no_text',
+ };
+};
+
+// Losing the stream claim says the turn's result may never have reached the
+// user, which is worth recording over "completed". It says nothing about a
+// failure that already happened, so those keep their own phase — otherwise a
+// turn that ran out of credits during a claim handover is filed as a plain
+// cancellation and the billing signal disappears.
+export const resolveSupersededTurnOutcome = (
+ outcome: AgentChatTurnOutcome,
+): AgentChatTurnOutcome =>
+ outcome.kind === 'failed'
+ ? outcome
+ : { kind: 'cancelled', reason: 'superseded' };