The open alternative to Salesforce, designed for AI.
TypeScript crmcrm-systemcustomergood-first-issuegraphqlhacktoberfestjavascriptmarketingmonoreponestjsopen-sourcepostgresqlreactreactjssalestypescriptweb
Agent engine: release a conversation's claim in one place (#28306)
Félix Malfait570bdc28033b
diff --git a/packages/twenty-server/src/engine/metadata-modules/ai/ai-agent-execution/services/agent-runner.service.ts b/packages/twenty-server/src/engine/metadata-modules/ai/ai-agent-execution/services/agent-runner.service.tsindex c8212685af4..c9bd2f85773 100644--- a/packages/twenty-server/src/engine/metadata-modules/ai/ai-agent-execution/services/agent-runner.service.ts+++ b/packages/twenty-server/src/engine/metadata-modules/ai/ai-agent-execution/services/agent-runner.service.ts@@ -35,10 +35,13 @@ import { type RunAgentJobData } from 'src/engine/metadata-modules/ai/ai-agent-eximport { buildAgentRunSummary } from 'src/engine/metadata-modules/ai/ai-agent-execution/utils/build-agent-run-summary.util';import { sumAgentRunSummaries } from 'src/engine/metadata-modules/ai/ai-agent-execution/utils/sum-agent-run-summaries.util';import { AgentEntity } from 'src/engine/metadata-modules/ai/ai-agent/entities/agent.entity';-import { type AgentChatStreamClaim } from 'src/engine/metadata-modules/ai/ai-chat/services/agent-chat-stream-recovery.service';import { AgentChatEventPublisherService } from 'src/engine/metadata-modules/ai/ai-chat/services/agent-chat-event-publisher.service';+import {+ type AgentChatStreamClaim,+ AgentChatStreamRecoveryService,+} from 'src/engine/metadata-modules/ai/ai-chat/services/agent-chat-stream-recovery.service';import { AgentTurnStreamService } from 'src/engine/metadata-modules/ai/ai-chat/services/agent-turn-stream.service';-import { AgentTurnRecorderService } from 'src/engine/metadata-modules/ai/ai-history/services/agent-turn-recorder.service';+import { mapErrorToStreamError } from 'src/engine/metadata-modules/ai/ai-history/utils/map-error-to-stream-error.util';import { AgentConversationReaderService } from 'src/engine/metadata-modules/ai/ai-history/services/agent-conversation-reader.service';import { withDedicatedAiTrace } from 'src/engine/metadata-modules/ai/ai-models/utils/with-dedicated-ai-trace.util';import {@@ -74,8 +77,8 @@ export class AgentRunnerService {private readonly agentRunSuspensionService: AgentRunSuspensionService,private readonly callerHandlerRegistry: AgentRunCallerHandlerRegistryService,private readonly pendingWakeUpService: PendingWakeUpService,- private readonly turnRecorderService: AgentTurnRecorderService,private readonly turnStreamService: AgentTurnStreamService,+ private readonly streamRecoveryService: AgentChatStreamRecoveryService,private readonly eventPublisherService: AgentChatEventPublisherService,@InjectMessageQueue(MessageQueue.aiQueue)private readonly messageQueueService: MessageQueueService,@@ -455,7 +458,11 @@ export class AgentRunnerService {streamId: stream.claim.streamId,});// handled here instead of thrown, so the interrupted turn is failed here as a thrown error would be- await this.turnStreamService.failStream({ claim: stream.claim, error });+ await this.streamRecoveryService.release({+ ...stream.claim,+ turnError: mapErrorToStreamError(error),+ shouldFlush: true,+ });return null;}@@ -553,71 +560,53 @@ export class AgentRunnerService {};// the toolset and prompt come from the spec alone, so a continued run gets those it paused with- const streamed = await withDedicatedAiTrace(() =>- this.turnStreamService.stream({+ const {+ streamed,+ completion: { execution, wait },+ finish,+ } = await withDedicatedAiTrace(() =>+ this.turnStreamService.streamTurn({claim,turnId,+ abortSignal,startTurn: () => this.agentRunTurnService.startTurn(turnInput),- }),- );-- // a turn that fails keeps the usage of the steps it finished, as billing does- const recordStreamedUsage = () =>- this.turnRecorderService.recordUsage({- workspaceId,- threadId,- turnId,- usage: streamed.usage,- });-- if (isDefined(streamed.streamError)) {- await recordStreamedUsage();-- throw streamed.streamError;- }-- const execution = await this.agentRunTurnService- .completeTurn({ ...turnInput, started: streamed.started })- .catch(async (error: unknown) => {- await recordStreamedUsage();-- throw error;- });-- const wait = execution.isPaused- ? findAgentRunWait(- execution.steps[execution.steps.length - 1]?.toolResults ?? [],- )- : undefined;+ completeTurn: async (streamed) => {+ if (isDefined(streamed.streamError)) {+ throw streamed.streamError;+ }- const finished = await this.turnStreamService.finishTurn({- claim,- turnId,- result: {- ...streamed,- usage: execution.structuredOutputSteps.reduce(- (usage, step) => ({- ...usage,- ...addStepToAgentTurnUsage(- streamed.started.source.modelConfig,- usage,- step,+ const execution = await this.agentRunTurnService.completeTurn({+ ...turnInput,+ started: streamed.started,+ });+ const wait = execution.isPaused+ ? findAgentRunWait(+ execution.steps[execution.steps.length - 1]?.toolResults ?? [],+ )+ : undefined;++ return {+ execution,+ wait,+ modelId: execution.modelId,+ stopReason: execution.stopReason,+ isWaiting: isDefined(wait),+ usage: execution.structuredOutputSteps.reduce(+ (usage, step) => ({+ ...usage,+ ...addStepToAgentTurnUsage(+ streamed.started.source.modelConfig,+ usage,+ step,+ ),+ }),+ streamed.usage,),- }),- streamed.usage,- ),- },- modelId: execution.modelId,- stopReason: execution.stopReason,- isWaiting: isDefined(wait),- isBackgroundRun: true,- });-- await this.turnStreamService.publishTurnEnd({- claim,- assistantMessageId: streamed.assistantMessageId,- stopReason: execution.stopReason,- });+ };+ },+ isBackgroundRun: true,+ }),+ );const summary = sumAgentRunSummaries({previousSummary: suspension?.summary ?? null,@@ -637,7 +626,7 @@ export class AgentRunnerService {input,execution,wait,- isAwaitingAnswer: finished?.isAwaitingAnswer ?? false,+ isAwaitingAnswer: finish?.isAwaitingAnswer ?? false,summary,suspension,streamId: claim.streamId,diff --git a/packages/twenty-server/src/engine/metadata-modules/ai/ai-chat/jobs/stream-agent-chat.job.ts b/packages/twenty-server/src/engine/metadata-modules/ai/ai-chat/jobs/stream-agent-chat.job.tsindex 10e5f4e6f32..892d93902ef 100644--- a/packages/twenty-server/src/engine/metadata-modules/ai/ai-chat/jobs/stream-agent-chat.job.ts+++ b/packages/twenty-server/src/engine/metadata-modules/ai/ai-chat/jobs/stream-agent-chat.job.ts@@ -2,10 +2,7 @@ import { Logger, Scope } from '@nestjs/common';import { InjectRepository } from '@nestjs/typeorm';import { isNonEmptyString } from '@sniptt/guards';-import {- type CodeExecutionData,- isSilentAgentChatMessageParts,-} from 'twenty-shared/ai';+import { type CodeExecutionData } from 'twenty-shared/ai';import { assertUnreachable, isDefined } from 'twenty-shared/utils';import { Repository } from 'typeorm';@@ -18,22 +15,16 @@ import { MessageQueue } from 'src/engine/core-modules/message-queue/message-queuimport { MetricsService } from 'src/engine/core-modules/metrics/metrics.service';import { MetricsKeys } from 'src/engine/core-modules/metrics/types/metrics-keys.type';import { WorkspaceEntity } from 'src/engine/core-modules/workspace/workspace.entity';-import { type StepDebitResult } from 'src/engine/metadata-modules/ai/ai-billing/types/step-debit-result.type';-import { findAwaitingPausingToolParts } from 'src/engine/metadata-modules/ai/ai-agent-execution/pausing-tools/utils/find-awaiting-pausing-tool-parts.util';import { STREAM_AGENT_CHAT_JOB_NAME } from 'src/engine/metadata-modules/ai/ai-chat/jobs/stream-agent-chat-job-name.constant';import { type StreamAgentChatJobData } from 'src/engine/metadata-modules/ai/ai-chat/jobs/stream-agent-chat-job.types';import { AgentChatActorService } from 'src/engine/metadata-modules/ai/ai-chat/services/agent-chat-actor.service';import { AgentChatEventPublisherService } from 'src/engine/metadata-modules/ai/ai-chat/services/agent-chat-event-publisher.service';import { AgentChatService } from 'src/engine/metadata-modules/ai/ai-chat/services/agent-chat.service';-import {- type AgentTurnStreamResult,- AgentTurnStreamService,-} from 'src/engine/metadata-modules/ai/ai-chat/services/agent-turn-stream.service';+import { AgentTurnStreamService } from 'src/engine/metadata-modules/ai/ai-chat/services/agent-turn-stream.service';import { ChatExecutionService } from 'src/engine/metadata-modules/ai/ai-chat/services/chat-execution.service';import { type AgentChatTurnOutcome } from 'src/engine/metadata-modules/ai/ai-chat/types/agent-chat-turn-outcome.type';import { type AgentChatStreamClaim } from 'src/engine/metadata-modules/ai/ai-chat/services/agent-chat-stream-recovery.service';import { formatErrorWithCause } from 'src/engine/metadata-modules/ai/ai-chat/utils/format-error-with-cause.util';-import { hasReplyText } from 'src/engine/metadata-modules/ai/ai-chat/utils/has-reply-text.util';import { tagAiChatStreamScope } from 'src/engine/metadata-modules/ai/ai-chat/utils/tag-ai-chat-stream-scope.util';import { AgentHistoryRepository } from 'src/engine/metadata-modules/ai/ai-history/repositories/agent-history-repository';import { InjectAgentHistoryRepository } from 'src/engine/metadata-modules/ai/ai-history/repositories/inject-agent-history-repository.decorator';@@ -206,9 +197,10 @@ export class StreamAgentChatJob {}).catch(() => null);- const result = await this.turnStreamService.stream({+ const { finish } = await this.turnStreamService.streamTurn({claim,turnId,+ abortSignal,startTurn: async (writer) => {const turn = await this.chatExecutionService.streamChat({workspace,@@ -278,24 +270,10 @@ export class StreamAgentChatJob {getStopReason: turn.getStopReason,};},- });-- const stopReason = result.started.getStopReason();-- this.logTurnWithoutText({- result,- stopReason,- threadId: data.threadId,- workspaceId: data.workspaceId,- modelId: turnModelId,- });-- const finish = await this.turnStreamService.finishTurn({- claim,- turnId,- result,- modelId: turnModelId,- stopReason,+ completeTurn: async ({ started }) => ({+ modelId: turnModelId,+ stopReason: started.getStopReason(),+ }),});if (isDefined(finish)) {@@ -309,21 +287,6 @@ export class StreamAgentChatJob {});}}-- // a worker shutting down interrupted the turn, which fails rather than ends as stopped- if (abortSignal.reason instanceof AiException) {- throw abortSignal.reason;- }-- if (isDefined(result.streamError)) {- throw result.streamError;- }-- await this.turnStreamService.publishTurnEnd({- claim,- assistantMessageId: result.assistantMessageId,- stopReason,- });}// resolve auto-select ids here so turn-start and outcome metrics carry the same model label@@ -394,58 +357,4 @@ export class StreamAgentChatJob {return assertUnreachable(outcome);}}-- private logTurnWithoutText({- result: { replyParts, isAborted, streamError, usage },- stopReason,- threadId,- workspaceId,- modelId,- }: {- result: AgentTurnStreamResult;- stopReason: StepDebitResult['stopReason'];- threadId: string;- workspaceId: string;- modelId: string;- }): void {- const hasText = hasReplyText(replyParts);-- if (- (!isAborted && hasText) ||- findAwaitingPausingToolParts(replyParts).length > 0 ||- isSilentAgentChatMessageParts(replyParts)- ) {- return;- }-- const reason = isAborted- ? 'user-cancelled'- : streamError- ? 'stream-error'- : stopReason === 'userAllowance'- ? 'user-credits-exhausted'- : stopReason === 'workspaceAllowance'- ? 'credits-exhausted'- : 'empty-completion';-- const errorDetail = isDefined(streamError)- ? formatErrorWithCause(streamError)- : 'none';-- this.logger.warn(- `[AI_CHAT_NO_TEXT] Assistant turn ended without a text reply — ` +- `reason=${reason}, model=${modelId}, ` +- `threadId=${threadId}, workspaceId=${workspaceId}, ` +- `isAborted=${isAborted}, stopReason=${stopReason}, hasText=${hasText}, ` +- `streamError=${errorDetail}, ` +- `inputTokens=${usage.inputTokens},` +- `responseMessage.parts=${JSON.stringify(replyParts)}`,- );-- if (streamError instanceof Error && isDefined(streamError.stack)) {- this.logger.warn(- `[AI_CHAT_NO_TEXT] streamError stack — threadId=${threadId}: ${streamError.stack}`,- );- }- }}diff --git a/packages/twenty-server/src/engine/metadata-modules/ai/ai-chat/resolvers/agent-chat.resolver.ts b/packages/twenty-server/src/engine/metadata-modules/ai/ai-chat/resolvers/agent-chat.resolver.tsindex 35874c757b7..a4455431d33 100644--- a/packages/twenty-server/src/engine/metadata-modules/ai/ai-chat/resolvers/agent-chat.resolver.ts+++ b/packages/twenty-server/src/engine/metadata-modules/ai/ai-chat/resolvers/agent-chat.resolver.ts@@ -331,6 +331,11 @@ export class AgentChatResolver {event: { type: 'queue-updated' },});+ await this.agentChatStreamingService.flushNextQueuedMessage({+ threadId,+ workspaceId: workspace.id,+ });+return { messageId: queuedMessage.id, queued: true };}diff --git a/packages/twenty-server/src/engine/metadata-modules/ai/ai-chat/services/agent-chat-event-publisher.service.ts b/packages/twenty-server/src/engine/metadata-modules/ai/ai-chat/services/agent-chat-event-publisher.service.tsindex 5966491ec0b..14e6514fc53 100644--- a/packages/twenty-server/src/engine/metadata-modules/ai/ai-chat/services/agent-chat-event-publisher.service.ts+++ b/packages/twenty-server/src/engine/metadata-modules/ai/ai-chat/services/agent-chat-event-publisher.service.ts@@ -7,6 +7,11 @@ import { SubscriptionService } from 'src/engine/subscriptions/subscription.serviconst STREAM_CHUNKS_TTL_SECONDS = 3600;+const RESET_OWNED_STREAM_STATE_SCRIPT = `+if redis.call('GET', KEYS[2]) == ARGV[1] then+ redis.call('DEL', KEYS[1], KEYS[2])+end`;+@Injectable()export class AgentChatEventPublisherService {constructor(@@ -18,6 +23,10 @@ export class AgentChatEventPublisherService {return `agent-chat-stream-chunks:${threadId}`;}+ private getStreamOwnerKey(threadId: string): string {+ return `agent-chat-stream-owner:${threadId}`;+ }+async publish({threadId,workspaceId,@@ -38,6 +47,10 @@ export class AgentChatEventPublisherService {const [seq] = await Promise.all([redis.rpush(key, JSON.stringify(event.chunk)),redis.expire(key, STREAM_CHUNKS_TTL_SECONDS),+ redis.expire(+ this.getStreamOwnerKey(threadId),+ STREAM_CHUNKS_TTL_SECONDS,+ ),]);publishedEvent = { ...event, seq };@@ -67,6 +80,45 @@ export class AgentChatEventPublisherService {await redis.del(this.getStreamChunksKey(threadId));}+ async startStreamState({+ threadId,+ streamId,+ }: {+ threadId: string;+ streamId: string;+ }): Promise<void> {+ await this.redisClientService+ .getClient()+ .multi()+ .del(this.getStreamChunksKey(threadId))+ .set(+ this.getStreamOwnerKey(threadId),+ streamId,+ 'EX',+ STREAM_CHUNKS_TTL_SECONDS,+ )+ .exec();+ }++ // a stale release of a dead stream must not delete the chunks of the stream that started after it+ async resetOwnedStreamState({+ threadId,+ streamId,+ }: {+ threadId: string;+ streamId: string;+ }): Promise<void> {+ await this.redisClientService+ .getClient()+ .eval(+ RESET_OWNED_STREAM_STATE_SCRIPT,+ 2,+ this.getStreamChunksKey(threadId),+ this.getStreamOwnerKey(threadId),+ streamId,+ );+ }+async getAccumulatedChunks(threadId: string): Promise<{chunks: Record<string, unknown>[];maxSeq: number;diff --git a/packages/twenty-server/src/engine/metadata-modules/ai/ai-chat/services/agent-chat-stream-recovery.service.ts b/packages/twenty-server/src/engine/metadata-modules/ai/ai-chat/services/agent-chat-stream-recovery.service.tsindex 0020172d90f..1a8bcb44212 100644--- a/packages/twenty-server/src/engine/metadata-modules/ai/ai-chat/services/agent-chat-stream-recovery.service.ts+++ b/packages/twenty-server/src/engine/metadata-modules/ai/ai-chat/services/agent-chat-stream-recovery.service.ts@@ -2,16 +2,20 @@ import { Injectable, Logger } from '@nestjs/common';import { type FindOptionsWhere, IsNull } from 'typeorm';import { isDefined, isNonEmptyString } from 'twenty-shared/utils';+import { InjectMessageQueue } from 'src/engine/core-modules/message-queue/decorators/message-queue.decorator';+import { MessageQueue } from 'src/engine/core-modules/message-queue/message-queue.constants';+import { MessageQueueService } from 'src/engine/core-modules/message-queue/services/message-queue.service';import { MetricsService } from 'src/engine/core-modules/metrics/metrics.service';import { MetricsKeys } from 'src/engine/core-modules/metrics/types/metrics-keys.type';import { AgentChatThreadWorkspaceEntity } from 'src/engine/metadata-modules/ai/ai-history/standard-objects/agent-chat-thread.workspace-entity';+import { type AgentMessageWorkspaceEntity } from 'src/engine/metadata-modules/ai/ai-history/standard-objects/agent-message.workspace-entity';+import { AgentMessageStatus } from 'src/engine/metadata-modules/ai/ai-history/enums/agent-message-status.enum';+import { FLUSH_QUEUED_AGENT_CHAT_MESSAGES_JOB_NAME } from 'src/engine/metadata-modules/ai/ai-chat/jobs/flush-queued-agent-chat-messages-job-name.constant';+import { type FlushQueuedAgentChatMessagesJobData } from 'src/engine/metadata-modules/ai/ai-chat/jobs/flush-queued-agent-chat-messages-job.types';import { AgentChatEventPublisherService } from 'src/engine/metadata-modules/ai/ai-chat/services/agent-chat-event-publisher.service';import { AgentChatStreamHeartbeatService } from 'src/engine/metadata-modules/ai/ai-chat/services/agent-chat-stream-heartbeat.service';import { formatErrorWithCause } from 'src/engine/metadata-modules/ai/ai-chat/utils/format-error-with-cause.util';-import {- mapErrorToStreamError,- type StreamErrorPayload,-} from 'src/engine/metadata-modules/ai/ai-history/utils/map-error-to-stream-error.util';+import { type StreamErrorPayload } from 'src/engine/metadata-modules/ai/ai-history/utils/map-error-to-stream-error.util';import { AgentTurnStatus } from 'src/engine/metadata-modules/ai/ai-history/enums/agent-turn-status.enum';import { AgentHistoryRepository } from 'src/engine/metadata-modules/ai/ai-history/repositories/agent-history-repository';import { AgentTurnRecorderService } from 'src/engine/metadata-modules/ai/ai-history/services/agent-turn-recorder.service';@@ -32,51 +36,67 @@ export class AgentChatStreamRecoveryService {constructor(@InjectAgentHistoryRepository('agentChatThread')private readonly threadRepository: AgentHistoryRepository<AgentChatThreadWorkspaceEntity>,+ @InjectAgentHistoryRepository('agentMessage')+ private readonly messageRepository: AgentHistoryRepository<AgentMessageWorkspaceEntity>,+ @InjectMessageQueue(MessageQueue.aiStreamQueue)+ private readonly chatMessageQueueService: MessageQueueService,private readonly streamHeartbeatService: AgentChatStreamHeartbeatService,private readonly eventPublisherService: AgentChatEventPublisherService,private readonly metricsService: MetricsService,private readonly turnRecorderService: AgentTurnRecorderService,) {}- async releaseStreamClaim({- threadId,- workspaceId,- streamId,+ // a member who stopped the turn, or a run about to take the queue itself, leaves the queue where it is+ async release({turnError,+ shouldFlush,+ ...claim}: AgentChatStreamClaim & {turnError?: StreamErrorPayload;+ shouldFlush: boolean;}): Promise<boolean> {- const isReleased = await this.clearStreamClaim({- threadId,- workspaceId,- streamId,- turnError,- }).catch((error: unknown) => {- this.logger.error(- `Failed to release stream claim for thread ${threadId}: ${formatErrorWithCause(error)}`,- );+ const { threadId, workspaceId, streamId } = claim;++ if (isDefined(turnError)) {+ await this.eventPublisherService+ .resetOwnedStreamState({ threadId, streamId })+ .catch((error: unknown) =>+ this.logger.warn(+ `Failed to reset the stream state of thread ${threadId} in workspace ${workspaceId}: ${formatErrorWithCause(error)}`,+ ),+ );+ }- return false;- });+ const isReleased = await this.turnRecorderService+ .releaseStreamClaim({+ ...claim,+ endRunningTurn: isDefined(turnError)+ ? { status: AgentTurnStatus.FAILED, error: turnError }+ : undefined,+ })+ .catch((error: unknown) => {+ this.logger.error(+ `Failed to release stream claim for thread ${threadId}: ${formatErrorWithCause(error)}`,+ );++ return false;+ });await this.streamHeartbeatService.clear(streamId);- return isReleased;- }+ if (!isReleased) {+ return false;+ }- // the error belongs to the turn the stream was running, and only the caller still holding the claim writes it- private clearStreamClaim({- turnError,- ...claim- }: AgentChatStreamClaim & {- turnError?: StreamErrorPayload;- }): Promise<boolean> {- return this.turnRecorderService.releaseStreamClaim({- ...claim,- endRunningTurn: isDefined(turnError)- ? { status: AgentTurnStatus.FAILED, error: turnError }- : undefined,- });+ if (isDefined(turnError)) {+ await this.publishStreamError({ threadId, workspaceId, turnError });+ }++ if (shouldFlush) {+ await this.flushQueuedMessages({ threadId, workspaceId });+ }++ return true;}async tryClaimStream({@@ -104,26 +124,6 @@ export class AgentChatStreamRecoveryService {return true;}- async failStream({- threadId,- workspaceId,- streamId,- error,- }: AgentChatStreamClaim & { error: unknown }): Promise<void> {- const turnError = mapErrorToStreamError(error);-- const isReleased = await this.releaseStreamClaim({- threadId,- workspaceId,- streamId,- turnError,- });-- if (isReleased) {- await this.publishStreamError({ threadId, workspaceId, turnError });- }- }-async reapDeadStream({thread,workspaceId,@@ -144,11 +144,12 @@ export class AgentChatStreamRecoveryService {message: 'The response was interrupted before it could finish.',};- const hasReaped = await this.clearStreamClaim({+ const hasReaped = await this.release({threadId: thread.id,workspaceId,streamId: thread.activeStreamId,turnError: interruptedError,+ shouldFlush: true,});if (!hasReaped) {@@ -168,16 +169,35 @@ export class AgentChatStreamRecoveryService {`[AI_CHAT_TURN_FAILED] failurePhase=interrupted, threadId=${thread.id}, workspaceId=${workspaceId}: stream ${thread.activeStreamId} stopped sending heartbeats`,);- await this.eventPublisherService.resetStreamState(thread.id);- await this.publishStreamError({- threadId: thread.id,- workspaceId,- turnError: interruptedError,- });-return interruptedError;}+ private async flushQueuedMessages({+ threadId,+ workspaceId,+ }: {+ threadId: string;+ workspaceId: string;+ }): Promise<void> {+ try {+ const hasQueuedMessages = await this.messageRepository.existsBy(+ workspaceId,+ { threadId, status: AgentMessageStatus.QUEUED },+ );++ if (hasQueuedMessages) {+ await this.chatMessageQueueService.add<FlushQueuedAgentChatMessagesJobData>(+ FLUSH_QUEUED_AGENT_CHAT_MESSAGES_JOB_NAME,+ { workspaceId, threadId },+ );+ }+ } catch (error) {+ this.logger.error(+ `Failed to send the messages queued in thread ${threadId} of workspace ${workspaceId}: ${formatErrorWithCause(error)}`,+ );+ }+ }+private async publishStreamError({threadId,workspaceId,diff --git a/packages/twenty-server/src/engine/metadata-modules/ai/ai-chat/services/agent-chat-streaming.service.ts b/packages/twenty-server/src/engine/metadata-modules/ai/ai-chat/services/agent-chat-streaming.service.tsindex de7bd4d2a67..f5f291918b4 100644--- a/packages/twenty-server/src/engine/metadata-modules/ai/ai-chat/services/agent-chat-streaming.service.ts+++ b/packages/twenty-server/src/engine/metadata-modules/ai/ai-chat/services/agent-chat-streaming.service.ts@@ -156,7 +156,8 @@ export class AgentChatStreamingService {wakeUpId: suspendedRun.id,outcome: { type: 'MESSAGE_RECEIVED' },});- } else if (hasQueuedBacklog) {+ } else {+ // the stream may have let go just before this message was queued, finding nothing to sendawait this.flushNextQueuedMessage({ threadId, workspaceId });}@@ -312,12 +313,12 @@ export class AgentChatStreamingService {if (await this.agentChatService.hasMessages({ threadId, workspaceId })) {- await this.streamRecoveryService.releaseStreamClaim({+ await this.streamRecoveryService.release({threadId,workspaceId,streamId,+ shouldFlush: true,});- await this.flushNextQueuedMessage({ threadId, workspaceId });return null;}@@ -637,10 +638,11 @@ export class AgentChatStreamingService {});if (turnId === null) {- await this.streamRecoveryService.releaseStreamClaim({+ await this.streamRecoveryService.release({threadId,workspaceId,streamId,+ shouldFlush: true,});return;@@ -748,9 +750,10 @@ export class AgentChatStreamingService {return await enqueue();} catch (error) {// a turn the claim already started fails with the enqueue error- await this.streamRecoveryService.releaseStreamClaim({+ await this.streamRecoveryService.release({...claim,turnError: mapErrorToStreamError(error),+ shouldFlush: true,});const model = modelId ?? 'unknown';diff --git a/packages/twenty-server/src/engine/metadata-modules/ai/ai-chat/services/agent-turn-stream.service.ts b/packages/twenty-server/src/engine/metadata-modules/ai/ai-chat/services/agent-turn-stream.service.tsindex 792aa62ef79..5cbe7b59504 100644--- a/packages/twenty-server/src/engine/metadata-modules/ai/ai-chat/services/agent-turn-stream.service.ts+++ b/packages/twenty-server/src/engine/metadata-modules/ai/ai-chat/services/agent-turn-stream.service.ts@@ -16,17 +16,12 @@ import {import { isDefined } from 'twenty-shared/utils';import { v5 as uuidv5 } from 'uuid';-import { InjectMessageQueue } from 'src/engine/core-modules/message-queue/decorators/message-queue.decorator';-import { MessageQueue } from 'src/engine/core-modules/message-queue/message-queue.constants';-import { MessageQueueService } from 'src/engine/core-modules/message-queue/services/message-queue.service';import { toDisplayCredits } from 'src/engine/core-modules/usage/utils/to-display-credits.util';import { findAwaitingPausingToolParts } from 'src/engine/metadata-modules/ai/ai-agent-execution/pausing-tools/utils/find-awaiting-pausing-tool-parts.util';import { type AgentTurn } from 'src/engine/metadata-modules/ai/ai-agent-execution/services/agent-turn.service';import { type AgentTurnUsage } from 'src/engine/metadata-modules/ai/ai-billing/types/agent-turn-usage.type';import { type StepDebitResult } from 'src/engine/metadata-modules/ai/ai-billing/types/step-debit-result.type';import { addStepToAgentTurnUsage } from 'src/engine/metadata-modules/ai/ai-billing/utils/add-step-to-agent-turn-usage.util';-import { FLUSH_QUEUED_AGENT_CHAT_MESSAGES_JOB_NAME } from 'src/engine/metadata-modules/ai/ai-chat/jobs/flush-queued-agent-chat-messages-job-name.constant';-import { type FlushQueuedAgentChatMessagesJobData } from 'src/engine/metadata-modules/ai/ai-chat/jobs/flush-queued-agent-chat-messages-job.types';import { AGENT_CHAT_CHECKPOINT_INTERVAL_MS } from 'src/engine/metadata-modules/ai/ai-chat/constants/agent-chat-checkpoint-interval-ms.constant';import { AgentChatCancelSubscriberService } from 'src/engine/metadata-modules/ai/ai-chat/services/agent-chat-cancel-subscriber.service';import { AgentChatEventPublisherService } from 'src/engine/metadata-modules/ai/ai-chat/services/agent-chat-event-publisher.service';@@ -45,6 +40,10 @@ import { findAwaitingCallText } from 'src/engine/metadata-modules/ai/ai-chat/utiimport { hasReplyText } from 'src/engine/metadata-modules/ai/ai-chat/utils/has-reply-text.util';import { findLastMessageText } from 'src/engine/metadata-modules/ai/ai-chat/utils/find-last-message-text.util';import { formatErrorWithCause } from 'src/engine/metadata-modules/ai/ai-chat/utils/format-error-with-cause.util';+import {+ mapErrorToStreamError,+ type StreamErrorPayload,+} from 'src/engine/metadata-modules/ai/ai-history/utils/map-error-to-stream-error.util';import { getCancelChannel } from 'src/engine/metadata-modules/ai/ai-chat/utils/get-cancel-channel.util';import { mapAgentChatTurnOutcomeToTurnStatus } from 'src/engine/metadata-modules/ai/ai-chat/utils/map-agent-chat-turn-outcome-to-turn-status.util';import { updateAgentChatThreadUsage } from 'src/engine/metadata-modules/ai/ai-chat/utils/update-agent-chat-thread-usage.util';@@ -52,8 +51,6 @@ import { AgentHistoryRepository } from 'src/engine/metadata-modules/ai/ai-historimport { InjectAgentHistoryRepository } from 'src/engine/metadata-modules/ai/ai-history/repositories/inject-agent-history-repository.decorator';import { AgentConversationWriterService } from 'src/engine/metadata-modules/ai/ai-history/services/agent-conversation-writer.service';import { AgentTurnRecorderService } from 'src/engine/metadata-modules/ai/ai-history/services/agent-turn-recorder.service';-import { AgentMessageStatus } from 'src/engine/metadata-modules/ai/ai-history/enums/agent-message-status.enum';-import { type AgentMessageWorkspaceEntity } from 'src/engine/metadata-modules/ai/ai-history/standard-objects/agent-message.workspace-entity';import { type AgentChatThreadWorkspaceEntity } from 'src/engine/metadata-modules/ai/ai-history/standard-objects/agent-chat-thread.workspace-entity';import { type AiModelConfig } from 'src/engine/metadata-modules/ai/ai-models/types/ai-model-config.type';import {@@ -84,6 +81,13 @@ export type AgentTurnStreamResult = {modelConfig: AiModelConfig | null;};+export type AgentTurnCompletion = {+ modelId: string;+ stopReason: StepDebitResult['stopReason'];+ isWaiting?: boolean;+ usage?: AgentTurnStreamUsage;+};+export type AgentTurnFinish = {outcome: AgentChatTurnOutcome;isAwaitingAnswer: boolean;@@ -91,6 +95,34 @@ export type AgentTurnFinish = {threadBefore: AgentChatThreadWorkspaceEntity | null;};+const getTurnWithoutTextReason = ({+ isAborted,+ streamError,+ stopReason,+}: {+ isAborted: boolean;+ streamError: unknown;+ stopReason: StepDebitResult['stopReason'];+}): string => {+ if (isAborted) {+ return 'user-cancelled';+ }++ if (isDefined(streamError)) {+ return 'stream-error';+ }++ if (stopReason === 'userAllowance') {+ return 'user-credits-exhausted';+ }++ if (stopReason === 'workspaceAllowance') {+ return 'credits-exhausted';+ }++ return 'empty-completion';+};+@Injectable()export class AgentTurnStreamService {private readonly logger = new Logger(AgentTurnStreamService.name);@@ -98,10 +130,6 @@ export class AgentTurnStreamService {constructor(@InjectAgentHistoryRepository('agentChatThread')private readonly threadRepository: AgentHistoryRepository<AgentChatThreadWorkspaceEntity>,- @InjectAgentHistoryRepository('agentMessage')- private readonly messageRepository: AgentHistoryRepository<AgentMessageWorkspaceEntity>,- @InjectMessageQueue(MessageQueue.aiStreamQueue)- private readonly chatMessageQueueService: MessageQueueService,private readonly eventPublisherService: AgentChatEventPublisherService,private readonly cancelSubscriberService: AgentChatCancelSubscriberService,private readonly streamHeartbeatService: AgentChatStreamHeartbeatService,@@ -111,8 +139,7 @@ export class AgentTurnStreamService {private readonly threadService: AgentChatThreadService,) {}- // Keeps the claim alive and stoppable while the work runs, and releases it however the work ends.- // Messages queued meanwhile are sent then, unless the member stopped the turn+ // Keeps the claim alive and stoppable while the work runs, and releases it however the work endsasync runClaimed<TResult>({claim,jobAbortSignal,@@ -123,83 +150,132 @@ export class AgentTurnStreamService {work: (abortSignal: AbortSignal) => Promise<TResult>;}): Promise<TResult> {const { threadId, streamId } = claim;-- await this.eventPublisherService.resetStreamState(threadId);-const abortController = new AbortController();const cancelChannel = getCancelChannel(threadId, streamId);- const stopHeartbeat = this.streamHeartbeatService.startRunning(streamId);+ let stopHeartbeat: (() => void) | undefined;+ let isStoppedByMember = false;+ let turnError: StreamErrorPayload | undefined;- await this.cancelSubscriberService.subscribe(cancelChannel, () => {- abortController.abort();- });+ try {+ await this.eventPublisherService.startStreamState({+ threadId,+ streamId,+ });+ stopHeartbeat = this.streamHeartbeatService.startRunning(streamId);- jobAbortSignal?.addEventListener(- 'abort',- () => {- abortController.abort(- new AiException(- 'The response was interrupted before it could finish.',- AiExceptionCode.STREAM_INTERRUPTED,- ),- );- },- { once: true },- );+ await this.cancelSubscriberService.subscribe(cancelChannel, () => {+ isStoppedByMember = true;+ abortController.abort();+ });++ jobAbortSignal?.addEventListener(+ 'abort',+ () => {+ abortController.abort(+ new AiException(+ 'The response was interrupted before it could finish.',+ AiExceptionCode.STREAM_INTERRUPTED,+ ),+ );+ },+ { once: true },+ );- try {return await work(abortController.signal);} catch (error) {- await this.failStream({ claim, error });+ turnError = mapErrorToStreamError(error);throw error;} finally {- stopHeartbeat();+ stopHeartbeat?.();await this.cancelSubscriberService.unsubscribe(cancelChannel);- await this.streamRecoveryService.releaseStreamClaim(claim);-- if (!abortController.signal.aborted) {- await this.flushQueuedMessages(claim);- }- }- }-- private async flushQueuedMessages({- workspaceId,- threadId,- }: AgentChatStreamClaim): Promise<void> {- try {- const hasQueuedMessages = await this.messageRepository.existsBy(- workspaceId,- { threadId, status: AgentMessageStatus.QUEUED },- );-- if (hasQueuedMessages) {- await this.chatMessageQueueService.add<FlushQueuedAgentChatMessagesJobData>(- FLUSH_QUEUED_AGENT_CHAT_MESSAGES_JOB_NAME,- { workspaceId, threadId },- );- }- } catch (error) {- this.logger.error(- `Failed to send the messages queued in thread ${threadId} of workspace ${workspaceId}: ${formatErrorWithCause(error)}`,- );+ // what a member queued while they stopped the turn stays queued; a worker shutdown sends it+ await this.streamRecoveryService.release({+ ...claim,+ turnError,+ shouldFlush: !isStoppedByMember,+ });}}- // ends the turn the claim runs as failed, and tells the conversation's subscribers- async failStream({+ // a turn whose completion fails still records the usage it streamed, as billing does+ async streamTurn<+ TStarted extends { source: AgentTurnStreamSource },+ TCompletion extends AgentTurnCompletion,+ >({claim,- error,+ turnId,+ abortSignal,+ startTurn,+ completeTurn,+ isBackgroundRun = false,}: {claim: AgentChatStreamClaim;- error: unknown;- }): Promise<void> {- await this.streamRecoveryService.failStream({ ...claim, error });+ turnId: string;+ abortSignal: AbortSignal;+ startTurn: (+ writer: UIMessageStreamWriter<ExtendedUIMessage>,+ ) => Promise<TStarted>;+ completeTurn: (+ streamed: AgentTurnStreamResult & { started: TStarted },+ ) => Promise<TCompletion>;+ isBackgroundRun?: boolean;+ }): Promise<{+ streamed: AgentTurnStreamResult & { started: TStarted };+ completion: TCompletion;+ finish: AgentTurnFinish | null;+ }> {+ const { workspaceId, threadId } = claim;+ const streamed = await this.stream({ claim, turnId, startTurn });+ const completion = await completeTurn(streamed).catch(+ async (error: unknown) => {+ await this.turnRecorderService.recordUsage({+ workspaceId,+ threadId,+ turnId,+ usage: streamed.usage,+ });++ throw error;+ },+ );++ const finish = await this.finishTurn({+ claim,+ turnId,+ result: { ...streamed, usage: completion.usage ?? streamed.usage },+ modelId: completion.modelId,+ stopReason: completion.stopReason,+ isWaiting: completion.isWaiting,+ isBackgroundRun,+ });++ // a worker shutting down interrupted the turn, which fails rather than ends as stopped+ if (abortSignal.reason instanceof AiException) {+ throw abortSignal.reason;+ }++ if (isDefined(streamed.streamError)) {+ throw streamed.streamError;+ }++ // the end event follows every chunk, so a subscriber that sees it has the whole reply+ await this.eventPublisherService.publish({+ threadId,+ workspaceId,+ event:+ completion.stopReason === 'workspaceAllowance'+ ? { type: 'credits-exhausted' }+ : {+ type: 'message-persisted',+ messageId: streamed.assistantMessageId,+ },+ });++ return { streamed, completion, finish };}- // a turn that could not start throws its error; otherwise the caller gets what startTurn returned- async stream<TStarted extends { source: AgentTurnStreamSource }>({+ private async stream<TStarted extends { source: AgentTurnStreamSource }>({claim,turnId,startTurn,@@ -359,7 +435,7 @@ export class AgentTurnStreamService {// Saves the reply and ends the turn, unless another stream took the thread over meanwhile. A turn that// waits on a person or an event is recorded as waiting rather than done. A turn whose stream failed// only records its usage: its failure is recorded with the stream's claim- async finishTurn({+ private async finishTurn({claim,turnId,result,@@ -398,6 +474,19 @@ export class AgentTurnStreamService {stopReason,isTextReplyRequired: !isBackgroundRun,});++ if (!isBackgroundRun) {+ this.logTurnWithoutText({+ claim,+ result,+ modelId,+ stopReason,+ hasText,+ isSilent,+ hasAwaitingCall: awaitingParts.length > 0,+ });+ }+const finishTurn = () =>this.turnRecorderService.finish({workspaceId,@@ -490,24 +579,55 @@ export class AgentTurnStreamService {return { outcome, isAwaitingAnswer, threadBefore };}- // the end event follows every chunk, so a subscriber that sees it has the whole reply- async publishTurnEnd({- claim: { workspaceId, threadId },- assistantMessageId,+ private logTurnWithoutText({+ claim: { threadId, workspaceId },+ result: { replyParts, isAborted, streamError, usage },+ modelId,stopReason,+ hasText,+ isSilent,+ hasAwaitingCall,}: {- claim: AgentChatStreamClaim;- assistantMessageId: string;+ claim: Pick<AgentChatStreamClaim, 'threadId' | 'workspaceId'>;+ result: Pick<+ AgentTurnStreamResult,+ 'replyParts' | 'isAborted' | 'streamError' | 'usage'+ >;+ modelId: string;stopReason: StepDebitResult['stopReason'];- }): Promise<void> {- await this.eventPublisherService.publish({- threadId,- workspaceId,- event:- stopReason === 'workspaceAllowance'- ? { type: 'credits-exhausted' }- : { type: 'message-persisted', messageId: assistantMessageId },+ hasText: boolean;+ isSilent: boolean;+ hasAwaitingCall: boolean;+ }): void {+ if ((!isAborted && hasText) || hasAwaitingCall || isSilent) {+ return;+ }++ const reason = getTurnWithoutTextReason({+ isAborted,+ streamError,+ stopReason,});++ const errorDetail = isDefined(streamError)+ ? formatErrorWithCause(streamError)+ : 'none';++ this.logger.warn(+ `[AI_CHAT_NO_TEXT] Assistant turn ended without a text reply — ` ++ `reason=${reason}, model=${modelId}, ` ++ `threadId=${threadId}, workspaceId=${workspaceId}, ` ++ `isAborted=${isAborted}, stopReason=${stopReason}, hasText=${hasText}, ` ++ `streamError=${errorDetail}, ` ++ `inputTokens=${usage.inputTokens},` ++ `responseMessage.parts=${JSON.stringify(replyParts)}`,+ );++ if (streamError instanceof Error && isDefined(streamError.stack)) {+ this.logger.warn(+ `[AI_CHAT_NO_TEXT] streamError stack — threadId=${threadId}: ${streamError.stack}`,+ );+ }}private computeMessageMetadata({diff --git a/packages/twenty-server/src/engine/metadata-modules/ai/ai-tool-call-answer/services/tool-call-answer.service.ts b/packages/twenty-server/src/engine/metadata-modules/ai/ai-tool-call-answer/services/tool-call-answer.service.tsindex 318e1b75aa4..b0570120f49 100644--- a/packages/twenty-server/src/engine/metadata-modules/ai/ai-tool-call-answer/services/tool-call-answer.service.ts+++ b/packages/twenty-server/src/engine/metadata-modules/ai/ai-tool-call-answer/services/tool-call-answer.service.ts@@ -29,6 +29,7 @@ import { AgentChatThreadService } from 'src/engine/metadata-modules/ai/ai-chat/simport { AgentChatTurnPreflightService } from 'src/engine/metadata-modules/ai/ai-chat/services/agent-chat-turn-preflight.service';import { AgentChatService } from 'src/engine/metadata-modules/ai/ai-chat/services/agent-chat.service';import { formatErrorWithCause } from 'src/engine/metadata-modules/ai/ai-chat/utils/format-error-with-cause.util';+import { mapErrorToStreamError } from 'src/engine/metadata-modules/ai/ai-history/utils/map-error-to-stream-error.util';import { AgentHistoryRepository } from 'src/engine/metadata-modules/ai/ai-history/repositories/agent-history-repository';import { AgentConversationReaderService } from 'src/engine/metadata-modules/ai/ai-history/services/agent-conversation-reader.service';import { InjectAgentHistoryRepository } from 'src/engine/metadata-modules/ai/ai-history/repositories/inject-agent-history-repository.decorator';@@ -278,10 +279,11 @@ export class ToolCallAnswerService {error,});} else {- await this.streamRecoveryService.releaseStreamClaim({+ await this.streamRecoveryService.release({threadId,workspaceId,streamId,+ shouldFlush: true,});}@@ -322,12 +324,13 @@ export class ToolCallAnswerService {),);- // a suspended run continues in its own job, and only the last answer resumes a chat+ // a suspended run continues in its own job and takes the queue itself; a chat waits for its last answerif (isDefined(wakeUp) || !isLastAnswer) {- await this.streamRecoveryService.releaseStreamClaim({+ await this.streamRecoveryService.release({threadId,workspaceId,streamId,+ shouldFlush: false,});if (isLastAnswer && isDefined(wakeUp)) {@@ -387,20 +390,22 @@ export class ToolCallAnswerService {error: unknown;}): Promise<void> {if (!isDefined(wakeUp)) {- await this.streamRecoveryService.failStream({+ await this.streamRecoveryService.release({threadId,workspaceId,streamId,- error,+ turnError: mapErrorToStreamError(error),+ shouldFlush: true,});return;}- await this.streamRecoveryService.releaseStreamClaim({+ await this.streamRecoveryService.release({threadId,workspaceId,streamId,+ shouldFlush: false,});await this.pendingWakeUpResolverService.resolve({workspaceId,diff --git a/packages/twenty-server/test/integration/ai/suites/agent-chat-stream-state.integration-spec.ts b/packages/twenty-server/test/integration/ai/suites/agent-chat-stream-state.integration-spec.tsnew file mode 100644index 00000000000..8e47fefef40--- /dev/null+++ b/packages/twenty-server/test/integration/ai/suites/agent-chat-stream-state.integration-spec.ts@@ -0,0 +1,73 @@+import { v4 } from 'uuid';++import { getAppProviderByClassName } from 'test/integration/utils/get-app-provider-by-class-name.util';++import { type AgentChatEventPublisherService } from 'src/engine/metadata-modules/ai/ai-chat/services/agent-chat-event-publisher.service';++describe('Agent chat stream state', () => {+ let eventPublisherService: AgentChatEventPublisherService;++ const workspaceId = v4();++ const publishChunk = (threadId: string, text: string) =>+ eventPublisherService.publish({+ threadId,+ workspaceId,+ event: { type: 'stream-chunk', chunk: { type: 'text-delta', text } },+ });++ beforeAll(() => {+ eventPublisherService =+ getAppProviderByClassName<AgentChatEventPublisherService>(+ 'AgentChatEventPublisherService',+ );+ });++ it('resets the chunks of the stream that failed', async () => {+ const threadId = v4();+ const streamId = v4();++ await eventPublisherService.startStreamState({ threadId, streamId });+ await publishChunk(threadId, 'partial');++ await eventPublisherService.resetOwnedStreamState({ threadId, streamId });++ const { chunks } =+ await eventPublisherService.getAccumulatedChunks(threadId);++ expect(chunks).toEqual([]);+ });++ it('keeps the chunks of a stream that started after a dead one was released', async () => {+ const threadId = v4();+ const deadStreamId = v4();+ const nextStreamId = v4();++ await eventPublisherService.startStreamState({+ threadId,+ streamId: deadStreamId,+ });+ await publishChunk(threadId, 'dead');+ await eventPublisherService.resetOwnedStreamState({+ threadId,+ streamId: deadStreamId,+ });++ await eventPublisherService.startStreamState({+ threadId,+ streamId: nextStreamId,+ });+ await publishChunk(threadId, 'next');++ await eventPublisherService.resetOwnedStreamState({+ threadId,+ streamId: deadStreamId,+ });++ const { chunks, maxSeq } =+ await eventPublisherService.getAccumulatedChunks(threadId);++ expect(chunks).toEqual([{ type: 'text-delta', text: 'next' }]);+ expect(maxSeq).toBe(1);+ });+});