The open alternative to Salesforce, designed for AI.
TypeScript crmcrm-systemcustomergood-first-issuegraphqlhacktoberfestjavascriptmarketingmonoreponestjsopen-sourcepostgresqlreactreactjssalestypescriptweb
Agent engine: runs wait for a busy conversation in one ordered line (#28506)
Félix Malfaitfbcf898c0ab5
diff --git a/packages/twenty-server/src/engine/core-modules/pending-wake-up/services/pending-wake-up.service.ts b/packages/twenty-server/src/engine/core-modules/pending-wake-up/services/pending-wake-up.service.tsindex 4ccdc3f8522..fbfaa8dd985 100644--- a/packages/twenty-server/src/engine/core-modules/pending-wake-up/services/pending-wake-up.service.ts+++ b/packages/twenty-server/src/engine/core-modules/pending-wake-up/services/pending-wake-up.service.ts@@ -89,6 +89,33 @@ export class PendingWakeUpService {});}+ // unlike arm, a wake-up claimed meanwhile stays claimed, and one still pending keeps its createdAt+ async rearm({+ wakeUp,+ resumeAt,+ }: {+ wakeUp: ScheduledWakeUp;+ resumeAt: Date;+ }): Promise<void> {+ const wakeUpId = v4();+ const { affected } = await this.pendingWakeUpRepository.update(+ wakeUp.workspaceId,+ { id: wakeUp.id },+ {+ id: wakeUpId,+ condition: { type: 'TIME', resumeAt: resumeAt.toISOString() },+ resumeAt,+ },+ );++ if (affected) {+ await this.scheduleResolution({+ wakeUp: { ...wakeUp, id: wakeUpId },+ delayMs: resumeAt.getTime() - Date.now(),+ });+ }+ }+async scheduleResolution({wakeUp,event,@@ -139,6 +166,19 @@ export class PendingWakeUpService {});}+ async findOldest({+ workspaceId,+ owner: { type, id },+ }: {+ workspaceId: string;+ owner: Omit<PendingWakeUpOwner, 'key'>;+ }): Promise<PendingWakeUpEntity | null> {+ return this.pendingWakeUpRepository.findOne(workspaceId, {+ where: { ownerType: type, ownerId: id },+ order: { createdAt: 'ASC' },+ });+ }+async claim({workspaceId,wakeUpId,diff --git a/packages/twenty-server/src/engine/metadata-modules/ai/ai-agent-execution/services/agent-runner.service.ts b/packages/twenty-server/src/engine/metadata-modules/ai/ai-agent-execution/services/agent-runner.service.tsindex 6bbcbf84ba7..97233e4ba18 100644--- a/packages/twenty-server/src/engine/metadata-modules/ai/ai-agent-execution/services/agent-runner.service.ts+++ b/packages/twenty-server/src/engine/metadata-modules/ai/ai-agent-execution/services/agent-runner.service.ts@@ -33,12 +33,12 @@ import { type RunAgentJobData } from 'src/engine/metadata-modules/ai/ai-agent-eximport { buildAgentRunSummary } from 'src/engine/metadata-modules/ai/ai-agent-execution/utils/build-agent-run-summary.util';import { sumAgentRunSummaries } from 'src/engine/metadata-modules/ai/ai-agent-execution/utils/sum-agent-run-summaries.util';import { AgentEntity } from 'src/engine/metadata-modules/ai/ai-agent/entities/agent.entity';-import { AgentChatParkedRunsService } from 'src/engine/metadata-modules/ai/ai-chat/services/agent-chat-parked-runs.service';import { AgentChatEventPublisherService } from 'src/engine/metadata-modules/ai/ai-chat/services/agent-chat-event-publisher.service';import {type AgentChatStreamClaim,AgentChatStreamRecoveryService,} from 'src/engine/metadata-modules/ai/ai-chat/services/agent-chat-stream-recovery.service';+import { AgentChatWaitingRunService } from 'src/engine/metadata-modules/ai/ai-chat/services/agent-chat-waiting-run.service';import { AgentTurnStreamService } from 'src/engine/metadata-modules/ai/ai-chat/services/agent-turn-stream.service';import { mapErrorToStreamError } from 'src/engine/metadata-modules/ai/ai-history/utils/map-error-to-stream-error.util';import { AgentConversationReaderService } from 'src/engine/metadata-modules/ai/ai-history/services/agent-conversation-reader.service';@@ -79,7 +79,7 @@ export class AgentRunnerService {private readonly turnStreamService: AgentTurnStreamService,private readonly streamRecoveryService: AgentChatStreamRecoveryService,private readonly eventPublisherService: AgentChatEventPublisherService,- private readonly parkedRunsService: AgentChatParkedRunsService,+ private readonly waitingRunService: AgentChatWaitingRunService,@InjectMessageQueue(MessageQueue.aiQueue)private readonly messageQueueService: MessageQueueService,@InjectWorkspaceScopedRepository(AgentEntity)@@ -97,7 +97,7 @@ export class AgentRunnerService {const { workspaceId, conversation, caller, parkId } = jobData;if (- !(await this.parkedRunsService.shouldRunParkedJob({+ !(await this.waitingRunService.shouldRunLegacyParkedCopy({threadId: conversation.threadId,parkId,}))@@ -130,11 +130,13 @@ export class AgentRunnerService {}if (result === CONVERSATION_BUSY) {- await this.parkedRunsService.park({+ await this.waitingRunService.wait({workspaceId,threadId: conversation.threadId,- jobName: RUN_AGENT_JOB_NAME,- jobData,+ run: {+ jobName: RUN_AGENT_JOB_NAME,+ jobData: { ...jobData, parkId: undefined },+ },});return;@@ -224,7 +226,10 @@ export class AgentRunnerService {const { workspaceId, threadId, wakeUpId, outcome, parkId } = jobData;if (- !(await this.parkedRunsService.shouldRunParkedJob({ threadId, parkId }))+ !(await this.waitingRunService.shouldRunLegacyParkedCopy({+ threadId,+ parkId,+ }))) {return;}@@ -270,11 +275,13 @@ export class AgentRunnerService {});if (continued === CONVERSATION_BUSY) {- await this.parkedRunsService.park({+ await this.waitingRunService.wait({workspaceId,threadId,- jobName: CONTINUE_AGENT_RUN_JOB_NAME,- jobData,+ run: {+ jobName: CONTINUE_AGENT_RUN_JOB_NAME,+ jobData: { ...jobData, parkId: undefined },+ },});return;diff --git a/packages/twenty-server/src/engine/metadata-modules/ai/ai-agent-execution/types/continue-agent-run-job-data.type.ts b/packages/twenty-server/src/engine/metadata-modules/ai/ai-agent-execution/types/continue-agent-run-job-data.type.tsindex 197cc3da1ba..b094c1a2c45 100644--- a/packages/twenty-server/src/engine/metadata-modules/ai/ai-agent-execution/types/continue-agent-run-job-data.type.ts+++ b/packages/twenty-server/src/engine/metadata-modules/ai/ai-agent-execution/types/continue-agent-run-job-data.type.ts@@ -6,6 +6,6 @@ export type ContinueAgentRunJobData = {// the run's wake-up, which the continuation removes once the run went on, and what resolved itwakeUpId: string;outcome: AgentRunWakeUpOutcome;- // set on the delayed copy of a run parked behind a busy conversation+ // only on a delayed copy parked in Redis by the previous releaseparkId?: string;};diff --git a/packages/twenty-server/src/engine/metadata-modules/ai/ai-agent-execution/types/run-agent-job-data.type.ts b/packages/twenty-server/src/engine/metadata-modules/ai/ai-agent-execution/types/run-agent-job-data.type.tsindex 2e03c52478a..825049dc0f3 100644--- a/packages/twenty-server/src/engine/metadata-modules/ai/ai-agent-execution/types/run-agent-job-data.type.ts+++ b/packages/twenty-server/src/engine/metadata-modules/ai/ai-agent-execution/types/run-agent-job-data.type.ts@@ -5,5 +5,6 @@ export type RunAgentJobData = Omit<AgentRunnerRunInput,'agent' | 'executionContext'> & {+ // only on a delayed copy parked in Redis by the previous releaseparkId?: string;};diff --git a/packages/twenty-server/src/engine/metadata-modules/ai/ai-chat/agent-chat-stream-state.module.ts b/packages/twenty-server/src/engine/metadata-modules/ai/ai-chat/agent-chat-stream-state.module.tsindex 63f0280ebb0..6a0d038b8e9 100644--- a/packages/twenty-server/src/engine/metadata-modules/ai/ai-chat/agent-chat-stream-state.module.ts+++ b/packages/twenty-server/src/engine/metadata-modules/ai/ai-chat/agent-chat-stream-state.module.ts@@ -1,30 +1,36 @@import { Module } from '@nestjs/common';import { MetricsModule } from 'src/engine/core-modules/metrics/metrics.module';+import { PendingWakeUpModule } from 'src/engine/core-modules/pending-wake-up/pending-wake-up.module';import { AgentChatThreadModule } from 'src/engine/metadata-modules/ai/ai-chat/agent-chat-thread.module';import { AgentChatCancelSubscriberService } from 'src/engine/metadata-modules/ai/ai-chat/services/agent-chat-cancel-subscriber.service';import { AgentChatEventPublisherService } from 'src/engine/metadata-modules/ai/ai-chat/services/agent-chat-event-publisher.service';-import { AgentChatParkedRunsService } from 'src/engine/metadata-modules/ai/ai-chat/services/agent-chat-parked-runs.service';import { AgentChatStreamHeartbeatService } from 'src/engine/metadata-modules/ai/ai-chat/services/agent-chat-stream-heartbeat.service';import { AgentChatStreamRecoveryService } from 'src/engine/metadata-modules/ai/ai-chat/services/agent-chat-stream-recovery.service';+import { AgentChatWaitingRunService } from 'src/engine/metadata-modules/ai/ai-chat/services/agent-chat-waiting-run.service';import { AgentTurnStreamService } from 'src/engine/metadata-modules/ai/ai-chat/services/agent-turn-stream.service';import { AgentHistoryModule } from 'src/engine/metadata-modules/ai/ai-history/ai-history.module';@Module({- imports: [AgentChatThreadModule, AgentHistoryModule, MetricsModule],+ imports: [+ AgentChatThreadModule,+ AgentHistoryModule,+ MetricsModule,+ PendingWakeUpModule,+ ],providers: [AgentChatCancelSubscriberService,AgentChatEventPublisherService,- AgentChatParkedRunsService,AgentChatStreamHeartbeatService,AgentChatStreamRecoveryService,+ AgentChatWaitingRunService,AgentTurnStreamService,],exports: [AgentChatEventPublisherService,- AgentChatParkedRunsService,AgentChatStreamHeartbeatService,AgentChatStreamRecoveryService,+ AgentChatWaitingRunService,AgentTurnStreamService,],})diff --git a/packages/twenty-server/src/engine/metadata-modules/ai/ai-chat/constants/agent-chat-parked-run-fallback-delay-ms.constant.ts b/packages/twenty-server/src/engine/metadata-modules/ai/ai-chat/constants/agent-chat-parked-run-fallback-delay-ms.constant.tsdeleted file mode 100644index 6d42f996ebf..00000000000--- a/packages/twenty-server/src/engine/metadata-modules/ai/ai-chat/constants/agent-chat-parked-run-fallback-delay-ms.constant.ts+++ /dev/null@@ -1 +0,0 @@-export const AGENT_CHAT_PARKED_RUN_FALLBACK_DELAY_MS = 60_000;diff --git a/packages/twenty-server/src/engine/metadata-modules/ai/ai-chat/constants/agent-chat-waiting-run-owner-type.constant.ts b/packages/twenty-server/src/engine/metadata-modules/ai/ai-chat/constants/agent-chat-waiting-run-owner-type.constant.tsnew file mode 100644index 00000000000..07b899501b5--- /dev/null+++ b/packages/twenty-server/src/engine/metadata-modules/ai/ai-chat/constants/agent-chat-waiting-run-owner-type.constant.ts@@ -0,0 +1 @@+export const AGENT_CHAT_WAITING_RUN_OWNER_TYPE = 'AGENT_CHAT_WAITING_RUN';diff --git a/packages/twenty-server/src/engine/metadata-modules/ai/ai-chat/constants/agent-chat-waiting-run-recheck-delay-ms.constant.ts b/packages/twenty-server/src/engine/metadata-modules/ai/ai-chat/constants/agent-chat-waiting-run-recheck-delay-ms.constant.tsnew file mode 100644index 00000000000..e0d8f637be9--- /dev/null+++ b/packages/twenty-server/src/engine/metadata-modules/ai/ai-chat/constants/agent-chat-waiting-run-recheck-delay-ms.constant.ts@@ -0,0 +1 @@+export const AGENT_CHAT_WAITING_RUN_RECHECK_DELAY_MS = 60_000;diff --git a/packages/twenty-server/src/engine/metadata-modules/ai/ai-chat/services/agent-chat-parked-runs.service.ts b/packages/twenty-server/src/engine/metadata-modules/ai/ai-chat/services/agent-chat-parked-runs.service.tsdeleted file mode 100644index 63744bf1257..00000000000--- a/packages/twenty-server/src/engine/metadata-modules/ai/ai-chat/services/agent-chat-parked-runs.service.ts+++ /dev/null@@ -1,136 +0,0 @@-import { Injectable, Logger } from '@nestjs/common';--import { isNonEmptyString } from '@sniptt/guards';-import { isDefined } from 'twenty-shared/utils';-import { v4 } from 'uuid';--import { InjectMessageQueue } from 'src/engine/core-modules/message-queue/decorators/message-queue.decorator';-import { MessageQueue } from 'src/engine/core-modules/message-queue/message-queue.constants';-import { MessageQueueService } from 'src/engine/core-modules/message-queue/services/message-queue.service';-import { RedisClientService } from 'src/engine/core-modules/redis-client/redis-client.service';-import { AGENT_CHAT_PARKED_RUN_FALLBACK_DELAY_MS } from 'src/engine/metadata-modules/ai/ai-chat/constants/agent-chat-parked-run-fallback-delay-ms.constant';-import { formatErrorWithCause } from 'src/engine/metadata-modules/ai/ai-chat/utils/format-error-with-cause.util';-import { AgentHistoryRepository } from 'src/engine/metadata-modules/ai/ai-history/repositories/agent-history-repository';-import { InjectAgentHistoryRepository } from 'src/engine/metadata-modules/ai/ai-history/repositories/inject-agent-history-repository.decorator';-import { type AgentChatThreadWorkspaceEntity } from 'src/engine/metadata-modules/ai/ai-history/standard-objects/agent-chat-thread.workspace-entity';--type ParkedRun = {- jobName: string;- jobData: object;-};--// the delayed copy of each parked run covers a claim whose worker died, which nothing else releases-@Injectable()-export class AgentChatParkedRunsService {- private readonly logger = new Logger(AgentChatParkedRunsService.name);-- constructor(- @InjectAgentHistoryRepository('agentChatThread')- private readonly threadRepository: AgentHistoryRepository<AgentChatThreadWorkspaceEntity>,- @InjectMessageQueue(MessageQueue.aiQueue)- private readonly messageQueueService: MessageQueueService,- private readonly redisClientService: RedisClientService,- ) {}-- private getKey(threadId: string): string {- return `agent-chat-parked-runs:${threadId}`;- }-- async park<TJobData extends { parkId?: string }>({- workspaceId,- threadId,- jobName,- jobData: { parkId: _previousParkId, ...jobData },- }: {- workspaceId: string;- threadId: string;- jobName: string;- jobData: TJobData;- }): Promise<void> {- const parkId = v4();- const parkedRun: ParkedRun = { jobName, jobData };-- await this.redisClientService- .getClient()- .hset(this.getKey(threadId), parkId, JSON.stringify(parkedRun));-- await this.messageQueueService.add<TJobData>(- jobName,- { ...jobData, parkId } as TJobData,- { delay: AGENT_CHAT_PARKED_RUN_FALLBACK_DELAY_MS },- );-- // the conversation may have been released between the failed claim and the park- const thread = await this.threadRepository.findOne(workspaceId, {- where: { id: threadId },- select: ['id', 'activeStreamId'],- });-- if (!isNonEmptyString(thread?.activeStreamId)) {- await this.wake({ workspaceId, threadId });- }- }-- // whether the job goes on: the delayed copy of a parked run does only when nothing woke the run first- async shouldRunParkedJob({- threadId,- parkId,- }: {- threadId: string;- parkId?: string;- }): Promise<boolean> {- if (!isDefined(parkId)) {- return true;- }-- const removedCount = await this.redisClientService- .getClient()- .hdel(this.getKey(threadId), parkId);-- return removedCount === 1;- }-- async wake({- workspaceId,- threadId,- }: {- workspaceId: string;- threadId: string;- }): Promise<void> {- const parkedRunsHashKey = this.getKey(threadId);- const results = await this.redisClientService- .getClient()- .multi()- .hgetall(parkedRunsHashKey)- .del(parkedRunsHashKey)- .exec()- .catch((error: unknown) => {- this.logger.error(- `Failed to read the runs parked on thread ${threadId} in workspace ${workspaceId}: ${formatErrorWithCause(error)}`,- );-- return null;- });- const parkedRuns = (results?.[0]?.[1] ?? {}) as Record<string, string>;-- for (const [parkId, parkedRun] of Object.entries(parkedRuns)) {- const { jobName, jobData } = JSON.parse(parkedRun) as ParkedRun;-- try {- await this.messageQueueService.add(jobName, jobData);- } catch (error) {- this.logger.error(- `Failed to wake a run parked on thread ${threadId} in workspace ${workspaceId}: ${formatErrorWithCause(error)}`,- );- await this.redisClientService- .getClient()- .hset(parkedRunsHashKey, parkId, parkedRun)- .catch((reParkError: unknown) =>- this.logger.error(- `Failed to park again a run on thread ${threadId} in workspace ${workspaceId}, only its delayed copy will run it: ${formatErrorWithCause(reParkError)}`,- ),- );- }- }- }-}diff --git a/packages/twenty-server/src/engine/metadata-modules/ai/ai-chat/services/agent-chat-stream-recovery.service.ts b/packages/twenty-server/src/engine/metadata-modules/ai/ai-chat/services/agent-chat-stream-recovery.service.tsindex ab7a0328498..6eac4fa9c56 100644--- a/packages/twenty-server/src/engine/metadata-modules/ai/ai-chat/services/agent-chat-stream-recovery.service.ts+++ b/packages/twenty-server/src/engine/metadata-modules/ai/ai-chat/services/agent-chat-stream-recovery.service.ts@@ -5,15 +5,16 @@ import { isDefined, isNonEmptyString } from 'twenty-shared/utils';import { InjectMessageQueue } from 'src/engine/core-modules/message-queue/decorators/message-queue.decorator';import { MessageQueue } from 'src/engine/core-modules/message-queue/message-queue.constants';import { MessageQueueService } from 'src/engine/core-modules/message-queue/services/message-queue.service';+import { PendingWakeUpService } from 'src/engine/core-modules/pending-wake-up/services/pending-wake-up.service';import { MetricsService } from 'src/engine/core-modules/metrics/metrics.service';import { MetricsKeys } from 'src/engine/core-modules/metrics/types/metrics-keys.type';import { AgentChatThreadWorkspaceEntity } from 'src/engine/metadata-modules/ai/ai-history/standard-objects/agent-chat-thread.workspace-entity';import { type AgentMessageWorkspaceEntity } from 'src/engine/metadata-modules/ai/ai-history/standard-objects/agent-message.workspace-entity';import { AgentMessageStatus } from 'src/engine/metadata-modules/ai/ai-history/enums/agent-message-status.enum';+import { AGENT_CHAT_WAITING_RUN_OWNER_TYPE } from 'src/engine/metadata-modules/ai/ai-chat/constants/agent-chat-waiting-run-owner-type.constant';import { FLUSH_QUEUED_AGENT_CHAT_MESSAGES_JOB_NAME } from 'src/engine/metadata-modules/ai/ai-chat/jobs/flush-queued-agent-chat-messages-job-name.constant';import { type FlushQueuedAgentChatMessagesJobData } from 'src/engine/metadata-modules/ai/ai-chat/jobs/flush-queued-agent-chat-messages-job.types';import { AgentChatEventPublisherService } from 'src/engine/metadata-modules/ai/ai-chat/services/agent-chat-event-publisher.service';-import { AgentChatParkedRunsService } from 'src/engine/metadata-modules/ai/ai-chat/services/agent-chat-parked-runs.service';import { AgentChatStreamHeartbeatService } from 'src/engine/metadata-modules/ai/ai-chat/services/agent-chat-stream-heartbeat.service';import { formatErrorWithCause } from 'src/engine/metadata-modules/ai/ai-chat/utils/format-error-with-cause.util';import { type StreamErrorPayload } from 'src/engine/metadata-modules/ai/ai-history/utils/map-error-to-stream-error.util';@@ -42,7 +43,7 @@ export class AgentChatStreamRecoveryService {@InjectMessageQueue(MessageQueue.aiStreamQueue)private readonly chatMessageQueueService: MessageQueueService,private readonly streamHeartbeatService: AgentChatStreamHeartbeatService,- private readonly parkedRunsService: AgentChatParkedRunsService,+ private readonly pendingWakeUpService: PendingWakeUpService,private readonly eventPublisherService: AgentChatEventPublisherService,private readonly metricsService: MetricsService,private readonly turnRecorderService: AgentTurnRecorderService,@@ -94,11 +95,7 @@ export class AgentChatStreamRecoveryService {await this.publishStreamError({ threadId, workspaceId, turnError });}- if (shouldFlush) {- await this.flushQueuedMessages({ threadId, workspaceId });- }-- await this.parkedRunsService.wake({ workspaceId, threadId });+ await this.wakeNext({ threadId, workspaceId, shouldFlush });return true;}@@ -176,28 +173,45 @@ export class AgentChatStreamRecoveryService {return interruptedError;}- private async flushQueuedMessages({+ // the oldest queued message is the queue's place in line, so it and the runs go in the order they came+ async wakeNext({threadId,workspaceId,+ shouldFlush,}: {threadId: string;workspaceId: string;+ shouldFlush: boolean;}): Promise<void> {try {- const hasQueuedMessages = await this.messageRepository.existsBy(- workspaceId,- { threadId, status: AgentMessageStatus.QUEUED },- );-- if (hasQueuedMessages) {+ const [nextRun, oldestQueuedMessage] = await Promise.all([+ this.pendingWakeUpService.findOldest({+ workspaceId,+ owner: { type: AGENT_CHAT_WAITING_RUN_OWNER_TYPE, id: threadId },+ }),+ shouldFlush+ ? this.messageRepository.findOne(workspaceId, {+ where: { threadId, status: AgentMessageStatus.QUEUED },+ order: { createdAt: 'ASC' },+ })+ : null,+ ]);++ if (+ isDefined(oldestQueuedMessage) &&+ (!isDefined(nextRun) ||+ new Date(oldestQueuedMessage.createdAt) <= nextRun.createdAt)+ ) {await this.chatMessageQueueService.add<FlushQueuedAgentChatMessagesJobData>(FLUSH_QUEUED_AGENT_CHAT_MESSAGES_JOB_NAME,{ workspaceId, threadId },);+ } else if (isDefined(nextRun)) {+ await this.pendingWakeUpService.scheduleResolution({ wakeUp: nextRun });}} catch (error) {this.logger.error(- `Failed to send the messages queued in thread ${threadId} of workspace ${workspaceId}: ${formatErrorWithCause(error)}`,+ `Failed to wake what waits on thread ${threadId} of workspace ${workspaceId}: ${formatErrorWithCause(error)}`,);}}diff --git a/packages/twenty-server/src/engine/metadata-modules/ai/ai-chat/services/agent-chat-streaming.service.ts b/packages/twenty-server/src/engine/metadata-modules/ai/ai-chat/services/agent-chat-streaming.service.tsindex 0911bcaec1e..a57afbbf391 100644--- a/packages/twenty-server/src/engine/metadata-modules/ai/ai-chat/services/agent-chat-streaming.service.ts+++ b/packages/twenty-server/src/engine/metadata-modules/ai/ai-chat/services/agent-chat-streaming.service.ts@@ -3,10 +3,7 @@ import { AgentChatQueueService } from 'src/engine/metadata-modules/ai/ai-chat/seimport { AuthException } from 'src/engine/core-modules/auth/auth.exception';import { PermissionsException } from 'src/engine/metadata-modules/permissions/permissions.exception';import { AgentChatActorService } from 'src/engine/metadata-modules/ai/ai-chat/services/agent-chat-actor.service';-import {- type AgentChatStreamClaim,- AgentChatStreamRecoveryService,-} from 'src/engine/metadata-modules/ai/ai-chat/services/agent-chat-stream-recovery.service';+import { AgentChatStreamRecoveryService } from 'src/engine/metadata-modules/ai/ai-chat/services/agent-chat-stream-recovery.service';import { InjectAgentHistoryRepository } from 'src/engine/metadata-modules/ai/ai-history/repositories/inject-agent-history-repository.decorator';import { AgentHistoryRepository } from 'src/engine/metadata-modules/ai/ai-history/repositories/agent-history-repository';import { Injectable, Logger } from '@nestjs/common';@@ -37,7 +34,6 @@ import { AgentChatThreadWorkspaceEntity } from 'src/engine/metadata-modules/ai/aimport { STREAM_AGENT_CHAT_JOB_NAME } from 'src/engine/metadata-modules/ai/ai-chat/jobs/stream-agent-chat-job-name.constant';import { type StreamAgentChatJobData } from 'src/engine/metadata-modules/ai/ai-chat/jobs/stream-agent-chat-job.types';import { AgentChatEventPublisherService } from 'src/engine/metadata-modules/ai/ai-chat/services/agent-chat-event-publisher.service';-import { AgentChatStreamHeartbeatService } from 'src/engine/metadata-modules/ai/ai-chat/services/agent-chat-stream-heartbeat.service';import { AgentChatService } from 'src/engine/metadata-modules/ai/ai-chat/services/agent-chat.service';import { AgentChatThreadService } from 'src/engine/metadata-modules/ai/ai-chat/services/agent-chat-thread.service';import { AgentConversationReaderService } from 'src/engine/metadata-modules/ai/ai-history/services/agent-conversation-reader.service';@@ -54,22 +50,20 @@ import { WorkspaceScopedRepository } from 'src/engine/twenty-orm/workspace-scopeimport { AgentTurnStatus } from 'src/engine/metadata-modules/ai/ai-history/enums/agent-turn-status.enum';import { AgentTurnRecorderService } from 'src/engine/metadata-modules/ai/ai-history/services/agent-turn-recorder.service';-type StreamAgentChatOptions = {+type MemberMessageOptions = {thread: AgentChatThreadWorkspaceEntity;userWorkspaceId: string;workspaceMemberId: string;workspace: WorkspaceEntity;text: string;- browsingContext: BrowsingContextType | null;- modelId?: string;messageId?: string;fileAttachments?: AiChatFileAttachment[];};-type StreamJobThread = Pick<- AgentChatThreadWorkspaceEntity,- 'id' | 'title' | 'conversationSize'->;+type StreamAgentChatOptions = MemberMessageOptions & {+ browsingContext: BrowsingContextType | null;+ modelId?: string;+};@Injectable()export class AgentChatStreamingService {@@ -87,7 +81,6 @@ export class AgentChatStreamingService {private readonly threadService: AgentChatThreadService,private readonly eventPublisherService: AgentChatEventPublisherService,private readonly conversationReaderService: AgentConversationReaderService,- private readonly streamHeartbeatService: AgentChatStreamHeartbeatService,private readonly metricsService: MetricsService,private readonly streamRecoveryService: AgentChatStreamRecoveryService,private readonly actorService: AgentChatActorService,@@ -97,20 +90,23 @@ export class AgentChatStreamingService {) {}// a message sent while a reply or a run streams in the conversation waits in its queue- async streamAgentChat({- thread,- userWorkspaceId,- workspaceMemberId,- workspace,- text,- browsingContext,- modelId,- messageId,- fileAttachments,- }: StreamAgentChatOptions): Promise<+ async streamAgentChat(+ options: StreamAgentChatOptions,+ ): Promise<| { queued: false; streamId: string; messageId: string; turnId: string }| { queued: true; messageId: string }> {+ const {+ thread,+ userWorkspaceId,+ workspaceMemberId,+ workspace,+ text,+ browsingContext,+ modelId,+ messageId,+ fileAttachments,+ } = options;const threadId = thread.id;const workspaceId = workspace.id;@@ -129,151 +125,74 @@ export class AgentChatStreamingService {hasPostedCall,});- const streamId = generateId();+ const started =+ hasQueuedBacklog || isDefined(suspendedRun)+ ? null+ : await this.startClaimedTurn(+ { threadId, workspaceId, modelId },+ async (streamId) => {+ const savedUserMessage = await this.addMemberMessage({+ ...options,+ eventType: 'message-persisted',+ });++ await this.enqueueStreamJob({+ threadId,+ workspaceId,+ workspaceMemberId,+ streamId,+ userWorkspaceId,+ browsingContext,+ modelId,+ turnId: savedUserMessage.turnId,+ messageId: savedUserMessage.id,+ });++ return {+ queued: false as const,+ streamId,+ messageId: savedUserMessage.id,+ turnId: savedUserMessage.turnId,+ };+ },+ );++ if (isDefined(started)) {+ return started;+ }- const claimed =- !hasQueuedBacklog &&- !isDefined(suspendedRun) &&- (await this.streamRecoveryService.tryClaimStream({- threadId,- workspaceId,- streamId,- }));+ const queuedMessage = await this.agentChatQueueService.queueMessage({+ threadId,+ text,+ id: messageId,+ fileAttachments,+ workspaceId,+ userWorkspaceId,+ workspaceMemberId,+ });- if (!claimed) {- const queuedMessage = await this.agentChatQueueService.queueMessage({- threadId,- text,- id: messageId,- fileAttachments,+ if (isDefined(suspendedRun)) {+ await this.agentRunSuspensionService.scheduleContinuation({workspaceId,- userWorkspaceId,- workspaceMemberId,+ threadId,+ wakeUpId: suspendedRun.id,+ outcome: { type: 'MESSAGE_RECEIVED' },});-- if (isDefined(suspendedRun)) {- await this.agentRunSuspensionService.scheduleContinuation({- workspaceId,- threadId,- wakeUpId: suspendedRun.id,- outcome: { type: 'MESSAGE_RECEIVED' },- });- } else {- // the stream may have let go just before this message was queued, finding nothing to send- await this.flushNextQueuedMessage({ threadId, workspaceId });- }-- return { queued: true, messageId: queuedMessage.id };+ } else {+ // the stream may have let go just before this message was queued, finding nothing to send+ await this.flushNextQueuedMessage({ threadId, workspaceId });}- return this.releaseClaimOnEnqueueFailure(- { threadId, workspaceId, streamId, modelId },- async () => {- const fileParts = await this.buildFilePartsFromAttachments(- fileAttachments,- workspaceId,- );-- const savedUserMessage = await this.agentChatService.addMessage({- threadId,- userWorkspaceId,- id: messageId,- uiMessage: {- role: AgentMessageRole.USER,- parts: [{ type: 'text' as const, text }, ...fileParts],- },- workspaceId,- });-- await this.eventPublisherService.publish({- workspaceId,- threadId,- event: { type: 'message-persisted', messageId: savedUserMessage.id },- });-- await this.threadService.notifyThreadActivityUpdated({- threadId,- workspaceMemberId,- workspaceId,- text,- });-- await this.enqueueStreamJob({- thread,- streamId,- userWorkspaceId,- workspaceId,- messages: await this.loadMessagesFromDB(- threadId,- workspaceId,- workspaceMemberId,- ),- browsingContext,- modelId,- lastUserMessageText: text,- turnId: savedUserMessage.turnId,- messageId: savedUserMessage.id,- });-- return {- queued: false as const,- streamId,- messageId: savedUserMessage.id,- turnId: savedUserMessage.turnId,- };- },- );+ return { queued: true, messageId: queuedMessage.id };}- async saveMessageWithoutReply({- thread,- userWorkspaceId,- workspaceMemberId,- workspace,- text,- messageId,- fileAttachments,- }: Pick<- StreamAgentChatOptions,- | 'thread'- | 'userWorkspaceId'- | 'workspaceMemberId'- | 'workspace'- | 'text'- | 'messageId'- | 'fileAttachments'- >): Promise<{ messageId: string }> {- const threadId = thread.id;- const workspaceId = workspace.id;-- const fileParts = await this.buildFilePartsFromAttachments(- fileAttachments,- workspaceId,- );-- const savedMessage = await this.agentChatService.addMessage({- threadId,- userWorkspaceId,- id: messageId,- uiMessage: {- role: AgentMessageRole.USER,- parts: [{ type: 'text' as const, text }, ...fileParts],- },+ async saveMessageWithoutReply(+ options: MemberMessageOptions,+ ): Promise<{ messageId: string }> {+ const savedMessage = await this.addMemberMessage({+ ...options,turnStatus: AgentTurnStatus.COMPLETED,- workspaceId,- });-- await this.eventPublisherService.publish({- workspaceId,- threadId,- event: { type: 'message-added', messageId: savedMessage.id },- });-- await this.threadService.notifyThreadActivityUpdated({- threadId,- workspaceMemberId,- workspaceId,- text,+ eventType: 'message-added',});return { messageId: savedMessage.id };@@ -297,31 +216,13 @@ export class AgentChatStreamingService {}): Promise<{ streamId: string; turnId: string } | null> {const threadId = thread.id;const workspaceId = workspace.id;- const streamId = generateId();- const claimed = await this.streamRecoveryService.tryClaimStream({- threadId,- workspaceId,- streamId,- });-- if (!claimed) {- return null;- }-- return this.releaseClaimOnEnqueueFailure(- { threadId, workspaceId, streamId, modelId },- async () => {+ return this.startClaimedTurn(+ { threadId, workspaceId, modelId },+ async (streamId) => {if (await this.agentChatService.hasMessages({ threadId, workspaceId })) {- await this.streamRecoveryService.release({- threadId,- workspaceId,- streamId,- shouldFlush: true,- });-return null;}@@ -332,18 +233,12 @@ export class AgentChatStreamingService {});await this.enqueueStreamJob({- thread,+ threadId,+ workspaceId,+ workspaceMemberId,streamId,userWorkspaceId,- workspaceId,- messages: await this.loadMessagesFromDB(- threadId,- workspaceId,- workspaceMemberId,- ),- browsingContext: null,modelId,- lastUserMessageText: '',turnId,});@@ -393,21 +288,9 @@ export class AgentChatStreamingService {userWorkspaceId,});- const streamId = generateId();-- const claimed = await this.streamRecoveryService.tryClaimStream({- threadId,- workspaceId,- streamId,- });-- if (!claimed) {- throw this.noFailedTurnToRetry();- }-- return this.releaseClaimOnEnqueueFailure(- { threadId, workspaceId, streamId, modelId },- async () => {+ const started = await this.startClaimedTurn(+ { threadId, workspaceId, modelId },+ async (streamId) => {// the latest turn may have changed while claiming, and its output is not ours to deleteconst claimedTurn = await this.turnRecorderService.findLatestTurn({threadId,@@ -436,25 +319,13 @@ export class AgentChatStreamingService {workspaceId,});- const messages = await this.loadMessagesFromDB(+ await this.enqueueStreamJob({threadId,workspaceId,workspaceMemberId,- );-- const textPart = messages- .find(({ id }) => id === message?.id)- ?.parts.find((part): part is TextUIPart => part.type === 'text');-- await this.enqueueStreamJob({- thread,streamId,userWorkspaceId,- workspaceId,- messages,- browsingContext: null,modelId,- lastUserMessageText: textPart?.text ?? '',turnId,messageId: message?.id,});@@ -462,47 +333,12 @@ export class AgentChatStreamingService {return { streamId, messageId: message?.id ?? null, turnId };},);- }- async enqueueResumeStream({- threadId,- userWorkspaceId,- workspaceMemberId,- workspace,- turnId,- streamId,- modelId,- messageId,- }: {- messageId: string;- threadId: string;- userWorkspaceId: string;- workspaceMemberId: string;- workspace: WorkspaceEntity;- turnId: string;- streamId: string;- modelId?: string;- }): Promise<void> {- const thread = await this.threadRepository.findOneOrFail(workspace.id, {- where: { id: threadId },- });+ if (!isDefined(started)) {+ throw this.noFailedTurnToRetry();+ }- await this.enqueueStreamJob({- thread,- streamId,- userWorkspaceId,- workspaceId: workspace.id,- messages: await this.loadMessagesFromDB(- threadId,- workspace.id,- workspaceMemberId,- ),- browsingContext: null,- modelId,- lastUserMessageText: '',- turnId,- messageId,- });+ return started;}async flushNextQueuedMessage({@@ -512,20 +348,200 @@ export class AgentChatStreamingService {threadId: string;workspaceId: string;}): Promise<void> {- const threadStatus = await this.threadRepository.findOne(workspaceId, {+ const thread = await this.threadRepository.findOne(workspaceId, {where: { id: threadId },select: ['id', 'deletedAt', 'pendingQuestionMessageId', 'activeStreamId'],});// the stream flushes the queue when it lets go, so a send behind it reads nothing moreif (- !threadStatus ||- threadStatus.deletedAt ||- isNonEmptyString(threadStatus.activeStreamId)+ !thread ||+ thread.deletedAt ||+ isNonEmptyString(thread.activeStreamId)) {return;}+ const next = await this.findQueuedMessageToSend({ thread, workspaceId });++ if (!isDefined(next)) {+ await this.streamRecoveryService.wakeNext({+ threadId,+ workspaceId,+ shouldFlush: false,+ });++ return;+ }++ const { message, userWorkspaceId, workspaceMemberId } = next;+ const textPart = message.parts?.find((part) => part.type === 'text');+ const hasFileAttachment = (message.parts ?? []).some(+ (part) => part.type === 'file',+ );++ if (!isNonEmptyString(textPart?.textContent) && !hasFileAttachment) {+ await this.agentChatQueueService.deleteQueuedMessage({+ messageId: message.id,+ workspaceId,+ });++ return;+ }++ await this.startClaimedTurn({ threadId, workspaceId }, async (streamId) => {+ const turnId = await this.agentChatQueueService.promoteQueuedMessage({+ messageId: message.id,+ threadId,+ workspaceId,+ });++ if (turnId === null) {+ return null;+ }++ await this.eventPublisherService.publish({+ threadId,+ workspaceId,+ event: { type: 'queue-updated' },+ });++ await this.eventPublisherService.publish({+ threadId,+ workspaceId,+ event: { type: 'message-persisted', messageId: message.id },+ });++ await this.enqueueStreamJob({+ threadId,+ workspaceId,+ workspaceMemberId,+ streamId,+ userWorkspaceId,+ turnId,+ messageId: message.id,+ });++ return turnId;+ });+ }++ // the thread's claim must be held; the title comes from the turn's message unless the caller passes its own+ async enqueueStreamJob({+ threadId,+ workspaceId,+ workspaceMemberId,+ turnId,+ messageId,+ browsingContext = null,+ lastUserMessageText,+ ...data+ }: Pick<+ StreamAgentChatJobData,+ | 'threadId'+ | 'workspaceId'+ | 'streamId'+ | 'userWorkspaceId'+ | 'modelId'+ | 'messageId'+ > & {+ workspaceMemberId: string;+ turnId: string;+ browsingContext?: BrowsingContextType | null;+ lastUserMessageText?: string;+ }): Promise<void> {+ const [thread, messages] = await Promise.all([+ this.threadRepository.findOneOrFail(workspaceId, {+ where: { id: threadId },+ select: ['id', 'title', 'conversationSize'],+ }),+ this.loadMessagesFromDB(threadId, workspaceId, workspaceMemberId),+ ]);++ await this.messageQueueService.add<StreamAgentChatJobData>(+ STREAM_AGENT_CHAT_JOB_NAME,+ {+ ...data,+ threadId,+ workspaceId,+ messageId,+ messages,+ browsingContext,+ existingTurnId: turnId,+ lastUserMessageText:+ lastUserMessageText ??+ messages+ .find(({ id }) => id === messageId)+ ?.parts.find((part): part is TextUIPart => part.type === 'text')+ ?.text ??+ '',+ hasTitle: isNonEmptyString(thread.title),+ conversationSizeTokens: thread.conversationSize,+ },+ );+ }++ private async addMemberMessage({+ thread,+ userWorkspaceId,+ workspaceMemberId,+ workspace,+ text,+ messageId,+ fileAttachments,+ turnStatus,+ eventType,+ }: MemberMessageOptions & {+ turnStatus?: AgentTurnStatus;+ eventType: 'message-added' | 'message-persisted';+ }) {+ const threadId = thread.id;+ const workspaceId = workspace.id;++ const fileParts = await this.buildFilePartsFromAttachments(+ fileAttachments,+ workspaceId,+ );++ const savedMessage = await this.agentChatService.addMessage({+ threadId,+ userWorkspaceId,+ id: messageId,+ uiMessage: {+ role: AgentMessageRole.USER,+ parts: [{ type: 'text' as const, text }, ...fileParts],+ },+ turnStatus,+ workspaceId,+ });++ await this.eventPublisherService.publish({+ workspaceId,+ threadId,+ event: { type: eventType, messageId: savedMessage.id },+ });++ await this.threadService.notifyThreadActivityUpdated({+ threadId,+ workspaceMemberId,+ workspaceId,+ text,+ });++ return savedMessage;+ }++ private async findQueuedMessageToSend({+ thread,+ workspaceId,+ }: {+ thread: Pick<+ AgentChatThreadWorkspaceEntity,+ 'id' | 'pendingQuestionMessageId'+ >;+ workspaceId: string;+ }) {+ const threadId = thread.id;const [queuedMessages, { suspendedRun }] = await Promise.all([this.agentChatQueueService.getQueuedMessages({ threadId, workspaceId }),this.agentRunSuspensionService.findConversationWait({@@ -543,33 +559,32 @@ export class AgentChatStreamingService {outcome: { type: 'MESSAGE_RECEIVED' },});- return;+ return null;}// queued messages wait behind a pending tool call: they follow the answer rather than replace it- if (isDefined(threadStatus.pendingQuestionMessageId)) {- return;+ if (isDefined(thread.pendingQuestionMessageId)) {+ return null;}- let nextQueued: (typeof queuedMessages)[number] | undefined;- let userWorkspaceId: string | undefined;- let workspaceMemberId: string | undefined;- for (const candidate of queuedMessages) {+ for (const message of queuedMessages) {try {const { sender } = await this.actorService.resolveMessage({workspaceId,threadId,- messageId: candidate.id,+ messageId: message.id,});const authorization = await this.actorService.authorize({workspaceId,threadId,sender,});- workspaceMemberId = authorization.authContext.workspaceMemberId;- nextQueued = candidate;- userWorkspaceId = sender.userWorkspaceId;- break;++ return {+ message,+ userWorkspaceId: sender.userWorkspaceId,+ workspaceMemberId: authorization.authContext.workspaceMemberId,+ };} catch (error) {// during a rolling upgrade a worker may run an older access policy, so keep the request for another workerif (@@ -590,7 +605,7 @@ export class AgentChatStreamingService {throw error;}await this.agentChatQueueService.deleteQueuedMessage({- messageId: candidate.id,+ messageId: message.id,workspaceId,});await this.eventPublisherService.publish({@@ -600,93 +615,8 @@ export class AgentChatStreamingService {});}}- if (- !isDefined(nextQueued) ||- !isDefined(userWorkspaceId) ||- !isDefined(workspaceMemberId)- ) {- return;- }-- const textPart = nextQueued.parts?.find((part) => part.type === 'text');- const messageText = textPart?.textContent ?? '';- const hasFileAttachment = (nextQueued.parts ?? []).some(- (part) => part.type === 'file',- );- if (messageText === '' && !hasFileAttachment) {- await this.agentChatQueueService.deleteQueuedMessage({- messageId: nextQueued.id,- workspaceId,- });-- return;- }-- const streamId = generateId();-- const claimed = await this.streamRecoveryService.tryClaimStream({- threadId,- workspaceId,- streamId,- });-- if (!claimed) {- return;- }-- await this.releaseClaimOnEnqueueFailure(- { threadId, workspaceId, streamId },- async () => {- const turnId = await this.agentChatQueueService.promoteQueuedMessage({- messageId: nextQueued.id,- threadId,- workspaceId,- });-- if (turnId === null) {- await this.streamRecoveryService.release({- threadId,- workspaceId,- streamId,- shouldFlush: true,- });-- return;- }-- await this.eventPublisherService.publish({- threadId,- workspaceId,- event: { type: 'queue-updated' },- });-- await this.eventPublisherService.publish({- threadId,- workspaceId,- event: { type: 'message-persisted', messageId: nextQueued.id },- });-- const [messages, thread] = await Promise.all([- this.loadMessagesFromDB(threadId, workspaceId, workspaceMemberId),- this.threadRepository.findOneOrFail(workspaceId, {- where: { id: threadId },- }),- ]);-- await this.enqueueStreamJob({- thread,- streamId,- userWorkspaceId,- workspaceId,- messages,- browsingContext: null,- lastUserMessageText: messageText,- turnId,- messageId: nextQueued.id,- });- },- );+ return null;}// a message sent while an agent waits on a person closes its pending calls as skipped, so the model sees why.@@ -726,35 +656,32 @@ export class AgentChatStreamingService {});}- private async enqueueStreamJob({- thread,- turnId,- ...data- }: Omit<- StreamAgentChatJobData,- 'threadId' | 'hasTitle' | 'conversationSizeTokens' | 'existingTurnId'- > & {- thread: StreamJobThread;- turnId: string;- }): Promise<void> {- await this.messageQueueService.add<StreamAgentChatJobData>(- STREAM_AGENT_CHAT_JOB_NAME,- {- ...data,- existingTurnId: turnId,- threadId: thread.id,- hasTitle: isNonEmptyString(thread.title),- conversationSizeTokens: thread.conversationSize,- },- );- }+ // starts a turn under a new claim; null when the thread is busy or there turned out to be nothing to start+ private async startClaimedTurn<TResult>(+ {+ threadId,+ workspaceId,+ modelId,+ }: { threadId: string; workspaceId: string; modelId?: string },+ start: (streamId: string) => Promise<TResult | null>,+ ): Promise<TResult | null> {+ const claim = { threadId, workspaceId, streamId: generateId() };++ if (!(await this.streamRecoveryService.tryClaimStream(claim))) {+ return null;+ }- private async releaseClaimOnEnqueueFailure<TResult>(- { modelId, ...claim }: AgentChatStreamClaim & { modelId?: string },- enqueue: () => Promise<TResult>,- ): Promise<TResult> {try {- return await enqueue();+ const started = await start(claim.streamId);++ if (!isDefined(started)) {+ await this.streamRecoveryService.release({+ ...claim,+ shouldFlush: true,+ });+ }++ return started;} catch (error) {// a turn the claim already started fails with the enqueue errorawait this.streamRecoveryService.release({@@ -776,7 +703,7 @@ export class AgentChatStreamingService {});this.logger.error(- `[AI_CHAT_TURN_FAILED] failurePhase=enqueue, model=${model}, threadId=${claim.threadId}, workspaceId=${claim.workspaceId}: ${formatErrorWithCause(error)}`,+ `[AI_CHAT_TURN_FAILED] failurePhase=enqueue, model=${model}, threadId=${threadId}, workspaceId=${workspaceId}: ${formatErrorWithCause(error)}`,);throw error;diff --git a/packages/twenty-server/src/engine/metadata-modules/ai/ai-chat/services/agent-chat-waiting-run.service.ts b/packages/twenty-server/src/engine/metadata-modules/ai/ai-chat/services/agent-chat-waiting-run.service.tsnew file mode 100644index 00000000000..5e0106897ab--- /dev/null+++ b/packages/twenty-server/src/engine/metadata-modules/ai/ai-chat/services/agent-chat-waiting-run.service.ts@@ -0,0 +1,150 @@+import { Injectable, type OnModuleInit } from '@nestjs/common';++import { isNonEmptyString } from '@sniptt/guards';+import { isDefined } from 'twenty-shared/utils';+import { IsNull } from 'typeorm';+import { v4 } from 'uuid';++import { InjectMessageQueue } from 'src/engine/core-modules/message-queue/decorators/message-queue.decorator';+import { MessageQueue } from 'src/engine/core-modules/message-queue/message-queue.constants';+import { MessageQueueService } from 'src/engine/core-modules/message-queue/services/message-queue.service';+import { type PendingWakeUpEntity } from 'src/engine/core-modules/pending-wake-up/entities/pending-wake-up.entity';+import { PendingWakeUpOwnerHandlerRegistryService } from 'src/engine/core-modules/pending-wake-up/services/pending-wake-up-owner-handler-registry.service';+import { PendingWakeUpService } from 'src/engine/core-modules/pending-wake-up/services/pending-wake-up.service';+import { RedisClientService } from 'src/engine/core-modules/redis-client/redis-client.service';+import { type PendingWakeUpOwnerHandler } from 'src/engine/core-modules/pending-wake-up/types/pending-wake-up-owner-handler.type';+import { AGENT_CHAT_WAITING_RUN_OWNER_TYPE } from 'src/engine/metadata-modules/ai/ai-chat/constants/agent-chat-waiting-run-owner-type.constant';+import { AGENT_CHAT_WAITING_RUN_RECHECK_DELAY_MS } from 'src/engine/metadata-modules/ai/ai-chat/constants/agent-chat-waiting-run-recheck-delay-ms.constant';+import { AgentChatStreamRecoveryService } from 'src/engine/metadata-modules/ai/ai-chat/services/agent-chat-stream-recovery.service';+import { AgentHistoryRepository } from 'src/engine/metadata-modules/ai/ai-history/repositories/agent-history-repository';+import { InjectAgentHistoryRepository } from 'src/engine/metadata-modules/ai/ai-history/repositories/inject-agent-history-repository.decorator';+import { type AgentChatThreadWorkspaceEntity } from 'src/engine/metadata-modules/ai/ai-history/standard-objects/agent-chat-thread.workspace-entity';++type AgentChatWaitingRun = { jobName: string; jobData: object };++// the recheck reaps a holder whose worker died, which nothing else releases+const buildRecheckAt = () =>+ new Date(Date.now() + AGENT_CHAT_WAITING_RUN_RECHECK_DELAY_MS);++@Injectable()+export class AgentChatWaitingRunService+ implements PendingWakeUpOwnerHandler<PendingWakeUpEntity>, OnModuleInit+{+ constructor(+ @InjectAgentHistoryRepository('agentChatThread')+ private readonly threadRepository: AgentHistoryRepository<AgentChatThreadWorkspaceEntity>,+ private readonly pendingWakeUpOwnerHandlerRegistryService: PendingWakeUpOwnerHandlerRegistryService,+ private readonly pendingWakeUpService: PendingWakeUpService,+ private readonly streamRecoveryService: AgentChatStreamRecoveryService,+ @InjectMessageQueue(MessageQueue.aiQueue)+ private readonly messageQueueService: MessageQueueService,+ private readonly redisClientService: RedisClientService,+ ) {}++ onModuleInit(): void {+ this.pendingWakeUpOwnerHandlerRegistryService.register(+ AGENT_CHAT_WAITING_RUN_OWNER_TYPE,+ this,+ );+ }++ async wait({+ workspaceId,+ threadId,+ run,+ }: {+ workspaceId: string;+ threadId: string;+ run: AgentChatWaitingRun;+ }): Promise<void> {+ await this.pendingWakeUpService.arm({+ workspaceId,+ owner: {+ type: AGENT_CHAT_WAITING_RUN_OWNER_TYPE,+ id: threadId,+ key: v4(),+ },+ condition: { type: 'TIME', resumeAt: buildRecheckAt().toISOString() },+ payload: run,+ });++ // the conversation may have been released between the failed claim and the wait+ if (+ await this.threadRepository.existsBy(workspaceId, {+ id: threadId,+ activeStreamId: IsNull(),+ })+ ) {+ await this.streamRecoveryService.wakeNext({+ workspaceId,+ threadId,+ shouldFlush: true,+ });+ }+ }++ // a run the previous release parked in Redis runs from its delayed copy only if no release woke it first+ async shouldRunLegacyParkedCopy({+ threadId,+ parkId,+ }: {+ threadId: string;+ parkId?: string;+ }): Promise<boolean> {+ if (!isDefined(parkId)) {+ return true;+ }++ const removedCount = await this.redisClientService+ .getClient()+ .hdel(`agent-chat-parked-runs:${threadId}`, parkId);++ return removedCount === 1;+ }++ async getOwnerState(wakeUp: PendingWakeUpEntity) {+ return { status: 'WAITING' as const, owner: wakeUp };+ }++ async getReadPermissions(): Promise<never> {+ throw new Error('A waiting run reads no event record');+ }++ async resolve({ wakeUp }: { wakeUp: PendingWakeUpEntity }): Promise<void> {+ const { workspaceId, ownerId: threadId } = wakeUp;+ const thread = await this.threadRepository.findOne(workspaceId, {+ where: { id: threadId },+ select: ['id', 'activeStreamId'],+ });++ if (isDefined(thread) && isNonEmptyString(thread.activeStreamId)) {+ // rearmed before the reap, so the release it makes wakes the fresh id+ await this.pendingWakeUpService.rearm({+ wakeUp,+ resumeAt: buildRecheckAt(),+ });+ await this.streamRecoveryService.reapDeadStream({ thread, workspaceId });++ return;+ }++ const claimedWakeUp = await this.pendingWakeUpService.claim({+ workspaceId,+ wakeUpId: wakeUp.id,+ });++ if (!isDefined(thread) || !isDefined(claimedWakeUp)) {+ return;+ }++ const run = claimedWakeUp.payload as AgentChatWaitingRun;++ try {+ await this.messageQueueService.add(run.jobName, run.jobData);+ } catch (error) {+ await this.wait({ workspaceId, threadId, run });++ throw error;+ }+ }+}diff --git a/packages/twenty-server/src/engine/metadata-modules/ai/ai-tool-call-answer/services/tool-call-answer.service.ts b/packages/twenty-server/src/engine/metadata-modules/ai/ai-tool-call-answer/services/tool-call-answer.service.tsindex 8c7e720c5a7..41047aff0c6 100644--- a/packages/twenty-server/src/engine/metadata-modules/ai/ai-tool-call-answer/services/tool-call-answer.service.ts+++ b/packages/twenty-server/src/engine/metadata-modules/ai/ai-tool-call-answer/services/tool-call-answer.service.ts@@ -353,15 +353,17 @@ export class ToolCallAnswerService {return { streamId: null, turnId: answerMessage.turnId };}- await this.agentChatStreamingService.enqueueResumeStream({+ await this.agentChatStreamingService.enqueueStreamJob({threadId,+ workspaceId,userWorkspaceId,workspaceMemberId: args.workspaceMemberId,- workspace,turnId: answerMessage.turnId,streamId,modelId: args.modelId,messageId: answerMessage.id,+ // the answer resumes the turn rather than opens the thread, so it never titles it+ lastUserMessageText: '',});return { streamId, turnId: answerMessage.turnId };diff --git a/packages/twenty-server/test/integration/ai/suites/agent-run-wait.integration-spec.ts b/packages/twenty-server/test/integration/ai/suites/agent-run-wait.integration-spec.tsindex f02bc8c7415..5df5f5a4e76 100644--- a/packages/twenty-server/test/integration/ai/suites/agent-run-wait.integration-spec.ts+++ b/packages/twenty-server/test/integration/ai/suites/agent-run-wait.integration-spec.ts@@ -6,6 +6,7 @@ import { createOneAgent } from 'test/integration/metadata/suites/agent/utils/creimport { deleteOneAgent } from 'test/integration/metadata/suites/agent/utils/delete-one-agent.util';import { updateOneAgent } from 'test/integration/metadata/suites/agent/utils/update-one-agent.util';import { makeMetadataApiRequest } from 'test/integration/metadata/suites/utils/make-metadata-api-request.util';+import { destroyAgentChatThread } from 'test/integration/utils/destroy-agent-chat-thread.util';import { expectEventually } from 'test/integration/utils/expect-eventually.util';import { getAppProviderByClassName } from 'test/integration/utils/get-app-provider-by-class-name.util';import { getCoreRepository } from 'test/integration/utils/get-core-repository.util';@@ -17,18 +18,30 @@ import {import { type AgentTrigger } from 'twenty-shared/application';import { type ApplicationLookupService } from 'src/engine/core-modules/application/application-lookup/application-lookup.service';+import { MessageQueue } from 'src/engine/core-modules/message-queue/message-queue.constants';+import { type MessageQueueService } from 'src/engine/core-modules/message-queue/services/message-queue.service';+import { getQueueToken } from 'src/engine/core-modules/message-queue/utils/get-queue-token.util';import { type PendingWakeUpResolverService } from 'src/engine/core-modules/pending-wake-up/services/pending-wake-up-resolver.service';import { fromWorkspaceEntityToFlat } from 'src/engine/core-modules/workspace/utils/from-workspace-entity-to-flat.util';import { WorkspaceEntity } from 'src/engine/core-modules/workspace/workspace.entity';import { type AgentRunService } from 'src/engine/metadata-modules/ai/ai-agent-execution/services/agent-run.service';import { type AgentExecutionResult } from 'src/engine/metadata-modules/ai/ai-agent-execution/types/agent-execution-result.type';+import { buildAgentRunThreadId } from 'src/engine/metadata-modules/ai/ai-agent-execution/utils/build-agent-run-thread-id.util';+import { AGENT_CHAT_WAITING_RUN_OWNER_TYPE } from 'src/engine/metadata-modules/ai/ai-chat/constants/agent-chat-waiting-run-owner-type.constant';+import { FLUSH_QUEUED_AGENT_CHAT_MESSAGES_JOB_NAME } from 'src/engine/metadata-modules/ai/ai-chat/jobs/flush-queued-agent-chat-messages-job-name.constant';import { type AgentChatStreamHeartbeatService } from 'src/engine/metadata-modules/ai/ai-chat/services/agent-chat-stream-heartbeat.service';import { type AgentChatStreamRecoveryService } from 'src/engine/metadata-modules/ai/ai-chat/services/agent-chat-stream-recovery.service';+import { type AgentChatThreadService } from 'src/engine/metadata-modules/ai/ai-chat/services/agent-chat-thread.service';+import { type AgentChatWaitingRunService } from 'src/engine/metadata-modules/ai/ai-chat/services/agent-chat-waiting-run.service';+import { type RedisClientService } from 'src/engine/core-modules/redis-client/redis-client.service';+import { type StreamAgentChatJobData } from 'src/engine/metadata-modules/ai/ai-chat/jobs/stream-agent-chat-job.types';+import { type AiModelRegistryService } from 'src/engine/metadata-modules/ai/ai-models/services/ai-model-registry.service';import { type AgentTriggerRunnerService } from 'src/engine/metadata-modules/ai/ai-agent-trigger/services/agent-trigger-runner.service';import { AiExceptionCode } from 'src/engine/metadata-modules/ai/ai.exception';import { SEED_APPLE_WORKSPACE_ID } from 'src/engine/workspace-manager/dev-seeder/core/constants/seeder-workspaces.constant';import { USER_WORKSPACE_DATA_SEED_IDS } from 'src/engine/workspace-manager/dev-seeder/core/utils/seed-user-workspaces.util';import { API_KEY_DATA_SEED_IDS } from 'src/engine/workspace-manager/dev-seeder/data/constants/api-key-data-seeds.constant';+import { WORKSPACE_MEMBER_DATA_SEED_IDS } from 'src/engine/workspace-manager/dev-seeder/data/constants/workspace-member-data-seeds.constant';import { getWorkspaceSchemaName } from 'src/engine/workspace-datasource/utils/get-workspace-schema-name.util';const workspaceId = SEED_APPLE_WORKSPACE_ID;@@ -95,6 +108,43 @@ const findTurnStatuses = async (threadId: string) =>)).map(({ status }: { status: string }) => status);+const findWaiters = async (threadId: string) =>+ global.testDataSource.query(+ `SELECT id FROM core."pendingWakeUp"+ WHERE "ownerId" = $1 AND "ownerType" = $2 ORDER BY "createdAt"`,+ [threadId, AGENT_CHAT_WAITING_RUN_OWNER_TYPE],+ );++const findUserMessageTexts = async (threadId: string) =>+ (+ await global.testDataSource.query(+ `SELECT part."textContent" AS text FROM "${schema}"."agentMessagePart" part+ JOIN "${schema}"."agentMessage" message ON message.id = part."messageId"+ WHERE message."threadId" = $1 AND message.role = 'user' AND part.type = 'text'+ ORDER BY message."createdAt"`,+ [threadId],+ )+ ).map(({ text }: { text: string }) => text);++const holdConversation = async (threadId: string): Promise<string> => {+ const streamId = randomUUID();++ await getAppProviderByClassName<AgentChatStreamHeartbeatService>(+ 'AgentChatStreamHeartbeatService',+ ).markClaimed(streamId);+ await global.testDataSource.query(+ `UPDATE "${schema}"."agentChatThread" SET "activeStreamId" = $2 WHERE id = $1`,+ [threadId, streamId],+ );++ return streamId;+};++const releaseConversation = (threadId: string, streamId: string) =>+ getAppProviderByClassName<AgentChatStreamRecoveryService>(+ 'AgentChatStreamRecoveryService',+ ).release({ threadId, workspaceId, streamId, shouldFlush: true });+describe('agent runs that wait (integration)', () => {const triggerId = randomUUID();const trigger: AgentTrigger = {@@ -162,6 +212,40 @@ describe('agent runs that wait (integration)', () => {});};+ const buildRunOnThread = async ({+ threadKey = `lead:${randomUUID()}`,+ waitForCompletion,+ }: {+ threadKey?: string;+ waitForCompletion?: boolean;+ } = {}) => {+ const agentRunService =+ getAppProviderByClassName<AgentRunService>('AgentRunService');+ const workspace = fromWorkspaceEntityToFlat(+ await getCoreRepository<WorkspaceEntity>(WorkspaceEntity).findOneByOrFail(+ { id: workspaceId },+ ),+ );+ const callerApplication =+ await getAppProviderByClassName<ApplicationLookupService>(+ 'ApplicationLookupService',+ ).findById({ id: agentApplicationId, workspaceId });++ return (content: string) =>+ agentRunService.run({+ workspace,+ requestUserWorkspaceId: null,+ requestWorkspaceMemberId: null,+ callerApplication: callerApplication ?? undefined,+ input: {+ agentUniversalIdentifier,+ input: [{ role: 'user', content }],+ thread: { key: threadKey },+ waitForCompletion,+ },+ });+ };+it('lets a triggered agent wait for a duration and go on once the wake-up resolves', async () => {const runTurn = mockAgent(buildWaitingAgentExecutionResult(),@@ -424,30 +508,7 @@ describe('agent runs that wait (integration)', () => {replyingResult,replyingResult,);- const agentRunService =- getAppProviderByClassName<AgentRunService>('AgentRunService');- const workspace = fromWorkspaceEntityToFlat(- await getCoreRepository<WorkspaceEntity>(WorkspaceEntity).findOneByOrFail(- { id: workspaceId },- ),- );- const callerApplication =- await getAppProviderByClassName<ApplicationLookupService>(- 'ApplicationLookupService',- ).findById({ id: agentApplicationId, workspaceId });- const thread = { key: `lead:${randomUUID()}` };- const runOnSameThread = (content: string) =>- agentRunService.run({- workspace,- requestUserWorkspaceId: null,- requestWorkspaceMemberId: null,- callerApplication: callerApplication ?? undefined,- input: {- agentUniversalIdentifier,- input: [{ role: 'user', content }],- thread,- },- });+ const runOnSameThread = await buildRunOnThread();const waiting = await runOnSameThread('Follow up tomorrow');@@ -470,33 +531,11 @@ describe('agent runs that wait (integration)', () => {expect(runTurn).toHaveBeenCalledTimes(3);});- it('parks a run whose conversation is busy and starts it once the conversation is released', async () => {+ it('waits in line for a busy conversation and starts the run once it is released', async () => {const runTurn = mockAgent(replyingResult, replyingResult);- const agentRunService =- getAppProviderByClassName<AgentRunService>('AgentRunService');- const workspace = fromWorkspaceEntityToFlat(- await getCoreRepository<WorkspaceEntity>(WorkspaceEntity).findOneByOrFail(- { id: workspaceId },- ),- );- const callerApplication =- await getAppProviderByClassName<ApplicationLookupService>(- 'ApplicationLookupService',- ).findById({ id: agentApplicationId, workspaceId });- const thread = { key: `lead:${randomUUID()}` };- const runOnSameThread = (content: string) =>- agentRunService.run({- workspace,- requestUserWorkspaceId: null,- requestWorkspaceMemberId: null,- callerApplication: callerApplication ?? undefined,- input: {- agentUniversalIdentifier,- input: [{ role: 'user', content }],- thread,- waitForCompletion: false,- },- });+ const runOnSameThread = await buildRunOnThread({+ waitForCompletion: false,+ });const { threadId } = await runOnSameThread('Follow up tomorrow');@@ -504,32 +543,246 @@ describe('agent runs that wait (integration)', () => {expect(await findTurnStatuses(threadId)).toEqual(['completed']);});- // a live stream, such as a member's chat reply, holds the conversation- const streamId = randomUUID();+ const streamId = await holdConversation(threadId);- await getAppProviderByClassName<AgentChatStreamHeartbeatService>(- 'AgentChatStreamHeartbeatService',- ).markClaimed(streamId);- await global.testDataSource.query(- `UPDATE "${schema}"."agentChatThread" SET "activeStreamId" = $2 WHERE id = $1`,- [threadId, streamId],+ await runOnSameThread('Any news?');++ await expectEventually(async () => {+ expect(await findWaiters(threadId)).toHaveLength(1);+ });+ expect(await findTurnStatuses(threadId)).toEqual(['completed']);++ await releaseConversation(threadId, streamId);++ await expectEventually(async () => {+ expect(await findTurnStatuses(threadId)).toEqual([+ 'completed',+ 'completed',+ ]);+ expect(await findWaiters(threadId)).toEqual([]);+ });+ expect(runTurn).toHaveBeenCalledTimes(2);+ });++ it('starts the runs waiting on a conversation one at a time, in the order they started waiting', async () => {+ mockAgent(replyingResult, replyingResult, replyingResult);+ const runOnSameThread = await buildRunOnThread({+ waitForCompletion: false,+ });++ const { threadId } = await runOnSameThread('Follow up tomorrow');++ await expectEventually(async () => {+ expect(await findTurnStatuses(threadId)).toEqual(['completed']);+ });++ const streamId = await holdConversation(threadId);++ for (const [index, content] of [+ 'First in line',+ 'Second in line',+ ].entries()) {+ await runOnSameThread(content);++ await expectEventually(async () => {+ expect(await findWaiters(threadId)).toHaveLength(index + 1);+ });+ }++ await releaseConversation(threadId, streamId);++ await expectEventually(async () => {+ expect(await findTurnStatuses(threadId)).toEqual([+ 'completed',+ 'completed',+ 'completed',+ ]);+ expect(await findWaiters(threadId)).toEqual([]);+ });+ expect(await findUserMessageTexts(threadId)).toEqual([+ 'Follow up tomorrow',+ 'First in line',+ 'Second in line',+ ]);+ });++ it('checks a busy conversation again and reaps a holder whose worker died', async () => {+ const runTurn = mockAgent(replyingResult, replyingResult);+ const runOnSameThread = await buildRunOnThread({+ waitForCompletion: false,+ });+ const resolver = getAppProviderByClassName<PendingWakeUpResolverService>(+ 'PendingWakeUpResolverService',);+ const { threadId } = await runOnSameThread('Follow up tomorrow');++ await expectEventually(async () => {+ expect(await findTurnStatuses(threadId)).toEqual(['completed']);+ });++ const streamId = await holdConversation(threadId);+await runOnSameThread('Any news?');- await new Promise((resolve) => setTimeout(resolve, 1_000));+ await expectEventually(async () => {+ expect(await findWaiters(threadId)).toHaveLength(1);+ });++ const [{ id: firstWakeUpId }] = await findWaiters(threadId);++ await resolver.resolve({ workspaceId, wakeUpId: firstWakeUpId });++ const [{ id: secondWakeUpId }] = await findWaiters(threadId);++ expect(secondWakeUpId).not.toBe(firstWakeUpId);expect(await findTurnStatuses(threadId)).toEqual(['completed']);- await getAppProviderByClassName<AgentChatStreamRecoveryService>(- 'AgentChatStreamRecoveryService',- ).release({ threadId, workspaceId, streamId, shouldFlush: true });+ await getAppProviderByClassName<AgentChatStreamHeartbeatService>(+ 'AgentChatStreamHeartbeatService',+ ).clear(streamId);+ await resolver.resolve({ workspaceId, wakeUpId: secondWakeUpId });await expectEventually(async () => {expect(await findTurnStatuses(threadId)).toEqual(['completed','completed',]);+ expect(await findWaiters(threadId)).toEqual([]);});expect(runTurn).toHaveBeenCalledTimes(2);});++ it('sends the messages a member queued before a run that started waiting after them', async () => {+ const runTurn = mockAgent(replyingResult);+ const threadKey = `lead:${randomUUID()}`;+ const threadId = buildAgentRunThreadId({+ applicationId: agentApplicationId,+ agentId,+ threadKey,+ });+ const runOnSameThread = await buildRunOnThread({+ threadKey,+ waitForCompletion: false,+ });+ const aiModelRegistryService =+ getAppProviderByClassName<AiModelRegistryService>(+ 'AiModelRegistryService',+ );++ jest+ .spyOn(aiModelRegistryService, 'getAvailableModels')+ .mockReturnValue([{ modelId: 'test-model' }] as never);+ jest+ .spyOn(aiModelRegistryService, 'validateModelAvailability')+ .mockReturnValue(undefined as never);+ const chatQueue = global.app.get<MessageQueueService>(+ getQueueToken(MessageQueue.aiStreamQueue),+ );+ const enqueueChatJob = chatQueue.add.bind(chatQueue);+ // chat replies are only enqueued, and each one lets go of the conversation below+ const enqueueChatReply = jest+ .spyOn(chatQueue, 'add')+ .mockImplementation(((jobName, data, options) =>+ jobName === FLUSH_QUEUED_AGENT_CHAT_MESSAGES_JOB_NAME+ ? enqueueChatJob(jobName, data, options)+ : Promise.resolve()) as typeof chatQueue.add);+ const findChatReply = (messageId: string) =>+ enqueueChatReply.mock.calls+ .map(([, data]) => data as Partial<StreamAgentChatJobData>)+ .find((data) => data.messageId === messageId);++ await getAppProviderByClassName<AgentChatThreadService>(+ 'AgentChatThreadService',+ ).createThread({+ workspaceId,+ workspaceMemberId: WORKSPACE_MEMBER_DATA_SEED_IDS.JANE,+ id: threadId,+ title: 'Waiting line',+ });++ try {+ const holderStreamId = await holdConversation(threadId);+ const queuedMessageIds = [randomUUID(), randomUUID()];++ for (const [index, messageId] of queuedMessageIds.entries()) {+ const response = await makeMetadataApiRequest({+ query: gql`+ mutation Send($threadId: UUID!, $text: String!, $messageId: UUID!) {+ sendChatMessage(+ threadId: $threadId+ text: $text+ messageId: $messageId+ ) {+ queued+ }+ }+ `,+ variables: { threadId, text: `Queued ${index + 1}`, messageId },+ });++ expect(response.body.errors).toBeUndefined();+ expect(response.body.data.sendChatMessage.queued).toBe(true);+ }++ await runOnSameThread('From the app');++ await expectEventually(async () => {+ expect(await findWaiters(threadId)).toHaveLength(1);+ });++ let streamId = holderStreamId;++ for (const messageId of queuedMessageIds) {+ await releaseConversation(threadId, streamId);++ await expectEventually(async () => {+ expect(findChatReply(messageId)).toBeDefined();+ });++ streamId = findChatReply(messageId)?.streamId ?? '';+ }++ expect(runTurn).not.toHaveBeenCalled();+ expect(await findWaiters(threadId)).toHaveLength(1);++ await releaseConversation(threadId, streamId);++ await expectEventually(async () => {+ expect(runTurn).toHaveBeenCalledTimes(1);+ expect(await findWaiters(threadId)).toEqual([]);+ expect((await findTurnStatuses(threadId)).at(-1)).toBe('completed');+ });+ expect(await findUserMessageTexts(threadId)).toEqual([+ 'Queued 1',+ 'Queued 2',+ 'From the app',+ ]);+ } finally {+ await destroyAgentChatThread({ threadId });+ }+ });++ it('runs a delayed copy the previous release parked in Redis only when no release woke it', async () => {+ const threadId = randomUUID();+ const parkId = randomUUID();+ const waitingRunService =+ getAppProviderByClassName<AgentChatWaitingRunService>(+ 'AgentChatWaitingRunService',+ );++ await getAppProviderByClassName<RedisClientService>('RedisClientService')+ .getClient()+ .hset(`agent-chat-parked-runs:${threadId}`, parkId, '{}');++ expect(+ await waitingRunService.shouldRunLegacyParkedCopy({ threadId, parkId }),+ ).toBe(true);+ expect(+ await waitingRunService.shouldRunLegacyParkedCopy({ threadId, parkId }),+ ).toBe(false);+ expect(+ await waitingRunService.shouldRunLegacyParkedCopy({ threadId }),+ ).toBe(true);+ });});