The open alternative to Salesforce, designed for AI.
TypeScript crmcrm-systemcustomergood-first-issuegraphqlhacktoberfestjavascriptmarketingmonoreponestjsopen-sourcepostgresqlreactreactjssalestypescriptweb
Agent engine: one turn engine for chat and background agents, background runs in a job (#28076)
Félix Malfait453a38a12c06
diff --git a/packages/twenty-client-sdk/src/metadata/generated/schema.graphql b/packages/twenty-client-sdk/src/metadata/generated/schema.graphqlindex ad32fde51ac..53522b48a0e 100644--- a/packages/twenty-client-sdk/src/metadata/generated/schema.graphql+++ b/packages/twenty-client-sdk/src/metadata/generated/schema.graphql@@ -4197,6 +4197,7 @@ input RunAgentInput {responseFormat: JSONrunAsWorkspaceMemberId: UUIDthread: RunAgentThreadInput+ waitForCompletion: Boolean}input RunAgentMessageAttachmentInput {diff --git a/packages/twenty-client-sdk/src/metadata/generated/schema.ts b/packages/twenty-client-sdk/src/metadata/generated/schema.tsindex 4c034c4c6a0..aee502926ee 100644--- a/packages/twenty-client-sdk/src/metadata/generated/schema.ts+++ b/packages/twenty-client-sdk/src/metadata/generated/schema.ts@@ -7370,7 +7370,7 @@ export interface RowLevelPermissionPredicateGroupInput {id?: (Scalars['UUID'] |export interface RowLevelPermissionPredicateInput {fieldMetadataId: Scalars['UUID'],id?: (Scalars['UUID'] | null),operand: RowLevelPermissionPredicateOperand,positionInRowLevelPermissionPredicateGroup?: (Scalars['Float'] | null),rowLevelPermissionPredicateGroupId?: (Scalars['UUID'] | null),subFieldName?: (Scalars['String'] | null),value?: (Scalars['JSON'] | null),workspaceMemberFieldMetadataId?: (Scalars['String'] | null),workspaceMemberSubFieldName?: (Scalars['String'] | null)}-export interface RunAgentInput {additionalInstructions?: (Scalars['String'] | null),agentUniversalIdentifier: Scalars['String'],input?: (RunAgentMessageInput[] | null),messages?: (RunAgentMessageInput[] | null),prompt?: (Scalars['String'] | null),responseFormat?: (Scalars['JSON'] | null),runAsWorkspaceMemberId?: (Scalars['UUID'] | null),thread?: (RunAgentThreadInput | null)}+export interface RunAgentInput {additionalInstructions?: (Scalars['String'] | null),agentUniversalIdentifier: Scalars['String'],input?: (RunAgentMessageInput[] | null),messages?: (RunAgentMessageInput[] | null),prompt?: (Scalars['String'] | null),responseFormat?: (Scalars['JSON'] | null),runAsWorkspaceMemberId?: (Scalars['UUID'] | null),thread?: (RunAgentThreadInput | null),waitForCompletion?: (Scalars['Boolean'] | null)}export interface RunAgentMessageAttachmentInput {fileId: Scalars['UUID'],filename?: (Scalars['String'] | null)}diff --git a/packages/twenty-client-sdk/src/metadata/generated/types.ts b/packages/twenty-client-sdk/src/metadata/generated/types.tsindex 8dbca20e1b7..146c78a7cd7 100644--- a/packages/twenty-client-sdk/src/metadata/generated/types.ts+++ b/packages/twenty-client-sdk/src/metadata/generated/types.ts@@ -11333,6 +11333,9 @@ export default {"thread": [437],+ "waitForCompletion": [+ 4+ ],"__typename": [1]diff --git a/packages/twenty-docs/developers/extend/apps/logic/skills-and-agents.mdx b/packages/twenty-docs/developers/extend/apps/logic/skills-and-agents.mdxindex f9549ca25c8..2b683fcdc1c 100644--- a/packages/twenty-docs/developers/extend/apps/logic/skills-and-agents.mdx+++ b/packages/twenty-docs/developers/extend/apps/logic/skills-and-agents.mdx@@ -119,8 +119,12 @@ Key points:(`{ role: 'user' | 'assistant', content: string }`). A `user` message canalso carry file attachments (see below). To hold a conversation, pass a`thread` rather than replaying earlier turns yourself.-- The agent runs **synchronously** and can read/update records itself via its- own tools — `runAgent()` resolves once the run completes.+- By default the call waits for the agent, which can read/update records+ itself via its own tools — `runAgent()` resolves once the run completes.+ Pass `waitForCompletion: false` to resolve as soon as the run starts:+ `status` is then `RUNNING` and `result: null`, and the reply lands in the+ `threadId` conversation. A run that outlasts 15 minutes also resolves as+ `RUNNING`.- The agent can also pause until a record changes or some time has passed.`runAgent()` then resolves with `success: true`, `status: 'SUSPENDED'` and`result: null`; the agent goes on by itself later, and its reply lands in the@@ -138,8 +142,9 @@ Key points:several seconds.- `status` is `COMPLETED` with a non-null `result` when the run completeswithout waiting; `SUSPENDED` with `success: true` and `result: null` while it- waits; `FAILED` with `success: false` and `error` holding the reason (for- example, when the workspace ran out of AI credits mid-run).+ waits; `RUNNING` with `success: true` and `result: null` when the call did not+ wait for the run; `FAILED` with `success: false` and `error` holding the+ reason (for example, when the workspace ran out of AI credits mid-run).```ts src/roles/default-role.tsimport { defineApplicationRole, SystemPermissionFlag } from 'twenty-sdk/define';diff --git a/packages/twenty-front/src/generated-metadata/graphql.ts b/packages/twenty-front/src/generated-metadata/graphql.tsindex 73cc258ddb7..c16cf85b8dc 100644--- a/packages/twenty-front/src/generated-metadata/graphql.ts+++ b/packages/twenty-front/src/generated-metadata/graphql.ts@@ -6302,6 +6302,7 @@ export type RunAgentInput = {responseFormat?: InputMaybe<Scalars['JSON']['input']>;runAsWorkspaceMemberId?: InputMaybe<Scalars['UUID']['input']>;thread?: InputMaybe<RunAgentThreadInput>;+ waitForCompletion?: InputMaybe<Scalars['Boolean']['input']>;};export type RunAgentMessageAttachmentInput = {diff --git a/packages/twenty-server/src/database/commands/agent-history/__tests__/agent-history-migration.postgres.spec.ts b/packages/twenty-server/src/database/commands/agent-history/__tests__/agent-history-migration.postgres.spec.tsindex 1208c60d326..9c717b0a2e5 100644--- a/packages/twenty-server/src/database/commands/agent-history/__tests__/agent-history-migration.postgres.spec.ts+++ b/packages/twenty-server/src/database/commands/agent-history/__tests__/agent-history-migration.postgres.spec.ts@@ -213,6 +213,18 @@ const SCHEMA = getWorkspaceSchemaName(WORKSPACE_ID);}),} as never,);+ const conversationWriter = new AgentConversationWriterService(+ turns as never,+ new AgentHistoryTransactionService(orm as never),+ {+ getOrRecompute: async () => ({+ flatWorkspaceMemberMaps: {+ idByUserWorkspaceId: { [OWNER_ID]: MEMBER_ID },+ },+ }),+ } as never,+ );+const createChatService = (messageRepository: typeof messages) =>new AgentChatService(threads,@@ -223,17 +235,7 @@ const SCHEMA = getWorkspaceSchemaName(WORKSPACE_ID);{} as never,chatSharing as never,chatRecordEvents as never,- new AgentConversationWriterService(- turns as never,- new AgentHistoryTransactionService(orm as never),- {- getOrRecompute: async () => ({- flatWorkspaceMemberMaps: {- idByUserWorkspaceId: { [OWNER_ID]: MEMBER_ID },- },- }),- } as never,- ),+ conversationWriter,chatThreadService,);@@ -706,10 +708,9 @@ const SCHEMA = getWorkspaceSchemaName(WORKSPACE_ID);expect(created.totalInputCredits).toBe('0');expect(created).not.toHaveProperty('workspaceId');const checkpointId = '20202020-8888-4888-8888-888888888888';- const checkpointChatService = createChatService(messages);await Promise.all(['First checkpoint', 'Second checkpoint'].map((text) =>- checkpointChatService.upsertAssistantMessage({+ conversationWriter.upsertAssistantMessage({id: checkpointId,threadId: THREAD_ID,turnId: TURN_ID,diff --git a/packages/twenty-server/src/engine/core-modules/message-queue/message-queue-worker-config.constant.ts b/packages/twenty-server/src/engine/core-modules/message-queue/message-queue-worker-config.constant.tsindex a457c845cd2..71c74ae0db9 100644--- a/packages/twenty-server/src/engine/core-modules/message-queue/message-queue-worker-config.constant.ts+++ b/packages/twenty-server/src/engine/core-modules/message-queue/message-queue-worker-config.constant.ts@@ -248,7 +248,8 @@ export const MESSAGE_QUEUE_WORKER_CONFIG: Record<[MessageQueue.aiQueue]: {priority: 5,workerOptions: {- concurrency: 1,+ // agent runs wait on the model and their tools most of the time, so several share a worker+ concurrency: 10,lockDuration: 30_000,maxStalledCount: 1,boundedShutdownDrain: false,diff --git a/packages/twenty-server/src/engine/metadata-modules/ai/ai-agent-execution/ai-agent-execution.module.ts b/packages/twenty-server/src/engine/metadata-modules/ai/ai-agent-execution/ai-agent-execution.module.tsindex 8595cea8100..58aa23d851a 100644--- a/packages/twenty-server/src/engine/metadata-modules/ai/ai-agent-execution/ai-agent-execution.module.ts+++ b/packages/twenty-server/src/engine/metadata-modules/ai/ai-agent-execution/ai-agent-execution.module.ts@@ -3,7 +3,6 @@ import { TypeOrmModule } from '@nestjs/typeorm';import { ApiKeyModule } from 'src/engine/core-modules/api-key/api-key.module';import { ApplicationLookupModule } from 'src/engine/core-modules/application/application-lookup/application-lookup.module';-import { CacheLockModule } from 'src/engine/core-modules/cache-lock/cache-lock.module';import { FileEntity } from 'src/engine/core-modules/file/entities/file.entity';import { FileUrlModule } from 'src/engine/core-modules/file/file-url/file-url.module';import { MetricsModule } from 'src/engine/core-modules/metrics/metrics.module';@@ -13,19 +12,26 @@ import { ToolProviderModule } from 'src/engine/core-modules/tool-provider/tool-pimport { UserWorkspaceModule } from 'src/engine/core-modules/user-workspace/user-workspace.module';import { WorkspaceEntity } from 'src/engine/core-modules/workspace/workspace.entity';import { ContinueAgentRunJob } from 'src/engine/metadata-modules/ai/ai-agent-execution/jobs/continue-agent-run.job';+import { DeliverAgentRunOutcomeJob } from 'src/engine/metadata-modules/ai/ai-agent-execution/jobs/deliver-agent-run-outcome.job';+import { RunAgentJob } from 'src/engine/metadata-modules/ai/ai-agent-execution/jobs/run-agent.job';import { AgentMessagePartResolver } from 'src/engine/metadata-modules/ai/ai-agent-execution/resolvers/agent-message-part.resolver';import { AgentMessageResolver } from 'src/engine/metadata-modules/ai/ai-agent-execution/resolvers/agent-message.resolver';import { AgentRunResolver } from 'src/engine/metadata-modules/ai/ai-agent-execution/resolvers/agent-run.resolver';import { AgentActorContextService } from 'src/engine/metadata-modules/ai/ai-agent-execution/services/agent-actor-context.service';-import { AgentAsyncExecutorService } from 'src/engine/metadata-modules/ai/ai-agent-execution/services/agent-async-executor.service';import { AgentCallerConversationService } from 'src/engine/metadata-modules/ai/ai-agent-execution/services/agent-caller-conversation.service';import { AgentRunCallerHandlerRegistryService } from 'src/engine/metadata-modules/ai/ai-agent-execution/services/agent-run-caller-handler-registry.service';import { AgentRunConversationService } from 'src/engine/metadata-modules/ai/ai-agent-execution/services/agent-run-conversation.service';import { AgentRunPendingWakeUpHandlerService } from 'src/engine/metadata-modules/ai/ai-agent-execution/services/agent-run-pending-wake-up-handler.service';import { AgentRunService } from 'src/engine/metadata-modules/ai/ai-agent-execution/services/agent-run.service';+import { AgentRunSettlementService } from 'src/engine/metadata-modules/ai/ai-agent-execution/services/agent-run-settlement.service';import { AgentRunSuspensionService } from 'src/engine/metadata-modules/ai/ai-agent-execution/services/agent-run-suspension.service';import { AgentRunnerService } from 'src/engine/metadata-modules/ai/ai-agent-execution/services/agent-runner.service';+import { AgentStepLoopService } from 'src/engine/metadata-modules/ai/ai-agent-execution/services/agent-step-loop.service';+import { AgentToolsetService } from 'src/engine/metadata-modules/ai/ai-agent-execution/services/agent-toolset.service';+import { AgentRunTurnService } from 'src/engine/metadata-modules/ai/ai-agent-execution/services/agent-run-turn.service';+import { AgentTurnService } from 'src/engine/metadata-modules/ai/ai-agent-execution/services/agent-turn.service';import { RunAgentAttachmentService } from 'src/engine/metadata-modules/ai/ai-agent-execution/services/run-agent-attachment.service';+import { AgentChatStreamStateModule } from 'src/engine/metadata-modules/ai/ai-chat/agent-chat-stream-state.module';import { AgentChatThreadModule } from 'src/engine/metadata-modules/ai/ai-chat/agent-chat-thread.module';import { AiAgentRoleModule } from 'src/engine/metadata-modules/ai/ai-agent-role/ai-agent-role.module';import { AiAgentModule } from 'src/engine/metadata-modules/ai/ai-agent/ai-agent.module';@@ -34,12 +40,14 @@ import { AgentChatThreadLifecycleModule } from 'src/engine/metadata-modules/ai/aimport { AiBillingModule } from 'src/engine/metadata-modules/ai/ai-billing/ai-billing.module';import { AgentHistoryModule } from 'src/engine/metadata-modules/ai/ai-history/ai-history.module';import { PermissionsModule } from 'src/engine/metadata-modules/permissions/permissions.module';+import { SkillModule } from 'src/engine/metadata-modules/skill/skill.module';import { UserRoleModule } from 'src/engine/metadata-modules/user-role/user-role.module';import { provideWorkspaceScopedRepository } from 'src/engine/twenty-orm/workspace-scoped-repository/provide-workspace-scoped-repository';@Module({imports: [AgentChatThreadLifecycleModule,+ AgentChatStreamStateModule,AgentChatThreadModule,AgentHistoryModule,AiBillingModule,@@ -47,13 +55,13 @@ import { provideWorkspaceScopedRepository } from 'src/engine/twenty-orm/workspacAiAgentRoleModule,ApiKeyModule,ApplicationLookupModule,- CacheLockModule,FileUrlModule,MetricsModule,PendingWakeUpModule,UserWorkspaceModule,UserRoleModule,PermissionsModule,+ SkillModule,ToolProviderModule,TypeOrmModule.forFeature([AgentEntity,@@ -63,7 +71,6 @@ import { provideWorkspaceScopedRepository } from 'src/engine/twenty-orm/workspac]),],providers: [- AgentAsyncExecutorService,AgentActorContextService,AgentCallerConversationService,AgentRunCallerHandlerRegistryService,@@ -73,11 +80,18 @@ import { provideWorkspaceScopedRepository } from 'src/engine/twenty-orm/workspacAgentMessageResolver,AgentRunResolver,AgentRunService,+ AgentRunSettlementService,AgentRunnerService,+ AgentStepLoopService,+ AgentToolsetService,+ AgentRunTurnService,+ AgentTurnService,// continues suspended runs once answered or woken up, on the server and the worker alikeAgentRunPendingWakeUpHandlerService,ContinueAgentRunJob,+ DeliverAgentRunOutcomeJob,RunAgentAttachmentService,+ RunAgentJob,provideWorkspaceScopedRepository(AgentEntity),provideWorkspaceScopedRepository(FileEntity),provideWorkspaceScopedRepository(PendingWakeUpEntity),@@ -89,6 +103,7 @@ import { provideWorkspaceScopedRepository } from 'src/engine/twenty-orm/workspacAgentRunConversationService,AgentRunSuspensionService,AgentRunnerService,+ AgentTurnService,],})export class AiAgentExecutionModule {}diff --git a/packages/twenty-server/src/engine/metadata-modules/ai/ai-agent-execution/constants/agent-run-busy-retry-delay-ms.const.ts b/packages/twenty-server/src/engine/metadata-modules/ai/ai-agent-execution/constants/agent-run-busy-retry-delay-ms.const.tsnew file mode 100644index 00000000000..ff612b2aa32--- /dev/null+++ b/packages/twenty-server/src/engine/metadata-modules/ai/ai-agent-execution/constants/agent-run-busy-retry-delay-ms.const.ts@@ -0,0 +1,2 @@+// a chat reply holding the conversation lasts seconds to minutes+export const AGENT_RUN_BUSY_RETRY_DELAY_MS = 5_000;diff --git a/packages/twenty-server/src/engine/metadata-modules/ai/ai-agent-execution/constants/agent-run-outcome-delivery-max-attempts.const.ts b/packages/twenty-server/src/engine/metadata-modules/ai/ai-agent-execution/constants/agent-run-outcome-delivery-max-attempts.const.tsnew file mode 100644index 00000000000..a8505fb4961--- /dev/null+++ b/packages/twenty-server/src/engine/metadata-modules/ai/ai-agent-execution/constants/agent-run-outcome-delivery-max-attempts.const.ts@@ -0,0 +1 @@+export const AGENT_RUN_OUTCOME_DELIVERY_MAX_ATTEMPTS = 60;diff --git a/packages/twenty-server/src/engine/metadata-modules/ai/ai-agent-execution/constants/agent-run-outcome-delivery-retry-delay-ms.const.ts b/packages/twenty-server/src/engine/metadata-modules/ai/ai-agent-execution/constants/agent-run-outcome-delivery-retry-delay-ms.const.tsnew file mode 100644index 00000000000..0bb2cb903d0--- /dev/null+++ b/packages/twenty-server/src/engine/metadata-modules/ai/ai-agent-execution/constants/agent-run-outcome-delivery-retry-delay-ms.const.ts@@ -0,0 +1,2 @@+// a caller such as a workflow step is marked waiting just after it starts its run+export const AGENT_RUN_OUTCOME_DELIVERY_RETRY_DELAY_MS = 1_000;diff --git a/packages/twenty-server/src/engine/metadata-modules/ai/ai-agent-execution/constants/agent-run-thread-lock.const.ts b/packages/twenty-server/src/engine/metadata-modules/ai/ai-agent-execution/constants/agent-run-thread-lock.const.tsdeleted file mode 100644index 7a8901767df..00000000000--- a/packages/twenty-server/src/engine/metadata-modules/ai/ai-agent-execution/constants/agent-run-thread-lock.const.ts+++ /dev/null@@ -1,3 +0,0 @@-export const AGENT_RUN_THREAD_LOCK_TTL_MS = 900_000;--export const AGENT_RUN_THREAD_LOCK_RETRY_INTERVAL_MS = 500;diff --git a/packages/twenty-server/src/engine/metadata-modules/ai/ai-agent-execution/constants/deliver-agent-run-outcome-job-name.constant.ts b/packages/twenty-server/src/engine/metadata-modules/ai/ai-agent-execution/constants/deliver-agent-run-outcome-job-name.constant.tsnew file mode 100644index 00000000000..5a0adf1a4d2--- /dev/null+++ b/packages/twenty-server/src/engine/metadata-modules/ai/ai-agent-execution/constants/deliver-agent-run-outcome-job-name.constant.ts@@ -0,0 +1 @@+export const DELIVER_AGENT_RUN_OUTCOME_JOB_NAME = 'DeliverAgentRunOutcomeJob';diff --git a/packages/twenty-server/src/engine/metadata-modules/ai/ai-agent-execution/constants/run-agent-job-name.constant.ts b/packages/twenty-server/src/engine/metadata-modules/ai/ai-agent-execution/constants/run-agent-job-name.constant.tsnew file mode 100644index 00000000000..4b35bc7c811--- /dev/null+++ b/packages/twenty-server/src/engine/metadata-modules/ai/ai-agent-execution/constants/run-agent-job-name.constant.ts@@ -0,0 +1 @@+export const RUN_AGENT_JOB_NAME = 'RunAgentJob';diff --git a/packages/twenty-server/src/engine/metadata-modules/ai/ai-agent-execution/constants/run-agent-max-wait-ms.const.ts b/packages/twenty-server/src/engine/metadata-modules/ai/ai-agent-execution/constants/run-agent-max-wait-ms.const.tsnew file mode 100644index 00000000000..43c87c267c5--- /dev/null+++ b/packages/twenty-server/src/engine/metadata-modules/ai/ai-agent-execution/constants/run-agent-max-wait-ms.const.ts@@ -0,0 +1,2 @@+// as long as a run may hold its conversation, so a run that waited this long is still going on+export const RUN_AGENT_MAX_WAIT_MS = 15 * 60_000;diff --git a/packages/twenty-server/src/engine/metadata-modules/ai/ai-agent-execution/constants/run-agent-settled-run-reconcile-interval-ms.const.ts b/packages/twenty-server/src/engine/metadata-modules/ai/ai-agent-execution/constants/run-agent-settled-run-reconcile-interval-ms.const.tsnew file mode 100644index 00000000000..83fed255272--- /dev/null+++ b/packages/twenty-server/src/engine/metadata-modules/ai/ai-agent-execution/constants/run-agent-settled-run-reconcile-interval-ms.const.ts@@ -0,0 +1,2 @@+// pub/sub drops a notice sent while the subscriber reconnects, so the stored settlement is read again+export const RUN_AGENT_SETTLED_RUN_RECONCILE_INTERVAL_MS = 10_000;diff --git a/packages/twenty-server/src/engine/metadata-modules/ai/ai-agent-execution/constants/run-agent-settled-run-ttl-seconds.const.ts b/packages/twenty-server/src/engine/metadata-modules/ai/ai-agent-execution/constants/run-agent-settled-run-ttl-seconds.const.tsnew file mode 100644index 00000000000..af94e77a5bd--- /dev/null+++ b/packages/twenty-server/src/engine/metadata-modules/ai/ai-agent-execution/constants/run-agent-settled-run-ttl-seconds.const.ts@@ -0,0 +1 @@+export const RUN_AGENT_SETTLED_RUN_TTL_SECONDS = 3_600;diff --git a/packages/twenty-server/src/engine/metadata-modules/ai/ai-agent-execution/dtos/run-agent.input.ts b/packages/twenty-server/src/engine/metadata-modules/ai/ai-agent-execution/dtos/run-agent.input.tsindex 086daae26ba..c7bee0cb72b 100644--- a/packages/twenty-server/src/engine/metadata-modules/ai/ai-agent-execution/dtos/run-agent.input.ts+++ b/packages/twenty-server/src/engine/metadata-modules/ai/ai-agent-execution/dtos/run-agent.input.ts@@ -7,6 +7,7 @@ import {ArrayMaxSize,ArrayNotEmpty,IsArray,+ IsBoolean,IsNotEmpty,IsOptional,IsString,@@ -72,4 +73,9 @@ export class RunAgentInputDTO {@Type(() => RunAgentThreadInputDTO)@Field(() => RunAgentThreadInputDTO, { nullable: true })thread?: RunAgentThreadInputDTO;++ @IsOptional()+ @IsBoolean()+ @Field({ nullable: true })+ waitForCompletion?: boolean;}diff --git a/packages/twenty-server/src/engine/metadata-modules/ai/ai-agent-execution/jobs/deliver-agent-run-outcome.job.ts b/packages/twenty-server/src/engine/metadata-modules/ai/ai-agent-execution/jobs/deliver-agent-run-outcome.job.tsnew file mode 100644index 00000000000..41c46c16236--- /dev/null+++ b/packages/twenty-server/src/engine/metadata-modules/ai/ai-agent-execution/jobs/deliver-agent-run-outcome.job.ts@@ -0,0 +1,18 @@+import { Process } from 'src/engine/core-modules/message-queue/decorators/process.decorator';+import { Processor } from 'src/engine/core-modules/message-queue/decorators/processor.decorator';+import { MessageQueue } from 'src/engine/core-modules/message-queue/message-queue.constants';+import { DELIVER_AGENT_RUN_OUTCOME_JOB_NAME } from 'src/engine/metadata-modules/ai/ai-agent-execution/constants/deliver-agent-run-outcome-job-name.constant';+import { AgentRunSuspensionService } from 'src/engine/metadata-modules/ai/ai-agent-execution/services/agent-run-suspension.service';+import { type AgentRunOutcomeDelivery } from 'src/engine/metadata-modules/ai/ai-agent-execution/types/agent-run-outcome-delivery.type';++@Processor(MessageQueue.aiQueue)+export class DeliverAgentRunOutcomeJob {+ constructor(+ private readonly agentRunSuspensionService: AgentRunSuspensionService,+ ) {}++ @Process(DELIVER_AGENT_RUN_OUTCOME_JOB_NAME)+ async handle(delivery: AgentRunOutcomeDelivery) {+ await this.agentRunSuspensionService.deliverOutcome(delivery);+ }+}diff --git a/packages/twenty-server/src/engine/metadata-modules/ai/ai-agent-execution/jobs/run-agent.job.ts b/packages/twenty-server/src/engine/metadata-modules/ai/ai-agent-execution/jobs/run-agent.job.tsnew file mode 100644index 00000000000..ce0babe5d3b--- /dev/null+++ b/packages/twenty-server/src/engine/metadata-modules/ai/ai-agent-execution/jobs/run-agent.job.ts@@ -0,0 +1,16 @@+import { Process } from 'src/engine/core-modules/message-queue/decorators/process.decorator';+import { Processor } from 'src/engine/core-modules/message-queue/decorators/processor.decorator';+import { MessageQueue } from 'src/engine/core-modules/message-queue/message-queue.constants';+import { RUN_AGENT_JOB_NAME } from 'src/engine/metadata-modules/ai/ai-agent-execution/constants/run-agent-job-name.constant';+import { AgentRunnerService } from 'src/engine/metadata-modules/ai/ai-agent-execution/services/agent-runner.service';+import { type RunAgentJobData } from 'src/engine/metadata-modules/ai/ai-agent-execution/types/run-agent-job-data.type';++@Processor(MessageQueue.aiQueue)+export class RunAgentJob {+ constructor(private readonly agentRunnerService: AgentRunnerService) {}++ @Process(RUN_AGENT_JOB_NAME)+ async handle(jobData: RunAgentJobData): Promise<void> {+ await this.agentRunnerService.runStarted(jobData);+ }+}diff --git a/packages/twenty-server/src/engine/metadata-modules/ai/ai-agent-execution/pausing-tools/wait-tools/build-wait-outcome-tool-output.util.ts b/packages/twenty-server/src/engine/metadata-modules/ai/ai-agent-execution/pausing-tools/wait-tools/build-wait-outcome-tool-output.util.tsindex 18ca1d93adf..a1c3acf0208 100644--- a/packages/twenty-server/src/engine/metadata-modules/ai/ai-agent-execution/pausing-tools/wait-tools/build-wait-outcome-tool-output.util.ts+++ b/packages/twenty-server/src/engine/metadata-modules/ai/ai-agent-execution/pausing-tools/wait-tools/build-wait-outcome-tool-output.util.ts@@ -1,9 +1,9 @@-import { type PendingWakeUpOutcome } from 'src/engine/core-modules/pending-wake-up/types/pending-wake-up-outcome.type';+import { type AgentRunWakeUpOutcome } from 'src/engine/metadata-modules/ai/ai-agent-execution/types/agent-run-wake-up-outcome.type';export const buildWaitOutcomeToolOutput = (// a run dropped, or continued on an answer, while it waits cancels its waitoutcome:- | Exclude<PendingWakeUpOutcome, { type: 'ANSWERED' }>+ | Exclude<AgentRunWakeUpOutcome, { type: 'ANSWERED' }>| { type: 'CANCELLED' },): Record<string, unknown> => {switch (outcome.type) {@@ -33,6 +33,13 @@ export const buildWaitOutcomeToolOutput = ('Stopped waiting: the event did not happen before the timeout.',result: { status: 'expired' },};+ case 'MESSAGE_RECEIVED':+ return {+ success: true,+ message:+ 'Stopped waiting: a member wrote in the conversation. Read what they wrote and decide whether to wait again.',+ result: { status: 'cancelled' },+ };case 'CANCELLED':return {success: true,diff --git a/packages/twenty-server/src/engine/metadata-modules/ai/ai-agent-execution/services/__tests__/agent-run-conversation.service.spec.ts b/packages/twenty-server/src/engine/metadata-modules/ai/ai-agent-execution/services/__tests__/agent-run-conversation.service.spec.tsdeleted file mode 100644index c469e3e9e35..00000000000--- a/packages/twenty-server/src/engine/metadata-modules/ai/ai-agent-execution/services/__tests__/agent-run-conversation.service.spec.ts+++ /dev/null@@ -1,207 +0,0 @@-import { FieldActorSource } from 'twenty-shared/types';--import { AgentRunConversationService } from 'src/engine/metadata-modules/ai/ai-agent-execution/services/agent-run-conversation.service';-import { type AgentExecutionResult } from 'src/engine/metadata-modules/ai/ai-agent-execution/types/agent-execution-result.type';-import { AgentMessageRole } from 'src/engine/metadata-modules/ai/ai-history/enums/agent-message-role.enum';--const execution = {- modelId: 'model-id',- hasNoMoreAvailableCredits: false,-} as AgentExecutionResult;--const turn = {- workspaceId: 'workspace-id',- threadId: 'thread-id',- turnId: 'turn-id',-};--const buildService = () => {- const scope = { insert: jest.fn().mockResolvedValue(undefined) };- const conversationWriterService = {- runInTransaction: jest- .fn()- .mockImplementation(async (_workspaceId, work) => work(scope)),- insertTurn: jest.fn().mockResolvedValue('turn-id'),- insertMessage: jest.fn().mockResolvedValue(undefined),- insertExecutionReply: jest- .fn()- .mockResolvedValue({ isAwaitingAnswer: false, replyParts: [] }),- };- const turnRecorderService = {- finishExecutedTurn: jest.fn().mockResolvedValue(undefined),- };- const threadService = {- recordThreadActivity: jest.fn().mockResolvedValue(undefined),- };-- return {- service: new AgentRunConversationService(- { findOne: jest.fn().mockResolvedValue(null) } as never,- conversationWriterService as never,- {} as never,- turnRecorderService as never,- threadService as never,- ),- conversationWriterService,- turnRecorderService,- threadService,- };-};--const closeTurn = (service: AgentRunConversationService) =>- service.closeTurn({- ...turn,- title: 'Draft the quote',- agentId: 'agent-id',- execution,- });--describe('AgentRunConversationService', () => {- it('stores handed-over replies as assistant messages with no sender', async () => {- const { service, conversationWriterService } = buildService();-- await service.openTurn({- workspaceId: 'workspace-id',- threadId: 'thread-id',- title: 'Helper',- agentId: 'agent-id',- senderUserWorkspaceId: 'user-workspace-id',- senderApplicationId: null,- createdBy: {- source: FieldActorSource.MANUAL,- name: 'Tim Apple',- workspaceMemberId: 'workspace-member-id',- context: {},- },- messages: [- { role: 'user', content: 'Who is our biggest customer?' },- { role: 'assistant', content: 'Acme.' },- { role: 'user', content: 'And the second one?' },- ],- });-- expect(conversationWriterService.insertMessage).toHaveBeenCalledTimes(3);- expect(- conversationWriterService.insertMessage.mock.calls.map(([message]) => ({- role: message.role,- agentId: message.agentId,- senderUserWorkspaceId: message.senderUserWorkspaceId,- text: message.parts[0].text,- })),- ).toEqual([- {- role: AgentMessageRole.USER,- agentId: null,- senderUserWorkspaceId: 'user-workspace-id',- text: 'Who is our biggest customer?',- },- {- role: AgentMessageRole.ASSISTANT,- agentId: null,- senderUserWorkspaceId: null,- text: 'Acme.',- },- {- role: AgentMessageRole.USER,- agentId: null,- senderUserWorkspaceId: 'user-workspace-id',- text: 'And the second one?',- },- ]);- });-- it('leaves the turn waiting and brings the conversation back to the inbox when the agent asked a question', async () => {- const {- service,- conversationWriterService,- turnRecorderService,- threadService,- } = buildService();-- conversationWriterService.insertExecutionReply.mockResolvedValue({- isAwaitingAnswer: true,- replyParts: [{ type: 'text', text: 'Which plan?' }],- });-- await expect(closeTurn(service)).resolves.toEqual({- isAwaitingAnswer: true,- });- expect(turnRecorderService.finishExecutedTurn).toHaveBeenCalledWith({- ...turn,- execution,- isWaiting: true,- });- expect(threadService.recordThreadActivity).toHaveBeenCalledWith({- workspaceId: 'workspace-id',- threadId: 'thread-id',- text: 'Which plan?',- });- });-- it('leaves the turn waiting without asking anything when the run paused on a wait', async () => {- const { service, conversationWriterService, turnRecorderService } =- buildService();- const pausedExecution = { ...execution, isPaused: true };-- conversationWriterService.insertExecutionReply.mockResolvedValue({- isAwaitingAnswer: false,- replyParts: [- {- type: 'tool-wait_for_duration',- toolCallId: 'wait-call-id',- state: 'output-available',- input: { minutes: 5 },- output: { success: true, result: { status: 'pending' } },- },- ],- });-- await expect(- service.closeTurn({- ...turn,- title: 'Draft the quote',- agentId: 'agent-id',- execution: pausedExecution,- }),- ).resolves.toEqual({ isAwaitingAnswer: false });- expect(turnRecorderService.finishExecutedTurn).toHaveBeenCalledTimes(1);- expect(turnRecorderService.finishExecutedTurn).toHaveBeenCalledWith({- ...turn,- execution: pausedExecution,- isWaiting: true,- });- });-- it('keeps the answer awaited when the inbox cannot be told', async () => {- const { service, conversationWriterService, threadService } =- buildService();-- conversationWriterService.insertExecutionReply.mockResolvedValue({- isAwaitingAnswer: true,- replyParts: [],- });- threadService.recordThreadActivity.mockRejectedValue(new Error('boom'));-- await expect(closeTurn(service)).resolves.toEqual({- isAwaitingAnswer: true,- });- });-- it('fails the turn when its reply cannot be saved', async () => {- const { service, conversationWriterService, turnRecorderService } =- buildService();-- conversationWriterService.insertExecutionReply.mockRejectedValue(- new Error('question slot taken'),- );-- await expect(closeTurn(service)).rejects.toThrow('question slot taken');-- expect(turnRecorderService.finishExecutedTurn).toHaveBeenCalledTimes(1);- expect(turnRecorderService.finishExecutedTurn).toHaveBeenCalledWith({- ...turn,- execution,- error: expect.objectContaining({ code: expect.any(String) }),- });- });-});diff --git a/packages/twenty-server/src/engine/metadata-modules/ai/ai-agent-execution/services/__tests__/agent-run-suspension.service.spec.ts b/packages/twenty-server/src/engine/metadata-modules/ai/ai-agent-execution/services/__tests__/agent-run-suspension.service.spec.tsdeleted file mode 100644index 8dd98384c53..00000000000--- a/packages/twenty-server/src/engine/metadata-modules/ai/ai-agent-execution/services/__tests__/agent-run-suspension.service.spec.ts+++ /dev/null@@ -1,276 +0,0 @@-import { AgentRunSuspensionService } from 'src/engine/metadata-modules/ai/ai-agent-execution/services/agent-run-suspension.service';-import { AgentTurnStatus } from 'src/engine/metadata-modules/ai/ai-history/enums/agent-turn-status.enum';--const CALLER = {- type: 'WORKFLOW_STEP' as const,- ref: { workflowRunId: 'run-id', stepId: 'step-id' },-};--const CLOSED_QUESTION = {- workspaceId: 'workspace-id',- threadId: 'thread-id',- messageId: 'question-message-id',- activeStreamId: null,- turnStatus: AgentTurnStatus.CANCELLED,-};--const buildService = ({- pendingQuestionMessageId = 'question-message-id' as string | null,- hasPostedCall = true,- wakeUps = [] as object[],- waitMessageIds = [] as string[],-} = {}) => {- const wakeUpRepository = { find: jest.fn().mockResolvedValue(wakeUps) };- const threadRepository = {- findOne: jest- .fn()- .mockResolvedValue(- pendingQuestionMessageId === null- ? null- : { id: 'thread-id', pendingQuestionMessageId },- ),- };- const messagePartRepository = {- query: jest- .fn()- .mockResolvedValue(waitMessageIds.map((messageId) => ({ messageId }))),- existsBy: jest.fn().mockResolvedValue(hasPostedCall),- };- const threadLifecycleService = {- closePendingQuestion: jest.fn().mockResolvedValue(undefined),- };- const turnRecorderService = {- endWaitingTurn: jest.fn().mockResolvedValue(undefined),- };- const pendingWakeUpService = {- claim: jest- .fn()- .mockImplementation(async ({ wakeUpId }) =>- wakeUps.find((wakeUp) => (wakeUp as { id: string }).id === wakeUpId),- ),- };- const onOutcome = jest.fn().mockResolvedValue(undefined);-- const service = new AgentRunSuspensionService(- wakeUpRepository as never,- threadRepository as never,- messagePartRepository as never,- threadLifecycleService as never,- turnRecorderService as never,- pendingWakeUpService as never,- { getHandlerOrThrow: () => ({ onOutcome }) } as never,- {} as never,- );-- return {- service,- threadLifecycleService,- messagePartRepository,- turnRecorderService,- pendingWakeUpService,- onOutcome,- };-};--describe('AgentRunSuspensionService', () => {- describe('closeAwaitedCalls', () => {- it('closes the question a run waiting on an answer asked, and its wait call', async () => {- const { service, threadLifecycleService, messagePartRepository } =- buildService();-- await service.closeAwaitedCalls({- workspaceId: 'workspace-id',- threadId: 'thread-id',- isAwaitingAnswer: true,- });-- expect(messagePartRepository.query).toHaveBeenCalledTimes(1);- expect(threadLifecycleService.closePendingQuestion).toHaveBeenCalledWith(- CLOSED_QUESTION,- );- });-- it('keeps open a question the run did not wait on', async () => {- const { service, threadLifecycleService } = buildService();-- await service.closeAwaitedCalls({- workspaceId: 'workspace-id',- threadId: 'thread-id',- isAwaitingAnswer: false,- });-- expect(- threadLifecycleService.closePendingQuestion,- ).not.toHaveBeenCalled();- });-- it('does nothing more when the conversation waits on no question', async () => {- const { service, threadLifecycleService } = buildService({- pendingQuestionMessageId: null,- });-- await service.closeAwaitedCalls({- workspaceId: 'workspace-id',- threadId: 'thread-id',- isAwaitingAnswer: true,- });-- expect(- threadLifecycleService.closePendingQuestion,- ).not.toHaveBeenCalled();- });-- it('cancels the turn that waited on the dropped wait', async () => {- const { service, turnRecorderService } = buildService({- pendingQuestionMessageId: null,- waitMessageIds: ['wait-message-id'],- });-- await service.closeAwaitedCalls({- workspaceId: 'workspace-id',- threadId: 'thread-id',- isAwaitingAnswer: false,- });-- expect(turnRecorderService.endWaitingTurn).toHaveBeenCalledTimes(1);- expect(turnRecorderService.endWaitingTurn).toHaveBeenCalledWith({- workspaceId: 'workspace-id',- messageId: 'wait-message-id',- status: AgentTurnStatus.CANCELLED,- });- });- });-- describe('recordWaitOutcome', () => {- it('completes the turn that waited on a wait that is over', async () => {- const { service, turnRecorderService } = buildService({- waitMessageIds: ['wait-message-id'],- });-- await service.recordWaitOutcome({- workspaceId: 'workspace-id',- threadId: 'thread-id',- outcome: { type: 'TIME_ELAPSED' },- });-- expect(turnRecorderService.endWaitingTurn).toHaveBeenCalledTimes(1);- expect(turnRecorderService.endWaitingTurn).toHaveBeenCalledWith({- workspaceId: 'workspace-id',- messageId: 'wait-message-id',- status: AgentTurnStatus.COMPLETED,- });- });-- it('goes on when the turn cannot be ended', async () => {- const { service, turnRecorderService } = buildService({- waitMessageIds: ['wait-message-id'],- });-- turnRecorderService.endWaitingTurn.mockRejectedValue(new Error('boom'));-- await expect(- service.recordWaitOutcome({- workspaceId: 'workspace-id',- threadId: 'thread-id',- outcome: { type: 'EXPIRED' },- }),- ).resolves.toBeUndefined();- });- });-- describe('closePostedCalls', () => {- const cancelledWakeUp = {- condition: { type: 'ANSWER', threadId: 'thread-id', toolCallId: 'call' },- };-- it('closes the question that holds the call a cancelled wake-up waited on', async () => {- const { service, threadLifecycleService } = buildService();-- await service.closePostedCalls({- workspaceId: 'workspace-id',- cancelledWakeUps: [cancelledWakeUp, { condition: { type: 'TIME' } }],- } as never);-- expect(threadLifecycleService.closePendingQuestion).toHaveBeenCalledTimes(- 1,- );- expect(threadLifecycleService.closePendingQuestion).toHaveBeenCalledWith(- CLOSED_QUESTION,- );- });-- it('keeps open a question that asks something else', async () => {- const { service, threadLifecycleService } = buildService({- hasPostedCall: false,- });-- await service.closePostedCalls({- workspaceId: 'workspace-id',- cancelledWakeUps: [cancelledWakeUp],- } as never);-- expect(- threadLifecycleService.closePendingQuestion,- ).not.toHaveBeenCalled();- });- });-- describe('releaseForCaller', () => {- it('drops the runs the caller waits on and closes the questions they asked', async () => {- const { service, pendingWakeUpService, threadLifecycleService } =- buildService({- wakeUps: [- {- id: 'wake-up-id',- ownerId: 'thread-id',- condition: { type: 'ANSWER', threadId: 'thread-id' },- payload: { caller: CALLER },- },- ],- });-- await service.releaseForCaller({- workspaceId: 'workspace-id',- caller: { type: 'WORKFLOW_STEP', ref: { workflowRunId: 'run-id' } },- });-- expect(pendingWakeUpService.claim).toHaveBeenCalledWith({- workspaceId: 'workspace-id',- wakeUpId: 'wake-up-id',- });- expect(threadLifecycleService.closePendingQuestion).toHaveBeenCalledWith(- CLOSED_QUESTION,- );- });- });-- describe('settle', () => {- it('hands the outcome and the run so far to the caller', async () => {- const { service, onOutcome, threadLifecycleService } = buildService();- const summary = { totalCredits: 1 } as never;-- await service.settle({- workspaceId: 'workspace-id',- threadId: 'thread-id',- suspension: {- caller: CALLER,- runSpec: {} as never,- summary,- continuationCount: 2,- },- outcome: { status: 'FAILED', error: 'boom' },- });-- expect(- threadLifecycleService.closePendingQuestion,- ).not.toHaveBeenCalled();- expect(onOutcome).toHaveBeenCalledWith({- workspaceId: 'workspace-id',- caller: CALLER,- threadId: 'thread-id',- outcome: { status: 'FAILED', error: 'boom' },- summary,- });- });- });-});diff --git a/packages/twenty-server/src/engine/metadata-modules/ai/ai-agent-execution/services/__tests__/agent-run.service.spec.ts b/packages/twenty-server/src/engine/metadata-modules/ai/ai-agent-execution/services/__tests__/agent-run.service.spec.tsdeleted file mode 100644index 4e55cbcebe1..00000000000--- a/packages/twenty-server/src/engine/metadata-modules/ai/ai-agent-execution/services/__tests__/agent-run.service.spec.ts+++ /dev/null@@ -1,564 +0,0 @@-import { Logger } from '@nestjs/common';--import {- ApiKeyException,- ApiKeyExceptionCode,-} from 'src/engine/core-modules/api-key/exceptions/api-key.exception';-import { AgentRunService } from 'src/engine/metadata-modules/ai/ai-agent-execution/services/agent-run.service';-import { buildAgentRunThreadId } from 'src/engine/metadata-modules/ai/ai-agent-execution/utils/build-agent-run-thread-id.util';-import {- AiException,- AiExceptionCode,-} from 'src/engine/metadata-modules/ai/ai.exception';--const WORKSPACE = { id: 'workspace-id' } as never;-const APPLICATION = { id: 'application-id' };-const AGENT = {- id: 'agent-id',- label: 'Slack Assistant',- applicationId: APPLICATION.id,- universalIdentifier: 'slack-assistant',-};-const RUN_AS_USER_WORKSPACE_ID = 'user-workspace-id';-const RUN_AS_ACTOR = {- source: 'API',- name: 'Tim Apple',- workspaceMemberId: 'workspace-member-id',- context: {},-};-const buildService = () => {- const agentRunnerService = {- run: jest.fn().mockImplementation(async ({ conversation }) => ({- threadId: conversation.threadId,- outcome: {- status: 'COMPLETED',- result: { response: 'Acme is your biggest customer' },- },- })),- };- const agentActorContextService = {- buildRunAsWorkspaceMemberContext: jest.fn().mockResolvedValue({- actorContext: RUN_AS_ACTOR,- authContext: {- type: 'user',- userWorkspaceId: RUN_AS_USER_WORKSPACE_ID,- workspaceMemberId: RUN_AS_ACTOR.workspaceMemberId,- },- roleId: 'role-id',- }),- buildApplicationAgentContext: jest.fn().mockResolvedValue({- application: APPLICATION,- authContext: {- type: 'application',- workspace: WORKSPACE,- application: APPLICATION,- },- agentRoleId: 'agent-role-id',- }),- };- const apiKeyRoleService = {- getRoleIdForApiKeyId: jest.fn().mockResolvedValue('api-key-role-id'),- };- const apiKeyService = {- validateApiKey: jest.fn().mockResolvedValue({- id: 'api-key-id',- workspaceId: 'workspace-id',- name: 'Agent integration',- expiresAt: new Date('2099-01-01'),- createdAt: new Date('2026-01-01'),- updatedAt: new Date('2026-01-01'),- revokedAt: null,- }),- };-- const service = new AgentRunService(- agentActorContextService as never,- agentRunnerService as never,- {} as never,- apiKeyRoleService as never,- apiKeyService as never,- { findById: jest.fn().mockResolvedValue(APPLICATION) } as never,- { findOne: jest.fn().mockResolvedValue(AGENT) } as never,- );-- return {- service,- agentRunnerService,- agentActorContextService,- apiKeyRoleService,- apiKeyService,- };-};--const runInput = (agentRunnerService: { run: jest.Mock }) =>- agentRunnerService.run.mock.calls[0][0];--const run = (- service: AgentRunService,- input: Record<string, unknown>,- {- isCalledByApplication = true,- requestUserWorkspaceId = null,- requestWorkspaceMemberId = null,- }: {- isCalledByApplication?: boolean;- requestUserWorkspaceId?: string | null;- requestWorkspaceMemberId?: string | null;- } = {},-) =>- service.run({- workspace: WORKSPACE,- requestUserWorkspaceId,- requestWorkspaceMemberId,- callerApplication: isCalledByApplication- ? (APPLICATION as never)- : undefined,- input: { agentUniversalIdentifier: AGENT.universalIdentifier, ...input },- });--const userInput = (content: string) => [{ role: 'user', content }];--describe('AgentRunService', () => {- it('runs without a thread in a new conversation of its own', async () => {- const { service, agentRunnerService } = buildService();-- const result = await run(service, {- input: userInput('Who is our biggest customer?'),- });- const secondResult = await run(service, {- input: userInput('Who is our biggest customer?'),- });-- expect(result).toEqual({- threadId: expect.any(String),- status: 'COMPLETED',- result: { response: 'Acme is your biggest customer' },- error: null,- success: true,- });- expect(secondResult.threadId).not.toBe(result.threadId);-- const { conversation, spec, prompt, executionContext } =- runInput(agentRunnerService);-- expect(conversation).toEqual({- threadId: result.threadId,- isCreated: true,- });- expect(spec).toMatchObject({- title: AGENT.label,- toolLoadingStrategy: 'lazy',- capabilities: {- canAskHumans: false,- },- });- expect(prompt).toEqual({- senderUserWorkspaceId: null,- senderApplicationId: APPLICATION.id,- messages: userInput('Who is our biggest customer?'),- });- expect(executionContext).toMatchObject({- rolePermissionConfig: {- intersectionOf: { agentRoleId: 'agent-role-id' },- },- conversationActor: { type: 'application', applicationId: APPLICATION.id },- turnCreatedBy: { source: 'APPLICATION' },- });- });-- it('keeps the replies a run without a thread hands over as its input', async () => {- const { service, agentRunnerService } = buildService();- const history = [- { role: 'user', content: 'Who is our biggest customer?' },- { role: 'assistant', content: 'Acme.' },- { role: 'user', content: 'And the second one?' },- ];-- await run(service, { input: history });-- expect(runInput(agentRunnerService).prompt.messages).toEqual(history);- });-- it('keeps accepting a prompt', async () => {- const { service, agentRunnerService } = buildService();-- await run(service, { prompt: 'Who is our biggest customer?' });-- expect(runInput(agentRunnerService).prompt.messages).toEqual(- userInput('Who is our biggest customer?'),- );- });-- it('continues the thread as the member it runs as', async () => {- const { service, agentRunnerService } = buildService();- const threadId = buildAgentRunThreadId({- applicationId: APPLICATION.id,- agentId: AGENT.id,- threadKey: 'C123:1700000000.000100',- });-- const result = await run(service, {- input: userInput('And the second one?'),- thread: { key: 'C123:1700000000.000100' },- runAsWorkspaceMemberId: 'workspace-member-id',- });-- const { conversation, caller, prompt, executionContext } =- runInput(agentRunnerService);-- expect(result.threadId).toBe(threadId);- expect(conversation).toEqual({ threadId, isCreated: false });- expect(caller).toEqual({- type: 'AGENT_API_RUN',- ref: {- agentId: AGENT.id,- runAsWorkspaceMemberId: 'workspace-member-id',- requestUserWorkspaceId: null,- createdBy: RUN_AS_ACTOR,- },- });- expect(prompt).toMatchObject({- senderUserWorkspaceId: RUN_AS_USER_WORKSPACE_ID,- });- expect(executionContext).toMatchObject({- actorContext: RUN_AS_ACTOR,- turnCreatedBy: RUN_AS_ACTOR,- userWorkspaceId: RUN_AS_USER_WORKSPACE_ID,- agentRunRoleIds: { userWorkspaceRoleId: 'role-id' },- rolePermissionConfig: {- intersectionOf: {- agentRoleId: 'agent-role-id',- userWorkspaceRoleId: 'role-id',- },- },- conversationActor: {- type: 'user',- workspaceMemberId: RUN_AS_ACTOR.workspaceMemberId,- },- });- });-- it('keeps the additional instructions with the run rather than in its messages', async () => {- const { service, agentRunnerService } = buildService();-- await run(service, {- input: userInput('And the second one?'),- thread: { key: 'thread', title: 'Acme renewal' },- additionalInstructions: 'Answer in Slack markdown',- });-- const { spec, prompt } = runInput(agentRunnerService);-- expect(spec).toMatchObject({- title: 'Acme renewal',- instructions: 'Answer in Slack markdown',- });- expect(prompt).toMatchObject({- messages: userInput('And the second one?'),- senderUserWorkspaceId: null,- });- });-- it('keeps the output format the call asks for with the run', async () => {- const { service, agentRunnerService } = buildService();- const responseFormat = {- type: 'json' as const,- schema: {- type: 'object' as const,- properties: { summary: { type: 'string' as const } },- },- };-- await run(service, { input: userInput('Summarize'), responseFormat });-- expect(runInput(agentRunnerService).spec).toMatchObject({ responseFormat });- });-- it('refuses an output format that is not one', async () => {- const { service, agentRunnerService } = buildService();-- await expect(- run(service, {- input: userInput('Summarize'),- responseFormat: { type: 'json', schema: { properties: 'nope' } },- }),- ).rejects.toThrow('Invalid response format');- expect(agentRunnerService.run).not.toHaveBeenCalled();- });-- it('records the member who called without an application as the sender', async () => {- const { service, agentRunnerService } = buildService();-- await run(- service,- { input: userInput('Who is our biggest customer?') },- {- isCalledByApplication: false,- requestUserWorkspaceId: RUN_AS_USER_WORKSPACE_ID,- requestWorkspaceMemberId: RUN_AS_ACTOR.workspaceMemberId,- },- );-- expect(runInput(agentRunnerService).prompt).toMatchObject({- senderUserWorkspaceId: RUN_AS_USER_WORKSPACE_ID,- senderApplicationId: null,- });- });-- it.each([false, true])(- 'keeps the member identity when an implicit member run resumes (application caller: %s)',- async (isCalledByApplication) => {- const { service, agentRunnerService, agentActorContextService } =- buildService();-- await run(- service,- { prompt: 'Hello' },- {- isCalledByApplication,- requestUserWorkspaceId: RUN_AS_USER_WORKSPACE_ID,- requestWorkspaceMemberId: RUN_AS_ACTOR.workspaceMemberId,- },- );-- const { caller, executionContext } = runInput(agentRunnerService);-- expect(executionContext).toMatchObject({- authContext: {- type: 'user',- userWorkspaceId: RUN_AS_USER_WORKSPACE_ID,- },- rolePermissionConfig: {- intersectionOf: {- agentRoleId: 'agent-role-id',- userWorkspaceRoleId: 'role-id',- },- },- turnCreatedBy: RUN_AS_ACTOR,- });-- agentActorContextService.buildRunAsWorkspaceMemberContext.mockResolvedValue(- {- actorContext: RUN_AS_ACTOR,- authContext: {- type: 'user',- userWorkspaceId: RUN_AS_USER_WORKSPACE_ID,- workspaceMemberId: RUN_AS_ACTOR.workspaceMemberId,- },- roleId: 'updated-member-role-id',- },- );-- const resumedContext = await service.buildExecutionContext({- workspaceId: 'workspace-id',- caller: JSON.parse(JSON.stringify(caller)),- });-- expect(resumedContext).toMatchObject({- authContext: executionContext.authContext,- conversationActor: {- type: 'user',- workspaceMemberId: RUN_AS_ACTOR.workspaceMemberId,- },- rolePermissionConfig: {- intersectionOf: {- agentRoleId: 'agent-role-id',- userWorkspaceRoleId: 'updated-member-role-id',- },- },- });- },- );-- it('releases an implicit member run when the member leaves', async () => {- const { service, agentRunnerService, agentActorContextService } =- buildService();-- await run(- service,- { prompt: 'Hello' },- {- isCalledByApplication: false,- requestUserWorkspaceId: RUN_AS_USER_WORKSPACE_ID,- requestWorkspaceMemberId: RUN_AS_ACTOR.workspaceMemberId,- },- );-- agentActorContextService.buildRunAsWorkspaceMemberContext.mockRejectedValue(- new AiException(- 'Member left',- AiExceptionCode.RUN_AS_WORKSPACE_MEMBER_NOT_FOUND,- ),- );-- await expect(- service.getWaitingState({- workspaceId: 'workspace-id',- caller: runInput(agentRunnerService).caller,- }),- ).resolves.toBe('GONE');- });-- it('keeps the API key and reloads its role when a run resumes', async () => {- const { service, agentRunnerService, apiKeyRoleService } = buildService();-- await service.run({- workspace: WORKSPACE,- requestUserWorkspaceId: null,- requestWorkspaceMemberId: null,- callerApiKey: { id: 'api-key-id' } as never,- input: {- agentUniversalIdentifier: AGENT.universalIdentifier,- prompt: 'Hello',- },- });-- const { caller, executionContext } = runInput(agentRunnerService);-- expect(executionContext).toMatchObject({- authContext: { type: 'apiKey', apiKey: { id: 'api-key-id' } },- rolePermissionConfig: {- intersectionOf: {- agentRoleId: 'agent-role-id',- apiKeyRoleId: 'api-key-role-id',- },- },- });-- apiKeyRoleService.getRoleIdForApiKeyId.mockResolvedValue(- 'updated-key-role-id',- );-- await expect(- service.buildExecutionContext({- workspaceId: 'workspace-id',- caller: JSON.parse(JSON.stringify(caller)),- }),- ).resolves.toMatchObject({- authContext: { type: 'apiKey', apiKey: { id: 'api-key-id' } },- rolePermissionConfig: {- intersectionOf: {- agentRoleId: 'agent-role-id',- apiKeyRoleId: 'updated-key-role-id',- },- },- });- });-- it.each([- ApiKeyExceptionCode.API_KEY_REVOKED,- ApiKeyExceptionCode.API_KEY_EXPIRED,- ApiKeyExceptionCode.API_KEY_NOT_FOUND,- ])(- 'releases a waiting run when its API key is unavailable: %s',- async (code) => {- const { service, agentRunnerService, apiKeyService } = buildService();-- await service.run({- workspace: WORKSPACE,- requestUserWorkspaceId: null,- requestWorkspaceMemberId: null,- callerApiKey: { id: 'api-key-id' } as never,- input: {- agentUniversalIdentifier: AGENT.universalIdentifier,- prompt: 'Hello',- },- });-- apiKeyService.validateApiKey.mockRejectedValue(- new ApiKeyException('Key unavailable', code),- );-- await expect(- service.getWaitingState({- workspaceId: 'workspace-id',- caller: runInput(agentRunnerService).caller,- }),- ).resolves.toBe('GONE');- await expect(- service.buildExecutionContext({- workspaceId: 'workspace-id',- caller: runInput(agentRunnerService).caller,- }),- ).rejects.toMatchObject({ code });- },- );-- it('refuses a caller without a member, application or API key', async () => {- const { service, agentRunnerService } = buildService();-- await expect(- run(service, { prompt: 'Hello' }, { isCalledByApplication: false }),- ).rejects.toMatchObject({ code: AiExceptionCode.RUN_AGENT_NOT_ALLOWED });- expect(agentRunnerService.run).not.toHaveBeenCalled();- });-- it('reports a run that ran out of credits', async () => {- const { service, agentRunnerService } = buildService();-- agentRunnerService.run.mockResolvedValue({- threadId: 'thread-id',- outcome: {- status: 'FAILED',- error: 'Agent stopped: no more available credits.',- },- });-- await expect(- run(service, { input: userInput('Hello') }),- ).resolves.toMatchObject({- status: 'FAILED',- success: false,- error: 'Agent stopped: no more available credits.',- });- });-- it('reports a failed run without throwing', async () => {- const { service, agentRunnerService } = buildService();-- jest.spyOn(Logger.prototype, 'error').mockImplementation();- agentRunnerService.run.mockRejectedValue(new Error('provider down'));-- await expect(- run(service, { input: userInput('Hello') }),- ).resolves.toMatchObject({- success: false,- error: 'Agent execution failed.',- });- });-- it('refuses a thread without an application token', async () => {- const { service } = buildService();-- await expect(- run(- service,- { input: userInput('Hello'), thread: { key: 'thread' } },- { isCalledByApplication: false },- ),- ).rejects.toMatchObject({ code: AiExceptionCode.RUN_AGENT_NOT_ALLOWED });- });-- it('refuses a thread with a blank key', async () => {- const { service, agentRunnerService } = buildService();-- await expect(- run(service, { input: userInput('Hello'), thread: { key: ' ' } }),- ).rejects.toMatchObject({ code: AiExceptionCode.INVALID_AGENT_INPUT });- expect(agentRunnerService.run).not.toHaveBeenCalled();- });-- it('refuses assistant messages sent to a thread', async () => {- const { service } = buildService();-- await expect(- run(service, {- thread: { key: 'thread' },- input: [- { role: 'user', content: 'Hello' },- { role: 'assistant', content: 'Hi' },- ],- }),- ).rejects.toMatchObject({ code: AiExceptionCode.INVALID_AGENT_INPUT });- });-});diff --git a/packages/twenty-server/src/engine/metadata-modules/ai/ai-agent-execution/services/__tests__/agent-runner.service.spec.ts b/packages/twenty-server/src/engine/metadata-modules/ai/ai-agent-execution/services/__tests__/agent-runner.service.spec.tsdeleted file mode 100644index e5588bf89fa..00000000000--- a/packages/twenty-server/src/engine/metadata-modules/ai/ai-agent-execution/services/__tests__/agent-runner.service.spec.ts+++ /dev/null@@ -1,535 +0,0 @@-import { Logger } from '@nestjs/common';--import { UsageOperationType } from 'src/engine/core-modules/usage/enums/usage-operation-type.enum';-import { AgentRunnerService } from 'src/engine/metadata-modules/ai/ai-agent-execution/services/agent-runner.service';-import { type AgentExecutionResult } from 'src/engine/metadata-modules/ai/ai-agent-execution/types/agent-execution-result.type';-import { type AgentRunnerRunInput } from 'src/engine/metadata-modules/ai/ai-agent-execution/types/agent-runner-run-input.type';-import { AgentEntity } from 'src/engine/metadata-modules/ai/ai-agent/entities/agent.entity';-import { type AgentResponseFormat } from 'src/engine/metadata-modules/ai/ai-agent/types/agent-response-format.type';--const CREATED_BY = {- source: 'WORKFLOW',- name: 'New deals',- workspaceMemberId: null,- context: {},-} as AgentRunnerRunInput['executionContext']['turnCreatedBy'];--const PRIOR_MESSAGES = [{ id: 'message-id', role: 'assistant', parts: [] }];--const MESSAGES = [{ role: 'user' as const, content: 'Draft a quote' }];--const AGENT = { id: 'agent-id' } as AgentEntity;--const buildExecution = (- overrides: Partial<AgentExecutionResult> = {},-): AgentExecutionResult => ({- result: { answer: 'done' },- usage: {- inputTokens: 1,- outputTokens: 2,- totalTokens: 3,- } as AgentExecutionResult['usage'],- cacheCreationTokens: 0,- nativeWebSearchCallCount: 0,- hasNoMoreAvailableCredits: false,- isPaused: false,- steps: [],- modelId: 'model-id',- totalCostInDollars: 0,- creditsUsedMicro: 0,- turnUsage: {- inputTokens: 1,- outputTokens: 2,- cacheReadTokens: 0,- cacheCreationTokens: 0,- inputCredits: 0,- outputCredits: 0,- },- ...overrides,-});--const buildAgent = (responseFormat: AgentResponseFormat): AgentEntity =>- Object.assign(new AgentEntity(), { id: 'agent-id', responseFormat });--const RUN_INPUT: AgentRunnerRunInput = {- workspaceId: 'workspace-id',- conversation: { threadId: 'thread-id', isCreated: false },- caller: {- type: 'WORKFLOW_STEP',- ref: { workflowRunId: 'run', stepId: 'step' },- },- spec: {- agentId: 'agent-id',- title: 'Draft the quote',- baseSystemPrompt: 'base prompt',- instructions: null,- capabilities: {- canAskHumans: false,- },- },- agent: AGENT,- prompt: {- messages: MESSAGES,- senderUserWorkspaceId: 'user-workspace-id',- senderApplicationId: null,- },- executionContext: {- authContext: {} as never,- userWorkspaceId: 'user-workspace-id',- rolePermissionConfig: { intersectionOf: {} },- conversationActor: { type: 'application', applicationId: 'app-id' },- turnCreatedBy: CREATED_BY,- usageOperationType: UsageOperationType.AI_WORKFLOW_TOKEN,- },-};--const SUSPENSION = {- caller: RUN_INPUT.caller,- runSpec: RUN_INPUT.spec,- summary: null,- continuationCount: 0,-};--const CONTINUATION = {- workspaceId: 'workspace-id',- threadId: 'thread-id',- wakeUpId: 'wake-up-id',- outcome: { type: 'ANSWERED' as const, answer: { result: {} } },-};--const buildService = (execution = buildExecution()) => {- const agentAsyncExecutorService = {- executeAgent: jest.fn().mockResolvedValue(execution),- };- const agentRunConversationService = {- openTurn: jest.fn().mockResolvedValue('turn-id'),- closeTurn: jest.fn().mockResolvedValue({ isAwaitingAnswer: false }),- failTurn: jest.fn().mockResolvedValue(undefined),- withThreadLock: jest.fn(({ work }) => work()),- };- const conversationReaderService = {- loadMessages: jest.fn().mockResolvedValue(PRIOR_MESSAGES),- };-- const agentRunSuspensionService = {- assertConversationNotSuspended: jest.fn().mockResolvedValue(undefined),- closeAwaitedCalls: jest.fn().mockResolvedValue(undefined),- suspend: jest.fn().mockResolvedValue(undefined),- settle: jest.fn().mockResolvedValue(undefined),- recordWaitOutcome: jest.fn().mockResolvedValue(undefined),- };- const callerHandler = {- getWaitingState: jest.fn().mockResolvedValue('WAITING'),- buildExecutionContext: jest- .fn()- .mockResolvedValue(RUN_INPUT.executionContext),- };-- const pendingWakeUpService = {- find: jest.fn().mockResolvedValue({- condition: { type: 'ANSWER', threadId: 'thread-id' },- payload: SUSPENSION,- }),- claim: jest.fn().mockResolvedValue(null),- cancel: jest.fn().mockResolvedValue([]),- };-- jest.spyOn(Logger.prototype, 'error').mockImplementation();-- const service = new AgentRunnerService(- agentAsyncExecutorService as never,- agentRunConversationService as never,- conversationReaderService as never,- agentRunSuspensionService as never,- { getHandlerOrThrow: () => callerHandler } as never,- pendingWakeUpService as never,- { findOne: jest.fn().mockResolvedValue(AGENT) } as never,- );-- return {- service,- agentAsyncExecutorService,- agentRunConversationService,- conversationReaderService,- agentRunSuspensionService,- callerHandler,- pendingWakeUpService,- };-};--describe('AgentRunnerService', () => {- it('continues a conversation under its lock and records the turn', async () => {- const {- service,- agentAsyncExecutorService,- agentRunConversationService,- conversationReaderService,- } = buildService();-- const { threadId, outcome, summary } = await service.run(RUN_INPUT);-- expect(threadId).toBe('thread-id');- expect(outcome).toEqual({- status: 'COMPLETED',- result: { answer: 'done' },- });- expect(summary).toMatchObject({ modelId: 'model-id', toolCalls: [] });- expect(agentRunConversationService.withThreadLock).toHaveBeenCalledWith(- expect.objectContaining({- workspaceId: 'workspace-id',- threadId: 'thread-id',- }),- );- expect(conversationReaderService.loadMessages).toHaveBeenCalledWith({- workspaceId: 'workspace-id',- threadId: 'thread-id',- actor: { type: 'application', applicationId: 'app-id' },- });- expect(agentRunConversationService.openTurn).toHaveBeenCalledWith({- workspaceId: 'workspace-id',- threadId: 'thread-id',- title: 'Draft the quote',- agentId: 'agent-id',- senderUserWorkspaceId: 'user-workspace-id',- senderApplicationId: null,- createdBy: CREATED_BY,- messages: MESSAGES,- });- expect(agentAsyncExecutorService.executeAgent).toHaveBeenCalledWith(- expect.objectContaining({- messages: MESSAGES,- baseSystemPrompt: expect.stringMatching(- /^base prompt\n\n.*wait_for_event/,- ),- priorMessages: PRIOR_MESSAGES,- executionContext: RUN_INPUT.executionContext,- }),- );- expect(agentRunConversationService.closeTurn).toHaveBeenCalledWith(- expect.objectContaining({- turnId: 'turn-id',- title: 'Draft the quote',- }),- );- });-- it("asks for the caller's output format over the agent's own", async () => {- const { service, agentAsyncExecutorService } = buildService();- const callerResponseFormat = {- type: 'json' as const,- schema: {- type: 'object' as const,- properties: { summary: { type: 'string' as const } },- },- };-- await service.run({- ...RUN_INPUT,- agent: buildAgent({ type: 'text' }),- spec: { ...RUN_INPUT.spec, responseFormat: callerResponseFormat },- });-- expect(agentAsyncExecutorService.executeAgent).toHaveBeenCalledTimes(1);- expect(agentAsyncExecutorService.executeAgent).toHaveBeenCalledWith(- expect.objectContaining({ responseFormat: callerResponseFormat }),- );- });-- it("falls back to the agent's output format when the caller gives none", async () => {- const { service, agentAsyncExecutorService } = buildService();- const agentResponseFormat = {- type: 'json' as const,- schema: {- type: 'object' as const,- properties: { summary: { type: 'string' as const } },- },- };-- await service.run({- ...RUN_INPUT,- agent: buildAgent(agentResponseFormat),- });-- expect(agentAsyncExecutorService.executeAgent).toHaveBeenCalledTimes(1);- expect(agentAsyncExecutorService.executeAgent).toHaveBeenCalledWith(- expect.objectContaining({ responseFormat: agentResponseFormat }),- );- });-- it.each([true, false])(- 'offers every human-input tool only to a run that can ask humans (%s)',- async (canAskHumans) => {- const { service, agentAsyncExecutorService } = buildService();-- await service.run({- ...RUN_INPUT,- spec: { ...RUN_INPUT.spec, capabilities: { canAskHumans } },- });-- expect(agentAsyncExecutorService.executeAgent).toHaveBeenCalledWith(- expect.objectContaining({ canAskHumans }),- );- },- );-- it('locks a conversation it just created without reading it', async () => {- const {- service,- agentRunConversationService,- conversationReaderService,- agentAsyncExecutorService,- } = buildService();-- await service.run({- ...RUN_INPUT,- conversation: { threadId: 'thread-id', isCreated: true },- });-- expect(agentRunConversationService.withThreadLock).toHaveBeenCalledWith(- expect.objectContaining({ threadId: 'thread-id' }),- );- expect(conversationReaderService.loadMessages).not.toHaveBeenCalled();- expect(agentAsyncExecutorService.executeAgent).toHaveBeenCalledWith(- expect.objectContaining({ priorMessages: [] }),- );- });-- it('still runs the agent when the turn cannot be opened', async () => {- const { service, agentAsyncExecutorService, agentRunConversationService } =- buildService();-- agentRunConversationService.openTurn.mockRejectedValue(- new Error('db down'),- );-- const { outcome } = await service.run(RUN_INPUT);-- expect(agentAsyncExecutorService.executeAgent).toHaveBeenCalled();- expect(agentRunConversationService.closeTurn).not.toHaveBeenCalled();- expect(outcome).toEqual({- status: 'COMPLETED',- result: { answer: 'done' },- });- });-- it('fails the turn of a run that throws', async () => {- const { service, agentAsyncExecutorService, agentRunConversationService } =- buildService();- const error = new Error('provider down');-- agentAsyncExecutorService.executeAgent.mockRejectedValue(error);-- await expect(service.run(RUN_INPUT)).rejects.toThrow('provider down');- expect(agentRunConversationService.failTurn).toHaveBeenCalledWith({- workspaceId: 'workspace-id',- turnId: 'turn-id',- error,- });- expect(agentRunConversationService.closeTurn).not.toHaveBeenCalled();- });-- it('fails a run that ran out of credits', async () => {- const { service } = buildService(- buildExecution({ hasNoMoreAvailableCredits: true }),- );-- await expect(service.run(RUN_INPUT)).resolves.toMatchObject({- outcome: {- status: 'FAILED',- error: 'Agent stopped: no more available credits.',- },- });- });-- it('suspends a run that asked a question on an answer to it', async () => {- const { service, agentRunConversationService, agentRunSuspensionService } =- buildService(buildExecution({ isPaused: true }));-- agentRunConversationService.closeTurn.mockResolvedValue({- isAwaitingAnswer: true,- });-- const { outcome } = await service.run(RUN_INPUT);-- expect(outcome).toEqual({ status: 'SUSPENDED' });- expect(agentRunSuspensionService.suspend).toHaveBeenCalledWith({- workspaceId: 'workspace-id',- threadId: 'thread-id',- condition: { type: 'ANSWER', threadId: 'thread-id' },- suspension: expect.objectContaining({- caller: RUN_INPUT.caller,- runSpec: RUN_INPUT.spec,- continuationCount: 0,- }),- });- expect(agentRunSuspensionService.closeAwaitedCalls).not.toHaveBeenCalled();- });-- it('stops a continued run that keeps pausing and closes what it asked', async () => {- const {- service,- agentRunConversationService,- agentRunSuspensionService,- conversationReaderService,- pendingWakeUpService,- } = buildService(buildExecution({ isPaused: true }));-- agentRunConversationService.closeTurn.mockResolvedValue({- isAwaitingAnswer: true,- });- pendingWakeUpService.find.mockResolvedValue({- condition: { type: 'ANSWER', threadId: 'thread-id' },- payload: { ...SUSPENSION, continuationCount: 49 },- });-- await service.continue(CONTINUATION);-- expect(agentRunConversationService.withThreadLock).toHaveBeenCalledTimes(1);- expect(pendingWakeUpService.claim).toHaveBeenCalledWith({- workspaceId: 'workspace-id',- wakeUpId: 'wake-up-id',- });- expect(conversationReaderService.loadMessages).toHaveBeenCalled();- expect(agentRunSuspensionService.suspend).not.toHaveBeenCalled();- expect(agentRunSuspensionService.closeAwaitedCalls).toHaveBeenCalledWith({- workspaceId: 'workspace-id',- threadId: 'thread-id',- isAwaitingAnswer: true,- });- expect(agentRunSuspensionService.settle).toHaveBeenCalledWith(- expect.objectContaining({- threadId: 'thread-id',- outcome: {- status: 'FAILED',- error: 'Agent stopped: it paused more than 50 times in one run.',- },- }),- );- });-- it('drops a continuation whose caller stopped waiting', async () => {- const { service, callerHandler, agentAsyncExecutorService } =- buildService();-- callerHandler.getWaitingState.mockResolvedValue('GONE');-- await service.continue(CONTINUATION);-- expect(agentAsyncExecutorService.executeAgent).not.toHaveBeenCalled();- });-- it('continues a pause once, as a second continuation finds its wake-up removed', async () => {- const { service, pendingWakeUpService, agentAsyncExecutorService } =- buildService();-- pendingWakeUpService.find.mockResolvedValue(null);-- await service.continue(CONTINUATION);-- expect(agentAsyncExecutorService.executeAgent).not.toHaveBeenCalled();- });-- it('continues a run again when the worker running it died, as its wake-up is still there', async () => {- const {- service,- pendingWakeUpService,- agentAsyncExecutorService,- agentRunSuspensionService,- } = buildService();-- agentAsyncExecutorService.executeAgent.mockReturnValueOnce(- new Promise(() => {}),- );-- void service.continue(CONTINUATION);-- while (agentAsyncExecutorService.executeAgent.mock.calls.length === 0) {- await Promise.resolve();- }-- expect(pendingWakeUpService.claim).not.toHaveBeenCalled();-- await service.continue(CONTINUATION);-- expect(agentAsyncExecutorService.executeAgent).toHaveBeenCalledTimes(2);- expect(agentRunSuspensionService.settle).toHaveBeenCalledTimes(1);- expect(pendingWakeUpService.claim).toHaveBeenCalledWith({- workspaceId: 'workspace-id',- wakeUpId: 'wake-up-id',- });- });-- it('keeps the wake-up of a run whose caller could not be handed the outcome', async () => {- const { service, pendingWakeUpService, agentRunSuspensionService } =- buildService();-- agentRunSuspensionService.settle.mockRejectedValue(new Error('db down'));-- await expect(service.continue(CONTINUATION)).rejects.toThrow('db down');-- expect(pendingWakeUpService.claim).not.toHaveBeenCalled();- });-- it('fails the run when its answer could not be delivered', async () => {- const { service, agentRunSuspensionService, agentAsyncExecutorService } =- buildService();-- await service.continue({- ...CONTINUATION,- outcome: { type: 'ANSWERED', answer: { error: 'answer lost' } },- });-- expect(agentAsyncExecutorService.executeAgent).not.toHaveBeenCalled();- expect(agentRunSuspensionService.settle).toHaveBeenCalledWith(- expect.objectContaining({- outcome: { status: 'FAILED', error: 'answer lost' },- isAwaitingAnswer: true,- }),- );- });-- it('fails a claimed run whose caller cannot be read, rather than dropping it', async () => {- const { service, callerHandler, agentRunSuspensionService } =- buildService();-- callerHandler.getWaitingState.mockRejectedValue(new Error('db down'));-- await service.continue(CONTINUATION);-- expect(agentRunSuspensionService.settle).toHaveBeenCalledWith(- expect.objectContaining({- outcome: { status: 'FAILED', error: 'db down' },- }),- );- });-- it('drops the pause of a continued run whose caller stopped waiting while it ran', async () => {- const {- service,- callerHandler,- agentRunConversationService,- agentRunSuspensionService,- pendingWakeUpService,- } = buildService(buildExecution({ isPaused: true }));-- agentRunConversationService.closeTurn.mockResolvedValue({- isAwaitingAnswer: true,- });- callerHandler.getWaitingState- .mockResolvedValueOnce('WAITING')- .mockResolvedValueOnce('GONE');-- await service.continue(CONTINUATION);-- expect(agentRunSuspensionService.suspend).toHaveBeenCalled();- expect(pendingWakeUpService.cancel).toHaveBeenCalledWith({- workspaceId: 'workspace-id',- owner: { type: 'AGENT_RUN', id: 'thread-id' },- });- expect(agentRunSuspensionService.closeAwaitedCalls).toHaveBeenCalledWith({- workspaceId: 'workspace-id',- threadId: 'thread-id',- isAwaitingAnswer: true,- });- });-});diff --git a/packages/twenty-server/src/engine/metadata-modules/ai/ai-agent-execution/services/agent-async-executor.service.ts b/packages/twenty-server/src/engine/metadata-modules/ai/ai-agent-execution/services/agent-async-executor.service.tsdeleted file mode 100644index 698ddc40f1d..00000000000--- a/packages/twenty-server/src/engine/metadata-modules/ai/ai-agent-execution/services/agent-async-executor.service.ts+++ /dev/null@@ -1,770 +0,0 @@-import { Injectable, Logger } from '@nestjs/common';-import { InjectRepository } from '@nestjs/typeorm';--import {- convertToModelMessages,- generateText,- jsonSchema,- type LanguageModelUsage,- Output,- isStepCount,- type StepResult,- type ToolSet,-} from 'ai';-import {- type RunAgentMessage,- TWENTY_STANDARD_APPLICATION_UNIVERSAL_IDENTIFIER,-} from 'twenty-shared/application';-import {- ASK_QUESTION_TOOL_NAME,- type AgentResponseFormat,- type ExtendedUIMessage,- PROPOSE_TOOL_CALL_TOOL_NAME,- REQUEST_FORM_TOOL_NAME,-} from 'twenty-shared/ai';-import {- isDefined,- isNonEmptyArray,- sortKeysDeeply,- tipTapDocumentToMarkdown,-} from 'twenty-shared/utils';-import { type Repository } from 'typeorm';--import { type FlatApplication } from 'src/engine/core-modules/application/types/flat-application.type';-import { isApplicationAuthContext } from 'src/engine/core-modules/auth/guards/is-application-auth-context.guard';-import { isUserAuthContext } from 'src/engine/core-modules/auth/guards/is-user-auth-context.guard';-import { type WorkspaceAuthContext } from 'src/engine/core-modules/auth/types/workspace-auth-context.type';-import { TOOL_EXECUTION_DURATION_MS_BUCKET_BOUNDARIES } from 'src/engine/core-modules/metrics/constants/tool-execution-duration-ms-bucket-boundaries.constant';-import { TOOL_OUTPUT_TOKENS_BUCKET_BOUNDARIES } from 'src/engine/core-modules/metrics/constants/tool-output-tokens-bucket-boundaries.constant';-import { MetricsService } from 'src/engine/core-modules/metrics/metrics.service';-import { MetricsKeys } from 'src/engine/core-modules/metrics/types/metrics-keys.type';-import { ToolRegistryService } from 'src/engine/core-modules/tool-provider/services/tool-registry.service';-import {- createExecuteToolTool,- createLearnToolsTool,- EXECUTE_TOOL_TOOL_NAME,- LEARN_TOOLS_TOOL_NAME,-} from 'src/engine/core-modules/tool-provider/tools';-import { type ToolContext } from 'src/engine/core-modules/tool-provider/types/tool-context.type';-import { type ToolIndexEntry } from 'src/engine/core-modules/tool-provider/types/tool-index-entry.type';-import { buildToolCatalogSection } from 'src/engine/core-modules/tool-provider/utils/build-tool-catalog-section.util';-import { estimateToolOutputTokens } from 'src/engine/core-modules/tool-provider/utils/estimate-tool-output-tokens.util';-import { getToolMetricName } from 'src/engine/core-modules/tool-provider/utils/get-tool-metric-name.util';-import { isToolOutputSuccessful } from 'src/engine/core-modules/tool-provider/utils/is-tool-output-successful.util';-import { OUTPUT_NAVIGATION_TOOL_NAMES } from 'src/engine/core-modules/tool/tools/output-navigation-tool/constants/output-navigation-tool-names.constant';-import { type ToolOutput } from 'src/engine/core-modules/tool/types/tool-output.type';-import { UsageOperationType } from 'src/engine/core-modules/usage/enums/usage-operation-type.enum';-import { type UsageSpenders } from 'src/engine/core-modules/usage/types/usage-spenders.type';-import { WorkspaceEntity } from 'src/engine/core-modules/workspace/workspace.entity';-import { OPEN_ENDED_AGENT_REGISTRY_TOOL_CATEGORIES } from 'src/engine/metadata-modules/ai/ai-agent-execution/constants/open-ended-agent-registry-tool-categories.const';-import { AGENT_RUN_EXCLUDED_TOOL_NAMES } from 'src/engine/metadata-modules/ai/ai-agent-execution/constants/agent-run-excluded-tool-names.const';-import { PRELOADED_AGENT_REGISTRY_TOOL_CATEGORIES } from 'src/engine/metadata-modules/ai/ai-agent-execution/constants/preloaded-agent-registry-tool-categories.const';-import { type PausingToolCompletionContext } from 'src/engine/metadata-modules/ai/ai-agent-execution/pausing-tools/types/pausing-tool-completion-context.type';-import { endsOnPausingToolCall } from 'src/engine/metadata-modules/ai/ai-agent-execution/pausing-tools/utils/ends-on-pausing-tool-call.util';-import { resolveProposedToolCall } from 'src/engine/metadata-modules/ai/ai-agent-execution/pausing-tools/utils/resolve-proposed-tool-call.util';-import { RunAgentAttachmentService } from 'src/engine/metadata-modules/ai/ai-agent-execution/services/run-agent-attachment.service';-import { type AgentExecutionResult } from 'src/engine/metadata-modules/ai/ai-agent-execution/types/agent-execution-result.type';-import { type AgentRunExecutionContext } from 'src/engine/metadata-modules/ai/ai-agent-execution/types/agent-run-execution-context.type';-import { type AgentToolLoadingStrategy } from 'src/engine/metadata-modules/ai/ai-agent-execution/types/agent-tool-loading-strategy.type';-import { assertAgentResponseFormatHasOutputFieldsOrThrow } from 'src/engine/metadata-modules/ai/ai-agent-execution/utils/assert-agent-response-format-has-output-fields-or-throw.util';-import { type AgentRunRoleIds } from 'src/engine/metadata-modules/ai/ai-agent-execution/types/agent-run-role-ids.type';-import { buildAgentRolePermissionConfig } from 'src/engine/metadata-modules/ai/ai-agent-execution/utils/build-agent-role-permission-config.util';-import { buildStrictAgentResponseSchema } from 'src/engine/metadata-modules/ai/ai-agent-execution/utils/build-strict-agent-response-schema.util';-import { AGENT_CONFIG } from 'src/engine/metadata-modules/ai/ai-agent/constants/agent-config.const';-import { STRUCTURED_OUTPUT_SYSTEM_PROMPT } from 'src/engine/metadata-modules/ai/ai-agent/constants/structured-output-system-prompt.const';-import { type AgentEntity } from 'src/engine/metadata-modules/ai/ai-agent/entities/agent.entity';-import { repairToolCall } from 'src/engine/metadata-modules/ai/ai-agent/utils/repair-tool-call.util';-import { NATIVE_WEB_SEARCH_COST_PER_CALL_DOLLARS } from 'src/engine/metadata-modules/ai/ai-billing/constants/native-web-search-cost-per-call-dollars';-import { AiAgentRoleService } from 'src/engine/metadata-modules/ai/ai-agent-role/ai-agent-role.service';-import { AiBillingService } from 'src/engine/metadata-modules/ai/ai-billing/services/ai-billing.service';-import { convertDollarsToCreditsMicro } from 'src/engine/metadata-modules/ai/ai-billing/utils/convert-dollars-to-credits-micro.util';-import { countNativeWebSearchCallsFromSteps } from 'src/engine/metadata-modules/ai/ai-billing/utils/count-native-web-search-calls-from-steps.util';-import {- extractCacheCreationTokens,- extractCacheCreationTokensFromSteps,-} from 'src/engine/metadata-modules/ai/ai-billing/utils/extract-cache-creation-tokens.util';-import { mergeLanguageModelUsage } from 'src/engine/metadata-modules/ai/ai-billing/utils/merge-language-model-usage.util';-import { getCallLevelProviderOptions } from 'src/engine/metadata-modules/ai/ai-agent-execution/utils/provider-options.util';-import { replaceUnsupportedFileParts } from 'src/engine/metadata-modules/ai/ai-agent-execution/utils/replace-unsupported-file-parts.util';-import { createProposeToolCallTool } from 'src/engine/metadata-modules/ai/ai-agent-execution/tools/propose-tool-call.tool';-import { createAskQuestionTool } from 'src/engine/metadata-modules/ai/ai-chat/tools/ask-question.tool';-import { createRequestFormTool } from 'src/engine/metadata-modules/ai/ai-chat/tools/request-form.tool';-import { buildAiTelemetry } from 'src/engine/metadata-modules/ai/ai-models/utils/build-ai-telemetry.util';-import { AiModelConfigService } from 'src/engine/metadata-modules/ai/ai-models/services/ai-model-config.service';-import { AiModelRegistryService } from 'src/engine/metadata-modules/ai/ai-models/services/ai-model-registry.service';-import { type NativeModelToolOptions } from 'src/engine/metadata-modules/ai/ai-models/types/native-model-tool-options.type';-import {- AiException,- AiExceptionCode,-} from 'src/engine/metadata-modules/ai/ai.exception';--const buildUnavailableToolOutput = (toolName: string): ToolOutput => ({- success: false,- message: `Tool "${toolName}" is not available`,- error: `Tool "${toolName}" is not available to this agent.`,-});--type ProposableTools = {- isToolAllowed: (toolName: string) => boolean;- findTool: (toolName: string) => Promise<ToolIndexEntry | undefined>;- toolContext: ToolContext;-};--type RegistryToolContext = Pick<- ToolContext,- | 'workspaceId'- | 'authContext'- | 'application'- | 'actorContext'- | 'userId'- | 'userWorkspaceId'- | 'agentId'->;--const EMPTY_USAGE: LanguageModelUsage = {- inputTokens: 0,- outputTokens: 0,- totalTokens: 0,- inputTokenDetails: {- noCacheTokens: 0,- cacheReadTokens: 0,- cacheWriteTokens: 0,- },- outputTokenDetails: {- textTokens: 0,- reasoningTokens: 0,- },-};--// workflow tools are offered only to open-ended agents, so a workflow step agent cannot recursively run workflows-@Injectable()-export class AgentAsyncExecutorService {- private readonly logger = new Logger(AgentAsyncExecutorService.name);-- constructor(- private readonly aiModelRegistryService: AiModelRegistryService,- private readonly aiModelConfigService: AiModelConfigService,- private readonly toolRegistry: ToolRegistryService,- private readonly aiBillingService: AiBillingService,- private readonly metricsService: MetricsService,- private readonly runAgentAttachmentService: RunAgentAttachmentService,- private readonly aiAgentRoleService: AiAgentRoleService,- @InjectRepository(WorkspaceEntity)- private readonly workspaceRepository: Repository<WorkspaceEntity>,- ) {}-- private resolveUserIdentity(authContext?: WorkspaceAuthContext): {- userId?: string;- userWorkspaceId?: string;- } {- if (isDefined(authContext) && isUserAuthContext(authContext)) {- return {- userId: authContext.user.id,- userWorkspaceId: authContext.userWorkspaceId,- };- }-- return {};- }-- private resolveCallingApplication(- authContext?: WorkspaceAuthContext,- ): FlatApplication | undefined {- if (!isDefined(authContext)) {- return undefined;- }-- const application = this.getAuthContextApplication(authContext);-- if (- application?.id === authContext.workspace.workspaceCustomApplicationId ||- application?.universalIdentifier ===- TWENTY_STANDARD_APPLICATION_UNIVERSAL_IDENTIFIER- ) {- return undefined;- }-- return application;- }-- private getAuthContextApplication(- authContext: WorkspaceAuthContext,- ): FlatApplication | undefined {- if (isApplicationAuthContext(authContext)) {- return authContext.application;- }-- if (isUserAuthContext(authContext)) {- return authContext.application ?? authContext.viaApplication;- }-- return undefined;- }-- // preloading the few granted object schemas saves the model a learn_tools round trip- private async buildPreloadedRegistryTools({- toolContext,- agentRoleId,- agentRunRoleIds,- additionalExcludedToolNames = [],- }: {- toolContext: RegistryToolContext;- agentRoleId: string;- agentRunRoleIds?: AgentRunRoleIds;- additionalExcludedToolNames?: readonly string[];- }): Promise<{ tools: ToolSet; proposableTools: ProposableTools }> {- const rolePermissionConfig = buildAgentRolePermissionConfig({- agentRoleId,- agentRunRoleIds,- });- const preloadedToolContext = { ...toolContext, rolePermissionConfig };-- const tools = await this.toolRegistry.getToolsByCategories(- { ...preloadedToolContext, explicitObjectGrantsRoleId: agentRoleId },- {- categories: PRELOADED_AGENT_REGISTRY_TOOL_CATEGORIES,- excludeTools: [- ...OUTPUT_NAVIGATION_TOOL_NAMES,- ...AGENT_RUN_EXCLUDED_TOOL_NAMES,- ...additionalExcludedToolNames,- ],- },- );-- // the context lacks the explicit grants the preloaded tools were built with, so calls stay within those tools- const isToolAllowed = (toolName: string): boolean =>- Object.prototype.hasOwnProperty.call(tools, toolName);-- return {- tools,- proposableTools: {- isToolAllowed,- findTool: async (toolName) =>- isToolAllowed(toolName)- ? this.toolRegistry.findCatalogEntry(toolName, preloadedToolContext)- : undefined,- toolContext: preloadedToolContext,- },- };- }-- // open-ended agents have broad access, so preloading would ship every schema: expose a compact catalog plus- // learn_tools / execute_tool instead, scoped by composed role permissions rather than explicit grants only- private async buildLazyRegistryTools({- toolContext: baseToolContext,- agentRoleId,- agentRunRoleIds,- additionalExcludedToolNames = [],- }: {- toolContext: RegistryToolContext;- agentRoleId: string;- agentRunRoleIds?: AgentRunRoleIds;- additionalExcludedToolNames?: readonly string[];- }): Promise<{- tools: ToolSet;- catalogSection: string;- proposableTools: ProposableTools;- }> {- const toolContext: ToolContext = {- ...baseToolContext,- rolePermissionConfig: buildAgentRolePermissionConfig({- agentRoleId,- agentRunRoleIds,- }),- };- const {- workspaceId,- userId,- userWorkspaceId,- rolePermissionConfig,- application,- } = toolContext;-- const fullCatalog = await this.toolRegistry.buildToolIndex(workspaceId, {- userId,- userWorkspaceId,- rolePermissionConfig,- application,- });-- const allowedCategories = new Set(- OPEN_ENDED_AGENT_REGISTRY_TOOL_CATEGORIES,- );- const excludedToolNames = new Set<string>([- ...OUTPUT_NAVIGATION_TOOL_NAMES,- ...AGENT_RUN_EXCLUDED_TOOL_NAMES,- ...additionalExcludedToolNames,- ]);-- const catalog = fullCatalog.filter(- (entry) =>- allowedCategories.has(entry.category) &&- !excludedToolNames.has(entry.name),- );-- // meta-tools are limited to the shown catalog, checked at call time so a tool added later stays unreachable (recursion guard)- const allowedToolNames = new Set(catalog.map((entry) => entry.name));- const isToolAllowed = (toolName: string): boolean =>- allowedToolNames.has(toolName);-- const tools: ToolSet = {- [LEARN_TOOLS_TOOL_NAME]: createLearnToolsTool(- this.toolRegistry,- toolContext,- { isToolAllowed, spillLargeOutput: true },- ),- [EXECUTE_TOOL_TOOL_NAME]: createExecuteToolTool(- this.toolRegistry,- toolContext,- { isToolAllowed, compactOutput: true, spillLargeOutput: true },- ),- };-- return {- tools,- catalogSection: buildToolCatalogSection(catalog, []),- proposableTools: {- isToolAllowed,- findTool: async (toolName) =>- catalog.find((toolIndexEntry) => toolIndexEntry.name === toolName),- toolContext,- },- };- }-- private buildProposeToolCallTool({- isToolAllowed,- findTool,- toolContext,- }: ProposableTools) {- const executeTool: PausingToolCompletionContext['executeTool'] = ({- toolName,- args,- }) =>- isToolAllowed(toolName)- ? this.toolRegistry.resolveAndExecute(toolName, args, toolContext)- : Promise.resolve(buildUnavailableToolOutput(toolName));-- return createProposeToolCallTool({- resolveProposal: (input) =>- resolveProposedToolCall({ input, findTool, executeTool }),- });- }-- async executeAgent({- agent,- messages,- baseSystemPrompt,- workspaceId,- executionContext: {- actorContext,- authContext,- userWorkspaceId,- agentRunRoleIds,- usageOperationType,- },- additionalExcludedToolNames,- toolLoadingStrategy = 'preload',- priorMessages = [],- pausingTools = {},- canAskHumans = false,- responseFormat,- }: {- agent: AgentEntity;- responseFormat?: AgentResponseFormat;- messages: RunAgentMessage[];- // a continued conversation, with the tool calls and results plain run messages cannot carry- priorMessages?: ExtendedUIMessage[];- pausingTools?: ToolSet;- canAskHumans?: boolean;- baseSystemPrompt: string;- workspaceId: string;- executionContext: AgentRunExecutionContext;- additionalExcludedToolNames?: readonly string[];- toolLoadingStrategy?: AgentToolLoadingStrategy;- }): Promise<AgentExecutionResult> {- if (!isNonEmptyArray(messages) && !isNonEmptyArray(priorMessages)) {- throw new AiException(- 'Provide at least one message to run an agent',- AiExceptionCode.INVALID_AGENT_INPUT,- );- }-- assertAgentResponseFormatHasOutputFieldsOrThrow(responseFormat);-- const usageSpenders: UsageSpenders = {- userWorkspaceId,- agentId: agent.id,- applicationId: agent.applicationId,- };-- await this.aiBillingService.assertAiExecutionAllowed({- workspaceId,- operationType: usageOperationType,- spenders: usageSpenders,- });-- let accumulatedUsage: LanguageModelUsage = EMPTY_USAGE;- let cacheCreationTokens = 0;- let nativeWebSearchCallCount = 0;- let executionSteps: StepResult<ToolSet>[] = [];- let resolvedModelId: string | undefined;-- try {- const workspace = await this.workspaceRepository.findOneBy({- id: workspaceId,- });-- this.aiModelRegistryService.validateModelAvailability(agent.modelId);-- const registeredModel =- await this.aiModelRegistryService.resolveModelForAgent(- agent,- workspace ?? undefined,- );-- resolvedModelId = registeredModel.modelId;-- let toolCatalogSection = '';- let proposableTools: ProposableTools | undefined;- const providerOptions = getCallLevelProviderOptions({- sdkPackage: registeredModel.sdkPackage,- providerOptions:- this.aiModelConfigService.getReasoningProviderOptions(- registeredModel,- ),- promptCacheKey: agent.id,- });-- const agentRoleId = await this.aiAgentRoleService.findAgentRoleId({- workspaceId: agent.workspaceId,- agentId: agent.id,- });-- const nativeModelToolOptions: NativeModelToolOptions = {- webSearch: agent.modelConfiguration?.webSearch?.enabled === true,- twitterSearch:- agent.modelConfiguration?.twitterSearch?.enabled === true,- };-- let registryTools: ToolSet = {};-- if (isDefined(agentRoleId)) {- const toolContext: RegistryToolContext = {- workspaceId: agent.workspaceId,- agentId: agent.id,- authContext,- application: this.resolveCallingApplication(authContext),- actorContext,- ...this.resolveUserIdentity(authContext),- };- const registryToolset =- toolLoadingStrategy === 'lazy'- ? await this.buildLazyRegistryTools({- toolContext,- agentRoleId,- agentRunRoleIds,- additionalExcludedToolNames,- })- : {- ...(await this.buildPreloadedRegistryTools({- toolContext,- agentRoleId,- agentRunRoleIds,- additionalExcludedToolNames,- })),- catalogSection: '',- };-- registryTools = registryToolset.tools;- toolCatalogSection = registryToolset.catalogSection;- proposableTools = registryToolset.proposableTools;- }-- const nativeTools = this.aiModelConfigService.getNativeModelTools(- registeredModel,- nativeModelToolOptions,- );-- const tools: ToolSet = {- ...registryTools,- ...nativeTools,- };-- this.logger.log(`Generated ${Object.keys(tools).length} tools for agent`);-- let hasNoMoreAvailableCredits = false;-- const decrementCreditsForStep = async (- step: Pick<StepResult<ToolSet>, 'usage' | 'providerMetadata'>,- ) => {- const { stopReason } =- await this.aiBillingService.decrementAndCheckAvailableCredits({- modelId: registeredModel.modelId,- billingInput: {- usage: step.usage,- cacheCreationTokens: extractCacheCreationTokens(- step.providerMetadata,- ),- },- workspaceId,- operationType: usageOperationType,- spenders: usageSpenders,- });-- if (isDefined(stopReason)) {- hasNoMoreAvailableCredits = true;- }- };-- const modalities = this.aiModelRegistryService.getModelConfig(- registeredModel.modelId,- )?.modalities;-- const priorModelMessages = await convertToModelMessages(- replaceUnsupportedFileParts(priorMessages, modalities, false),- );-- const modelMessages =- await this.runAgentAttachmentService.buildModelMessagesOrThrow({- messages,- workspaceId,- modalities,- });-- const offeredPausingTools: ToolSet = {- ...pausingTools,- ...(canAskHumans- ? {- [ASK_QUESTION_TOOL_NAME]: createAskQuestionTool({- isWorkspaceSetupThread: false,- }),- [REQUEST_FORM_TOOL_NAME]: createRequestFormTool(),- }- : {}),- ...(canAskHumans && isDefined(proposableTools)- ? {- [PROPOSE_TOOL_CALL_TOOL_NAME]:- this.buildProposeToolCallTool(proposableTools),- }- : {}),- };- const offeredToolNames = Object.keys(offeredPausingTools);-- const textResponse = await generateText({- instructions: `${baseSystemPrompt}\n\n${tipTapDocumentToMarkdown(agent.prompt)}${toolCatalogSection}`,- tools: { ...tools, ...offeredPausingTools },- model: registeredModel.model,- messages: [...priorModelMessages, ...modelMessages],- prepareStep: ({ messages }) => ({- messages: messages.map(sortKeysDeeply),- }),- stopWhen: (step) =>- isStepCount(AGENT_CONFIG.MAX_STEPS)(step) ||- endsOnPausingToolCall({ steps: step.steps, offeredToolNames }) ||- hasNoMoreAvailableCredits,- providerOptions,- ...buildAiTelemetry({- functionId: 'agent-execution',- workspaceId,- userWorkspaceId,- agentId: agent.id,- }),- onToolExecutionEnd: (event) => {- this.metricsService.recordHistogram({- key: MetricsKeys.WorkflowAgentToolExecutionDurationMs,- value: event.toolExecutionMs,- unit: 'ms',- attributes: {- model: registeredModel.modelId,- tool: getToolMetricName(event.toolCall.toolName),- },- bucketBoundaries: TOOL_EXECUTION_DURATION_MS_BUCKET_BOUNDARIES,- });- },- onStepEnd: async (step) => {- await decrementCreditsForStep(step);-- for (const part of step.content) {- if (part.type !== 'tool-result' && part.type !== 'tool-error') {- continue;- }-- const succeeded =- part.type === 'tool-result' &&- isToolOutputSuccessful(part.output);-- const toolAttributes = {- model: registeredModel.modelId,- tool: getToolMetricName(part.toolName),- };-- this.metricsService.incrementCounterBy({- key: succeeded- ? MetricsKeys.WorkflowAgentToolExecutionSucceeded- : MetricsKeys.WorkflowAgentToolExecutionFailed,- amount: 1,- attributes: toolAttributes,- });-- this.metricsService.recordHistogram({- key: MetricsKeys.WorkflowAgentToolOutputTokens,- value: estimateToolOutputTokens(- part.type === 'tool-result' ? part.output : part.error,- ),- unit: 'token',- attributes: toolAttributes,- bucketBoundaries: TOOL_OUTPUT_TOKENS_BUCKET_BOUNDARIES,- });- }- },- repairToolCall: async ({- toolCall,- tools: toolsForRepair,- inputSchema,- error,- }) => {- return repairToolCall({- toolCall,- tools: toolsForRepair,- inputSchema,- error,- model: registeredModel.model,- billingContext: {- aiBillingService: this.aiBillingService,- modelId: registeredModel.modelId,- workspaceId,- spenders: usageSpenders,- operationType: usageOperationType,- },- });- },- });-- accumulatedUsage = textResponse.usage;- cacheCreationTokens = extractCacheCreationTokensFromSteps(- textResponse.steps,- );- nativeWebSearchCallCount = countNativeWebSearchCallsFromSteps(- textResponse.steps,- );- executionSteps = textResponse.steps;-- const agentSchema =- responseFormat?.type === 'json' ? responseFormat.schema : undefined;-- let result: object = { response: textResponse.text };-- const endsOnPausingTool = endsOnPausingToolCall({- steps: textResponse.steps,- offeredToolNames,- });-- if (isDefined(agentSchema) && !endsOnPausingTool) {- const structuredResult = await generateText({- instructions: STRUCTURED_OUTPUT_SYSTEM_PROMPT,- model: registeredModel.model,- prompt: `Based on the following execution results, generate the structured output according to the schema:-- Execution Results: ${textResponse.text}-- Please generate the structured output based on the execution results and context above.`,- output: Output.object({- schema: jsonSchema(buildStrictAgentResponseSchema(agentSchema)),- }),- providerOptions: getCallLevelProviderOptions({- sdkPackage: registeredModel.sdkPackage,- providerOptions: undefined,- promptCacheKey: agent.id,- }),- ...buildAiTelemetry({- functionId: 'agent-structured-output',- workspaceId,- userWorkspaceId,- agentId: agent.id,- }),- onStepEnd: decrementCreditsForStep,- });-- accumulatedUsage = mergeLanguageModelUsage(- textResponse.usage,- structuredResult.usage,- );- executionSteps = [...textResponse.steps, ...structuredResult.steps];-- if (structuredResult.output == null) {- throw new AiException(- 'Failed to generate structured output from execution results',- AiExceptionCode.AGENT_EXECUTION_FAILED,- );- }-- result = structuredResult.output as object;- }-- const tokenCostInDollars = this.aiBillingService.calculateStepsCost(- registeredModel.modelId,- executionSteps,- );- const totalCostInDollars =- tokenCostInDollars +- nativeWebSearchCallCount * NATIVE_WEB_SEARCH_COST_PER_CALL_DOLLARS;- const creditsUsedMicro = convertDollarsToCreditsMicro(totalCostInDollars);-- return {- result,- usage: accumulatedUsage,- cacheCreationTokens,- nativeWebSearchCallCount,- hasNoMoreAvailableCredits,- // out of credits fails the execution even if it asked something, so it is never left waiting for an answer- isPaused: endsOnPausingTool && !hasNoMoreAvailableCredits,- steps: executionSteps,- modelId: registeredModel.modelId,- totalCostInDollars,- creditsUsedMicro,- turnUsage: this.aiBillingService.calculateStepsTurnUsage(- registeredModel.modelId,- executionSteps,- ),- };- } catch (error) {- if (error instanceof AiException) {- throw error;- }- throw new AiException(- error instanceof Error ? error.message : 'Agent execution failed',- AiExceptionCode.AGENT_EXECUTION_FAILED,- );- } finally {- const modelId = resolvedModelId ?? agent.modelId;- // pricing an unresolved model id would throw over the original error- const costInDollars = isDefined(resolvedModelId)- ? this.aiBillingService.calculateStepsCost(- resolvedModelId,- executionSteps,- )- : 0;- const creditsUsedMicro = convertDollarsToCreditsMicro(costInDollars);- const totalTokens =- (accumulatedUsage.inputTokens ?? 0) +- (accumulatedUsage.outputTokens ?? 0);-- void this.aiBillingService.emitAiTokenUsageEvent({- workspaceId,- creditsUsedMicro,- totalTokens,- modelId,- operationType: usageOperationType,- spenders: usageSpenders,- });-- void this.aiBillingService.billNativeWebSearchUsage({- nativeWebSearchCallCount,- workspaceId,- operationType: UsageOperationType.AI_WORKFLOW_WEB_SEARCH,- spenders: usageSpenders,- });- }- }-}diff --git a/packages/twenty-server/src/engine/metadata-modules/ai/ai-agent-execution/services/agent-run-conversation.service.ts b/packages/twenty-server/src/engine/metadata-modules/ai/ai-agent-execution/services/agent-run-conversation.service.tsindex b46bca33097..721f63b3c03 100644--- a/packages/twenty-server/src/engine/metadata-modules/ai/ai-agent-execution/services/agent-run-conversation.service.ts+++ b/packages/twenty-server/src/engine/metadata-modules/ai/ai-agent-execution/services/agent-run-conversation.service.ts@@ -1,6 +1,4 @@-import { Injectable, Logger } from '@nestjs/common';--import { getToolName, isToolUIPart } from 'ai';+import { Injectable } from '@nestjs/common';import { isNonEmptyString } from '@sniptt/guards';import {@@ -11,109 +9,76 @@ import { type RunAgentMessage } from 'twenty-shared/application';import { type ActorMetadata } from 'twenty-shared/types';import { isDefined } from 'twenty-shared/utils';-import { CacheLockService } from 'src/engine/core-modules/cache-lock/cache-lock.service';-import { CacheLockException } from 'src/engine/core-modules/cache-lock/exceptions/cache-lock.exception';-import { AGENT_WAIT_TOOL_NAMES } from 'src/engine/metadata-modules/ai/ai-agent-execution/pausing-tools/constants/agent-wait-tool-names.constant';-import { AgentChatThreadService } from 'src/engine/metadata-modules/ai/ai-chat/services/agent-chat-thread.service';-import { findLastMessageText } from 'src/engine/metadata-modules/ai/ai-chat/utils/find-last-message-text.util';-import { mapErrorToStreamError } from 'src/engine/metadata-modules/ai/ai-history/utils/map-error-to-stream-error.util';-import { AgentTurnStatus } from 'src/engine/metadata-modules/ai/ai-history/enums/agent-turn-status.enum';-import { AgentTurnRecorderService } from 'src/engine/metadata-modules/ai/ai-history/services/agent-turn-recorder.service';-import {- AGENT_RUN_THREAD_LOCK_RETRY_INTERVAL_MS,- AGENT_RUN_THREAD_LOCK_TTL_MS,-} from 'src/engine/metadata-modules/ai/ai-agent-execution/constants/agent-run-thread-lock.const';+import { AgentChatStreamRecoveryService } from 'src/engine/metadata-modules/ai/ai-chat/services/agent-chat-stream-recovery.service';import { AgentMessageRole } from 'src/engine/metadata-modules/ai/ai-history/enums/agent-message-role.enum';+import { AgentMessageStatus } from 'src/engine/metadata-modules/ai/ai-history/enums/agent-message-status.enum';+import { AgentTurnStatus } from 'src/engine/metadata-modules/ai/ai-history/enums/agent-turn-status.enum';import { AgentHistoryRepository } from 'src/engine/metadata-modules/ai/ai-history/repositories/agent-history-repository';import { InjectAgentHistoryRepository } from 'src/engine/metadata-modules/ai/ai-history/repositories/inject-agent-history-repository.decorator';import { AgentConversationWriterService } from 'src/engine/metadata-modules/ai/ai-history/services/agent-conversation-writer.service';import { type AgentChatThreadWorkspaceEntity } from 'src/engine/metadata-modules/ai/ai-history/standard-objects/agent-chat-thread.workspace-entity';-import { isAwaitingPausingToolOutput } from 'src/engine/metadata-modules/ai/ai-history/utils/is-awaiting-pausing-tool-output.util';-import { type RecordableAgentExecution } from 'src/engine/metadata-modules/ai/ai-history/types/recordable-agent-execution.type';-import {- AiException,- AiExceptionCode,-} from 'src/engine/metadata-modules/ai/ai.exception';--// long enough to wait for another message being sent, not for a run-const MESSAGE_THREAD_LOCK_MAX_WAIT_MS = 2_000;@Injectable()export class AgentRunConversationService {- private readonly logger = new Logger(AgentRunConversationService.name);-constructor(@InjectAgentHistoryRepository('agentChatThread')private readonly threadRepository: AgentHistoryRepository<AgentChatThreadWorkspaceEntity>,private readonly conversationWriterService: AgentConversationWriterService,- private readonly cacheLockService: CacheLockService,- private readonly turnRecorderService: AgentTurnRecorderService,- private readonly threadService: AgentChatThreadService,+ private readonly streamRecoveryService: AgentChatStreamRecoveryService,) {}- withThreadLock<TResult>({- workspaceId,- threadId,- work,- maxWaitMs = AGENT_RUN_THREAD_LOCK_TTL_MS,- }: {- workspaceId: string;- threadId: string;- work: () => Promise<TResult>;- maxWaitMs?: number;- }): Promise<TResult> {- return this.cacheLockService.withLock(- work,- `agent-run-thread:${workspaceId}:${threadId}`,- {- ttl: AGENT_RUN_THREAD_LOCK_TTL_MS,- ms: AGENT_RUN_THREAD_LOCK_RETRY_INTERVAL_MS,- maxRetries: maxWaitMs / AGENT_RUN_THREAD_LOCK_RETRY_INTERVAL_MS,- },- );- }-- // A member's message waits a moment for a run going on in the conversation, then is refused like- // one sent while the run waits- async withThreadLockForMessage<TResult>({+ // A run holds its conversation's stream while it goes on, as a chat reply does, so nothing else+ // writes to the conversation meanwhile. A stream whose worker died is reaped first+ async claimThread({workspaceId,threadId,- work,+ title,+ streamId,}: {workspaceId: string;threadId: string;- work: () => Promise<TResult>;- }): Promise<TResult> {- let isLocked = false;+ title: string;+ streamId: string;+ }): Promise<boolean> {+ const thread = await this.threadRepository.findOne(workspaceId, {+ where: { id: threadId },+ select: ['id', 'activeStreamId'],+ });- try {- return await this.withThreadLock({+ if (isDefined(thread)) {+ const interruptedError = await this.streamRecoveryService.reapDeadStream({+ thread,workspaceId,- threadId,- maxWaitMs: MESSAGE_THREAD_LOCK_MAX_WAIT_MS,- work: () => {- isLocked = true;-- return work();- },});- } catch (error) {- if (!isLocked && error instanceof CacheLockException) {- throw new AiException(- 'A run is going on in the conversation; send the next message once it has finished',- AiExceptionCode.THREAD_AWAITING_ANSWER,- );- }- throw error;+ // a live stream keeps the conversation, so a claim could only fail+ if (+ isNonEmptyString(thread.activeStreamId) &&+ !isDefined(interruptedError)+ ) {+ return false;+ }+ } else {+ await this.conversationWriterService.runInTransaction(+ workspaceId,+ // two runs starting one keyed conversation can both find it missing+ (scope) =>+ scope.upsert('agentChatThread', { id: threadId, title }, ['id']),+ );}++ return this.streamRecoveryService.tryClaimStream({+ workspaceId,+ threadId,+ streamId,+ });}- // The turn is written before the agent runs, so a run in progress or one that fails is on record+ // The turn is written before the agent runs, so a run in progress or one that fails is on record.+ // Messages members sent while the conversation was busy, or while the run waited, are its input tooasync openTurn({workspaceId,threadId,- title,agentId,senderUserWorkspaceId,senderApplicationId,@@ -122,25 +87,19 @@ export class AgentRunConversationService {}: {workspaceId: string;threadId: string;- title: string;agentId: string | null;senderUserWorkspaceId: string | null;senderApplicationId: string | null;createdBy: ActorMetadata;messages: RunAgentMessage[];- }): Promise<string> {- const existingThread = await this.threadRepository.findOne(workspaceId, {- where: { id: threadId },- select: ['id'],- });-+ }): Promise<{+ turnId: string;+ promptMessageIds: string[];+ hasTakenQueuedMessages: boolean;+ }> {return this.conversationWriterService.runInTransaction(workspaceId,async (scope) => {- if (!isDefined(existingThread)) {- await scope.insert('agentChatThread', { id: threadId, title });- }-const turnId = await this.conversationWriterService.insertTurn({workspaceId,threadId,@@ -150,119 +109,51 @@ export class AgentRunConversationService {scope,});+ const takenMessageCount = await scope.update(+ 'agentMessage',+ { threadId, status: AgentMessageStatus.QUEUED },+ {+ status: AgentMessageStatus.SENT,+ turnId,+ processedAt: new Date().toISOString(),+ },+ );++ const promptMessageIds: string[] = [];+for (const message of messages) {const isAssistantMessage = message.role === 'assistant';- await this.conversationWriterService.insertMessage({- workspaceId,- threadId,- turnId,- role: isAssistantMessage- ? AgentMessageRole.ASSISTANT- : AgentMessageRole.USER,- agentId: null,- senderUserWorkspaceId: isAssistantMessage- ? null- : senderUserWorkspaceId,- senderApplicationId: isAssistantMessage- ? null- : senderApplicationId,- parts: this.buildMessageParts(message),- scope,- });+ promptMessageIds.push(+ await this.conversationWriterService.insertMessage({+ workspaceId,+ threadId,+ turnId,+ role: isAssistantMessage+ ? AgentMessageRole.ASSISTANT+ : AgentMessageRole.USER,+ agentId: null,+ senderUserWorkspaceId: isAssistantMessage+ ? null+ : senderUserWorkspaceId,+ senderApplicationId: isAssistantMessage+ ? null+ : senderApplicationId,+ parts: this.buildMessageParts(message),+ scope,+ }),+ );}- return turnId;+ return {+ turnId,+ promptMessageIds,+ hasTakenQueuedMessages: takenMessageCount > 0,+ };},);}- async closeTurn({- workspaceId,- threadId,- turnId,- title,- agentId,- execution,- }: {- workspaceId: string;- threadId: string;- turnId: string;- title: string;- agentId: string | null;- execution: RecordableAgentExecution;- }): Promise<{ isAwaitingAnswer: boolean }> {- const turn = { workspaceId, threadId, turnId, execution };-- const { isAwaitingAnswer, replyParts } =- await this.conversationWriterService- .insertExecutionReply({- workspaceId,- threadId,- turnId,- agentId,- execution,- })- .catch(async (error: unknown) => {- await this.turnRecorderService.finishExecutedTurn({- ...turn,- error: mapErrorToStreamError(error),- });-- throw error;- });-- // a run that paused on a wait goes on later, so its turn is not done yet- const isWaitingOnWait =- execution.isPaused === true &&- replyParts.some(- (part) =>- isToolUIPart(part) &&- AGENT_WAIT_TOOL_NAMES.includes(getToolName(part)) &&- isAwaitingPausingToolOutput(part.output),- );-- await this.turnRecorderService.finishExecutedTurn({- ...turn,- isWaiting: isAwaitingAnswer || isWaitingOnWait,- });-- // the waiting call is already saved and can be answered from the conversation, so a- // failure to bring it back to the inbox must not fail the run- if (isAwaitingAnswer) {- await this.threadService- .recordThreadActivity({- workspaceId,- threadId,- text: findLastMessageText(replyParts) ?? title,- })- .catch((error: unknown) =>- this.logger.warn(- `Could not record waiting activity on thread ${threadId}: ${error instanceof Error ? error.message : String(error)}`,- ),- );- }-- return { isAwaitingAnswer };- }-- async failTurn({- workspaceId,- turnId,- error,- }: {- workspaceId: string;- turnId: string;- error: unknown;- }): Promise<void> {- await this.turnRecorderService.finish({- workspaceId,- turnId,- status: AgentTurnStatus.FAILED,- error: mapErrorToStreamError(error),- });- }-private buildMessageParts(message: RunAgentMessage): ExtendedUIMessagePart[] {const fileParts = (message.attachments ?? []).map((attachment): ExtendedFileUIPart => ({diff --git a/packages/twenty-server/src/engine/metadata-modules/ai/ai-agent-execution/services/agent-run-settlement.service.ts b/packages/twenty-server/src/engine/metadata-modules/ai/ai-agent-execution/services/agent-run-settlement.service.tsnew file mode 100644index 00000000000..44d8ee11eff--- /dev/null+++ b/packages/twenty-server/src/engine/metadata-modules/ai/ai-agent-execution/services/agent-run-settlement.service.ts@@ -0,0 +1,115 @@+import { Injectable, Logger, type OnModuleDestroy } from '@nestjs/common';++import type { Redis } from 'ioredis';+import { isDefined } from 'twenty-shared/utils';++import { RedisClientService } from 'src/engine/core-modules/redis-client/redis-client.service';+import { RUN_AGENT_SETTLED_RUN_RECONCILE_INTERVAL_MS } from 'src/engine/metadata-modules/ai/ai-agent-execution/constants/run-agent-settled-run-reconcile-interval-ms.const';+import { RUN_AGENT_SETTLED_RUN_TTL_SECONDS } from 'src/engine/metadata-modules/ai/ai-agent-execution/constants/run-agent-settled-run-ttl-seconds.const';+import { type SettledAgentRun } from 'src/engine/metadata-modules/ai/ai-agent-execution/types/settled-agent-run.type';+import { formatErrorWithCause } from 'src/engine/metadata-modules/ai/ai-chat/utils/format-error-with-cause.util';++const getSettledRunKey = (runId: string) => `agent-run-settled:${runId}`;++// One subscriber connection per process carries every waiting call's channel+@Injectable()+export class AgentRunSettlementService implements OnModuleDestroy {+ private readonly logger = new Logger(AgentRunSettlementService.name);+ private subscriber: Redis | null = null;+ private readonly waiters = new Map<+ string,+ (settledRun: SettledAgentRun) => void+ >();++ constructor(private readonly redisClientService: RedisClientService) {}++ async publish({+ runId,+ settledRun,+ }: {+ runId: string;+ settledRun: SettledAgentRun;+ }): Promise<void> {+ const key = getSettledRunKey(runId);+ const payload = JSON.stringify(settledRun);++ await this.redisClientService+ .getClient()+ .multi()+ .set(key, payload, 'EX', RUN_AGENT_SETTLED_RUN_TTL_SECONDS)+ .publish(key, payload)+ .exec();+ }++ // Redis queues commands while it is down, so only the timer decides when the call stops waiting+ async waitFor({+ runId,+ timeoutMs,+ }: {+ runId: string;+ timeoutMs: number;+ }): Promise<SettledAgentRun | null> {+ const key = getSettledRunKey(runId);+ let timeout: NodeJS.Timeout | undefined;++ const settledRun = new Promise<SettledAgentRun | null>((resolve) => {+ this.waiters.set(key, resolve);+ timeout = setTimeout(() => resolve(null), timeoutMs);+ });++ const readStoredSettlement = () =>+ this.redisClientService+ .getClient()+ .get(key)+ .then((payload) => {+ if (isDefined(payload)) {+ this.waiters.get(key)?.(JSON.parse(payload) as SettledAgentRun);+ }+ })+ .catch((error: unknown) =>+ this.logger.warn(+ `Could not read how run ${runId} settled: ${formatErrorWithCause(error)}`,+ ),+ );++ // a run that settled before the subscription took is read from its key+ this.ensureSubscriber()+ .subscribe(key)+ .then(readStoredSettlement)+ .catch((error: unknown) =>+ this.logger.warn(+ `Could not wait for run ${runId} to settle: ${formatErrorWithCause(error)}`,+ ),+ );++ const reconcile = setInterval(+ () => void readStoredSettlement(),+ RUN_AGENT_SETTLED_RUN_RECONCILE_INTERVAL_MS,+ );++ try {+ return await settledRun;+ } finally {+ clearTimeout(timeout);+ clearInterval(reconcile);+ this.waiters.delete(key);+ this.subscriber?.unsubscribe(key).catch(() => {});+ }+ }++ async onModuleDestroy(): Promise<void> {+ await this.subscriber?.quit().catch(() => {});+ this.subscriber = null;+ }++ private ensureSubscriber(): Redis {+ if (!isDefined(this.subscriber)) {+ this.subscriber = this.redisClientService.getClient().duplicate();+ this.subscriber.on('message', (channel: string, payload: string) =>+ this.waiters.get(channel)?.(JSON.parse(payload) as SettledAgentRun),+ );+ }++ return this.subscriber;+ }+}diff --git a/packages/twenty-server/src/engine/metadata-modules/ai/ai-agent-execution/services/agent-run-suspension.service.ts b/packages/twenty-server/src/engine/metadata-modules/ai/ai-agent-execution/services/agent-run-suspension.service.tsindex 1ceb3c7c2c3..b0d018b4dfc 100644--- a/packages/twenty-server/src/engine/metadata-modules/ai/ai-agent-execution/services/agent-run-suspension.service.ts+++ b/packages/twenty-server/src/engine/metadata-modules/ai/ai-agent-execution/services/agent-run-suspension.service.ts@@ -5,17 +5,22 @@ import { type PendingWakeUpCondition } from 'twenty-shared/pending-wake-up';import { isDefined } from 'twenty-shared/utils';import { IsNull, Not, Raw } from 'typeorm';+import { formatErrorWithCause } from 'src/engine/metadata-modules/ai/ai-chat/utils/format-error-with-cause.util';import { InjectMessageQueue } from 'src/engine/core-modules/message-queue/decorators/message-queue.decorator';import { MessageQueue } from 'src/engine/core-modules/message-queue/message-queue.constants';import { MessageQueueService } from 'src/engine/core-modules/message-queue/services/message-queue.service';import { PendingWakeUpEntity } from 'src/engine/core-modules/pending-wake-up/entities/pending-wake-up.entity';import { PendingWakeUpService } from 'src/engine/core-modules/pending-wake-up/services/pending-wake-up.service';+import { AGENT_RUN_OUTCOME_DELIVERY_MAX_ATTEMPTS } from 'src/engine/metadata-modules/ai/ai-agent-execution/constants/agent-run-outcome-delivery-max-attempts.const';+import { AGENT_RUN_OUTCOME_DELIVERY_RETRY_DELAY_MS } from 'src/engine/metadata-modules/ai/ai-agent-execution/constants/agent-run-outcome-delivery-retry-delay-ms.const';import { CONTINUE_AGENT_RUN_JOB_NAME } from 'src/engine/metadata-modules/ai/ai-agent-execution/constants/continue-agent-run-job-name.constant';+import { DELIVER_AGENT_RUN_OUTCOME_JOB_NAME } from 'src/engine/metadata-modules/ai/ai-agent-execution/constants/deliver-agent-run-outcome-job-name.constant';import { AGENT_WAIT_TOOL_NAMES } from 'src/engine/metadata-modules/ai/ai-agent-execution/pausing-tools/constants/agent-wait-tool-names.constant';import { buildWaitOutcomeToolOutput } from 'src/engine/metadata-modules/ai/ai-agent-execution/pausing-tools/wait-tools/build-wait-outcome-tool-output.util';import { AgentRunCallerHandlerRegistryService } from 'src/engine/metadata-modules/ai/ai-agent-execution/services/agent-run-caller-handler-registry.service';import { type AgentRunCaller } from 'src/engine/metadata-modules/ai/ai-agent-execution/types/agent-run-caller.type';import { type AgentRunCallerOutcome } from 'src/engine/metadata-modules/ai/ai-agent-execution/types/agent-run-caller-outcome.type';+import { type AgentRunOutcomeDelivery } from 'src/engine/metadata-modules/ai/ai-agent-execution/types/agent-run-outcome-delivery.type';import { type AgentRunSuspension } from 'src/engine/metadata-modules/ai/ai-agent-execution/types/agent-run-suspension.type';import { type ContinueAgentRunJobData } from 'src/engine/metadata-modules/ai/ai-agent-execution/types/continue-agent-run-job-data.type';import { AgentChatThreadLifecycleService } from 'src/engine/metadata-modules/ai/ai-chat/services/agent-chat-thread-lifecycle.service';@@ -35,6 +40,11 @@ import { WorkspaceScopedRepository } from 'src/engine/twenty-orm/workspace-scope// one key for every run, so the owner's unique key leaves a conversation one suspended runconst SUSPENDED_RUN_WAKE_UP_KEY = 'RUN';+const buildAnswerInThreadCondition = (threadId: string) =>+ Raw((alias) => `${alias} @> :condition::jsonb`, {+ condition: JSON.stringify({ type: 'ANSWER', threadId }),+ });+// A suspended run is the AGENT_RUN wake-up its conversation owns: what wakes it up, and in its// payload what continues it and who gets its outcome@Injectable()@@ -79,8 +89,8 @@ export class AgentRunSuspensionService {});}- // a run waiting in the conversation would read a new message on continuing, and a caller waiting- // on a call it posted there would see the message close it+ // a run waiting in the conversation would read another run's prompt on continuing, and a caller+ // waiting on a call it posted there would see the run close itasync isConversationWaiting({workspaceId,threadId,@@ -91,11 +101,7 @@ export class AgentRunSuspensionService {return this.wakeUpRepository.exists(workspaceId, {where: [{ ownerType: 'AGENT_RUN', ownerId: threadId },- {- condition: Raw((alias) => `${alias} @> :condition::jsonb`, {- condition: JSON.stringify({ type: 'ANSWER', threadId }),- }),- },+ { condition: buildAnswerInThreadCondition(threadId) },],});}@@ -115,6 +121,43 @@ export class AgentRunSuspensionService {}}+ // a caller, such as a workflow step, waits on the answer to a call it posted, which a message+ // would leave unanswered+ async assertNoPostedCallAwaited({+ workspaceId,+ threadId,+ }: {+ workspaceId: string;+ threadId: string;+ }): Promise<void> {+ const isAwaited = await this.wakeUpRepository.exists(workspaceId, {+ where: {+ ownerType: Not('AGENT_RUN'),+ condition: buildAnswerInThreadCondition(threadId),+ },+ });++ if (isAwaited) {+ throw new AiException(+ 'The conversation is waiting on an answer; answer it before sending a message',+ AiExceptionCode.THREAD_AWAITING_ANSWER,+ );+ }+ }++ async findSuspendedRun({+ workspaceId,+ threadId,+ }: {+ workspaceId: string;+ threadId: string;+ }): Promise<Pick<PendingWakeUpEntity, 'id'> | null> {+ return this.wakeUpRepository.findOne(workspaceId, {+ where: { ownerType: 'AGENT_RUN', ownerId: threadId },+ select: ['id'],+ });+ }+// the wake-up stays while its run goes on, so another resolution of it is let go rather than queued behind itasync scheduleContinuation(jobData: ContinueAgentRunJobData): Promise<void> {await this.messageQueueService.add<ContinueAgentRunJobData>(@@ -132,6 +175,7 @@ export class AgentRunSuspensionService {outcome,summary = suspendedSummary,isAwaitingAnswer = false,+ streamId,}: {workspaceId: string;threadId: string;@@ -139,12 +183,70 @@ export class AgentRunSuspensionService {outcome: AgentRunCallerOutcome;summary?: AgentRunSummary | null;isAwaitingAnswer?: boolean;+ streamId?: string;}): Promise<void> {- await this.closeAwaitedCalls({ workspaceId, threadId, isAwaitingAnswer });+ await this.closeAwaitedCalls({+ workspaceId,+ threadId,+ isAwaitingAnswer,+ streamId,+ });++ await this.deliverOutcome({+ workspaceId,+ caller,+ threadId,+ outcome,+ summary: summary ?? null,+ });+ }++ // a workflow step is marked pending only after its executor returns, so an outcome that comes sooner is delivered later+ async deliverOutcome(delivery: AgentRunOutcomeDelivery): Promise<void> {+ const { workspaceId, caller, attempt = 0 } = delivery;+ const handler = this.callerHandlerRegistry.getHandlerOrThrow(caller.type);++ if (!isDefined(handler.onOutcome)) {+ return;+ }++ try {+ const waitingState = await handler.getWaitingState({+ workspaceId,+ caller,+ });++ if (waitingState === 'GONE') {+ await handler.onCallerGone?.(delivery);- await this.callerHandlerRegistry- .getHandlerOrThrow(caller.type)- .onOutcome?.({ workspaceId, caller, threadId, outcome, summary });+ return;+ }++ if (waitingState === 'WAITING') {+ await handler.onOutcome(delivery);++ return;+ }+ } catch (error) {+ // the run already ended, so a failed delivery is tried again rather than lost with the job+ this.logger.warn(+ `Could not deliver the outcome of the run in thread ${delivery.threadId} of workspace ${workspaceId}: ${formatErrorWithCause(error)}`,+ );+ }++ if (attempt + 1 >= AGENT_RUN_OUTCOME_DELIVERY_MAX_ATTEMPTS) {+ this.logger.error(+ `Dropped the outcome of the run in thread ${delivery.threadId} of workspace ${workspaceId}: its ${caller.type} caller never took it`,+ );++ return;+ }++ await this.messageQueueService.add<AgentRunOutcomeDelivery>(+ DELIVER_AGENT_RUN_OUTCOME_JOB_NAME,+ { ...delivery, attempt: attempt + 1 },+ { delay: AGENT_RUN_OUTCOME_DELIVERY_RETRY_DELAY_MS },+ );}// A caller that stops waiting drops its suspended runs, such as every step of a run that ended@@ -198,10 +300,13 @@ export class AgentRunSuspensionService {workspaceId,threadId,isAwaitingAnswer,+ streamId,}: {workspaceId: string;threadId: string;isAwaitingAnswer: boolean;+ // the stream of a run that closes its calls while it holds the conversation+ streamId?: string;}): Promise<void> {await this.recordWaitOutcome({workspaceId,@@ -210,7 +315,7 @@ export class AgentRunSuspensionService {});if (isAwaitingAnswer) {- await this.closePendingQuestion({ workspaceId, threadId });+ await this.closePendingQuestion({ workspaceId, threadId, streamId });}}@@ -257,7 +362,7 @@ export class AgentRunSuspensionService {}).catch((error: unknown) =>this.logger.warn(- `Could not end the turn of wait message ${messageId}: ${error instanceof Error ? error.message : String(error)}`,+ `Could not end the turn of wait message ${messageId}: ${formatErrorWithCause(error)}`,),);}@@ -283,20 +388,24 @@ export class AgentRunSuspensionService {}// without a call to look for, the pending question is the one waited on- private async closePendingQuestion({+ async closePendingQuestion({workspaceId,threadId,toolCallId,+ streamId,+ turnStatus = AgentTurnStatus.CANCELLED,}: {workspaceId: string;threadId: string;toolCallId?: string;+ streamId?: string;+ turnStatus?: AgentTurnStatus.COMPLETED | AgentTurnStatus.CANCELLED;}): Promise<void> {const thread = await this.threadRepository.findOne(workspaceId, {where: {id: threadId,pendingQuestionMessageId: Not(IsNull()),- activeStreamId: IsNull(),+ activeStreamId: streamId ?? IsNull(),},select: ['id', 'pendingQuestionMessageId'],});@@ -317,8 +426,8 @@ export class AgentRunSuspensionService {workspaceId,threadId,messageId,- activeStreamId: null,- turnStatus: AgentTurnStatus.CANCELLED,+ activeStreamId: streamId ?? null,+ turnStatus,});}}diff --git a/packages/twenty-server/src/engine/metadata-modules/ai/ai-agent-execution/services/agent-run-turn.service.ts b/packages/twenty-server/src/engine/metadata-modules/ai/ai-agent-execution/services/agent-run-turn.service.tsnew file mode 100644index 00000000000..78b13756183--- /dev/null+++ b/packages/twenty-server/src/engine/metadata-modules/ai/ai-agent-execution/services/agent-run-turn.service.ts@@ -0,0 +1,406 @@+import { Injectable } from '@nestjs/common';+import { InjectRepository } from '@nestjs/typeorm';++import {+ convertToModelMessages,+ generateText,+ jsonSchema,+ Output,+ type TextStreamPart,+ type ToolSet,+} from 'ai';+import { type RunAgentMessage } from 'twenty-shared/application';+import {+ type AgentResponseFormat,+ type ExtendedUIMessage,+} from 'twenty-shared/ai';+import {+ isDefined,+ isNonEmptyArray,+ tipTapDocumentToMarkdown,+} from 'twenty-shared/utils';+import { type Repository } from 'typeorm';++import { isUserAuthContext } from 'src/engine/core-modules/auth/guards/is-user-auth-context.guard';+import { type ToolContext } from 'src/engine/core-modules/tool-provider/types/tool-context.type';+import { UsageOperationType } from 'src/engine/core-modules/usage/enums/usage-operation-type.enum';+import { type UsageSpenders } from 'src/engine/core-modules/usage/types/usage-spenders.type';+import { buildToolCatalogSection } from 'src/engine/core-modules/tool-provider/utils/build-tool-catalog-section.util';+import { OUTPUT_NAVIGATION_TOOL_NAMES } from 'src/engine/core-modules/tool/tools/output-navigation-tool/constants/output-navigation-tool-names.constant';+import { WorkspaceEntity } from 'src/engine/core-modules/workspace/workspace.entity';+import { AGENT_RUN_EXCLUDED_TOOL_NAMES } from 'src/engine/metadata-modules/ai/ai-agent-execution/constants/agent-run-excluded-tool-names.const';+import { OPEN_ENDED_AGENT_REGISTRY_TOOL_CATEGORIES } from 'src/engine/metadata-modules/ai/ai-agent-execution/constants/open-ended-agent-registry-tool-categories.const';+import { PRELOADED_AGENT_REGISTRY_TOOL_CATEGORIES } from 'src/engine/metadata-modules/ai/ai-agent-execution/constants/preloaded-agent-registry-tool-categories.const';+import { endsOnPausingToolCall } from 'src/engine/metadata-modules/ai/ai-agent-execution/pausing-tools/utils/ends-on-pausing-tool-call.util';+import { type AgentRegistryAccess } from 'src/engine/metadata-modules/ai/ai-agent-execution/services/agent-toolset.service';+import {+ type AgentTurn,+ AgentTurnService,+} from 'src/engine/metadata-modules/ai/ai-agent-execution/services/agent-turn.service';+import { RunAgentAttachmentService } from 'src/engine/metadata-modules/ai/ai-agent-execution/services/run-agent-attachment.service';+import { type AgentExecutionResult } from 'src/engine/metadata-modules/ai/ai-agent-execution/types/agent-execution-result.type';+import { type AgentRunExecutionContext } from 'src/engine/metadata-modules/ai/ai-agent-execution/types/agent-run-execution-context.type';+import { type AgentToolLoadingStrategy } from 'src/engine/metadata-modules/ai/ai-agent-execution/types/agent-tool-loading-strategy.type';+import { assertAgentResponseFormatHasOutputFieldsOrThrow } from 'src/engine/metadata-modules/ai/ai-agent-execution/utils/assert-agent-response-format-has-output-fields-or-throw.util';+import { buildAgentRolePermissionConfig } from 'src/engine/metadata-modules/ai/ai-agent-execution/utils/build-agent-role-permission-config.util';+import { buildStrictAgentResponseSchema } from 'src/engine/metadata-modules/ai/ai-agent-execution/utils/build-strict-agent-response-schema.util';+import { getCallLevelProviderOptions } from 'src/engine/metadata-modules/ai/ai-agent-execution/utils/provider-options.util';+import { resolveAgentRunCallingApplication } from 'src/engine/metadata-modules/ai/ai-agent-execution/utils/resolve-agent-run-calling-application.util';+import { replaceUnsupportedFileParts } from 'src/engine/metadata-modules/ai/ai-agent-execution/utils/replace-unsupported-file-parts.util';+import { AiAgentRoleService } from 'src/engine/metadata-modules/ai/ai-agent-role/ai-agent-role.service';+import { STRUCTURED_OUTPUT_SYSTEM_PROMPT } from 'src/engine/metadata-modules/ai/ai-agent/constants/structured-output-system-prompt.const';+import { type AgentEntity } from 'src/engine/metadata-modules/ai/ai-agent/entities/agent.entity';+import { NATIVE_WEB_SEARCH_COST_PER_CALL_DOLLARS } from 'src/engine/metadata-modules/ai/ai-billing/constants/native-web-search-cost-per-call-dollars';+import { AiBillingService } from 'src/engine/metadata-modules/ai/ai-billing/services/ai-billing.service';+import { type AgentTurnUsageStep } from 'src/engine/metadata-modules/ai/ai-billing/utils/add-step-to-agent-turn-usage.util';+import { convertDollarsToCreditsMicro } from 'src/engine/metadata-modules/ai/ai-billing/utils/convert-dollars-to-credits-micro.util';+import { countNativeWebSearchCallsFromSteps } from 'src/engine/metadata-modules/ai/ai-billing/utils/count-native-web-search-calls-from-steps.util';+import { extractCacheCreationTokensFromSteps } from 'src/engine/metadata-modules/ai/ai-billing/utils/extract-cache-creation-tokens.util';+import { mergeLanguageModelUsage } from 'src/engine/metadata-modules/ai/ai-billing/utils/merge-language-model-usage.util';+import { type AgentTurnStreamSource } from 'src/engine/metadata-modules/ai/ai-chat/services/agent-turn-stream.service';+import { buildAiTelemetry } from 'src/engine/metadata-modules/ai/ai-models/utils/build-ai-telemetry.util';+import {+ AiException,+ AiExceptionCode,+} from 'src/engine/metadata-modules/ai/ai.exception';++type AgentRunTurnInput = {+ agent: AgentEntity;+ responseFormat?: AgentResponseFormat;+ messages: RunAgentMessage[];+ // a continued conversation, with the tool calls and results plain run messages cannot carry+ priorMessages?: ExtendedUIMessage[];+ pausingTools?: ToolSet;+ canAskHumans?: boolean;+ baseSystemPrompt: string;+ workspaceId: string;+ executionContext: AgentRunExecutionContext;+ additionalExcludedToolNames?: readonly string[];+ toolLoadingStrategy?: AgentToolLoadingStrategy;+ abortSignal?: AbortSignal;+};++export type StartedAgentRunTurn = {+ turn: AgentTurn;+ source: AgentTurnStreamSource;+};++export type CompletedAgentRunTurn = AgentExecutionResult & {+ // the structured output request is not part of the stream, so the turn's usage must add it+ structuredOutputSteps: AgentTurnUsageStep[];+};++const buildSpenders = ({+ agent,+ executionContext,+}: Pick<AgentRunTurnInput, 'agent' | 'executionContext'>): UsageSpenders => ({+ userWorkspaceId: executionContext.userWorkspaceId,+ agentId: agent.id,+ applicationId: agent.applicationId,+});++@Injectable()+export class AgentRunTurnService {+ constructor(+ private readonly agentTurnService: AgentTurnService,+ private readonly aiBillingService: AiBillingService,+ private readonly aiAgentRoleService: AiAgentRoleService,+ private readonly runAgentAttachmentService: RunAgentAttachmentService,+ @InjectRepository(WorkspaceEntity)+ private readonly workspaceRepository: Repository<WorkspaceEntity>,+ ) {}++ async startTurn({+ agent,+ responseFormat,+ messages,+ baseSystemPrompt,+ workspaceId,+ executionContext,+ additionalExcludedToolNames = [],+ toolLoadingStrategy = 'preload',+ priorMessages = [],+ pausingTools = {},+ canAskHumans = false,+ abortSignal,+ }: AgentRunTurnInput): Promise<StartedAgentRunTurn> {+ if (!isNonEmptyArray(messages) && !isNonEmptyArray(priorMessages)) {+ throw new AiException(+ 'Provide at least one message to run an agent',+ AiExceptionCode.INVALID_AGENT_INPUT,+ );+ }++ assertAgentResponseFormatHasOutputFieldsOrThrow(responseFormat);++ const { userWorkspaceId, usageOperationType } = executionContext;+ const spenders = buildSpenders({ agent, executionContext });++ await this.aiBillingService.assertAiExecutionAllowed({+ workspaceId,+ operationType: usageOperationType,+ spenders,+ });++ const workspace = await this.workspaceRepository.findOneBy({+ id: workspaceId,+ });+ const registryAccess = await this.buildAgentRegistryAccess({+ agent,+ executionContext,+ additionalExcludedToolNames,+ toolLoadingStrategy,+ });++ const turn = await this.agentTurnService.start({+ workspace: workspace ?? undefined,+ modelId: agent.modelId,+ toolset: {+ registryAccess,+ nativeToolOptions: {+ webSearch: agent.modelConfiguration?.webSearch?.enabled === true,+ twitterSearch:+ agent.modelConfiguration?.twitterSearch?.enabled === true,+ },+ canAskHumans,+ waitTools: pausingTools,+ },+ buildContext: async ({ modelConfig, toolset }) => ({+ systemPrompt: `${baseSystemPrompt}\n\n${tipTapDocumentToMarkdown(agent.prompt)}${+ registryAccess.type === 'catalog'+ ? buildToolCatalogSection(toolset.toolCatalog, [])+ : ''+ }`,+ messages: [+ ...(await convertToModelMessages(+ replaceUnsupportedFileParts(+ priorMessages,+ modelConfig.modalities,+ false,+ ),+ )),+ ...(await this.runAgentAttachmentService.buildModelMessagesOrThrow({+ messages,+ workspaceId,+ modalities: modelConfig.modalities,+ })),+ ],+ }),+ promptCacheKey: agent.id,+ abortSignal,+ billing: {+ workspaceId,+ operationType: usageOperationType,+ webSearchOperationType: UsageOperationType.AI_WORKFLOW_WEB_SEARCH,+ spenders,+ },+ telemetry: {+ functionId: 'agent-execution',+ workspaceId,+ userWorkspaceId,+ agentId: agent.id,+ },+ // nobody resends a background run, so a provider error after streaming started is retried here+ streamRetries: 2,+ metricsScope: 'agent',+ });++ let streamError: unknown;++ return {+ turn,+ source: {+ // a retried provider error never reaches the stream, so only an error part fails the run+ parts: turn.stream.stream.pipeThrough(+ new TransformStream<TextStreamPart<ToolSet>, TextStreamPart<ToolSet>>(+ {+ transform: (part, controller) => {+ if (part.type === 'error') {+ streamError ??= part.error;+ }++ controller.enqueue(part);+ },+ },+ ),+ ),+ modelConfig: turn.modelConfig,+ getStreamError: () => streamError,+ },+ };+ }++ // the structured output request is not part of the stream, so its usage is added to the turn's here+ async completeTurn({+ started: { turn },+ agent,+ responseFormat,+ workspaceId,+ executionContext,+ abortSignal,+ }: Pick<+ AgentRunTurnInput,+ | 'agent'+ | 'responseFormat'+ | 'workspaceId'+ | 'executionContext'+ | 'abortSignal'+ > & {+ started: StartedAgentRunTurn;+ }): Promise<CompletedAgentRunTurn> {+ const { userWorkspaceId, usageOperationType } = executionContext;+ const spenders = buildSpenders({ agent, executionContext });+ const [text, loopUsage, loopSteps] = await Promise.all([+ turn.stream.text,+ turn.stream.usage,+ turn.stream.steps,+ ]);+ const { registeredModel, toolset } = turn;+ const isPaused = endsOnPausingToolCall({+ steps: loopSteps,+ offeredToolNames: toolset.offeredPausingToolNames,+ });+ const agentSchema =+ responseFormat?.type === 'json' ? responseFormat.schema : undefined;++ const stopReason = turn.getStopReason();+ let result: object = { response: text };+ let usage = loopUsage;+ let steps = loopSteps;+ let structuredOutputSteps: AgentTurnUsageStep[] = [];++ if (isDefined(agentSchema) && !isPaused) {+ const structuredResult = await generateText({+ instructions: STRUCTURED_OUTPUT_SYSTEM_PROMPT,+ model: registeredModel.model,+ prompt: `Based on the following execution results, generate the structured output according to the schema:++ Execution Results: ${text}++ Please generate the structured output based on the execution results and context above.`,+ output: Output.object({+ schema: jsonSchema(buildStrictAgentResponseSchema(agentSchema)),+ }),+ abortSignal,+ providerOptions: getCallLevelProviderOptions({+ sdkPackage: registeredModel.sdkPackage,+ providerOptions: undefined,+ promptCacheKey: agent.id,+ }),+ ...buildAiTelemetry({+ functionId: 'agent-structured-output',+ workspaceId,+ userWorkspaceId,+ agentId: agent.id,+ }),+ });++ void this.aiBillingService.calculateAndBillUsage({+ modelId: registeredModel.modelId,+ billingInput: {+ usage: structuredResult.usage,+ cacheCreationTokens: extractCacheCreationTokensFromSteps(+ structuredResult.steps,+ ),+ },+ workspaceId,+ operationType: usageOperationType,+ spenders,+ });++ usage = mergeLanguageModelUsage(usage, structuredResult.usage);+ steps = [...steps, ...structuredResult.steps];+ structuredOutputSteps = structuredResult.steps;++ if (structuredResult.output == null) {+ throw new AiException(+ 'Failed to generate structured output from execution results',+ AiExceptionCode.AGENT_EXECUTION_FAILED,+ );+ }++ result = structuredResult.output as object;+ }++ const nativeWebSearchCallCount = countNativeWebSearchCallsFromSteps(steps);+ const totalCostInDollars =+ this.aiBillingService.calculateStepsCost(registeredModel.modelId, steps) ++ nativeWebSearchCallCount * NATIVE_WEB_SEARCH_COST_PER_CALL_DOLLARS;++ return {+ result,+ usage,+ cacheCreationTokens: extractCacheCreationTokensFromSteps(steps),+ nativeWebSearchCallCount,+ stopReason,+ // out of credits fails the execution even if it asked something, so it is never left waiting for an answer+ isPaused: isPaused && !isDefined(stopReason),+ steps,+ modelId: registeredModel.modelId,+ totalCostInDollars,+ creditsUsedMicro: convertDollarsToCreditsMicro(totalCostInDollars),+ structuredOutputSteps,+ };+ }++ private async buildAgentRegistryAccess({+ agent,+ executionContext: { authContext, actorContext, agentRunRoleIds },+ additionalExcludedToolNames,+ toolLoadingStrategy,+ }: Pick<+ Required<AgentRunTurnInput>,+ | 'agent'+ | 'executionContext'+ | 'additionalExcludedToolNames'+ | 'toolLoadingStrategy'+ >): Promise<AgentRegistryAccess> {+ const agentRoleId = await this.aiAgentRoleService.findAgentRoleId({+ workspaceId: agent.workspaceId,+ agentId: agent.id,+ });++ if (!isDefined(agentRoleId)) {+ return { type: 'none' };+ }++ const toolContext: ToolContext = {+ workspaceId: agent.workspaceId,+ agentId: agent.id,+ authContext,+ application: resolveAgentRunCallingApplication(authContext),+ actorContext,+ rolePermissionConfig: buildAgentRolePermissionConfig({+ agentRoleId,+ agentRunRoleIds,+ }),+ ...(isDefined(authContext) && isUserAuthContext(authContext)+ ? {+ userId: authContext.user.id,+ userWorkspaceId: authContext.userWorkspaceId,+ }+ : {}),+ };+ const excludedToolNames = [+ ...OUTPUT_NAVIGATION_TOOL_NAMES,+ ...AGENT_RUN_EXCLUDED_TOOL_NAMES,+ ...additionalExcludedToolNames,+ ];++ // open-ended agents reach too many tools to send them all; a few granted ones are cheaper sent than learned+ return toolLoadingStrategy === 'lazy'+ ? {+ type: 'catalog',+ toolContext,+ categories: OPEN_ENDED_AGENT_REGISTRY_TOOL_CATEGORIES,+ excludedToolNames,+ }+ : {+ type: 'granted',+ toolContext,+ agentRoleId,+ categories: PRELOADED_AGENT_REGISTRY_TOOL_CATEGORIES,+ excludedToolNames,+ };+ }+}diff --git a/packages/twenty-server/src/engine/metadata-modules/ai/ai-agent-execution/services/agent-run.service.ts b/packages/twenty-server/src/engine/metadata-modules/ai/ai-agent-execution/services/agent-run.service.tsindex 4c40878f741..c1b302bf538 100644--- a/packages/twenty-server/src/engine/metadata-modules/ai/ai-agent-execution/services/agent-run.service.ts+++ b/packages/twenty-server/src/engine/metadata-modules/ai/ai-agent-execution/services/agent-run.service.ts@@ -36,16 +36,22 @@ import { UsageOperationType } from 'src/engine/core-modules/usage/enums/usage-opimport { type FlatWorkspace } from 'src/engine/core-modules/workspace/types/flat-workspace.type';import { AgentActorContextService } from 'src/engine/metadata-modules/ai/ai-agent-execution/services/agent-actor-context.service';import { AgentRunCallerHandlerRegistryService } from 'src/engine/metadata-modules/ai/ai-agent-execution/services/agent-run-caller-handler-registry.service';+import { assertAgentResponseFormatHasOutputFieldsOrThrow } from 'src/engine/metadata-modules/ai/ai-agent-execution/utils/assert-agent-response-format-has-output-fields-or-throw.util';+import { RUN_AGENT_MAX_WAIT_MS } from 'src/engine/metadata-modules/ai/ai-agent-execution/constants/run-agent-max-wait-ms.const';+import { AgentRunSettlementService } from 'src/engine/metadata-modules/ai/ai-agent-execution/services/agent-run-settlement.service';+import { AgentRunSuspensionService } from 'src/engine/metadata-modules/ai/ai-agent-execution/services/agent-run-suspension.service';import { AgentRunnerService } from 'src/engine/metadata-modules/ai/ai-agent-execution/services/agent-runner.service';import { resolveApplicationRoleId } from 'src/engine/twenty-orm/utils/resolve-application-role-id.util';import { type AgentRunRoleIds } from 'src/engine/metadata-modules/ai/ai-agent-execution/types/agent-run-role-ids.type';import { type AgentRunCallerContext } from 'src/engine/metadata-modules/ai/ai-agent-execution/types/agent-run-caller-context.type';+import { type AgentRunCallerOutcome } from 'src/engine/metadata-modules/ai/ai-agent-execution/types/agent-run-caller-outcome.type';import { type AgentRunCaller } from 'src/engine/metadata-modules/ai/ai-agent-execution/types/agent-run-caller.type';import { type AgentRunCallerInput } from 'src/engine/metadata-modules/ai/ai-agent-execution/types/agent-run-caller-input.type';import { type AgentRunCallerHandler } from 'src/engine/metadata-modules/ai/ai-agent-execution/types/agent-run-caller-handler.type';import { type OwnerWaitingState } from 'src/engine/core-modules/pending-wake-up/types/owner-waiting-state.type';import { type AgentRunExecutionContext } from 'src/engine/metadata-modules/ai/ai-agent-execution/types/agent-run-execution-context.type';import { type RunAsWorkspaceMemberContext } from 'src/engine/metadata-modules/ai/ai-agent-execution/types/run-as-workspace-member-context.type';+import { type SettledAgentRun } from 'src/engine/metadata-modules/ai/ai-agent-execution/types/settled-agent-run.type';import { buildAgentRolePermissionConfig } from 'src/engine/metadata-modules/ai/ai-agent-execution/utils/build-agent-role-permission-config.util';import { buildAgentRunThreadId } from 'src/engine/metadata-modules/ai/ai-agent-execution/utils/build-agent-run-thread-id.util';import { resolveRunAgentMessagesOrThrow } from 'src/engine/metadata-modules/ai/ai-agent-execution/utils/resolve-run-agent-messages-or-throw.util';@@ -67,9 +73,12 @@ type RunAgentServiceInput = {responseFormat?: unknown;thread?: RunAgentThread | null;runAsWorkspaceMemberId?: string;+ waitForCompletion?: boolean | null;};const agentApiRunCallerRefSchema = z.object({+ // what a call waiting on the run listens to; absent on runs suspended before calls could wait+ runId: z.string().optional(),agentId: z.string(),runAsWorkspaceMemberId: z.string().nullable(),// who a run that names no member acts as, when it is not the agent's application@@ -94,6 +103,8 @@ export class AgentRunService implements AgentRunCallerHandler, OnModuleInit {constructor(private readonly agentActorContextService: AgentActorContextService,private readonly agentRunnerService: AgentRunnerService,+ private readonly agentRunSettlementService: AgentRunSettlementService,+ private readonly agentRunSuspensionService: AgentRunSuspensionService,private readonly callerHandlerRegistry: AgentRunCallerHandlerRegistryService,private readonly apiKeyRoleService: ApiKeyRoleService,private readonly apiKeyService: ApiKeyService,@@ -236,67 +247,135 @@ export class AgentRunService implements AgentRunCallerHandler, OnModuleInit {executionContext.actorContext ??this.buildRunCreator({ callerApplication, application });- const caller: AgentRunCaller = {- type: 'AGENT_API_RUN',- ref: { ...callerRef, createdBy },- };+ const isContinuingConversation = isDefined(thread);- try {- const { outcome } = await this.agentRunnerService.run({+ // a conversation waiting on an earlier run refuses the call, rather than a run that fails later+ if (isContinuingConversation) {+ await this.agentRunSuspensionService.assertConversationNotSuspended({workspaceId: workspace.id,- conversation: { threadId, isCreated: !isDefined(thread) },- caller,- spec: {- agentId: agent.id,- title: isNonEmptyString(thread?.title) ? thread.title : agent.label,- baseSystemPrompt: AGENT_RUN_BASE_SYSTEM_PROMPT,- // kept with the run rather than in its messages, so a run that waits still has them when it goes on- instructions: input.additionalInstructions ?? null,- ...(isDefined(responseFormat) ? { responseFormat } : {}),- // the call returns before anyone could answer, so the run can wait but not ask- capabilities: {- canAskHumans: false,- },- toolLoadingStrategy: 'lazy',- },- agent,- prompt: {- messages,- senderUserWorkspaceId: memberUserWorkspaceId,- senderApplicationId: callerApplication?.id ?? null,- },- executionContext: { ...executionContext, turnCreatedBy: createdBy },+ threadId,});+ }- return {- threadId,- status: outcome.status,- result: outcome.status === 'COMPLETED' ? outcome.result : null,- error: outcome.status === 'FAILED' ? outcome.error : null,- success: outcome.status !== 'FAILED',- };- } catch (error) {- if (- error instanceof AiException &&- (error.code === AiExceptionCode.INVALID_AGENT_INPUT ||- error.code === AiExceptionCode.THREAD_AWAITING_ANSWER)- ) {- throw error;- }+ assertAgentResponseFormatHasOutputFieldsOrThrow(+ responseFormat ?? agent.responseFormat,+ );- this.logger.error(- `Agent execution failed for ${input.agentUniversalIdentifier}`,- error instanceof Error ? error.stack : error,- );+ const runId = randomUUID();- return {- threadId,+ await this.agentRunnerService.start({+ workspaceId: workspace.id,+ conversation: { threadId, isCreated: !isContinuingConversation },+ caller: {+ type: 'AGENT_API_RUN',+ ref: { ...callerRef, runId, createdBy },+ },+ spec: {+ agentId: agent.id,+ title: isNonEmptyString(thread?.title) ? thread.title : agent.label,+ baseSystemPrompt: AGENT_RUN_BASE_SYSTEM_PROMPT,+ // kept with the run rather than in its messages, so a run that waits still has them when it goes on+ instructions: input.additionalInstructions ?? null,+ ...(isDefined(responseFormat) ? { responseFormat } : {}),+ // the call returns before anyone could answer, so the run can wait but not ask+ capabilities: {+ canAskHumans: false,+ },+ toolLoadingStrategy: 'lazy',+ },+ prompt: {+ messages,+ senderUserWorkspaceId: memberUserWorkspaceId,+ senderApplicationId: callerApplication?.id ?? null,+ },+ });++ if (input.waitForCompletion === false) {+ return this.buildRunAgentResult({ threadId, status: 'RUNNING' });+ }++ const settledRun = await this.agentRunSettlementService.waitFor({+ runId,+ timeoutMs: RUN_AGENT_MAX_WAIT_MS,+ });++ return this.buildRunAgentResult({+ threadId,+ ...(settledRun ?? { status: 'RUNNING' }),+ });+ }++ // a call that waits on the run gets how it settled; a run that goes on later lands in its conversation+ async onSuspended({ caller }: AgentRunCallerInput): Promise<void> {+ await this.publishSettledRun({+ caller,+ settledRun: { status: 'SUSPENDED' },+ });+ }++ async onOutcome({+ caller,+ outcome,+ }: AgentRunCallerInput & { outcome: AgentRunCallerOutcome }): Promise<void> {+ await this.publishSettledRun({+ caller,+ settledRun:+ outcome.status === 'COMPLETED'+ ? { status: 'COMPLETED', result: outcome.result }+ : { status: 'FAILED', error: outcome.error },+ });+ }++ // the request that waits on the run is still there when its agent or key went away meanwhile+ async onCallerGone({+ caller,+ outcome,+ }: AgentRunCallerInput & { outcome: AgentRunCallerOutcome }): Promise<void> {+ await this.publishSettledRun({+ caller,+ settledRun: {status: 'FAILED',- result: null,- error: 'Agent execution failed.',- success: false,- };+ error:+ outcome.status === 'FAILED'+ ? outcome.error+ : 'The agent or the key that started this run is gone',+ },+ });+ }++ private async publishSettledRun({+ caller,+ settledRun,+ }: {+ caller: AgentRunCaller;+ settledRun: Pick<SettledAgentRun, 'status'> & Partial<SettledAgentRun>;+ }): Promise<void> {+ const { runId } = agentApiRunCallerRefSchema.parse(caller.ref);++ if (!isDefined(runId)) {+ return;}++ await this.agentRunSettlementService.publish({+ runId,+ settledRun: { result: null, error: null, ...settledRun },+ });+ }++ private buildRunAgentResult({+ threadId,+ status,+ result = null,+ error = null,+ }: Pick<RunAgentResult, 'threadId' | 'status'> &+ Partial<Pick<RunAgentResult, 'result' | 'error'>>): RunAgentResult {+ return {+ threadId,+ status,+ result,+ error,+ success: status !== 'FAILED',+ };}// The member a run acts as is looked up again, so a waiting run goes on with their current rolediff --git a/packages/twenty-server/src/engine/metadata-modules/ai/ai-agent-execution/services/agent-runner.service.ts b/packages/twenty-server/src/engine/metadata-modules/ai/ai-agent-execution/services/agent-runner.service.tsindex 911714d3444..c8212685af4 100644--- a/packages/twenty-server/src/engine/metadata-modules/ai/ai-agent-execution/services/agent-runner.service.ts+++ b/packages/twenty-server/src/engine/metadata-modules/ai/ai-agent-execution/services/agent-runner.service.ts@@ -1,27 +1,44 @@import { Injectable, Logger } from '@nestjs/common';import { isNonEmptyString } from '@sniptt/guards';+import { generateId } from 'ai';import { type AgentRunSummary } from 'twenty-shared/ai';import { type PendingWakeUpCondition } from 'twenty-shared/pending-wake-up';import { isDefined } from 'twenty-shared/utils';-import { type PendingWakeUpEntity } from 'src/engine/core-modules/pending-wake-up/entities/pending-wake-up.entity';+import { workspaceAuthContextStorage } from 'src/engine/core-modules/auth/storage/workspace-auth-context.storage';+import { InjectMessageQueue } from 'src/engine/core-modules/message-queue/decorators/message-queue.decorator';+import { MessageQueue } from 'src/engine/core-modules/message-queue/message-queue.constants';+import { MessageQueueService } from 'src/engine/core-modules/message-queue/services/message-queue.service';import { PendingWakeUpService } from 'src/engine/core-modules/pending-wake-up/services/pending-wake-up.service';+import { RUN_AGENT_MAX_WAIT_MS } from 'src/engine/metadata-modules/ai/ai-agent-execution/constants/run-agent-max-wait-ms.const';+import { AGENT_RUN_BUSY_RETRY_DELAY_MS } from 'src/engine/metadata-modules/ai/ai-agent-execution/constants/agent-run-busy-retry-delay-ms.const';+import { CONTINUE_AGENT_RUN_JOB_NAME } from 'src/engine/metadata-modules/ai/ai-agent-execution/constants/continue-agent-run-job-name.constant';+import { RUN_AGENT_JOB_NAME } from 'src/engine/metadata-modules/ai/ai-agent-execution/constants/run-agent-job-name.constant';import { AGENT_WAIT_PROMPT } from 'src/engine/metadata-modules/ai/ai-agent-execution/pausing-tools/constants/agent-wait-prompt.constant';import { createAgentWaitTools } from 'src/engine/metadata-modules/ai/ai-agent-execution/pausing-tools/wait-tools/create-agent-wait-tools.util';import { findAgentRunWait } from 'src/engine/metadata-modules/ai/ai-agent-execution/pausing-tools/wait-tools/find-agent-run-wait.util';-import { AgentAsyncExecutorService } from 'src/engine/metadata-modules/ai/ai-agent-execution/services/agent-async-executor.service';import { AgentRunCallerHandlerRegistryService } from 'src/engine/metadata-modules/ai/ai-agent-execution/services/agent-run-caller-handler-registry.service';import { AgentRunConversationService } from 'src/engine/metadata-modules/ai/ai-agent-execution/services/agent-run-conversation.service';import { AgentRunSuspensionService } from 'src/engine/metadata-modules/ai/ai-agent-execution/services/agent-run-suspension.service';+import { AgentRunTurnService } from 'src/engine/metadata-modules/ai/ai-agent-execution/services/agent-run-turn.service';+import { AgentTurnStatus } from 'src/engine/metadata-modules/ai/ai-history/enums/agent-turn-status.enum';+import { addStepToAgentTurnUsage } from 'src/engine/metadata-modules/ai/ai-billing/utils/add-step-to-agent-turn-usage.util';+import { type AgentRunCaller } from 'src/engine/metadata-modules/ai/ai-agent-execution/types/agent-run-caller.type';import { type AgentExecutionResult } from 'src/engine/metadata-modules/ai/ai-agent-execution/types/agent-execution-result.type';import { type AgentRunnerOutcome } from 'src/engine/metadata-modules/ai/ai-agent-execution/types/agent-runner-outcome.type';import { type AgentRunnerRunInput } from 'src/engine/metadata-modules/ai/ai-agent-execution/types/agent-runner-run-input.type';import { type AgentRunSuspension } from 'src/engine/metadata-modules/ai/ai-agent-execution/types/agent-run-suspension.type';import { type ContinueAgentRunJobData } from 'src/engine/metadata-modules/ai/ai-agent-execution/types/continue-agent-run-job-data.type';+import { type AgentRunCallerHandler } from 'src/engine/metadata-modules/ai/ai-agent-execution/types/agent-run-caller-handler.type';+import { type RunAgentJobData } from 'src/engine/metadata-modules/ai/ai-agent-execution/types/run-agent-job-data.type';import { buildAgentRunSummary } from 'src/engine/metadata-modules/ai/ai-agent-execution/utils/build-agent-run-summary.util';import { sumAgentRunSummaries } from 'src/engine/metadata-modules/ai/ai-agent-execution/utils/sum-agent-run-summaries.util';import { AgentEntity } from 'src/engine/metadata-modules/ai/ai-agent/entities/agent.entity';+import { type AgentChatStreamClaim } from 'src/engine/metadata-modules/ai/ai-chat/services/agent-chat-stream-recovery.service';+import { AgentChatEventPublisherService } from 'src/engine/metadata-modules/ai/ai-chat/services/agent-chat-event-publisher.service';+import { AgentTurnStreamService } from 'src/engine/metadata-modules/ai/ai-chat/services/agent-turn-stream.service';+import { AgentTurnRecorderService } from 'src/engine/metadata-modules/ai/ai-history/services/agent-turn-recorder.service';import { AgentConversationReaderService } from 'src/engine/metadata-modules/ai/ai-history/services/agent-conversation-reader.service';import { withDedicatedAiTrace } from 'src/engine/metadata-modules/ai/ai-models/utils/with-dedicated-ai-trace.util';import {@@ -33,86 +50,337 @@ import { WorkspaceScopedRepository } from 'src/engine/twenty-orm/workspace-scopeconst AGENT_RUN_MAX_CONTINUATIONS = 50;+const CONVERSATION_BUSY = Symbol('conversation-busy');++type AgentRunStream = {+ claim: AgentChatStreamClaim;+ abortSignal: AbortSignal;+};+type AgentRunnerResult = {- threadId: string;outcome: AgentRunnerOutcome;// the run so far, across every segment of a continued runsummary: AgentRunSummary;};-// Runs an agent turn and records it in its conversation. A run that pauses on an answer or a wait-// is suspended: the engine continues it on its own, through this same runner, and hands the-// outcome to its caller's handler@Injectable()export class AgentRunnerService {private readonly logger = new Logger(AgentRunnerService.name);constructor(- private readonly agentAsyncExecutorService: AgentAsyncExecutorService,+ private readonly agentRunTurnService: AgentRunTurnService,private readonly agentRunConversationService: AgentRunConversationService,private readonly conversationReaderService: AgentConversationReaderService,private readonly agentRunSuspensionService: AgentRunSuspensionService,private readonly callerHandlerRegistry: AgentRunCallerHandlerRegistryService,private readonly pendingWakeUpService: PendingWakeUpService,+ private readonly turnRecorderService: AgentTurnRecorderService,+ private readonly turnStreamService: AgentTurnStreamService,+ private readonly eventPublisherService: AgentChatEventPublisherService,+ @InjectMessageQueue(MessageQueue.aiQueue)+ private readonly messageQueueService: MessageQueueService,@InjectWorkspaceScopedRepository(AgentEntity)private readonly agentRepository: WorkspaceScopedRepository<AgentEntity>,) {}- // nothing else writes to the conversation while the run goes on: another run would read it half- // written, and a run that suspends would read a member's message once it goes on- run(input: AgentRunnerRunInput): Promise<AgentRunnerResult> {- return this.agentRunConversationService.withThreadLock({- workspaceId: input.workspaceId,- threadId: input.conversation.threadId,- work: () => this.runTurn({ input, suspension: null }),+ async start(input: RunAgentJobData): Promise<void> {+ await this.messageQueueService.add<RunAgentJobData>(+ RUN_AGENT_JOB_NAME,+ input,+ );+ }++ async runStarted(jobData: RunAgentJobData): Promise<void> {+ const { workspaceId, conversation, caller } = jobData;+ const handler = this.callerHandlerRegistry.getHandlerOrThrow(caller.type);++ // a caller that stopped waiting while the run was queued, such as a stopped workflow, must not have it act+ if ((await handler.getWaitingState({ workspaceId, caller })) === 'GONE') {+ await this.deliverFailure({+ workspaceId,+ caller,+ threadId: conversation.threadId,+ error: 'The caller stopped waiting before the run started',+ });++ return;+ }++ const result = await this.runClaimedOrDeliverFailure({ jobData, handler });++ if (!isDefined(result)) {+ return;+ }++ if (result === CONVERSATION_BUSY) {+ if (await this.retryWhenFree({ jobName: RUN_AGENT_JOB_NAME, jobData })) {+ return;+ }++ await this.deliverFailure({+ workspaceId,+ caller,+ threadId: conversation.threadId,+ error: 'The conversation stayed busy',+ });++ return;+ }++ if (result.outcome.status === 'SUSPENDED') {+ await this.showSuspension({+ workspaceId,+ caller,+ threadId: conversation.threadId,+ summary: result.summary,+ });++ return;+ }++ await this.agentRunSuspensionService.deliverOutcome({+ workspaceId,+ caller,+ threadId: conversation.threadId,+ outcome: result.outcome,+ summary: result.summary,});}- // Picks a suspended run up where it paused once its wake-up resolved, under its conversation's lock.+ // a run that fails before it settles is delivered as failed here, so null means nothing is left to do+ private async runClaimedOrDeliverFailure({+ jobData: { workspaceId, conversation, caller, spec, prompt },+ handler,+ }: {+ jobData: RunAgentJobData;+ handler: AgentRunCallerHandler;+ }): Promise<AgentRunnerResult | typeof CONVERSATION_BUSY | null> {+ try {+ const [executionContext, agent] = await Promise.all([+ handler.buildExecutionContext({ workspaceId, caller }),+ this.findAgentOrThrow({ workspaceId, agentId: spec.agentId }),+ ]);++ return await this.withClaimedConversation({+ workspaceId,+ threadId: conversation.threadId,+ title: spec.title,+ work: (stream) =>+ workspaceAuthContextStorage.run(executionContext.authContext, () =>+ this.runTurn({+ input: {+ workspaceId,+ conversation,+ caller,+ spec,+ agent,+ prompt,+ executionContext,+ },+ suspension: null,+ stream,+ }),+ ),+ });+ } catch (error) {+ this.logger.error(+ `Agent run in thread ${conversation.threadId} of workspace ${workspaceId} failed`,+ error instanceof Error ? error.stack : error,+ );++ await this.deliverFailure({+ workspaceId,+ caller,+ threadId: conversation.threadId,+ error: error instanceof Error ? error.message : String(error),+ });++ return null;+ }+ }++ // Picks a suspended run up where it paused once its wake-up resolved, holding its conversation.// The wake-up is removed only once the run went on and its caller has the outcome, so a job that// died or failed midway finds it again, and a pause the run made again replaced it with another// id, so each pause goes on once- continue({- workspaceId,- threadId,- wakeUpId,- outcome,- }: ContinueAgentRunJobData): Promise<void> {- return this.agentRunConversationService.withThreadLock({+ async continue(jobData: ContinueAgentRunJobData): Promise<void> {+ const { workspaceId, threadId, wakeUpId, outcome } = jobData;+ const wakeUp = await this.pendingWakeUpService.find({workspaceId,- threadId,- work: async () => {- const wakeUp = await this.pendingWakeUpService.find({- workspaceId,- wakeUpId,- });+ wakeUpId,+ });++ if (!isDefined(wakeUp)) {+ return;+ }++ const suspension = wakeUp.payload as AgentRunSuspension;- if (!isDefined(wakeUp)) {- return;+ const continued = await this.withClaimedConversation({+ workspaceId,+ threadId,+ title: suspension.runSpec.title,+ work: async (stream) => {+ // a duplicate job waited for the conversation while the first one went on and claimed the wake-up+ if (+ !isDefined(+ await this.pendingWakeUpService.find({ workspaceId, wakeUpId }),+ )+ ) {+ return null;}- await this.continueSuspendedRun({+ const result = await this.continueSuspendedRun({workspaceId,threadId,- wakeUp,+ suspension,+ isAwaitingAnswer: wakeUp.condition.type === 'ANSWER',outcome,+ stream,});await this.pendingWakeUpService.claim({ workspaceId, wakeUpId });++ return result;},});++ if (continued === CONVERSATION_BUSY) {+ if (+ !(await this.retryWhenFree({+ jobName: CONTINUE_AGENT_RUN_JOB_NAME,+ jobData,+ }))+ ) {+ await this.agentRunSuspensionService.settle({+ workspaceId,+ threadId,+ suspension,+ outcome: { status: 'FAILED', error: 'The conversation stayed busy' },+ isAwaitingAnswer: wakeUp.condition.type === 'ANSWER',+ });+ }++ return;+ }++ if (continued?.outcome.status === 'SUSPENDED') {+ await this.showSuspension({+ workspaceId,+ caller: suspension.caller,+ threadId,+ summary: continued.summary,+ });+ }+ }++ // shown once the run let go of its conversation, so a caller that sees it waiting can answer it+ private async showSuspension({+ workspaceId,+ caller,+ threadId,+ summary,+ }: {+ workspaceId: string;+ caller: AgentRunCaller;+ threadId: string;+ summary: AgentRunSummary;+ }): Promise<void> {+ await this.tryRecording(+ `show the caller of thread ${threadId} where its run waits`,+ async () =>+ this.callerHandlerRegistry+ .getHandlerOrThrow(caller.type)+ .onSuspended?.({ workspaceId, caller, threadId, summary }),+ );+ }++ private async withClaimedConversation<TResult>({+ workspaceId,+ threadId,+ title,+ work,+ }: {+ workspaceId: string;+ threadId: string;+ title: string;+ work: (stream: AgentRunStream) => Promise<TResult>;+ }): Promise<TResult | typeof CONVERSATION_BUSY> {+ const streamId = generateId();+ const isClaimed = await this.agentRunConversationService.claimThread({+ workspaceId,+ threadId,+ title,+ streamId,+ });++ if (!isClaimed) {+ return CONVERSATION_BUSY;+ }++ const claim = { workspaceId, threadId, streamId };++ return this.turnStreamService.runClaimed({+ claim,+ work: (abortSignal) => work({ claim, abortSignal }),+ });+ }++ private async retryWhenFree<TJobData extends { busyAttempt?: number }>({+ jobName,+ jobData,+ }: {+ jobName: string;+ jobData: TJobData;+ }): Promise<boolean> {+ const busyAttempt = (jobData.busyAttempt ?? 0) + 1;++ // a run holding the conversation for longer than a runAgent call waits on it is taken as stuck+ if (busyAttempt * AGENT_RUN_BUSY_RETRY_DELAY_MS >= RUN_AGENT_MAX_WAIT_MS) {+ return false;+ }++ await this.messageQueueService.add<TJobData>(+ jobName,+ { ...jobData, busyAttempt },+ { delay: AGENT_RUN_BUSY_RETRY_DELAY_MS },+ );++ return true;+ }++ private async deliverFailure({+ workspaceId,+ caller,+ threadId,+ error,+ }: {+ workspaceId: string;+ caller: AgentRunCaller;+ threadId: string;+ error: string;+ }): Promise<void> {+ await this.agentRunSuspensionService.deliverOutcome({+ workspaceId,+ caller,+ threadId,+ outcome: { status: 'FAILED', error },+ summary: null,+ });}private async continueSuspendedRun({workspaceId,threadId,- wakeUp,+ suspension,+ isAwaitingAnswer,outcome,+ stream,}: Omit<ContinueAgentRunJobData, 'wakeUpId'> & {- wakeUp: Pick<PendingWakeUpEntity, 'payload' | 'condition'>;- }): Promise<void> {- const suspension = wakeUp.payload as AgentRunSuspension;- const isAwaitingAnswer = wakeUp.condition.type === 'ANSWER';+ suspension: AgentRunSuspension;+ isAwaitingAnswer: boolean;+ stream: AgentRunStream;+ }): Promise<AgentRunnerResult | null> {const { caller, runSpec } = suspension;const handler = this.callerHandlerRegistry.getHandlerOrThrow(caller.type);@@ -125,9 +393,10 @@ export class AgentRunnerService {workspaceId,threadId,isAwaitingAnswer,+ streamId: stream.claim.streamId,});- return;+ return null;}if (outcome.type === 'ANSWERED' && 'error' in outcome.answer) {@@ -141,32 +410,38 @@ export class AgentRunnerService {outcome: outcome.type === 'ANSWERED' ? { type: 'CANCELLED' } : outcome,});- const agent = await this.agentRepository.findOne(workspaceId, {- where: { id: runSpec.agentId },- });-- if (!isDefined(agent)) {- throw new AiException(- `Agent with id ${runSpec.agentId} not found`,- AiExceptionCode.AGENT_NOT_FOUND,- );+ if (outcome.type === 'MESSAGE_RECEIVED' && isAwaitingAnswer) {+ // a member who writes instead of answering moves on from the question, as in chat+ await this.agentRunSuspensionService.closePendingQuestion({+ workspaceId,+ threadId,+ streamId: stream.claim.streamId,+ turnStatus: AgentTurnStatus.COMPLETED,+ });}- result = await this.runTurn({- input: {- workspaceId,- conversation: { threadId, isCreated: false },- caller,- spec: runSpec,- agent,- prompt: null,- executionContext: await handler.buildExecutionContext({- workspaceId,- caller,+ const [agent, executionContext] = await Promise.all([+ this.findAgentOrThrow({ workspaceId, agentId: runSpec.agentId }),+ handler.buildExecutionContext({ workspaceId, caller }),+ ]);++ result = await workspaceAuthContextStorage.run(+ executionContext.authContext,+ () =>+ this.runTurn({+ input: {+ workspaceId,+ conversation: { threadId, isCreated: false },+ caller,+ spec: runSpec,+ agent,+ prompt: null,+ executionContext,+ },+ suspension,+ stream,}),- },- suspension,- });+ );} catch (error) {await this.agentRunSuspensionService.settle({workspaceId,@@ -177,9 +452,12 @@ export class AgentRunnerService {error: error instanceof Error ? error.message : String(error),},isAwaitingAnswer,+ streamId: stream.claim.streamId,});+ // handled here instead of thrown, so the interrupted turn is failed here as a thrown error would be+ await this.turnStreamService.failStream({ claim: stream.claim, error });- return;+ return null;}if (result.outcome.status !== 'SUSPENDED') {@@ -189,17 +467,22 @@ export class AgentRunnerService {suspension,outcome: result.outcome,summary: result.summary,+ streamId: stream.claim.streamId,});}++ return result;}private async runTurn({input,suspension,+ stream: { claim, abortSignal },}: {input: AgentRunnerRunInput;// what a continued run resumes fromsuspension: AgentRunSuspension | null;+ stream: AgentRunStream;}): Promise<AgentRunnerResult> {const {workspaceId,@@ -217,145 +500,186 @@ export class AgentRunnerService {});}- const priorMessages = isCreated- ? []- : await this.conversationReaderService.loadMessages({- workspaceId,- threadId,- actor: executionContext.conversationActor,- });+ const { turnId, promptMessageIds, hasTakenQueuedMessages } =+ await this.agentRunConversationService.openTurn({+ workspaceId,+ threadId,+ agentId: agent.id,+ senderUserWorkspaceId: prompt?.senderUserWorkspaceId ?? null,+ senderApplicationId: prompt?.senderApplicationId ?? null,+ createdBy: executionContext.turnCreatedBy,+ messages: prompt?.messages ?? [],+ });- const turnId = await this.tryRecording(- `record the agent turn in thread ${threadId}`,- () =>- this.agentRunConversationService.openTurn({- workspaceId,- threadId,- title: spec.title,- agentId: agent.id,- senderUserWorkspaceId: prompt?.senderUserWorkspaceId ?? null,- senderApplicationId: prompt?.senderApplicationId ?? null,- createdBy: executionContext.turnCreatedBy,- messages: prompt?.messages ?? [],- }),+ if (hasTakenQueuedMessages) {+ await this.eventPublisherService.publish({+ workspaceId,+ threadId,+ event: { type: 'queue-updated' },+ });+ }++ // read once the turn is open, so it holds the messages the turn took, while its prompt goes to the model as is+ const conversationMessages =+ await this.conversationReaderService.loadMessages({+ workspaceId,+ threadId,+ actor: executionContext.conversationActor,+ });+ const priorMessages = conversationMessages.filter(+ ({ id }) => !promptMessageIds.includes(id),);const startedAtMs = Date.now();+ const turnInput = {+ agent,+ messages: prompt?.messages ?? [],+ priorMessages,+ // every run here goes on in the background, unlike a chat, so it can wait. The wait tools are+ // the engine's, so it explains them; the caller's own instructions come last+ baseSystemPrompt: [+ spec.baseSystemPrompt,+ AGENT_WAIT_PROMPT,+ ...(isNonEmptyString(spec.instructions) ? [spec.instructions] : []),+ ].join('\n\n'),+ pausingTools: createAgentWaitTools(),+ canAskHumans: spec.capabilities.canAskHumans,+ responseFormat: spec.responseFormat ?? agent.responseFormat,+ workspaceId,+ executionContext,+ additionalExcludedToolNames: spec.additionalExcludedToolNames,+ toolLoadingStrategy: spec.toolLoadingStrategy,+ abortSignal,+ };- let execution: AgentExecutionResult;+ // the toolset and prompt come from the spec alone, so a continued run gets those it paused with+ const streamed = await withDedicatedAiTrace(() =>+ this.turnStreamService.stream({+ claim,+ turnId,+ startTurn: () => this.agentRunTurnService.startTurn(turnInput),+ }),+ );- try {- // the toolset and prompt come from the spec alone, so a continued run gets those it paused with- execution = await withDedicatedAiTrace(() =>- this.agentAsyncExecutorService.executeAgent({- agent,- messages: prompt?.messages ?? [],- priorMessages,- // every run here goes on in the background, unlike a chat, so it can wait. The wait tools are- // the engine's, so it explains them; the caller's own instructions come last- baseSystemPrompt: [- spec.baseSystemPrompt,- AGENT_WAIT_PROMPT,- ...(isNonEmptyString(spec.instructions) ? [spec.instructions] : []),- ].join('\n\n'),- pausingTools: createAgentWaitTools(),- canAskHumans: spec.capabilities.canAskHumans,- responseFormat: spec.responseFormat ?? agent.responseFormat,- workspaceId,- executionContext,- additionalExcludedToolNames: spec.additionalExcludedToolNames,- toolLoadingStrategy: spec.toolLoadingStrategy,- }),- );- } catch (error) {- if (isDefined(turnId)) {- await this.tryRecording(- `record the agent turn in thread ${threadId}`,- () =>- this.agentRunConversationService.failTurn({- workspaceId,- turnId,- error,- }),- );- }+ // a turn that fails keeps the usage of the steps it finished, as billing does+ const recordStreamedUsage = () =>+ this.turnRecorderService.recordUsage({+ workspaceId,+ threadId,+ turnId,+ usage: streamed.usage,+ });- throw error;+ if (isDefined(streamed.streamError)) {+ await recordStreamedUsage();++ throw streamed.streamError;}- const durationMs = Date.now() - startedAtMs;+ const execution = await this.agentRunTurnService+ .completeTurn({ ...turnInput, started: streamed.started })+ .catch(async (error: unknown) => {+ await recordStreamedUsage();- const closedTurn = isDefined(turnId)- ? await this.tryRecording(- `record the agent turn in thread ${threadId}`,- () =>- this.agentRunConversationService.closeTurn({- workspaceId,- threadId,- turnId,- title: spec.title,- agentId: agent.id,- execution,- }),+ throw error;+ });++ const wait = execution.isPaused+ ? findAgentRunWait(+ execution.steps[execution.steps.length - 1]?.toolResults ?? [],)- : null;+ : undefined;++ const finished = await this.turnStreamService.finishTurn({+ claim,+ turnId,+ result: {+ ...streamed,+ usage: execution.structuredOutputSteps.reduce(+ (usage, step) => ({+ ...usage,+ ...addStepToAgentTurnUsage(+ streamed.started.source.modelConfig,+ usage,+ step,+ ),+ }),+ streamed.usage,+ ),+ },+ modelId: execution.modelId,+ stopReason: execution.stopReason,+ isWaiting: isDefined(wait),+ isBackgroundRun: true,+ });++ await this.turnStreamService.publishTurnEnd({+ claim,+ assistantMessageId: streamed.assistantMessageId,+ stopReason: execution.stopReason,+ });const summary = sumAgentRunSummaries({previousSummary: suspension?.summary ?? null,- nextSummary: buildAgentRunSummary({ execution, durationMs }),+ nextSummary: buildAgentRunSummary({+ execution,+ durationMs: Date.now() - startedAtMs,+ }),});return {- threadId,summary,- outcome: await this.settleTurn({- input,- execution,- closedTurn,- summary,- suspension,- }),+ // a stop can also land while the structured output is asked for, after the stream ended+ outcome:+ streamed.isAborted || abortSignal.aborted+ ? { status: 'FAILED', error: 'The run was stopped' }+ : await this.settleTurn({+ input,+ execution,+ wait,+ isAwaitingAnswer: finished?.isAwaitingAnswer ?? false,+ summary,+ suspension,+ streamId: claim.streamId,+ }),};}private async settleTurn({input: { workspaceId, conversation, caller, spec },execution,- closedTurn,+ wait,+ isAwaitingAnswer,summary,suspension,+ streamId,}: {input: AgentRunnerRunInput;execution: AgentExecutionResult;- // null when the turn could not be recorded- closedTurn: { isAwaitingAnswer: boolean } | null;+ wait: ReturnType<typeof findAgentRunWait>;+ isAwaitingAnswer: boolean;summary: AgentRunSummary;suspension: AgentRunSuspension | null;+ streamId: string;}): Promise<AgentRunnerOutcome> {- if (!execution.hasNoMoreAvailableCredits && !execution.isPaused) {+ if (!isDefined(execution.stopReason) && !execution.isPaused) {return { status: 'COMPLETED', result: execution.result };}const { threadId } = conversation;- const wait = findAgentRunWait(- execution.steps[execution.steps.length - 1]?.toolResults ?? [],- );const continuationCount = isDefined(suspension)? suspension.continuationCount + 1: 0;const hasPausedTooOften = continuationCount >= AGENT_RUN_MAX_CONTINUATIONS;// a run waits on the questions it asked rather than on a wait it called alongside them- const condition: PendingWakeUpCondition | undefined =- closedTurn?.isAwaitingAnswer === true- ? { type: 'ANSWER', threadId }- : wait?.condition;+ const condition: PendingWakeUpCondition | undefined = isAwaitingAnswer+ ? { type: 'ANSWER', threadId }+ : wait?.condition;- // continuing reads the conversation, so a pause it does not hold could never be continued.// A failure to suspend only fails the run, like a pause that could not be recordedconst isSuspended =- !execution.hasNoMoreAvailableCredits &&+ !isDefined(execution.stopReason) &&!hasPausedTooOften &&- isDefined(closedTurn) &&isDefined(condition) &&(await this.tryRecording(`suspend the agent run in thread ${threadId}`,@@ -396,10 +720,11 @@ export class AgentRunnerService {await this.agentRunSuspensionService.closeAwaitedCalls({workspaceId,threadId,- isAwaitingAnswer: closedTurn?.isAwaitingAnswer === true,+ isAwaitingAnswer,+ streamId,});- if (execution.hasNoMoreAvailableCredits) {+ if (isDefined(execution.stopReason)) {return {status: 'FAILED',error: 'Agent stopped: no more available credits.',@@ -413,20 +738,35 @@ export class AgentRunnerService {};}- if (isDefined(wait)) {- return {- status: 'FAILED',- error:- 'Agent paused to wait but its conversation could not be recorded.',- };- }-return {status: 'FAILED',- error: 'Agent asked a question that could not be recorded.',+ error: isDefined(wait)+ ? 'Agent paused to wait but its conversation could not be recorded.'+ : 'Agent asked a question that could not be recorded.',};}+ private async findAgentOrThrow({+ workspaceId,+ agentId,+ }: {+ workspaceId: string;+ agentId: string;+ }): Promise<AgentEntity> {+ const agent = await this.agentRepository.findOne(workspaceId, {+ where: { id: agentId },+ });++ if (!isDefined(agent)) {+ throw new AiException(+ `Agent with id ${agentId} not found`,+ AiExceptionCode.AGENT_NOT_FOUND,+ );+ }++ return agent;+ }+// A record of the run, not its outcome, so a write failure must not fail the runprivate async tryRecording<TResult>(description: string,diff --git a/packages/twenty-server/src/engine/metadata-modules/ai/ai-agent-execution/services/agent-step-loop.service.ts b/packages/twenty-server/src/engine/metadata-modules/ai/ai-agent-execution/services/agent-step-loop.service.tsnew file mode 100644index 00000000000..7189a35c956--- /dev/null+++ b/packages/twenty-server/src/engine/metadata-modules/ai/ai-agent-execution/services/agent-step-loop.service.ts@@ -0,0 +1,458 @@+import { Injectable, Logger } from '@nestjs/common';++import { type ProviderOptions } from '@ai-sdk/provider-utils';+import {+ isStepCount,+ type ModelMessage,+ NoOutputGeneratedError,+ type StepResult,+ type StopCondition,+ streamText,+ type SystemModelMessage,+ type ToolSet,+} from 'ai';+import { isDefined, sortKeysDeeply } from 'twenty-shared/utils';++import { AI_LATENCY_MS_BUCKET_BOUNDARIES } from 'src/engine/core-modules/metrics/constants/ai-latency-ms-bucket-boundaries.constant';+import { TOOL_EXECUTION_DURATION_MS_BUCKET_BOUNDARIES } from 'src/engine/core-modules/metrics/constants/tool-execution-duration-ms-bucket-boundaries.constant';+import { TOOL_OUTPUT_TOKENS_BUCKET_BOUNDARIES } from 'src/engine/core-modules/metrics/constants/tool-output-tokens-bucket-boundaries.constant';+import { ExceptionHandlerService } from 'src/engine/core-modules/exception-handler/exception-handler.service';+import { MetricsService } from 'src/engine/core-modules/metrics/metrics.service';+import { MetricsKeys } from 'src/engine/core-modules/metrics/types/metrics-keys.type';+import { estimateToolOutputTokens } from 'src/engine/core-modules/tool-provider/utils/estimate-tool-output-tokens.util';+import { getToolMetricName } from 'src/engine/core-modules/tool-provider/utils/get-tool-metric-name.util';+import { isToolOutputSuccessful } from 'src/engine/core-modules/tool-provider/utils/is-tool-output-successful.util';+import { resolveToolName } from 'src/engine/core-modules/tool-provider/utils/resolve-tool-name.util';+import { type UsageOperationType } from 'src/engine/core-modules/usage/enums/usage-operation-type.enum';+import { type UsageSpenders } from 'src/engine/core-modules/usage/types/usage-spenders.type';+import { endsOnPausingToolCall } from 'src/engine/metadata-modules/ai/ai-agent-execution/pausing-tools/utils/ends-on-pausing-tool-call.util';+import {+ getCacheProviderOptions,+ injectCacheBreakpoint,+} from 'src/engine/metadata-modules/ai/ai-agent-execution/utils/provider-options.util';+import { AGENT_CONFIG } from 'src/engine/metadata-modules/ai/ai-agent/constants/agent-config.const';+import { repairToolCall } from 'src/engine/metadata-modules/ai/ai-agent/utils/repair-tool-call.util';+import {+ AiBillingService,+ type WebSearchOperationType,+} from 'src/engine/metadata-modules/ai/ai-billing/services/ai-billing.service';+import { type StepDebitResult } from 'src/engine/metadata-modules/ai/ai-billing/types/step-debit-result.type';+import {+ extractCacheCreationTokens,+ extractCacheCreationTokensFromSteps,+} from 'src/engine/metadata-modules/ai/ai-billing/utils/extract-cache-creation-tokens.util';+import { type RegisteredAiModel } from 'src/engine/metadata-modules/ai/ai-models/services/ai-model-registry.service';+import { type AiModelConfig } from 'src/engine/metadata-modules/ai/ai-models/types/ai-model-config.type';+import { buildAiTelemetry } from 'src/engine/metadata-modules/ai/ai-models/utils/build-ai-telemetry.util';+import {+ AiException,+ AiExceptionCode,+} from 'src/engine/metadata-modules/ai/ai.exception';++// chat and agent runs report to their own metric keys, so existing dashboards keep their series+type AgentStepLoopMetricsScope = 'chat' | 'agent';++type ToolMetricKeys = {+ succeeded: MetricsKeys;+ failed: MetricsKeys;+ durationMs: MetricsKeys;+ outputTokens: MetricsKeys;+};++const METRICS_CONFIG_BY_SCOPE: Record<+ AgentStepLoopMetricsScope,+ {+ toolMetricKeys: ToolMetricKeys;+ stepTokensLogTag: string;+ recordsTurnMetrics: boolean;+ }+> = {+ chat: {+ toolMetricKeys: {+ succeeded: MetricsKeys.AiChatToolExecutionSucceeded,+ failed: MetricsKeys.AiChatToolExecutionFailed,+ durationMs: MetricsKeys.AiChatToolExecutionDurationMs,+ outputTokens: MetricsKeys.AiChatToolOutputTokens,+ },+ stepTokensLogTag: 'AI_CHAT_TOKENS',+ recordsTurnMetrics: true,+ },+ agent: {+ toolMetricKeys: {+ succeeded: MetricsKeys.WorkflowAgentToolExecutionSucceeded,+ failed: MetricsKeys.WorkflowAgentToolExecutionFailed,+ durationMs: MetricsKeys.WorkflowAgentToolExecutionDurationMs,+ outputTokens: MetricsKeys.WorkflowAgentToolOutputTokens,+ },+ stepTokensLogTag: 'AI_AGENT_TOKENS',+ recordsTurnMetrics: false,+ },+};++export type AgentStepLoopInput = {+ registeredModel: RegisteredAiModel;+ modelConfig: Pick<AiModelConfig, 'sdkPackage' | 'modelFamily'>;+ systemPrompt: string;+ contextMessages?: SystemModelMessage[];+ messages: ModelMessage[];+ tools: ToolSet;+ toolChoice?: 'auto' | 'required';+ abortSignal?: AbortSignal;+ offeredPausingToolNames: string[];+ shouldStopAfterStep?: StopCondition<ToolSet>;+ billing: {+ workspaceId: string;+ operationType: UsageOperationType;+ webSearchOperationType: WebSearchOperationType;+ spenders: UsageSpenders;+ };+ onUserCreditsExhausted?: (notice: {+ renewsAt: Date;+ isFallingBackToWorkspaceCredits: boolean;+ }) => Promise<void>;+ telemetry: Parameters<typeof buildAiTelemetry>[0];+ providerOptions?: ProviderOptions;+ streamRetries?: number;+ beforeStep?: () => Promise<unknown>;+ metricsScope: AgentStepLoopMetricsScope;+ onError?: (error: unknown) => void;+};++export type AgentStepLoop = {+ stream: ReturnType<typeof streamText<ToolSet>>;+ getStopReason: () => StepDebitResult['stopReason'];+};++const sumSteps = (+ steps: StepResult<ToolSet>[],+ read: (usage: StepResult<ToolSet>['usage']) => number | undefined,+): number => steps.reduce((total, { usage }) => total + (read(usage) ?? 0), 0);++const isExpectedLoopEnd = (error: unknown): boolean =>+ (error instanceof Error && error.name === 'AbortError') ||+ (error instanceof AiException &&+ error.code === AiExceptionCode.STREAM_INTERRUPTED) ||+ NoOutputGeneratedError.isInstance(error);++@Injectable()+export class AgentStepLoopService {+ private readonly logger = new Logger(AgentStepLoopService.name);++ constructor(+ private readonly aiBillingService: AiBillingService,+ private readonly metricsService: MetricsService,+ private readonly exceptionHandlerService: ExceptionHandlerService,+ ) {}++ start({+ registeredModel,+ modelConfig,+ systemPrompt,+ contextMessages = [],+ messages,+ tools,+ toolChoice = 'auto',+ abortSignal,+ offeredPausingToolNames,+ shouldStopAfterStep,+ billing,+ telemetry,+ providerOptions,+ streamRetries,+ beforeStep,+ metricsScope,+ onError,+ onUserCreditsExhausted,+ }: AgentStepLoopInput): AgentStepLoop {+ const { toolMetricKeys, stepTokensLogTag, recordsTurnMetrics } =+ METRICS_CONFIG_BY_SCOPE[metricsScope];+ const modelAttributes = { model: registeredModel.modelId };+ const finishedSteps: StepResult<ToolSet>[] = [];+ const startedAt = performance.now();+ let stopReason: StepDebitResult['stopReason'] = null;+ let userCreditsUsedMicro = 0;+ let stepStartedAt = startedAt;+ let isUsageRecorded = false;+ let isFirstTokenRecorded = !recordsTurnMetrics;++ const notifyUserCreditsCrossing = async (debit: StepDebitResult) => {+ if (+ isDefined(debit.userAllowanceExhaustedUntil) &&+ debit.userCreditsUsedMicro > 0+ ) {+ await onUserCreditsExhausted?.({+ renewsAt: debit.userAllowanceExhaustedUntil,+ isFallingBackToWorkspaceCredits: debit.stopReason === null,+ });+ }+ };++ // credits were taken step by step; the run's usage is recorded once, however it ends+ const recordUsage = async (steps: StepResult<ToolSet>[]) => {+ if (isUsageRecorded) {+ return;+ }++ isUsageRecorded = true;++ const webSearchDebit = await this.aiBillingService.recordStepsUsage({+ modelId: registeredModel.modelId,+ steps,+ ...billing,+ userCreditsUsedMicro,+ });++ if (isDefined(webSearchDebit) && stopReason !== 'userAllowance') {+ await notifyUserCreditsCrossing(webSearchDebit);+ }++ if (!recordsTurnMetrics) {+ return;+ }++ for (const [key, amount] of [+ [+ MetricsKeys.AiChatInputTokens,+ sumSteps(steps, (usage) => usage.inputTokens),+ ],+ [+ MetricsKeys.AiChatOutputTokens,+ sumSteps(steps, (usage) => usage.outputTokens),+ ],+ [+ MetricsKeys.AiChatCacheReadTokens,+ sumSteps(steps, (usage) => usage.inputTokenDetails?.cacheReadTokens),+ ],+ [+ MetricsKeys.AiChatCacheWriteTokens,+ extractCacheCreationTokensFromSteps(steps),+ ],+ ] as const) {+ this.metricsService.incrementCounterBy({+ key,+ amount,+ attributes: modelAttributes,+ });+ }++ this.metricsService.recordHistogram({+ key: MetricsKeys.AiChatTurnLatencyMs,+ value: performance.now() - startedAt,+ unit: 'ms',+ attributes: modelAttributes,+ bucketBoundaries: AI_LATENCY_MS_BUCKET_BOUNDARIES,+ });+ };++ const systemMessage: SystemModelMessage = {+ role: 'system',+ content: systemPrompt,+ providerOptions: getCacheProviderOptions(modelConfig),+ };++ const stream = streamText({+ model: registeredModel.model,+ instructions: [systemMessage, ...contextMessages],+ messages,+ tools,+ toolChoice,+ abortSignal,+ streamRetries,+ stopWhen: async (step) =>+ isDefined(stopReason) ||+ endsOnPausingToolCall({+ steps: step.steps,+ offeredToolNames: offeredPausingToolNames,+ }) ||+ (await isStepCount(AGENT_CONFIG.MAX_STEPS)(step)) ||+ (await (shouldStopAfterStep?.(step) ?? false)),+ ...buildAiTelemetry(telemetry),+ providerOptions,+ // keys are sorted and the last message carries the cache point, so the prompt prefix stays cacheable+ prepareStep: async ({ messages: stepMessages }) => {+ await beforeStep?.();+ stepStartedAt = performance.now();++ return {+ messages: injectCacheBreakpoint(+ stepMessages.map(sortKeysDeeply),+ modelConfig,+ ),+ };+ },+ onChunk: ({ chunk }) => {+ if (+ isFirstTokenRecorded ||+ (chunk.type !== 'text-delta' && chunk.type !== 'tool-call')+ ) {+ return;+ }++ isFirstTokenRecorded = true;+ this.metricsService.recordHistogram({+ key: MetricsKeys.AiChatTtftMs,+ value: performance.now() - startedAt,+ unit: 'ms',+ attributes: modelAttributes,+ bucketBoundaries: AI_LATENCY_MS_BUCKET_BOUNDARIES,+ });+ },+ onError: onError ? ({ error }) => onError(error) : undefined,+ onToolExecutionEnd: (event) => {+ this.metricsService.recordHistogram({+ key: toolMetricKeys.durationMs,+ value: event.toolExecutionMs,+ unit: 'ms',+ attributes: {+ ...modelAttributes,+ tool: getToolMetricName(event.toolCall.toolName),+ },+ bucketBoundaries: TOOL_EXECUTION_DURATION_MS_BUCKET_BOUNDARIES,+ });+ },+ onStepEnd: async (step) => {+ finishedSteps.push(step);++ if (recordsTurnMetrics) {+ this.metricsService.recordHistogram({+ key: MetricsKeys.AiChatStepLatencyMs,+ value: performance.now() - stepStartedAt,+ unit: 'ms',+ attributes: modelAttributes,+ bucketBoundaries: AI_LATENCY_MS_BUCKET_BOUNDARIES,+ });+ }++ const stepDebit =+ await this.aiBillingService.decrementAndCheckAvailableCredits({+ modelId: registeredModel.modelId,+ billingInput: {+ usage: step.usage,+ cacheCreationTokens: extractCacheCreationTokens(+ step.providerMetadata,+ ),+ },+ workspaceId: billing.workspaceId,+ operationType: billing.operationType,+ spenders: billing.spenders,+ });++ stopReason ??= stepDebit.stopReason;+ userCreditsUsedMicro += stepDebit.userCreditsUsedMicro;++ await notifyUserCreditsCrossing(stepDebit);++ this.logStepTokens({+ step,+ stepTokensLogTag,+ stepIndex: finishedSteps.length,+ });+ this.recordToolResultMetrics({ step, toolMetricKeys, modelAttributes });+ },+ onAbort: ({ steps }) => recordUsage(steps),+ repairToolCall: ({+ toolCall,+ tools: toolsForRepair,+ inputSchema,+ error,+ }) =>+ repairToolCall({+ toolCall,+ tools: toolsForRepair,+ inputSchema,+ error,+ model: registeredModel.model,+ billingContext: {+ aiBillingService: this.aiBillingService,+ modelId: registeredModel.modelId,+ workspaceId: billing.workspaceId,+ spenders: billing.spenders,+ operationType: billing.operationType,+ },+ }),+ });++ Promise.all([stream.usage, stream.steps])+ .then(+ ([, steps]) => steps,+ (error) => {+ if (!isExpectedLoopEnd(error)) {+ this.exceptionHandlerService.captureExceptions([error]);+ }++ return [...finishedSteps];+ },+ )+ .then(recordUsage)+ .catch((error) =>+ this.exceptionHandlerService.captureExceptions([error]),+ );++ return {+ stream,+ getStopReason: () => stopReason,+ };+ }++ private logStepTokens({+ step,+ stepTokensLogTag,+ stepIndex,+ }: {+ step: StepResult<ToolSet>;+ stepTokensLogTag: string;+ stepIndex: number;+ }): void {+ this.logger.log(+ `[${stepTokensLogTag}] step #${stepIndex} — ` ++ `toolCallIds=[${step.toolCalls.map((toolCall) => toolCall.toolCallId).join(', ')}]: ` ++ `outputTokens=${step.usage.outputTokens ?? 0}, ` ++ `reasoningTokens=${step.usage.outputTokenDetails?.reasoningTokens ?? 0}, ` ++ `inputTokens(fullContext)=${step.usage.inputTokens ?? 0}, ` ++ `cacheReadTokens=${step.usage.inputTokenDetails?.cacheReadTokens ?? 0}, ` ++ `cacheWriteTokens=${step.usage.inputTokenDetails?.cacheWriteTokens ?? 0}, ` ++ `cacheCreationTokens=${extractCacheCreationTokens(step.providerMetadata)}, ` ++ `totalTokens=${step.usage.totalTokens ?? 0}`,+ );+ }++ private recordToolResultMetrics({+ step,+ toolMetricKeys,+ modelAttributes,+ }: {+ step: StepResult<ToolSet>;+ toolMetricKeys: ToolMetricKeys;+ modelAttributes: { model: string };+ }): void {+ for (const part of step.content) {+ if (part.type !== 'tool-result' && part.type !== 'tool-error') {+ continue;+ }++ const succeeded =+ part.type === 'tool-result' && isToolOutputSuccessful(part.output);++ const attributes = {+ ...modelAttributes,+ tool: getToolMetricName(resolveToolName(part)),+ };++ this.metricsService.incrementCounterBy({+ key: succeeded ? toolMetricKeys.succeeded : toolMetricKeys.failed,+ amount: 1,+ attributes,+ });++ this.metricsService.recordHistogram({+ key: toolMetricKeys.outputTokens,+ value: estimateToolOutputTokens(+ part.type === 'tool-result' ? part.output : part.error,+ ),+ unit: 'token',+ attributes,+ bucketBoundaries: TOOL_OUTPUT_TOKENS_BUCKET_BOUNDARIES,+ });+ }+ }+}diff --git a/packages/twenty-server/src/engine/metadata-modules/ai/ai-agent-execution/services/agent-toolset.service.ts b/packages/twenty-server/src/engine/metadata-modules/ai/ai-agent-execution/services/agent-toolset.service.tsnew file mode 100644index 00000000000..24d26b7caa3--- /dev/null+++ b/packages/twenty-server/src/engine/metadata-modules/ai/ai-agent-execution/services/agent-toolset.service.ts@@ -0,0 +1,290 @@+import { Injectable } from '@nestjs/common';++import { type ToolSet } from 'ai';+import {+ ASK_QUESTION_TOOL_NAME,+ PROPOSE_TOOL_CALL_TOOL_NAME,+ type ProposeToolCallToolInput,+ REQUEST_FORM_TOOL_NAME,+ type ToolCategory,+} from 'twenty-shared/ai';++import { ToolRegistryService } from 'src/engine/core-modules/tool-provider/services/tool-registry.service';+import {+ createExecuteToolTool,+ createLearnToolsTool,+ createLoadSkillTool,+ EXECUTE_TOOL_TOOL_NAME,+ LEARN_TOOLS_TOOL_NAME,+ LOAD_SKILL_TOOL_NAME,+} from 'src/engine/core-modules/tool-provider/tools';+import { type ToolContext } from 'src/engine/core-modules/tool-provider/types/tool-context.type';+import { type ToolIndexEntry } from 'src/engine/core-modules/tool-provider/types/tool-index-entry.type';+import { type ToolOutput } from 'src/engine/core-modules/tool/types/tool-output.type';+import { resolveProposedToolCall } from 'src/engine/metadata-modules/ai/ai-agent-execution/pausing-tools/utils/resolve-proposed-tool-call.util';+import { createProposeToolCallTool } from 'src/engine/metadata-modules/ai/ai-agent-execution/tools/propose-tool-call.tool';+import { createAskQuestionTool } from 'src/engine/metadata-modules/ai/ai-chat/tools/ask-question.tool';+import { createRequestFormTool } from 'src/engine/metadata-modules/ai/ai-chat/tools/request-form.tool';+import { AiModelConfigService } from 'src/engine/metadata-modules/ai/ai-models/services/ai-model-config.service';+import { type RegisteredAiModel } from 'src/engine/metadata-modules/ai/ai-models/services/ai-model-registry.service';+import { type NativeModelToolOptions } from 'src/engine/metadata-modules/ai/ai-models/types/native-model-tool-options.type';+import { SkillService } from 'src/engine/metadata-modules/skill/skill.service';++export type AgentRegistryAccess =+ // the workspace assistant: every tool its member may use, checked against the registry on each call+ | {+ type: 'assistant';+ toolContext: ToolContext;+ excludedToolNames: ReadonlySet<string>;+ preloadedToolNames: string[];+ }+ // calls stay within the catalog shown when the turn started, so a tool added since is out of reach+ | {+ type: 'catalog';+ toolContext: ToolContext;+ categories: readonly ToolCategory[];+ excludedToolNames: readonly string[];+ }+ // an agent limited to the objects it was granted, whose few tools are sent with the turn+ | {+ type: 'granted';+ toolContext: ToolContext;+ agentRoleId: string;+ categories: readonly ToolCategory[];+ excludedToolNames: readonly string[];+ }+ | { type: 'none' };++export type AgentToolsetInput = {+ registryAccess: AgentRegistryAccess;+ registeredModel: RegisteredAiModel;+ nativeToolOptions: NativeModelToolOptions;+ canAskHumans: boolean;+ isWorkspaceSetupThread?: boolean;+ waitTools?: ToolSet;+ extraTools?: ToolSet;+};++export type AgentToolset = {+ tools: ToolSet;+ toolCatalog: ToolIndexEntry[];+ directToolNames: string[];+ offeredPausingToolNames: string[];+};++type RegistryToolset = {+ directTools: ToolSet;+ metaTools: ToolSet;+ toolCatalog: ToolIndexEntry[];+ isToolAllowed: (toolName: string) => boolean;+};++const buildUnavailableToolOutput = (toolName: string): ToolOutput => ({+ success: false,+ message: `Tool "${toolName}" is not available`,+ error: `Tool "${toolName}" is not available to this agent.`,+});++@Injectable()+export class AgentToolsetService {+ constructor(+ private readonly toolRegistry: ToolRegistryService,+ private readonly aiModelConfigService: AiModelConfigService,+ private readonly skillService: SkillService,+ ) {}++ async build({+ registryAccess,+ registeredModel,+ nativeToolOptions,+ canAskHumans,+ isWorkspaceSetupThread = false,+ waitTools = {},+ extraTools = {},+ }: AgentToolsetInput): Promise<AgentToolset> {+ const { directTools, metaTools, toolCatalog, isToolAllowed } =+ await this.buildRegistryToolset(registryAccess);++ const resolveProposal = async (+ input: ProposeToolCallToolInput,+ toolContext: ToolContext,+ ) =>+ resolveProposedToolCall({+ input,+ findTool: async (toolName) => {+ if (!isToolAllowed(toolName)) {+ return undefined;+ }++ return registryAccess.type === 'granted'+ ? this.toolRegistry.findCatalogEntry(toolName, toolContext)+ : toolCatalog.find((entry) => entry.name === toolName);+ },+ executeTool: async ({ toolName, args }) =>+ isToolAllowed(toolName)+ ? this.toolRegistry.resolveAndExecute(toolName, args, toolContext)+ : buildUnavailableToolOutput(toolName),+ });++ const pausingTools: ToolSet = {+ ...waitTools,+ ...(canAskHumans+ ? {+ [ASK_QUESTION_TOOL_NAME]: createAskQuestionTool({+ isWorkspaceSetupThread,+ }),+ [REQUEST_FORM_TOOL_NAME]: createRequestFormTool(),+ }+ : {}),+ ...(canAskHumans && registryAccess.type !== 'none'+ ? {+ [PROPOSE_TOOL_CALL_TOOL_NAME]: createProposeToolCallTool({+ resolveProposal: (input) =>+ resolveProposal(input, registryAccess.toolContext),+ }),+ }+ : {}),+ };++ const sentTools: ToolSet = {+ ...directTools,+ ...this.aiModelConfigService.getNativeModelTools(+ registeredModel,+ nativeToolOptions,+ ),+ ...pausingTools,+ ...extraTools,+ };++ return {+ tools: { ...sentTools, ...metaTools },+ toolCatalog,+ directToolNames: Object.keys(sentTools),+ offeredPausingToolNames: Object.keys(pausingTools),+ };+ }++ private async buildRegistryToolset(+ registryAccess: AgentRegistryAccess,+ ): Promise<RegistryToolset> {+ if (registryAccess.type === 'none') {+ return {+ directTools: {},+ metaTools: {},+ toolCatalog: [],+ isToolAllowed: () => false,+ };+ }++ const { toolContext } = registryAccess;+ const { workspaceId, userId, userWorkspaceId, locale } = toolContext;++ switch (registryAccess.type) {+ case 'assistant': {+ const { excludedToolNames, preloadedToolNames } = registryAccess;+ const toolCatalog = await this.toolRegistry.buildToolIndex(+ workspaceId,+ {+ userId,+ userWorkspaceId,+ locale,+ excludeTools: new Set(excludedToolNames),+ rolePermissionConfig: toolContext.rolePermissionConfig,+ application: toolContext.application,+ },+ );+ const isToolAllowed = (toolName: string) =>+ !excludedToolNames.has(toolName);++ return {+ directTools: await this.toolRegistry.getToolsByName(+ preloadedToolNames,+ toolContext,+ { compactOutput: true, spillLargeOutput: true },+ ),+ metaTools: {+ ...this.buildMetaTools(toolContext, isToolAllowed),+ [LOAD_SKILL_TOOL_NAME]: createLoadSkillTool(+ (skillNames) =>+ this.skillService.findFlatSkillsByNames(+ skillNames,+ workspaceId,+ ),+ async () =>+ (await this.skillService.findAllFlatSkills(workspaceId)).map(+ (skill) => skill.name,+ ),+ ),+ },+ toolCatalog,+ isToolAllowed,+ };+ }+ case 'catalog': {+ const toolCatalog = await this.toolRegistry.buildToolIndex(+ workspaceId,+ {+ userId,+ userWorkspaceId,+ rolePermissionConfig: toolContext.rolePermissionConfig,+ application: toolContext.application,+ categories: [...registryAccess.categories],+ excludeTools: new Set(registryAccess.excludedToolNames),+ },+ );+ const catalogToolNames = new Set(+ toolCatalog.map((entry) => entry.name),+ );+ const isToolAllowed = (toolName: string) =>+ catalogToolNames.has(toolName);++ return {+ directTools: {},+ metaTools: this.buildMetaTools(toolContext, isToolAllowed),+ toolCatalog,+ isToolAllowed,+ };+ }+ case 'granted': {+ const directTools = await this.toolRegistry.getToolsByCategories(+ {+ ...toolContext,+ explicitObjectGrantsRoleId: registryAccess.agentRoleId,+ },+ {+ categories: [...registryAccess.categories],+ excludeTools: [...registryAccess.excludedToolNames],+ },+ );+ // the context lacks the explicit grants the tools were built with, so calls stay within them+ const isToolAllowed = (toolName: string) =>+ Object.prototype.hasOwnProperty.call(directTools, toolName);++ return {+ directTools,+ metaTools: {},+ toolCatalog: [],+ isToolAllowed,+ };+ }+ }+ }++ private buildMetaTools(+ toolContext: ToolContext,+ isToolAllowed: (toolName: string) => boolean,+ ): ToolSet {+ return {+ [LEARN_TOOLS_TOOL_NAME]: createLearnToolsTool(+ this.toolRegistry,+ toolContext,+ { isToolAllowed, spillLargeOutput: true },+ ),+ [EXECUTE_TOOL_TOOL_NAME]: createExecuteToolTool(+ this.toolRegistry,+ toolContext,+ { isToolAllowed, compactOutput: true, spillLargeOutput: true },+ ),+ };+ }+}diff --git a/packages/twenty-server/src/engine/metadata-modules/ai/ai-agent-execution/services/agent-turn.service.ts b/packages/twenty-server/src/engine/metadata-modules/ai/ai-agent-execution/services/agent-turn.service.tsnew file mode 100644index 00000000000..461213f39eb--- /dev/null+++ b/packages/twenty-server/src/engine/metadata-modules/ai/ai-agent-execution/services/agent-turn.service.ts@@ -0,0 +1,134 @@+import { Injectable, Logger } from '@nestjs/common';++import { type WorkspaceEntity } from 'src/engine/core-modules/workspace/workspace.entity';+import {+ type AgentStepLoop,+ type AgentStepLoopInput,+ AgentStepLoopService,+} from 'src/engine/metadata-modules/ai/ai-agent-execution/services/agent-step-loop.service';+import {+ type AgentToolset,+ type AgentToolsetInput,+ AgentToolsetService,+} from 'src/engine/metadata-modules/ai/ai-agent-execution/services/agent-toolset.service';+import { getCallLevelProviderOptions } from 'src/engine/metadata-modules/ai/ai-agent-execution/utils/provider-options.util';+import { AiModelConfigService } from 'src/engine/metadata-modules/ai/ai-models/services/ai-model-config.service';+import {+ AiModelRegistryService,+ type RegisteredAiModel,+} from 'src/engine/metadata-modules/ai/ai-models/services/ai-model-registry.service';+import { type AiModelConfig } from 'src/engine/metadata-modules/ai/ai-models/types/ai-model-config.type';+import { type NativeModelToolOptions } from 'src/engine/metadata-modules/ai/ai-models/types/native-model-tool-options.type';+import { getNativeModelCapabilities } from 'src/engine/metadata-modules/ai/ai-models/utils/get-native-model-capabilities.util';++type AgentTurnContext = Pick<+ AgentStepLoopInput,+ 'systemPrompt' | 'contextMessages' | 'messages'+>;++type AgentTurnInput = Omit<+ AgentStepLoopInput,+ | 'registeredModel'+ | 'modelConfig'+ | 'systemPrompt'+ | 'contextMessages'+ | 'messages'+ | 'tools'+ | 'offeredPausingToolNames'+ | 'providerOptions'+> & {+ workspace?: WorkspaceEntity;+ modelId: string;+ // the model's own native tools when nativeToolOptions is absent+ toolset: Omit<AgentToolsetInput, 'registeredModel' | 'nativeToolOptions'> & {+ nativeToolOptions?: NativeModelToolOptions;+ };+ buildContext: (turn: {+ registeredModel: RegisteredAiModel;+ modelConfig: AiModelConfig;+ toolset: AgentToolset;+ }) => Promise<AgentTurnContext>;+ promptCacheKey: string;+ shouldIncludeReasoningSummary?: boolean;+};++export type AgentTurn = AgentStepLoop & {+ registeredModel: RegisteredAiModel;+ modelConfig: AiModelConfig;+ toolset: AgentToolset;+};++@Injectable()+export class AgentTurnService {+ private readonly logger = new Logger(AgentTurnService.name);++ constructor(+ private readonly aiModelRegistryService: AiModelRegistryService,+ private readonly aiModelConfigService: AiModelConfigService,+ private readonly agentToolsetService: AgentToolsetService,+ private readonly agentStepLoopService: AgentStepLoopService,+ ) {}++ async start({+ workspace,+ modelId,+ toolset: toolsetInput,+ buildContext,+ promptCacheKey,+ shouldIncludeReasoningSummary = false,+ ...loopInput+ }: AgentTurnInput): Promise<AgentTurn> {+ this.aiModelRegistryService.validateModelAvailability(modelId);++ const registeredModel = this.aiModelRegistryService.resolveModelForAgent(+ { modelId },+ workspace,+ );+ const modelConfig = this.aiModelRegistryService.getEffectiveModelConfig(+ registeredModel.modelId,+ );+ const nativeCapabilities = getNativeModelCapabilities(+ registeredModel.sdkPackage,+ );++ const toolset = await this.agentToolsetService.build({+ ...toolsetInput,+ registeredModel,+ nativeToolOptions: toolsetInput.nativeToolOptions ?? {+ webSearch: nativeCapabilities?.webSearch === true,+ twitterSearch: nativeCapabilities?.twitterSearch === true,+ },+ });++ const { systemPrompt, contextMessages, messages } = await buildContext({+ registeredModel,+ modelConfig,+ toolset,+ });++ this.logger.log(+ `Starting ${loopInput.metricsScope} turn with model ${registeredModel.modelId}, ${Object.keys(toolset.tools).length} tools`,+ );++ const stepLoop = this.agentStepLoopService.start({+ ...loopInput,+ registeredModel,+ modelConfig,+ systemPrompt,+ contextMessages,+ messages,+ tools: toolset.tools,+ offeredPausingToolNames: toolset.offeredPausingToolNames,+ providerOptions: getCallLevelProviderOptions({+ sdkPackage: registeredModel.sdkPackage,+ providerOptions: this.aiModelConfigService.getReasoningProviderOptions(+ registeredModel,+ { shouldIncludeReasoningSummary },+ ),+ promptCacheKey,+ }),+ });++ return { ...stepLoop, registeredModel, modelConfig, toolset };+ }+}diff --git a/packages/twenty-server/src/engine/metadata-modules/ai/ai-agent-execution/types/agent-execution-result.type.ts b/packages/twenty-server/src/engine/metadata-modules/ai/ai-agent-execution/types/agent-execution-result.type.tsindex 42b6bae80a5..0f2dbada590 100644--- a/packages/twenty-server/src/engine/metadata-modules/ai/ai-agent-execution/types/agent-execution-result.type.ts+++ b/packages/twenty-server/src/engine/metadata-modules/ai/ai-agent-execution/types/agent-execution-result.type.ts@@ -1,17 +1,16 @@import { type LanguageModelUsage, type StepResult, type ToolSet } from 'ai';-import { type AgentTurnUsage } from 'src/engine/metadata-modules/ai/ai-billing/types/agent-turn-usage.type';+import { type StepDebitResult } from 'src/engine/metadata-modules/ai/ai-billing/types/step-debit-result.type';export type AgentExecutionResult = {result: object;usage: LanguageModelUsage;cacheCreationTokens: number;nativeWebSearchCallCount: number;- hasNoMoreAvailableCredits: boolean;+ stopReason: StepDebitResult['stopReason'];isPaused: boolean;steps: StepResult<ToolSet>[];modelId: string;totalCostInDollars: number;creditsUsedMicro: number;- turnUsage: AgentTurnUsage;};diff --git a/packages/twenty-server/src/engine/metadata-modules/ai/ai-agent-execution/types/agent-run-caller-handler.type.ts b/packages/twenty-server/src/engine/metadata-modules/ai/ai-agent-execution/types/agent-run-caller-handler.type.tsindex 5e4572995eb..d49f6233a58 100644--- a/packages/twenty-server/src/engine/metadata-modules/ai/ai-agent-execution/types/agent-run-caller-handler.type.ts+++ b/packages/twenty-server/src/engine/metadata-modules/ai/ai-agent-execution/types/agent-run-caller-handler.type.ts@@ -12,6 +12,14 @@ export type AgentRunCallerHandler = {getWaitingState(input: AgentRunCallerInput): Promise<OwnerWaitingState>;+ // a run that paused goes on later; its caller can show where it waits meanwhile+ onSuspended?(+ input: AgentRunCallerInput & {+ threadId: string;+ summary: AgentRunSummary;+ },+ ): Promise<void>;+// absent for a caller that only records the run, whose outcome is already in its conversationonOutcome?(input: AgentRunCallerInput & {@@ -21,4 +29,12 @@ export type AgentRunCallerHandler = {summary: AgentRunSummary | null;},): Promise<void>;++ // the caller can no longer take the outcome, but something may still wait on the run, such as a request+ onCallerGone?(+ input: AgentRunCallerInput & {+ threadId: string;+ outcome: AgentRunCallerOutcome;+ },+ ): Promise<void>;};diff --git a/packages/twenty-server/src/engine/metadata-modules/ai/ai-agent-execution/types/agent-run-outcome-delivery.type.ts b/packages/twenty-server/src/engine/metadata-modules/ai/ai-agent-execution/types/agent-run-outcome-delivery.type.tsnew file mode 100644index 00000000000..07d785dc70b--- /dev/null+++ b/packages/twenty-server/src/engine/metadata-modules/ai/ai-agent-execution/types/agent-run-outcome-delivery.type.ts@@ -0,0 +1,13 @@+import { type AgentRunSummary } from 'twenty-shared/ai';++import { type AgentRunCaller } from 'src/engine/metadata-modules/ai/ai-agent-execution/types/agent-run-caller.type';+import { type AgentRunCallerOutcome } from 'src/engine/metadata-modules/ai/ai-agent-execution/types/agent-run-caller-outcome.type';++export type AgentRunOutcomeDelivery = {+ workspaceId: string;+ caller: AgentRunCaller;+ threadId: string;+ outcome: AgentRunCallerOutcome;+ summary: AgentRunSummary | null;+ attempt?: number;+};diff --git a/packages/twenty-server/src/engine/metadata-modules/ai/ai-agent-execution/types/agent-run-wake-up-outcome.type.ts b/packages/twenty-server/src/engine/metadata-modules/ai/ai-agent-execution/types/agent-run-wake-up-outcome.type.tsnew file mode 100644index 00000000000..7596ea3e045--- /dev/null+++ b/packages/twenty-server/src/engine/metadata-modules/ai/ai-agent-execution/types/agent-run-wake-up-outcome.type.ts@@ -0,0 +1,5 @@+import { type PendingWakeUpOutcome } from 'src/engine/core-modules/pending-wake-up/types/pending-wake-up-outcome.type';++export type AgentRunWakeUpOutcome =+ | PendingWakeUpOutcome+ | { type: 'MESSAGE_RECEIVED' };diff --git a/packages/twenty-server/src/engine/metadata-modules/ai/ai-agent-execution/types/continue-agent-run-job-data.type.ts b/packages/twenty-server/src/engine/metadata-modules/ai/ai-agent-execution/types/continue-agent-run-job-data.type.tsindex 0936ec9a0ee..a4b32f3f455 100644--- a/packages/twenty-server/src/engine/metadata-modules/ai/ai-agent-execution/types/continue-agent-run-job-data.type.ts+++ b/packages/twenty-server/src/engine/metadata-modules/ai/ai-agent-execution/types/continue-agent-run-job-data.type.ts@@ -1,9 +1,10 @@-import { type PendingWakeUpOutcome } from 'src/engine/core-modules/pending-wake-up/types/pending-wake-up-outcome.type';+import { type AgentRunWakeUpOutcome } from 'src/engine/metadata-modules/ai/ai-agent-execution/types/agent-run-wake-up-outcome.type';export type ContinueAgentRunJobData = {workspaceId: string;threadId: string;// the run's wake-up, which the continuation removes once the run went on, and what resolved itwakeUpId: string;- outcome: PendingWakeUpOutcome;+ outcome: AgentRunWakeUpOutcome;+ busyAttempt?: number;};diff --git a/packages/twenty-server/src/engine/metadata-modules/ai/ai-agent-execution/types/run-agent-job-data.type.ts b/packages/twenty-server/src/engine/metadata-modules/ai/ai-agent-execution/types/run-agent-job-data.type.tsnew file mode 100644index 00000000000..caccc5cd1a6--- /dev/null+++ b/packages/twenty-server/src/engine/metadata-modules/ai/ai-agent-execution/types/run-agent-job-data.type.ts@@ -0,0 +1,9 @@+import { type AgentRunnerRunInput } from 'src/engine/metadata-modules/ai/ai-agent-execution/types/agent-runner-run-input.type';++// the agent and how the run acts are read again in the job from the spec and the caller+export type RunAgentJobData = Omit<+ AgentRunnerRunInput,+ 'agent' | 'executionContext'+> & {+ busyAttempt?: number;+};diff --git a/packages/twenty-server/src/engine/metadata-modules/ai/ai-agent-execution/types/settled-agent-run.type.ts b/packages/twenty-server/src/engine/metadata-modules/ai/ai-agent-execution/types/settled-agent-run.type.tsnew file mode 100644index 00000000000..fba6de8e007--- /dev/null+++ b/packages/twenty-server/src/engine/metadata-modules/ai/ai-agent-execution/types/settled-agent-run.type.ts@@ -0,0 +1,6 @@+import { type RunAgentResult } from 'twenty-shared/application';++export type SettledAgentRun = Pick<+ RunAgentResult,+ 'status' | 'result' | 'error'+>;diff --git a/packages/twenty-server/src/engine/metadata-modules/ai/ai-agent-execution/utils/__tests__/build-agent-run-summary.util.spec.ts b/packages/twenty-server/src/engine/metadata-modules/ai/ai-agent-execution/utils/__tests__/build-agent-run-summary.util.spec.tsindex c1462071725..e2df530968b 100644--- a/packages/twenty-server/src/engine/metadata-modules/ai/ai-agent-execution/utils/__tests__/build-agent-run-summary.util.spec.ts+++ b/packages/twenty-server/src/engine/metadata-modules/ai/ai-agent-execution/utils/__tests__/build-agent-run-summary.util.spec.ts@@ -25,19 +25,11 @@ const baseExecution: AgentExecutionResult = {} as AgentExecutionResult['usage'],cacheCreationTokens: 5,nativeWebSearchCallCount: 2,- hasNoMoreAvailableCredits: false,+ stopReason: null,isPaused: false,modelId: 'claude-sonnet-4',totalCostInDollars: 0.012,creditsUsedMicro: 12_000,- turnUsage: {- inputTokens: 100,- outputTokens: 50,- cacheReadTokens: 20,- cacheCreationTokens: 5,- inputCredits: 0,- outputCredits: 0,- },steps: [],};diff --git a/packages/twenty-server/src/engine/metadata-modules/ai/ai-agent-execution/utils/build-started-agent-run-summary.util.ts b/packages/twenty-server/src/engine/metadata-modules/ai/ai-agent-execution/utils/build-started-agent-run-summary.util.tsnew file mode 100644index 00000000000..e81010e3f75--- /dev/null+++ b/packages/twenty-server/src/engine/metadata-modules/ai/ai-agent-execution/utils/build-started-agent-run-summary.util.ts@@ -0,0 +1,12 @@+import { type AgentRunSummary } from 'twenty-shared/ai';++export const buildStartedAgentRunSummary = (+ modelId: string,+): AgentRunSummary => ({+ modelId,+ usage: { inputTokens: 0, outputTokens: 0, totalTokens: 0 },+ cost: { totalCostInDollars: 0, creditsUsedMicro: 0 },+ nativeWebSearchCallCount: 0,+ toolCalls: [],+ durationMs: 0,+});diff --git a/packages/twenty-server/src/engine/metadata-modules/ai/ai-agent-execution/utils/resolve-agent-run-calling-application.util.ts b/packages/twenty-server/src/engine/metadata-modules/ai/ai-agent-execution/utils/resolve-agent-run-calling-application.util.tsnew file mode 100644index 00000000000..7ad6ae912b6--- /dev/null+++ b/packages/twenty-server/src/engine/metadata-modules/ai/ai-agent-execution/utils/resolve-agent-run-calling-application.util.ts@@ -0,0 +1,41 @@+import { TWENTY_STANDARD_APPLICATION_UNIVERSAL_IDENTIFIER } from 'twenty-shared/application';+import { isDefined } from 'twenty-shared/utils';++import { type FlatApplication } from 'src/engine/core-modules/application/types/flat-application.type';+import { isApplicationAuthContext } from 'src/engine/core-modules/auth/guards/is-application-auth-context.guard';+import { isUserAuthContext } from 'src/engine/core-modules/auth/guards/is-user-auth-context.guard';+import { type WorkspaceAuthContext } from 'src/engine/core-modules/auth/types/workspace-auth-context.type';++const getAuthContextApplication = (+ authContext: WorkspaceAuthContext,+): FlatApplication | undefined => {+ if (isApplicationAuthContext(authContext)) {+ return authContext.application;+ }++ if (isUserAuthContext(authContext)) {+ return authContext.application ?? authContext.viaApplication;+ }++ return undefined;+};++export const resolveAgentRunCallingApplication = (+ authContext?: WorkspaceAuthContext,+): FlatApplication | undefined => {+ if (!isDefined(authContext)) {+ return undefined;+ }++ const application = getAuthContextApplication(authContext);++ if (+ application?.id === authContext.workspace.workspaceCustomApplicationId ||+ application?.universalIdentifier ===+ TWENTY_STANDARD_APPLICATION_UNIVERSAL_IDENTIFIER+ ) {+ return undefined;+ }++ return application;+};diff --git a/packages/twenty-server/src/engine/metadata-modules/ai/ai-agent-trigger/services/__tests__/agent-trigger-runner.service.spec.ts b/packages/twenty-server/src/engine/metadata-modules/ai/ai-agent-trigger/services/__tests__/agent-trigger-runner.service.spec.tsdeleted file mode 100644index 622c68df96d..00000000000--- a/packages/twenty-server/src/engine/metadata-modules/ai/ai-agent-trigger/services/__tests__/agent-trigger-runner.service.spec.ts+++ /dev/null@@ -1,211 +0,0 @@-import { Test, type TestingModule } from '@nestjs/testing';--import { AgentActorContextService } from 'src/engine/metadata-modules/ai/ai-agent-execution/services/agent-actor-context.service';-import { AgentRunCallerHandlerRegistryService } from 'src/engine/metadata-modules/ai/ai-agent-execution/services/agent-run-caller-handler-registry.service';-import { AgentRunnerService } from 'src/engine/metadata-modules/ai/ai-agent-execution/services/agent-runner.service';-import { AgentTriggerRunnerService } from 'src/engine/metadata-modules/ai/ai-agent-trigger/services/agent-trigger-runner.service';-import { type RunAgentTriggerJobData } from 'src/engine/metadata-modules/ai/ai-agent-trigger/types/run-agent-trigger-job-data.type';-import { AgentEntity } from 'src/engine/metadata-modules/ai/ai-agent/entities/agent.entity';-import { getWorkspaceScopedRepositoryToken } from 'src/engine/twenty-orm/workspace-scoped-repository/get-workspace-scoped-repository-token.util';--const WORKSPACE_ID = 'workspace-id';-const AGENT_ID = 'agent-id';-const AGENT_ROLE_ID = 'agent-role-id';-const TRIGGER_ID = '6f1b5a3e-3c3f-4f4a-9a43-0a7f5d6c2b11';--const JOB_DATA: RunAgentTriggerJobData = {- workspaceId: WORKSPACE_ID,- agentId: AGENT_ID,- triggerId: TRIGGER_ID,- dispatchedRoleId: AGENT_ROLE_ID,- payload: {- type: 'DATABASE_EVENT',- eventName: 'company.created',- objectNameSingular: 'company',- events: [],- },-};--const buildAgent = (isTriggerActive = true) => ({- id: AGENT_ID,- label: 'Enricher',- applicationId: 'application-id',- triggers: [- {- id: TRIGGER_ID,- type: 'DATABASE_EVENT',- isActive: isTriggerActive,- instructions: 'Qualify the company',- settings: { eventName: 'company.created' },- },- ],-});--describe('AgentTriggerRunnerService', () => {- let service: AgentTriggerRunnerService;- let runAgent: jest.Mock;- let currentRoleId: string | undefined;- let findAgent: jest.Mock;- let findApplication: jest.Mock;-- beforeEach(async () => {- currentRoleId = AGENT_ROLE_ID;- findAgent = jest.fn().mockResolvedValue(buildAgent());- findApplication = jest- .fn()- .mockResolvedValue({ id: 'application-id', name: 'App' });- runAgent = jest.fn().mockResolvedValue(undefined);-- const module: TestingModule = await Test.createTestingModule({- providers: [- AgentTriggerRunnerService,- {- provide: AgentRunnerService,- useValue: { run: runAgent },- },- {- provide: AgentRunCallerHandlerRegistryService,- useValue: { register: jest.fn() },- },- {- provide: AgentActorContextService,- useValue: {- buildApplicationAgentContext: jest.fn(async () => {- const application = await findApplication();-- return application- ? {- application,- authContext: { type: 'application', application },- agentRoleId: currentRoleId,- }- : null;- }),- },- },- {- provide: getWorkspaceScopedRepositoryToken(AgentEntity),- useValue: { findOne: findAgent },- },- ],- }).compile();-- service = module.get(AgentTriggerRunnerService);- });-- it('should run the agent as itself in a new conversation, able to wait', async () => {- await service.run(JOB_DATA);-- expect(runAgent).toHaveBeenCalledWith(- expect.objectContaining({- conversation: { threadId: expect.any(String), isCreated: true },- caller: {- type: 'AGENT_TRIGGER',- ref: {- agentId: AGENT_ID,- triggerId: TRIGGER_ID,- dispatchedRoleId: AGENT_ROLE_ID,- },- },- spec: expect.objectContaining({- title: 'Enricher',- toolLoadingStrategy: 'lazy',- capabilities: {- canAskHumans: false,- },- }),- prompt: expect.objectContaining({- senderUserWorkspaceId: null,- senderApplicationId: 'application-id',- }),- executionContext: expect.objectContaining({- actorContext: expect.objectContaining({ name: 'Enricher' }),- authContext: expect.objectContaining({- type: 'application',- actingAgent: { id: AGENT_ID, label: 'Enricher' },- }),- rolePermissionConfig: {- intersectionOf: { agentRoleId: AGENT_ROLE_ID },- },- }),- }),- );- });-- it("should keep the agent's role alone when its application has no role", async () => {- findApplication.mockResolvedValue({- id: 'application-id',- name: 'App',- defaultRoleId: null,- });-- await service.run(JOB_DATA);-- expect(runAgent).toHaveBeenCalledTimes(1);- expect(runAgent).toHaveBeenCalledWith(- expect.objectContaining({- executionContext: expect.objectContaining({- rolePermissionConfig: {- intersectionOf: { agentRoleId: AGENT_ROLE_ID },- },- agentRunRoleIds: {},- }),- }),- );- });-- it("should restrict the agent's role to its application's default role", async () => {- findApplication.mockResolvedValue({- id: 'application-id',- name: 'App',- defaultRoleId: 'application-role-id',- });-- await service.run(JOB_DATA);-- expect(runAgent).toHaveBeenCalledTimes(1);- expect(runAgent).toHaveBeenCalledWith(- expect.objectContaining({- executionContext: expect.objectContaining({- rolePermissionConfig: {- intersectionOf: {- agentRoleId: AGENT_ROLE_ID,- applicationRoleId: 'application-role-id',- },- },- agentRunRoleIds: { applicationRoleId: 'application-role-id' },- }),- }),- );- });-- it.each([- ['the agent was deleted', () => findAgent.mockResolvedValue(null)],- [- 'the trigger was turned off',- () => findAgent.mockResolvedValue(buildAgent(false)),- ],- [- 'the trigger was removed',- () => findAgent.mockResolvedValue({ ...buildAgent(), triggers: [] }),- ],- [- 'the application was not found',- () => findApplication.mockResolvedValue(null),- ],- ])('should not run when %s', async (_, arrange) => {- arrange();-- await service.run(JOB_DATA);-- expect(runAgent).not.toHaveBeenCalled();- });-- it('should not run when the agent role changed since dispatch', async () => {- currentRoleId = 'other-role-id';-- await service.run(JOB_DATA);-- expect(runAgent).not.toHaveBeenCalled();- });-});diff --git a/packages/twenty-server/src/engine/metadata-modules/ai/ai-agent-trigger/services/agent-trigger-runner.service.ts b/packages/twenty-server/src/engine/metadata-modules/ai/ai-agent-trigger/services/agent-trigger-runner.service.tsindex 268a7f51608..a6222ed588c 100644--- a/packages/twenty-server/src/engine/metadata-modules/ai/ai-agent-trigger/services/agent-trigger-runner.service.ts+++ b/packages/twenty-server/src/engine/metadata-modules/ai/ai-agent-trigger/services/agent-trigger-runner.service.ts@@ -68,15 +68,15 @@ export class AgentTriggerRunnerServicepayload,...ref}: RunAgentTriggerJobData): Promise<void> {- const triggeredRun = await this.findTriggeredRun({ workspaceId, ref });+ const activeTrigger = await this.findActiveTrigger({ workspaceId, ref });- if (!isDefined(triggeredRun)) {+ if (!isDefined(activeTrigger)) {return;}- const { agent, trigger, executionContext } = triggeredRun;+ const { agent, trigger } = activeTrigger;- await this.agentRunnerService.run({+ await this.agentRunnerService.start({workspaceId,conversation: {threadId: buildAgentRunThreadId({@@ -98,7 +98,6 @@ export class AgentTriggerRunnerService},toolLoadingStrategy: 'lazy',},- agent,prompt: {messages: buildAgentTriggerMessages({instructions: trigger.instructions,@@ -107,7 +106,6 @@ export class AgentTriggerRunnerServicesenderUserWorkspaceId: null,senderApplicationId: agent.applicationId,},- executionContext,});}@@ -139,14 +137,13 @@ export class AgentTriggerRunnerService: 'GONE';}- // The trigger may have been turned off or removed, or the agent's role changed, since the run was queued- private async findTriggeredRun({+ private async findActiveTrigger({workspaceId,- ref: { agentId, triggerId, dispatchedRoleId },+ ref: { agentId, triggerId },}: {workspaceId: string;ref: AgentTriggerCallerRef;- }): Promise<TriggeredRun | null> {+ }): Promise<Pick<TriggeredRun, 'agent' | 'trigger'> | null> {const agent = await this.agentRepository.findOne(workspaceId, {where: { id: agentId },});@@ -155,10 +152,28 @@ export class AgentTriggerRunnerService(agentTrigger) => agentTrigger.id === triggerId,);- if (!isDefined(agent) || !isDefined(trigger) || !trigger.isActive) {+ return isDefined(agent) && isDefined(trigger) && trigger.isActive+ ? { agent, trigger }+ : null;+ }++ // The trigger may have been turned off or removed, or the agent's role changed, since the run was queued+ private async findTriggeredRun({+ workspaceId,+ ref,+ }: {+ workspaceId: string;+ ref: AgentTriggerCallerRef;+ }): Promise<TriggeredRun | null> {+ const { agentId, triggerId, dispatchedRoleId } = ref;+ const activeTrigger = await this.findActiveTrigger({ workspaceId, ref });++ if (!isDefined(activeTrigger)) {return null;}+ const { agent, trigger } = activeTrigger;+const agentContext =await this.agentActorContextService.buildApplicationAgentContext({workspaceId,diff --git a/packages/twenty-server/src/engine/metadata-modules/ai/ai-billing/services/ai-billing.service.ts b/packages/twenty-server/src/engine/metadata-modules/ai/ai-billing/services/ai-billing.service.tsindex 40cfeab7c66..afe8cb9bc8f 100644--- a/packages/twenty-server/src/engine/metadata-modules/ai/ai-billing/services/ai-billing.service.ts+++ b/packages/twenty-server/src/engine/metadata-modules/ai/ai-billing/services/ai-billing.service.ts@@ -10,16 +10,20 @@ import { UsageRecorderService } from 'src/engine/core-modules/usage/services/usaimport { type RecordUsageInput } from 'src/engine/core-modules/usage/types/record-usage-input.type';import { type UsageSpenders } from 'src/engine/core-modules/usage/types/usage-spenders.type';import { NATIVE_WEB_SEARCH_COST_PER_CALL_DOLLARS } from 'src/engine/metadata-modules/ai/ai-billing/constants/native-web-search-cost-per-call-dollars';+import { type BillableStep } from 'src/engine/metadata-modules/ai/ai-billing/types/billable-step.type';import { type BillingTokenUsage } from 'src/engine/metadata-modules/ai/ai-billing/types/billing-token-usage.type';import { type StepDebitResult } from 'src/engine/metadata-modules/ai/ai-billing/types/step-debit-result.type';-import { computeAgentTurnUsageFromSteps } from 'src/engine/metadata-modules/ai/ai-billing/utils/compute-agent-turn-usage-from-steps.util';import { computeStepCostBreakdown } from 'src/engine/metadata-modules/ai/ai-billing/utils/compute-step-cost-breakdown.util';import { convertDollarsToCreditsMicro } from 'src/engine/metadata-modules/ai/ai-billing/utils/convert-dollars-to-credits-micro.util';+import { countNativeWebSearchCallsFromSteps } from 'src/engine/metadata-modules/ai/ai-billing/utils/count-native-web-search-calls-from-steps.util';import { extractCacheCreationTokens } from 'src/engine/metadata-modules/ai/ai-billing/utils/extract-cache-creation-tokens.util';import { AiModelRegistryService } from 'src/engine/metadata-modules/ai/ai-models/services/ai-model-registry.service';import { type AiModelCostConfig } from 'src/engine/metadata-modules/ai/ai-models/types/ai-model-cost-config.type';import { type ModelId } from 'src/engine/metadata-modules/ai/ai-models/types/model-id.type';-import { type AgentTurnUsage } from 'src/engine/metadata-modules/ai/ai-billing/types/agent-turn-usage.type';++export type WebSearchOperationType =+ | UsageOperationType.WEB_SEARCH+ | UsageOperationType.AI_WORKFLOW_WEB_SEARCH;export type BillingUsageInput = {usage: BillingTokenUsage;@@ -62,13 +66,7 @@ export class AiBillingService {);}- calculateStepsCost(- modelId: ModelId,- steps: {- usage: BillingTokenUsage;- providerMetadata?: Record<string, Record<string, unknown> | undefined>;- }[],- ): number {+ calculateStepsCost(modelId: ModelId, steps: BillableStep[]): number {const model = this.getCostConfig(modelId);return steps.reduce(@@ -84,16 +82,6 @@ export class AiBillingService {);}- calculateStepsTurnUsage(- modelId: ModelId,- steps: {- usage: BillingTokenUsage;- providerMetadata?: Record<string, Record<string, unknown> | undefined>;- }[],- ): AgentTurnUsage {- return computeAgentTurnUsageFromSteps(this.getCostConfig(modelId), steps);- }-calculateCost(modelId: ModelId, billingInput: BillingUsageInput): number {const model = this.getCostConfig(modelId);const breakdown = computeStepCostBreakdown(model, billingInput);@@ -178,7 +166,7 @@ export class AiBillingService {);}- async billNativeWebSearchUsage({+ private async billNativeWebSearchUsage({nativeWebSearchCallCount,workspaceId,operationType,@@ -186,9 +174,7 @@ export class AiBillingService {}: {nativeWebSearchCallCount: number;workspaceId: string;- operationType:- | UsageOperationType.WEB_SEARCH- | UsageOperationType.AI_WORKFLOW_WEB_SEARCH;+ operationType: WebSearchOperationType;spenders: UsageSpenders;}): Promise<StepDebitResult | null> {if (nativeWebSearchCallCount <= 0) {@@ -226,35 +212,59 @@ export class AiBillingService {return this.toStepDebitResult(webSearchDebit);}- async emitAiTokenUsageEvent({- workspaceId,- creditsUsedMicro,- totalTokens,+ // returns the web search debit, so a caller can tell when it crossed the user's allowance+ async recordStepsUsage({modelId,+ steps,+ workspaceId,operationType,+ webSearchOperationType,spenders,- userCreditsUsedMicro = 0,+ userCreditsUsedMicro,}: {- workspaceId: string;- creditsUsedMicro: number;- totalTokens: number;modelId: ModelId;+ steps: (BillableStep & { toolCalls: { toolName: string }[] })[];+ workspaceId: string;operationType: UsageOperationType;+ webSearchOperationType: WebSearchOperationType;spenders: UsageSpenders;- userCreditsUsedMicro?: number;- }): Promise<void> {- await this.usageRecorderService.record(workspaceId, [- {- ...this.buildAiTokenUsageEvent({- creditsUsedMicro,- totalTokens,- modelId,- operationType,- spenders,- }),- userCreditsUsedMicro,- },+ userCreditsUsedMicro: number;+ }): Promise<StepDebitResult | null> {+ const totalTokens = steps.reduce(+ (total, { usage }) =>+ total + (usage.inputTokens ?? 0) + (usage.outputTokens ?? 0),+ 0,+ );+ const creditsUsedMicro = convertDollarsToCreditsMicro(+ this.calculateStepsCost(modelId, steps),+ );++ // a failed token record must not also skip charging for searches already made+ const [, webSearchDebit] = await Promise.all([+ this.usageRecorderService.record(workspaceId, [+ {+ ...this.buildAiTokenUsageEvent({+ creditsUsedMicro,+ totalTokens,+ modelId,+ operationType,+ spenders,+ }),+ userCreditsUsedMicro: Math.min(+ userCreditsUsedMicro,+ creditsUsedMicro,+ ),+ },+ ]),+ this.billNativeWebSearchUsage({+ nativeWebSearchCallCount: countNativeWebSearchCallsFromSteps(steps),+ workspaceId,+ operationType: webSearchOperationType,+ spenders,+ }),]);++ return webSearchDebit;}private toStepDebitResult({diff --git a/packages/twenty-server/src/engine/metadata-modules/ai/ai-billing/types/billable-step.type.ts b/packages/twenty-server/src/engine/metadata-modules/ai/ai-billing/types/billable-step.type.tsnew file mode 100644index 00000000000..0b8013019c0--- /dev/null+++ b/packages/twenty-server/src/engine/metadata-modules/ai/ai-billing/types/billable-step.type.ts@@ -0,0 +1,6 @@+import { type BillingTokenUsage } from 'src/engine/metadata-modules/ai/ai-billing/types/billing-token-usage.type';++export type BillableStep = {+ usage: BillingTokenUsage;+ providerMetadata?: Record<string, Record<string, unknown> | undefined>;+};diff --git a/packages/twenty-server/src/engine/metadata-modules/ai/ai-billing/utils/__tests__/compute-agent-turn-usage-from-steps.util.spec.ts b/packages/twenty-server/src/engine/metadata-modules/ai/ai-billing/utils/__tests__/add-step-to-agent-turn-usage.util.spec.tssimilarity index 62%rename from packages/twenty-server/src/engine/metadata-modules/ai/ai-billing/utils/__tests__/compute-agent-turn-usage-from-steps.util.spec.tsrename to packages/twenty-server/src/engine/metadata-modules/ai/ai-billing/utils/__tests__/add-step-to-agent-turn-usage.util.spec.tsindex a7bf48ea6ff..ce89a840ab5 100644--- a/packages/twenty-server/src/engine/metadata-modules/ai/ai-billing/utils/__tests__/compute-agent-turn-usage-from-steps.util.spec.ts+++ b/packages/twenty-server/src/engine/metadata-modules/ai/ai-billing/utils/__tests__/add-step-to-agent-turn-usage.util.spec.ts@@ -1,4 +1,5 @@-import { computeAgentTurnUsageFromSteps } from 'src/engine/metadata-modules/ai/ai-billing/utils/compute-agent-turn-usage-from-steps.util';+import { addStepToAgentTurnUsage } from 'src/engine/metadata-modules/ai/ai-billing/utils/add-step-to-agent-turn-usage.util';+import { type AgentTurnUsage } from 'src/engine/metadata-modules/ai/ai-billing/types/agent-turn-usage.type';import { type AiModelCostConfig } from 'src/engine/metadata-modules/ai/ai-models/types/ai-model-cost-config.type';const MODEL_COST_CONFIG: AiModelCostConfig = {@@ -13,24 +14,27 @@ const MODEL_COST_CONFIG: AiModelCostConfig = {},};-describe('computeAgentTurnUsageFromSteps', () => {- it('is empty for an execution without steps', () => {- expect(computeAgentTurnUsageFromSteps(MODEL_COST_CONFIG, [])).toEqual({- inputTokens: 0,- outputTokens: 0,- cacheReadTokens: 0,- cacheCreationTokens: 0,- inputCredits: 0,- outputCredits: 0,- });- });+const EMPTY_TURN_USAGE: AgentTurnUsage = {+ inputTokens: 0,+ outputTokens: 0,+ cacheReadTokens: 0,+ cacheCreationTokens: 0,+ inputCredits: 0,+ outputCredits: 0,+};++const addSteps = (steps: Parameters<typeof addStepToAgentTurnUsage>[2][]) =>+ steps.reduce(+ (turnUsage, step) =>+ addStepToAgentTurnUsage(MODEL_COST_CONFIG, turnUsage, step),+ EMPTY_TURN_USAGE,+ );+describe('addStepToAgentTurnUsage', () => {it('prices each step on its own, so two steps under the long-context threshold keep the base rate', () => {const step = { usage: { inputTokens: 150_000, outputTokens: 1_000 } };- expect(- computeAgentTurnUsageFromSteps(MODEL_COST_CONFIG, [step, step]),- ).toEqual({+ expect(addSteps([step, step])).toEqual({inputTokens: 300_000,outputTokens: 2_000,cacheReadTokens: 0,@@ -41,7 +45,7 @@ describe('computeAgentTurnUsageFromSteps', () => {});it('counts cache reads and cache writes', () => {- const turnUsage = computeAgentTurnUsageFromSteps(MODEL_COST_CONFIG, [+ const turnUsage = addSteps([{usage: {inputTokens: 1_000,diff --git a/packages/twenty-server/src/engine/metadata-modules/ai/ai-billing/utils/compute-agent-turn-usage-from-steps.util.ts b/packages/twenty-server/src/engine/metadata-modules/ai/ai-billing/utils/add-step-to-agent-turn-usage.util.tssimilarity index 74%rename from packages/twenty-server/src/engine/metadata-modules/ai/ai-billing/utils/compute-agent-turn-usage-from-steps.util.tsrename to packages/twenty-server/src/engine/metadata-modules/ai/ai-billing/utils/add-step-to-agent-turn-usage.util.tsindex 9bb16b71c49..27214978d4f 100644--- a/packages/twenty-server/src/engine/metadata-modules/ai/ai-billing/utils/compute-agent-turn-usage-from-steps.util.ts+++ b/packages/twenty-server/src/engine/metadata-modules/ai/ai-billing/utils/add-step-to-agent-turn-usage.util.ts@@ -5,22 +5,12 @@ import { extractCacheCreationTokens } from 'src/engine/metadata-modules/ai/ai-biimport { type AgentTurnUsage } from 'src/engine/metadata-modules/ai/ai-billing/types/agent-turn-usage.type';import { type AiModelCostConfig } from 'src/engine/metadata-modules/ai/ai-models/types/ai-model-cost-config.type';-type AgentTurnUsageStep = {+export type AgentTurnUsageStep = {usage?: BillingTokenUsage;providerMetadata?: Record<string, Record<string, unknown> | undefined>;};-const EMPTY_AGENT_TURN_USAGE: AgentTurnUsage = {- inputTokens: 0,- outputTokens: 0,- cacheReadTokens: 0,- cacheCreationTokens: 0,- inputCredits: 0,- outputCredits: 0,-};--// each step is priced on its own, so a chat stream adding steps as they end-// and an execution adding them at the end reach the same totals+// each step is priced on its own, so steps under the long-context threshold never reach its rate togetherexport const addStepToAgentTurnUsage = (model: AiModelCostConfig,turnUsage: AgentTurnUsage,@@ -46,12 +36,3 @@ export const addStepToAgentTurnUsage = (convertDollarsToCreditsMicro(breakdown.outputCostInDollars),};};--export const computeAgentTurnUsageFromSteps = (- model: AiModelCostConfig,- steps: AgentTurnUsageStep[],-): AgentTurnUsage =>- steps.reduce<AgentTurnUsage>(- (turnUsage, step) => addStepToAgentTurnUsage(model, turnUsage, step),- EMPTY_AGENT_TURN_USAGE,- );diff --git a/packages/twenty-server/src/engine/metadata-modules/ai/ai-billing/utils/count-native-web-search-calls-from-steps.util.ts b/packages/twenty-server/src/engine/metadata-modules/ai/ai-billing/utils/count-native-web-search-calls-from-steps.util.tsindex c7829761fbd..03ff9f293c7 100644--- a/packages/twenty-server/src/engine/metadata-modules/ai/ai-billing/utils/count-native-web-search-calls-from-steps.util.ts+++ b/packages/twenty-server/src/engine/metadata-modules/ai/ai-billing/utils/count-native-web-search-calls-from-steps.util.ts@@ -1,9 +1,7 @@-import { type StepResult, type ToolSet } from 'ai';-const WEB_SEARCH_TOOL_NAME = 'web_search';export const countNativeWebSearchCallsFromSteps = (- steps: StepResult<ToolSet>[],+ steps: { toolCalls: { toolName: string }[] }[],): number =>steps.reduce((count, step) =>diff --git a/packages/twenty-server/src/engine/metadata-modules/ai/ai-chat/agent-chat-stream-state.module.ts b/packages/twenty-server/src/engine/metadata-modules/ai/ai-chat/agent-chat-stream-state.module.tsindex 7eebe78272b..af6b0762263 100644--- a/packages/twenty-server/src/engine/metadata-modules/ai/ai-chat/agent-chat-stream-state.module.ts+++ b/packages/twenty-server/src/engine/metadata-modules/ai/ai-chat/agent-chat-stream-state.module.ts@@ -1,22 +1,28 @@-import { AgentChatStreamRecoveryService } from 'src/engine/metadata-modules/ai/ai-chat/services/agent-chat-stream-recovery.service';-import { AgentHistoryModule } from 'src/engine/metadata-modules/ai/ai-history/ai-history.module';-import { MetricsModule } from 'src/engine/core-modules/metrics/metrics.module';import { Module } from '@nestjs/common';+import { MetricsModule } from 'src/engine/core-modules/metrics/metrics.module';+import { AgentChatThreadModule } from 'src/engine/metadata-modules/ai/ai-chat/agent-chat-thread.module';+import { AgentChatCancelSubscriberService } from 'src/engine/metadata-modules/ai/ai-chat/services/agent-chat-cancel-subscriber.service';import { AgentChatEventPublisherService } from 'src/engine/metadata-modules/ai/ai-chat/services/agent-chat-event-publisher.service';import { AgentChatStreamHeartbeatService } from 'src/engine/metadata-modules/ai/ai-chat/services/agent-chat-stream-heartbeat.service';+import { AgentChatStreamRecoveryService } from 'src/engine/metadata-modules/ai/ai-chat/services/agent-chat-stream-recovery.service';+import { AgentTurnStreamService } from 'src/engine/metadata-modules/ai/ai-chat/services/agent-turn-stream.service';+import { AgentHistoryModule } from 'src/engine/metadata-modules/ai/ai-history/ai-history.module';@Module({- imports: [AgentHistoryModule, MetricsModule],+ imports: [AgentChatThreadModule, AgentHistoryModule, MetricsModule],providers: [- AgentChatStreamRecoveryService,+ AgentChatCancelSubscriberService,AgentChatEventPublisherService,AgentChatStreamHeartbeatService,+ AgentChatStreamRecoveryService,+ AgentTurnStreamService,],exports: [- AgentChatStreamRecoveryService,AgentChatEventPublisherService,AgentChatStreamHeartbeatService,+ AgentChatStreamRecoveryService,+ AgentTurnStreamService,],})export class AgentChatStreamStateModule {}diff --git a/packages/twenty-server/src/engine/metadata-modules/ai/ai-chat/ai-chat.module.ts b/packages/twenty-server/src/engine/metadata-modules/ai/ai-chat/ai-chat.module.tsindex 32f9dce2e23..f64a9f2ab5a 100644--- a/packages/twenty-server/src/engine/metadata-modules/ai/ai-chat/ai-chat.module.ts+++ b/packages/twenty-server/src/engine/metadata-modules/ai/ai-chat/ai-chat.module.ts@@ -27,6 +27,7 @@ import { WorkspaceCacheModule } from 'src/engine/workspace-cache/workspace-cacheimport { DashboardToolsModule } from 'src/modules/dashboard/tools/dashboard-tools.module';import { AgentInboxProposalService } from 'src/engine/metadata-modules/ai/ai-chat/services/agent-inbox-proposal.service';+import { FlushQueuedAgentChatMessagesJob } from 'src/engine/metadata-modules/ai/ai-chat/jobs/flush-queued-agent-chat-messages.job';import { StreamAgentChatJob } from 'src/engine/metadata-modules/ai/ai-chat/jobs/stream-agent-chat.job';import { AgentChatResolver } from 'src/engine/metadata-modules/ai/ai-chat/resolvers/agent-chat.resolver';import { AgentChatThreadParticipantResolver } from 'src/engine/metadata-modules/ai/ai-chat/resolvers/agent-chat-thread-participant.resolver';@@ -36,7 +37,6 @@ import { AgentChatSubscriptionResolver } from 'src/engine/metadata-modules/ai/aiimport { AgentInboxResolver } from 'src/engine/metadata-modules/ai/ai-chat/resolvers/agent-inbox.resolver';import { WorkspaceSetupChatResolver } from 'src/engine/metadata-modules/ai/ai-chat/resolvers/workspace-setup-chat.resolver';import { WorkspaceSetupChatService } from 'src/engine/metadata-modules/ai/ai-chat/services/workspace-setup-chat.service';-import { AgentChatCancelSubscriberService } from 'src/engine/metadata-modules/ai/ai-chat/services/agent-chat-cancel-subscriber.service';import { AgentChatStreamingService } from 'src/engine/metadata-modules/ai/ai-chat/services/agent-chat-streaming.service';import { AgentChatService } from 'src/engine/metadata-modules/ai/ai-chat/services/agent-chat.service';import { AgentChatThreadTargetService } from 'src/engine/metadata-modules/ai/ai-chat/services/agent-chat-thread-target.service';@@ -72,7 +72,6 @@ import { SystemPromptBuilderService } from 'src/engine/metadata-modules/ai/ai-chAgentChatActorService,AiChatUsageService,AiChatUsageResolver,- AgentChatCancelSubscriberService,AgentChatResolver,AgentChatThreadParticipantResolver,AgentChatMessageReactionResolver,@@ -89,6 +88,7 @@ import { SystemPromptBuilderService } from 'src/engine/metadata-modules/ai/ai-chAgentTitleGenerationService,ChatExecutionService,MessagePruningService,+ FlushQueuedAgentChatMessagesJob,StreamAgentChatJob,SystemPromptBuilderService,provideWorkspaceScopedRepository(FileEntity),diff --git a/packages/twenty-server/src/engine/metadata-modules/ai/ai-chat/jobs/__tests__/stream-agent-chat.job.spec.ts b/packages/twenty-server/src/engine/metadata-modules/ai/ai-chat/jobs/__tests__/stream-agent-chat.job.spec.tsdeleted file mode 100644index c8e18c9e188..00000000000--- a/packages/twenty-server/src/engine/metadata-modules/ai/ai-chat/jobs/__tests__/stream-agent-chat.job.spec.ts+++ /dev/null@@ -1,1238 +0,0 @@-import { type LanguageModelUsage, type TextStreamPart, type ToolSet } from 'ai';-import { ASK_QUESTION_TOOL_NAME, NO_REPLY_TOOL_NAME } from 'twenty-shared/ai';-import { isDefined } from 'twenty-shared/utils';--import { type WorkspaceEntity } from 'src/engine/core-modules/workspace/workspace.entity';-import { StreamAgentChatJob } from 'src/engine/metadata-modules/ai/ai-chat/jobs/stream-agent-chat.job';-import { AgentChatStreamRecoveryService } from 'src/engine/metadata-modules/ai/ai-chat/services/agent-chat-stream-recovery.service';-import { type StreamAgentChatJobData } from 'src/engine/metadata-modules/ai/ai-chat/jobs/stream-agent-chat-job.types';-import { type AiModelConfig } from 'src/engine/metadata-modules/ai/ai-models/types/ai-model-config.type';-import { AiExceptionCode } from 'src/engine/metadata-modules/ai/ai.exception';-import { AgentTurnStatus } from 'src/engine/metadata-modules/ai/ai-history/enums/agent-turn-status.enum';--type PublishedEvent = { type: string } & Record<string, unknown>;--type ModelStreamPart = TextStreamPart<ToolSet>;--const USAGE: LanguageModelUsage = {- inputTokens: 12,- inputTokenDetails: {- noCacheTokens: 12,- cacheReadTokens: 0,- cacheWriteTokens: 0,- },- outputTokens: 3,- outputTokenDetails: { textTokens: 3, reasoningTokens: 0 },- totalTokens: 15,-};--const START_PARTS: ModelStreamPart[] = [- { type: 'start' },- { type: 'start-step', request: {}, warnings: [] },-];--const buildFinishStepPart = (usage: LanguageModelUsage): ModelStreamPart => ({- type: 'finish-step',- response: {- id: 'response-id',- timestamp: new Date(0),- modelId: 'gpt-5.6-luna',- },- usage,- performance: {- effectiveOutputTokensPerSecond: 0,- outputTokensPerSecond: undefined,- inputTokensPerSecond: undefined,- effectiveTotalTokensPerSecond: 0,- stepTimeMs: 0,- responseTimeMs: 0,- timeToFirstOutputMs: 0,- toolExecutionMs: {},- },- finishReason: 'stop',- rawFinishReason: undefined,- providerMetadata: undefined,-});--const buildFinishPart = (totalUsage: LanguageModelUsage): ModelStreamPart => ({- type: 'finish',- finishReason: 'stop',- rawFinishReason: undefined,- totalUsage,-});--const FINISH_PARTS: ModelStreamPart[] = [- buildFinishStepPart(USAGE),- buildFinishPart(USAGE),-];--const TEXT_PARTS: ModelStreamPart[] = [- ...START_PARTS,- { type: 'text-start', id: 'text-1' },- { type: 'text-delta', id: 'text-1', text: 'Hello' },- { type: 'text-end', id: 'text-1' },- ...FINISH_PARTS,-];--const EMPTY_REPLY_PARTS: ModelStreamPart[] = [...START_PARTS, ...FINISH_PARTS];--const QUESTIONS = [- {- header: 'Plan',- question: 'Which plan?',- options: [{ label: 'Pro' }, { label: 'Team' }],- },-];--const PENDING_QUESTION_PARTS: ModelStreamPart[] = [- ...START_PARTS,- {- type: 'tool-input-start',- id: 'tool-call-id',- toolName: ASK_QUESTION_TOOL_NAME,- },- {- type: 'tool-call',- toolCallId: 'tool-call-id',- toolName: ASK_QUESTION_TOOL_NAME,- input: QUESTIONS[0],- },- {- type: 'tool-result',- toolCallId: 'tool-call-id',- toolName: ASK_QUESTION_TOOL_NAME,- input: QUESTIONS[0],- output: { result: { question: QUESTIONS[0], status: 'pending' } },- },- ...FINISH_PARTS,-];--const NO_REPLY_PARTS: ModelStreamPart[] = [- ...START_PARTS,- {- type: 'tool-call',- toolCallId: 'tool-call-id',- toolName: NO_REPLY_TOOL_NAME,- input: {},- },- {- type: 'tool-result',- toolCallId: 'tool-call-id',- toolName: NO_REPLY_TOOL_NAME,- input: {},- output: { success: true },- },- ...FINISH_PARTS,-];--const RECOVERED_TOOL_CALL_PARTS: ModelStreamPart[] = [- ...START_PARTS,- {- type: 'tool-call',- toolCallId: 'tool-call-id',- toolName: 'code_interpreter',- input: {},- dynamic: true,- invalid: true,- error: new Error("Model tried to call unavailable tool 'code_interpreter'"),- },- {- type: 'tool-error',- toolCallId: 'tool-call-id',- toolName: 'code_interpreter',- input: {},- dynamic: true,- error:- "AI_NoSuchToolError: Model tried to call unavailable tool 'code_interpreter'",- },- { type: 'text-start', id: 'text-1' },- { type: 'text-delta', id: 'text-1', text: 'Here is the answer' },- { type: 'text-end', id: 'text-1' },- ...FINISH_PARTS,-];--const createFakeChatStream = ({- parts = TEXT_PARTS,- midStreamError,- onFirstPart,- isAborted = false,-}: {- parts?: ModelStreamPart[];- midStreamError?: Error;- onFirstPart?: () => void;- isAborted?: boolean;-} = {}) => ({- streamError: midStreamError,- stream: new ReadableStream<ModelStreamPart>(- {- pull(controller) {- parts.forEach((part, index) => {- controller.enqueue(part);-- if (index === 0) {- onFirstPart?.();- }- });-- if (midStreamError) {- controller.enqueue({ type: 'error', error: midStreamError });- }-- if (isAborted) {- controller.enqueue({ type: 'abort' });- }-- controller.close();- },- },- // Produced on the first read, so the tests' triggers fire while the job is streaming.- { highWaterMark: 0 },- ),-});--describe('StreamAgentChatJob', () => {- const workspace = {- id: 'workspace-id',- aiChatModelTier: 'smart',- aiAgentModelTier: 'smart',- isAutoModelSelectionEnabled: true,- aiModelIdByTier: {},- } as WorkspaceEntity;-- const jobData: StreamAgentChatJobData = {- threadId: 'thread-id',- streamId: 'stream-id',- userWorkspaceId: 'user-workspace-id',- workspaceId: 'workspace-id',- messages: [],- browsingContext: null,- lastUserMessageText: 'hello',- hasTitle: true,- conversationSizeTokens: 0,- existingTurnId: 'turn-id',- messageId: 'user-message-id',- };-- const buildJob = ({- workspaceFound = true,- chatStream = createFakeChatStream(),- streamChatRejection,- assistantPersistRejection,- totalsUpdateAffected = 1,- finalPublishRejection,- modelConfig = {- contextWindowTokens: 100000,- inputCostPerMillionTokens: 1,- outputCostPerMillionTokens: 2,- },- }: {- workspaceFound?: boolean;- chatStream?: ReturnType<typeof createFakeChatStream>;- streamChatRejection?: Error;- assistantPersistRejection?: Error;- totalsUpdateAffected?: number;- finalPublishRejection?: Error;- modelConfig?: Partial<AiModelConfig>;- } = {}) => {- const publishedEvents: PublishedEvent[] = [];-- const threadUsageQuery = jest.fn(- async (_sql?: string, _parameters?: unknown[]) =>- Array.from({ length: totalsUpdateAffected }, () => ({- id: 'thread-id',- })),- );- const threadRepository = {- findOne: jest.fn().mockResolvedValue({- id: 'thread-id',- deletedAt: null,- activeStreamId: 'stream-id',- }),- update: jest.fn().mockResolvedValue({ affected: 1 }),- query: jest.fn().mockImplementation(async (_workspaceId, work) =>- work({- table: () => '"workspace"."agentChatThread"',- manager: { query: threadUsageQuery },- }),- ),- };- const workspaceRepository = {- findOne: jest.fn().mockResolvedValue(workspaceFound ? workspace : null),- };- const agentChatService = {- upsertAssistantMessage: assistantPersistRejection- ? jest.fn().mockRejectedValue(assistantPersistRejection)- : jest.fn().mockResolvedValue(undefined),- generateTitleIfNeeded: jest.fn().mockResolvedValue(null),- notifyThreadUsageUpdated: jest.fn().mockResolvedValue(undefined),- };- const threadService = {- recordThreadActivity: jest.fn().mockResolvedValue(undefined),- };- const chatExecutionService = {- streamChat: streamChatRejection- ? jest.fn().mockRejectedValue(streamChatRejection)- : jest.fn().mockResolvedValue({- stream: chatStream,- modelConfig,- getStopReason: () => null,- getStreamError: () => chatStream.streamError,- }),- };- const eventPublisherService = {- resetStreamState: jest.fn().mockResolvedValue(undefined),- publish: jest- .fn()- .mockImplementation(({ event }: { event: PublishedEvent }) => {- if (- isDefined(finalPublishRejection) &&- event.type === 'message-persisted'- ) {- return Promise.reject(finalPublishRejection);- }-- publishedEvents.push(event);-- return Promise.resolve();- }),- };- const cancelCallbacks: Array<() => void> = [];- const cancelSubscriberService = {- subscribe: jest- .fn()- .mockImplementation((_channel: string, callback: () => void) => {- cancelCallbacks.push(callback);-- return Promise.resolve();- }),- unsubscribe: jest.fn().mockResolvedValue(undefined),- };- const agentChatStreamingService = {- flushNextQueuedMessage: jest.fn().mockResolvedValue(undefined),- };- const streamHeartbeatService = {- startRunning: jest.fn().mockReturnValue(() => {}),- markClaimed: jest.fn().mockResolvedValue(undefined),- clear: jest.fn().mockResolvedValue(undefined),- };- const metricsService = { incrementCounterBy: jest.fn() };- const aiModelRegistryService = {- getEffectiveModelConfig: jest- .fn()- .mockReturnValue({ modelId: 'openai/gpt-5.6-luna' }),- };- const sender = {- userWorkspaceId: 'user-workspace-id',- applicationId: null,- };- const authorization = {- authContext: {- type: 'user',- workspaceMemberId: 'member',- workspaceMember: { name: { firstName: 'Tim', lastName: 'Apple' } },- },- };- const actorService = {- authorizeJob: jest.fn().mockResolvedValue({- sender,- authorization,- message: { id: 'user-message-id', turnId: 'turn-id' },- }),- };- const turnRecorderService = {- markRunning: jest.fn().mockResolvedValue(true),- recordUsage: jest.fn().mockResolvedValue(undefined),- finish: jest.fn().mockResolvedValue(true),- };- const claimReleases = () =>- threadUsageQuery.mock.calls- .filter(([sql]) => String(sql).includes('"activeStreamId" = NULL'))- .map(([, parameters]) => {- const turnError = (parameters as unknown[])[3] as string | null;-- return {- turnError: turnError === null ? null : JSON.parse(turnError),- };- });- const job = new StreamAgentChatJob(- threadRepository as never,- workspaceRepository as never,- agentChatService as never,- chatExecutionService as never,- eventPublisherService as never,- cancelSubscriberService as never,- agentChatStreamingService as never,- streamHeartbeatService as never,- new AgentChatStreamRecoveryService(- threadRepository as never,- streamHeartbeatService as never,- eventPublisherService as never,- metricsService as never,- ),- threadService as never,- metricsService as never,- aiModelRegistryService as never,- actorService as never,- turnRecorderService as never,- );-- const turnCounts = (key: string) =>- metricsService.incrementCounterBy.mock.calls- .map(([call]) => call)- .filter((call: { key: string }) => call.key === key);-- return {- job,- actorService,- turnRecorderService,- claimReleases,- chatExecutionService,- publishedEvents,- threadRepository,- threadUsageQuery,- agentChatService,- threadService,- eventPublisherService,- agentChatStreamingService,- cancelCallbacks,- metricsService,- aiModelRegistryService,- turnCounts,- };- };-- it('streams the authorized turn without authorizing it again', async () => {- const { job, actorService, agentChatService, chatExecutionService } =- buildJob();- await job.handle(jobData);- expect(actorService.authorizeJob).toHaveBeenCalledTimes(1);- expect(actorService.authorizeJob).toHaveBeenCalledWith(- expect.objectContaining({- messageId: 'user-message-id',- turnId: 'turn-id',- }),- );- expect(chatExecutionService.streamChat).toHaveBeenCalledWith(- expect.objectContaining({- turnId: 'turn-id',- sender: expect.objectContaining({- userWorkspaceId: 'user-workspace-id',- }),- authorization: expect.objectContaining({- authContext: expect.objectContaining({ workspaceMemberId: 'member' }),- }),- }),- );- expect(agentChatService.upsertAssistantMessage).toHaveBeenCalledWith(- expect.objectContaining({ turnId: 'turn-id' }),- );- });-- it('runs the turn as its sender before streaming', async () => {- const { job, turnRecorderService, chatExecutionService } = buildJob();-- await job.handle(jobData);-- expect(turnRecorderService.markRunning).toHaveBeenCalledWith({- workspaceId: 'workspace-id',- turnId: 'turn-id',- modelId: 'openai/gpt-5.6-luna',- createdBy: expect.objectContaining({- source: 'MANUAL',- workspaceMemberId: 'member',- name: 'Tim Apple',- }),- streamClaim: { threadId: 'thread-id', streamId: 'stream-id' },- });- expect(- turnRecorderService.markRunning.mock.invocationCallOrder[0],- ).toBeLessThan(chatExecutionService.streamChat.mock.invocationCallOrder[0]);- });-- it('does not stream a turn whose stream was stopped while starting', async () => {- const { job, turnRecorderService, chatExecutionService } = buildJob();-- turnRecorderService.markRunning.mockResolvedValue(false);-- await job.handle(jobData);-- expect(chatExecutionService.streamChat).not.toHaveBeenCalled();- });-- it('finishes the turn as completed when the reply is saved', async () => {- const { job, turnRecorderService } = buildJob();-- await job.handle(jobData);-- expect(turnRecorderService.finish).toHaveBeenCalledWith({- workspaceId: 'workspace-id',- turnId: 'turn-id',- status: AgentTurnStatus.COMPLETED,- error: null,- modelId: 'openai/gpt-5.6-luna',- streamClaim: { threadId: 'thread-id', streamId: 'stream-id' },- });- });-- it('runs a job queued before sender attribution with only its turn', async () => {- const { job, actorService, agentChatService } = buildJob();- await job.handle({ ...jobData, messageId: undefined });- expect(actorService.authorizeJob).toHaveBeenCalledWith(- expect.objectContaining({ messageId: undefined, turnId: 'turn-id' }),- );- expect(agentChatService.upsertAssistantMessage).toHaveBeenCalledWith(- expect.objectContaining({ turnId: 'turn-id' }),- );- });-- it('uses the persisted turn when a job supplies only its message ID', async () => {- const { job, agentChatService } = buildJob();- await job.handle({ ...jobData, existingTurnId: undefined });- expect(agentChatService.upsertAssistantMessage).toHaveBeenCalledWith(- expect.objectContaining({ turnId: 'turn-id' }),- );- });-- it('rejects a persisted message without a turn', async () => {- const { job, actorService, chatExecutionService } = buildJob();- actorService.authorizeJob.mockResolvedValue({- message: { id: 'user-message-id', turnId: null },- } as never);- await expect(- job.handle({ ...jobData, existingTurnId: undefined }),- ).rejects.toMatchObject({- code: 'MESSAGE_NOT_FOUND',- });- expect(chatExecutionService.streamChat).not.toHaveBeenCalled();- });-- it('generates a title as the turn sender when the thread has none', async () => {- const { job, agentChatService } = buildJob();- await job.handle({ ...jobData, hasTitle: false });- expect(agentChatService.generateTitleIfNeeded).toHaveBeenCalledWith({- userWorkspaceId: 'user-workspace-id',- threadId: 'thread-id',- messageContent: 'hello',- workspaceId: 'workspace-id',- });- });-- it('does not invoke the model when the saved sender lost access before execution', async () => {- const { job, actorService, chatExecutionService, claimReleases } =- buildJob();- actorService.authorizeJob.mockRejectedValue(- new Error('Sender access revoked'),- );- await expect(job.handle(jobData)).rejects.toThrow('Sender access revoked');- expect(chatExecutionService.streamChat).not.toHaveBeenCalled();- expect(claimReleases()).toContainEqual({ turnError: null });- });-- it('publishes all chunks in order with message-persisted last on success', async () => {- const {- job,- publishedEvents,- threadRepository,- agentChatService,- agentChatStreamingService,- claimReleases,- } = buildJob();-- await job.handle(jobData);-- const chunkEvents = publishedEvents.filter(- (event) => event.type === 'stream-chunk',- );-- expect(- chunkEvents.map((event) => (event.chunk as { type: string }).type),- ).toEqual(TEXT_PARTS.map((part) => part.type));- expect(publishedEvents[publishedEvents.length - 1]).toMatchObject({- type: 'message-persisted',- });- expect(agentChatService.upsertAssistantMessage).toHaveBeenCalledWith(- expect.objectContaining({- id: expect.any(String),- turnId: 'turn-id',- parts: expect.arrayContaining([- expect.objectContaining({ type: 'text' }),- ]),- }),- );- expect(threadRepository.query).toHaveBeenCalledWith(- 'workspace-id',- expect.any(Function),- );- expect(claimReleases()).toContainEqual({ turnError: null });- expect(agentChatStreamingService.flushNextQueuedMessage).toHaveBeenCalled();- });-- it('gates the thread totals on still owning the stream so a prior completion is not double-counted', async () => {- const { job, agentChatService, threadRepository } = buildJob({- totalsUpdateAffected: 0,- });-- await job.handle(jobData);-- expect(agentChatService.upsertAssistantMessage).toHaveBeenCalledWith(- expect.objectContaining({ turnId: 'turn-id' }),- );- expect(threadRepository.query).toHaveBeenCalledWith(- 'workspace-id',- expect.any(Function),- );- expect(agentChatService.notifyThreadUsageUpdated).not.toHaveBeenCalled();- });-- it('prices each step on its own, so steps under the long-context threshold never reach its rate together', async () => {- const stepUsage: LanguageModelUsage = {- inputTokens: 150_000,- inputTokenDetails: {- noCacheTokens: 150_000,- cacheReadTokens: 0,- cacheWriteTokens: 0,- },- outputTokens: 1_000,- outputTokenDetails: { textTokens: 1_000, reasoningTokens: 0 },- totalTokens: 151_000,- };- const turnUsage: LanguageModelUsage = {- inputTokens: 300_000,- inputTokenDetails: {- noCacheTokens: 300_000,- cacheReadTokens: 0,- cacheWriteTokens: 0,- },- outputTokens: 2_000,- outputTokenDetails: { textTokens: 2_000, reasoningTokens: 0 },- totalTokens: 302_000,- };- const { job, turnRecorderService } = buildJob({- chatStream: createFakeChatStream({- parts: [- ...START_PARTS,- { type: 'text-start', id: 'text-1' },- { type: 'text-delta', id: 'text-1', text: 'Hello' },- { type: 'text-end', id: 'text-1' },- buildFinishStepPart(stepUsage),- { type: 'start-step', request: {}, warnings: [] },- buildFinishStepPart(stepUsage),- buildFinishPart(turnUsage),- ],- }),- modelConfig: {- contextWindowTokens: 1_000_000,- inputCostPerMillionTokens: 1,- outputCostPerMillionTokens: 2,- longContextCost: {- inputCostPerMillionTokens: 10,- outputCostPerMillionTokens: 20,- thresholdTokens: 200_000,- },- },- });-- await job.handle(jobData);-- expect(turnRecorderService.recordUsage).toHaveBeenCalledTimes(1);-- const [{ usage }] = turnRecorderService.recordUsage.mock.calls[0] as [- { usage: Record<string, number> },- ];- const { inputTokens, outputTokens, inputCredits, outputCredits } = usage;-- expect({ inputTokens, outputTokens, inputCredits, outputCredits }).toEqual({- inputTokens: 300_000,- outputTokens: 2_000,- inputCredits: 300_000,- outputCredits: 4_000,- });- });-- it('applies thread totals when the claim is still held even if the message already exists from a checkpoint', async () => {- const { job, agentChatService } = buildJob({ totalsUpdateAffected: 1 });-- await job.handle(jobData);-- expect(agentChatService.upsertAssistantMessage).toHaveBeenCalled();- expect(agentChatService.notifyThreadUsageUpdated).toHaveBeenCalled();- });-- it('never publishes the opaque error chunk to subscribers', async () => {- const { job, publishedEvents } = buildJob({- chatStream: createFakeChatStream({- midStreamError: new Error('provider exploded'),- }),- });-- await expect(job.handle(jobData)).rejects.toThrow('provider exploded');-- const chunkTypes = publishedEvents- .filter((event) => event.type === 'stream-chunk')- .map((event) => (event.chunk as { type: string }).type);-- expect(chunkTypes).not.toContain('error');- });-- it('rejects, persists the error, and unblocks the thread when the model stream fails mid-stream', async () => {- const { job, publishedEvents, claimReleases } = buildJob({- chatStream: createFakeChatStream({- midStreamError: new Error('provider exploded'),- }),- });-- await expect(job.handle(jobData)).rejects.toThrow('provider exploded');-- const chunkEvents = publishedEvents.filter(- (event) => event.type === 'stream-chunk',- );-- expect(chunkEvents).toHaveLength(TEXT_PARTS.length);- expect(publishedEvents[publishedEvents.length - 2]).toMatchObject({- type: 'stream-error',- code: 'STREAM_EXECUTION_FAILED',- message: 'provider exploded',- });- expect(publishedEvents[publishedEvents.length - 1]).toMatchObject({- type: 'queue-updated',- });- expect(publishedEvents.map((event) => event.type)).not.toContain(- 'message-persisted',- );- expect(claimReleases()).toContainEqual({- turnError: expect.objectContaining({- code: 'STREAM_EXECUTION_FAILED',- message: 'provider exploded',- }),- });- expect(claimReleases()).toContainEqual({ turnError: null });- });-- it('rejects promptly when execution setup throws instead of hanging until the queue lock expires', async () => {- const { job, publishedEvents, claimReleases } = buildJob({- streamChatRejection: new Error('model resolution failed'),- });-- await expect(job.handle(jobData)).rejects.toThrow(- 'model resolution failed',- );-- expect(publishedEvents[publishedEvents.length - 2]).toMatchObject({- type: 'stream-error',- message: 'model resolution failed',- });- expect(publishedEvents[publishedEvents.length - 1]).toMatchObject({- type: 'queue-updated',- });- expect(claimReleases()).toContainEqual({ turnError: null });- });-- it('terminates the stream with an error when assistant persistence fails after draining chunks', async () => {- const { job, publishedEvents } = buildJob({- assistantPersistRejection: new Error('insert failed'),- });-- await expect(job.handle(jobData)).rejects.toThrow('insert failed');-- const chunkEvents = publishedEvents.filter(- (event) => event.type === 'stream-chunk',- );-- expect(chunkEvents).toHaveLength(TEXT_PARTS.length);- expect(publishedEvents[publishedEvents.length - 2]).toMatchObject({- type: 'stream-error',- });- expect(publishedEvents[publishedEvents.length - 1]).toMatchObject({- type: 'queue-updated',- });- expect(publishedEvents.map((event) => event.type)).not.toContain(- 'message-persisted',- );- });-- it('persists the error and unblocks the thread when the workspace is missing', async () => {- const { job, publishedEvents, claimReleases } = buildJob({- workspaceFound: false,- });-- await expect(job.handle(jobData)).rejects.toMatchObject({- code: AiExceptionCode.WORKSPACE_NOT_FOUND,- });-- expect(publishedEvents[publishedEvents.length - 2]).toMatchObject({- type: 'stream-error',- code: AiExceptionCode.WORKSPACE_NOT_FOUND,- });- expect(publishedEvents[publishedEvents.length - 1]).toMatchObject({- type: 'queue-updated',- });- expect(claimReleases()).toContainEqual({- turnError: expect.objectContaining({- code: AiExceptionCode.WORKSPACE_NOT_FOUND,- }),- });- expect(claimReleases()).toContainEqual({ turnError: null });- });-- it('bails out without streaming when the thread no longer holds the claim for this stream', async () => {- const {- job,- publishedEvents,- threadRepository,- eventPublisherService,- agentChatStreamingService,- } = buildJob();-- threadRepository.findOne.mockResolvedValueOnce({- id: 'thread-id',- deletedAt: null,- activeStreamId: 'newer-stream-id',- });-- await job.handle(jobData);-- expect(publishedEvents).toHaveLength(0);- expect(eventPublisherService.resetStreamState).not.toHaveBeenCalled();- expect(threadRepository.update).not.toHaveBeenCalled();- expect(- agentChatStreamingService.flushNextQueuedMessage,- ).not.toHaveBeenCalled();- });-- it('bails out without streaming when the thread was deleted', async () => {- const { job, publishedEvents, threadRepository, eventPublisherService } =- buildJob();-- threadRepository.findOne.mockResolvedValueOnce(null);-- await job.handle(jobData);-- expect(publishedEvents).toHaveLength(0);- expect(eventPublisherService.resetStreamState).not.toHaveBeenCalled();- expect(threadRepository.update).not.toHaveBeenCalled();- });-- it('persists the interrupted error and publishes the terminal sequence when aborted by a worker shutdown', async () => {- let triggerShutdown: (() => void) | undefined;-- const { job, publishedEvents, agentChatStreamingService, claimReleases } =- buildJob({- chatStream: createFakeChatStream({- onFirstPart: () => triggerShutdown?.(),- }),- });-- const shutdownController = new AbortController();-- triggerShutdown = () => shutdownController.abort();-- await expect(- job.handle(jobData, {- jobId: 'job-id',- abortSignal: shutdownController.signal,- }),- ).rejects.toMatchObject({ code: AiExceptionCode.STREAM_INTERRUPTED });-- const eventTypes = publishedEvents.map((event) => event.type);- const streamErrorIndex = eventTypes.indexOf('stream-error');- const queueUpdatedIndex = eventTypes.indexOf('queue-updated');-- expect(publishedEvents[streamErrorIndex]).toMatchObject({- type: 'stream-error',- code: AiExceptionCode.STREAM_INTERRUPTED,- });- expect(queueUpdatedIndex).toBeGreaterThan(streamErrorIndex);- expect(eventTypes).not.toContain('message-persisted');- expect(claimReleases()).toContainEqual({- turnError: expect.objectContaining({- code: AiExceptionCode.STREAM_INTERRUPTED,- }),- });- expect(claimReleases()).toContainEqual({ turnError: null });- expect(- agentChatStreamingService.flushNextQueuedMessage,- ).not.toHaveBeenCalled();- });-- it('keeps user-cancel semantics when a shutdown signal is wired but never aborted', async () => {- let triggerUserCancel: (() => void) | undefined;-- const { job, publishedEvents, agentChatStreamingService, cancelCallbacks } =- buildJob({- chatStream: createFakeChatStream({- onFirstPart: () => triggerUserCancel?.(),- }),- });-- triggerUserCancel = () => cancelCallbacks.forEach((callback) => callback());-- const shutdownController = new AbortController();-- await job.handle(jobData, {- jobId: 'job-id',- abortSignal: shutdownController.signal,- });-- expect(publishedEvents.map((event) => event.type)).not.toContain(- 'stream-error',- );- expect(- agentChatStreamingService.flushNextQueuedMessage,- ).not.toHaveBeenCalled();- });-- it('resolves without flushing the queue when the stream is cancelled', async () => {- let triggerCancel: (() => void) | undefined;-- const {- job,- publishedEvents,- agentChatService,- agentChatStreamingService,- cancelCallbacks,- } = buildJob({- chatStream: createFakeChatStream({- onFirstPart: () => triggerCancel?.(),- }),- });-- triggerCancel = () => cancelCallbacks.forEach((callback) => callback());-- await job.handle(jobData);-- expect(publishedEvents.map((event) => event.type)).not.toContain(- 'stream-error',- );- expect(- agentChatStreamingService.flushNextQueuedMessage,- ).not.toHaveBeenCalled();- expect(agentChatService.notifyThreadUsageUpdated).toHaveBeenCalled();- });- it('labels turn-started with the resolved model, not the auto-select id', async () => {- const { job, aiModelRegistryService, turnCounts } = buildJob();-- await job.handle({ ...jobData, modelId: 'default-fast-model' });-- expect(aiModelRegistryService.getEffectiveModelConfig).toHaveBeenCalledWith(- 'default-fast-model',- workspace,- );- expect(turnCounts('ai-chat/turn-started')).toEqual([- expect.objectContaining({- attributes: { model: 'openai/gpt-5.6-luna' },- }),- ]);- });-- it('falls back to the workspace chat tier when the turn did not pick one', async () => {- const { job, aiModelRegistryService } = buildJob();-- await job.handle(jobData);-- expect(aiModelRegistryService.getEffectiveModelConfig).toHaveBeenCalledWith(- 'default-smart-model',- workspace,- );- });-- it('counts a text reply once, as an answered completion', async () => {- const { job, turnCounts } = buildJob();-- await job.handle(jobData);-- expect(turnCounts('ai-chat/turn-completed')).toEqual([- expect.objectContaining({- attributes: { model: 'openai/gpt-5.6-luna', outcome: 'answered' },- }),- ]);- expect(turnCounts('ai-chat/turn-failed')).toEqual([]);- expect(turnCounts('ai-chat/turn-cancelled')).toEqual([]);- });-- it('counts a turn that ended on a question as completed and awaiting the user', async () => {- const { job, turnCounts } = buildJob({- chatStream: createFakeChatStream({- parts: PENDING_QUESTION_PARTS,- }),- });-- await job.handle(jobData);-- expect(turnCounts('ai-chat/turn-completed')).toEqual([- expect.objectContaining({- attributes: { model: 'openai/gpt-5.6-luna', outcome: 'awaiting_user' },- }),- ]);- expect(turnCounts('ai-chat/turn-failed')).toEqual([]);- });-- const pendingQuestionMessageIdOf = (threadUsageQuery: jest.Mock) =>- (threadUsageQuery.mock.calls[0] as unknown as [string, unknown[]])[1][4];-- it('marks the conversation as waiting on the message that paused, in the totals write, and leaves the queue waiting', async () => {- const { job, threadUsageQuery, agentChatStreamingService } = buildJob({- chatStream: createFakeChatStream({ parts: PENDING_QUESTION_PARTS }),- });-- await job.handle(jobData);-- expect(pendingQuestionMessageIdOf(threadUsageQuery)).toEqual(- expect.any(String),- );- expect(- agentChatStreamingService.flushNextQueuedMessage,- ).not.toHaveBeenCalled();- });-- it('leaves a conversation whose turn did not pause waiting on nothing', async () => {- const { job, threadUsageQuery } = buildJob();-- await job.handle(jobData);-- expect(pendingQuestionMessageIdOf(threadUsageQuery)).toBeNull();- });-- it('fails the turn, leaving it retryable, when a paused call cannot be read', async () => {- const { job, threadUsageQuery, publishedEvents, turnCounts } = buildJob({- chatStream: createFakeChatStream({- parts: PENDING_QUESTION_PARTS.map((part) =>- part.type === 'tool-call' || part.type === 'tool-result'- ? { ...part, input: { questions: [] } }- : part,- ),- }),- });-- await expect(job.handle(jobData)).rejects.toThrow(- 'A call waiting on the user could not be read',- );-- expect(pendingQuestionMessageIdOf(threadUsageQuery)).toBeNull();- expect(publishedEvents.map((event) => event.type)).toContain(- 'stream-error',- );- expect(turnCounts('ai-chat/turn-completed')).toEqual([]);- });-- it('counts a reply as answered when the model recovers from a failed tool call', async () => {- const { job, publishedEvents, turnCounts } = buildJob({- chatStream: createFakeChatStream({- parts: RECOVERED_TOOL_CALL_PARTS,- }),- });-- await job.handle(jobData);-- expect(turnCounts('ai-chat/turn-completed')).toEqual([- expect.objectContaining({- attributes: { model: 'openai/gpt-5.6-luna', outcome: 'answered' },- }),- ]);- expect(turnCounts('ai-chat/turn-failed')).toEqual([]);- expect(publishedEvents.map((event) => event.type)).not.toContain(- 'stream-error',- );- });-- it('counts an aborted turn as cancelled rather than leaving it unaccounted', async () => {- const { job, turnCounts } = buildJob({- chatStream: createFakeChatStream({- parts: START_PARTS,- isAborted: true,- }),- });-- await job.handle(jobData);-- expect(turnCounts('ai-chat/turn-cancelled')).toEqual([- expect.objectContaining({- attributes: {- model: 'openai/gpt-5.6-luna',- reason: 'user_cancelled',- },- }),- ]);- expect(turnCounts('ai-chat/turn-completed')).toEqual([]);- });-- it('adds the steps an aborted turn completed to the thread totals', async () => {- const { job, turnRecorderService } = buildJob({- chatStream: createFakeChatStream({- parts: [- ...START_PARTS,- { type: 'text-start', id: 'text-1' },- { type: 'text-delta', id: 'text-1', text: 'Hello' },- { type: 'text-end', id: 'text-1' },- buildFinishStepPart(USAGE),- ],- isAborted: true,- }),- });-- await job.handle(jobData);-- const [{ usage }] = turnRecorderService.recordUsage.mock.calls[0] as [- { usage: Record<string, number> },- ];- const { inputTokens, outputTokens } = usage;-- expect({ inputTokens, outputTokens }).toEqual({- inputTokens: 12,- outputTokens: 3,- });- });-- it('counts a turn whose claim moved on as superseded', async () => {- const { job, turnCounts, threadService } = buildJob({- totalsUpdateAffected: 0,- });-- await job.handle(jobData);-- expect(threadService.recordThreadActivity).toHaveBeenCalledWith({- workspaceId: jobData.workspaceId,- threadId: jobData.threadId,- text: 'Hello',- });- expect(turnCounts('ai-chat/turn-cancelled')).toEqual([- expect.objectContaining({- attributes: { model: 'openai/gpt-5.6-luna', reason: 'superseded' },- }),- ]);- expect(turnCounts('ai-chat/turn-completed')).toEqual([]);- });-- it('keeps a superseded reply when recording its activity fails', async () => {- const { job, turnCounts, threadService } = buildJob({- totalsUpdateAffected: 0,- });-- threadService.recordThreadActivity.mockRejectedValue(- new Error('database unavailable'),- );-- await job.handle(jobData);-- expect(threadService.recordThreadActivity).toHaveBeenCalledWith({- workspaceId: jobData.workspaceId,- threadId: jobData.threadId,- text: 'Hello',- });-- expect(turnCounts('ai-chat/turn-cancelled')).toEqual([- expect.objectContaining({- attributes: { model: 'openai/gpt-5.6-luna', reason: 'superseded' },- }),- ]);- expect(turnCounts('ai-chat/turn-failed')).toEqual([]);- });-- it('counts a turn that chose not to reply as completed, without marking the chat as active', async () => {- const { job, turnCounts, threadUsageQuery } = buildJob({- chatStream: createFakeChatStream({ parts: NO_REPLY_PARTS }),- });-- await job.handle(jobData);-- expect(turnCounts('ai-chat/turn-completed')).toEqual([- expect.objectContaining({- attributes: { model: 'openai/gpt-5.6-luna', outcome: 'silent' },- }),- ]);- expect(turnCounts('ai-chat/turn-failed')).toEqual([]);-- const [usageSql] = threadUsageQuery.mock.calls[0] as unknown as [string];-- expect(usageSql).not.toContain('lastActivityAt');- });-- it('marks the chat as active when the turn replied', async () => {- const { job, threadUsageQuery } = buildJob();-- await job.handle(jobData);-- const [usageSql] = threadUsageQuery.mock.calls[0] as unknown as [string];-- expect(usageSql).toContain('lastActivityAt');- });-- it('counts an empty reply as a no_text failure exactly once', async () => {- const { job, turnCounts } = buildJob({- chatStream: createFakeChatStream({- parts: EMPTY_REPLY_PARTS,- }),- });-- await job.handle(jobData);-- expect(turnCounts('ai-chat/turn-failed')).toEqual([- expect.objectContaining({- attributes: { model: 'openai/gpt-5.6-luna', failure_phase: 'no_text' },- }),- ]);- });-- it('leaves a stream error to the execution counter so it is not counted twice', async () => {- const { job, turnCounts } = buildJob({- streamChatRejection: new Error('provider exploded'),- });-- await job.handle(jobData).catch(() => {});-- expect(turnCounts('ai-chat/turn-failed')).toEqual([- expect.objectContaining({- attributes: expect.objectContaining({- model: 'openai/gpt-5.6-luna',- failure_phase: 'execution',- }),- }),- ]);- expect(turnCounts('ai-chat/turn-completed')).toEqual([]);- });-- it('records exactly one outcome for every started turn', async () => {- const scenarios = [- buildJob(),- buildJob({ totalsUpdateAffected: 0 }),- buildJob({- chatStream: createFakeChatStream({- parts: PENDING_QUESTION_PARTS,- }),- }),- buildJob({- chatStream: createFakeChatStream({- parts: START_PARTS,- isAborted: true,- }),- }),- buildJob({ streamChatRejection: new Error('provider exploded') }),- ];-- for (const scenario of scenarios) {- await scenario.job.handle(jobData).catch(() => {});-- const started = scenario.turnCounts('ai-chat/turn-started').length;- const terminal =- scenario.turnCounts('ai-chat/turn-completed').length +- scenario.turnCounts('ai-chat/turn-cancelled').length +- scenario.turnCounts('ai-chat/turn-failed').length;-- expect({ started, terminal }).toEqual({ started: 1, terminal: 1 });- }- });- it('does not count a turn twice when the final publish fails after the outcome was recorded', async () => {- const { job, turnCounts } = buildJob({- finalPublishRejection: new Error('redis is down'),- });-- await job.handle(jobData).catch(() => {});-- expect(turnCounts('ai-chat/turn-completed')).toEqual([- expect.objectContaining({- attributes: { model: 'openai/gpt-5.6-luna', outcome: 'answered' },- }),- ]);- expect(turnCounts('ai-chat/turn-failed')).toEqual([]);- expect(turnCounts('ai-chat/turn-started')).toHaveLength(1);- });-});diff --git a/packages/twenty-server/src/engine/metadata-modules/ai/ai-chat/jobs/flush-queued-agent-chat-messages-job-name.constant.ts b/packages/twenty-server/src/engine/metadata-modules/ai/ai-chat/jobs/flush-queued-agent-chat-messages-job-name.constant.tsnew file mode 100644index 00000000000..e052015c53f--- /dev/null+++ b/packages/twenty-server/src/engine/metadata-modules/ai/ai-chat/jobs/flush-queued-agent-chat-messages-job-name.constant.ts@@ -0,0 +1,2 @@+export const FLUSH_QUEUED_AGENT_CHAT_MESSAGES_JOB_NAME =+ 'FlushQueuedAgentChatMessagesJob';diff --git a/packages/twenty-server/src/engine/metadata-modules/ai/ai-chat/jobs/flush-queued-agent-chat-messages-job.types.ts b/packages/twenty-server/src/engine/metadata-modules/ai/ai-chat/jobs/flush-queued-agent-chat-messages-job.types.tsnew file mode 100644index 00000000000..bb4b393bdb7--- /dev/null+++ b/packages/twenty-server/src/engine/metadata-modules/ai/ai-chat/jobs/flush-queued-agent-chat-messages-job.types.ts@@ -0,0 +1,4 @@+export type FlushQueuedAgentChatMessagesJobData = {+ workspaceId: string;+ threadId: string;+};diff --git a/packages/twenty-server/src/engine/metadata-modules/ai/ai-chat/jobs/flush-queued-agent-chat-messages.job.ts b/packages/twenty-server/src/engine/metadata-modules/ai/ai-chat/jobs/flush-queued-agent-chat-messages.job.tsnew file mode 100644index 00000000000..ce806dba613--- /dev/null+++ b/packages/twenty-server/src/engine/metadata-modules/ai/ai-chat/jobs/flush-queued-agent-chat-messages.job.ts@@ -0,0 +1,21 @@+import { Scope } from '@nestjs/common';++import { Process } from 'src/engine/core-modules/message-queue/decorators/process.decorator';+import { Processor } from 'src/engine/core-modules/message-queue/decorators/processor.decorator';+import { MessageQueue } from 'src/engine/core-modules/message-queue/message-queue.constants';+import { FLUSH_QUEUED_AGENT_CHAT_MESSAGES_JOB_NAME } from 'src/engine/metadata-modules/ai/ai-chat/jobs/flush-queued-agent-chat-messages-job-name.constant';+import { type FlushQueuedAgentChatMessagesJobData } from 'src/engine/metadata-modules/ai/ai-chat/jobs/flush-queued-agent-chat-messages-job.types';+import { AgentChatStreamingService } from 'src/engine/metadata-modules/ai/ai-chat/services/agent-chat-streaming.service';++// messages sent while a run held the conversation are sent once it lets go+@Processor({ queueName: MessageQueue.aiStreamQueue, scope: Scope.REQUEST })+export class FlushQueuedAgentChatMessagesJob {+ constructor(+ private readonly agentChatStreamingService: AgentChatStreamingService,+ ) {}++ @Process(FLUSH_QUEUED_AGENT_CHAT_MESSAGES_JOB_NAME)+ async handle(data: FlushQueuedAgentChatMessagesJobData): Promise<void> {+ await this.agentChatStreamingService.flushNextQueuedMessage(data);+ }+}diff --git a/packages/twenty-server/src/engine/metadata-modules/ai/ai-chat/jobs/stream-agent-chat.job.ts b/packages/twenty-server/src/engine/metadata-modules/ai/ai-chat/jobs/stream-agent-chat.job.tsindex 9c66c55d0a7..10e5f4e6f32 100644--- a/packages/twenty-server/src/engine/metadata-modules/ai/ai-chat/jobs/stream-agent-chat.job.ts+++ b/packages/twenty-server/src/engine/metadata-modules/ai/ai-chat/jobs/stream-agent-chat.job.ts@@ -1,87 +1,51 @@-import { AgentChatActorService } from 'src/engine/metadata-modules/ai/ai-chat/services/agent-chat-actor.service';-import { findAwaitingCallText } from 'src/engine/metadata-modules/ai/ai-chat/utils/find-awaiting-call-text.util';-import { findLastMessageText } from 'src/engine/metadata-modules/ai/ai-chat/utils/find-last-message-text.util';-import { updateAgentChatThreadUsage } from 'src/engine/metadata-modules/ai/ai-chat/utils/update-agent-chat-thread-usage.util';-import { mapAgentChatTurnOutcomeToTurnStatus } from 'src/engine/metadata-modules/ai/ai-chat/utils/map-agent-chat-turn-outcome-to-turn-status.util';-import { AgentTurnRecorderService } from 'src/engine/metadata-modules/ai/ai-history/services/agent-turn-recorder.service';-import { buildActorMetadataFromAuthContext } from 'src/engine/core-modules/actor/utils/build-actor-metadata-from-auth-context.util';-import { InjectAgentHistoryRepository } from 'src/engine/metadata-modules/ai/ai-history/repositories/inject-agent-history-repository.decorator';-import { AgentHistoryRepository } from 'src/engine/metadata-modules/ai/ai-history/repositories/agent-history-repository';import { Logger, Scope } from '@nestjs/common';import { InjectRepository } from '@nestjs/typeorm';import { isNonEmptyString } from '@sniptt/guards';-import {- createUIMessageStream,- readUIMessageStream,- toUIMessageStream,-} from 'ai';import {type CodeExecutionData,- type ExtendedUIMessage,isSilentAgentChatMessageParts,} from 'twenty-shared/ai';import { assertUnreachable, isDefined } from 'twenty-shared/utils';import { Repository } from 'typeorm';-import { v5 as uuidv5 } from 'uuid';import { type MessageQueueJobContext } from 'src/engine/core-modules/message-queue/interfaces/message-queue-job.interface';+import { buildActorMetadataFromAuthContext } from 'src/engine/core-modules/actor/utils/build-actor-metadata-from-auth-context.util';import { Process } from 'src/engine/core-modules/message-queue/decorators/process.decorator';import { Processor } from 'src/engine/core-modules/message-queue/decorators/processor.decorator';import { MessageQueue } from 'src/engine/core-modules/message-queue/message-queue.constants';import { MetricsService } from 'src/engine/core-modules/metrics/metrics.service';import { MetricsKeys } from 'src/engine/core-modules/metrics/types/metrics-keys.type';-import { toDisplayCredits } from 'src/engine/core-modules/usage/utils/to-display-credits.util';import { WorkspaceEntity } from 'src/engine/core-modules/workspace/workspace.entity';import { type StepDebitResult } from 'src/engine/metadata-modules/ai/ai-billing/types/step-debit-result.type';-import { addStepToAgentTurnUsage } from 'src/engine/metadata-modules/ai/ai-billing/utils/compute-agent-turn-usage-from-steps.util';-import {- AiException,- AiExceptionCode,-} from 'src/engine/metadata-modules/ai/ai.exception';-import { AgentChatThreadWorkspaceEntity } from 'src/engine/metadata-modules/ai/ai-history/standard-objects/agent-chat-thread.workspace-entity';import { findAwaitingPausingToolParts } from 'src/engine/metadata-modules/ai/ai-agent-execution/pausing-tools/utils/find-awaiting-pausing-tool-parts.util';-import { AgentChatCancelSubscriberService } from 'src/engine/metadata-modules/ai/ai-chat/services/agent-chat-cancel-subscriber.service';+import { STREAM_AGENT_CHAT_JOB_NAME } from 'src/engine/metadata-modules/ai/ai-chat/jobs/stream-agent-chat-job-name.constant';+import { type StreamAgentChatJobData } from 'src/engine/metadata-modules/ai/ai-chat/jobs/stream-agent-chat-job.types';+import { AgentChatActorService } from 'src/engine/metadata-modules/ai/ai-chat/services/agent-chat-actor.service';import { AgentChatEventPublisherService } from 'src/engine/metadata-modules/ai/ai-chat/services/agent-chat-event-publisher.service';-import { AgentChatStreamHeartbeatService } from 'src/engine/metadata-modules/ai/ai-chat/services/agent-chat-stream-heartbeat.service';-import { AgentChatStreamRecoveryService } from 'src/engine/metadata-modules/ai/ai-chat/services/agent-chat-stream-recovery.service';-import { AgentChatThreadService } from 'src/engine/metadata-modules/ai/ai-chat/services/agent-chat-thread.service';-import { AgentChatStreamingService } from 'src/engine/metadata-modules/ai/ai-chat/services/agent-chat-streaming.service';import { AgentChatService } from 'src/engine/metadata-modules/ai/ai-chat/services/agent-chat.service';import {- type ChatExecutionOptions,- ChatExecutionService,-} from 'src/engine/metadata-modules/ai/ai-chat/services/chat-execution.service';+ type AgentTurnStreamResult,+ AgentTurnStreamService,+} from 'src/engine/metadata-modules/ai/ai-chat/services/agent-turn-stream.service';+import { ChatExecutionService } from 'src/engine/metadata-modules/ai/ai-chat/services/chat-execution.service';import { type AgentChatTurnOutcome } from 'src/engine/metadata-modules/ai/ai-chat/types/agent-chat-turn-outcome.type';-import {- classifyAgentChatTurnOutcome,- resolveSupersededTurnOutcome,-} from 'src/engine/metadata-modules/ai/ai-chat/utils/classify-agent-chat-turn-outcome.util';-import { AGENT_CHAT_CHECKPOINT_INTERVAL_MS } from 'src/engine/metadata-modules/ai/ai-chat/constants/agent-chat-checkpoint-interval-ms.constant';+import { type AgentChatStreamClaim } from 'src/engine/metadata-modules/ai/ai-chat/services/agent-chat-stream-recovery.service';import { formatErrorWithCause } from 'src/engine/metadata-modules/ai/ai-chat/utils/format-error-with-cause.util';-import { getCancelChannel } from 'src/engine/metadata-modules/ai/ai-chat/utils/get-cancel-channel.util';-import { mapErrorToStreamError } from 'src/engine/metadata-modules/ai/ai-history/utils/map-error-to-stream-error.util';+import { hasReplyText } from 'src/engine/metadata-modules/ai/ai-chat/utils/has-reply-text.util';import { tagAiChatStreamScope } from 'src/engine/metadata-modules/ai/ai-chat/utils/tag-ai-chat-stream-scope.util';+import { AgentHistoryRepository } from 'src/engine/metadata-modules/ai/ai-history/repositories/agent-history-repository';+import { InjectAgentHistoryRepository } from 'src/engine/metadata-modules/ai/ai-history/repositories/inject-agent-history-repository.decorator';+import { AgentTurnRecorderService } from 'src/engine/metadata-modules/ai/ai-history/services/agent-turn-recorder.service';+import { AgentChatThreadWorkspaceEntity } from 'src/engine/metadata-modules/ai/ai-history/standard-objects/agent-chat-thread.workspace-entity';+import { mapErrorToStreamError } from 'src/engine/metadata-modules/ai/ai-history/utils/map-error-to-stream-error.util';import { AiModelRegistryService } from 'src/engine/metadata-modules/ai/ai-models/services/ai-model-registry.service';-import type { AiModelConfig } from 'src/engine/metadata-modules/ai/ai-models/types/ai-model-config.type';--import { STREAM_AGENT_CHAT_JOB_NAME } from 'src/engine/metadata-modules/ai/ai-chat/jobs/stream-agent-chat-job-name.constant';-import { type StreamAgentChatJobData } from 'src/engine/metadata-modules/ai/ai-chat/jobs/stream-agent-chat-job.types';import { getChatModelId } from 'src/engine/metadata-modules/ai/ai-models/utils/get-chat-model-id.util';--// assistantMessageId derives from streamId, so a retried job skips persistence while each resume persists its own message-const ASSISTANT_MESSAGE_ID_NAMESPACE = '0b9c2a3d-4e5f-4a1b-8c2d-3e4f5a6b7c8d';--type StreamUsageTotals = {- inputTokens: number;- outputTokens: number;- inputCredits: number;- outputCredits: number;- cacheReadTokens: number;- cacheCreationTokens: number;- conversationSize: number;-};+import {+ AiException,+ AiExceptionCode,+} from 'src/engine/metadata-modules/ai/ai.exception';@Processor({ queueName: MessageQueue.aiStreamQueue, scope: Scope.REQUEST })export class StreamAgentChatJob {@@ -90,9 +54,6 @@ export class StreamAgentChatJob {// keeps the catch in handle() from counting a turn the stream already classifiedprivate hasRecordedTurnOutcome = false;- // set once the turn pauses on a person: a queued message drained then would bypass the pending answer- private isAwaitingInput = false;-constructor(@InjectAgentHistoryRepository('agentChatThread')private readonly threadRepository: AgentHistoryRepository<AgentChatThreadWorkspaceEntity>,@@ -101,11 +62,7 @@ export class StreamAgentChatJob {private readonly agentChatService: AgentChatService,private readonly chatExecutionService: ChatExecutionService,private readonly eventPublisherService: AgentChatEventPublisherService,- private readonly cancelSubscriberService: AgentChatCancelSubscriberService,- private readonly agentChatStreamingService: AgentChatStreamingService,- private readonly streamHeartbeatService: AgentChatStreamHeartbeatService,- private readonly streamRecoveryService: AgentChatStreamRecoveryService,- private readonly threadService: AgentChatThreadService,+ private readonly turnStreamService: AgentTurnStreamService,private readonly metricsService: MetricsService,private readonly aiModelRegistryService: AiModelRegistryService,private readonly actorService: AgentChatActorService,@@ -149,91 +106,18 @@ export class StreamAgentChatJob {attributes: { model: turnModelId },});- await this.eventPublisherService.resetStreamState(data.threadId);-- const abortController = new AbortController();- const cancelChannel = getCancelChannel(data.threadId, data.streamId);-- const stopHeartbeat = this.streamHeartbeatService.startRunning(- data.streamId,- );-- await this.cancelSubscriberService.subscribe(cancelChannel, () => {- abortController.abort();- });-- context?.abortSignal?.addEventListener(- 'abort',- () => {- abortController.abort(- new AiException(- 'The response was interrupted before it could finish.',- AiExceptionCode.STREAM_INTERRUPTED,- ),- );- },- { once: true },- );+ const claim = {+ threadId: data.threadId,+ workspaceId: data.workspaceId,+ streamId: data.streamId,+ };try {- if (!workspace) {- throw new AiException(- `Workspace ${data.workspaceId} not found`,- AiExceptionCode.WORKSPACE_NOT_FOUND,- );- }-- const { message, sender, authorization } =- await this.actorService.authorizeJob({- workspaceId: data.workspaceId,- threadId: data.threadId,- messageId: data.messageId,- turnId: data.existingTurnId,- userWorkspaceId: data.userWorkspaceId,- });-- const turnId = data.existingTurnId ?? message?.turnId;-- if (!isDefined(turnId)) {- throw new AiException(- 'Message turn not found',- AiExceptionCode.MESSAGE_NOT_FOUND,- );- }-- const isRunning = await this.turnRecorderService.markRunning({- workspaceId: data.workspaceId,- turnId,- modelId: turnModelId,- createdBy: buildActorMetadataFromAuthContext(authorization.authContext),- streamClaim: { threadId: data.threadId, streamId: data.streamId },- });-- // the stream was stopped while starting, and its turn already ended- if (!isRunning) {- return;- }-- const titlePromise = data.hasTitle- ? Promise.resolve(null)- : this.agentChatService- .generateTitleIfNeeded({- userWorkspaceId: data.userWorkspaceId,- threadId: data.threadId,- messageContent: data.lastUserMessageText,- workspaceId: data.workspaceId,- })- .catch(() => null);-- await this.buildAndPublishStream({- workspace,- data,- turnId,- sender,- authorization,- titlePromise,- abortSignal: abortController.signal,- turnModelId,+ await this.turnStreamService.runClaimed({+ claim,+ jobAbortSignal: context?.abortSignal,+ work: (abortSignal) =>+ this.runTurn({ workspace, data, claim, turnModelId, abortSignal }),});} catch (error) {this.logger.error(@@ -249,13 +133,6 @@ export class StreamAgentChatJob {turnModelId,);- await this.streamRecoveryService.failStream({- threadId: data.threadId,- workspaceId: data.workspaceId,- streamId: data.streamId,- error,- });-await this.eventPublisherService.publish({threadId: data.threadId,@@ -264,28 +141,189 @@ export class StreamAgentChatJob {}).catch(() => {});throw error;- } finally {- stopHeartbeat();- await this.cancelSubscriberService.unsubscribe(cancelChannel);- await this.streamRecoveryService.releaseStreamClaim({- threadId: data.threadId,+ }+ }++ private async runTurn({+ workspace,+ data,+ claim,+ turnModelId,+ abortSignal,+ }: {+ workspace: WorkspaceEntity | null;+ data: StreamAgentChatJobData;+ claim: AgentChatStreamClaim;+ turnModelId: string;+ abortSignal: AbortSignal;+ }): Promise<void> {+ if (!workspace) {+ throw new AiException(+ `Workspace ${data.workspaceId} not found`,+ AiExceptionCode.WORKSPACE_NOT_FOUND,+ );+ }++ const { message, sender, authorization } =+ await this.actorService.authorizeJob({workspaceId: data.workspaceId,- streamId: data.streamId,+ threadId: data.threadId,+ messageId: data.messageId,+ turnId: data.existingTurnId,+ userWorkspaceId: data.userWorkspaceId,});- if (!abortController.signal.aborted && !this.isAwaitingInput) {- await this.agentChatStreamingService- .flushNextQueuedMessage({+ const turnId = data.existingTurnId ?? message?.turnId;++ if (!isDefined(turnId)) {+ throw new AiException(+ 'Message turn not found',+ AiExceptionCode.MESSAGE_NOT_FOUND,+ );+ }++ const isRunning = await this.turnRecorderService.markRunning({+ workspaceId: data.workspaceId,+ turnId,+ modelId: turnModelId,+ createdBy: buildActorMetadataFromAuthContext(authorization.authContext),+ streamClaim: claim,+ });++ // the stream was stopped while starting, and its turn already ended+ if (!isRunning) {+ return;+ }++ const titlePromise = data.hasTitle+ ? Promise.resolve(null)+ : this.agentChatService+ .generateTitleIfNeeded({+ userWorkspaceId: data.userWorkspaceId,threadId: data.threadId,+ messageContent: data.lastUserMessageText,workspaceId: data.workspaceId,})- .catch((error) => {- this.logger.error(- `Failed to flush queued message for thread ${data.threadId}: ${formatErrorWithCause(error)}`,- );- });+ .catch(() => null);++ const result = await this.turnStreamService.stream({+ claim,+ turnId,+ startTurn: async (writer) => {+ const turn = await this.chatExecutionService.streamChat({+ workspace,+ userWorkspaceId: data.userWorkspaceId,+ sender,+ authorization,+ threadId: data.threadId,+ streamId: data.streamId,+ turnId,+ messages: data.messages,+ browsingContext: data.browsingContext,+ modelId: data.modelId,+ onCodeExecutionUpdate: (codeExecutionData: CodeExecutionData) => {+ writer.write({+ type: 'data-code-execution' as const,+ id: `code-execution-${codeExecutionData.executionId}`,+ data: codeExecutionData,+ });+ },+ onCompaction: () => {+ writer.write({+ type: 'data-compaction' as const,+ id: `compaction-${data.threadId}`,+ data: {},+ });+ },+ abortSignal,+ conversationSizeTokens: data.conversationSizeTokens,+ onUserCreditsExhausted: ({+ renewsAt,+ isFallingBackToWorkspaceCredits,+ }) =>+ this.eventPublisherService+ .publish({+ threadId: data.threadId,+ workspaceId: data.workspaceId,+ event: {+ type: 'user-credits-exhausted',+ workspaceMemberId:+ authorization.authContext.workspaceMemberId,+ isFallingBackToWorkspaceCredits,+ renewsAt: renewsAt.toISOString(),+ },+ })+ .catch((error: unknown) =>+ this.logger.warn(+ `Could not publish the user credits notice for stream ${data.streamId}: ${formatErrorWithCause(error)}`,+ ),+ ),+ });++ return {+ source: {+ parts: turn.stream.stream,+ modelConfig: turn.modelConfig,+ getStreamError: turn.getStreamError,+ pendingWrites: titlePromise.then((generatedTitle) => {+ if (generatedTitle) {+ writer.write({+ type: 'data-thread-title' as const,+ id: `thread-title-${data.threadId}`,+ data: { title: generatedTitle },+ });+ }+ }),+ },+ getStopReason: turn.getStopReason,+ };+ },+ });++ const stopReason = result.started.getStopReason();++ this.logTurnWithoutText({+ result,+ stopReason,+ threadId: data.threadId,+ workspaceId: data.workspaceId,+ modelId: turnModelId,+ });++ const finish = await this.turnStreamService.finishTurn({+ claim,+ turnId,+ result,+ modelId: turnModelId,+ stopReason,+ });++ if (isDefined(finish)) {+ this.recordTurnOutcome(finish.outcome, turnModelId);++ if (isDefined(finish.threadBefore)) {+ await this.agentChatService.notifyThreadUsageUpdated({+ threadBefore: finish.threadBefore,+ workspaceMemberId: authorization.authContext.workspaceMemberId,+ workspaceId: data.workspaceId,+ });}}++ // a worker shutting down interrupted the turn, which fails rather than ends as stopped+ if (abortSignal.reason instanceof AiException) {+ throw abortSignal.reason;+ }++ if (isDefined(result.streamError)) {+ throw result.streamError;+ }++ await this.turnStreamService.publishTurnEnd({+ claim,+ assistantMessageId: result.assistantMessageId,+ stopReason,+ });}// resolve auto-select ids here so turn-start and outcome metrics carry the same model label@@ -357,592 +395,29 @@ export class StreamAgentChatJob {}}- private async buildAndPublishStream({- workspace,- data,- turnId,- sender,- authorization,- titlePromise,- abortSignal,- turnModelId,- }: {- workspace: WorkspaceEntity;- data: StreamAgentChatJobData;- turnId: string;- sender: ChatExecutionOptions['sender'];- authorization: ChatExecutionOptions['authorization'];- titlePromise: Promise<string | null>;- abortSignal: AbortSignal;- turnModelId: string;- }): Promise<void> {- const assistantMessageId = uuidv5(- data.streamId,- ASSISTANT_MESSAGE_ID_NAMESPACE,- );-- return new Promise<void>((resolve, reject) => {- const usageTotals: StreamUsageTotals = {- inputTokens: 0,- outputTokens: 0,- inputCredits: 0,- outputCredits: 0,- cacheReadTokens: 0,- cacheCreationTokens: 0,- conversationSize: 0,- };- let streamError: unknown;- let streamFinishError: unknown;- let getStopReason: () => StepDebitResult['stopReason'] = () => null;-- let persistChain: Promise<void> = Promise.resolve();- let lastCheckpointAt = 0;- let isFinalizingPersist = false;-- const enqueueAssistantPersist = (- persist: () => Promise<void>,- ): Promise<void> => {- persistChain = persistChain.then(persist).catch((error) => {- this.logger.warn(- `Failed to checkpoint assistant message for stream ${data.streamId}: ${formatErrorWithCause(error)}`,- );- });-- return persistChain;- };-- // onEnd fires before the uiStream drains, so message-persisted waits on this- let resolveStreamFinished: () => void;- const streamFinishedPromise = new Promise<void>((res) => {- resolveStreamFinished = res;- });-- abortSignal.addEventListener(- 'abort',- () => {- const reason = abortSignal.reason;-- void streamFinishedPromise.then(() => {- if (reason instanceof AiException) {- reject(reason);- } else {- resolve();- }- });- },- { once: true },- );-- const uiStream = createUIMessageStream<ExtendedUIMessage>({- execute: async ({ writer }) => {- const onCodeExecutionUpdate = (- codeExecutionData: CodeExecutionData,- ) => {- writer.write({- type: 'data-code-execution' as const,- id: `code-execution-${codeExecutionData.executionId}`,- data: codeExecutionData,- });- };-- const onCompaction = () => {- writer.write({- type: 'data-compaction' as const,- id: `compaction-${data.threadId}`,- data: {},- });- };-- const onUserCreditsExhausted: ChatExecutionOptions['onUserCreditsExhausted'] =- ({ renewsAt, isFallingBackToWorkspaceCredits }) =>- this.eventPublisherService- .publish({- threadId: data.threadId,- workspaceId: data.workspaceId,- event: {- type: 'user-credits-exhausted',- workspaceMemberId:- authorization.authContext.workspaceMemberId,- isFallingBackToWorkspaceCredits,- renewsAt: renewsAt.toISOString(),- },- })- .catch((error: unknown) =>- this.logger.warn(- `Could not publish the user credits notice for stream ${data.streamId}: ${formatErrorWithCause(error)}`,- ),- );-- const {- stream,- modelConfig,- getStopReason: getChatStopReason,- getStreamError,- } = await this.chatExecutionService.streamChat({- workspace,- userWorkspaceId: data.userWorkspaceId,- sender,- authorization,- threadId: data.threadId,- streamId: data.streamId,- turnId,- messages: data.messages,- browsingContext: data.browsingContext,- modelId: data.modelId,- onCodeExecutionUpdate,- onCompaction,- abortSignal,- conversationSizeTokens: data.conversationSizeTokens,- onUserCreditsExhausted,- });-- getStopReason = getChatStopReason;-- const titleWritePromise = titlePromise.then((generatedTitle) => {- if (generatedTitle) {- writer.write({- type: 'data-thread-title' as const,- id: `thread-title-${data.threadId}`,- data: { title: generatedTitle },- });- }- });-- writer.merge(- toUIMessageStream({- stream: stream.stream,- onError: formatErrorWithCause,- sendStart: true,- generateMessageId: () => assistantMessageId,- messageMetadata: ({ part }) =>- this.computeMessageMetadata({ part, modelConfig, usageTotals }),- onEnd: async ({ responseMessage, isAborted }) => {- // Rejecting here would race chunks still draining.- try {- streamError ??= getStreamError();- isFinalizingPersist = true;- await persistChain;- const outcome = await this.persistStreamFinish({- assistantMessageId,- streamId: data.streamId,- responseMessage,- isAborted,- streamError,- stopReason: getStopReason(),- threadId: data.threadId,- workspaceId: data.workspaceId,- workspaceMemberId:- authorization.authContext.workspaceMemberId,- usageTotals,- modelConfig,- turnModelId,- turnId,- });-- if (isDefined(outcome)) {- this.recordTurnOutcome(outcome, turnModelId);- }- await titleWritePromise;- } catch (error) {- streamFinishError = error;- } finally {- resolveStreamFinished();- }- },- sendReasoning: true,- }),- );- },- // Errors thrown before the model stream merges never reach onEnd.- onError: (error) => {- streamError = error;- resolveStreamFinished();-- return formatErrorWithCause(error);- },- });-- const [publishStream, checkpointStream] = uiStream.tee();-- void (async () => {- try {- for await (const message of readUIMessageStream<ExtendedUIMessage>({- stream: checkpointStream,- terminateOnError: false,- })) {- if (isFinalizingPersist || message.parts.length === 0) {- continue;- }-- const now = Date.now();-- if (now - lastCheckpointAt < AGENT_CHAT_CHECKPOINT_INTERVAL_MS) {- continue;- }-- lastCheckpointAt = now;- const parts = message.parts;-- void enqueueAssistantPersist(async () => {- if (isFinalizingPersist) {- return;- }-- await this.agentChatService.upsertAssistantMessage({- id: assistantMessageId,- threadId: data.threadId,- turnId,- parts,- workspaceId: data.workspaceId,- });- });- }- } catch {- // best-effort; the authoritative persist runs onEnd- }- })();-- // message-persisted must reach the client after every stream-chunk- void (async () => {- try {- for await (const chunk of publishStream) {- if ((chunk as { type?: string }).type === 'error') {- continue;- }-- await this.eventPublisherService.publish({- threadId: data.threadId,- workspaceId: data.workspaceId,- event: {- type: 'stream-chunk',- chunk: chunk as Record<string, unknown>,- },- });- }-- await streamFinishedPromise;-- if (streamError) {- reject(streamError);- } else if (streamFinishError) {- reject(streamFinishError);- } else if (getStopReason() === 'workspaceAllowance') {- await this.eventPublisherService.publish({- threadId: data.threadId,- workspaceId: data.workspaceId,- event: { type: 'credits-exhausted' },- });- resolve();- } else {- await this.eventPublisherService.publish({- threadId: data.threadId,- workspaceId: data.workspaceId,- event: {- type: 'message-persisted',- messageId: assistantMessageId,- },- });- resolve();- }- } catch (error) {- reject(error);- }- })();- });- }-- private computeMessageMetadata({- part,- modelConfig,- usageTotals,- }: {- part: {- type: string;- usage?: {- inputTokens?: number;- outputTokens?: number;- inputTokenDetails?: { cacheReadTokens?: number };- outputTokenDetails?: { reasoningTokens?: number };- };- providerMetadata?: Record<string, Record<string, unknown> | undefined>;- };- modelConfig: AiModelConfig;- usageTotals: StreamUsageTotals;- }) {- if (part.type === 'finish-step') {- Object.assign(- usageTotals,- addStepToAgentTurnUsage(modelConfig, usageTotals, part),- );- usageTotals.conversationSize = part.usage?.inputTokens ?? 0;- }-- if (part.type === 'start') {- return { startedAt: new Date().toISOString() };- }-- if (part.type === 'finish') {- const finishedAt = new Date().toISOString();-- return {- createdAt: finishedAt,- finishedAt,- usage: {- inputTokens: usageTotals.inputTokens,- outputTokens: usageTotals.outputTokens,- cachedInputTokens: usageTotals.cacheReadTokens,- inputCredits: toDisplayCredits(usageTotals.inputCredits),- outputCredits: toDisplayCredits(usageTotals.outputCredits),- conversationSize: usageTotals.conversationSize,- },- model: {- contextWindowTokens: modelConfig.contextWindowTokens,- },- };- }-- return undefined;- }-- // null on stream error: the catch in handle() already counts that turn- private async persistStreamFinish({- assistantMessageId,- streamId,- responseMessage,- isAborted,- streamError,- stopReason,- threadId,- workspaceId,- workspaceMemberId,- usageTotals,- modelConfig,- turnModelId,- turnId,- }: {- assistantMessageId: string;- streamId: string;- responseMessage: Omit<ExtendedUIMessage, 'id'>;- isAborted: boolean;- streamError: unknown;- stopReason: StepDebitResult['stopReason'];- threadId: string;- workspaceId: string;- workspaceMemberId: string;- usageTotals: StreamUsageTotals;- modelConfig: AiModelConfig;- turnModelId: string;- turnId: string;- }): Promise<AgentChatTurnOutcome | null> {- // A turn that only waits on the member still needs a preview in chat lists- const replyText =- findLastMessageText(responseMessage.parts) ??- findAwaitingCallText(responseMessage.parts);- const hasText = responseMessage.parts.some(- (part) => part.type === 'text' && isNonEmptyString(part.text),- );- const isSilent = isSilentAgentChatMessageParts(responseMessage.parts);-- const awaitingParts = findAwaitingPausingToolParts(responseMessage.parts);- // without an answerable call nothing can resume the turn, so fail rather than wait forever- const isAwaitingAnswer =- awaitingParts.length > 0 &&- awaitingParts.every(({ isAnswerable }) => isAnswerable);-- if ((isAborted || !hasText) && awaitingParts.length === 0 && !isSilent) {- this.logAssistantTurnWithoutText({- responseMessage,- isAborted,- streamError,- stopReason,- hasText,- threadId,- workspaceId,- inputTokens: usageTotals.inputTokens,- modelId: turnModelId,- });- }-- await this.recordTurnUsage({ workspaceId, threadId, turnId, usageTotals });-- if (isDefined(streamError)) {- return null;- }-- const outcome = classifyAgentChatTurnOutcome({- hasText,- isAborted,- isAwaitingUserAnswer: awaitingParts.length > 0,- isSilent,- stopReason,- });-- if (responseMessage.parts.length === 0) {- await this.finishTurn({- workspaceId,- turnId,- threadId,- streamId,- outcome,- turnModelId,- });-- return outcome;- }-- const threadBeforeUsage = await this.threadRepository.findOne(workspaceId, {- where: { id: threadId },- });-- if (!threadBeforeUsage || threadBeforeUsage.deletedAt) {- return resolveSupersededTurnOutcome(outcome);- }-- await this.agentChatService.upsertAssistantMessage({- id: assistantMessageId,- threadId,- turnId,- parts: responseMessage.parts,- workspaceId,- });-- const totalsUpdate = await updateAgentChatThreadUsage({- repository: this.threadRepository,- workspaceId,- threadId,- streamId,- lastMessageText: replyText,- shouldRecordActivity: !isSilent,- usage: {- contextWindowTokens: modelConfig.contextWindowTokens,- conversationSize: usageTotals.conversationSize,- pendingQuestionMessageId: isAwaitingAnswer ? assistantMessageId : null,- },- });-- if (!totalsUpdate.affected && isSilent) {- return resolveSupersededTurnOutcome(outcome);- }-- if (!totalsUpdate.affected) {- // The reply is saved even though a newer stream owns the thread. That- // activity is best-effort and must not turn the saved reply into an error- await this.threadService- .recordThreadActivity({ workspaceId, threadId, text: replyText })- .catch((error: unknown) =>- this.logger.warn(- `Could not record reply activity on thread ${threadId}: ${formatErrorWithCause(error)}`,- ),- );-- return resolveSupersededTurnOutcome(outcome);- }-- if (awaitingParts.length > 0 && !isAwaitingAnswer) {- throw new AiException(- 'A call waiting on the user could not be read',- AiExceptionCode.INVALID_TOOL_CALL_OUTPUT,- );- }-- this.isAwaitingInput = isAwaitingAnswer;-- await this.finishTurn({- workspaceId,- turnId,- threadId,- streamId,- outcome,- turnModelId,- });-- await this.agentChatService.notifyThreadUsageUpdated({- threadBefore: threadBeforeUsage,- workspaceMemberId,- workspaceId,- });-- return outcome;- }-- // Usage is recorded on its own: a failure to count it must not fail a reply already saved- private async recordTurnUsage({- workspaceId,- threadId,- turnId,- usageTotals,- }: {- workspaceId: string;- threadId: string;- turnId: string;- usageTotals: StreamUsageTotals;- }): Promise<void> {- await this.turnRecorderService- .recordUsage({- workspaceId,- threadId,- turnId,- usage: {- inputTokens: usageTotals.inputTokens,- outputTokens: usageTotals.outputTokens,- cacheReadTokens: usageTotals.cacheReadTokens,- cacheCreationTokens: usageTotals.cacheCreationTokens,- inputCredits: usageTotals.inputCredits,- outputCredits: usageTotals.outputCredits,- },- })- .catch((error: unknown) =>- this.logger.warn(- `Could not record the usage of turn ${turnId}: ${formatErrorWithCause(error)}`,- ),- );- }-- private async finishTurn({- workspaceId,- turnId,- threadId,- streamId,- outcome,- turnModelId,- }: {- workspaceId: string;- turnId: string;- threadId: string;- streamId: string;- outcome: AgentChatTurnOutcome;- turnModelId: string;- }): Promise<void> {- await this.turnRecorderService.finish({- workspaceId,- turnId,- ...mapAgentChatTurnOutcomeToTurnStatus(outcome),- modelId: turnModelId,- streamClaim: { threadId, streamId },- });- }-- private logAssistantTurnWithoutText({- responseMessage,- isAborted,- streamError,+ private logTurnWithoutText({+ result: { replyParts, isAborted, streamError, usage },stopReason,- hasText,threadId,workspaceId,- inputTokens,modelId,}: {- responseMessage: Omit<ExtendedUIMessage, 'id'>;- isAborted: boolean;- streamError: unknown;+ result: AgentTurnStreamResult;stopReason: StepDebitResult['stopReason'];- hasText: boolean;threadId: string;workspaceId: string;- inputTokens: number;modelId: string;}): void {+ const hasText = hasReplyText(replyParts);++ if (+ (!isAborted && hasText) ||+ findAwaitingPausingToolParts(replyParts).length > 0 ||+ isSilentAgentChatMessageParts(replyParts)+ ) {+ return;+ }+const reason = isAborted? 'user-cancelled': streamError@@ -963,8 +438,8 @@ export class StreamAgentChatJob {`threadId=${threadId}, workspaceId=${workspaceId}, ` +`isAborted=${isAborted}, stopReason=${stopReason}, hasText=${hasText}, ` +`streamError=${errorDetail}, ` +- `inputTokens=${inputTokens},` +- `responseMessage.parts=${JSON.stringify(responseMessage.parts)}`,+ `inputTokens=${usage.inputTokens},` ++ `responseMessage.parts=${JSON.stringify(replyParts)}`,);if (streamError instanceof Error && isDefined(streamError.stack)) {diff --git a/packages/twenty-server/src/engine/metadata-modules/ai/ai-chat/services/__tests__/agent-chat-streaming.service.claim.spec.ts b/packages/twenty-server/src/engine/metadata-modules/ai/ai-chat/services/__tests__/agent-chat-streaming.service.claim.spec.tsdeleted file mode 100644index 1258863fade..00000000000--- a/packages/twenty-server/src/engine/metadata-modules/ai/ai-chat/services/__tests__/agent-chat-streaming.service.claim.spec.ts+++ /dev/null@@ -1,440 +0,0 @@-import { AgentChatStreamRecoveryService } from 'src/engine/metadata-modules/ai/ai-chat/services/agent-chat-stream-recovery.service';-import { type WorkspaceEntity } from 'src/engine/core-modules/workspace/workspace.entity';-import {- AiException,- AiExceptionCode,-} from 'src/engine/metadata-modules/ai/ai.exception';-import { AgentTurnStatus } from 'src/engine/metadata-modules/ai/ai-history/enums/agent-turn-status.enum';-import { AgentChatStreamingService } from 'src/engine/metadata-modules/ai/ai-chat/services/agent-chat-streaming.service';--describe('AgentChatStreamingService claim & reap', () => {- const workspace = { id: 'workspace-id' } as WorkspaceEntity;-- const idleThread = {- id: 'thread-id',- title: 'Thread',- conversationSize: 0,- activeStreamId: null,- };-- const sendArguments = {- workspaceMemberId: 'member',- userWorkspaceId: 'user-workspace-id',- workspace,- text: 'hello',- browsingContext: null,- };-- const buildService = ({- thread = idleThread,- claimAffected = 1,- queuedMessages = [] as unknown[],- heartbeatAlive = true,- isConversationAwaited = false,- }: {- thread?: typeof idleThread & {- pendingQuestionMessageId?: string;- };- isConversationAwaited?: boolean;- claimAffected?: number;- queuedMessages?: unknown[];- heartbeatAlive?: boolean;- } = {}) => {- const publishedEvents: Array<{ type: string }> = [];- const releaseQuery = jest.fn(async (_sql: string, _parameters: unknown[]) =>- Array.from({ length: claimAffected }, () => ({ id: 'thread-id' })),- );- const claimReleases = () =>- releaseQuery.mock.calls.map(([, parameters]) => {- const [threadId, streamId, , turnError] = parameters as [- string,- string,- string | null,- string | null,- ];-- return {- threadId,- streamId,- turnError: turnError === null ? null : JSON.parse(turnError),- };- });- const threadRepository = {- findOne: jest.fn().mockResolvedValue(thread),- findOneOrFail: jest.fn().mockResolvedValue(thread),- update: jest.fn().mockResolvedValue({ affected: claimAffected }),- query: jest.fn().mockImplementation(async (_workspaceId, work) =>- work({- table: (name: string) => name,- manager: { query: releaseQuery },- }),- ),- };- const messageQueueService = { add: jest.fn().mockResolvedValue(undefined) };- const agentChatService = {- addMessage: jest- .fn()- .mockResolvedValue({ id: 'user-message-id', turnId: 'turn-id' }),- getMessagesForThread: jest.fn().mockResolvedValue([]),- getThreadContexts: jest.fn().mockResolvedValue([]),- getQueuedMessages: jest.fn().mockResolvedValue(queuedMessages),- hasQueuedMessages: jest- .fn()- .mockImplementation(() => Promise.resolve(queuedMessages.length > 0)),- queueMessage: jest.fn().mockResolvedValue({ id: 'queued-message-id' }),- promoteQueuedMessage: jest.fn().mockResolvedValue('turn-id'),- deleteQueuedMessage: jest.fn().mockResolvedValue(true),- };- const threadLifecycleService = {- closePendingQuestion: jest.fn().mockResolvedValue(undefined),- };- const eventPublisherService = {- publish: jest.fn().mockImplementation(({ event }) => {- publishedEvents.push(event);-- return Promise.resolve();- }),- resetStreamState: jest.fn().mockResolvedValue(undefined),- };- const streamHeartbeatService = {- markClaimed: jest.fn().mockResolvedValue(undefined),- isAlive: jest.fn().mockResolvedValue(heartbeatAlive),- clear: jest.fn().mockResolvedValue(undefined),- };-- const metricsService = { incrementCounterBy: jest.fn() };- const streamRecoveryService = new AgentChatStreamRecoveryService(- threadRepository as never,- streamHeartbeatService as never,- eventPublisherService as never,- metricsService as never,- );-- const service = new AgentChatStreamingService(- threadRepository as never,- { find: jest.fn().mockResolvedValue([]) } as never,- messageQueueService as never,- agentChatService as never,- {- notifyThreadActivityUpdated: jest.fn().mockResolvedValue(undefined),- } as never,- eventPublisherService as never,- { signFileByIdUrl: jest.fn() } as never,- streamHeartbeatService as never,- metricsService as never,- streamRecoveryService,- {- authorizeRetry: jest.fn().mockResolvedValue(undefined),- authorize: jest- .fn()- .mockResolvedValue({ authContext: { workspaceMemberId: 'member' } }),- resolveMessage: jest.fn().mockResolvedValue({- sender: {- userWorkspaceId: 'user-workspace-id',- applicationId: null,- },- }),- } as never,- {- findLatestTurn: jest.fn().mockResolvedValue(null),- markRunning: jest.fn().mockResolvedValue(true),- } as never,- {- assertConversationNotSuspended: isConversationAwaited- ? jest- .fn()- .mockRejectedValue(- new AiException(- 'awaited',- AiExceptionCode.THREAD_AWAITING_ANSWER,- ),- )- : jest.fn().mockResolvedValue(undefined),- } as never,- { withThreadLockForMessage: jest.fn(({ work }) => work()) } as never,- threadLifecycleService as never,- );-- return {- service,- streamRecoveryService,- claimReleases,- send: (overrides: { userWorkspaceId?: string } = {}) =>- service.streamAgentChat({- ...sendArguments,- thread: thread as never,- ...overrides,- }),- threadRepository,- messageQueueService,- agentChatService,- threadLifecycleService,- eventPublisherService,- streamHeartbeatService,- publishedEvents,- };- };-- describe('streamAgentChat', () => {- it('announces the participant prompt to the thread before enqueuing the reply', async () => {- const {- send,- agentChatService,- eventPublisherService,- messageQueueService,- } = buildService();- await send({ userWorkspaceId: 'other-participant' });- expect(agentChatService.addMessage).toHaveBeenCalledWith(- expect.objectContaining({ userWorkspaceId: 'other-participant' }),- );- expect(eventPublisherService.publish).toHaveBeenCalledWith({- workspaceId: workspace.id,- threadId: 'thread-id',- event: { type: 'message-persisted', messageId: 'user-message-id' },- });- expect(- eventPublisherService.publish.mock.invocationCallOrder[0],- ).toBeLessThan(messageQueueService.add.mock.invocationCallOrder[0]);- });-- it('claims the thread conditionally before enqueueing', async () => {- const { send, threadRepository, streamHeartbeatService } = buildService();-- const result = await send();-- expect(result.queued).toBe(false);- expect(result).toEqual(- expect.objectContaining({- messageId: 'user-message-id',- turnId: 'turn-id',- }),- );- expect(threadRepository.update).toHaveBeenCalledWith(- 'workspace-id',- expect.objectContaining({ id: 'thread-id' }),- { activeStreamId: expect.any(String) },- );- expect(streamHeartbeatService.markClaimed).toHaveBeenCalled();- expect(- streamHeartbeatService.markClaimed.mock.invocationCallOrder[0],- ).toBeLessThan(threadRepository.update.mock.invocationCallOrder[0]);- });-- it('queues the message when another stream wins the claim race', async () => {- const {- send,- agentChatService,- messageQueueService,- streamHeartbeatService,- } = buildService({- claimAffected: 0,- });-- const result = await send();-- expect(result.queued).toBe(true);- expect(agentChatService.queueMessage).toHaveBeenCalled();- expect(messageQueueService.add).not.toHaveBeenCalled();- expect(streamHeartbeatService.clear).toHaveBeenCalled();- });-- it('queues behind a halted backlog and kicks the drain from the front', async () => {- const { send, agentChatService } = buildService({- queuedMessages: [- {- id: 'older-queued-id',- parts: [{ type: 'text', textContent: 'first in line' }],- },- ],- });-- const result = await send();-- expect(result.queued).toBe(true);- expect(agentChatService.queueMessage).toHaveBeenCalled();- expect(agentChatService.promoteQueuedMessage).toHaveBeenCalledWith(- expect.objectContaining({ messageId: 'older-queued-id' }),- );- });-- const waitingThread = {- ...idleThread,- pendingQuestionMessageId: 'question-message-id',- };-- it('closes the pending call as skipped before streaming, unless an answer holds the stream', async () => {- const { send, threadLifecycleService, messageQueueService } =- buildService({- thread: waitingThread,- });-- const result = await send();-- expect(result.queued).toBe(false);- expect(threadLifecycleService.closePendingQuestion).toHaveBeenCalledWith({- workspaceId: 'workspace-id',- threadId: 'thread-id',- messageId: 'question-message-id',- activeStreamId: null,- turnStatus: AgentTurnStatus.COMPLETED,- });- expect(- threadLifecycleService.closePendingQuestion.mock.invocationCallOrder[0],- ).toBeLessThan(messageQueueService.add.mock.invocationCallOrder[0]);- });-- it('refuses a message while a caller waits on the pending call', async () => {- const {- send,- agentChatService,- threadLifecycleService,- messageQueueService,- } = buildService({- thread: waitingThread,- isConversationAwaited: true,- });-- await expect(send()).rejects.toMatchObject({- code: AiExceptionCode.THREAD_AWAITING_ANSWER,- });- expect(- threadLifecycleService.closePendingQuestion,- ).not.toHaveBeenCalled();- expect(agentChatService.addMessage).not.toHaveBeenCalled();- expect(agentChatService.queueMessage).not.toHaveBeenCalled();- expect(messageQueueService.add).not.toHaveBeenCalled();- });-- it('releases the claim when enqueueing the job fails', async () => {- const {- send,- claimReleases,- messageQueueService,- streamHeartbeatService,- } = buildService();-- messageQueueService.add.mockRejectedValue(new Error('redis down'));-- await expect(send()).rejects.toThrow('redis down');-- expect(claimReleases()).toEqual([- {- threadId: 'thread-id',- streamId: expect.any(String),- turnError: expect.objectContaining({ message: 'redis down' }),- },- ]);- expect(streamHeartbeatService.clear).toHaveBeenCalled();- });- });-- describe('flushNextQueuedMessage', () => {- const queuedMessages = [- {- id: 'queued-message-id',- parts: [{ type: 'text', textContent: 'next' }],- },- ];-- it('keeps queued messages waiting behind a pending call', async () => {- const { service, agentChatService, messageQueueService } = buildService({- queuedMessages,- thread: {- ...idleThread,- pendingQuestionMessageId: 'question-message-id',- },- });-- await service.flushNextQueuedMessage({- threadId: 'thread-id',- workspaceId: 'workspace-id',- });-- expect(agentChatService.promoteQueuedMessage).not.toHaveBeenCalled();- expect(messageQueueService.add).not.toHaveBeenCalled();- });-- it('promotes the next queued message once nothing is pending', async () => {- const { service, agentChatService, messageQueueService } = buildService({- queuedMessages,- });-- await service.flushNextQueuedMessage({- threadId: 'thread-id',- workspaceId: 'workspace-id',- });-- expect(agentChatService.promoteQueuedMessage).toHaveBeenCalledWith(- expect.objectContaining({ messageId: 'queued-message-id' }),- );- expect(messageQueueService.add).toHaveBeenCalled();- });- });-- describe('reapDeadStream', () => {- it('leaves a live stream alone', async () => {- const { streamRecoveryService, threadRepository } = buildService();-- const reaped = await streamRecoveryService.reapDeadStream({- thread: { id: 'thread-id', activeStreamId: 'stream-id' },- workspaceId: 'workspace-id',- });-- expect(reaped).toBeNull();- expect(threadRepository.update).not.toHaveBeenCalled();- });-- it('converts a heartbeat-less claim into a retryable interrupted error', async () => {- const {- streamRecoveryService,- claimReleases,- eventPublisherService,- publishedEvents,- } = buildService({ heartbeatAlive: false });-- const reaped = await streamRecoveryService.reapDeadStream({- thread: { id: 'thread-id', activeStreamId: 'stream-id' },- workspaceId: 'workspace-id',- });-- expect(reaped).toEqual(- expect.objectContaining({ code: AiExceptionCode.STREAM_INTERRUPTED }),- );- expect(claimReleases()).toEqual([- {- threadId: 'thread-id',- streamId: 'stream-id',- turnError: expect.objectContaining({- code: AiExceptionCode.STREAM_INTERRUPTED,- }),- },- ]);- expect(eventPublisherService.resetStreamState).toHaveBeenCalledWith(- 'thread-id',- );- expect(publishedEvents).toContainEqual(- expect.objectContaining({- type: 'stream-error',- code: AiExceptionCode.STREAM_INTERRUPTED,- }),- );- });-- it('does nothing when the claim moved to a newer stream mid-check', async () => {- const { streamRecoveryService, publishedEvents, claimReleases } =- buildService({- heartbeatAlive: false,- claimAffected: 0,- });-- const reaped = await streamRecoveryService.reapDeadStream({- thread: { id: 'thread-id', activeStreamId: 'stream-id' },- workspaceId: 'workspace-id',- });-- expect(reaped).toBeNull();- expect(claimReleases()).toHaveLength(1);- expect(publishedEvents).toHaveLength(0);- });- });-});diff --git a/packages/twenty-server/src/engine/metadata-modules/ai/ai-chat/services/__tests__/agent-chat-streaming.service.opening-turn.spec.ts b/packages/twenty-server/src/engine/metadata-modules/ai/ai-chat/services/__tests__/agent-chat-streaming.service.opening-turn.spec.tsdeleted file mode 100644index ef5aa348dfb..00000000000--- a/packages/twenty-server/src/engine/metadata-modules/ai/ai-chat/services/__tests__/agent-chat-streaming.service.opening-turn.spec.ts+++ /dev/null@@ -1,210 +0,0 @@-import { AgentChatStreamRecoveryService } from 'src/engine/metadata-modules/ai/ai-chat/services/agent-chat-stream-recovery.service';-import { type WorkspaceEntity } from 'src/engine/core-modules/workspace/workspace.entity';-import { AgentChatThreadWorkspaceEntity } from 'src/engine/metadata-modules/ai/ai-history/standard-objects/agent-chat-thread.workspace-entity';-import { AgentChatStreamingService } from 'src/engine/metadata-modules/ai/ai-chat/services/agent-chat-streaming.service';--describe('AgentChatStreamingService.startOpeningTurn', () => {- const workspace = { id: 'workspace-id' } as WorkspaceEntity;-- const thread = {- id: 'thread-id',- title: 'Workspace setup',- conversationSize: 0,- activeStreamId: null,- } as unknown as AgentChatThreadWorkspaceEntity;-- const buildService = ({ claimAffected = 1, hasMessages = false } = {}) => {- const releaseQuery = jest.fn(- async (_sql: string, _parameters: unknown[]) => [{ id: 'thread-id' }],- );- const threadRepository = {- findOne: jest.fn().mockResolvedValue(thread),- update: jest.fn().mockResolvedValue({ affected: claimAffected }),- query: jest.fn().mockImplementation(async (_workspaceId, work) =>- work({- table: (name: string) => name,- manager: { query: releaseQuery },- }),- ),- };- const messageQueueService = { add: jest.fn().mockResolvedValue(undefined) };- const agentChatService = {- hasMessages: jest.fn().mockResolvedValue(hasMessages),- getQueuedMessages: jest.fn().mockResolvedValue([]),- replaceOpeningTurn: jest.fn().mockResolvedValue('opening-turn-id'),- getMessagesForThread: jest.fn().mockResolvedValue([]),- getThreadContexts: jest.fn().mockResolvedValue(['Company: Acme Inc']),- };- const threadService = {- notifyThreadActivityUpdated: jest.fn().mockResolvedValue(undefined),- };- const streamHeartbeatService = {- markClaimed: jest.fn().mockResolvedValue(undefined),- isAlive: jest.fn().mockResolvedValue(true),- clear: jest.fn().mockResolvedValue(undefined),- };- const metricsService = { incrementCounterBy: jest.fn() };- const eventPublisherService = {- publish: jest.fn(),- resetStreamState: jest.fn(),- };-- const service = new AgentChatStreamingService(- threadRepository as never,- { find: jest.fn().mockResolvedValue([]) } as never,- messageQueueService as never,- agentChatService as never,- threadService as never,- eventPublisherService as never,- { signFileByIdUrl: jest.fn() } as never,- streamHeartbeatService as never,- metricsService as never,- new AgentChatStreamRecoveryService(- threadRepository as never,- streamHeartbeatService as never,- eventPublisherService as never,- metricsService as never,- ),- {} as never,- {} as never,- {} as never,- {} as never,- {} as never,- );-- return {- service,- releaseQuery,- threadRepository,- messageQueueService,- agentChatService,- threadService,- streamHeartbeatService,- metricsService,- };- };-- const openingTurnArguments = {- thread,- userWorkspaceId: 'user-workspace-id',- workspaceMemberId: 'member-id',- workspace,- context: 'Company: Acme Inc',- modelId: 'default-fast-model',- };-- it('returns null without starting a turn when the claim is lost', async () => {- const {- service,- agentChatService,- messageQueueService,- streamHeartbeatService,- } = buildService({ claimAffected: 0 });-- const result = await service.startOpeningTurn(openingTurnArguments);-- expect(result).toBeNull();- expect(agentChatService.replaceOpeningTurn).not.toHaveBeenCalled();- expect(messageQueueService.add).not.toHaveBeenCalled();- expect(streamHeartbeatService.clear).toHaveBeenCalled();- });-- it('releases the claim and flushes the queue when the conversation already started', async () => {- const { service, releaseQuery, agentChatService, messageQueueService } =- buildService({ hasMessages: true });-- const result = await service.startOpeningTurn(openingTurnArguments);-- expect(result).toBeNull();- expect(agentChatService.replaceOpeningTurn).not.toHaveBeenCalled();- expect(messageQueueService.add).not.toHaveBeenCalled();- expect(releaseQuery).toHaveBeenCalledWith(- expect.stringContaining('"activeStreamId" = NULL'),- ['thread-id', expect.any(String), null, null],- );- expect(agentChatService.getQueuedMessages).toHaveBeenCalledWith({- threadId: 'thread-id',- workspaceId: 'workspace-id',- });- });-- it('starts over an empty thread and opens a turn on its context alone', async () => {- const {- service,- threadRepository,- agentChatService,- threadService,- messageQueueService,- } = buildService();-- const result = await service.startOpeningTurn(openingTurnArguments);-- expect(agentChatService.replaceOpeningTurn).toHaveBeenCalledWith({- threadId: 'thread-id',- workspaceId: 'workspace-id',- context: 'Company: Acme Inc',- });- expect(messageQueueService.add).toHaveBeenCalledWith(- expect.any(String),- expect.objectContaining({- threadId: 'thread-id',- messages: [- {- id: 'context-0',- role: 'system',- parts: [{ type: 'text', text: 'Company: Acme Inc' }],- },- ],- browsingContext: null,- modelId: 'default-fast-model',- hasTitle: true,- existingTurnId: 'opening-turn-id',- }),- );- expect(messageQueueService.add.mock.calls[0][1]).not.toHaveProperty(- 'messageId',- );- expect(threadService.notifyThreadActivityUpdated).not.toHaveBeenCalled();- expect(result).toEqual({- streamId: expect.any(String),- turnId: 'opening-turn-id',- });- expect(threadRepository.update).toHaveBeenCalledWith(- 'workspace-id',- expect.objectContaining({ id: 'thread-id' }),- { activeStreamId: result?.streamId },- );- });-- it('releases the claim and reports an enqueue failure when adding the job fails', async () => {- const {- service,- releaseQuery,- messageQueueService,- streamHeartbeatService,- metricsService,- } = buildService();-- messageQueueService.add.mockRejectedValue(new Error('redis down'));-- await expect(- service.startOpeningTurn(openingTurnArguments),- ).rejects.toThrow('redis down');-- expect(releaseQuery).toHaveBeenLastCalledWith(- expect.stringContaining('"activeStreamId" = NULL'),- [- 'thread-id',- expect.any(String),- 'failed',- expect.stringContaining('redis down'),- ],- );- expect(streamHeartbeatService.clear).toHaveBeenCalled();- expect(metricsService.incrementCounterBy).toHaveBeenCalledWith(- expect.objectContaining({- attributes: expect.objectContaining({ failure_phase: 'enqueue' }),- }),- );- });-});diff --git a/packages/twenty-server/src/engine/metadata-modules/ai/ai-chat/services/__tests__/agent-chat-streaming.service.retry.spec.ts b/packages/twenty-server/src/engine/metadata-modules/ai/ai-chat/services/__tests__/agent-chat-streaming.service.retry.spec.tsdeleted file mode 100644index 764747bd28a..00000000000--- a/packages/twenty-server/src/engine/metadata-modules/ai/ai-chat/services/__tests__/agent-chat-streaming.service.retry.spec.ts+++ /dev/null@@ -1,285 +0,0 @@-import { AgentChatStreamRecoveryService } from 'src/engine/metadata-modules/ai/ai-chat/services/agent-chat-stream-recovery.service';-import { type WorkspaceEntity } from 'src/engine/core-modules/workspace/workspace.entity';-import { AgentTurnStatus } from 'src/engine/metadata-modules/ai/ai-history/enums/agent-turn-status.enum';-import { AgentMessageRole } from 'src/engine/metadata-modules/ai/ai-history/enums/agent-message-role.enum';-import { AgentMessageStatus } from 'src/engine/metadata-modules/ai/ai-history/enums/agent-message-status.enum';-import { AgentMessageWorkspaceEntity } from 'src/engine/metadata-modules/ai/ai-history/standard-objects/agent-message.workspace-entity';-import { AgentChatThreadWorkspaceEntity } from 'src/engine/metadata-modules/ai/ai-history/standard-objects/agent-chat-thread.workspace-entity';-import { AgentChatStreamingService } from 'src/engine/metadata-modules/ai/ai-chat/services/agent-chat-streaming.service';-import { AiExceptionCode } from 'src/engine/metadata-modules/ai/ai.exception';--describe('AgentChatStreamingService.retryLastFailedTurn', () => {- const workspace = { id: 'workspace-id' } as WorkspaceEntity;-- const failedThread = {- id: 'thread-id',- title: 'Thread title',- conversationSize: 42,- activeStreamId: null,- } as unknown as AgentChatThreadWorkspaceEntity;-- const failedTurn = {- id: 'turn-id',- status: AgentTurnStatus.FAILED,- error: { code: 'STREAM_EXECUTION_FAILED', message: 'Provider timed out' },- };-- const userMessageEntity = {- id: 'user-message-id',- role: AgentMessageRole.USER,- status: AgentMessageStatus.SENT,- parts: [{ type: 'text', textContent: 'hello', orderIndex: 0 }],- createdAt: new Date('2026-01-01T00:00:00.000Z'),- } as unknown as AgentMessageWorkspaceEntity;-- const buildService = ({- thread = failedThread,- latestTurn = failedTurn as typeof failedTurn | null,- turnUserMessage = { id: 'user-message-id' } as { id: string } | null,- threadMessages = [userMessageEntity],- } = {}) => {- const claimQuery = jest.fn(async (_sql: string, _parameters: unknown[]) => [- { id: 'thread-id' },- ]);- const threadRepository = {- findOneOrFail: jest- .fn()- .mockResolvedValue({ workspaceMemberId: 'member' }),- findOne: jest.fn().mockResolvedValue(thread),- update: jest.fn().mockResolvedValue({ affected: 1 }),- query: jest.fn().mockImplementation(async (_workspaceId, work) =>- work({- table: (name: string) => name,- manager: { query: claimQuery },- }),- ),- };- const turnRecorderService = {- findLatestTurn: jest.fn().mockResolvedValue(latestTurn),- markRunning: jest.fn().mockResolvedValue(true),- };- const messageQueueService = { add: jest.fn().mockResolvedValue(undefined) };- const agentChatService = {- deleteAssistantMessagesForTurn: jest.fn().mockResolvedValue(undefined),- getMessagesForThread: jest.fn().mockResolvedValue(threadMessages),- getThreadContexts: jest.fn().mockResolvedValue([]),- };- const threadService = {- getWritableThread: jest- .fn()- .mockImplementation(() => threadRepository.findOne()),- };-- const streamHeartbeatService = {- markClaimed: jest.fn().mockResolvedValue(undefined),- isAlive: jest.fn().mockResolvedValue(true),- clear: jest.fn().mockResolvedValue(undefined),- };-- const metricsService = { incrementCounterBy: jest.fn() };-- const eventPublisherService = {- publish: jest.fn(),- resetStreamState: jest.fn(),- };-- const service = new AgentChatStreamingService(- threadRepository as never,- { find: jest.fn() } as never,- messageQueueService as never,- agentChatService as never,- threadService as never,- eventPublisherService as never,- { signFileByIdUrl: jest.fn() } as never,- streamHeartbeatService as never,- metricsService as never,- new AgentChatStreamRecoveryService(- threadRepository as never,- streamHeartbeatService as never,- eventPublisherService as never,- metricsService as never,- ),- {- authorizeRetry: jest- .fn()- .mockResolvedValue({ message: turnUserMessage }),- } as never,- turnRecorderService as never,- {} as never,- {} as never,- {} as never,- );-- return {- service,- claimQuery,- turnRecorderService,- threadRepository,- messageQueueService,- agentChatService,- threadService,- };- };-- const retryArguments = {- workspaceMemberId: 'member',- threadId: 'thread-id',- userWorkspaceId: 'user-workspace-id',- workspace,- };-- it('rejects a shared viewer without deleting messages or scheduling execution', async () => {- const {- service,- threadRepository,- messageQueueService,- agentChatService,- threadService,- } = buildService();- threadService.getWritableThread.mockRejectedValue({- code: AiExceptionCode.THREAD_NOT_FOUND,- });- await expect(- service.retryLastFailedTurn(retryArguments),- ).rejects.toMatchObject({ code: AiExceptionCode.THREAD_NOT_FOUND });- expect(threadService.getWritableThread).toHaveBeenCalledWith({- workspaceId: workspace.id,- threadId: retryArguments.threadId,- workspaceMemberId: retryArguments.workspaceMemberId,- });- expect(threadRepository.update).not.toHaveBeenCalled();- expect(- agentChatService.deleteAssistantMessagesForTurn,- ).not.toHaveBeenCalled();- expect(messageQueueService.add).not.toHaveBeenCalled();- });-- it('rejects when the latest turn did not fail', async () => {- const { service, messageQueueService } = buildService({- latestTurn: { ...failedTurn, status: AgentTurnStatus.COMPLETED },- });-- await expect(- service.retryLastFailedTurn(retryArguments),- ).rejects.toMatchObject({- code: AiExceptionCode.NO_FAILED_TURN_TO_RETRY,- });- expect(messageQueueService.add).not.toHaveBeenCalled();- });-- it('rejects when a stream is already active', async () => {- const { service, messageQueueService } = buildService({- thread: { ...failedThread, activeStreamId: 'stream-id' },- });-- await expect(- service.retryLastFailedTurn(retryArguments),- ).rejects.toMatchObject({- code: AiExceptionCode.NO_FAILED_TURN_TO_RETRY,- });- expect(messageQueueService.add).not.toHaveBeenCalled();- });-- it('drops the failed output, re-enqueues the turn, and runs it again', async () => {- const {- service,- threadRepository,- messageQueueService,- agentChatService,- turnRecorderService,- } = buildService();-- const result = await service.retryLastFailedTurn({- ...retryArguments,- modelId: 'model-id',- });-- expect(- agentChatService.deleteAssistantMessagesForTurn,- ).toHaveBeenCalledWith({ turnId: 'turn-id', workspaceId: 'workspace-id' });- expect(messageQueueService.add).toHaveBeenCalledWith(- expect.any(String),- expect.objectContaining({- threadId: 'thread-id',- existingTurnId: 'turn-id',- lastUserMessageText: 'hello',- modelId: 'model-id',- hasTitle: true,- conversationSizeTokens: 42,- }),- );- expect(threadRepository.update).toHaveBeenCalledWith(- 'workspace-id',- expect.objectContaining({ id: 'thread-id' }),- { activeStreamId: result.streamId },- );- expect(turnRecorderService.markRunning).toHaveBeenCalledWith({- workspaceId: 'workspace-id',- turnId: 'turn-id',- streamClaim: { threadId: 'thread-id', streamId: result.streamId },- });- expect(result.messageId).toBe('user-message-id');- expect(result.turnId).toBe('turn-id');- });-- it('retries a turn the agent opened, which has no user message', async () => {- const { service, messageQueueService, agentChatService } = buildService({- turnUserMessage: null,- threadMessages: [],- });- agentChatService.getThreadContexts.mockResolvedValue(['Company: Acme Inc']);-- const result = await service.retryLastFailedTurn(retryArguments);-- expect(- agentChatService.deleteAssistantMessagesForTurn,- ).toHaveBeenCalledWith({ turnId: 'turn-id', workspaceId: 'workspace-id' });- expect(messageQueueService.add).toHaveBeenCalledWith(- expect.any(String),- expect.objectContaining({- messages: [- {- id: 'context-0',- role: 'system',- parts: [{ type: 'text', text: 'Company: Acme Inc' }],- },- ],- existingTurnId: 'turn-id',- lastUserMessageText: '',- }),- );- expect(result.messageId).toBeNull();- expect(result.turnId).toBe('turn-id');- });-- it('does not delete another participant’s output if the latest turn changes while claiming a retry', async () => {- const {- service,- agentChatService,- messageQueueService,- claimQuery,- turnRecorderService,- } = buildService();- turnRecorderService.findLatestTurn- .mockResolvedValueOnce(failedTurn)- .mockResolvedValueOnce({ ...failedTurn, id: 'another-turn' });- await expect(- service.retryLastFailedTurn(retryArguments),- ).rejects.toMatchObject({ code: AiExceptionCode.NO_FAILED_TURN_TO_RETRY });- expect(- agentChatService.deleteAssistantMessagesForTurn,- ).not.toHaveBeenCalled();- expect(messageQueueService.add).not.toHaveBeenCalled();- expect(turnRecorderService.markRunning).not.toHaveBeenCalled();-- const [releaseSql, releaseParameters] = claimQuery.mock.calls[- claimQuery.mock.calls.length - 1- ] as [string, unknown[]];-- expect(releaseSql).toContain('"activeStreamId" = NULL');- expect(releaseParameters.slice(0, 2)).toEqual([- 'thread-id',- expect.any(String),- ]);- });-});diff --git a/packages/twenty-server/src/engine/metadata-modules/ai/ai-chat/services/__tests__/agent-chat-streaming.service.sender.spec.ts b/packages/twenty-server/src/engine/metadata-modules/ai/ai-chat/services/__tests__/agent-chat-streaming.service.sender.spec.tsdeleted file mode 100644index 8a58debd258..00000000000--- a/packages/twenty-server/src/engine/metadata-modules/ai/ai-chat/services/__tests__/agent-chat-streaming.service.sender.spec.ts+++ /dev/null@@ -1,182 +0,0 @@-import {- AiException,- AiExceptionCode,-} from 'src/engine/metadata-modules/ai/ai.exception';-import { AgentChatStreamingService } from 'src/engine/metadata-modules/ai/ai-chat/services/agent-chat-streaming.service';-import { AgentChatStreamRecoveryService } from 'src/engine/metadata-modules/ai/ai-chat/services/agent-chat-stream-recovery.service';-import {- AuthException,- AuthExceptionCode,-} from 'src/engine/core-modules/auth/auth.exception';--const queued = {- id: 'message-b',- senderUserWorkspaceId: 'participant-b',- parts: [{ type: 'text', textContent: 'B’s request' }],-};-const build = () => {- const threads = {- findOne: jest.fn().mockResolvedValue({ id: 'thread' }),- findOneOrFail: jest- .fn()- .mockResolvedValue({ id: 'thread', conversationSize: 0 }),- update: jest.fn().mockResolvedValue({ affected: 1 }),- };- const queue = { add: jest.fn().mockResolvedValue(undefined) };- const chat = {- getQueuedMessages: jest.fn().mockResolvedValue([queued]),- promoteQueuedMessage: jest.fn().mockResolvedValue('turn-b'),- getMessagesForThread: jest.fn().mockResolvedValue([]),- getThreadContexts: jest.fn().mockResolvedValue([]),- deleteQueuedMessage: jest.fn().mockResolvedValue(true),- };- const actors = {- resolveMessage: jest.fn().mockResolvedValue({- sender: { userWorkspaceId: 'participant-b', applicationId: null },- }),- authorize: jest- .fn()- .mockResolvedValue({ authContext: { workspaceMemberId: 'member' } }),- };- const heartbeat = { markClaimed: jest.fn(), clear: jest.fn() };- const events = { publish: jest.fn() };- const metrics = { incrementCounterBy: jest.fn() };- const service = new AgentChatStreamingService(- threads as never,- {} as never,- queue as never,- chat as never,- {} as never,- events as never,- {} as never,- heartbeat as never,- metrics as never,- new AgentChatStreamRecoveryService(- threads as never,- heartbeat as never,- events as never,- metrics as never,- ),- actors as never,- {} as never,- {} as never,- {} as never,- {} as never,- );- return { service, threads, queue, chat, actors, heartbeat };-};-const args = { workspaceId: 'workspace', threadId: 'thread' };--describe('Sender-aware queue draining', () => {- it('starts the next turn as its saved sender', async () => {- const { service, queue, chat } = build();- await service.flushNextQueuedMessage(args);- expect(chat.getMessagesForThread).toHaveBeenCalledWith(- expect.objectContaining({ workspaceMemberId: 'member' }),- );- expect(queue.add).toHaveBeenCalledWith(- expect.any(String),- expect.objectContaining({- userWorkspaceId: 'participant-b',- messageId: 'message-b',- existingTurnId: 'turn-b',- }),- );- });- it('preserves a non-owner request when a worker cannot yet authorize thread access', async () => {- const { service, actors, chat, queue } = build();- actors.authorize.mockRejectedValue(- new AiException('Thread not found', AiExceptionCode.THREAD_NOT_FOUND),- );- await service.flushNextQueuedMessage(args);- expect(chat.deleteQueuedMessage).not.toHaveBeenCalled();- expect(chat.promoteQueuedMessage).not.toHaveBeenCalled();- expect(queue.add).not.toHaveBeenCalled();- });-- it('removes revoked queued work before promoting or invoking it', async () => {- const { service, actors, chat, queue } = build();- actors.authorize.mockRejectedValue(- new AuthException(- 'Membership removed',- AuthExceptionCode.UNAUTHENTICATED,- ),- );- await service.flushNextQueuedMessage(args);- expect(chat.deleteQueuedMessage).toHaveBeenCalledWith({- workspaceId: 'workspace',- messageId: 'message-b',- });- expect(chat.promoteQueuedMessage).not.toHaveBeenCalled();- expect(queue.add).not.toHaveBeenCalled();- });- it('continues with an authorized participant after a revoked queue entry', async () => {- const { service, actors, chat, queue } = build();- chat.getQueuedMessages.mockResolvedValue([- { ...queued, id: 'revoked' },- queued,- ]);- actors.authorize.mockRejectedValueOnce(- new AuthException(- 'Membership removed',- AuthExceptionCode.UNAUTHENTICATED,- ),- );- await service.flushNextQueuedMessage(args);- expect(chat.deleteQueuedMessage).toHaveBeenCalledWith({- workspaceId: 'workspace',- messageId: 'revoked',- });- expect(queue.add).toHaveBeenCalledWith(- expect.any(String),- expect.objectContaining({- messageId: 'message-b',- userWorkspaceId: 'participant-b',- }),- );- });- it('preserves the queue when authorization fails because of infrastructure', async () => {- const { service, actors, chat, queue } = build();- actors.authorize.mockRejectedValue(new Error('Database unavailable'));- await expect(service.flushNextQueuedMessage(args)).rejects.toThrow(- 'Database unavailable',- );- expect(chat.deleteQueuedMessage).not.toHaveBeenCalled();- expect(queue.add).not.toHaveBeenCalled();- });- it('does not enqueue when another worker wins message promotion', async () => {- const { service, chat, queue, heartbeat } = build();- chat.promoteQueuedMessage.mockResolvedValue(null);- await service.flushNextQueuedMessage(args);- expect(queue.add).not.toHaveBeenCalled();- expect(heartbeat.clear).toHaveBeenCalled();- });- it('opens the conversation with the thread contexts as system messages', async () => {- const { service, chat, queue } = build();- const message = (id: string, role: string, createdAt: string) => ({- id,- role,- createdAt,- parts: [{ type: 'text', textContent: id }],- });- chat.getMessagesForThread.mockResolvedValue([- message('inbox-message', 'assistant', '2026-01-01T11:00:01.000Z'),- message('follow-up', 'user', '2026-01-01T12:00:00.000Z'),- ]);- chat.getThreadContexts.mockResolvedValue([- 'Billing app started this conversation.',- ]);- await service.flushNextQueuedMessage(args);- const { messages } = queue.add.mock.calls[0][1];- expect(messages.map(({ id }: { id: string }) => id)).toEqual([- 'context-0',- 'inbox-message',- 'follow-up',- ]);- expect(messages[0]).toEqual({- id: 'context-0',- role: 'system',- parts: [{ type: 'text', text: 'Billing app started this conversation.' }],- });- });-});diff --git a/packages/twenty-server/src/engine/metadata-modules/ai/ai-chat/services/agent-chat-stream-recovery.service.ts b/packages/twenty-server/src/engine/metadata-modules/ai/ai-chat/services/agent-chat-stream-recovery.service.tsindex f8ff9050441..0020172d90f 100644--- a/packages/twenty-server/src/engine/metadata-modules/ai/ai-chat/services/agent-chat-stream-recovery.service.ts+++ b/packages/twenty-server/src/engine/metadata-modules/ai/ai-chat/services/agent-chat-stream-recovery.service.ts@@ -1,4 +1,5 @@import { Injectable, Logger } from '@nestjs/common';+import { type FindOptionsWhere, IsNull } from 'typeorm';import { isDefined, isNonEmptyString } from 'twenty-shared/utils';import { MetricsService } from 'src/engine/core-modules/metrics/metrics.service';@@ -13,7 +14,7 @@ import {} from 'src/engine/metadata-modules/ai/ai-history/utils/map-error-to-stream-error.util';import { AgentTurnStatus } from 'src/engine/metadata-modules/ai/ai-history/enums/agent-turn-status.enum';import { AgentHistoryRepository } from 'src/engine/metadata-modules/ai/ai-history/repositories/agent-history-repository';-import { buildReleaseStreamClaimQuery } from 'src/engine/metadata-modules/ai/ai-history/utils/build-release-stream-claim-query.util';+import { AgentTurnRecorderService } from 'src/engine/metadata-modules/ai/ai-history/services/agent-turn-recorder.service';import { AiExceptionCode } from 'src/engine/metadata-modules/ai/ai.exception';import { InjectAgentHistoryRepository } from 'src/engine/metadata-modules/ai/ai-history/repositories/inject-agent-history-repository.decorator';@@ -34,6 +35,7 @@ export class AgentChatStreamRecoveryService {private readonly streamHeartbeatService: AgentChatStreamHeartbeatService,private readonly eventPublisherService: AgentChatEventPublisherService,private readonly metricsService: MetricsService,+ private readonly turnRecorderService: AgentTurnRecorderService,) {}async releaseStreamClaim({@@ -62,32 +64,44 @@ export class AgentChatStreamRecoveryService {return isReleased;}- // The error belongs to the turn the stream was running, and is only written- // by the caller still holding the claim- private async clearStreamClaim({+ // the error belongs to the turn the stream was running, and only the caller still holding the claim writes it+ private clearStreamClaim({+ turnError,+ ...claim+ }: AgentChatStreamClaim & {+ turnError?: StreamErrorPayload;+ }): Promise<boolean> {+ return this.turnRecorderService.releaseStreamClaim({+ ...claim,+ endRunningTurn: isDefined(turnError)+ ? { status: AgentTurnStatus.FAILED, error: turnError }+ : undefined,+ });+ }++ async tryClaimStream({threadId,workspaceId,streamId,- turnError,+ where = {},}: AgentChatStreamClaim & {- turnError?: StreamErrorPayload;+ where?: FindOptionsWhere<AgentChatThreadWorkspaceEntity>;}): Promise<boolean> {- return this.threadRepository.query(+ await this.streamHeartbeatService.markClaimed(streamId);++ const claim = await this.threadRepository.update(workspaceId,- async ({ manager, table }) => {- const releasedThreads = await manager.query<{ id: string }[]>(- buildReleaseStreamClaimQuery({ table }),- [- threadId,- streamId,- isDefined(turnError) ? AgentTurnStatus.FAILED : null,- isDefined(turnError) ? JSON.stringify(turnError) : null,- ],- );-- return releasedThreads.length > 0;- },+ { id: threadId, activeStreamId: IsNull(), ...where },+ { activeStreamId: streamId },);++ if (!claim.affected) {+ await this.streamHeartbeatService.clear(streamId);++ return false;+ }++ return true;}async failStream({diff --git a/packages/twenty-server/src/engine/metadata-modules/ai/ai-chat/services/agent-chat-streaming.service.ts b/packages/twenty-server/src/engine/metadata-modules/ai/ai-chat/services/agent-chat-streaming.service.tsindex 5e6746e8b03..3777004ea86 100644--- a/packages/twenty-server/src/engine/metadata-modules/ai/ai-chat/services/agent-chat-streaming.service.ts+++ b/packages/twenty-server/src/engine/metadata-modules/ai/ai-chat/services/agent-chat-streaming.service.ts@@ -15,14 +15,12 @@ import {type ExtendedFileUIPart,type ExtendedUIMessage,type ExtendedUIMessagePart,- isExtendedFileUIPart,} from 'twenty-shared/ai';import { FileFolder } from 'twenty-shared/types';-import { isDefined } from 'twenty-shared/utils';-import { type FindOptionsWhere, In, IsNull, Like } from 'typeorm';+import { isDefined, isNonEmptyArray } from 'twenty-shared/utils';+import { In, Like } from 'typeorm';import { FileEntity } from 'src/engine/core-modules/file/entities/file.entity';-import { FileUrlService } from 'src/engine/core-modules/file/file-url/file-url.service';import { InjectMessageQueue } from 'src/engine/core-modules/message-queue/decorators/message-queue.decorator';import { MessageQueue } from 'src/engine/core-modules/message-queue/message-queue.constants';import { MessageQueueService } from 'src/engine/core-modules/message-queue/services/message-queue.service';@@ -31,7 +29,6 @@ import { MetricsKeys } from 'src/engine/core-modules/metrics/types/metrics-keys.import { type WorkspaceEntity } from 'src/engine/core-modules/workspace/workspace.entity';import { AgentMessageRole } from 'src/engine/metadata-modules/ai/ai-history/enums/agent-message-role.enum';import { AgentMessageStatus } from 'src/engine/metadata-modules/ai/ai-history/enums/agent-message-status.enum';-import { AgentRunConversationService } from 'src/engine/metadata-modules/ai/ai-agent-execution/services/agent-run-conversation.service';import { AgentRunSuspensionService } from 'src/engine/metadata-modules/ai/ai-agent-execution/services/agent-run-suspension.service';import { mapDBPartsToUIMessageParts } from 'src/engine/metadata-modules/ai/ai-history/utils/map-db-parts-to-ui-message-parts.util';import { type BrowsingContextType } from 'src/engine/metadata-modules/ai/ai-agent/types/browsing-context.type';@@ -42,6 +39,7 @@ import { AgentChatEventPublisherService } from 'src/engine/metadata-modules/ai/aimport { AgentChatStreamHeartbeatService } from 'src/engine/metadata-modules/ai/ai-chat/services/agent-chat-stream-heartbeat.service';import { AgentChatService } from 'src/engine/metadata-modules/ai/ai-chat/services/agent-chat.service';import { AgentChatThreadService } from 'src/engine/metadata-modules/ai/ai-chat/services/agent-chat-thread.service';+import { AgentConversationReaderService } from 'src/engine/metadata-modules/ai/ai-history/services/agent-conversation-reader.service';import { AgentChatThreadLifecycleService } from 'src/engine/metadata-modules/ai/ai-chat/services/agent-chat-thread-lifecycle.service';import { AiChatFileAttachment } from 'src/engine/metadata-modules/ai/ai-chat/types/ai-chat-file-attachment.type';import { formatErrorWithCause } from 'src/engine/metadata-modules/ai/ai-chat/utils/format-error-with-cause.util';@@ -86,53 +84,18 @@ export class AgentChatStreamingService {private readonly agentChatService: AgentChatService,private readonly threadService: AgentChatThreadService,private readonly eventPublisherService: AgentChatEventPublisherService,- private readonly fileUrlService: FileUrlService,+ private readonly conversationReaderService: AgentConversationReaderService,private readonly streamHeartbeatService: AgentChatStreamHeartbeatService,private readonly metricsService: MetricsService,private readonly streamRecoveryService: AgentChatStreamRecoveryService,private readonly actorService: AgentChatActorService,private readonly turnRecorderService: AgentTurnRecorderService,private readonly agentRunSuspensionService: AgentRunSuspensionService,- private readonly agentRunConversationService: AgentRunConversationService,private readonly threadLifecycleService: AgentChatThreadLifecycleService,) {}- async tryClaimStream({- threadId,- workspaceId,- streamId,- where = {},- }: AgentChatStreamClaim & {- where?: FindOptionsWhere<AgentChatThreadWorkspaceEntity>;- }): Promise<boolean> {- await this.streamHeartbeatService.markClaimed(streamId);-- const claim = await this.threadRepository.update(- workspaceId,- { id: threadId, activeStreamId: IsNull(), ...where },- { activeStreamId: streamId },- );-- if (!claim.affected) {- await this.streamHeartbeatService.clear(streamId);-- return false;- }-- return true;- }-- // a message that lands while a run goes on in the conversation would be read by that run once it- // goes on after a wait, so it is let in only between runs- streamAgentChat(options: StreamAgentChatOptions) {- return this.agentRunConversationService.withThreadLockForMessage({- workspaceId: options.workspace.id,- threadId: options.thread.id,- work: () => this.startOrQueueMessage(options),- });- }-- private async startOrQueueMessage({+ // a message sent while a reply or a run streams in the conversation waits in its queue+ async streamAgentChat({thread,userWorkspaceId,workspaceMemberId,@@ -149,16 +112,25 @@ export class AgentChatStreamingService {const threadId = thread.id;const workspaceId = workspace.id;- const [, hasQueuedBacklog] = await Promise.all([+ const [, hasQueuedBacklog, suspendedRun] = await Promise.all([this.settlePendingToolCallsBeforeSending({ thread, workspaceId }),this.agentChatService.hasQueuedMessages({ threadId, workspaceId }),+ this.agentRunSuspensionService.findSuspendedRun({+ workspaceId,+ threadId,+ }),]);const streamId = generateId();const claimed =!hasQueuedBacklog &&- (await this.tryClaimStream({ threadId, workspaceId, streamId }));+ !isDefined(suspendedRun) &&+ (await this.streamRecoveryService.tryClaimStream({+ threadId,+ workspaceId,+ streamId,+ }));if (!claimed) {const queuedMessage = await this.agentChatService.queueMessage({@@ -171,7 +143,14 @@ export class AgentChatStreamingService {workspaceMemberId,});- if (hasQueuedBacklog) {+ if (isDefined(suspendedRun)) {+ await this.agentRunSuspensionService.scheduleContinuation({+ workspaceId,+ threadId,+ wakeUpId: suspendedRun.id,+ outcome: { type: 'MESSAGE_RECEIVED' },+ });+ } else if (hasQueuedBacklog) {await this.flushNextQueuedMessage({ threadId, workspaceId });}@@ -237,7 +216,7 @@ export class AgentChatStreamingService {);}- saveMessageWithoutReply({+ async saveMessageWithoutReply({thread,userWorkspaceId,workspaceMemberId,@@ -258,43 +237,37 @@ export class AgentChatStreamingService {const threadId = thread.id;const workspaceId = workspace.id;- return this.agentRunConversationService.withThreadLockForMessage({+ const fileParts = await this.buildFilePartsFromAttachments(+ fileAttachments,workspaceId,- threadId,- work: async () => {- const fileParts = await this.buildFilePartsFromAttachments(- fileAttachments,- workspaceId,- );-- const savedMessage = await this.agentChatService.addMessage({- threadId,- userWorkspaceId,- id: messageId,- uiMessage: {- role: AgentMessageRole.USER,- parts: [{ type: 'text' as const, text }, ...fileParts],- },- turnStatus: AgentTurnStatus.COMPLETED,- workspaceId,- });+ );- await this.eventPublisherService.publish({- workspaceId,- threadId,- event: { type: 'message-added', messageId: savedMessage.id },- });+ const savedMessage = await this.agentChatService.addMessage({+ threadId,+ userWorkspaceId,+ id: messageId,+ uiMessage: {+ role: AgentMessageRole.USER,+ parts: [{ type: 'text' as const, text }, ...fileParts],+ },+ turnStatus: AgentTurnStatus.COMPLETED,+ workspaceId,+ });- await this.threadService.notifyThreadActivityUpdated({- threadId,- workspaceMemberId,- workspaceId,- text,- });+ await this.eventPublisherService.publish({+ workspaceId,+ threadId,+ event: { type: 'message-added', messageId: savedMessage.id },+ });- return { messageId: savedMessage.id };- },+ await this.threadService.notifyThreadActivityUpdated({+ threadId,+ workspaceMemberId,+ workspaceId,+ text,});++ return { messageId: savedMessage.id };}// the agent speaks first, from a context in place of a user message@@ -317,7 +290,7 @@ export class AgentChatStreamingService {const workspaceId = workspace.id;const streamId = generateId();- const claimed = await this.tryClaimStream({+ const claimed = await this.streamRecoveryService.tryClaimStream({threadId,workspaceId,streamId,@@ -413,7 +386,7 @@ export class AgentChatStreamingService {const streamId = generateId();- const claimed = await this.tryClaimStream({+ const claimed = await this.streamRecoveryService.tryClaimStream({threadId,workspaceId,streamId,@@ -535,19 +508,34 @@ export class AgentChatStreamingService {select: ['id', 'deletedAt', 'pendingQuestionMessageId'],});- // queued messages wait behind a pending tool call: they follow the answer rather than replace it- if (- !threadStatus ||- threadStatus.deletedAt ||- isDefined(threadStatus.pendingQuestionMessageId)- ) {+ if (!threadStatus || threadStatus.deletedAt) {return;}- const queuedMessages = await this.agentChatService.getQueuedMessages({- threadId,- workspaceId,- });+ const [queuedMessages, suspendedRun] = await Promise.all([+ this.agentChatService.getQueuedMessages({ threadId, workspaceId }),+ this.agentRunSuspensionService.findSuspendedRun({+ workspaceId,+ threadId,+ }),+ ]);++ // a run that went on to ask something reads what was sent meanwhile, as a message sent now would skip its question+ if (isNonEmptyArray(queuedMessages) && isDefined(suspendedRun)) {+ await this.agentRunSuspensionService.scheduleContinuation({+ workspaceId,+ threadId,+ wakeUpId: suspendedRun.id,+ outcome: { type: 'MESSAGE_RECEIVED' },+ });++ return;+ }++ // queued messages wait behind a pending tool call: they follow the answer rather than replace it+ if (isDefined(threadStatus.pendingQuestionMessageId)) {+ return;+ }let nextQueued: (typeof queuedMessages)[number] | undefined;let userWorkspaceId: string | undefined;@@ -623,7 +611,7 @@ export class AgentChatStreamingService {const streamId = generateId();- const claimed = await this.tryClaimStream({+ const claimed = await this.streamRecoveryService.tryClaimStream({threadId,workspaceId,streamId,@@ -686,8 +674,8 @@ export class AgentChatStreamingService {);}- // a message sent while the agent waits on a person closes its pending calls as skipped, so the model sees why.- // An answer holding the stream keeps them, and a chat message never closes calls a run or a caller waits on+ // a message sent while an agent waits on a person closes its pending calls as skipped, so the model sees why.+ // A call a caller such as a workflow step posted waits on its own answer, so it refuses the messageprivate async settlePendingToolCallsBeforeSending({thread,workspaceId,@@ -700,7 +688,7 @@ export class AgentChatStreamingService {}): Promise<void> {const messageId = thread.pendingQuestionMessageId;- await this.agentRunSuspensionService.assertConversationNotSuspended({+ await this.agentRunSuspensionService.assertNoPostedCallAwaited({workspaceId,threadId: thread.id,});@@ -808,24 +796,10 @@ export class AgentChatStreamingService {sentMessages.map(async (message) => ({id: message.id,role: message.role as 'user' | 'assistant' | 'system',- parts: await Promise.all(- mapDBPartsToUIMessageParts(message.parts ?? []).map(async (part) => {- if (isExtendedFileUIPart(part as Record<string, unknown>)) {- const filePart = part as ExtendedFileUIPart;-- return {- ...filePart,- url: await this.fileUrlService.signFileByIdUrl({- fileId: filePart.fileId,- workspaceId,- fileFolder: FileFolder.AgentChat,- }),- } as ExtendedFileUIPart;- }-- return part;- }),- ),+ parts: await this.conversationReaderService.signFileParts({+ workspaceId,+ parts: mapDBPartsToUIMessageParts(message.parts ?? []),+ }),metadata: {createdAt: new Date(message.createdAt).toISOString(),senderWorkspaceMemberId: message.senderWorkspaceMemberId,diff --git a/packages/twenty-server/src/engine/metadata-modules/ai/ai-chat/services/agent-chat.service.ts b/packages/twenty-server/src/engine/metadata-modules/ai/ai-chat/services/agent-chat.service.tsindex 05e4c93cd2e..dbe26f94681 100644--- a/packages/twenty-server/src/engine/metadata-modules/ai/ai-chat/services/agent-chat.service.ts+++ b/packages/twenty-server/src/engine/metadata-modules/ai/ai-chat/services/agent-chat.service.ts@@ -23,7 +23,6 @@ import { AgentTurnStatus } from 'src/engine/metadata-modules/ai/ai-history/enumsimport { buildEndWaitingAgentTurnQuery } from 'src/engine/metadata-modules/ai/ai-history/utils/build-end-waiting-agent-turn-query.util';import { buildActorMetadataFromAuthContext } from 'src/engine/core-modules/actor/utils/build-actor-metadata-from-auth-context.util';import { AgentTurnWorkspaceEntity } from 'src/engine/metadata-modules/ai/ai-history/standard-objects/agent-turn.workspace-entity';-import { mapUIMessagePartsToDBParts } from 'src/engine/metadata-modules/ai/ai-history/utils/map-ui-message-parts-to-db-parts.util';import { findAwaitingPausingTool } from 'src/engine/metadata-modules/ai/ai-agent-execution/pausing-tools/utils/find-awaiting-pausing-tool.util';import { AgentChatThreadWorkspaceEntity } from 'src/engine/metadata-modules/ai/ai-history/standard-objects/agent-chat-thread.workspace-entity';import { InjectWorkspaceScopedRepository } from 'src/engine/twenty-orm/workspace-scoped-repository/inject-workspace-scoped-repository.decorator';@@ -152,45 +151,6 @@ export class AgentChatService {};}- async upsertAssistantMessage({- id,- threadId,- turnId,- parts,- workspaceId,- }: {- id: string;- threadId: string;- turnId: string;- parts: ExtendedUIMessage['parts'];- workspaceId: string;- }): Promise<void> {- const dbParts = mapUIMessagePartsToDBParts(parts, id);-- // the message is replaced whole, so no reader or crash ever finds it without its parts- await this.conversationWriterService.runInTransaction(- workspaceId,- async (scope) => {- await scope.upsert(- 'agentMessage',- {- id,- threadId,- turnId,- role: AgentMessageRole.ASSISTANT,- processedAt: new Date().toISOString(),- },- ['id'],- );- await scope.delete('agentMessagePart', { messageId: id });-- if (dbParts.length > 0) {- await scope.insert('agentMessagePart', dbParts);- }- },- );- }-// the contexts are the thread owner's, so the turns of other participants run without themasync getThreadContexts({threadId,diff --git a/packages/twenty-server/src/engine/metadata-modules/ai/ai-chat/services/agent-turn-stream.service.ts b/packages/twenty-server/src/engine/metadata-modules/ai/ai-chat/services/agent-turn-stream.service.tsnew file mode 100644index 00000000000..792aa62ef79--- /dev/null+++ b/packages/twenty-server/src/engine/metadata-modules/ai/ai-chat/services/agent-turn-stream.service.ts@@ -0,0 +1,562 @@+import { Injectable, Logger } from '@nestjs/common';++import {+ createUIMessageStream,+ readUIMessageStream,+ type TextStreamPart,+ type ToolSet,+ toUIMessageStream,+ type UIMessageStreamWriter,+} from 'ai';+import {+ type ExtendedUIMessage,+ type ExtendedUIMessagePart,+ isSilentAgentChatMessageParts,+} from 'twenty-shared/ai';+import { isDefined } from 'twenty-shared/utils';+import { v5 as uuidv5 } from 'uuid';++import { InjectMessageQueue } from 'src/engine/core-modules/message-queue/decorators/message-queue.decorator';+import { MessageQueue } from 'src/engine/core-modules/message-queue/message-queue.constants';+import { MessageQueueService } from 'src/engine/core-modules/message-queue/services/message-queue.service';+import { toDisplayCredits } from 'src/engine/core-modules/usage/utils/to-display-credits.util';+import { findAwaitingPausingToolParts } from 'src/engine/metadata-modules/ai/ai-agent-execution/pausing-tools/utils/find-awaiting-pausing-tool-parts.util';+import { type AgentTurn } from 'src/engine/metadata-modules/ai/ai-agent-execution/services/agent-turn.service';+import { type AgentTurnUsage } from 'src/engine/metadata-modules/ai/ai-billing/types/agent-turn-usage.type';+import { type StepDebitResult } from 'src/engine/metadata-modules/ai/ai-billing/types/step-debit-result.type';+import { addStepToAgentTurnUsage } from 'src/engine/metadata-modules/ai/ai-billing/utils/add-step-to-agent-turn-usage.util';+import { FLUSH_QUEUED_AGENT_CHAT_MESSAGES_JOB_NAME } from 'src/engine/metadata-modules/ai/ai-chat/jobs/flush-queued-agent-chat-messages-job-name.constant';+import { type FlushQueuedAgentChatMessagesJobData } from 'src/engine/metadata-modules/ai/ai-chat/jobs/flush-queued-agent-chat-messages-job.types';+import { AGENT_CHAT_CHECKPOINT_INTERVAL_MS } from 'src/engine/metadata-modules/ai/ai-chat/constants/agent-chat-checkpoint-interval-ms.constant';+import { AgentChatCancelSubscriberService } from 'src/engine/metadata-modules/ai/ai-chat/services/agent-chat-cancel-subscriber.service';+import { AgentChatEventPublisherService } from 'src/engine/metadata-modules/ai/ai-chat/services/agent-chat-event-publisher.service';+import { AgentChatStreamHeartbeatService } from 'src/engine/metadata-modules/ai/ai-chat/services/agent-chat-stream-heartbeat.service';+import {+ type AgentChatStreamClaim,+ AgentChatStreamRecoveryService,+} from 'src/engine/metadata-modules/ai/ai-chat/services/agent-chat-stream-recovery.service';+import { AgentChatThreadService } from 'src/engine/metadata-modules/ai/ai-chat/services/agent-chat-thread.service';+import { type AgentChatTurnOutcome } from 'src/engine/metadata-modules/ai/ai-chat/types/agent-chat-turn-outcome.type';+import {+ classifyAgentChatTurnOutcome,+ resolveSupersededTurnOutcome,+} from 'src/engine/metadata-modules/ai/ai-chat/utils/classify-agent-chat-turn-outcome.util';+import { findAwaitingCallText } from 'src/engine/metadata-modules/ai/ai-chat/utils/find-awaiting-call-text.util';+import { hasReplyText } from 'src/engine/metadata-modules/ai/ai-chat/utils/has-reply-text.util';+import { findLastMessageText } from 'src/engine/metadata-modules/ai/ai-chat/utils/find-last-message-text.util';+import { formatErrorWithCause } from 'src/engine/metadata-modules/ai/ai-chat/utils/format-error-with-cause.util';+import { getCancelChannel } from 'src/engine/metadata-modules/ai/ai-chat/utils/get-cancel-channel.util';+import { mapAgentChatTurnOutcomeToTurnStatus } from 'src/engine/metadata-modules/ai/ai-chat/utils/map-agent-chat-turn-outcome-to-turn-status.util';+import { updateAgentChatThreadUsage } from 'src/engine/metadata-modules/ai/ai-chat/utils/update-agent-chat-thread-usage.util';+import { AgentHistoryRepository } from 'src/engine/metadata-modules/ai/ai-history/repositories/agent-history-repository';+import { InjectAgentHistoryRepository } from 'src/engine/metadata-modules/ai/ai-history/repositories/inject-agent-history-repository.decorator';+import { AgentConversationWriterService } from 'src/engine/metadata-modules/ai/ai-history/services/agent-conversation-writer.service';+import { AgentTurnRecorderService } from 'src/engine/metadata-modules/ai/ai-history/services/agent-turn-recorder.service';+import { AgentMessageStatus } from 'src/engine/metadata-modules/ai/ai-history/enums/agent-message-status.enum';+import { type AgentMessageWorkspaceEntity } from 'src/engine/metadata-modules/ai/ai-history/standard-objects/agent-message.workspace-entity';+import { type AgentChatThreadWorkspaceEntity } from 'src/engine/metadata-modules/ai/ai-history/standard-objects/agent-chat-thread.workspace-entity';+import { type AiModelConfig } from 'src/engine/metadata-modules/ai/ai-models/types/ai-model-config.type';+import {+ AiException,+ AiExceptionCode,+} from 'src/engine/metadata-modules/ai/ai.exception';++// the reply id derives from the stream, so a retried job overwrites its own reply while each resume writes its own+const ASSISTANT_MESSAGE_ID_NAMESPACE = '0b9c2a3d-4e5f-4a1b-8c2d-3e4f5a6b7c8d';++export type AgentTurnStreamUsage = AgentTurnUsage & {+ conversationSize: number;+};++export type AgentTurnStreamSource = Pick<AgentTurn, 'modelConfig'> & {+ parts: ReadableStream<TextStreamPart<ToolSet>>;+ getStreamError?: () => unknown;+ // writes that must land before the stream closes, such as a generated title+ pendingWrites?: Promise<unknown>;+};++export type AgentTurnStreamResult = {+ assistantMessageId: string;+ replyParts: ExtendedUIMessagePart[];+ isAborted: boolean;+ streamError: unknown;+ usage: AgentTurnStreamUsage;+ modelConfig: AiModelConfig | null;+};++export type AgentTurnFinish = {+ outcome: AgentChatTurnOutcome;+ isAwaitingAnswer: boolean;+ // the thread as it was before this turn, absent when another stream took the thread over+ threadBefore: AgentChatThreadWorkspaceEntity | null;+};++@Injectable()+export class AgentTurnStreamService {+ private readonly logger = new Logger(AgentTurnStreamService.name);++ constructor(+ @InjectAgentHistoryRepository('agentChatThread')+ private readonly threadRepository: AgentHistoryRepository<AgentChatThreadWorkspaceEntity>,+ @InjectAgentHistoryRepository('agentMessage')+ private readonly messageRepository: AgentHistoryRepository<AgentMessageWorkspaceEntity>,+ @InjectMessageQueue(MessageQueue.aiStreamQueue)+ private readonly chatMessageQueueService: MessageQueueService,+ private readonly eventPublisherService: AgentChatEventPublisherService,+ private readonly cancelSubscriberService: AgentChatCancelSubscriberService,+ private readonly streamHeartbeatService: AgentChatStreamHeartbeatService,+ private readonly streamRecoveryService: AgentChatStreamRecoveryService,+ private readonly conversationWriterService: AgentConversationWriterService,+ private readonly turnRecorderService: AgentTurnRecorderService,+ private readonly threadService: AgentChatThreadService,+ ) {}++ // Keeps the claim alive and stoppable while the work runs, and releases it however the work ends.+ // Messages queued meanwhile are sent then, unless the member stopped the turn+ async runClaimed<TResult>({+ claim,+ jobAbortSignal,+ work,+ }: {+ claim: AgentChatStreamClaim;+ jobAbortSignal?: AbortSignal;+ work: (abortSignal: AbortSignal) => Promise<TResult>;+ }): Promise<TResult> {+ const { threadId, streamId } = claim;++ await this.eventPublisherService.resetStreamState(threadId);++ const abortController = new AbortController();+ const cancelChannel = getCancelChannel(threadId, streamId);+ const stopHeartbeat = this.streamHeartbeatService.startRunning(streamId);++ await this.cancelSubscriberService.subscribe(cancelChannel, () => {+ abortController.abort();+ });++ jobAbortSignal?.addEventListener(+ 'abort',+ () => {+ abortController.abort(+ new AiException(+ 'The response was interrupted before it could finish.',+ AiExceptionCode.STREAM_INTERRUPTED,+ ),+ );+ },+ { once: true },+ );++ try {+ return await work(abortController.signal);+ } catch (error) {+ await this.failStream({ claim, error });++ throw error;+ } finally {+ stopHeartbeat();+ await this.cancelSubscriberService.unsubscribe(cancelChannel);+ await this.streamRecoveryService.releaseStreamClaim(claim);++ if (!abortController.signal.aborted) {+ await this.flushQueuedMessages(claim);+ }+ }+ }++ private async flushQueuedMessages({+ workspaceId,+ threadId,+ }: AgentChatStreamClaim): Promise<void> {+ try {+ const hasQueuedMessages = await this.messageRepository.existsBy(+ workspaceId,+ { threadId, status: AgentMessageStatus.QUEUED },+ );++ if (hasQueuedMessages) {+ await this.chatMessageQueueService.add<FlushQueuedAgentChatMessagesJobData>(+ FLUSH_QUEUED_AGENT_CHAT_MESSAGES_JOB_NAME,+ { workspaceId, threadId },+ );+ }+ } catch (error) {+ this.logger.error(+ `Failed to send the messages queued in thread ${threadId} of workspace ${workspaceId}: ${formatErrorWithCause(error)}`,+ );+ }+ }++ // ends the turn the claim runs as failed, and tells the conversation's subscribers+ async failStream({+ claim,+ error,+ }: {+ claim: AgentChatStreamClaim;+ error: unknown;+ }): Promise<void> {+ await this.streamRecoveryService.failStream({ ...claim, error });+ }++ // a turn that could not start throws its error; otherwise the caller gets what startTurn returned+ async stream<TStarted extends { source: AgentTurnStreamSource }>({+ claim,+ turnId,+ startTurn,+ }: {+ claim: AgentChatStreamClaim;+ turnId: string;+ startTurn: (+ writer: UIMessageStreamWriter<ExtendedUIMessage>,+ ) => Promise<TStarted>;+ }): Promise<AgentTurnStreamResult & { started: TStarted }> {+ const { workspaceId, threadId, streamId } = claim;+ const assistantMessageId = uuidv5(streamId, ASSISTANT_MESSAGE_ID_NAMESPACE);+ const usage: AgentTurnStreamUsage = {+ inputTokens: 0,+ outputTokens: 0,+ inputCredits: 0,+ outputCredits: 0,+ cacheReadTokens: 0,+ cacheCreationTokens: 0,+ conversationSize: 0,+ };+ const result: AgentTurnStreamResult = {+ assistantMessageId,+ replyParts: [],+ isAborted: false,+ streamError: undefined,+ usage,+ modelConfig: null,+ };++ let started: TStarted | undefined;+ let isFinalizing = false;+ let checkpoints: Promise<void> = Promise.resolve();+ let lastCheckpointAt = 0;++ // onEnd fires before the stream drains, so the result is read once it has run+ let markEnded: () => void = () => {};+ const ended = new Promise<void>((resolve) => {+ markEnded = resolve;+ });++ const uiStream = createUIMessageStream<ExtendedUIMessage>({+ execute: async ({ writer }) => {+ started = await startTurn(writer);++ const { source } = started;++ result.modelConfig = source.modelConfig;++ writer.merge(+ toUIMessageStream({+ stream: source.parts,+ onError: formatErrorWithCause,+ sendStart: true,+ sendReasoning: true,+ generateMessageId: () => assistantMessageId,+ messageMetadata: ({ part }) =>+ this.computeMessageMetadata({+ part,+ modelConfig: source.modelConfig,+ usage,+ }),+ onEnd: async ({ responseMessage, isAborted }) => {+ try {+ result.streamError ??= source.getStreamError?.();+ result.replyParts = responseMessage.parts;+ result.isAborted = isAborted;+ isFinalizing = true;+ await source.pendingWrites;+ } finally {+ markEnded();+ }+ },+ }),+ );+ },+ // errors thrown before the model stream merges never reach onEnd+ onError: (error) => {+ result.streamError = error;+ markEnded();++ return formatErrorWithCause(error);+ },+ });++ const [publishStream, checkpointStream] = uiStream.tee();++ void (async () => {+ try {+ for await (const message of readUIMessageStream<ExtendedUIMessage>({+ stream: checkpointStream,+ terminateOnError: false,+ })) {+ const now = Date.now();++ if (+ isFinalizing ||+ message.parts.length === 0 ||+ now - lastCheckpointAt < AGENT_CHAT_CHECKPOINT_INTERVAL_MS+ ) {+ continue;+ }++ lastCheckpointAt = now;+ const parts = message.parts;++ checkpoints = checkpoints+ .then(async () => {+ if (isFinalizing) {+ return;+ }++ await this.conversationWriterService.upsertAssistantMessage({+ workspaceId,+ id: assistantMessageId,+ threadId,+ turnId,+ parts,+ });+ })+ .catch((error) => {+ this.logger.warn(+ `Failed to checkpoint assistant message for stream ${streamId}: ${formatErrorWithCause(error)}`,+ );+ });+ }+ } catch {+ // best-effort; the reply is saved whole once the turn ends+ }+ })();++ for await (const chunk of publishStream) {+ if ((chunk as { type?: string }).type === 'error') {+ continue;+ }++ await this.eventPublisherService.publish({+ threadId,+ workspaceId,+ event: {+ type: 'stream-chunk',+ chunk: chunk as Record<string, unknown>,+ },+ });+ }++ await ended;+ await checkpoints;++ if (!isDefined(started)) {+ throw result.streamError;+ }++ return { ...result, started };+ }++ // Saves the reply and ends the turn, unless another stream took the thread over meanwhile. A turn that+ // waits on a person or an event is recorded as waiting rather than done. A turn whose stream failed+ // only records its usage: its failure is recorded with the stream's claim+ async finishTurn({+ claim,+ turnId,+ result,+ modelId,+ stopReason,+ isWaiting = false,+ isBackgroundRun = false,+ }: {+ claim: AgentChatStreamClaim;+ turnId: string;+ result: AgentTurnStreamResult;+ modelId: string;+ stopReason: StepDebitResult['stopReason'];+ isWaiting?: boolean;+ // a background run's reply stays out of the inbox unless it asks something, and its result+ // stands in for a written reply+ isBackgroundRun?: boolean;+ }): Promise<AgentTurnFinish | null> {+ const { workspaceId, threadId, streamId } = claim;+ const { replyParts, assistantMessageId } = result;+ // a turn that only waits on the member still needs a preview in chat lists+ const replyText =+ findLastMessageText(replyParts) ?? findAwaitingCallText(replyParts);+ const hasText = hasReplyText(replyParts);+ const isSilent = isSilentAgentChatMessageParts(replyParts);+ const awaitingParts = findAwaitingPausingToolParts(replyParts);+ // without an answerable call nothing can resume the turn, so fail rather than wait forever+ const isAwaitingAnswer =+ awaitingParts.length > 0 &&+ awaitingParts.every(({ isAnswerable }) => isAnswerable);+ const outcome = classifyAgentChatTurnOutcome({+ hasText,+ isAborted: result.isAborted,+ isAwaitingUserAnswer: awaitingParts.length > 0 || isWaiting,+ isSilent,+ stopReason,+ isTextReplyRequired: !isBackgroundRun,+ });+ const finishTurn = () =>+ this.turnRecorderService.finish({+ workspaceId,+ turnId,+ ...mapAgentChatTurnOutcomeToTurnStatus(outcome),+ modelId,+ streamClaim: { threadId, streamId },+ });++ await this.turnRecorderService.recordUsage({+ workspaceId,+ threadId,+ turnId,+ usage: result.usage,+ });++ if (isDefined(result.streamError)) {+ return null;+ }++ if (replyParts.length === 0) {+ await finishTurn();++ return { outcome, isAwaitingAnswer, threadBefore: null };+ }++ const threadBefore = await this.threadRepository.findOne(workspaceId, {+ where: { id: threadId },+ });++ if (!isDefined(threadBefore) || isDefined(threadBefore.deletedAt)) {+ return {+ outcome: resolveSupersededTurnOutcome(outcome),+ isAwaitingAnswer,+ threadBefore: null,+ };+ }++ await this.conversationWriterService.upsertAssistantMessage({+ workspaceId,+ id: assistantMessageId,+ threadId,+ turnId,+ parts: replyParts,+ });++ const totalsUpdate = await updateAgentChatThreadUsage({+ repository: this.threadRepository,+ workspaceId,+ threadId,+ streamId,+ lastMessageText: replyText,+ shouldRecordActivity: !isSilent && (!isBackgroundRun || isAwaitingAnswer),+ usage: {+ contextWindowTokens: result.modelConfig?.contextWindowTokens ?? null,+ conversationSize: result.usage.conversationSize,+ pendingQuestionMessageId: isAwaitingAnswer ? assistantMessageId : null,+ },+ });++ if (!totalsUpdate.affected) {+ // the reply is saved even though a newer stream owns the thread, and its activity is+ // best-effort so it never turns the saved reply into an error+ if (!isSilent && !isBackgroundRun) {+ await this.threadService+ .recordThreadActivity({ workspaceId, threadId, text: replyText })+ .catch((error: unknown) =>+ this.logger.warn(+ `Could not record reply activity on thread ${threadId}: ${formatErrorWithCause(error)}`,+ ),+ );+ }++ return {+ outcome: resolveSupersededTurnOutcome(outcome),+ isAwaitingAnswer,+ threadBefore: null,+ };+ }++ if (awaitingParts.length > 0 && !isAwaitingAnswer) {+ throw new AiException(+ 'A call waiting on the user could not be read',+ AiExceptionCode.INVALID_TOOL_CALL_OUTPUT,+ );+ }++ await finishTurn();++ return { outcome, isAwaitingAnswer, threadBefore };+ }++ // the end event follows every chunk, so a subscriber that sees it has the whole reply+ async publishTurnEnd({+ claim: { workspaceId, threadId },+ assistantMessageId,+ stopReason,+ }: {+ claim: AgentChatStreamClaim;+ assistantMessageId: string;+ stopReason: StepDebitResult['stopReason'];+ }): Promise<void> {+ await this.eventPublisherService.publish({+ threadId,+ workspaceId,+ event:+ stopReason === 'workspaceAllowance'+ ? { type: 'credits-exhausted' }+ : { type: 'message-persisted', messageId: assistantMessageId },+ });+ }++ private computeMessageMetadata({+ part,+ modelConfig,+ usage,+ }: {+ part: {+ type: string;+ usage?: {+ inputTokens?: number;+ outputTokens?: number;+ inputTokenDetails?: { cacheReadTokens?: number };+ outputTokenDetails?: { reasoningTokens?: number };+ };+ providerMetadata?: Record<string, Record<string, unknown> | undefined>;+ };+ modelConfig: AiModelConfig;+ usage: AgentTurnStreamUsage;+ }) {+ if (part.type === 'finish-step') {+ Object.assign(usage, addStepToAgentTurnUsage(modelConfig, usage, part));+ usage.conversationSize = part.usage?.inputTokens ?? 0;+ }++ if (part.type === 'start') {+ return { startedAt: new Date().toISOString() };+ }++ if (part.type === 'finish') {+ const finishedAt = new Date().toISOString();++ return {+ createdAt: finishedAt,+ finishedAt,+ usage: {+ inputTokens: usage.inputTokens,+ outputTokens: usage.outputTokens,+ cachedInputTokens: usage.cacheReadTokens,+ inputCredits: toDisplayCredits(usage.inputCredits),+ outputCredits: toDisplayCredits(usage.outputCredits),+ conversationSize: usage.conversationSize,+ },+ model: {+ contextWindowTokens: modelConfig.contextWindowTokens,+ },+ };+ }++ return undefined;+ }+}diff --git a/packages/twenty-server/src/engine/metadata-modules/ai/ai-chat/services/chat-execution.service.ts b/packages/twenty-server/src/engine/metadata-modules/ai/ai-chat/services/chat-execution.service.tsindex 4a555f5ff3b..287ce1e5055 100644--- a/packages/twenty-server/src/engine/metadata-modules/ai/ai-chat/services/chat-execution.service.ts+++ b/packages/twenty-server/src/engine/metadata-modules/ai/ai-chat/services/chat-execution.service.ts@@ -1,27 +1,11 @@-import { injectChatMessageSenders } from 'src/engine/metadata-modules/ai/ai-chat/utils/inject-chat-message-senders.util';-import { splitContextInstructions } from 'src/engine/metadata-modules/ai/ai-chat/utils/split-context-instructions.util';-import { AgentChatActorService } from 'src/engine/metadata-modules/ai/ai-chat/services/agent-chat-actor.service';-import { type ToolContext } from 'src/engine/core-modules/tool-provider/types/tool-context.type';import { Injectable, Logger } from '@nestjs/common';+import { convertToModelMessages, hasToolCall } from 'ai';import {- convertToModelMessages,- hasToolCall,- NoOutputGeneratedError,- isStepCount,- type StepResult,- streamText,- type SystemModelMessage,- type ToolSet,-} from 'ai';-import {- ASK_QUESTION_TOOL_NAME,ATTACH_CONVERSATION_TO_RECORD_TOOL_NAME,COMPLETE_WORKSPACE_SETUP_TOOL_NAME,type ExtendedUIMessage,NO_REPLY_TOOL_NAME,- PROPOSE_TOOL_CALL_TOOL_NAME,- REQUEST_FORM_TOOL_NAME,extractRecordReferenceIds,} from 'twenty-shared/ai';import { type APP_LOCALES } from 'twenty-shared/translations';@@ -30,97 +14,62 @@ import {CoreObjectNameSingular,FeatureFlagKey,} from 'twenty-shared/types';-import { getAppPath, isDefined, sortKeysDeeply } from 'twenty-shared/utils';--import { AI_LATENCY_MS_BUCKET_BOUNDARIES } from 'src/engine/core-modules/metrics/constants/ai-latency-ms-bucket-boundaries.constant';-import { TOOL_EXECUTION_DURATION_MS_BUCKET_BOUNDARIES } from 'src/engine/core-modules/metrics/constants/tool-execution-duration-ms-bucket-boundaries.constant';-import { TOOL_OUTPUT_TOKENS_BUCKET_BOUNDARIES } from 'src/engine/core-modules/metrics/constants/tool-output-tokens-bucket-boundaries.constant';-import { MetricsService } from 'src/engine/core-modules/metrics/metrics.service';-import { MetricsKeys } from 'src/engine/core-modules/metrics/types/metrics-keys.type';-import { UsageOperationType } from 'src/engine/core-modules/usage/enums/usage-operation-type.enum';+import { getAppPath, isDefined } from 'twenty-shared/utils';import { type CodeExecutionStreamEmitter } from 'src/engine/core-modules/tool-provider/interfaces/code-execution-stream-emitter.type';import { CodeInterpreterService } from 'src/engine/core-modules/code-interpreter/code-interpreter.service';import { WorkspaceDomainsService } from 'src/engine/core-modules/domain/workspace-domains/services/workspace-domains.service';-import { ExceptionHandlerService } from 'src/engine/core-modules/exception-handler/exception-handler.service';import { FeatureFlagService } from 'src/engine/core-modules/feature-flag/services/feature-flag.service';-import { ToolRegistryService } from 'src/engine/core-modules/tool-provider/services/tool-registry.service';-import {- createExecuteToolTool,- createLearnToolsTool,- createLoadSkillTool,- EXECUTE_TOOL_TOOL_NAME,- LEARN_TOOLS_TOOL_NAME,- LOAD_SKILL_TOOL_NAME,-} from 'src/engine/core-modules/tool-provider/tools';-import { estimateToolOutputTokens } from 'src/engine/core-modules/tool-provider/utils/estimate-tool-output-tokens.util';-import { getToolMetricName } from 'src/engine/core-modules/tool-provider/utils/get-tool-metric-name.util';-import { isToolOutputSuccessful } from 'src/engine/core-modules/tool-provider/utils/is-tool-output-successful.util';-import { resolveToolName } from 'src/engine/core-modules/tool-provider/utils/resolve-tool-name.util';+import { type ToolContext } from 'src/engine/core-modules/tool-provider/types/tool-context.type';+import { UsageOperationType } from 'src/engine/core-modules/usage/enums/usage-operation-type.enum';import { type WorkspaceEntity } from 'src/engine/core-modules/workspace/workspace.entity';import { AgentActorContextService } from 'src/engine/metadata-modules/ai/ai-agent-execution/services/agent-actor-context.service';-import { resolveProposedToolCall } from 'src/engine/metadata-modules/ai/ai-agent-execution/pausing-tools/utils/resolve-proposed-tool-call.util';-import { endsOnPausingToolCall } from 'src/engine/metadata-modules/ai/ai-agent-execution/pausing-tools/utils/ends-on-pausing-tool-call.util';-import { finalizeDanglingToolParts } from 'src/engine/metadata-modules/ai/ai-history/utils/finalize-dangling-tool-parts.util';-import { guideUncallableToolCallsToMetaTool } from 'src/engine/metadata-modules/ai/ai-agent-execution/utils/guide-uncallable-tool-calls-to-meta-tool.util';-import { AGENT_CONFIG } from 'src/engine/metadata-modules/ai/ai-agent/constants/agent-config.const';-import { BrowsingContextType } from 'src/engine/metadata-modules/ai/ai-agent/types/browsing-context.type';-import { repairToolCall } from 'src/engine/metadata-modules/ai/ai-agent/utils/repair-tool-call.util';-import { AiBillingService } from 'src/engine/metadata-modules/ai/ai-billing/services/ai-billing.service';-import { type StepDebitResult } from 'src/engine/metadata-modules/ai/ai-billing/types/step-debit-result.type';-import { convertDollarsToCreditsMicro } from 'src/engine/metadata-modules/ai/ai-billing/utils/convert-dollars-to-credits-micro.util';-import { countNativeWebSearchCallsFromSteps } from 'src/engine/metadata-modules/ai/ai-billing/utils/count-native-web-search-calls-from-steps.util';+import { type AgentStepLoopInput } from 'src/engine/metadata-modules/ai/ai-agent-execution/services/agent-step-loop.service';import {- extractCacheCreationTokens,- extractCacheCreationTokensFromSteps,-} from 'src/engine/metadata-modules/ai/ai-billing/utils/extract-cache-creation-tokens.util';+ type AgentTurn,+ AgentTurnService,+} from 'src/engine/metadata-modules/ai/ai-agent-execution/services/agent-turn.service';+import { guideUncallableToolCallsToMetaTool } from 'src/engine/metadata-modules/ai/ai-agent-execution/utils/guide-uncallable-tool-calls-to-meta-tool.util';+import { replaceUnsupportedFileParts } from 'src/engine/metadata-modules/ai/ai-agent-execution/utils/replace-unsupported-file-parts.util';+import { type BrowsingContextType } from 'src/engine/metadata-modules/ai/ai-agent/types/browsing-context.type';+import { AGENT_CHAT_KEEPALIVE_INTERVAL_MS } from 'src/engine/metadata-modules/ai/ai-chat/constants/agent-chat-keepalive-interval-ms.constant';import { AI_CHAT_EXCLUDED_TOOL_NAMES } from 'src/engine/metadata-modules/ai/ai-chat/constants/ai-chat-excluded-tool-names.const';import { AI_CHAT_STREAM_FUNCTION_ID } from 'src/engine/metadata-modules/ai/ai-chat/constants/ai-chat-stream-function-id.constant';import { AI_CHAT_TOOL_NAMES_TO_PRELOAD } from 'src/engine/metadata-modules/ai/ai-chat/constants/ai-chat-tool-names-to-preload.const';import { AI_CHAT_WORKSPACE_SETUP_STREAM_FUNCTION_ID } from 'src/engine/metadata-modules/ai/ai-chat/constants/ai-chat-workspace-setup-stream-function-id.constant';+import { AgentChatActorService } from 'src/engine/metadata-modules/ai/ai-chat/services/agent-chat-actor.service';import { AgentChatThreadParticipantService } from 'src/engine/metadata-modules/ai/ai-chat/services/agent-chat-thread-participant.service';import { AgentChatThreadTargetService } from 'src/engine/metadata-modules/ai/ai-chat/services/agent-chat-thread-target.service';import { MessagePruningService } from 'src/engine/metadata-modules/ai/ai-chat/services/message-pruning.service';-import { createAskQuestionTool } from 'src/engine/metadata-modules/ai/ai-chat/tools/ask-question.tool';import { createAttachConversationToRecordTool } from 'src/engine/metadata-modules/ai/ai-chat/tools/attach-conversation-to-record.tool';-import { createProposeToolCallTool } from 'src/engine/metadata-modules/ai/ai-agent-execution/tools/propose-tool-call.tool';-import { createRequestFormTool } from 'src/engine/metadata-modules/ai/ai-chat/tools/request-form.tool';import { createCompleteWorkspaceSetupTool } from 'src/engine/metadata-modules/ai/ai-chat/tools/complete-workspace-setup.tool';import { createNoReplyTool } from 'src/engine/metadata-modules/ai/ai-chat/tools/no-reply.tool';import { type AgentChatSender } from 'src/engine/metadata-modules/ai/ai-chat/types/agent-chat-sender.type';import { type UploadedFileReference } from 'src/engine/metadata-modules/ai/ai-chat/types/uploaded-file-reference.type';-import { findLastMessageText } from 'src/engine/metadata-modules/ai/ai-chat/utils/find-last-message-text.util';-import { formatErrorWithCause } from 'src/engine/metadata-modules/ai/ai-chat/utils/format-error-with-cause.util';-import { buildWorkspaceSetupChatThreadId } from 'src/engine/metadata-modules/ai/ai-chat/utils/build-workspace-setup-chat-thread-id.util';import { buildFullSystemPrompt } from 'src/engine/metadata-modules/ai/ai-chat/utils/build-full-system-prompt.util';-import { hasNoAssistantMessage } from 'src/engine/metadata-modules/ai/ai-chat/utils/has-no-assistant-message.util';-import { hasSucceededWorkspaceSetupCompletion } from 'src/engine/metadata-modules/ai/ai-chat/utils/has-succeeded-workspace-setup-completion.util';+import { buildWorkspaceSetupChatThreadId } from 'src/engine/metadata-modules/ai/ai-chat/utils/build-workspace-setup-chat-thread-id.util';import { collectReferencedSkillIds } from 'src/engine/metadata-modules/ai/ai-chat/utils/collect-referenced-skill-ids.util';import { collectUploadedFileReferences } from 'src/engine/metadata-modules/ai/ai-chat/utils/collect-uploaded-file-references.util';+import { createTurnAuthorizer } from 'src/engine/metadata-modules/ai/ai-chat/utils/create-turn-authorizer.util';import { extractCodeInterpreterFiles } from 'src/engine/metadata-modules/ai/ai-chat/utils/extract-code-interpreter-files.util';+import { findLastMessageText } from 'src/engine/metadata-modules/ai/ai-chat/utils/find-last-message-text.util';+import { formatErrorWithCause } from 'src/engine/metadata-modules/ai/ai-chat/utils/format-error-with-cause.util';+import { hasNoAssistantMessage } from 'src/engine/metadata-modules/ai/ai-chat/utils/has-no-assistant-message.util';+import { hasSucceededWorkspaceSetupCompletion } from 'src/engine/metadata-modules/ai/ai-chat/utils/has-succeeded-workspace-setup-completion.util';+import { injectChatMessageSenders } from 'src/engine/metadata-modules/ai/ai-chat/utils/inject-chat-message-senders.util';import { injectMessageTimestamps } from 'src/engine/metadata-modules/ai/ai-chat/utils/inject-message-timestamps.util';-import {- getCacheProviderOptions,- getCallLevelProviderOptions,- injectCacheBreakpoint,-} from 'src/engine/metadata-modules/ai/ai-agent-execution/utils/provider-options.util';-import { replaceUnsupportedFileParts } from 'src/engine/metadata-modules/ai/ai-agent-execution/utils/replace-unsupported-file-parts.util';+import { splitContextInstructions } from 'src/engine/metadata-modules/ai/ai-chat/utils/split-context-instructions.util';import { tagAiChatExecutionScope } from 'src/engine/metadata-modules/ai/ai-chat/utils/tag-ai-chat-execution-scope.util';-import { buildAiTelemetry } from 'src/engine/metadata-modules/ai/ai-models/utils/build-ai-telemetry.util';-import { AiModelConfigService } from 'src/engine/metadata-modules/ai/ai-models/services/ai-model-config.service';-import { AiModelRegistryService } from 'src/engine/metadata-modules/ai/ai-models/services/ai-model-registry.service';+import { AgentMessageRole } from 'src/engine/metadata-modules/ai/ai-history/enums/agent-message-role.enum';+import { finalizeDanglingToolParts } from 'src/engine/metadata-modules/ai/ai-history/utils/finalize-dangling-tool-parts.util';import { type AiModelConfig } from 'src/engine/metadata-modules/ai/ai-models/types/ai-model-config.type';-import { getNativeModelCapabilities } from 'src/engine/metadata-modules/ai/ai-models/utils/get-native-model-capabilities.util';+import { getChatModelId } from 'src/engine/metadata-modules/ai/ai-models/utils/get-chat-model-id.util';import {AiException,AiExceptionCode,} from 'src/engine/metadata-modules/ai/ai.exception';import { SkillService } from 'src/engine/metadata-modules/skill/skill.service';-import { getChatModelId } from 'src/engine/metadata-modules/ai/ai-models/utils/get-chat-model-id.util';-import { AGENT_CHAT_KEEPALIVE_INTERVAL_MS } from 'src/engine/metadata-modules/ai/ai-chat/constants/agent-chat-keepalive-interval-ms.constant';-import { createTurnAuthorizer } from 'src/engine/metadata-modules/ai/ai-chat/utils/create-turn-authorizer.util';-import { AgentMessageRole } from 'src/engine/metadata-modules/ai/ai-history/enums/agent-message-role.enum';export type ChatExecutionOptions = {workspace: WorkspaceEntity;@@ -137,16 +86,15 @@ export type ChatExecutionOptions = {modelId?: string;abortSignal: AbortSignal;conversationSizeTokens: number;- onUserCreditsExhausted: (notice: {- renewsAt: Date;- isFallingBackToWorkspaceCredits: boolean;- }) => Promise<void>;+ onUserCreditsExhausted: NonNullable<+ AgentStepLoopInput['onUserCreditsExhausted']+ >;};-export type ChatExecutionResult = {- stream: ReturnType<typeof streamText>;- modelConfig: AiModelConfig;- getStopReason: () => StepDebitResult['stopReason'];+type ChatExecutionResult = Pick<+ AgentTurn,+ 'stream' | 'modelConfig' | 'getStopReason'+> & {getStreamError: () => unknown;};@@ -155,21 +103,16 @@ export class ChatExecutionService {private readonly logger = new Logger(ChatExecutionService.name);constructor(- private readonly toolRegistry: ToolRegistryService,private readonly skillService: SkillService,- private readonly aiModelRegistryService: AiModelRegistryService,- private readonly aiModelConfigService: AiModelConfigService,- private readonly aiBillingService: AiBillingService,private readonly agentActorContextService: AgentActorContextService,private readonly workspaceDomainsService: WorkspaceDomainsService,private readonly codeInterpreterService: CodeInterpreterService,- private readonly exceptionHandlerService: ExceptionHandlerService,private readonly messagePruningService: MessagePruningService,- private readonly metricsService: MetricsService,private readonly chatActorService: AgentChatActorService,private readonly agentChatThreadTargetService: AgentChatThreadTargetService,private readonly agentChatThreadParticipantService: AgentChatThreadParticipantService,private readonly featureFlagService: FeatureFlagService,+ private readonly agentTurnService: AgentTurnService,) {}async streamChat({@@ -224,56 +167,6 @@ export class ChatExecutionService {resolveExecutionContext,};- const toolCatalog = await this.toolRegistry.buildToolIndex(workspace.id, {- userId,- userWorkspaceId,- locale,- excludeTools: AI_CHAT_EXCLUDED_TOOL_NAMES,- rolePermissionConfig: toolContext.rolePermissionConfig,- });-- const skillCatalog = await this.skillService.findAllFlatSkills(- workspace.id,- );-- this.logger.log(- `Built tool catalog with ${toolCatalog.length} tools, ${skillCatalog.length} skills available`,- );-- const preloadedTools = await this.toolRegistry.getToolsByName(- AI_CHAT_TOOL_NAMES_TO_PRELOAD,- toolContext,- { compactOutput: true, spillLargeOutput: true },- );-- const resolvedModelId = getChatModelId({- requestedModelId: modelId,- workspace,- });-- this.aiModelRegistryService.validateModelAvailability(resolvedModelId);-- const registeredModel =- await this.aiModelRegistryService.resolveModelForAgent(- { modelId: resolvedModelId },- workspace,- );-- const modelConfig = this.aiModelRegistryService.getEffectiveModelConfig(- registeredModel.modelId,- );-- const nativeCapabilities = getNativeModelCapabilities(- registeredModel.sdkPackage,- );- const nativeTools = this.aiModelConfigService.getNativeModelTools(- registeredModel,- {- webSearch: nativeCapabilities?.webSearch === true,- twitterSearch: nativeCapabilities?.twitterSearch === true,- },- );-const isWorkspaceSetupConversation =threadId ===buildWorkspaceSetupChatThreadId({@@ -288,11 +181,6 @@ export class ChatExecutionService {const isWorkspaceSetupKickoffTurn =isWorkspaceSetupThread && hasNoAssistantMessage(messages);- tagAiChatExecutionScope({- isWorkspaceSetupThread,- modelId: registeredModel.modelId,- });-// judged on the conversation, not setup status: onboarding continues here after setup completesconst isInboxConversation =!isWorkspaceSetupConversation &&@@ -317,294 +205,130 @@ export class ChatExecutionService {}),})) > 1;- const isToolAllowed = (toolName: string) =>- !AI_CHAT_EXCLUDED_TOOL_NAMES.has(toolName);-- const preloadedToolSet: ToolSet = {- ...preloadedTools,- ...nativeTools,- [ASK_QUESTION_TOOL_NAME]: createAskQuestionTool({- isWorkspaceSetupThread,- }),- [REQUEST_FORM_TOOL_NAME]: createRequestFormTool(),- [PROPOSE_TOOL_CALL_TOOL_NAME]: createProposeToolCallTool({- resolveProposal: (input) =>- resolveProposedToolCall({- input,- findTool: async (toolName) =>- toolCatalog.find(- (toolIndexEntry) => toolIndexEntry.name === toolName,- ),- executeTool: ({ toolName, args }) =>- this.toolRegistry.resolveAndExecute(toolName, args, toolContext),- }),- }),- ...(isWorkspaceSetupThread- ? {- [COMPLETE_WORKSPACE_SETUP_TOOL_NAME]:- createCompleteWorkspaceSetupTool(),- }- : {}),- ...(isGroupConversation- ? { [NO_REPLY_TOOL_NAME]: createNoReplyTool() }- : {}),- ...(isInboxConversation- ? {- [ATTACH_CONVERSATION_TO_RECORD_TOOL_NAME]:- createAttachConversationToRecordTool({- agentChatThreadTargetService: this.agentChatThreadTargetService,- toolContext,- }),- }- : {}),- };+ let lastUnderlyingStreamError: unknown;- const activeTools: ToolSet = {- ...preloadedToolSet,- [LEARN_TOOLS_TOOL_NAME]: createLearnToolsTool(- this.toolRegistry,- toolContext,- {- spillLargeOutput: true,- isToolAllowed,- },- ),- [EXECUTE_TOOL_TOOL_NAME]: createExecuteToolTool(- this.toolRegistry,- toolContext,- {- compactOutput: true,- spillLargeOutput: true,- isToolAllowed,+ const turn = await this.agentTurnService.start({+ workspace,+ modelId: getChatModelId({ requestedModelId: modelId, workspace }),+ toolset: {+ registryAccess: {+ type: 'assistant',+ toolContext,+ excludedToolNames: AI_CHAT_EXCLUDED_TOOL_NAMES,+ preloadedToolNames: AI_CHAT_TOOL_NAMES_TO_PRELOAD,},- ),- [LOAD_SKILL_TOOL_NAME]: createLoadSkillTool(- (skillNames) =>- this.skillService.findFlatSkillsByNames(skillNames, workspace.id),- async () => {- const allSkills = await this.skillService.findAllFlatSkills(- workspace.id,- );-- return allSkills.map((skill) => skill.name);+ canAskHumans: true,+ isWorkspaceSetupThread,+ extraTools: {+ ...(isWorkspaceSetupThread+ ? {+ [COMPLETE_WORKSPACE_SETUP_TOOL_NAME]:+ createCompleteWorkspaceSetupTool(),+ }+ : {}),+ ...(isGroupConversation+ ? { [NO_REPLY_TOOL_NAME]: createNoReplyTool() }+ : {}),+ ...(isInboxConversation+ ? {+ [ATTACH_CONVERSATION_TO_RECORD_TOOL_NAME]:+ createAttachConversationToRecordTool({+ agentChatThreadTargetService:+ this.agentChatThreadTargetService,+ toolContext,+ }),+ }+ : {}),},- ),- };-- const isCodeInterpreterEnabled = this.codeInterpreterService.isEnabled();-- const uploadedFiles = collectUploadedFileReferences(messages);-- // inline tagged skills to save the model a load_skills round trip- const referencedSkills = await this.skillService.findFlatSkillsByIds(- collectReferencedSkillIds(messages),- workspace.id,- );-- let processedMessages: ExtendedUIMessage[] = replaceUnsupportedFileParts(- messages,- modelConfig.modalities,- isCodeInterpreterEnabled,- );-- let codeInterpreterFiles: UploadedFileReference[] = [];-- if (isCodeInterpreterEnabled) {- const extracted = extractCodeInterpreterFiles(processedMessages);-- processedMessages = extracted.processedMessages;- codeInterpreterFiles = extracted.extractedFiles.map(- ({ filename, fileId }) => ({ filename, fileId }),- );- }-- if (isDefined(browsingContext)) {- const contextString = this.buildContextFromBrowsingContext(- workspace,- browsingContext,- );-- processedMessages = this.injectBrowsingContextIntoLastUserMessage(- processedMessages,- contextString,- );- }-- processedMessages = injectChatMessageSenders({- messages: processedMessages,- currentWorkspaceMemberId: authorization.authContext.workspaceMemberId,- });- processedMessages = injectMessageTimestamps(- processedMessages,- userContext.timezone,- );-- const systemPrompt = buildFullSystemPrompt({- toolCatalog,- skillCatalog,- referencedSkills,- preloadedTools: Object.keys(preloadedToolSet),- uploadedFilesContext: { uploadedFiles, codeInterpreterFiles },- workspaceInstructions: workspace.aiAdditionalInstructions ?? undefined,- userContext,- userWorkspaceId,- workspaceId: workspace.id,- isWorkspaceSetupThread,- canAttachConversationToRecords: isInboxConversation,- });-- this.logger.log(- `Starting chat execution with model ${registeredModel.modelId}, ${Object.keys(activeTools).length} active tools`,- );-- const systemMessage: SystemModelMessage = {- role: 'system',- content: systemPrompt,- providerOptions: getCacheProviderOptions(modelConfig),- };-- const sanitizedMessages = this.sanitizeMessagePartsForModel(- processedMessages,- new Set(Object.keys(activeTools)),- );-- const { contexts, conversation } = splitContextInstructions(- await convertToModelMessages(sanitizedMessages),- );-- const pruningResult =- this.messagePruningService.pruneIfOverContextWindowLimit(- conversation,- modelConfig.contextWindowTokens,- conversationSizeTokens,- );-- if (pruningResult.isStillOverLimit) {- throw new AiException(- 'This conversation is too long for the model to process. Please start a new thread.',- AiExceptionCode.CONTEXT_WINDOW_EXCEEDED,- );- }-- if (pruningResult.wasPruned) {- onCompaction();- }-- const modelMessages = pruningResult.messages;-- let stopReason: StepDebitResult['stopReason'] = null;- let turnUserCreditsUsedMicro = 0;- const streamStartedAt = performance.now();- let stepStartedAt = streamStartedAt;- let ttftRecorded = false;- let stepIndex = 0;- let lastUnderlyingStreamError: unknown;-- const notifyUserCreditsCrossing = async (debit: StepDebitResult) => {- if (- isDefined(debit.userAllowanceExhaustedUntil) &&- debit.userCreditsUsedMicro > 0- ) {- await onUserCreditsExhausted({- renewsAt: debit.userAllowanceExhaustedUntil,- isFallingBackToWorkspaceCredits: debit.stopReason === null,+ },+ buildContext: async ({ registeredModel, modelConfig, toolset }) => {+ tagAiChatExecutionScope({+ isWorkspaceSetupThread,+ modelId: registeredModel.modelId,});- }- };-- const emitTurnUsageEvent = async (steps: StepResult<ToolSet>[]) => {- const usage = steps.reduce(- (totals, step) => ({- inputTokens: totals.inputTokens + (step.usage.inputTokens ?? 0),- outputTokens: totals.outputTokens + (step.usage.outputTokens ?? 0),- cacheReadTokens:- totals.cacheReadTokens +- (step.usage.inputTokenDetails?.cacheReadTokens ?? 0),- }),- { inputTokens: 0, outputTokens: 0, cacheReadTokens: 0 },- );- const cacheCreationTokens = extractCacheCreationTokensFromSteps(steps);- const totalTokens = usage.inputTokens + usage.outputTokens;-- const costInDollars = this.aiBillingService.calculateStepsCost(- registeredModel.modelId,- steps,- );- const creditsUsedMicro = convertDollarsToCreditsMicro(costInDollars);-- await this.aiBillingService.emitAiTokenUsageEvent({- workspaceId: workspace.id,- creditsUsedMicro,- totalTokens,- modelId: registeredModel.modelId,- operationType: UsageOperationType.AI_CHAT_TOKEN,- spenders: { userWorkspaceId },- userCreditsUsedMicro: Math.min(- turnUserCreditsUsedMicro,- creditsUsedMicro,- ),- });+ const { preparedMessages, codeInterpreterFiles } = this.prepareMessages(+ {+ workspace,+ messages,+ modelConfig,+ browsingContext,+ workspaceMemberId: authorization.authContext.workspaceMemberId,+ timezone: userContext.timezone,+ },+ );+ const { contexts, conversation } = splitContextInstructions(+ await convertToModelMessages(+ this.sanitizeMessagePartsForModel(+ preparedMessages,+ new Set(Object.keys(toolset.tools)),+ ),+ ),+ );- void this.aiBillingService- .billNativeWebSearchUsage({- nativeWebSearchCallCount: countNativeWebSearchCallsFromSteps(steps),- workspaceId: workspace.id,- operationType: UsageOperationType.WEB_SEARCH,- spenders: { userWorkspaceId },- })- .then(async (webSearchDebit) => {- if (isDefined(webSearchDebit) && stopReason !== 'userAllowance') {- await notifyUserCreditsCrossing(webSearchDebit);- }- });+ const pruningResult =+ this.messagePruningService.pruneIfOverContextWindowLimit(+ conversation,+ modelConfig.contextWindowTokens,+ conversationSizeTokens,+ );- const modelAttr = { model: registeredModel.modelId };+ if (pruningResult.isStillOverLimit) {+ throw new AiException(+ 'This conversation is too long for the model to process. Please start a new thread.',+ AiExceptionCode.CONTEXT_WINDOW_EXCEEDED,+ );+ }- this.metricsService.incrementCounterBy({- key: MetricsKeys.AiChatInputTokens,- amount: usage.inputTokens,- attributes: modelAttr,- });- this.metricsService.incrementCounterBy({- key: MetricsKeys.AiChatOutputTokens,- amount: usage.outputTokens,- attributes: modelAttr,- });- this.metricsService.incrementCounterBy({- key: MetricsKeys.AiChatCacheReadTokens,- amount: usage.cacheReadTokens,- attributes: modelAttr,- });- this.metricsService.incrementCounterBy({- key: MetricsKeys.AiChatCacheWriteTokens,- amount: cacheCreationTokens,- attributes: modelAttr,- });- this.metricsService.recordHistogram({- key: MetricsKeys.AiChatTurnLatencyMs,- value: performance.now() - streamStartedAt,- unit: 'ms',- attributes: modelAttr,- bucketBoundaries: AI_LATENCY_MS_BUCKET_BOUNDARIES,- });- };+ if (pruningResult.wasPruned) {+ onCompaction();+ }- const stream = streamText({- model: registeredModel.model,- instructions: [systemMessage, ...contexts],- messages: modelMessages,- tools: activeTools,+ return {+ systemPrompt: buildFullSystemPrompt({+ toolCatalog: toolset.toolCatalog,+ skillCatalog: await this.skillService.findAllFlatSkills(+ workspace.id,+ ),+ // inline tagged skills to save the model a load_skills round trip+ referencedSkills: await this.skillService.findFlatSkillsByIds(+ collectReferencedSkillIds(messages),+ workspace.id,+ ),+ preloadedTools: toolset.directToolNames,+ uploadedFilesContext: {+ uploadedFiles: collectUploadedFileReferences(messages),+ codeInterpreterFiles,+ },+ workspaceInstructions:+ workspace.aiAdditionalInstructions ?? undefined,+ userContext,+ userWorkspaceId,+ workspaceId: workspace.id,+ isWorkspaceSetupThread,+ canAttachConversationToRecords: isInboxConversation,+ }),+ contextMessages: contexts,+ messages: pruningResult.messages,+ };+ },+ promptCacheKey: threadId,+ shouldIncludeReasoningSummary: true,// Every step of the kickoff turn is forced so it cannot end in prose; stopWhen ends it at the first pausing tool call.toolChoice: isWorkspaceSetupKickoffTurn ? 'required' : 'auto',abortSignal,- stopWhen: (step) =>- isStepCount(AGENT_CONFIG.MAX_STEPS)(step) ||- endsOnPausingToolCall({ steps: step.steps }) ||- hasToolCall(COMPLETE_WORKSPACE_SETUP_TOOL_NAME)(step) ||- hasToolCall(NO_REPLY_TOOL_NAME)(step) ||- isDefined(stopReason),- ...buildAiTelemetry({+ shouldStopAfterStep: hasToolCall(+ COMPLETE_WORKSPACE_SETUP_TOOL_NAME,+ NO_REPLY_TOOL_NAME,+ ),+ billing: {+ workspaceId: workspace.id,+ operationType: UsageOperationType.AI_CHAT_TOKEN,+ webSearchOperationType: UsageOperationType.WEB_SEARCH,+ spenders: { userWorkspaceId },+ },+ onUserCreditsExhausted,+ telemetry: {functionId: isWorkspaceSetupThread? AI_CHAT_WORKSPACE_SETUP_STREAM_FUNCTION_ID: AI_CHAT_STREAM_FUNCTION_ID,@@ -613,187 +337,72 @@ export class ChatExecutionService {threadId,turnId,streamId,- }),- providerOptions: getCallLevelProviderOptions({- sdkPackage: registeredModel.sdkPackage,- providerOptions: this.aiModelConfigService.getReasoningProviderOptions(- registeredModel,- { shouldIncludeReasoningSummary: true },- ),- promptCacheKey: threadId,- }),- prepareStep: async ({ messages }) => {- await getAuthorization();- stepStartedAt = performance.now();-- return {- messages: injectCacheBreakpoint(- messages.map(sortKeysDeeply),- modelConfig,- ),- };},- onChunk: ({ chunk }) => {- if (- !ttftRecorded &&- (chunk.type === 'text-delta' || chunk.type === 'tool-call')- ) {- ttftRecorded = true;- this.metricsService.recordHistogram({- key: MetricsKeys.AiChatTtftMs,- value: performance.now() - streamStartedAt,- unit: 'ms',- attributes: { model: registeredModel.modelId },- bucketBoundaries: AI_LATENCY_MS_BUCKET_BOUNDARIES,- });- }- },- onError: ({ error }) => {+ beforeStep: getAuthorization,+ metricsScope: 'chat',+ onError: (error) => {lastUnderlyingStreamError = error;this.logger.error(`Stream ${streamId} emitted an error: ${formatErrorWithCause(error)}`,);},- onToolExecutionEnd: (event) => {- this.metricsService.recordHistogram({- key: MetricsKeys.AiChatToolExecutionDurationMs,- value: event.toolExecutionMs,- unit: 'ms',- attributes: {- model: registeredModel.modelId,- tool: getToolMetricName(event.toolCall.toolName),- },- bucketBoundaries: TOOL_EXECUTION_DURATION_MS_BUCKET_BOUNDARIES,- });- },- onStepEnd: async (step) => {- this.metricsService.recordHistogram({- key: MetricsKeys.AiChatStepLatencyMs,- value: performance.now() - stepStartedAt,- unit: 'ms',- attributes: { model: registeredModel.modelId },- bucketBoundaries: AI_LATENCY_MS_BUCKET_BOUNDARIES,- });-- const stepDebit =- await this.aiBillingService.decrementAndCheckAvailableCredits({- modelId: registeredModel.modelId,- billingInput: {- usage: step.usage,- cacheCreationTokens: extractCacheCreationTokens(- step.providerMetadata,- ),- },- workspaceId: workspace.id,- operationType: UsageOperationType.AI_CHAT_TOKEN,- spenders: { userWorkspaceId },- });-- stopReason ??= stepDebit.stopReason;- turnUserCreditsUsedMicro += stepDebit.userCreditsUsedMicro;-- await notifyUserCreditsCrossing(stepDebit);-- this.logger.log(- `[AI_CHAT_TOKENS] step #${++stepIndex} — ` +- `toolCallIds=[${step.toolCalls.map((toolCall) => toolCall.toolCallId).join(', ')}]: ` +- `outputTokens=${step.usage.outputTokens ?? 0}, ` +- `reasoningTokens=${step.usage.outputTokenDetails?.reasoningTokens ?? 0}, ` +- `inputTokens(fullContext)=${step.usage.inputTokens ?? 0}, ` +- `cacheReadTokens=${step.usage.inputTokenDetails?.cacheReadTokens ?? 0}, ` +- `cacheWriteTokens=${step.usage.inputTokenDetails?.cacheWriteTokens ?? 0}, ` +- `cacheCreationTokens=${extractCacheCreationTokens(step.providerMetadata)}, ` +- `totalTokens=${step.usage.totalTokens ?? 0}`,- );-- for (const part of step.content) {- if (part.type !== 'tool-result' && part.type !== 'tool-error') {- continue;- }-- const succeeded =- part.type === 'tool-result' && isToolOutputSuccessful(part.output);+ });- const outputTokens = estimateToolOutputTokens(- part.type === 'tool-result' ? part.output : part.error,- );+ return { ...turn, getStreamError: () => lastUnderlyingStreamError };+ }- const executionAttributes = {- model: registeredModel.modelId,- tool: getToolMetricName(resolveToolName(part)),- };-- this.metricsService.incrementCounterBy({- key: succeeded- ? MetricsKeys.AiChatToolExecutionSucceeded- : MetricsKeys.AiChatToolExecutionFailed,- amount: 1,- attributes: executionAttributes,- });-- this.metricsService.recordHistogram({- key: MetricsKeys.AiChatToolOutputTokens,- value: outputTokens,- unit: 'token',- attributes: executionAttributes,- bucketBoundaries: TOOL_OUTPUT_TOKENS_BUCKET_BOUNDARIES,- });- }- },- onAbort: async ({ steps }) => {- await emitTurnUsageEvent(steps);- },- repairToolCall: async ({- toolCall,- tools: toolsForRepair,- inputSchema,- error,- }) => {- return repairToolCall({- toolCall,- tools: toolsForRepair,- inputSchema,- error,- model: registeredModel.model,- billingContext: {- aiBillingService: this.aiBillingService,- modelId: registeredModel.modelId,- workspaceId: workspace.id,- spenders: { userWorkspaceId },- operationType: UsageOperationType.AI_CHAT_TOKEN,- },- });- },- });+ private prepareMessages({+ workspace,+ messages,+ modelConfig,+ browsingContext,+ workspaceMemberId,+ timezone,+ }: {+ workspace: WorkspaceEntity;+ messages: ExtendedUIMessage[];+ modelConfig: AiModelConfig;+ browsingContext: BrowsingContextType | null;+ workspaceMemberId: string;+ timezone: string | null;+ }): {+ preparedMessages: ExtendedUIMessage[];+ codeInterpreterFiles: UploadedFileReference[];+ } {+ const isCodeInterpreterEnabled = this.codeInterpreterService.isEnabled();- Promise.all([stream.usage, stream.steps])- .then(async ([, steps]) => {- await emitTurnUsageEvent(steps);- })- .catch((error) => {- if (error?.name === 'AbortError') {- return;- }+ let processedMessages = replaceUnsupportedFileParts(+ messages,+ modelConfig.modalities,+ isCodeInterpreterEnabled,+ );+ let codeInterpreterFiles: UploadedFileReference[] = [];- if (- error instanceof AiException &&- error.code === AiExceptionCode.STREAM_INTERRUPTED- ) {- return;- }+ if (isCodeInterpreterEnabled) {+ const extracted = extractCodeInterpreterFiles(processedMessages);- if (NoOutputGeneratedError.isInstance(error)) {- return;- }+ processedMessages = extracted.processedMessages;+ codeInterpreterFiles = extracted.extractedFiles.map(+ ({ filename, fileId }) => ({ filename, fileId }),+ );+ }- this.exceptionHandlerService.captureExceptions([error]);- });+ if (isDefined(browsingContext)) {+ processedMessages = this.injectBrowsingContextIntoLastUserMessage(+ processedMessages,+ this.buildContextFromBrowsingContext(workspace, browsingContext),+ );+ }return {- stream,- modelConfig,- getStopReason: () => stopReason,- getStreamError: () => lastUnderlyingStreamError,+ preparedMessages: injectMessageTimestamps(+ injectChatMessageSenders({+ messages: processedMessages,+ currentWorkspaceMemberId: workspaceMemberId,+ }),+ timezone,+ ),+ codeInterpreterFiles,};}diff --git a/packages/twenty-server/src/engine/metadata-modules/ai/ai-chat/utils/__tests__/classify-agent-chat-turn-outcome.util.spec.ts b/packages/twenty-server/src/engine/metadata-modules/ai/ai-chat/utils/__tests__/classify-agent-chat-turn-outcome.util.spec.tsindex c0d1e31869f..b5bb703e39b 100644--- a/packages/twenty-server/src/engine/metadata-modules/ai/ai-chat/utils/__tests__/classify-agent-chat-turn-outcome.util.spec.ts+++ b/packages/twenty-server/src/engine/metadata-modules/ai/ai-chat/utils/__tests__/classify-agent-chat-turn-outcome.util.spec.ts@@ -110,6 +110,41 @@ describe('classifyAgentChatTurnOutcome', () => {});});+describe('classifyAgentChatTurnOutcome for a background run', () => {+ const runTurn = {+ hasText: false,+ isAborted: false,+ isAwaitingUserAnswer: false,+ isSilent: false,+ isTextReplyRequired: false,+ };++ it('completes a run that ends on tool calls without writing a reply', () => {+ expect(+ classifyAgentChatTurnOutcome({ ...runTurn, stopReason: null }),+ ).toEqual({ kind: 'completed', outcome: 'answered' });+ });++ it('still fails a run that ran out of credits before writing anything', () => {+ expect(+ classifyAgentChatTurnOutcome({+ ...runTurn,+ stopReason: 'workspaceAllowance',+ }),+ ).toEqual({ kind: 'failed', failurePhase: 'credits_exhausted' });+ });++ it('fails a run that ran out of credits even after writing a reply', () => {+ expect(+ classifyAgentChatTurnOutcome({+ ...runTurn,+ hasText: true,+ stopReason: 'workspaceAllowance',+ }),+ ).toEqual({ kind: 'failed', failurePhase: 'credits_exhausted' });+ });+});+describe('resolveSupersededTurnOutcome', () => {it('keeps a credits failure rather than filing it as a cancellation', () => {expect(diff --git a/packages/twenty-server/src/engine/metadata-modules/ai/ai-chat/utils/classify-agent-chat-turn-outcome.util.ts b/packages/twenty-server/src/engine/metadata-modules/ai/ai-chat/utils/classify-agent-chat-turn-outcome.util.tsindex 149c8a177cb..c006ccea24c 100644--- a/packages/twenty-server/src/engine/metadata-modules/ai/ai-chat/utils/classify-agent-chat-turn-outcome.util.ts+++ b/packages/twenty-server/src/engine/metadata-modules/ai/ai-chat/utils/classify-agent-chat-turn-outcome.util.ts@@ -1,18 +1,27 @@+import { isDefined } from 'twenty-shared/utils';+import { type StepDebitResult } from 'src/engine/metadata-modules/ai/ai-billing/types/step-debit-result.type';import { type AgentChatTurnOutcome } from 'src/engine/metadata-modules/ai/ai-chat/types/agent-chat-turn-outcome.type';+const CREDITS_FAILURE_PHASE = {+ userAllowance: 'user_credits_exhausted',+ workspaceAllowance: 'credits_exhausted',+} as const satisfies Record<NonNullable<StepDebitResult['stopReason']>, string>;+export const classifyAgentChatTurnOutcome = ({hasText,isAborted,isAwaitingUserAnswer,isSilent,stopReason,+ isTextReplyRequired = true,}: {hasText: boolean;isAborted: boolean;isAwaitingUserAnswer: boolean;isSilent: boolean;stopReason: StepDebitResult['stopReason'];+ isTextReplyRequired?: boolean;}): AgentChatTurnOutcome => {// stopWhen ends the stream on a question, so this is a completed turn, not an abandoned oneif (isAwaitingUserAnswer) {@@ -23,6 +32,11 @@ export const classifyAgentChatTurnOutcome = ({return { kind: 'cancelled', reason: 'user_cancelled' };}+ // a background run that ran out of credits fails, whatever it wrote before+ if (!isTextReplyRequired && isDefined(stopReason)) {+ return { kind: 'failed', failurePhase: CREDITS_FAILURE_PHASE[stopReason] };+ }+if (hasText) {return { kind: 'completed', outcome: 'answered' };}@@ -31,14 +45,16 @@ export const classifyAgentChatTurnOutcome = ({return { kind: 'completed', outcome: 'silent' };}- if (stopReason === 'userAllowance') {- return { kind: 'failed', failurePhase: 'user_credits_exhausted' };+ // a background run's outcome is its result, so it may end on tool calls without a written reply+ if (!isTextReplyRequired) {+ return { kind: 'completed', outcome: 'answered' };}return {kind: 'failed',- failurePhase:- stopReason === 'workspaceAllowance' ? 'credits_exhausted' : 'no_text',+ failurePhase: isDefined(stopReason)+ ? CREDITS_FAILURE_PHASE[stopReason]+ : 'no_text',};};diff --git a/packages/twenty-server/src/engine/metadata-modules/ai/ai-chat/utils/has-reply-text.util.ts b/packages/twenty-server/src/engine/metadata-modules/ai/ai-chat/utils/has-reply-text.util.tsnew file mode 100644index 00000000000..218e87e434c--- /dev/null+++ b/packages/twenty-server/src/engine/metadata-modules/ai/ai-chat/utils/has-reply-text.util.ts@@ -0,0 +1,5 @@+import { isNonEmptyString } from '@sniptt/guards';+import { type ExtendedUIMessagePart } from 'twenty-shared/ai';++export const hasReplyText = (parts: ExtendedUIMessagePart[]): boolean =>+ parts.some((part) => part.type === 'text' && isNonEmptyString(part.text));diff --git a/packages/twenty-server/src/engine/metadata-modules/ai/ai-history/services/__tests__/agent-turn-recorder.service.finish-executed-turn.spec.ts b/packages/twenty-server/src/engine/metadata-modules/ai/ai-history/services/__tests__/agent-turn-recorder.service.finish-executed-turn.spec.tsdeleted file mode 100644index 2691a32e0d3..00000000000--- a/packages/twenty-server/src/engine/metadata-modules/ai/ai-history/services/__tests__/agent-turn-recorder.service.finish-executed-turn.spec.ts+++ /dev/null@@ -1,96 +0,0 @@-import { AgentTurnStatus } from 'src/engine/metadata-modules/ai/ai-history/enums/agent-turn-status.enum';-import { AgentTurnRecorderService } from 'src/engine/metadata-modules/ai/ai-history/services/agent-turn-recorder.service';--const TURN_USAGE = {- inputTokens: 10,- outputTokens: 5,- cacheReadTokens: 0,- cacheCreationTokens: 0,- inputCredits: 100,- outputCredits: 50,-};--const buildService = () => {- const service = new AgentTurnRecorderService({} as never);- const recordUsage = jest- .spyOn(service, 'recordUsage')- .mockResolvedValue(undefined);- const finish = jest.spyOn(service, 'finish').mockResolvedValue(true);-- return { service, recordUsage, finish };-};--const turn = {- workspaceId: 'workspace-id',- threadId: 'thread-id',- turnId: 'turn-id',-};--describe('AgentTurnRecorderService.finishExecutedTurn', () => {- it('records the usage and completes the turn', async () => {- const { service, recordUsage, finish } = buildService();-- await service.finishExecutedTurn({- ...turn,- execution: { modelId: 'model-id', turnUsage: TURN_USAGE },- });-- expect(recordUsage).toHaveBeenCalledWith({ ...turn, usage: TURN_USAGE });- expect(finish).toHaveBeenCalledWith({- workspaceId: 'workspace-id',- turnId: 'turn-id',- modelId: 'model-id',- status: AgentTurnStatus.COMPLETED,- });- });-- it('leaves a turn that asked a question waiting', async () => {- const { service, finish } = buildService();-- await service.finishExecutedTurn({- ...turn,- execution: { turnUsage: TURN_USAGE },- isWaiting: true,- });-- expect(finish).toHaveBeenCalledWith(- expect.objectContaining({ status: AgentTurnStatus.WAITING_FOR_INPUT }),- );- });-- it('fails a turn that ran out of credits', async () => {- const { service, finish } = buildService();-- await service.finishExecutedTurn({- ...turn,- execution: { turnUsage: TURN_USAGE, hasNoMoreAvailableCredits: true },- isWaiting: true,- });-- expect(finish).toHaveBeenCalledWith(- expect.objectContaining({- status: AgentTurnStatus.FAILED,- error: expect.objectContaining({ code: 'CREDITS_EXHAUSTED' }),- }),- );- });-- it('still ends the turn when its usage cannot be written', async () => {- const { service, recordUsage, finish } = buildService();-- recordUsage.mockRejectedValue(new Error('database down'));-- await service.finishExecutedTurn({- ...turn,- execution: { turnUsage: TURN_USAGE },- error: { code: 'EXECUTION_FAILED', message: 'Reply not saved.' },- });-- expect(finish).toHaveBeenCalledWith(- expect.objectContaining({- status: AgentTurnStatus.FAILED,- error: { code: 'EXECUTION_FAILED', message: 'Reply not saved.' },- }),- );- });-});diff --git a/packages/twenty-server/src/engine/metadata-modules/ai/ai-history/services/agent-conversation-reader.service.ts b/packages/twenty-server/src/engine/metadata-modules/ai/ai-history/services/agent-conversation-reader.service.tsindex 7be383be346..efbd5a40c3d 100644--- a/packages/twenty-server/src/engine/metadata-modules/ai/ai-history/services/agent-conversation-reader.service.ts+++ b/packages/twenty-server/src/engine/metadata-modules/ai/ai-history/services/agent-conversation-reader.service.ts@@ -14,6 +14,7 @@ import { finalizeDanglingToolParts } from 'src/engine/metadata-modules/ai/ai-hisimport { mapDBPartsToUIMessageParts } from 'src/engine/metadata-modules/ai/ai-history/utils/map-db-parts-to-ui-message-parts.util';import { AgentHistoryRepository } from 'src/engine/metadata-modules/ai/ai-history/repositories/agent-history-repository';import { InjectAgentHistoryRepository } from 'src/engine/metadata-modules/ai/ai-history/repositories/inject-agent-history-repository.decorator';+import { AgentMessageStatus } from 'src/engine/metadata-modules/ai/ai-history/enums/agent-message-status.enum';import { AgentTurnStatus } from 'src/engine/metadata-modules/ai/ai-history/enums/agent-turn-status.enum';import { type AgentMessagePartWorkspaceEntity } from 'src/engine/metadata-modules/ai/ai-history/standard-objects/agent-message-part.workspace-entity';import { type AgentMessageWorkspaceEntity } from 'src/engine/metadata-modules/ai/ai-history/standard-objects/agent-message.workspace-entity';@@ -92,10 +93,13 @@ export class AgentConversationReaderService {}),]);- // a failed run is kept on record but leaves the conversation as it was+ // a failed run is kept on record but leaves the conversation as it was, and a chat message+ // queued behind a stream is not sent yetconst failedTurnIds = new Set(failedTurns.map(({ id }) => id));const messages = threadMessages.filter(- ({ turnId }) => !isDefined(turnId) || !failedTurnIds.has(turnId),+ ({ turnId, status }) =>+ status !== AgentMessageStatus.QUEUED &&+ (!isDefined(turnId) || !failedTurnIds.has(turnId)),);const turnIdsActedByOthers = isDefined(actor)@@ -121,7 +125,7 @@ export class AgentConversationReaderService {);}- private signFileParts({+ signFileParts({workspaceId,parts,}: {diff --git a/packages/twenty-server/src/engine/metadata-modules/ai/ai-history/services/agent-conversation-writer.service.ts b/packages/twenty-server/src/engine/metadata-modules/ai/ai-history/services/agent-conversation-writer.service.tsindex a0a3cad7c2c..5778ac4b8b7 100644--- a/packages/twenty-server/src/engine/metadata-modules/ai/ai-history/services/agent-conversation-writer.service.ts+++ b/packages/twenty-server/src/engine/metadata-modules/ai/ai-history/services/agent-conversation-writer.service.ts@@ -7,8 +7,6 @@ import { type ActorMetadata } from 'twenty-shared/types';import { isDefined } from 'twenty-shared/utils';import { IsNull } from 'typeorm';-import { findAwaitingPausingToolParts } from 'src/engine/metadata-modules/ai/ai-agent-execution/pausing-tools/utils/find-awaiting-pausing-tool-parts.util';-import { mapAiStepsToUIMessageParts } from 'src/engine/metadata-modules/ai/ai-history/utils/map-ai-steps-to-ui-message-parts.util';import { mapUIMessagePartsToDBParts } from 'src/engine/metadata-modules/ai/ai-history/utils/map-ui-message-parts-to-db-parts.util';import { AgentMessageRole } from 'src/engine/metadata-modules/ai/ai-history/enums/agent-message-role.enum';import { AgentTurnStatus } from 'src/engine/metadata-modules/ai/ai-history/enums/agent-turn-status.enum';@@ -17,7 +15,6 @@ import { InjectAgentHistoryRepository } from 'src/engine/metadata-modules/ai/ai-import { AgentHistoryTransactionService } from 'src/engine/metadata-modules/ai/ai-history/services/agent-history-transaction.service';import { type AgentTurnWorkspaceEntity } from 'src/engine/metadata-modules/ai/ai-history/standard-objects/agent-turn.workspace-entity';import { type AgentHistoryTransactionScope } from 'src/engine/metadata-modules/ai/ai-history/types/agent-history-transaction-scope.type';-import { type RecordableAgentExecution } from 'src/engine/metadata-modules/ai/ai-history/types/recordable-agent-execution.type';import { isAgentTurnStatusFinal } from 'src/engine/metadata-modules/ai/ai-history/utils/is-agent-turn-status-final.util';import { WorkspaceCacheService } from 'src/engine/workspace-cache/services/workspace-cache.service';import {@@ -41,6 +38,42 @@ export class AgentConversationWriterService {return this.transactionService.run(workspaceId, work);}+ // the message is replaced whole, so no reader or crash ever finds it without its parts+ async upsertAssistantMessage({+ workspaceId,+ id,+ threadId,+ turnId,+ parts,+ }: {+ workspaceId: string;+ id: string;+ threadId: string;+ turnId: string;+ parts: ExtendedUIMessagePart[];+ }): Promise<void> {+ const dbParts = mapUIMessagePartsToDBParts(parts, id);++ await this.transactionService.run(workspaceId, async (scope) => {+ await scope.upsert(+ 'agentMessage',+ {+ id,+ threadId,+ turnId,+ role: AgentMessageRole.ASSISTANT,+ processedAt: new Date().toISOString(),+ },+ ['id'],+ );+ await scope.delete('agentMessagePart', { messageId: id });++ if (dbParts.length > 0) {+ await scope.insert('agentMessagePart', dbParts);+ }+ });+ }+async insertTurn({workspaceId,threadId,@@ -215,52 +248,6 @@ export class AgentConversationWriterService {return messageId;}- // One unanswerable call would keep the conversation waiting forever, so it is recorded as not waiting- async insertExecutionReply({- workspaceId,- threadId,- turnId,- agentId,- execution,- scope,- }: {- workspaceId: string;- threadId: string;- turnId: string;- agentId: string | null;- execution: RecordableAgentExecution;- scope?: AgentHistoryTransactionScope;- }): Promise<{- isAwaitingAnswer: boolean;- replyParts: ExtendedUIMessagePart[];- }> {- const replyParts = mapAiStepsToUIMessageParts(execution.steps ?? []);-- if (replyParts.length === 0) {- return { isAwaitingAnswer: false, replyParts };- }-- const awaitingParts = findAwaitingPausingToolParts(replyParts);- const isAwaitingAnswer =- execution.isPaused === true &&- awaitingParts.length > 0 &&- awaitingParts.every(({ isAnswerable }) => isAnswerable);-- await this.insertMessage({- workspaceId,- threadId,- turnId,- role: AgentMessageRole.ASSISTANT,- agentId,- senderUserWorkspaceId: null,- isAwaitingAnswer,- parts: replyParts,- scope,- });-- return { isAwaitingAnswer, replyParts };- }-// Callers name the sender by their membership; a sender who has left since// is recorded without oneprivate async findSenderWorkspaceMemberId({diff --git a/packages/twenty-server/src/engine/metadata-modules/ai/ai-history/services/agent-turn-recorder.service.ts b/packages/twenty-server/src/engine/metadata-modules/ai/ai-history/services/agent-turn-recorder.service.tsindex bd2832aa48f..cd2bba2101f 100644--- a/packages/twenty-server/src/engine/metadata-modules/ai/ai-history/services/agent-turn-recorder.service.ts+++ b/packages/twenty-server/src/engine/metadata-modules/ai/ai-history/services/agent-turn-recorder.service.ts@@ -3,7 +3,7 @@ import { Injectable, Logger } from '@nestjs/common';import { type ActorMetadata, FieldActorSource } from 'twenty-shared/types';import { isDefined } from 'twenty-shared/utils';-import { AGENT_TURN_CREDITS_EXHAUSTED_ERROR } from 'src/engine/metadata-modules/ai/ai-history/constants/agent-turn-credits-exhausted-error.constant';+import { formatErrorWithCause } from 'src/engine/metadata-modules/ai/ai-chat/utils/format-error-with-cause.util';import { type StreamErrorPayload } from 'src/engine/metadata-modules/ai/ai-history/utils/map-error-to-stream-error.util';import { AgentTurnStatus } from 'src/engine/metadata-modules/ai/ai-history/enums/agent-turn-status.enum';import { type AgentHistoryStorageContext } from 'src/engine/metadata-modules/ai/ai-history/services/agent-history-workspace-storage.service';@@ -11,7 +11,6 @@ import { AgentHistoryRepository } from 'src/engine/metadata-modules/ai/ai-historimport { InjectAgentHistoryRepository } from 'src/engine/metadata-modules/ai/ai-history/repositories/inject-agent-history-repository.decorator';import { type AgentTurnWorkspaceEntity } from 'src/engine/metadata-modules/ai/ai-history/standard-objects/agent-turn.workspace-entity';import { type AgentTurnUsage } from 'src/engine/metadata-modules/ai/ai-billing/types/agent-turn-usage.type';-import { type RecordableAgentExecution } from 'src/engine/metadata-modules/ai/ai-history/types/recordable-agent-execution.type';import { buildEndWaitingAgentTurnQuery } from 'src/engine/metadata-modules/ai/ai-history/utils/build-end-waiting-agent-turn-query.util';import { buildReleaseStreamClaimQuery } from 'src/engine/metadata-modules/ai/ai-history/utils/build-release-stream-claim-query.util';import { isAgentTurnStatusFinal } from 'src/engine/metadata-modules/ai/ai-history/utils/is-agent-turn-status-final.util';@@ -141,57 +140,8 @@ export class AgentTurnRecorderService {);}- // Ends a turn run outside chat. Its credits were spent even when its reply- // could not be saved, and a failed usage write must not leave it running- async finishExecutedTurn({- workspaceId,- threadId,- turnId,- execution,- isWaiting = false,- error,- }: {- workspaceId: string;- threadId: string;- turnId: string;- execution: RecordableAgentExecution;- isWaiting?: boolean;- error?: StreamErrorPayload;- }): Promise<void> {- if (isDefined(execution.turnUsage)) {- await this.recordUsage({- workspaceId,- threadId,- turnId,- usage: execution.turnUsage,- }).catch((usageError: unknown) => {- this.logger.error(- `Failed to record the usage of agent turn ${turnId}: ${usageError instanceof Error ? usageError.message : String(usageError)}`,- );- });- }-- const failureError =- error ??- (execution.hasNoMoreAvailableCredits === true- ? AGENT_TURN_CREDITS_EXHAUSTED_ERROR- : undefined);-- await this.finish({- workspaceId,- turnId,- modelId: execution.modelId,- ...(isDefined(failureError)- ? { status: AgentTurnStatus.FAILED, error: failureError }- : {- status: isWaiting- ? AgentTurnStatus.WAITING_FOR_INPUT- : AgentTurnStatus.COMPLETED,- }),- });- }-- // Credits were spent whatever became of the turn, so usage is never guarded by a stream claim+ // Credits were spent whatever became of the turn, so usage is never guarded by a stream claim, and a+ // failure to count it must not fail a reply already savedasync recordUsage({workspaceId,turnId,@@ -203,10 +153,11 @@ export class AgentTurnRecorderService {threadId: string;usage: AgentTurnUsage;}): Promise<void> {- await this.turnRepository.query(workspaceId, async ({ manager, table }) => {- // sum in Postgres: JS numbers must never be added to exact NUMERIC totals- await manager.query(- `WITH turn AS (+ await this.turnRepository+ .query(workspaceId, async ({ manager, table }) => {+ // sum in Postgres: JS numbers must never be added to exact NUMERIC totals+ await manager.query(+ `WITH turn AS (UPDATE ${table('agentTurn')} SET"inputTokens" = COALESCE("inputTokens", 0) + $3,"outputTokens" = COALESCE("outputTokens", 0) + $4,@@ -226,18 +177,23 @@ export class AgentTurnRecorderService {"totalInputCredits" = "totalInputCredits" + $7,"totalOutputCredits" = "totalOutputCredits" + $8WHERE id = $2`,- [- turnId,- threadId,- usage.inputTokens,- usage.outputTokens,- usage.cacheReadTokens,- usage.cacheCreationTokens,- usage.inputCredits,- usage.outputCredits,- ],+ [+ turnId,+ threadId,+ usage.inputTokens,+ usage.outputTokens,+ usage.cacheReadTokens,+ usage.cacheCreationTokens,+ usage.inputCredits,+ usage.outputCredits,+ ],+ );+ })+ .catch((error: unknown) =>+ this.logger.warn(+ `Could not record the usage of turn ${turnId}: ${formatErrorWithCause(error)}`,+ ),);- });}// the waiting turn is the one holding the question, whoever ends the waitdiff --git a/packages/twenty-server/src/engine/metadata-modules/ai/ai-history/types/recordable-agent-execution.type.ts b/packages/twenty-server/src/engine/metadata-modules/ai/ai-history/types/recordable-agent-execution.type.tsdeleted file mode 100644index 7969c1d1c7c..00000000000--- a/packages/twenty-server/src/engine/metadata-modules/ai/ai-history/types/recordable-agent-execution.type.ts+++ /dev/null@@ -1,12 +0,0 @@-import { type StepResult, type ToolSet } from 'ai';--import { type AgentTurnUsage } from 'src/engine/metadata-modules/ai/ai-billing/types/agent-turn-usage.type';--// what recording a turn reads from an execution; usage is absent when the execution ran elsewhere-export type RecordableAgentExecution = {- steps?: Pick<StepResult<ToolSet>, 'content'>[];- isPaused?: boolean;- modelId?: string;- turnUsage?: AgentTurnUsage;- hasNoMoreAvailableCredits?: boolean;-};diff --git a/packages/twenty-server/src/engine/metadata-modules/ai/ai-tool-call-answer/services/tool-call-answer.service.ts b/packages/twenty-server/src/engine/metadata-modules/ai/ai-tool-call-answer/services/tool-call-answer.service.tsindex 2926af1c9e6..318e1b75aa4 100644--- a/packages/twenty-server/src/engine/metadata-modules/ai/ai-tool-call-answer/services/tool-call-answer.service.ts+++ b/packages/twenty-server/src/engine/metadata-modules/ai/ai-tool-call-answer/services/tool-call-answer.service.ts@@ -16,7 +16,6 @@ import { type ToolContext } from 'src/engine/core-modules/tool-provider/types/toimport { type WorkspaceEntity } from 'src/engine/core-modules/workspace/workspace.entity';import { AgentMessageRole } from 'src/engine/metadata-modules/ai/ai-history/enums/agent-message-role.enum';import { type PausingToolCompletionContext } from 'src/engine/metadata-modules/ai/ai-agent-execution/pausing-tools/types/pausing-tool-completion-context.type';-import { AgentRunConversationService } from 'src/engine/metadata-modules/ai/ai-agent-execution/services/agent-run-conversation.service';import { isAwaitingPausingToolOutput } from 'src/engine/metadata-modules/ai/ai-history/utils/is-awaiting-pausing-tool-output.util';import { parsePausingToolCall } from 'src/engine/metadata-modules/ai/ai-agent-execution/pausing-tools/utils/parse-pausing-tool-call.util';import { readProposedToolCallAnswer } from 'src/engine/metadata-modules/ai/ai-agent-execution/pausing-tools/utils/read-proposed-tool-call-answer.util';@@ -79,22 +78,11 @@ export class ToolCallAnswerService {private readonly toolRegistryService: ToolRegistryService,private readonly conversationReaderService: AgentConversationReaderService,private readonly threadLifecycleService: AgentChatThreadLifecycleService,- private readonly agentRunConversationService: AgentRunConversationService,) {}- // a run holds its conversation until it has suspended, so an answer to the question it just- // asked finds the run waiting on it rather than starting a chat- answer(args: AnswerToolCallArgs): Promise<AnswerToolCallOutcome> {- return this.agentRunConversationService.withThreadLockForMessage({- workspaceId: args.workspace.id,- threadId: args.threadId,- work: () => this.answerUnderLock(args),- });- }-- private async answerUnderLock(- args: AnswerToolCallArgs,- ): Promise<AnswerToolCallOutcome> {+ // a run holds its conversation's stream until it has suspended, so an answer that comes sooner is+ // turned away as busy rather than starting a chat turn+ async answer(args: AnswerToolCallArgs): Promise<AnswerToolCallOutcome> {const { threadId, toolCallId, userWorkspaceId, workspace } = args;const workspaceId = workspace.id;@@ -159,13 +147,12 @@ export class ToolCallAnswerService {const streamId = generateId();- const hasClaimedStream =- await this.agentChatStreamingService.tryClaimStream({- threadId,- workspaceId,- streamId,- where: { pendingQuestionMessageId: toolPart.messageId },- });+ const hasClaimedStream = await this.streamRecoveryService.tryClaimStream({+ threadId,+ workspaceId,+ streamId,+ where: { pendingQuestionMessageId: toolPart.messageId },+ });if (!hasClaimedStream) {throw new AiException(diff --git a/packages/twenty-server/src/engine/workspace-manager/dev-seeder/data/services/dev-seeder-workflow-pending-input.workspace-service.ts b/packages/twenty-server/src/engine/workspace-manager/dev-seeder/data/services/dev-seeder-workflow-pending-input.workspace-service.tsindex eae9e5f5ca4..05a0b364811 100644--- a/packages/twenty-server/src/engine/workspace-manager/dev-seeder/data/services/dev-seeder-workflow-pending-input.workspace-service.ts+++ b/packages/twenty-server/src/engine/workspace-manager/dev-seeder/data/services/dev-seeder-workflow-pending-input.workspace-service.ts@@ -15,6 +15,9 @@ import {import { WorkflowEntity } from 'src/engine/core-modules/workflow/entities/workflow.entity';import { AgentRunConversationService } from 'src/engine/metadata-modules/ai/ai-agent-execution/services/agent-run-conversation.service';import { AgentRunSuspensionService } from 'src/engine/metadata-modules/ai/ai-agent-execution/services/agent-run-suspension.service';+import { AgentChatThreadService } from 'src/engine/metadata-modules/ai/ai-chat/services/agent-chat-thread.service';+import { AgentMessageRole } from 'src/engine/metadata-modules/ai/ai-history/enums/agent-message-role.enum';+import { AgentConversationWriterService } from 'src/engine/metadata-modules/ai/ai-history/services/agent-conversation-writer.service';import { AgentService } from 'src/engine/metadata-modules/ai/ai-agent/agent.service';import {AiException,@@ -36,6 +39,7 @@ import {type WorkflowAiAgentAction,} from 'src/modules/workflow/workflow-executor/workflow-actions/types/workflow-action.type';import { WorkflowRunWorkspaceService } from 'src/modules/workflow/workflow-runner/workflow-run/workflow-run.workspace-service';+import { mapAiStepsToUIMessageParts } from 'src/engine/metadata-modules/ai/ai-history/utils/map-ai-steps-to-ui-message-parts.util';import {type WorkflowManualTrigger,WorkflowTriggerType,@@ -163,6 +167,8 @@ export class DevSeederWorkflowPendingInputWorkspaceService {private readonly workflowAgentConversationService: WorkflowAgentConversationWorkspaceService,private readonly agentRunConversationService: AgentRunConversationService,private readonly agentRunSuspensionService: AgentRunSuspensionService,+ private readonly conversationWriterService: AgentConversationWriterService,+ private readonly threadService: AgentChatThreadService,private readonly agentService: AgentService,@InjectWorkspaceScopedRepository(WorkflowEntity)private readonly coreWorkflowRepository: WorkspaceScopedRepository<WorkflowEntity>,@@ -276,10 +282,9 @@ export class DevSeederWorkflowPendingInputWorkspaceService {throw error;});- const turnId = await this.agentRunConversationService.openTurn({+ const { turnId } = await this.agentRunConversationService.openTurn({workspaceId,threadId,- title: workflow.step.name,agentId: agent.id,senderUserWorkspaceId: null,senderApplicationId: null,@@ -309,16 +314,21 @@ export class DevSeederWorkflowPendingInputWorkspaceService {},});- await this.agentRunConversationService.closeTurn({+ await this.conversationWriterService.insertMessage({workspaceId,threadId,turnId,- title: workflow.step.name,+ role: AgentMessageRole.ASSISTANT,agentId: agent.id,- execution: {- isPaused: true,- steps: [{ content }],- },+ senderUserWorkspaceId: null,+ isAwaitingAnswer: true,+ parts: mapAiStepsToUIMessageParts([{ content }]),+ });++ await this.threadService.recordThreadActivity({+ workspaceId,+ threadId,+ text: workflow.step.name,});},});diff --git a/packages/twenty-server/src/engine/workspace-manager/dev-seeder/dev-seeder.module.ts b/packages/twenty-server/src/engine/workspace-manager/dev-seeder/dev-seeder.module.tsindex f6a388730d4..be930975d52 100644--- a/packages/twenty-server/src/engine/workspace-manager/dev-seeder/dev-seeder.module.ts+++ b/packages/twenty-server/src/engine/workspace-manager/dev-seeder/dev-seeder.module.ts@@ -1,5 +1,6 @@import { AiAgentExecutionModule } from 'src/engine/metadata-modules/ai/ai-agent-execution/ai-agent-execution.module';import { AiAgentModule } from 'src/engine/metadata-modules/ai/ai-agent/ai-agent.module';+import { AgentChatThreadModule } from 'src/engine/metadata-modules/ai/ai-chat/agent-chat-thread.module';import { AgentHistoryModule } from 'src/engine/metadata-modules/ai/ai-history/ai-history.module';import { Module } from '@nestjs/common';import { TypeOrmModule } from '@nestjs/typeorm';@@ -44,6 +45,7 @@ import { WorkflowRunModule } from 'src/modules/workflow/workflow-runner/workflow@Module({imports: [+ AgentChatThreadModule,AgentHistoryModule,AiAgentExecutionModule,AiAgentModule,diff --git a/packages/twenty-server/src/modules/workflow/workflow-executor/workflow-actions/ai-agent/ai-agent.workflow-action.ts b/packages/twenty-server/src/modules/workflow/workflow-executor/workflow-actions/ai-agent/ai-agent.workflow-action.tsindex f441a6e5319..1014e04ac64 100644--- a/packages/twenty-server/src/modules/workflow/workflow-executor/workflow-actions/ai-agent/ai-agent.workflow-action.ts+++ b/packages/twenty-server/src/modules/workflow/workflow-executor/workflow-actions/ai-agent/ai-agent.workflow-action.ts@@ -6,6 +6,7 @@ import { isDefined, isValidUuid, resolveInput } from 'twenty-shared/utils';import { type WorkflowAction } from 'src/modules/workflow/workflow-executor/interfaces/workflow-action.interface';import { AgentRunnerService } from 'src/engine/metadata-modules/ai/ai-agent-execution/services/agent-runner.service';+import { buildStartedAgentRunSummary } from 'src/engine/metadata-modules/ai/ai-agent-execution/utils/build-started-agent-run-summary.util';import { AgentEntity } from 'src/engine/metadata-modules/ai/ai-agent/entities/agent.entity';import { InjectWorkspaceScopedRepository } from 'src/engine/twenty-orm/workspace-scoped-repository/inject-workspace-scoped-repository.decorator';import { WorkspaceScopedRepository } from 'src/engine/twenty-orm/workspace-scoped-repository/workspace-scoped-repository';@@ -22,14 +23,14 @@ import { findStepOrThrow } from 'src/modules/workflow/workflow-executor/utils/fiimport { resolveConversationThreadKey } from 'src/modules/workflow/workflow-executor/utils/resolve-conversation-thread-key.util';import { WorkflowAgentConversationWorkspaceService } from 'src/modules/workflow/workflow-executor/workflow-actions/ai-agent/services/workflow-agent-conversation.workspace-service';import { type WorkflowAiAgentActionInput } from 'src/modules/workflow/workflow-executor/workflow-actions/ai-agent/types/workflow-ai-agent-action-input.type';-import { buildWorkflowAgentRunExecutionContext } from 'src/modules/workflow/workflow-executor/workflow-actions/ai-agent/utils/build-workflow-agent-run-execution-context.util';+import { getWorkflowRunUserWorkspaceId } from 'src/modules/workflow/workflow-executor/workflow-actions/ai-agent/utils/get-workflow-run-user-workspace-id.util';import { buildWorkflowAgentRunSpec } from 'src/modules/workflow/workflow-executor/workflow-actions/ai-agent/utils/build-workflow-agent-run-spec.util';import { WorkflowRunStepLogWorkspaceService } from 'src/modules/workflow/workflow-runner/workflow-run/workflow-run-step-log.workspace-service';import { isWorkflowAiAgentAction } from 'src/modules/workflow/workflow-executor/workflow-actions/ai-agent/guards/is-workflow-ai-agent-action.guard';-// Starts the step's agent and hands it to the engine: an agent that pauses is continued by the-// engine, which calls the step back with its outcome+// Starts the step's agent and hands it to the engine, which runs it on its own, continues it when it+// pauses and calls the step back with its outcome@Injectable()export class AiAgentWorkflowAction implements WorkflowAction {constructor(@@ -131,14 +132,18 @@ export class AiAgentWorkflowAction implements WorkflowAction {}),});- const agentExecutionContext = buildWorkflowAgentRunExecutionContext({- executionContext,- turnCreatedBy:- await this.workflowAgentConversationService.findTurnCreatedBy(runInfo),- });const resolvedPrompt = resolveInput(prompt, context) as string | undefined;- const { threadId, outcome, summary } = await this.agentRunnerService.run({+ // the conversation is linked from the start, so the run can be followed while it goes on+ await this.workflowRunStepLogService.setAiAgentStepLog({+ workflowRunId: runInfo.workflowRunId,+ workspaceId,+ stepId: currentStepId,+ summary: buildStartedAgentRunSummary(agent.modelId),+ threadId: conversation.threadId,+ });++ await this.agentRunnerService.start({workspaceId,conversation,caller: buildWorkflowStepCaller({@@ -149,32 +154,16 @@ export class AiAgentWorkflowAction implements WorkflowAction {step,isApplicationBound: isDefined(application),}),- agent,prompt: isDefined(resolvedPrompt)? {messages: [{ role: 'user', content: resolvedPrompt }],- senderUserWorkspaceId: agentExecutionContext.userWorkspaceId,+ senderUserWorkspaceId:+ getWorkflowRunUserWorkspaceId(executionContext),senderApplicationId: null,}: null,- executionContext: agentExecutionContext,});- await this.workflowRunStepLogService.setAiAgentStepLog({- workflowRunId: runInfo.workflowRunId,- workspaceId,- stepId: currentStepId,- summary,- threadId,- });-- switch (outcome.status) {- case 'COMPLETED':- return { result: outcome.result };- case 'SUSPENDED':- return { wait: { type: 'CALLBACK' } };- case 'FAILED':- return { error: outcome.error };- }+ return { wait: { type: 'CALLBACK' } };}}diff --git a/packages/twenty-server/src/modules/workflow/workflow-executor/workflow-actions/ai-agent/utils/build-workflow-agent-run-execution-context.util.ts b/packages/twenty-server/src/modules/workflow/workflow-executor/workflow-actions/ai-agent/utils/build-workflow-agent-run-execution-context.util.tsindex 1b03552dc3b..5ab274e6cc4 100644--- a/packages/twenty-server/src/modules/workflow/workflow-executor/workflow-actions/ai-agent/utils/build-workflow-agent-run-execution-context.util.ts+++ b/packages/twenty-server/src/modules/workflow/workflow-executor/workflow-actions/ai-agent/utils/build-workflow-agent-run-execution-context.util.ts@@ -6,6 +6,7 @@ import { type AgentRunExecutionContext } from 'src/engine/metadata-modules/ai/aiimport { type AgentRunRoleIds } from 'src/engine/metadata-modules/ai/ai-agent-execution/types/agent-run-role-ids.type';import { type RolePermissionConfig } from 'src/engine/twenty-orm/types/role-permission-config.type';import { type WorkflowExecutionContext } from 'src/modules/workflow/workflow-executor/types/workflow-execution-context.type';+import { getWorkflowRunUserWorkspaceId } from 'src/modules/workflow/workflow-executor/workflow-actions/ai-agent/utils/get-workflow-run-user-workspace-id.util';// A run that bypasses permission checks adds no restriction to the agent's roleconst getAgentRunRoleIdsFromRolePermissionConfig = (@@ -34,10 +35,7 @@ export const buildWorkflowAgentRunExecutionContext = ({? executionContext.initiator: undefined,turnCreatedBy,- userWorkspaceId:- executionContext.authContext.type === 'user'- ? executionContext.authContext.userWorkspaceId- : null,+ userWorkspaceId: getWorkflowRunUserWorkspaceId(executionContext),rolePermissionConfig: executionContext.rolePermissionConfig,usageOperationType: UsageOperationType.AI_WORKFLOW_TOKEN,agentRunRoleIds: getAgentRunRoleIdsFromRolePermissionConfig(diff --git a/packages/twenty-server/src/modules/workflow/workflow-executor/workflow-actions/ai-agent/utils/get-workflow-run-user-workspace-id.util.ts b/packages/twenty-server/src/modules/workflow/workflow-executor/workflow-actions/ai-agent/utils/get-workflow-run-user-workspace-id.util.tsnew file mode 100644index 00000000000..157ec150f98--- /dev/null+++ b/packages/twenty-server/src/modules/workflow/workflow-executor/workflow-actions/ai-agent/utils/get-workflow-run-user-workspace-id.util.ts@@ -0,0 +1,6 @@+import { type WorkflowExecutionContext } from 'src/modules/workflow/workflow-executor/types/workflow-execution-context.type';++export const getWorkflowRunUserWorkspaceId = ({+ authContext,+}: Pick<WorkflowExecutionContext, 'authContext'>): string | null =>+ authContext.type === 'user' ? authContext.userWorkspaceId : null;diff --git a/packages/twenty-server/src/modules/workflow/workflow-runner/workspace-services/workflow-waiting-step.workspace-service.ts b/packages/twenty-server/src/modules/workflow/workflow-runner/workspace-services/workflow-waiting-step.workspace-service.tsindex 14c79ccc636..c3ed8dcaba7 100644--- a/packages/twenty-server/src/modules/workflow/workflow-runner/workspace-services/workflow-waiting-step.workspace-service.ts+++ b/packages/twenty-server/src/modules/workflow/workflow-runner/workspace-services/workflow-waiting-step.workspace-service.ts@@ -108,6 +108,27 @@ export class WorkflowWaitingStepWorkspaceService: 'GONE';}+ async onSuspended({+ workspaceId,+ caller,+ threadId,+ summary,+ }: Parameters<+ NonNullable<AgentRunCallerHandler['onSuspended']>+ >[0]): Promise<void> {+ const { workflowRunId, stepId } = workflowStepCallerRefSchema.parse(+ caller.ref,+ );++ await this.workflowRunStepLogService.setAiAgentStepLog({+ workflowRunId,+ workspaceId,+ stepId,+ summary,+ threadId,+ });+ }+// The run job ends the step with the outcome through the executor's usual path, so a failure is// retried or continues on failure like any failed stepasync onOutcome({diff --git a/packages/twenty-server/test/integration/ai/suites/__snapshots__/agent-turn-context.integration-spec.ts.snap b/packages/twenty-server/test/integration/ai/suites/__snapshots__/agent-turn-context.integration-spec.ts.snapnew file mode 100644index 00000000000..df78953dfd3--- /dev/null+++ b/packages/twenty-server/test/integration/ai/suites/__snapshots__/agent-turn-context.integration-spec.ts.snap@@ -0,0 +1,23 @@+// Jest Snapshot v1, https://goo.gl/fbAQLP++exports[`what an agent turn offers the model (integration) offers a chat turn the assistant tools 1`] = `+[+ "ask_question",+ "attach_conversation_to_record",+ "execute_tool",+ "learn_tools",+ "load_skills",+ "propose_tool_call",+ "request_form",+ "search_help_center",+]+`;++exports[`what an agent turn offers the model (integration) offers a runAgent call its agent tools 1`] = `+[+ "execute_tool",+ "learn_tools",+ "wait_for_duration",+ "wait_for_event",+]+`;diff --git a/packages/twenty-server/test/integration/ai/suites/agent-run-wait.integration-spec.ts b/packages/twenty-server/test/integration/ai/suites/agent-run-wait.integration-spec.tsindex 2ffc1388a46..ea7e3a9bc1f 100644--- a/packages/twenty-server/test/integration/ai/suites/agent-run-wait.integration-spec.ts+++ b/packages/twenty-server/test/integration/ai/suites/agent-run-wait.integration-spec.ts@@ -9,16 +9,19 @@ import { makeMetadataApiRequest } from 'test/integration/metadata/suites/utils/mimport { expectEventually } from 'test/integration/utils/expect-eventually.util';import { getAppProviderByClassName } from 'test/integration/utils/get-app-provider-by-class-name.util';import { getCoreRepository } from 'test/integration/utils/get-core-repository.util';+import {+ buildAgentExecutionResult,+ buildWaitingAgentExecutionResult,+ mockAgentRunTurns,+} from 'test/utils/mock-agent-run-turns.util';import { type AgentTrigger } from 'twenty-shared/application';import { type ApplicationLookupService } from 'src/engine/core-modules/application/application-lookup/application-lookup.service';import { type PendingWakeUpResolverService } from 'src/engine/core-modules/pending-wake-up/services/pending-wake-up-resolver.service';import { fromWorkspaceEntityToFlat } from 'src/engine/core-modules/workspace/utils/from-workspace-entity-to-flat.util';import { WorkspaceEntity } from 'src/engine/core-modules/workspace/workspace.entity';-import { type AgentAsyncExecutorService } from 'src/engine/metadata-modules/ai/ai-agent-execution/services/agent-async-executor.service';import { type AgentRunService } from 'src/engine/metadata-modules/ai/ai-agent-execution/services/agent-run.service';import { type AgentExecutionResult } from 'src/engine/metadata-modules/ai/ai-agent-execution/types/agent-execution-result.type';-import { type AgentChatStreamingService } from 'src/engine/metadata-modules/ai/ai-chat/services/agent-chat-streaming.service';import { type AgentTriggerRunnerService } from 'src/engine/metadata-modules/ai/ai-agent-trigger/services/agent-trigger-runner.service';import { AiExceptionCode } from 'src/engine/metadata-modules/ai/ai.exception';import { SEED_APPLE_WORKSPACE_ID } from 'src/engine/workspace-manager/dev-seeder/core/constants/seeder-workspaces.constant';@@ -29,61 +32,7 @@ import { getWorkspaceSchemaName } from 'src/engine/workspace-datasource/utils/geconst workspaceId = SEED_APPLE_WORKSPACE_ID;const schema = getWorkspaceSchemaName(workspaceId);-const agentResult = (- overrides: Partial<AgentExecutionResult>,-): AgentExecutionResult =>- ({- result: { response: '' },- usage: { inputTokens: 1, outputTokens: 1, totalTokens: 2 },- cacheCreationTokens: 0,- nativeWebSearchCallCount: 0,- hasNoMoreAvailableCredits: false,- modelId: 'test-model',- ...overrides,- }) as AgentExecutionResult;--const waitingResult = ({- toolName = 'wait_for_duration',- wait = {- type: 'TIME',- resumeAt: new Date(Date.now() + 1_000).toISOString(),- },-}: {- toolName?: string;- wait?: Record<string, unknown>;-} = {}) => {- const output = {- success: true,- message: 'Waiting.',- result: { status: 'pending', wait },- };-- return agentResult({- isPaused: true,- steps: [- {- content: [- {- type: 'tool-call',- toolCallId: 'wait-1',- toolName,- input: {},- },- {- type: 'tool-result',- toolCallId: 'wait-1',- toolName,- input: {},- output,- },- ],- toolResults: [{ toolCallId: 'wait-1', toolName, output }],- },- ] as unknown as AgentExecutionResult['steps'],- });-};--const replyingResult = agentResult({+const replyingResult = buildAgentExecutionResult({result: { response: 'Followed up' },steps: [{ content: [{ type: 'text', text: 'Followed up' }] },@@ -91,15 +40,7 @@ const replyingResult = agentResult({});const mockAgent = (...results: AgentExecutionResult[]) =>- results.reduce(- (spy, result) => spy.mockResolvedValueOnce(result),- jest.spyOn(- getAppProviderByClassName<AgentAsyncExecutorService>(- 'AgentAsyncExecutorService',- ),- 'executeAgent',- ),- );+ mockAgentRunTurns(...results);const findSuspensions = (callerType: string, agentId: string) =>global.testDataSource.query(@@ -179,8 +120,13 @@ describe('agent runs that wait (integration)', () => {await deleteOneAgent({ expectToFail: false, input: { id: agentId } });});- const runTrigger = () =>- getAppProviderByClassName<AgentTriggerRunnerService>(+ // a trigger starts its run in a job, so the run is awaited until its first turn is no longer running+ const runTrigger = async () => {+ const [{ startedAt }] = await global.testDataSource.query(+ `SELECT now() AS "startedAt"`,+ );++ await getAppProviderByClassName<AgentTriggerRunnerService>('AgentTriggerRunnerService',).run({workspaceId,@@ -189,8 +135,22 @@ describe('agent runs that wait (integration)', () => {payload: { type: 'CRON', firedAt: new Date().toISOString() },});+ await expectEventually(async () => {+ const [turn] = await global.testDataSource.query(+ `SELECT status FROM "${schema}"."agentTurn"+ WHERE "agentId" = $1 AND "createdAt" >= $2 ORDER BY "createdAt" LIMIT 1`,+ [agentId, startedAt],+ );++ expect(turn?.status ?? 'running').not.toBe('running');+ });+ };+it('lets a triggered agent wait for a duration and go on once the wake-up resolves', async () => {- const executeAgent = mockAgent(waitingResult(), replyingResult);+ const runTurn = mockAgent(+ buildWaitingAgentExecutionResult(),+ replyingResult,+ );await runTrigger();@@ -201,22 +161,13 @@ describe('agent runs that wait (integration)', () => {condition: { type: 'TIME' },});- // the run reads its conversation when it goes on, so a chat message cannot slip in while it waits- await expect(- getAppProviderByClassName<AgentChatStreamingService>(- 'AgentChatStreamingService',- ).streamAgentChat({- thread: { id: suspension.threadId, pendingQuestionMessageId: null },- workspace: { id: workspaceId },- text: 'Any news?',- } as Parameters<AgentChatStreamingService['streamAgentChat']>[0]),- ).rejects.toMatchObject({ code: AiExceptionCode.THREAD_AWAITING_ANSWER });- expect(executeAgent.mock.calls[0][0].baseSystemPrompt).toContain(+ expect(runTurn.mock.calls[0][0].baseSystemPrompt).toContain('wait_for_duration',);- expect(- Object.keys(executeAgent.mock.calls[0][0].pausingTools ?? {}),- ).toEqual(['wait_for_event', 'wait_for_duration']);+ expect(Object.keys(runTurn.mock.calls[0][0].pausingTools ?? {})).toEqual([+ 'wait_for_event',+ 'wait_for_duration',+ ]);// the wake-up goes once the run went on, just after its turn closesawait expectEventually(async () => {@@ -227,57 +178,17 @@ describe('agent runs that wait (integration)', () => {expect(await findSuspensions('AGENT_TRIGGER', agentId)).toEqual([]);});- expect(executeAgent).toHaveBeenCalledTimes(2);- expect(executeAgent.mock.calls[1][0].messages).toEqual([]);- expect(- JSON.stringify(executeAgent.mock.calls[1][0].priorMessages),- ).toContain('The wait is over.');- });-- it('refuses a chat message while a run goes on in its conversation', async () => {- let chatAttempt: Promise<unknown> | undefined;-- jest- .spyOn(- getAppProviderByClassName<AgentAsyncExecutorService>(- 'AgentAsyncExecutorService',- ),- 'executeAgent',- )- .mockImplementationOnce(async () => {- const [{ threadId }] = await global.testDataSource.query(- `SELECT "threadId" FROM "${schema}"."agentTurn" WHERE "agentId" = $1 AND status = 'running'- ORDER BY "createdAt" DESC LIMIT 1`,- [agentId],- );-- // the run is not suspended yet, so only its lock on the conversation keeps the message out- chatAttempt = getAppProviderByClassName<AgentChatStreamingService>(- 'AgentChatStreamingService',- )- .streamAgentChat({- thread: { id: threadId, pendingQuestionMessageId: null },- workspace: { id: workspaceId },- text: 'Any news?',- } as Parameters<AgentChatStreamingService['streamAgentChat']>[0])- .catch((error: unknown) => error);-- await chatAttempt;-- return replyingResult;- });-- await runTrigger();-- expect(await chatAttempt).toMatchObject({- code: AiExceptionCode.THREAD_AWAITING_ANSWER,- });+ expect(runTurn).toHaveBeenCalledTimes(2);+ expect(runTurn.mock.calls[1][0].messages).toEqual([]);+ expect(JSON.stringify(runTurn.mock.calls[1][0].priorMessages)).toContain(+ 'The wait is over.',+ );});it('drops a waiting triggered run once its trigger is turned off', async () => {// an event that never happens, so only the explicit resolution below wakes the run up- const executeAgent = mockAgent(- waitingResult({+ const runTurn = mockAgent(+ buildWaitingAgentExecutionResult({toolName: 'wait_for_event',wait: {type: 'EVENT',@@ -312,7 +223,7 @@ describe('agent runs that wait (integration)', () => {expect(await findSuspensions('AGENT_TRIGGER', agentId)).toEqual([]);- expect(executeAgent).toHaveBeenCalledTimes(1);+ expect(runTurn).toHaveBeenCalledTimes(1);expect(await findWaitCallStatus(threadId)).toBe('cancelled');expect(await findTurnStatuses(threadId)).toEqual(['cancelled']);@@ -322,8 +233,53 @@ describe('agent runs that wait (integration)', () => {});});+ it('starts a runAgent call without waiting when asked not to, its reply landing in its thread', async () => {+ const runTurn = mockAgent(replyingResult);++ const response = await makeMetadataApiRequest({+ query: gql`+ mutation RunAgent($input: RunAgentInput!) {+ runAgent(input: $input) {+ threadId+ status+ result+ error+ success+ }+ }+ `,+ variables: {+ input: {+ agentUniversalIdentifier,+ input: [{ role: 'user', content: 'Follow up tomorrow' }],+ waitForCompletion: false,+ },+ },+ });++ expect(response.body.errors).toBeUndefined();+ expect(response.body.data.runAgent).toEqual({+ threadId: expect.any(String),+ status: 'RUNNING',+ result: null,+ error: null,+ success: true,+ });++ await expectEventually(async () => {+ expect(+ await findTurnStatuses(response.body.data.runAgent.threadId),+ ).toEqual(['completed']);+ });++ expect(runTurn).toHaveBeenCalledTimes(1);+ });+it('answers a runAgent call whose agent waits as waiting, with its thread', async () => {- const executeAgent = mockAgent(waitingResult(), replyingResult);+ const runTurn = mockAgent(+ buildWaitingAgentExecutionResult(),+ replyingResult,+ );const response = await makeMetadataApiRequest({query: gql`@@ -361,7 +317,7 @@ describe('agent runs that wait (integration)', () => {expect(await findSuspensions('AGENT_API_RUN', agentId)).toEqual([]);});- expect(executeAgent).toHaveBeenCalledTimes(2);+ expect(runTurn).toHaveBeenCalledTimes(2);});it.each([@@ -388,7 +344,10 @@ describe('agent runs that wait (integration)', () => {])('goes on with a waiting runAgent call as $title',async ({ token, authContext, roleTargetId, callerRoleSource }) => {- const executeAgent = mockAgent(waitingResult(), replyingResult);+ const runTurn = mockAgent(+ buildWaitingAgentExecutionResult(),+ replyingResult,+ );const response = await makeMetadataApiRequest({@@ -414,6 +373,7 @@ describe('agent runs that wait (integration)', () => {expect(status).toBe('SUSPENDED');+ // the wake-up goes once the run went on, just after its turn closesawait expectEventually(async () => {expect(await findTurnStatuses(threadId)).toEqual(['completed',@@ -421,9 +381,8 @@ describe('agent runs that wait (integration)', () => {]);expect(await findSuspensions('AGENT_API_RUN', agentId)).toEqual([]);});-expect(- executeAgent.mock.calls.map(+ runTurn.mock.calls.map(([execution]) => execution.executionContext.authContext,),).toEqual([@@ -436,7 +395,7 @@ describe('agent runs that wait (integration)', () => {};expect(- executeAgent.mock.calls.map(+ runTurn.mock.calls.map(([execution]) => execution.executionContext.agentRunRoleIds,),).toEqual([agentRunRoleIds, agentRunRoleIds]);@@ -444,8 +403,8 @@ describe('agent runs that wait (integration)', () => {);it('refuses a message to a thread whose run waits, and takes it once the run finished', async () => {- const executeAgent = mockAgent(- waitingResult(),+ const runTurn = mockAgent(+ buildWaitingAgentExecutionResult(),replyingResult,replyingResult,);@@ -492,6 +451,6 @@ describe('agent runs that wait (integration)', () => {result: { response: 'Followed up' },threadId: waiting.threadId,});- expect(executeAgent).toHaveBeenCalledTimes(3);+ expect(runTurn).toHaveBeenCalledTimes(3);});});diff --git a/packages/twenty-server/test/integration/ai/suites/agent-turn-context.integration-spec.ts b/packages/twenty-server/test/integration/ai/suites/agent-turn-context.integration-spec.tsnew file mode 100644index 00000000000..f83e4602497--- /dev/null+++ b/packages/twenty-server/test/integration/ai/suites/agent-turn-context.integration-spec.ts@@ -0,0 +1,214 @@+import { randomUUID } from 'node:crypto';++import { writeFileSync } from 'node:fs';++import gql from 'graphql-tag';+import { TEST_AI_MODEL_ID } from 'test/integration/constants/test-ai-model-ids.constants';+import { createOneAgent } from 'test/integration/metadata/suites/agent/utils/create-one-agent.util';+import { deleteOneAgent } from 'test/integration/metadata/suites/agent/utils/delete-one-agent.util';+import { createOneRole } from 'test/integration/metadata/suites/role/utils/create-one-role.util';+import { deleteOneRole } from 'test/integration/metadata/suites/role/utils/delete-one-role.util';+import { makeMetadataApiRequest } from 'test/integration/metadata/suites/utils/make-metadata-api-request.util';+import { destroyAgentChatThread } from 'test/integration/utils/destroy-agent-chat-thread.util';+import { expectEventually } from 'test/integration/utils/expect-eventually.util';+import { getAppProviderByClassName } from 'test/integration/utils/get-app-provider-by-class-name.util';+import { createScriptedLanguageModel } from 'test/utils/create-scripted-language-model.util';++import {+ type AiModelRegistryService,+ type RegisteredAiModel,+} from 'src/engine/metadata-modules/ai/ai-models/services/ai-model-registry.service';+import { getWorkspaceSchemaName } from 'src/engine/workspace-datasource/utils/get-workspace-schema-name.util';+import { SEED_APPLE_WORKSPACE_ID } from 'src/engine/workspace-manager/dev-seeder/core/constants/seeder-workspaces.constant';++const schema = getWorkspaceSchemaName(SEED_APPLE_WORKSPACE_ID);++const USAGE = {+ inputTokens: {+ total: 1,+ noCache: 1,+ cacheRead: undefined,+ cacheWrite: undefined,+ },+ outputTokens: { total: 1, text: 1, reasoning: undefined },+};++type ModelCall = {+ toolNames: string[];+ systemPrompt: string;+};++// The model answers at once, so a test sees what a turn offers it before any tool runs+const mockAnsweringModel = (): ModelCall[] => {+ const calls: ModelCall[] = [];+ const model = createScriptedLanguageModel((options) => {+ calls.push({+ toolNames: (options.tools ?? []).map((tool) => tool.name).sort(),+ systemPrompt: options.prompt+ .flatMap((message) =>+ message.role === 'system' ? [message.content] : [],+ )+ .join('\n\n'),+ });++ return {+ content: [{ type: 'text', text: 'Done' }],+ finishReason: { unified: 'stop', raw: undefined },+ usage: USAGE,+ };+ });++ jest+ .spyOn(+ getAppProviderByClassName<AiModelRegistryService>(+ 'AiModelRegistryService',+ ),+ 'resolveModelForAgent',+ )+ .mockReturnValue({+ modelId: TEST_AI_MODEL_ID,+ sdkPackage: '@ai-sdk/openai',+ model,+ } as unknown as RegisteredAiModel);++ return calls;+};++const recordSystemPrompt = (name: string, systemPrompt: string) => {+ if (process.env.AGENT_TURN_CONTEXT_PROMPT_DIR) {+ writeFileSync(+ `${process.env.AGENT_TURN_CONTEXT_PROMPT_DIR}/${name}.txt`,+ systemPrompt,+ );+ }+};++describe('what an agent turn offers the model (integration)', () => {+ afterEach(() => jest.restoreAllMocks());++ it('offers a chat turn the assistant tools', async () => {+ const calls = mockAnsweringModel();+ const aiModelRegistryService =+ getAppProviderByClassName<AiModelRegistryService>(+ 'AiModelRegistryService',+ );++ jest+ .spyOn(aiModelRegistryService, 'getAvailableModels')+ .mockReturnValue([{ modelId: TEST_AI_MODEL_ID }] as never);+ const threadId = randomUUID();++ const response = await makeMetadataApiRequest({+ query: gql`+ mutation SendChatMessage(+ $threadId: UUID!+ $text: String!+ $messageId: UUID!+ ) {+ sendChatMessage(+ threadId: $threadId+ text: $text+ messageId: $messageId+ ) {+ queued+ }+ }+ `,+ variables: { threadId, text: 'Hello', messageId: randomUUID() },+ });++ expect(response.body.errors).toBeUndefined();++ await expectEventually(async () => {+ const [assistantMessage] = await global.testDataSource.query(+ `SELECT id FROM "${schema}"."agentMessage" WHERE "threadId" = $1 AND role = 'assistant'`,+ [threadId],+ );++ expect(assistantMessage).toBeDefined();+ });++ expect(calls).toHaveLength(1);+ expect(calls[0].toolNames).toMatchSnapshot();+ for (const section of [+ '## User Context',+ '## Available Tools',+ '### Pre-loaded Tools (ready to use now)',+ '### Tool Catalog by Category',+ '## Available Skills',+ ]) {+ expect(calls[0].systemPrompt).toContain(section);+ }+ recordSystemPrompt('chat', calls[0].systemPrompt);++ await destroyAgentChatThread({ threadId });+ });++ it('offers a runAgent call its agent tools', async () => {+ const calls = mockAnsweringModel();+ const { data: roleData } = await createOneRole({+ expectToFail: false,+ input: {+ label: 'Turn context agent role',+ description: 'Reads every record',+ canUpdateAllSettings: false,+ canAccessAllTools: false,+ canReadAllObjectRecords: true,+ canUpdateAllObjectRecords: false,+ canSoftDeleteAllObjectRecords: false,+ canDestroyAllObjectRecords: false,+ canBeAssignedToUsers: false,+ canBeAssignedToAgents: true,+ canBeAssignedToApiKeys: false,+ },+ });+ const roleId = roleData.createOneRole.id;+ const { data } = await createOneAgent({+ expectToFail: false,+ input: {+ label: 'Turn Context Agent',+ prompt: 'Answer what you are asked',+ modelId: TEST_AI_MODEL_ID,+ roleId,+ },+ });+ const agentId = data.createOneAgent.id;+ const [{ universalIdentifier }] = await global.testDataSource.query(+ `SELECT "universalIdentifier" FROM core.agent WHERE id = $1`,+ [agentId],+ );++ const response = await makeMetadataApiRequest({+ query: gql`+ mutation RunAgent($input: RunAgentInput!) {+ runAgent(input: $input) {+ status+ }+ }+ `,+ variables: {+ input: {+ agentUniversalIdentifier: universalIdentifier,+ input: [{ role: 'user', content: 'Hello' }],+ },+ },+ });++ expect(response.body.errors).toBeUndefined();+ expect(response.body.data.runAgent.status).toBe('COMPLETED');+ expect(calls).toHaveLength(1);+ expect(calls[0].toolNames).toMatchSnapshot();+ for (const section of [+ 'invoked programmatically',+ 'wait_for_event',+ 'Answer what you are asked',+ '### Tool Catalog by Category',+ ]) {+ expect(calls[0].systemPrompt).toContain(section);+ }+ recordSystemPrompt('run-agent', calls[0].systemPrompt);++ await deleteOneAgent({ expectToFail: false, input: { id: agentId } });+ await deleteOneRole({ expectToFail: false, input: { idToDelete: roleId } });+ });+});diff --git a/packages/twenty-server/test/integration/graphql/suites/ai-chat-answer-tool-call.integration-spec.ts b/packages/twenty-server/test/integration/graphql/suites/ai-chat-answer-tool-call.integration-spec.tsindex 50078955029..388435ca43a 100644--- a/packages/twenty-server/test/integration/graphql/suites/ai-chat-answer-tool-call.integration-spec.ts+++ b/packages/twenty-server/test/integration/graphql/suites/ai-chat-answer-tool-call.integration-spec.ts@@ -12,6 +12,7 @@ import { getQueueToken } from 'src/engine/core-modules/message-queue/utils/get-qimport { AgentMessageRole } from 'src/engine/metadata-modules/ai/ai-history/enums/agent-message-role.enum';import { type AgentChatService } from 'src/engine/metadata-modules/ai/ai-chat/services/agent-chat.service';import { type AgentChatThreadService } from 'src/engine/metadata-modules/ai/ai-chat/services/agent-chat-thread.service';+import { type AgentConversationWriterService } from 'src/engine/metadata-modules/ai/ai-history/services/agent-conversation-writer.service';import { createAskQuestionTool } from 'src/engine/metadata-modules/ai/ai-chat/tools/ask-question.tool';import { type AiModelRegistryService } from 'src/engine/metadata-modules/ai/ai-models/services/ai-model-registry.service';import { getWorkspaceSchemaName } from 'src/engine/workspace-datasource/utils/get-workspace-schema-name.util';@@ -55,7 +56,9 @@ describe('Answering a chat tool call', () => {isWorkspaceSetupThread: false,}).execute(QUESTION);- await chat.upsertAssistantMessage({+ await getAppProviderByClassName<AgentConversationWriterService>(+ 'AgentConversationWriterService',+ ).upsertAssistantMessage({id: assistantMessageId,threadId,turnId: userMessage.turnId,diff --git a/packages/twenty-server/test/integration/graphql/suites/user-session/utils/capture-agent-run-execution.util.ts b/packages/twenty-server/test/integration/graphql/suites/user-session/utils/capture-agent-run-execution.util.tsindex bc993051d78..2cb3c60ee6a 100644--- a/packages/twenty-server/test/integration/graphql/suites/user-session/utils/capture-agent-run-execution.util.ts+++ b/packages/twenty-server/test/integration/graphql/suites/user-session/utils/capture-agent-run-execution.util.ts@@ -1,14 +1,17 @@import { runAgentQueryFactory } from 'test/integration/graphql/suites/user-session/utils/run-agent-query-factory.util';import { makeMetadataApiRequest } from 'test/integration/metadata/suites/utils/make-metadata-api-request.util';-import { getAppProviderByClassName } from 'test/integration/utils/get-app-provider-by-class-name.util';+import {+ EMPTY_USAGE,+ mockAgentRunTurns,+} from 'test/utils/mock-agent-run-turns.util';import { type RunAgentThread } from 'twenty-shared/application';import { isDefined } from 'twenty-shared/utils';-import { type AgentAsyncExecutorService } from 'src/engine/metadata-modules/ai/ai-agent-execution/services/agent-async-executor.service';+import { type AgentRunTurnService } from 'src/engine/metadata-modules/ai/ai-agent-execution/services/agent-run-turn.service';import { type AgentExecutionResult } from 'src/engine/metadata-modules/ai/ai-agent-execution/types/agent-execution-result.type';export type AgentRunExecution = Parameters<- AgentAsyncExecutorService['executeAgent']+ AgentRunTurnService['startTurn']>[0] & { threadId: string };export const captureAgentRunExecution = async ({@@ -24,43 +27,18 @@ export const captureAgentRunExecution = async ({runAsWorkspaceMemberId?: string;steps?: AgentExecutionResult['steps'];}): Promise<AgentRunExecution> => {- const executeAgentSpy = jest- .spyOn(- getAppProviderByClassName<AgentAsyncExecutorService>(- 'AgentAsyncExecutorService',- ),- 'executeAgent',- )- .mockResolvedValue({- result: {},- usage: {- inputTokens: 0,- outputTokens: 0,- totalTokens: 0,- inputTokenDetails: {- noCacheTokens: 0,- cacheReadTokens: 0,- cacheWriteTokens: 0,- },- outputTokenDetails: { textTokens: 0, reasoningTokens: 0 },- },- cacheCreationTokens: 0,- nativeWebSearchCallCount: 0,- hasNoMoreAvailableCredits: false,- isPaused: false,- steps,- modelId: 'stub-model',- totalCostInDollars: 0,- creditsUsedMicro: 0,- turnUsage: {- inputTokens: 0,- outputTokens: 0,- cacheReadTokens: 0,- cacheCreationTokens: 0,- inputCredits: 0,- outputCredits: 0,- },- });+ const startTurnSpy = mockAgentRunTurns({+ result: {},+ usage: EMPTY_USAGE,+ cacheCreationTokens: 0,+ nativeWebSearchCallCount: 0,+ stopReason: null,+ isPaused: false,+ steps,+ modelId: 'stub-model',+ totalCostInDollars: 0,+ creditsUsedMicro: 0,+ });try {const response = await makeMetadataApiRequest(@@ -73,7 +51,7 @@ export const captureAgentRunExecution = async ({token,);- const execution = executeAgentSpy.mock.calls[0]?.[0];+ const execution = startTurnSpy.mock.calls[0]?.[0];if (!isDefined(execution)) {throw new Error(@@ -83,6 +61,6 @@ export const captureAgentRunExecution = async ({return { ...execution, threadId: response.body.data.runAgent.threadId };} finally {- executeAgentSpy.mockRestore();+ jest.restoreAllMocks();}};diff --git a/packages/twenty-server/test/integration/graphql/suites/workflow/core-workflow-execution.integration-spec.ts b/packages/twenty-server/test/integration/graphql/suites/workflow/core-workflow-execution.integration-spec.tsindex fce5ecb730f..854c3a23f92 100644--- a/packages/twenty-server/test/integration/graphql/suites/workflow/core-workflow-execution.integration-spec.ts+++ b/packages/twenty-server/test/integration/graphql/suites/workflow/core-workflow-execution.integration-spec.ts@@ -43,7 +43,13 @@ import { submitFormStep } from 'test/integration/graphql/suites/workflow/utils/simport { workflowGraphqlRequest } from 'test/integration/graphql/suites/workflow/utils/workflow-graphql-request.util';import { createOneAgent } from 'test/integration/metadata/suites/agent/utils/create-one-agent.util';import { deleteOneAgent } from 'test/integration/metadata/suites/agent/utils/delete-one-agent.util';+import { expectEventually } from 'test/integration/utils/expect-eventually.util';import { getAppProviderByClassName } from 'test/integration/utils/get-app-provider-by-class-name.util';+import {+ buildAgentExecutionResult,+ buildWaitingAgentExecutionResult,+ mockAgentRunTurns,+} from 'test/utils/mock-agent-run-turns.util';import { type AgentChatService } from 'src/engine/metadata-modules/ai/ai-chat/services/agent-chat.service';import { type AgentChatThreadService } from 'src/engine/metadata-modules/ai/ai-chat/services/agent-chat-thread.service';import { type ProvisionedWorkspaceCommandRunner } from 'src/database/commands/command-runners/provisioned-workspace.command-runner';@@ -53,8 +59,10 @@ import { type WorkspaceMigrationValidateBuildAndRunService } from 'src/engine/woimport { WORKFLOW_CRON_TRIGGER_CACHE_KEY } from 'src/modules/workflow/workflow-trigger/automated-trigger/crons/constants/workflow-cron-trigger-cache-key.constant';import { type CodeStepBuildService } from 'src/modules/workflow/workflow-builder/workflow-version-step/code-step/services/code-step-build.service';-import { type AgentAsyncExecutorService } from 'src/engine/metadata-modules/ai/ai-agent-execution/services/agent-async-executor.service';import { type AgentExecutionResult } from 'src/engine/metadata-modules/ai/ai-agent-execution/types/agent-execution-result.type';+import { type AgentRunTurnService } from 'src/engine/metadata-modules/ai/ai-agent-execution/services/agent-run-turn.service';+import { type ChatExecutionService } from 'src/engine/metadata-modules/ai/ai-chat/services/chat-execution.service';+import { type AiModelRegistryService } from 'src/engine/metadata-modules/ai/ai-models/services/ai-model-registry.service';import { type WorkflowActionFactory } from 'src/modules/workflow/workflow-executor/factories/workflow-action.factory';const workspaceId = SEED_APPLE_WORKSPACE_ID;@@ -644,6 +652,26 @@ describe('core workflow execution and queue compatibility (e2e)', () => {);};+ // an agent step waits as soon as it starts its run, which pauses later in its own job+ const waitForPausedAgentStep = async (runId: string, stepId: string) => {+ const pendingRun = await waitForStep(runId, stepId, 'PENDING');+ const threadId: string = pendingRun.stepLogs[stepId].details.threadId;++ for (let attempt = 0; attempt < 150; attempt++) {+ const [turn] = await global.testDataSource.query(+ `SELECT status FROM "${schema}"."agentTurn" WHERE "threadId" = $1 ORDER BY "createdAt" DESC LIMIT 1`,+ [threadId],+ );++ if (isDefined(turn) && turn.status !== 'running') {+ return getRun(runId);+ }+ await new Promise((resolve) => setTimeout(resolve, 100));+ }++ throw new Error(`The agent of step ${stepId} never paused`);+ };+const backfill = async (dryRun = false) =>global.workflowTestServices.backfill.runOnWorkspace({workspaceId,@@ -1025,20 +1053,7 @@ describe('core workflow execution and queue compatibility (e2e)', () => {},}) as WorkflowAiAgentAction;- const agentResult = (- overrides: Partial<AgentExecutionResult>,- ): AgentExecutionResult =>- ({- result: { response: '' },- usage: { inputTokens: 1, outputTokens: 1, totalTokens: 2 },- cacheCreationTokens: 0,- nativeWebSearchCallCount: 0,- hasNoMoreAvailableCredits: false,- modelId: 'test-model',- ...overrides,- }) as AgentExecutionResult;-- const askingResult = agentResult({+ const askingResult = buildAgentExecutionResult({isPaused: true,steps: [{@@ -1066,23 +1081,14 @@ describe('core workflow execution and queue compatibility (e2e)', () => {] as AgentExecutionResult['steps'],});- const replyingResult = agentResult({+ const replyingResult = buildAgentExecutionResult({result: { response: 'Quote sent' },steps: [{ content: [{ type: 'text', text: 'Quote sent' }] },] as AgentExecutionResult['steps'],});- const mockAgent = () =>- jest- .spyOn(- getAppProviderByClassName<AgentAsyncExecutorService>(- 'AgentAsyncExecutorService',- ),- 'executeAgent',- )- .mockResolvedValueOnce(askingResult)- .mockResolvedValueOnce(replyingResult);+ const mockAgent = () => mockAgentRunTurns(askingResult, replyingResult);const getToolCalls = async (threadId: string) => {const [thread] = await global.testDataSource.query(@@ -1128,7 +1134,7 @@ describe('core workflow execution and queue compatibility (e2e)', () => {// A run reads through its workspace workflow record, which a mirrorless fixture lacks.const fixture = await createFixture({ steps: [agent, finalStep] });const runId = await runFixture(fixture);- const run = await waitForStep(runId, agent.id, 'PENDING');+ const run = await waitForPausedAgentStep(runId, agent.id);const threadId: string = run.stepLogs[agent.id].details.threadId;return { runId, agent, finalStep, threadId };@@ -1161,15 +1167,99 @@ describe('core workflow execution and queue compatibility (e2e)', () => {return state;};- it('records the conversation of an agent that answers without asking, filed under done', async () => {+ // the run surfaces its waiting conversation just after its turn stops running+ const waitForWaitingInboxState = async (threadId: string) => {+ for (let attempt = 0; attempt < 50; attempt++) {+ const state = await getInboxState(threadId);++ if (isDefined(state?.lastMessageText)) {+ return state;+ }+ await new Promise((resolve) => setTimeout(resolve, 100));+ }++ return getInboxState(threadId);+ };++ it('queues a message the recipient sends while the agent works, and sends it once the run ends', async () => {+ let sentWhileWorking: Promise<request.Response> | undefined;+ let threadId: string | undefined;++ mockAgentRunTurns(replyingResult);+ jest+ .spyOn(+ getAppProviderByClassName<AgentRunTurnService>('AgentRunTurnService'),+ 'completeTurn',+ )+ .mockReset()+ .mockImplementationOnce(async () => {+ [{ threadId }] = await global.testDataSource.query(+ `SELECT "threadId" FROM "${schema}"."agentTurn" WHERE "agentId" = $1 AND status = 'running'+ ORDER BY "createdAt" DESC LIMIT 1`,+ [agentId],+ );++ sentWhileWorking = request(`http://localhost:${APP_PORT}`)+ .post('/metadata')+ .set('Authorization', `Bearer ${APPLE_JANE_ADMIN_ACCESS_TOKEN}`)+ .send({+ query: `mutation SendChatMessage($threadId: UUID!, $text: String!, $messageId: UUID!) {+ sendChatMessage(threadId: $threadId, text: $text, messageId: $messageId) { queued }+ }`,+ variables: {+ threadId,+ text: 'Any news?',+ messageId: randomUUID(),+ },+ });++ await sentWhileWorking;++ return { ...replyingResult, structuredOutputSteps: [] };+ });+ jest+ .spyOn(+ getAppProviderByClassName<AiModelRegistryService>(+ 'AiModelRegistryService',+ ),+ 'getAvailableModels',+ )+ .mockReturnValue([{ modelId: TEST_AI_MODEL_ID }] as never);+ // the reply to the queued message is not under testjest.spyOn(- getAppProviderByClassName<AgentAsyncExecutorService>(- 'AgentAsyncExecutorService',+ getAppProviderByClassName<ChatExecutionService>(+ 'ChatExecutionService',),- 'executeAgent',+ 'streamChat',)- .mockResolvedValueOnce(replyingResult);+ .mockRejectedValue(new Error('No reply in this test'));++ const finalStep = emptyStep();+ const fixture = await createFixture({+ steps: [agentStep([finalStep.id]), finalStep],+ });++ await waitForRun(await runFixture(fixture), 'COMPLETED');++ expect((await sentWhileWorking)?.body.data.sendChatMessage).toEqual({+ queued: true,+ });++ await expectEventually(async () => {+ const [message] = await global.testDataSource.query(+ `SELECT message.status FROM "${schema}"."agentMessage" message+ JOIN "${schema}"."agentMessagePart" part ON part."messageId" = message.id+ WHERE message."threadId" = $1 AND part."textContent" = 'Any news?'`,+ [threadId],+ );++ expect(message?.status).toBe('sent');+ });+ });++ it('records the conversation of an agent that answers without asking, filed under done', async () => {+ mockAgentRunTurns(replyingResult);const finalStep = emptyStep();const agent = agentStep([finalStep.id]);const fixture = await createFixture({ steps: [agent, finalStep] });@@ -1198,37 +1288,30 @@ describe('core workflow execution and queue compatibility (e2e)', () => {const largeOutput = 'x'.repeat(64_000);const toolCallIds = ['find-1', 'find-2', 'find-3', 'find-4', 'find-5'];- jest- .spyOn(- getAppProviderByClassName<AgentAsyncExecutorService>(- 'AgentAsyncExecutorService',- ),- 'executeAgent',- )- .mockResolvedValueOnce(- agentResult({- result: { response: 'Quote sent' },- steps: [- {- content: toolCallIds.flatMap((toolCallId) => [- {- type: 'tool-call',- toolCallId,- toolName: 'find_companies',- input: {},- },- {- type: 'tool-result',- toolCallId,- toolName: 'find_companies',- input: {},- output: { success: true, result: largeOutput },- },- ]),- },- ] as AgentExecutionResult['steps'],- }),- );+ mockAgentRunTurns(+ buildAgentExecutionResult({+ result: { response: 'Quote sent' },+ steps: [+ {+ content: toolCallIds.flatMap((toolCallId) => [+ {+ type: 'tool-call',+ toolCallId,+ toolName: 'find_companies',+ input: {},+ },+ {+ type: 'tool-result',+ toolCallId,+ toolName: 'find_companies',+ input: {},+ output: { success: true, result: largeOutput },+ },+ ]),+ },+ ] as AgentExecutionResult['steps'],+ }),+ );const finalStep = emptyStep();const agent = agentStep([finalStep.id]);const fixture = await createFixture({ steps: [agent, finalStep] });@@ -1248,15 +1331,7 @@ describe('core workflow execution and queue compatibility (e2e)', () => {});it('continues the conversation its key names across runs', async () => {- const executeAgent = jest- .spyOn(- getAppProviderByClassName<AgentAsyncExecutorService>(- 'AgentAsyncExecutorService',- ),- 'executeAgent',- )- .mockResolvedValueOnce(replyingResult)- .mockResolvedValueOnce(replyingResult);+ const runTurn = mockAgentRunTurns(replyingResult, replyingResult);const agent = {...agentStep([]),settings: {@@ -1277,10 +1352,10 @@ describe('core workflow execution and queue compatibility (e2e)', () => {const threadId: string = firstRun.stepLogs[agent.id].details.threadId;expect(secondRun.stepLogs[agent.id].details.threadId).toBe(threadId);- expect(executeAgent.mock.calls[0][0].priorMessages).toEqual([]);- expect(- JSON.stringify(executeAgent.mock.calls[1][0].priorMessages),- ).toContain('Quote sent');+ expect(runTurn.mock.calls[0][0].priorMessages).toEqual([]);+ expect(JSON.stringify(runTurn.mock.calls[1][0].priorMessages)).toContain(+ 'Quote sent',+ );});it("routes the waiting conversation to the workflow creator's inbox when the step names no recipient", async () => {@@ -1310,12 +1385,12 @@ describe('core workflow execution and queue compatibility (e2e)', () => {);const runId = await runFixture(fixture);- const run = await waitForStep(runId, agent.id, 'PENDING');+ const run = await waitForPausedAgentStep(runId, agent.id);const threadId: string = run.stepLogs[agent.id].details.threadId;- expect(await getInboxState(threadId)).toMatchObject({+ expect(await waitForWaitingInboxState(threadId)).toMatchObject({workspaceMemberId: creator.workspaceMemberId,- lastMessageText: agent.name,+ lastMessageText: QUESTION.question,isArchived: true,isBackInInbox: true,});@@ -1351,7 +1426,7 @@ describe('core workflow execution and queue compatibility (e2e)', () => {);const runId = await runFixture(fixture);- const run = await waitForStep(runId, agent.id, 'PENDING');+ const run = await waitForPausedAgentStep(runId, agent.id);const threadId: string = run.stepLogs[agent.id].details.threadId;const [conversation] = await global.testDataSource.query(`SELECT "workspaceMemberId" FROM "${schema}"."agentChatThread" WHERE id = $1`,@@ -1389,16 +1464,11 @@ describe('core workflow execution and queue compatibility (e2e)', () => {});it('retries a step whose agent failed after its answer in the same conversation, which holds the answer', async () => {- const executeAgent = jest- .spyOn(- getAppProviderByClassName<AgentAsyncExecutorService>(- 'AgentAsyncExecutorService',- ),- 'executeAgent',- )- .mockResolvedValueOnce(askingResult)- .mockRejectedValueOnce(new Error('The model is overloaded'))- .mockResolvedValueOnce(replyingResult);+ const runTurn = mockAgentRunTurns(+ askingResult,+ new Error('The model is overloaded'),+ replyingResult,+ );const agent = {...agentStep([]),settings: {@@ -1411,7 +1481,7 @@ describe('core workflow execution and queue compatibility (e2e)', () => {} as WorkflowAction;const fixture = await createFixture({ steps: [agent] });const runId = await runFixture(fixture);- const pausedRun = await waitForStep(runId, agent.id, 'PENDING');+ const pausedRun = await waitForPausedAgentStep(runId, agent.id);const threadId: string = pausedRun.stepLogs[agent.id].details.threadId;expect((await answer({ threadId })).body.errors).toBeUndefined();@@ -1422,13 +1492,13 @@ describe('core workflow execution and queue compatibility (e2e)', () => {status: 'SUCCESS',});expect(run.stepLogs[agent.id].details.threadId).toBe(threadId);- expect(executeAgent).toHaveBeenCalledTimes(3);- expect(executeAgent.mock.calls[2][0].messages).toEqual([+ expect(runTurn).toHaveBeenCalledTimes(3);+ expect(runTurn.mock.calls[2][0].messages).toEqual([{ role: 'user', content: 'Draft the quote' },]);- expect(- JSON.stringify(executeAgent.mock.calls[2][0].priorMessages),- ).toContain('Send it');+ expect(JSON.stringify(runTurn.mock.calls[2][0].priorMessages)).toContain(+ 'Send it',+ );expect(await getToolCalls(threadId)).toMatchObject({isWaiting: false,calls: [{ toolCallId: 'ask-1', status: 'answered' }],@@ -1460,8 +1530,15 @@ describe('core workflow execution and queue compatibility (e2e)', () => {});it('pauses the run on the question and lets the engine continue the agent, without running the step again', async () => {- const executeAgent = mockAgent();+ const runTurn = mockAgent();const { runId, agent, finalStep, threadId } = await startAskingRun();+ const [{ activeStreamId }] = await global.testDataSource.query(+ `SELECT "activeStreamId" FROM "${schema}"."agentChatThread" WHERE id = $1`,+ [threadId],+ );++ // the step shows it waits only once its run let go of the conversation, so it can be answered+ expect(activeStreamId).toBeNull();expect((await getRun(runId)).state.stepInfos[finalStep.id].status).toBe('NOT_STARTED',@@ -1488,7 +1565,7 @@ describe('core workflow execution and queue compatibility (e2e)', () => {});expect(run.state.stepInfos[finalStep.id].status).toBe('SUCCESS');- const resumedWith = executeAgent.mock.calls[1][0];+ const resumedWith = runTurn.mock.calls[1][0];expect(resumedWith.messages).toEqual([]);expect(JSON.stringify(resumedWith.priorMessages)).toContain('Send it');@@ -1514,6 +1591,33 @@ describe('core workflow execution and queue compatibility (e2e)', () => {]);});+ it('goes on with the agent when the recipient writes instead of answering, which skips the question', async () => {+ const runTurn = mockAgent();+ const { runId, agent, threadId } = await startAskingRun();++ const response = await sendAsRecipient({+ threadId,+ text: 'Hold it until Monday',+ });++ expect(response.body.errors).toBeUndefined();+ expect(response.body.data.sendChatMessage.queued).toBe(true);++ const run = await waitForRun(runId, 'COMPLETED');++ expect(run.state.stepInfos[agent.id]).toMatchObject({+ status: 'SUCCESS',+ result: { response: 'Quote sent' },+ });+ expect(JSON.stringify(runTurn.mock.calls[1][0].priorMessages)).toContain(+ 'Hold it until Monday',+ );+ expect(await getToolCalls(threadId)).toEqual({+ isWaiting: false,+ calls: [{ toolCallId: 'ask-1', status: 'skipped' }],+ });+ });+it('refuses a second answer to the same question', async () => {mockAgent();const { runId, threadId } = await startAskingRun();@@ -1531,65 +1635,47 @@ describe('core workflow execution and queue compatibility (e2e)', () => {]);});- it('continues an agent that paused on a wait once its wake-up resolves, without running the step again', async () => {- const resumeAt = new Date(Date.now() + 1_000).toISOString();- const executeAgent = jest+ const sendAsRecipient = ({+ threadId,+ text,+ }: {+ threadId: string;+ text: string;+ }) => {+ jest.spyOn(- getAppProviderByClassName<AgentAsyncExecutorService>(- 'AgentAsyncExecutorService',+ getAppProviderByClassName<AiModelRegistryService>(+ 'AiModelRegistryService',),- 'executeAgent',- )- .mockResolvedValueOnce(- agentResult({- isPaused: true,- steps: [- {- content: [- {- type: 'tool-call',- toolCallId: 'wait-1',- toolName: 'wait_for_duration',- input: { durationInMinutes: 1 },- },- {- type: 'tool-result',- toolCallId: 'wait-1',- toolName: 'wait_for_duration',- input: { durationInMinutes: 1 },- output: {- success: true,- message: `Waiting until ${resumeAt}.`,- result: {- status: 'pending',- wait: { type: 'TIME', resumeAt },- },- },- },- ],- toolResults: [- {- toolCallId: 'wait-1',- toolName: 'wait_for_duration',- output: {- success: true,- result: {- status: 'pending',- wait: { type: 'TIME', resumeAt },- },- },- },- ],- },- ] as unknown as AgentExecutionResult['steps'],- }),+ 'getAvailableModels',)- .mockResolvedValueOnce(replyingResult);+ .mockReturnValue([{ modelId: TEST_AI_MODEL_ID }] as never);++ return request(`http://localhost:${APP_PORT}`)+ .post('/metadata')+ .set('Authorization', `Bearer ${APPLE_JANE_ADMIN_ACCESS_TOKEN}`)+ .send({+ query: `mutation SendChatMessage($threadId: UUID!, $text: String!, $messageId: UUID!) {+ sendChatMessage(threadId: $threadId, text: $text, messageId: $messageId) { queued }+ }`,+ variables: { threadId, text, messageId: randomUUID() },+ });+ };++ it('continues an agent that paused on a wait once its wake-up resolves, without running the step again', async () => {+ const resumeAt = new Date(Date.now() + 3_000).toISOString();+ const runTurn = mockAgentRunTurns(+ buildWaitingAgentExecutionResult({+ toolName: 'wait_for_duration',+ wait: { type: 'TIME', resumeAt },+ }),+ replyingResult,+ );const finalStep = emptyStep();const agent = agentStep([finalStep.id]);const fixture = await createFixture({ steps: [agent, finalStep] });const runId = await runFixture(fixture);- const pausedRun = await waitForStep(runId, agent.id, 'PENDING');+ const pausedRun = await waitForPausedAgentStep(runId, agent.id);const threadId: string = pausedRun.stepLogs[agent.id].details.threadId;expect(pausedRun.state.stepInfos[agent.id].wait).toEqual({@@ -1608,11 +1694,60 @@ describe('core workflow execution and queue compatibility (e2e)', () => {status: 'SUCCESS',result: { response: 'Quote sent' },});- expect(executeAgent).toHaveBeenCalledTimes(2);- expect(executeAgent.mock.calls[1][0].messages).toEqual([]);+ expect(runTurn).toHaveBeenCalledTimes(2);+ expect(runTurn.mock.calls[1][0].messages).toEqual([]);+ expect(JSON.stringify(runTurn.mock.calls[1][0].priorMessages)).toContain(+ 'The wait is over.',+ );expect(- JSON.stringify(executeAgent.mock.calls[1][0].priorMessages),- ).toContain('The wait is over.');+ await global.testDataSource.query(+ `SELECT id FROM core."pendingWakeUp" WHERE "ownerId" = $1`,+ [threadId],+ ),+ ).toEqual([]);+ });++ it('goes on with an agent waiting on an event as soon as the recipient writes to it', async () => {+ // an event that never happens, so only the message wakes the run up+ const runTurn = mockAgentRunTurns(+ buildWaitingAgentExecutionResult({+ toolName: 'wait_for_event',+ wait: {+ type: 'EVENT',+ eventName: 'opportunity.deleted',+ recordId: randomUUID(),+ },+ }),+ replyingResult,+ );+ const finalStep = emptyStep();+ const agent = agentStep([finalStep.id]);+ const fixture = await createFixture({ steps: [agent, finalStep] });+ const runId = await runFixture(fixture);+ const pausedRun = await waitForPausedAgentStep(runId, agent.id);+ const threadId: string = pausedRun.stepLogs[agent.id].details.threadId;++ const response = await sendAsRecipient({+ threadId,+ text: 'No need to wait, send it now',+ });++ expect(response.body.errors).toBeUndefined();+ expect(response.body.data.sendChatMessage.queued).toBe(true);++ const run = await waitForRun(runId, 'COMPLETED');++ expect(run.state.stepInfos[agent.id]).toMatchObject({+ status: 'SUCCESS',+ result: { response: 'Quote sent' },+ });++ const resumedWith = JSON.stringify(+ runTurn.mock.calls[1][0].priorMessages,+ );++ expect(resumedWith).toContain('a member wrote in the conversation');+ expect(resumedWith).toContain('No need to wait, send it now');expect(await global.testDataSource.query(`SELECT id FROM core."pendingWakeUp" WHERE "ownerId" = $1`,@@ -1622,7 +1757,7 @@ describe('core workflow execution and queue compatibility (e2e)', () => {});it('continues an agent step paused on a question by 2.45 once the upgrade suspends it', async () => {- const executeAgent = mockAgent();+ const runTurn = mockAgent();const { runId, agent, threadId } = await startAskingRun();// the shape a step paused by 2.45 left behind@@ -1647,8 +1782,8 @@ describe('core workflow execution and queue compatibility (e2e)', () => {status: 'SUCCESS',result: { response: 'Quote sent' },});- expect(executeAgent).toHaveBeenCalledTimes(2);- expect(executeAgent.mock.calls[1][0].messages).toEqual([]);+ expect(runTurn).toHaveBeenCalledTimes(2);+ expect(runTurn.mock.calls[1][0].messages).toEqual([]);expect(run.stepLogs[agent.id].details.usage.totalTokens).toBe(4);});@@ -1672,22 +1807,15 @@ describe('core workflow execution and queue compatibility (e2e)', () => {},},];- const executeAgent = jest- .spyOn(- getAppProviderByClassName<AgentAsyncExecutorService>(- 'AgentAsyncExecutorService',- ),- 'executeAgent',- )- .mockResolvedValueOnce(- agentResult({- isPaused: true,- steps: [- { content: [...askCall('ask-1'), ...askCall('ask-2')] },- ] as AgentExecutionResult['steps'],- }),- )- .mockResolvedValueOnce(replyingResult);+ const runTurn = mockAgentRunTurns(+ buildAgentExecutionResult({+ isPaused: true,+ steps: [+ { content: [...askCall('ask-1'), ...askCall('ask-2')] },+ ] as AgentExecutionResult['steps'],+ }),+ replyingResult,+ );const { runId, agent, threadId } = await startAskingRun();expect(await getToolCalls(threadId)).toEqual({@@ -1717,9 +1845,9 @@ describe('core workflow execution and queue compatibility (e2e)', () => {status: 'SUCCESS',result: { response: 'Quote sent' },});- expect(executeAgent).toHaveBeenCalledTimes(2);+ expect(runTurn).toHaveBeenCalledTimes(2);expect(- JSON.stringify(executeAgent.mock.calls[1][0].priorMessages),+ JSON.stringify(runTurn.mock.calls[1][0].priorMessages),).not.toContain('"pending"');});@@ -1750,7 +1878,7 @@ describe('core workflow execution and queue compatibility (e2e)', () => {});it('keeps the run alive while the agent continuation is queued, even once a parallel branch has finished', async () => {- const executeAgent = mockAgent();+ const runTurn = mockAgent();const agent = agentStep([]);const parallelBranch = emptyStep();const fixture = await createFixture({@@ -1758,7 +1886,7 @@ describe('core workflow execution and queue compatibility (e2e)', () => {triggerNextStepIds: [agent.id, parallelBranch.id],});const runId = await runFixture(fixture);- const pausedRun = await waitForStep(runId, agent.id, 'PENDING');+ const pausedRun = await waitForPausedAgentStep(runId, agent.id);await waitForStep(runId, parallelBranch.id, 'SUCCESS');@@ -1816,7 +1944,7 @@ describe('core workflow execution and queue compatibility (e2e)', () => {status: 'SUCCESS',result: { response: 'Quote sent' },});- expect(executeAgent).toHaveBeenCalledTimes(2);+ expect(runTurn).toHaveBeenCalledTimes(2);});it('refuses the answer, leaving the question open, when it cannot be recorded', async () => {diff --git a/packages/twenty-server/test/integration/graphql/suites/workflow/core-workflow-visibility.integration-spec.ts b/packages/twenty-server/test/integration/graphql/suites/workflow/core-workflow-visibility.integration-spec.tsindex 0553b6b67db..bd801300f76 100644--- a/packages/twenty-server/test/integration/graphql/suites/workflow/core-workflow-visibility.integration-spec.ts+++ b/packages/twenty-server/test/integration/graphql/suites/workflow/core-workflow-visibility.integration-spec.ts@@ -27,13 +27,15 @@ import {import { workflowGraphqlRequest } from 'test/integration/graphql/suites/workflow/utils/workflow-graphql-request.util';import { getAppProviderByClassName } from 'test/integration/utils/get-app-provider-by-class-name.util';-import { type AgentExecutionResult } from 'src/engine/metadata-modules/ai/ai-agent-execution/types/agent-execution-result.type';import { EngineComponentKey } from 'src/engine/metadata-modules/command-menu-item/enums/engine-component-key.enum';import { type UserWorkspaceService } from 'src/engine/core-modules/user-workspace/user-workspace.service';import { getWorkspaceSchemaName } from 'src/engine/workspace-datasource/utils/get-workspace-schema-name.util';import { SEED_APPLE_WORKSPACE_ID } from 'src/engine/workspace-manager/dev-seeder/core/constants/seeder-workspaces.constant';import { WORKSPACE_MEMBER_DATA_SEED_IDS } from 'src/engine/workspace-manager/dev-seeder/data/constants/workspace-member-data-seeds.constant';import { type AgentRunConversationService } from 'src/engine/metadata-modules/ai/ai-agent-execution/services/agent-run-conversation.service';+import { AgentTurnStatus } from 'src/engine/metadata-modules/ai/ai-history/enums/agent-turn-status.enum';+import { type AgentConversationWriterService } from 'src/engine/metadata-modules/ai/ai-history/services/agent-conversation-writer.service';+import { type AgentTurnRecorderService } from 'src/engine/metadata-modules/ai/ai-history/services/agent-turn-recorder.service';import { type WorkflowAgentConversationWorkspaceService } from 'src/modules/workflow/workflow-executor/workflow-actions/ai-agent/services/workflow-agent-conversation.workspace-service';const client = request(`http://localhost:${APP_PORT}`);@@ -790,10 +792,9 @@ describe('core workflow visibility (e2e)', () => {'AgentRunConversationService',);- const turnId = await runConversationService.openTurn({+ const { turnId } = await runConversationService.openTurn({workspaceId: SEED_APPLE_WORKSPACE_ID,threadId,- title: 'Summarize the lead',agentId: null,senderUserWorkspaceId: null,senderApplicationId: null,@@ -804,17 +805,21 @@ describe('core workflow visibility (e2e)', () => {}),});- await runConversationService.closeTurn({+ await getAppProviderByClassName<AgentConversationWriterService>(+ 'AgentConversationWriterService',+ ).upsertAssistantMessage({workspaceId: SEED_APPLE_WORKSPACE_ID,+ id: randomUUID(),threadId,turnId,- title: 'Summarize the lead',- agentId: null,- execution: {- steps: [- { content: [{ type: 'text', text: 'A warm lead.' }] },- ] as AgentExecutionResult['steps'],- },+ parts: [{ type: 'text', text: 'A warm lead.' }],+ });+ await getAppProviderByClassName<AgentTurnRecorderService>(+ 'AgentTurnRecorderService',+ ).finish({+ workspaceId: SEED_APPLE_WORKSPACE_ID,+ turnId,+ status: AgentTurnStatus.COMPLETED,});});diff --git a/packages/twenty-server/test/integration/graphql/suites/workflow/workflow-ai-step-run-roles.integration-spec.ts b/packages/twenty-server/test/integration/graphql/suites/workflow/workflow-ai-step-run-roles.integration-spec.tsindex 405f697aee4..be4065fe9b1 100644--- a/packages/twenty-server/test/integration/graphql/suites/workflow/workflow-ai-step-run-roles.integration-spec.ts+++ b/packages/twenty-server/test/integration/graphql/suites/workflow/workflow-ai-step-run-roles.integration-spec.ts@@ -1,7 +1,7 @@import { randomUUID } from 'node:crypto';-import { type LanguageModelV4CallOptions } from '@ai-sdk/provider';-import { MockLanguageModelV4 } from 'ai/test';+import { APICallError } from '@ai-sdk/provider';+import { TEST_AI_MODEL_ID } from 'test/integration/constants/test-ai-model-ids.constants';import {createCoreWorkflow,@@ -24,6 +24,7 @@ import { updateWorkspaceMemberRole } from 'test/integration/metadata/suites/roleimport { upsertPermissionFlags } from 'test/integration/metadata/suites/role-permission-flag/utils/upsert-permission-flags.util';import { upsertRowLevelPermissionPredicates } from 'test/integration/metadata/suites/row-level-permission-predicate/utils/upsert-row-level-permission-predicates.util';import { getAppProviderByClassName } from 'test/integration/utils/get-app-provider-by-class-name.util';+import { createScriptedLanguageModel } from 'test/utils/create-scripted-language-model.util';import { RowLevelPermissionPredicateOperand } from 'twenty-shared/types';import { WorkflowActionType } from 'twenty-shared/workflow';import { PermissionFlagType } from 'twenty-shared/constants';@@ -60,39 +61,54 @@ type ModelCall = {};// The model calls the find tool once, then answers, so the test sees the tools the step offers and what they return-const mockModelCallingFindTool = (): ModelCall[] => {+const mockModelCallingFindTool = ({+ failsOnceMidAnswer = false,+}: { failsOnceMidAnswer?: boolean } = {}): ModelCall[] => {const calls: ModelCall[] = [];- const model = new MockLanguageModelV4({- doGenerate: async (options: LanguageModelV4CallOptions) => {- calls.push({- offeredToolNames: (options.tools ?? []).map((tool) => tool.name),- toolOutputs: options.prompt- .filter((message) => message.role === 'tool')- .flatMap((message) => message.content)- .map((part) => JSON.stringify(part)),- });+ const model = createScriptedLanguageModel((options) => {+ calls.push({+ offeredToolNames: (options.tools ?? []).map((tool) => tool.name),+ toolOutputs: options.prompt+ .filter((message) => message.role === 'tool')+ .flatMap((message) => message.content)+ .map((part) => JSON.stringify(part)),+ });++ if (calls.length === 1) {+ return {+ content: [+ {+ type: 'tool-call',+ toolCallId: 'find-1',+ toolName: FIND_TOOL_NAME,+ input: JSON.stringify({ select: ['id', 'name'] }),+ providerMetadata: { openai: { itemId: 'find-item' } },+ },+ ],+ finishReason: { unified: 'tool-calls', raw: undefined },+ usage: USAGE,+ };+ }- return calls.length === 1- ? {- content: [- {- type: 'tool-call',- toolCallId: 'find-1',- toolName: FIND_TOOL_NAME,- input: JSON.stringify({ select: ['id', 'name'] }),- },- ],- finishReason: { unified: 'tool-calls', raw: undefined },- usage: USAGE,- warnings: [],- }- : {- content: [{ type: 'text', text: 'Done' }],- finishReason: { unified: 'stop', raw: undefined },- usage: USAGE,- warnings: [],- };- },+ if (failsOnceMidAnswer && calls.length === 2) {+ return {+ content: [{ type: 'text', text: 'Do' }],+ finishReason: { unified: 'error', raw: undefined },+ usage: USAGE,+ streamError: new APICallError({+ message: 'Service unavailable',+ url: 'https://provider.test',+ requestBodyValues: {},+ statusCode: 503,+ }),+ };+ }++ return {+ content: [{ type: 'text', text: 'Done' }],+ finishReason: { unified: 'stop', raw: undefined },+ usage: USAGE,+ };});const aiModelRegistryService =getAppProviderByClassName<AiModelRegistryService>('AiModelRegistryService');@@ -393,4 +409,61 @@ describe('workflow AI step when a member runs a workspace-owned workflow (integrexpect(calls[1].toolOutputs.join()).toContain(VISIBLE_RECORD_NAME);expect(calls[1].toolOutputs.join()).not.toContain(HIDDEN_RECORD_NAME);});++ const findSavedReplyParts = (): Promise<+ {+ type: string;+ state: string | null;+ textContent: string | null;+ hasToolOutput: boolean;+ providerMetadata: object | null;+ }[]+ > =>+ global.testDataSource.query(+ `SELECT part.type, part.state, part."textContent", part."toolOutput" IS NOT NULL AS "hasToolOutput",+ part."providerMetadata"+ FROM "${schema}"."agentMessagePart" part+ WHERE part."messageId" = (+ SELECT id FROM "${schema}"."agentMessage"+ WHERE role = 'assistant' ORDER BY "createdAt" DESC LIMIT 1+ )+ ORDER BY part."orderIndex"`,+ );++ it('saves the step reply in the parts chat saves', async () => {+ mockModelCallingFindTool();++ expect(await runAsMemberUntilDone(await createAgentStepWorkflow())).toBe(+ 'COMPLETED',+ );++ expect(await findSavedReplyParts()).toMatchObject([+ { type: 'step-start' },+ {+ type: `tool-${FIND_TOOL_NAME}`,+ state: 'output-available',+ hasToolOutput: true,+ providerMetadata: { openai: { itemId: 'find-item' } },+ },+ { type: 'step-start' },+ { type: 'text', textContent: 'Done' },+ ]);+ });++ it('completes the step when the provider fails midway through an answer once', async () => {+ const calls = mockModelCallingFindTool({ failsOnceMidAnswer: true });++ expect(await runAsMemberUntilDone(await createAgentStepWorkflow())).toBe(+ 'COMPLETED',+ );++ expect(calls).toHaveLength(3);++ const savedReplyParts = await findSavedReplyParts();++ expect(savedReplyParts[savedReplyParts.length - 1]).toMatchObject({+ type: 'text',+ textContent: 'Done',+ });+ });});diff --git a/packages/twenty-server/test/utils/create-scripted-language-model.util.ts b/packages/twenty-server/test/utils/create-scripted-language-model.util.tsnew file mode 100644index 00000000000..9892d3ddc31--- /dev/null+++ b/packages/twenty-server/test/utils/create-scripted-language-model.util.ts@@ -0,0 +1,67 @@+import {+ type LanguageModelV4CallOptions,+ type LanguageModelV4StreamPart,+} from '@ai-sdk/provider';+import { convertArrayToReadableStream, MockLanguageModelV4 } from 'ai/test';+import { type ScriptedModelResponse } from 'test/utils/scripted-model-response.type';++const toStreamParts = (+ { content, finishReason, usage, streamError }: ScriptedModelResponse,+ callIndex: number,+): LanguageModelV4StreamPart[] => [+ { type: 'stream-start', warnings: [] },+ ...content.flatMap((part, partIndex): LanguageModelV4StreamPart[] => {+ const id = `${callIndex}-${partIndex}`;++ switch (part.type) {+ case 'text':+ return [+ { type: 'text-start', id },+ { type: 'text-delta', id, delta: part.text },+ { type: 'text-end', id },+ ];+ case 'reasoning':+ return [+ { type: 'reasoning-start', id },+ { type: 'reasoning-delta', id, delta: part.text },+ { type: 'reasoning-end', id },+ ];+ default:+ return [part];+ }+ }),+ streamError === undefined+ ? { type: 'finish', finishReason, usage }+ : { type: 'error', error: streamError },+];++// Agent runs stream their steps, so a mock model answers both generate and stream calls from one script+export const createScriptedLanguageModel = (+ respond: (+ options: LanguageModelV4CallOptions,+ callIndex: number,+ ) => ScriptedModelResponse | Promise<ScriptedModelResponse>,+): MockLanguageModelV4 => {+ let callCount = 0;++ return new MockLanguageModelV4({+ doGenerate: async (options) => {+ const { streamError, ...response } = await respond(options, callCount++);++ if (streamError !== undefined) {+ throw streamError;+ }++ return { ...response, warnings: [] };+ },+ doStream: async (options) => {+ const callIndex = callCount++;++ return {+ stream: convertArrayToReadableStream(+ toStreamParts(await respond(options, callIndex), callIndex),+ ),+ };+ },+ });+};diff --git a/packages/twenty-server/test/utils/mock-agent-run-turns.util.ts b/packages/twenty-server/test/utils/mock-agent-run-turns.util.tsnew file mode 100644index 00000000000..4c5392b443f--- /dev/null+++ b/packages/twenty-server/test/utils/mock-agent-run-turns.util.ts@@ -0,0 +1,152 @@+import { type TextStreamPart, type ToolSet } from 'ai';+import { getAppProviderByClassName } from 'test/integration/utils/get-app-provider-by-class-name.util';++import { type AgentRunTurnService } from 'src/engine/metadata-modules/ai/ai-agent-execution/services/agent-run-turn.service';+import { type AgentExecutionResult } from 'src/engine/metadata-modules/ai/ai-agent-execution/types/agent-execution-result.type';+import { type AiModelConfig } from 'src/engine/metadata-modules/ai/ai-models/types/ai-model-config.type';++export const EMPTY_USAGE = {+ inputTokens: 0,+ outputTokens: 0,+ totalTokens: 0,+ inputTokenDetails: {+ noCacheTokens: 0,+ cacheReadTokens: 0,+ cacheWriteTokens: 0,+ },+ outputTokenDetails: { textTokens: 0, reasoningTokens: 0 },+};++export const buildAgentExecutionResult = (+ overrides: Partial<AgentExecutionResult>,+): AgentExecutionResult =>+ ({+ result: { response: '' },+ usage: { inputTokens: 1, outputTokens: 1, totalTokens: 2 },+ cacheCreationTokens: 0,+ nativeWebSearchCallCount: 0,+ stopReason: null,+ modelId: 'test-model',+ steps: [],+ ...overrides,+ }) as AgentExecutionResult;++// a run that called a wait tool, by default a short duration+export const buildWaitingAgentExecutionResult = ({+ toolName = 'wait_for_duration',+ wait = {+ type: 'TIME',+ resumeAt: new Date(Date.now() + 3_000).toISOString(),+ },+}: {+ toolName?: string;+ wait?: Record<string, unknown>;+} = {}): AgentExecutionResult => {+ const output = {+ success: true,+ message: 'Waiting.',+ result: { status: 'pending', wait },+ };++ return buildAgentExecutionResult({+ isPaused: true,+ steps: [+ {+ content: [+ { type: 'tool-call', toolCallId: 'wait-1', toolName, input: {} },+ {+ type: 'tool-result',+ toolCallId: 'wait-1',+ toolName,+ input: {},+ output,+ },+ ],+ toolResults: [{ toolCallId: 'wait-1', toolName, output }],+ },+ ] as unknown as AgentExecutionResult['steps'],+ });+};++const toStreamParts = (+ steps: AgentExecutionResult['steps'],+): TextStreamPart<ToolSet>[] =>+ [+ { type: 'start' },+ ...steps.flatMap((step, stepIndex) => [+ { type: 'start-step', request: {}, warnings: [] },+ ...step.content.flatMap((content, contentIndex): unknown[] => {+ switch (content.type) {+ case 'text': {+ const id = `text-${stepIndex}-${contentIndex}`;++ return [+ { type: 'text-start', id },+ { type: 'text-delta', id, text: content.text },+ { type: 'text-end', id },+ ];+ }+ case 'tool-call':+ case 'tool-result':+ return [content];+ default:+ return [];+ }+ }),+ {+ type: 'finish-step',+ response: {+ id: 'response-id',+ timestamp: new Date(0),+ modelId: 'test',+ },+ usage: EMPTY_USAGE,+ finishReason: 'stop',+ providerMetadata: undefined,+ },+ ]),+ { type: 'finish', finishReason: 'stop', totalUsage: EMPTY_USAGE },+ ] as TextStreamPart<ToolSet>[];++// Each turn streams the reply its steps hold through the real pipeline, and ends as the given+// execution; an error fails the turn as it starts+export const mockAgentRunTurns = (+ ...turns: (AgentExecutionResult | Error)[]+) => {+ const service = getAppProviderByClassName<AgentRunTurnService>(+ 'AgentRunTurnService',+ );+ const startTurn = jest.spyOn(service, 'startTurn');+ const completeTurn = jest.spyOn(service, 'completeTurn');++ for (const turn of turns) {+ if (turn instanceof Error) {+ startTurn.mockRejectedValueOnce(turn);++ continue;+ }++ startTurn.mockImplementationOnce(async () => ({+ turn: {} as never,+ spenders: {},+ source: {+ parts: new ReadableStream({+ start: (controller) => {+ toStreamParts(turn.steps ?? []).forEach((part) =>+ controller.enqueue(part),+ );+ controller.close();+ },+ }),+ modelConfig: {+ contextWindowTokens: 100_000,+ inputCostPerMillionTokens: 0,+ outputCostPerMillionTokens: 0,+ } as AiModelConfig,+ },+ }));+ completeTurn.mockResolvedValueOnce({ ...turn, structuredOutputSteps: [] });+ }++ return startTurn;+};diff --git a/packages/twenty-server/test/utils/scripted-model-response.type.ts b/packages/twenty-server/test/utils/scripted-model-response.type.tsnew file mode 100644index 00000000000..baa68d22cfb--- /dev/null+++ b/packages/twenty-server/test/utils/scripted-model-response.type.ts@@ -0,0 +1,13 @@+import {+ type LanguageModelV4Content,+ type LanguageModelV4FinishReason,+ type LanguageModelV4Usage,+} from '@ai-sdk/provider';++export type ScriptedModelResponse = {+ content: LanguageModelV4Content[];+ finishReason: LanguageModelV4FinishReason;+ usage: LanguageModelV4Usage;+ // ends the stream with this error after the content, as a provider failing mid-response does+ streamError?: unknown;+};diff --git a/packages/twenty-shared/src/application/runAgentType.ts b/packages/twenty-shared/src/application/runAgentType.tsindex bd16c2f342e..88e800cb49b 100644--- a/packages/twenty-shared/src/application/runAgentType.ts+++ b/packages/twenty-shared/src/application/runAgentType.ts@@ -21,6 +21,8 @@ export type RunAgentInput = {responseFormat?: AgentResponseFormat;thread?: RunAgentThread;runAsWorkspaceMemberId?: string;+ // false returns as soon as the run starts, with RUNNING; its reply lands in the thread+ waitForCompletion?: boolean;} & (| {input: string | RunAgentMessage[];@@ -44,7 +46,8 @@ export type RunAgentInput = {export type RunAgentResult = {threadId: string;// SUSPENDED: the agent paused on a wait and goes on by itself; its reply lands in the thread- status: 'COMPLETED' | 'SUSPENDED' | 'FAILED';+ // RUNNING: the call did not wait for the run, or the run outlasted the wait; its reply lands in the thread+ status: 'COMPLETED' | 'SUSPENDED' | 'FAILED' | 'RUNNING';result: object | null;error: string | null;success: boolean;