The open alternative to Salesforce, designed for AI.
TypeScript crmcrm-systemcustomergood-first-issuegraphqlhacktoberfestjavascriptmarketingmonoreponestjsopen-sourcepostgresqlreactreactjssalestypescriptweb
feat(ai): add a stream heartbeat and reap dead claims so a worker crash cannot brick a thread (#22482)
Félix Malfait5d3b6d05b32e
diff --git a/packages/twenty-server/src/engine/metadata-modules/ai/ai-chat/ai-chat.module.ts b/packages/twenty-server/src/engine/metadata-modules/ai/ai-chat/ai-chat.module.tsindex 2e8f2248cdb..7cf1f6dac84 100644--- a/packages/twenty-server/src/engine/metadata-modules/ai/ai-chat/ai-chat.module.ts+++ b/packages/twenty-server/src/engine/metadata-modules/ai/ai-chat/ai-chat.module.ts@@ -33,6 +33,7 @@ import { AgentChatResolver } from './resolvers/agent-chat.resolver';import { AgentChatSubscriptionResolver } from './resolvers/agent-chat-subscription.resolver';import { AgentChatCancelSubscriberService } from './services/agent-chat-cancel-subscriber.service';import { AgentChatEventPublisherService } from './services/agent-chat-event-publisher.service';+import { AgentChatStreamHeartbeatService } from './services/agent-chat-stream-heartbeat.service';import { AgentChatStreamingService } from './services/agent-chat-streaming.service';import { AgentChatService } from './services/agent-chat.service';import { AgentTitleGenerationService } from './services/agent-title-generation.service';@@ -69,6 +70,7 @@ import { SystemPromptBuilderService } from './services/system-prompt-builder.serproviders: [AgentChatCancelSubscriberService,AgentChatEventPublisherService,+ AgentChatStreamHeartbeatService,AgentChatResolver,AgentChatSubscriptionResolver,AgentChatService,diff --git a/packages/twenty-server/src/engine/metadata-modules/ai/ai-chat/constants/agent-chat-stream-reap-check-interval-ms.constant.ts b/packages/twenty-server/src/engine/metadata-modules/ai/ai-chat/constants/agent-chat-stream-reap-check-interval-ms.constant.tsnew file mode 100644index 00000000000..6374738a625--- /dev/null+++ b/packages/twenty-server/src/engine/metadata-modules/ai/ai-chat/constants/agent-chat-stream-reap-check-interval-ms.constant.ts@@ -0,0 +1 @@+export const AGENT_CHAT_STREAM_REAP_CHECK_INTERVAL_MS = 10_000;diff --git a/packages/twenty-server/src/engine/metadata-modules/ai/ai-chat/constants/stream-interrupted-code.constant.ts b/packages/twenty-server/src/engine/metadata-modules/ai/ai-chat/constants/stream-interrupted-code.constant.tsnew file mode 100644index 00000000000..98be7902d1d--- /dev/null+++ b/packages/twenty-server/src/engine/metadata-modules/ai/ai-chat/constants/stream-interrupted-code.constant.ts@@ -0,0 +1 @@+export const STREAM_INTERRUPTED_CODE = 'STREAM_INTERRUPTED';diff --git a/packages/twenty-server/src/engine/metadata-modules/ai/ai-chat/jobs/__tests__/stream-agent-chat.job.spec.ts b/packages/twenty-server/src/engine/metadata-modules/ai/ai-chat/jobs/__tests__/stream-agent-chat.job.spec.tsindex 755a9c098fb..f443b8014bc 100644--- a/packages/twenty-server/src/engine/metadata-modules/ai/ai-chat/jobs/__tests__/stream-agent-chat.job.spec.ts+++ b/packages/twenty-server/src/engine/metadata-modules/ai/ai-chat/jobs/__tests__/stream-agent-chat.job.spec.ts@@ -144,6 +144,11 @@ describe('StreamAgentChatJob', () => {const agentChatStreamingService = {flushNextQueuedMessage: jest.fn().mockResolvedValue(undefined),};+ const streamHeartbeatService = {+ startRunning: jest.fn().mockReturnValue(() => {}),+ markClaimed: jest.fn().mockResolvedValue(undefined),+ clear: jest.fn().mockResolvedValue(undefined),+ };const job = new StreamAgentChatJob(threadRepository as never,workspaceRepository as never,@@ -152,6 +157,7 @@ describe('StreamAgentChatJob', () => {eventPublisherService as never,cancelSubscriberService as never,agentChatStreamingService as never,+ streamHeartbeatService as never,);return {diff --git a/packages/twenty-server/src/engine/metadata-modules/ai/ai-chat/jobs/stream-agent-chat.job.ts b/packages/twenty-server/src/engine/metadata-modules/ai/ai-chat/jobs/stream-agent-chat.job.tsindex ee6871d9c9c..2ab12c30da0 100644--- a/packages/twenty-server/src/engine/metadata-modules/ai/ai-chat/jobs/stream-agent-chat.job.ts+++ b/packages/twenty-server/src/engine/metadata-modules/ai/ai-chat/jobs/stream-agent-chat.job.ts@@ -28,6 +28,7 @@ import {import { AgentChatThreadEntity } from 'src/engine/metadata-modules/ai/ai-chat/entities/agent-chat-thread.entity';import { AgentChatCancelSubscriberService } from 'src/engine/metadata-modules/ai/ai-chat/services/agent-chat-cancel-subscriber.service';import { AgentChatEventPublisherService } from 'src/engine/metadata-modules/ai/ai-chat/services/agent-chat-event-publisher.service';+import { AgentChatStreamHeartbeatService } from 'src/engine/metadata-modules/ai/ai-chat/services/agent-chat-stream-heartbeat.service';import { AgentChatStreamingService } from 'src/engine/metadata-modules/ai/ai-chat/services/agent-chat-streaming.service';import { AgentChatService } from 'src/engine/metadata-modules/ai/ai-chat/services/agent-chat.service';import { ChatExecutionService } from 'src/engine/metadata-modules/ai/ai-chat/services/chat-execution.service';@@ -62,6 +63,7 @@ export class StreamAgentChatJob {private readonly eventPublisherService: AgentChatEventPublisherService,private readonly cancelSubscriberService: AgentChatCancelSubscriberService,private readonly agentChatStreamingService: AgentChatStreamingService,+ private readonly streamHeartbeatService: AgentChatStreamHeartbeatService,) {}@Process(STREAM_AGENT_CHAT_JOB_NAME)@@ -71,6 +73,10 @@ export class StreamAgentChatJob {const abortController = new AbortController();const cancelChannel = getCancelChannel(data.threadId, data.streamId);+ const stopHeartbeat = this.streamHeartbeatService.startRunning(+ data.streamId,+ );+await this.cancelSubscriberService.subscribe(cancelChannel, () => {abortController.abort();});@@ -132,6 +138,8 @@ export class StreamAgentChatJob {.catch(() => {});throw error;} finally {+ stopHeartbeat();+ await this.streamHeartbeatService.clear(data.streamId);await this.cancelSubscriberService.unsubscribe(cancelChannel);await this.threadRepository.update(diff --git a/packages/twenty-server/src/engine/metadata-modules/ai/ai-chat/resolvers/agent-chat-subscription.resolver.ts b/packages/twenty-server/src/engine/metadata-modules/ai/ai-chat/resolvers/agent-chat-subscription.resolver.tsindex 4e37f7a7dd8..6a378d92044 100644--- a/packages/twenty-server/src/engine/metadata-modules/ai/ai-chat/resolvers/agent-chat-subscription.resolver.ts+++ b/packages/twenty-server/src/engine/metadata-modules/ai/ai-chat/resolvers/agent-chat-subscription.resolver.ts@@ -18,8 +18,10 @@ import {} from 'src/engine/metadata-modules/ai/ai.exception';import { AiGraphqlApiExceptionInterceptor } from 'src/engine/metadata-modules/ai/interceptors/ai-graphql-api-exception.interceptor';import { AGENT_CHAT_KEEPALIVE_INTERVAL_MS } from 'src/engine/metadata-modules/ai/ai-chat/constants/agent-chat-keepalive-interval-ms.constant';+import { AGENT_CHAT_STREAM_REAP_CHECK_INTERVAL_MS } from 'src/engine/metadata-modules/ai/ai-chat/constants/agent-chat-stream-reap-check-interval-ms.constant';import { AgentChatEventDTO } from 'src/engine/metadata-modules/ai/ai-chat/dtos/agent-chat-event.dto';import { AgentChatThreadEntity } from 'src/engine/metadata-modules/ai/ai-chat/entities/agent-chat-thread.entity';+import { AgentChatStreamingService } from 'src/engine/metadata-modules/ai/ai-chat/services/agent-chat-streaming.service';import { SubscriptionService } from 'src/engine/subscriptions/subscription.service';import { wrapAsyncIteratorWithLifecycle } from 'src/engine/subscriptions/utils/wrap-async-iterator-with-lifecycle';import { InjectWorkspaceScopedRepository } from 'src/engine/twenty-orm/workspace-scoped-repository/inject-workspace-scoped-repository.decorator';@@ -30,6 +32,7 @@ import { WorkspaceScopedRepository } from 'src/engine/twenty-orm/workspace-scopeexport class AgentChatSubscriptionResolver {constructor(private readonly subscriptionService: SubscriptionService,+ private readonly agentChatStreamingService: AgentChatStreamingService,@InjectWorkspaceScopedRepository(AgentChatThreadEntity)private readonly threadRepository: WorkspaceScopedRepository<AgentChatThreadEntity>,) {}@@ -72,9 +75,19 @@ export class AgentChatSubscriptionResolver {},};+ let lastReapCheckAt = 0;+return wrapAsyncIteratorWithLifecycle(iterator, {initialValue: keepalivePayload,onHeartbeat: async () => {+ if (+ Date.now() - lastReapCheckAt >=+ AGENT_CHAT_STREAM_REAP_CHECK_INTERVAL_MS+ ) {+ lastReapCheckAt = Date.now();+ await this.reapWatchedStreamIfDead(workspace.id, threadId);+ }+await this.subscriptionService.publishToAgentChat({workspaceId: workspace.id,threadId,@@ -86,4 +99,30 @@ export class AgentChatSubscriptionResolver {heartbeatIntervalMs: AGENT_CHAT_KEEPALIVE_INTERVAL_MS,});}++ // Reaping from the keep-alive loop means a user watching a stream whose+ // worker died sees the interrupted state without having to interact;+ // reapDeadStream publishes the stream-error event they are subscribed to.+ private async reapWatchedStreamIfDead(+ workspaceId: string,+ threadId: string,+ ): Promise<void> {+ try {+ const thread = await this.threadRepository.findOne(workspaceId, {+ where: { id: threadId },+ select: ['id', 'activeStreamId'],+ });++ if (!isDefined(thread) || !isDefined(thread.activeStreamId)) {+ return;+ }++ await this.agentChatStreamingService.reapDeadStream({+ thread,+ workspaceId,+ });+ } catch {+ // The keep-alive tick must never die with the check+ }+ }}diff --git a/packages/twenty-server/src/engine/metadata-modules/ai/ai-chat/resolvers/agent-chat.resolver.ts b/packages/twenty-server/src/engine/metadata-modules/ai/ai-chat/resolvers/agent-chat.resolver.tsindex 1310c267a0c..6841384edf1 100644--- a/packages/twenty-server/src/engine/metadata-modules/ai/ai-chat/resolvers/agent-chat.resolver.ts+++ b/packages/twenty-server/src/engine/metadata-modules/ai/ai-chat/resolvers/agent-chat.resolver.ts@@ -110,6 +110,17 @@ export class AgentChatResolver {workspaceId,});+ const interruptedError =+ await this.agentChatStreamingService.reapDeadStream({+ thread,+ workspaceId,+ });++ if (interruptedError) {+ thread.activeStreamId = null;+ thread.lastStreamError = interruptedError;+ }+const { chunks, maxSeq } =await this.eventPublisherService.getAccumulatedChunks(threadId);@@ -188,6 +199,19 @@ export class AgentChatResolver {});}+ if (isDefined(thread.activeStreamId)) {+ const interruptedError =+ await this.agentChatStreamingService.reapDeadStream({+ thread,+ workspaceId: workspace.id,+ });++ if (interruptedError) {+ thread.activeStreamId = null;+ thread.lastStreamError = interruptedError;+ }+ }+if (isDefined(thread.activeStreamId) ||isDefined(thread.pendingQuestionMessageId)diff --git a/packages/twenty-server/src/engine/metadata-modules/ai/ai-chat/services/__tests__/agent-chat-streaming.service.claim.spec.ts b/packages/twenty-server/src/engine/metadata-modules/ai/ai-chat/services/__tests__/agent-chat-streaming.service.claim.spec.tsindex a63668f9358..00c9e01cd8d 100644--- a/packages/twenty-server/src/engine/metadata-modules/ai/ai-chat/services/__tests__/agent-chat-streaming.service.claim.spec.ts+++ b/packages/twenty-server/src/engine/metadata-modules/ai/ai-chat/services/__tests__/agent-chat-streaming.service.claim.spec.ts@@ -1,4 +1,5 @@import { type WorkspaceEntity } from 'src/engine/core-modules/workspace/workspace.entity';+import { STREAM_INTERRUPTED_CODE } from 'src/engine/metadata-modules/ai/ai-chat/constants/stream-interrupted-code.constant';import { AgentChatStreamingService } from 'src/engine/metadata-modules/ai/ai-chat/services/agent-chat-streaming.service';describe('AgentChatStreamingService claim & reap', () => {@@ -17,6 +18,7 @@ describe('AgentChatStreamingService claim & reap', () => {thread = idleThread,claimAffected = 1,queuedMessages = [] as unknown[],+ heartbeatAlive = true,} = {}) => {const publishedEvents: Array<{ type: string }> = [];const threadRepository = {@@ -47,6 +49,12 @@ describe('AgentChatStreamingService claim & reap', () => {}),resetStreamState: jest.fn().mockResolvedValue(undefined),};+ const streamHeartbeatService = {+ markClaimed: jest.fn().mockResolvedValue(undefined),+ isAlive: jest.fn().mockResolvedValue(heartbeatAlive),+ clear: jest.fn().mockResolvedValue(undefined),+ };+const service = new AgentChatStreamingService(threadRepository as never,{ find: jest.fn().mockResolvedValue([]) } as never,@@ -54,6 +62,7 @@ describe('AgentChatStreamingService claim & reap', () => {agentChatService as never,eventPublisherService as never,{ signFileByIdUrl: jest.fn() } as never,+ streamHeartbeatService as never,);return {@@ -62,6 +71,7 @@ describe('AgentChatStreamingService claim & reap', () => {messageQueueService,agentChatService,eventPublisherService,+ streamHeartbeatService,publishedEvents,};};@@ -76,7 +86,8 @@ describe('AgentChatStreamingService claim & reap', () => {describe('streamAgentChat', () => {it('claims the thread conditionally before enqueueing', async () => {- const { service, threadRepository } = buildService();+ const { service, threadRepository, streamHeartbeatService } =+ buildService();const result = await service.streamAgentChat(sendArguments);@@ -86,6 +97,7 @@ describe('AgentChatStreamingService claim & reap', () => {expect.objectContaining({ id: 'thread-id' }),expect.objectContaining({ lastStreamError: null }),);+ expect(streamHeartbeatService.markClaimed).toHaveBeenCalled();});it('queues the message when another stream wins the claim race', async () => {@@ -135,4 +147,71 @@ describe('AgentChatStreamingService claim & reap', () => {);});});++ describe('reapDeadStream', () => {+ it('leaves a live stream alone', async () => {+ const { service, threadRepository } = buildService();++ const reaped = await service.reapDeadStream({+ thread: { id: 'thread-id', activeStreamId: 'stream-id' },+ workspaceId: 'workspace-id',+ });++ expect(reaped).toBeNull();+ expect(threadRepository.update).not.toHaveBeenCalled();+ });++ it('converts a heartbeat-less claim into a retryable interrupted error', async () => {+ const {+ service,+ threadRepository,+ eventPublisherService,+ publishedEvents,+ } = buildService({ heartbeatAlive: false });++ const reaped = await service.reapDeadStream({+ thread: { id: 'thread-id', activeStreamId: 'stream-id' },+ workspaceId: 'workspace-id',+ });++ expect(reaped).toEqual(+ expect.objectContaining({ code: STREAM_INTERRUPTED_CODE }),+ );+ expect(threadRepository.update).toHaveBeenCalledWith(+ 'workspace-id',+ { id: 'thread-id', activeStreamId: 'stream-id' },+ expect.objectContaining({+ activeStreamId: null,+ lastStreamError: expect.objectContaining({+ code: STREAM_INTERRUPTED_CODE,+ }),+ }),+ );+ expect(eventPublisherService.resetStreamState).toHaveBeenCalledWith(+ 'thread-id',+ );+ expect(publishedEvents).toContainEqual(+ expect.objectContaining({+ type: 'stream-error',+ code: STREAM_INTERRUPTED_CODE,+ }),+ );+ });++ it('does nothing when the claim moved to a newer stream mid-check', async () => {+ const { service, publishedEvents, threadRepository } = buildService({+ heartbeatAlive: false,+ claimAffected: 0,+ });++ const reaped = await service.reapDeadStream({+ thread: { id: 'thread-id', activeStreamId: 'stream-id' },+ workspaceId: 'workspace-id',+ });++ expect(reaped).toBeNull();+ expect(threadRepository.update).toHaveBeenCalledTimes(1);+ expect(publishedEvents).toHaveLength(0);+ });+ });});diff --git a/packages/twenty-server/src/engine/metadata-modules/ai/ai-chat/services/__tests__/agent-chat-streaming.service.retry.spec.ts b/packages/twenty-server/src/engine/metadata-modules/ai/ai-chat/services/__tests__/agent-chat-streaming.service.retry.spec.tsindex cb1fa2bf354..791eef09ec4 100644--- a/packages/twenty-server/src/engine/metadata-modules/ai/ai-chat/services/__tests__/agent-chat-streaming.service.retry.spec.ts+++ b/packages/twenty-server/src/engine/metadata-modules/ai/ai-chat/services/__tests__/agent-chat-streaming.service.retry.spec.ts@@ -47,6 +47,12 @@ describe('AgentChatStreamingService.retryLastFailedTurn', () => {getMessagesForThread: jest.fn().mockResolvedValue(threadMessages),};+ const streamHeartbeatService = {+ markClaimed: jest.fn().mockResolvedValue(undefined),+ isAlive: jest.fn().mockResolvedValue(true),+ clear: jest.fn().mockResolvedValue(undefined),+ };+const service = new AgentChatStreamingService(threadRepository as never,{ find: jest.fn() } as never,@@ -54,6 +60,7 @@ describe('AgentChatStreamingService.retryLastFailedTurn', () => {agentChatService as never,{ publish: jest.fn() } as never,{ signFileByIdUrl: jest.fn() } as never,+ streamHeartbeatService as never,);return { service, threadRepository, messageQueueService, agentChatService };diff --git a/packages/twenty-server/src/engine/metadata-modules/ai/ai-chat/services/agent-chat-stream-heartbeat.service.ts b/packages/twenty-server/src/engine/metadata-modules/ai/ai-chat/services/agent-chat-stream-heartbeat.service.tsnew file mode 100644index 00000000000..3bef3438e0c--- /dev/null+++ b/packages/twenty-server/src/engine/metadata-modules/ai/ai-chat/services/agent-chat-stream-heartbeat.service.ts@@ -0,0 +1,55 @@+import { Injectable } from '@nestjs/common';++import { RedisClientService } from 'src/engine/core-modules/redis-client/redis-client.service';++const CLAIM_TTL_SECONDS = 600;+const RUNNING_TTL_SECONDS = 30;+const REFRESH_INTERVAL_MS = 5_000;++@Injectable()+export class AgentChatStreamHeartbeatService {+ constructor(private readonly redisClientService: RedisClientService) {}++ private getKey(streamId: string): string {+ return `agent-chat-stream-alive:${streamId}`;+ }++ async markClaimed(streamId: string): Promise<void> {+ await this.redisClientService+ .getClient()+ .set(this.getKey(streamId), '1', 'EX', CLAIM_TTL_SECONDS);+ }++ startRunning(streamId: string): () => void {+ const refresh = () => {+ this.redisClientService+ .getClient()+ .set(this.getKey(streamId), '1', 'EX', RUNNING_TTL_SECONDS)+ .catch(() => {});+ };++ refresh();+ const interval = setInterval(refresh, REFRESH_INTERVAL_MS);++ return () => clearInterval(interval);+ }++ async isAlive(streamId: string): Promise<boolean> {+ try {+ const exists = await this.redisClientService+ .getClient()+ .exists(this.getKey(streamId));++ return exists === 1;+ } catch {+ return true;+ }+ }++ async clear(streamId: string): Promise<void> {+ await this.redisClientService+ .getClient()+ .del(this.getKey(streamId))+ .catch(() => {});+ }+}diff --git a/packages/twenty-server/src/engine/metadata-modules/ai/ai-chat/services/agent-chat-streaming.service.ts b/packages/twenty-server/src/engine/metadata-modules/ai/ai-chat/services/agent-chat-streaming.service.tsindex 19ca567a61e..0d00945a2e4 100644--- a/packages/twenty-server/src/engine/metadata-modules/ai/ai-chat/services/agent-chat-streaming.service.ts+++ b/packages/twenty-server/src/engine/metadata-modules/ai/ai-chat/services/agent-chat-streaming.service.ts@@ -16,6 +16,7 @@ import { InjectMessageQueue } from 'src/engine/core-modules/message-queue/decoraimport { MessageQueue } from 'src/engine/core-modules/message-queue/message-queue.constants';import { MessageQueueService } from 'src/engine/core-modules/message-queue/services/message-queue.service';import { type WorkspaceEntity } from 'src/engine/core-modules/workspace/workspace.entity';+import { STREAM_INTERRUPTED_CODE } from 'src/engine/metadata-modules/ai/ai-chat/constants/stream-interrupted-code.constant';import {AgentMessageRole,AgentMessageStatus,@@ -27,6 +28,7 @@ import { type AgentChatThreadLastStreamError } from 'src/engine/metadata-modulesimport { STREAM_AGENT_CHAT_JOB_NAME } from 'src/engine/metadata-modules/ai/ai-chat/jobs/stream-agent-chat-job-name.constant';import { type StreamAgentChatJobData } from 'src/engine/metadata-modules/ai/ai-chat/jobs/stream-agent-chat-job.types';import { AgentChatEventPublisherService } from 'src/engine/metadata-modules/ai/ai-chat/services/agent-chat-event-publisher.service';+import { AgentChatStreamHeartbeatService } from 'src/engine/metadata-modules/ai/ai-chat/services/agent-chat-stream-heartbeat.service';import { AgentChatService } from 'src/engine/metadata-modules/ai/ai-chat/services/agent-chat.service';import { AiChatFileAttachment } from 'src/engine/metadata-modules/ai/ai-chat/types/ai-chat-file-attachment.type';import {@@ -60,8 +62,56 @@ export class AgentChatStreamingService {private readonly agentChatService: AgentChatService,private readonly eventPublisherService: AgentChatEventPublisherService,private readonly fileUrlService: FileUrlService,+ private readonly streamHeartbeatService: AgentChatStreamHeartbeatService,) {}+ async reapDeadStream({+ thread,+ workspaceId,+ }: {+ thread: Pick<AgentChatThreadEntity, 'id' | 'activeStreamId'>;+ workspaceId: string;+ }): Promise<AgentChatThreadLastStreamError | null> {+ if (!isDefined(thread.activeStreamId)) {+ return null;+ }++ if (await this.streamHeartbeatService.isAlive(thread.activeStreamId)) {+ return null;+ }++ const interruptedError: AgentChatThreadLastStreamError = {+ code: STREAM_INTERRUPTED_CODE,+ message: 'The response was interrupted before it could finish.',+ failedAt: new Date().toISOString(),+ };++ const reap = await this.threadRepository.update(+ workspaceId,+ { id: thread.id, activeStreamId: thread.activeStreamId },+ { activeStreamId: null, lastStreamError: interruptedError },+ );++ if (!reap.affected) {+ return null;+ }++ await this.eventPublisherService.resetStreamState(thread.id);+ await this.eventPublisherService+ .publish({+ threadId: thread.id,+ workspaceId,+ event: {+ type: 'stream-error',+ code: interruptedError.code,+ message: interruptedError.message,+ },+ })+ .catch(() => {});++ return interruptedError;+ }+private async tryClaimStream({threadId,workspaceId,@@ -79,7 +129,13 @@ export class AgentChatStreamingService {{ activeStreamId: streamId, lastStreamError: null },);- return claim.affected === 1;+ if (!claim.affected) {+ return false;+ }++ await this.streamHeartbeatService.markClaimed(streamId);++ return true;}async streamAgentChat({@@ -343,6 +399,8 @@ export class AgentChatStreamingService {workspace.id,);+ await this.streamHeartbeatService.markClaimed(streamId);+await this.messageQueueService.add<StreamAgentChatJobData>(STREAM_AGENT_CHAT_JOB_NAME,{