The open alternative to Salesforce, designed for AI.
TypeScript crmcrm-systemcustomergood-first-issuegraphqlhacktoberfestjavascriptmarketingmonoreponestjsopen-sourcepostgresqlreactreactjssalestypescriptweb
refactor messaging jobs (#19626)
neo7737dfc556250a1
diff --git a/packages/twenty-server/project.json b/packages/twenty-server/project.jsonindex c4254421be3..54d809636e2 100644--- a/packages/twenty-server/project.json+++ b/packages/twenty-server/project.json@@ -117,10 +117,10 @@},"worker": {"executor": "nx:run-commands",- "dependsOn": ["build"],+ "dependsOn": ["^build"],"options": {"cwd": "packages/twenty-server",- "command": "node dist/queue-worker/queue-worker.js"+ "command": "NODE_ENV=development nest start --watch --entryFile queue-worker/queue-worker"}},"ts-node": {diff --git a/packages/twenty-server/src/modules/calendar/calendar-event-import-manager/crons/jobs/calendar-event-list-fetch.cron.job.ts b/packages/twenty-server/src/modules/calendar/calendar-event-import-manager/crons/jobs/calendar-event-list-fetch.cron.job.tsindex 890d33af0dc..220674febf0 100644--- a/packages/twenty-server/src/modules/calendar/calendar-event-import-manager/crons/jobs/calendar-event-list-fetch.cron.job.ts+++ b/packages/twenty-server/src/modules/calendar/calendar-event-import-manager/crons/jobs/calendar-event-list-fetch.cron.job.ts@@ -1,7 +1,8 @@-import { InjectDataSource, InjectRepository } from '@nestjs/typeorm';+import { Logger } from '@nestjs/common';+import { InjectRepository } from '@nestjs/typeorm';import { WorkspaceActivationStatus } from 'twenty-shared/workspace';-import { DataSource, Repository } from 'typeorm';+import { In, Repository } from 'typeorm';import { CalendarChannelSyncStage } from 'twenty-shared/types';import { SentryCronMonitor } from 'src/engine/core-modules/cron/sentry-cron-monitor.decorator';@@ -16,6 +17,9 @@ import {CalendarEventListFetchJob,type CalendarEventListFetchJobData,} from 'src/modules/calendar/calendar-event-import-manager/jobs/calendar-event-list-fetch.job';+import { CalendarChannelEntity } from 'src/engine/metadata-modules/calendar-channel/entities/calendar-channel.entity';+import { isThrottled } from 'src/modules/connected-account/utils/is-throttled';+import { toIsoStringOrNull } from 'src/utils/date/toIsoStringOrNull';export const CALENDAR_EVENT_LIST_FETCH_CRON_PATTERN = '*/5 * * * *';@@ -23,14 +27,16 @@ export const CALENDAR_EVENT_LIST_FETCH_CRON_PATTERN = '*/5 * * * *';queueName: MessageQueue.cronQueue,})export class CalendarEventListFetchCronJob {+ private readonly logger = new Logger(CalendarEventListFetchCronJob.name);+constructor(@InjectRepository(WorkspaceEntity)private readonly workspaceRepository: Repository<WorkspaceEntity>,@InjectMessageQueue(MessageQueue.calendarQueue)private readonly messageQueueService: MessageQueueService,private readonly exceptionHandlerService: ExceptionHandlerService,- @InjectDataSource()- private readonly coreDataSource: DataSource,+ @InjectRepository(CalendarChannelEntity)+ private readonly calendarChannelRepository: Repository<CalendarChannelEntity>,) {}@Process(CalendarEventListFetchCronJob.name)@@ -47,18 +53,68 @@ export class CalendarEventListFetchCronJob {for (const activeWorkspace of activeWorkspaces) {try {- const now = new Date().toISOString();+ const pendingCalendarChannels =+ await this.calendarChannelRepository.find({+ where: {+ workspaceId: activeWorkspace.id,+ isSyncEnabled: true,+ syncStage:+ CalendarChannelSyncStage.CALENDAR_EVENT_LIST_FETCH_PENDING,+ },+ });++ const calendarChannelsToSchedule = pendingCalendarChannels.filter(+ (calendarChannel) =>+ !isThrottled(+ toIsoStringOrNull(calendarChannel.syncStageStartedAt),+ calendarChannel.throttleFailureCount,+ ),+ );++ const throttledCount =+ pendingCalendarChannels.length - calendarChannelsToSchedule.length;++ if (throttledCount > 0) {+ this.logger.log(+ `Skipped ${throttledCount} throttled calendar channels for workspace ${activeWorkspace.id}`,+ );+ }++ if (calendarChannelsToSchedule.length === 0) {+ continue;+ }++ const calendarChannelIds = calendarChannelsToSchedule.map(+ (calendarChannel) => calendarChannel.id,+ );++ const updateResult = await this.calendarChannelRepository+ .createQueryBuilder()+ .update()+ .set({+ syncStage:+ CalendarChannelSyncStage.CALENDAR_EVENT_LIST_FETCH_SCHEDULED,+ syncStageStartedAt: new Date(),+ })+ .where({+ id: In(calendarChannelIds),+ workspaceId: activeWorkspace.id,+ isSyncEnabled: true,+ syncStage:+ CalendarChannelSyncStage.CALENDAR_EVENT_LIST_FETCH_PENDING,+ })+ .returning('id')+ .execute();- const [calendarChannels] = await this.coreDataSource.query(- `UPDATE core."calendarChannel" SET "syncStage" = '${CalendarChannelSyncStage.CALENDAR_EVENT_LIST_FETCH_SCHEDULED}', "syncStageStartedAt" = COALESCE("syncStageStartedAt", '${now}')- WHERE "workspaceId" = '${activeWorkspace.id}' AND "isSyncEnabled" = true AND "syncStage" = '${CalendarChannelSyncStage.CALENDAR_EVENT_LIST_FETCH_PENDING}' RETURNING *`,+ const updatedIds = updateResult.raw.map(+ (row: { id: string }) => row.id,);- for (const calendarChannel of calendarChannels) {+ for (const calendarChannelId of updatedIds) {await this.messageQueueService.add<CalendarEventListFetchJobData>(CalendarEventListFetchJob.name,{- calendarChannelId: calendarChannel.id,+ calendarChannelId,workspaceId: activeWorkspace.id,},);diff --git a/packages/twenty-server/src/modules/calendar/calendar-event-import-manager/crons/jobs/calendar-events-import.cron.job.ts b/packages/twenty-server/src/modules/calendar/calendar-event-import-manager/crons/jobs/calendar-events-import.cron.job.tsindex 060cfbc074d..c97daa4075f 100644--- a/packages/twenty-server/src/modules/calendar/calendar-event-import-manager/crons/jobs/calendar-events-import.cron.job.ts+++ b/packages/twenty-server/src/modules/calendar/calendar-event-import-manager/crons/jobs/calendar-events-import.cron.job.ts@@ -1,7 +1,8 @@-import { InjectDataSource, InjectRepository } from '@nestjs/typeorm';+import { Logger } from '@nestjs/common';+import { InjectRepository } from '@nestjs/typeorm';import { WorkspaceActivationStatus } from 'twenty-shared/workspace';-import { DataSource, Repository } from 'typeorm';+import { In, Repository } from 'typeorm';import { CalendarChannelSyncStage } from 'twenty-shared/types';import { SentryCronMonitor } from 'src/engine/core-modules/cron/sentry-cron-monitor.decorator';@@ -12,8 +13,13 @@ import { Processor } from 'src/engine/core-modules/message-queue/decorators/procimport { MessageQueue } from 'src/engine/core-modules/message-queue/message-queue.constants';import { MessageQueueService } from 'src/engine/core-modules/message-queue/services/message-queue.service';import { WorkspaceEntity } from 'src/engine/core-modules/workspace/workspace.entity';-import { type CalendarEventListFetchJobData } from 'src/modules/calendar/calendar-event-import-manager/jobs/calendar-event-list-fetch.job';-import { CalendarEventsImportJob } from 'src/modules/calendar/calendar-event-import-manager/jobs/calendar-events-import.job';+import {+ CalendarEventsImportJob,+ type CalendarEventsImportJobData,+} from 'src/modules/calendar/calendar-event-import-manager/jobs/calendar-events-import.job';+import { CalendarChannelEntity } from 'src/engine/metadata-modules/calendar-channel/entities/calendar-channel.entity';+import { isThrottled } from 'src/modules/connected-account/utils/is-throttled';+import { toIsoStringOrNull } from 'src/utils/date/toIsoStringOrNull';export const CALENDAR_EVENTS_IMPORT_CRON_PATTERN = '*/1 * * * *';@@ -21,13 +27,15 @@ export const CALENDAR_EVENTS_IMPORT_CRON_PATTERN = '*/1 * * * *';queueName: MessageQueue.cronQueue,})export class CalendarEventsImportCronJob {+ private readonly logger = new Logger(CalendarEventsImportCronJob.name);+constructor(@InjectRepository(WorkspaceEntity)private readonly workspaceRepository: Repository<WorkspaceEntity>,@InjectMessageQueue(MessageQueue.calendarQueue)private readonly messageQueueService: MessageQueueService,- @InjectDataSource()- private readonly coreDataSource: DataSource,+ @InjectRepository(CalendarChannelEntity)+ private readonly calendarChannelRepository: Repository<CalendarChannelEntity>,private readonly exceptionHandlerService: ExceptionHandlerService,) {}@@ -45,18 +53,67 @@ export class CalendarEventsImportCronJob {for (const activeWorkspace of activeWorkspaces) {try {- const now = new Date().toISOString();+ const pendingCalendarChannels =+ await this.calendarChannelRepository.find({+ where: {+ workspaceId: activeWorkspace.id,+ isSyncEnabled: true,+ syncStage:+ CalendarChannelSyncStage.CALENDAR_EVENTS_IMPORT_PENDING,+ },+ });++ const calendarChannelsToSchedule = pendingCalendarChannels.filter(+ (calendarChannel) =>+ !isThrottled(+ toIsoStringOrNull(calendarChannel.syncStageStartedAt),+ calendarChannel.throttleFailureCount,+ ),+ );++ const throttledCount =+ pendingCalendarChannels.length - calendarChannelsToSchedule.length;++ if (throttledCount > 0) {+ this.logger.log(+ `Skipped ${throttledCount} throttled calendar channels for workspace ${activeWorkspace.id}`,+ );+ }++ if (calendarChannelsToSchedule.length === 0) {+ continue;+ }++ const calendarChannelIds = calendarChannelsToSchedule.map(+ (calendarChannel) => calendarChannel.id,+ );++ const updateResult = await this.calendarChannelRepository+ .createQueryBuilder()+ .update()+ .set({+ syncStage:+ CalendarChannelSyncStage.CALENDAR_EVENTS_IMPORT_SCHEDULED,+ syncStageStartedAt: new Date(),+ })+ .where({+ id: In(calendarChannelIds),+ workspaceId: activeWorkspace.id,+ isSyncEnabled: true,+ syncStage: CalendarChannelSyncStage.CALENDAR_EVENTS_IMPORT_PENDING,+ })+ .returning('id')+ .execute();- const [calendarChannels] = await this.coreDataSource.query(- `UPDATE core."calendarChannel" SET "syncStage" = '${CalendarChannelSyncStage.CALENDAR_EVENTS_IMPORT_SCHEDULED}', "syncStageStartedAt" = COALESCE("syncStageStartedAt", '${now}')- WHERE "workspaceId" = '${activeWorkspace.id}' AND "isSyncEnabled" = true AND "syncStage" = '${CalendarChannelSyncStage.CALENDAR_EVENTS_IMPORT_PENDING}' RETURNING *`,+ const updatedIds = updateResult.raw.map(+ (row: { id: string }) => row.id,);- for (const calendarChannel of calendarChannels) {- await this.messageQueueService.add<CalendarEventListFetchJobData>(+ for (const calendarChannelId of updatedIds) {+ await this.messageQueueService.add<CalendarEventsImportJobData>(CalendarEventsImportJob.name,{- calendarChannelId: calendarChannel.id,+ calendarChannelId,workspaceId: activeWorkspace.id,},);diff --git a/packages/twenty-server/src/modules/calendar/calendar-event-import-manager/jobs/calendar-event-list-fetch.job.ts b/packages/twenty-server/src/modules/calendar/calendar-event-import-manager/jobs/calendar-event-list-fetch.job.tsindex 8d6f03e5a1c..22714322107 100644--- a/packages/twenty-server/src/modules/calendar/calendar-event-import-manager/jobs/calendar-event-list-fetch.job.ts+++ b/packages/twenty-server/src/modules/calendar/calendar-event-import-manager/jobs/calendar-event-list-fetch.job.ts@@ -10,8 +10,6 @@ import { MessageQueue } from 'src/engine/core-modules/message-queue/message-queuimport { GlobalWorkspaceOrmManager } from 'src/engine/twenty-orm/global-workspace-datasource/global-workspace-orm.manager';import { buildSystemAuthContext } from 'src/engine/twenty-orm/utils/build-system-auth-context.util';import { CalendarFetchEventsService } from 'src/modules/calendar/calendar-event-import-manager/services/calendar-fetch-events.service';-import { CalendarChannelSyncStatusService } from 'src/modules/calendar/common/services/calendar-channel-sync-status.service';-import { isThrottled } from 'src/modules/connected-account/utils/is-throttled';import { CalendarChannelEntity } from 'src/engine/metadata-modules/calendar-channel/entities/calendar-channel.entity';export type CalendarEventListFetchJobData = {@@ -28,7 +26,6 @@ export class CalendarEventListFetchJob {private readonly globalWorkspaceOrmManager: GlobalWorkspaceOrmManager,@InjectRepository(CalendarChannelEntity)private readonly calendarChannelRepository: Repository<CalendarChannelEntity>,- private readonly calendarChannelSyncStatusService: CalendarChannelSyncStatusService,private readonly calendarFetchEventsService: CalendarFetchEventsService,) {}@@ -59,23 +56,6 @@ export class CalendarEventListFetchJob {return;}- const syncStageStartedAt = calendarChannel.syncStageStartedAt;-- if (- isThrottled(- syncStageStartedAt?.toISOString() ?? null,- calendarChannel.throttleFailureCount,- )- ) {- await this.calendarChannelSyncStatusService.markAsCalendarEventListFetchPending(- [calendarChannel.id],- workspaceId,- true,- );-- return;- }-await this.calendarFetchEventsService.fetchCalendarEvents(calendarChannel as unknown as CalendarChannelEntity,calendarChannel.connectedAccount,diff --git a/packages/twenty-server/src/modules/calendar/calendar-event-import-manager/jobs/calendar-events-import.job.ts b/packages/twenty-server/src/modules/calendar/calendar-event-import-manager/jobs/calendar-events-import.job.tsindex f37338388c8..929aafa482a 100644--- a/packages/twenty-server/src/modules/calendar/calendar-event-import-manager/jobs/calendar-events-import.job.ts+++ b/packages/twenty-server/src/modules/calendar/calendar-event-import-manager/jobs/calendar-events-import.job.ts@@ -10,8 +10,6 @@ import { MessageQueue } from 'src/engine/core-modules/message-queue/message-queuimport { GlobalWorkspaceOrmManager } from 'src/engine/twenty-orm/global-workspace-datasource/global-workspace-orm.manager';import { buildSystemAuthContext } from 'src/engine/twenty-orm/utils/build-system-auth-context.util';import { CalendarEventsImportService } from 'src/modules/calendar/calendar-event-import-manager/services/calendar-events-import.service';-import { CalendarChannelSyncStatusService } from 'src/modules/calendar/common/services/calendar-channel-sync-status.service';-import { isThrottled } from 'src/modules/connected-account/utils/is-throttled';import { CalendarChannelEntity } from 'src/engine/metadata-modules/calendar-channel/entities/calendar-channel.entity';export type CalendarEventsImportJobData = {@@ -26,7 +24,6 @@ export type CalendarEventsImportJobData = {export class CalendarEventsImportJob {constructor(private readonly calendarEventsImportService: CalendarEventsImportService,- private readonly calendarChannelSyncStatusService: CalendarChannelSyncStatusService,private readonly globalWorkspaceOrmManager: GlobalWorkspaceOrmManager,@InjectRepository(CalendarChannelEntity)private readonly calendarChannelRepository: Repository<CalendarChannelEntity>,@@ -59,23 +56,6 @@ export class CalendarEventsImportJob {return;}- const syncStageStartedAt = calendarChannel.syncStageStartedAt;-- if (- isThrottled(- syncStageStartedAt?.toISOString() ?? null,- calendarChannel.throttleFailureCount,- )- ) {- await this.calendarChannelSyncStatusService.markAsCalendarEventsImportPending(- [calendarChannel.id],- workspaceId,- true,- );-- return;- }-await this.calendarEventsImportService.processCalendarEventsImport(calendarChannel as unknown as CalendarChannelEntity,calendarChannel.connectedAccount,diff --git a/packages/twenty-server/src/modules/connected-account/utils/__tests__/is-throttled.spec.ts b/packages/twenty-server/src/modules/connected-account/utils/__tests__/is-throttled.spec.tsindex 5ceb925ab74..80cb95fc8cc 100644--- a/packages/twenty-server/src/modules/connected-account/utils/__tests__/is-throttled.spec.ts+++ b/packages/twenty-server/src/modules/connected-account/utils/__tests__/is-throttled.spec.ts@@ -5,10 +5,28 @@ describe('isThrottled', () => {expect(isThrottled(null, 3)).toBe(false);});+ it('should throttle until retryAfter even when no sync stage is active', () => {+ const fiveMinutesFromNow = new Date(+ Date.now() + 5 * 60 * 1000,+ ).toISOString();++ expect(isThrottled(null, 3, fiveMinutesFromNow)).toBe(true);+ });+it('should not throttle when there have been no failures', () => {expect(isThrottled(new Date().toISOString(), 0)).toBe(false);});+ it('should throttle until retryAfter even when there have been no failures', () => {+ const fiveMinutesFromNow = new Date(+ Date.now() + 5 * 60 * 1000,+ ).toISOString();++ expect(isThrottled(new Date().toISOString(), 0, fiveMinutesFromNow)).toBe(+ true,+ );+ });+it('should keep throttling when retryAfter is in the future even though exponential backoff has expired', () => {const tenMinutesAgo = new Date(Date.now() - 10 * 60 * 1000).toISOString();const fiveMinutesFromNow = new Date(diff --git a/packages/twenty-server/src/modules/connected-account/utils/is-throttled.ts b/packages/twenty-server/src/modules/connected-account/utils/is-throttled.tsindex 1eaee3b8996..72be4240504 100644--- a/packages/twenty-server/src/modules/connected-account/utils/is-throttled.ts+++ b/packages/twenty-server/src/modules/connected-account/utils/is-throttled.ts@@ -8,6 +8,19 @@ export const isThrottled = (throttleFailureCount: number,throttleRetryAfter?: string | null,): boolean => {+ const now = new Date();++ const retryAfterCandidate = isDefined(throttleRetryAfter)+ ? new Date(throttleRetryAfter)+ : null;+ const retryAfterDate = isValidDate(retryAfterCandidate)+ ? retryAfterCandidate+ : null;++ if (isDefined(retryAfterDate) && retryAfterDate > now) {+ return true;+ }+if (!syncStageStartedAt) {return false;}@@ -16,25 +29,12 @@ export const isThrottled = (return false;}- const now = new Date();-const exponentialBackoffUntil = computeThrottlePauseUntil(syncStageStartedAt,throttleFailureCount,);- const retryAfterCandidate = isDefined(throttleRetryAfter)- ? new Date(throttleRetryAfter)- : null;- const retryAfterDate = isValidDate(retryAfterCandidate)- ? retryAfterCandidate- : null;-- const effectiveUntil =- isDefined(retryAfterDate) && retryAfterDate > exponentialBackoffUntil- ? retryAfterDate- : exponentialBackoffUntil;- return effectiveUntil > now;+ return exponentialBackoffUntil > now;};const computeThrottlePauseUntil = (diff --git a/packages/twenty-server/src/modules/messaging/message-import-manager/crons/jobs/messaging-message-list-fetch.cron.job.ts b/packages/twenty-server/src/modules/messaging/message-import-manager/crons/jobs/messaging-message-list-fetch.cron.job.tsindex ac945ccbc36..4e83cb4913f 100644--- a/packages/twenty-server/src/modules/messaging/message-import-manager/crons/jobs/messaging-message-list-fetch.cron.job.ts+++ b/packages/twenty-server/src/modules/messaging/message-import-manager/crons/jobs/messaging-message-list-fetch.cron.job.ts@@ -1,7 +1,8 @@-import { InjectDataSource, InjectRepository } from '@nestjs/typeorm';+import { Logger } from '@nestjs/common';+import { InjectRepository } from '@nestjs/typeorm';import { WorkspaceActivationStatus } from 'twenty-shared/workspace';-import { DataSource, Repository } from 'typeorm';+import { In, Repository } from 'typeorm';import { MessageChannelSyncStage } from 'twenty-shared/types';import { SentryCronMonitor } from 'src/engine/core-modules/cron/sentry-cron-monitor.decorator';@@ -16,18 +17,23 @@ import {MessagingMessageListFetchJob,type MessagingMessageListFetchJobData,} from 'src/modules/messaging/message-import-manager/jobs/messaging-message-list-fetch.job';+import { isThrottled } from 'src/modules/connected-account/utils/is-throttled';+import { MessageChannelEntity } from 'src/engine/metadata-modules/message-channel/entities/message-channel.entity';+import { toIsoStringOrNull } from 'src/utils/date/toIsoStringOrNull';export const MESSAGING_MESSAGE_LIST_FETCH_CRON_PATTERN = '2-59/5 * * * *';@Processor(MessageQueue.cronQueue)export class MessagingMessageListFetchCronJob {+ private readonly logger = new Logger(MessagingMessageListFetchCronJob.name);+constructor(@InjectRepository(WorkspaceEntity)private readonly workspaceRepository: Repository<WorkspaceEntity>,@InjectMessageQueue(MessageQueue.messagingQueue)private readonly messageQueueService: MessageQueueService,- @InjectDataSource()- private readonly coreDataSource: DataSource,+ @InjectRepository(MessageChannelEntity)+ private readonly messageChannelRepository: Repository<MessageChannelEntity>,private readonly exceptionHandlerService: ExceptionHandlerService,) {}@@ -45,19 +51,68 @@ export class MessagingMessageListFetchCronJob {for (const activeWorkspace of activeWorkspaces) {try {- const now = new Date().toISOString();+ const pendingMessageChannels = await this.messageChannelRepository.find(+ {+ where: {+ workspaceId: activeWorkspace.id,+ isSyncEnabled: true,+ syncStage: MessageChannelSyncStage.MESSAGE_LIST_FETCH_PENDING,+ },+ },+ );++ const messageChannelsToSchedule = pendingMessageChannels.filter(+ (messageChannel) =>+ !isThrottled(+ toIsoStringOrNull(messageChannel.syncStageStartedAt),+ messageChannel.throttleFailureCount,+ toIsoStringOrNull(messageChannel.throttleRetryAfter),+ ),+ );++ const throttledCount =+ pendingMessageChannels.length - messageChannelsToSchedule.length;++ if (throttledCount > 0) {+ this.logger.log(+ `Skipped ${throttledCount} throttled message channels for workspace ${activeWorkspace.id}`,+ );+ }++ if (messageChannelsToSchedule.length === 0) {+ continue;+ }++ const messageChannelIdsToSchedule = messageChannelsToSchedule.map(+ (messageChannel) => messageChannel.id,+ );++ const updateResult = await this.messageChannelRepository+ .createQueryBuilder()+ .update()+ .set({+ syncStage: MessageChannelSyncStage.MESSAGE_LIST_FETCH_SCHEDULED,+ syncStageStartedAt: new Date(),+ })+ .where({+ id: In(messageChannelIdsToSchedule),+ workspaceId: activeWorkspace.id,+ isSyncEnabled: true,+ syncStage: MessageChannelSyncStage.MESSAGE_LIST_FETCH_PENDING,+ })+ .returning('id')+ .execute();- const [messageChannels] = await this.coreDataSource.query(- `UPDATE core."messageChannel" SET "syncStage" = '${MessageChannelSyncStage.MESSAGE_LIST_FETCH_SCHEDULED}', "syncStageStartedAt" = COALESCE("syncStageStartedAt", '${now}')- WHERE "workspaceId" = '${activeWorkspace.id}' AND "isSyncEnabled" = true AND "syncStage" = '${MessageChannelSyncStage.MESSAGE_LIST_FETCH_PENDING}' RETURNING *`,+ const updatedIds = updateResult.raw.map(+ (row: { id: string }) => row.id,);- for (const messageChannel of messageChannels) {+ for (const messageChannelId of updatedIds) {await this.messageQueueService.add<MessagingMessageListFetchJobData>(MessagingMessageListFetchJob.name,{workspaceId: activeWorkspace.id,- messageChannelId: messageChannel.id,+ messageChannelId,},);}diff --git a/packages/twenty-server/src/modules/messaging/message-import-manager/crons/jobs/messaging-messages-import.cron.job.ts b/packages/twenty-server/src/modules/messaging/message-import-manager/crons/jobs/messaging-messages-import.cron.job.tsindex 6e884052b7a..02db9699a6d 100644--- a/packages/twenty-server/src/modules/messaging/message-import-manager/crons/jobs/messaging-messages-import.cron.job.ts+++ b/packages/twenty-server/src/modules/messaging/message-import-manager/crons/jobs/messaging-messages-import.cron.job.ts@@ -1,8 +1,9 @@-import { InjectDataSource, InjectRepository } from '@nestjs/typeorm';+import { Logger } from '@nestjs/common';+import { InjectRepository } from '@nestjs/typeorm';import { isDefined } from 'twenty-shared/utils';import { WorkspaceActivationStatus } from 'twenty-shared/workspace';-import { DataSource, Repository } from 'typeorm';+import { In, Repository } from 'typeorm';import { MessageChannelSyncStage } from 'twenty-shared/types';import { SentryCronMonitor } from 'src/engine/core-modules/cron/sentry-cron-monitor.decorator';@@ -17,19 +18,24 @@ import {MessagingMessagesImportJob,type MessagingMessagesImportJobData,} from 'src/modules/messaging/message-import-manager/jobs/messaging-messages-import.job';+import { isThrottled } from 'src/modules/connected-account/utils/is-throttled';+import { MessageChannelEntity } from 'src/engine/metadata-modules/message-channel/entities/message-channel.entity';+import { toIsoStringOrNull } from 'src/utils/date/toIsoStringOrNull';export const MESSAGING_MESSAGES_IMPORT_CRON_PATTERN = '*/1 * * * *';@Processor(MessageQueue.cronQueue)export class MessagingMessagesImportCronJob {+ private readonly logger = new Logger(MessagingMessagesImportCronJob.name);+constructor(@InjectRepository(WorkspaceEntity)private readonly workspaceRepository: Repository<WorkspaceEntity>,@InjectMessageQueue(MessageQueue.messagingQueue)private readonly messageQueueService: MessageQueueService,private readonly exceptionHandlerService: ExceptionHandlerService,- @InjectDataSource()- private readonly coreDataSource: DataSource,+ @InjectRepository(MessageChannelEntity)+ private readonly messageChannelRepository: Repository<MessageChannelEntity>,) {}@Process(MessagingMessagesImportCronJob.name)@@ -46,19 +52,68 @@ export class MessagingMessagesImportCronJob {for (const activeWorkspace of activeWorkspaces) {try {- const now = new Date().toISOString();+ const pendingMessageChannels = await this.messageChannelRepository.find(+ {+ where: {+ workspaceId: activeWorkspace.id,+ isSyncEnabled: true,+ syncStage: MessageChannelSyncStage.MESSAGES_IMPORT_PENDING,+ },+ },+ );++ const messageChannelsToSchedule = pendingMessageChannels.filter(+ (messageChannel) =>+ !isThrottled(+ toIsoStringOrNull(messageChannel.syncStageStartedAt),+ messageChannel.throttleFailureCount,+ toIsoStringOrNull(messageChannel.throttleRetryAfter),+ ),+ );++ const throttledCount =+ pendingMessageChannels.length - messageChannelsToSchedule.length;++ if (throttledCount > 0) {+ this.logger.log(+ `Skipped ${throttledCount} throttled message channels for workspace ${activeWorkspace.id}`,+ );+ }++ if (messageChannelsToSchedule.length === 0) {+ continue;+ }++ const messageChannelIdsToSchedule = messageChannelsToSchedule.map(+ (messageChannel) => messageChannel.id,+ );++ const updateResult = await this.messageChannelRepository+ .createQueryBuilder()+ .update()+ .set({+ syncStage: MessageChannelSyncStage.MESSAGES_IMPORT_SCHEDULED,+ syncStageStartedAt: new Date(),+ })+ .where({+ id: In(messageChannelIdsToSchedule),+ workspaceId: activeWorkspace.id,+ isSyncEnabled: true,+ syncStage: MessageChannelSyncStage.MESSAGES_IMPORT_PENDING,+ })+ .returning('id')+ .execute();- const [messageChannels] = await this.coreDataSource.query(- `UPDATE core."messageChannel" SET "syncStage" = '${MessageChannelSyncStage.MESSAGES_IMPORT_SCHEDULED}', "syncStageStartedAt" = COALESCE("syncStageStartedAt", '${now}')- WHERE "workspaceId" = '${activeWorkspace.id}' AND "isSyncEnabled" = true AND "syncStage" = '${MessageChannelSyncStage.MESSAGES_IMPORT_PENDING}' RETURNING *`,+ const updatedIds = updateResult.raw.map(+ (row: { id: string }) => row.id,);- for (const messageChannel of messageChannels) {+ for (const messageChannelId of updatedIds) {await this.messageQueueService.add<MessagingMessagesImportJobData>(MessagingMessagesImportJob.name,{workspaceId: activeWorkspace.id,- messageChannelId: messageChannel.id,+ messageChannelId,},);}diff --git a/packages/twenty-server/src/modules/messaging/message-import-manager/jobs/messaging-message-list-fetch.job.ts b/packages/twenty-server/src/modules/messaging/message-import-manager/jobs/messaging-message-list-fetch.job.tsindex 3a7152bc9d6..0cc6f868d84 100644--- a/packages/twenty-server/src/modules/messaging/message-import-manager/jobs/messaging-message-list-fetch.job.ts+++ b/packages/twenty-server/src/modules/messaging/message-import-manager/jobs/messaging-message-list-fetch.job.ts@@ -9,8 +9,6 @@ import { Processor } from 'src/engine/core-modules/message-queue/decorators/procimport { MessageQueue } from 'src/engine/core-modules/message-queue/message-queue.constants';import { GlobalWorkspaceOrmManager } from 'src/engine/twenty-orm/global-workspace-datasource/global-workspace-orm.manager';import { buildSystemAuthContext } from 'src/engine/twenty-orm/utils/build-system-auth-context.util';-import { isThrottled } from 'src/modules/connected-account/utils/is-throttled';-import { MessageChannelSyncStatusService } from 'src/modules/messaging/common/services/message-channel-sync-status.service';import {MessageImportExceptionHandlerService,MessageImportSyncStep,@@ -19,16 +17,6 @@ import { MessagingMessageListFetchService } from 'src/modules/messaging/message-import { MessagingMonitoringService } from 'src/modules/messaging/monitoring/services/messaging-monitoring.service';import { MessageChannelEntity } from 'src/engine/metadata-modules/message-channel/entities/message-channel.entity';-const toIsoStringOrNull = (- value: string | Date | null | undefined,-): string | null => {- if (value == null) {- return null;- }-- return value instanceof Date ? value.toISOString() : value;-};-export type MessagingMessageListFetchJobData = {messageChannelId: string;workspaceId: string;@@ -46,7 +34,6 @@ export class MessagingMessageListFetchJob {@InjectRepository(MessageChannelEntity)private readonly messageChannelRepository: Repository<MessageChannelEntity>,private readonly messageImportErrorHandlerService: MessageImportExceptionHandlerService,- private readonly messageChannelSyncStatusService: MessageChannelSyncStatusService,) {}@Process(MessagingMessageListFetchJob.name)@@ -88,22 +75,6 @@ export class MessagingMessageListFetchJob {}try {- if (- isThrottled(- toIsoStringOrNull(messageChannel.syncStageStartedAt),- messageChannel.throttleFailureCount,- toIsoStringOrNull(messageChannel.throttleRetryAfter),- )- ) {- await this.messageChannelSyncStatusService.markAsMessagesListFetchPending(- [messageChannel.id],- workspaceId,- true,- );-- return;- }-await this.messagingMonitoringService.track({eventName: 'message_list_fetch.started',workspaceId,diff --git a/packages/twenty-server/src/modules/messaging/message-import-manager/jobs/messaging-messages-import.job.ts b/packages/twenty-server/src/modules/messaging/message-import-manager/jobs/messaging-messages-import.job.tsindex c0c98d4b80a..a8a30d91cc6 100644--- a/packages/twenty-server/src/modules/messaging/message-import-manager/jobs/messaging-messages-import.job.ts+++ b/packages/twenty-server/src/modules/messaging/message-import-manager/jobs/messaging-messages-import.job.ts@@ -9,22 +9,10 @@ import { Processor } from 'src/engine/core-modules/message-queue/decorators/procimport { MessageQueue } from 'src/engine/core-modules/message-queue/message-queue.constants';import { GlobalWorkspaceOrmManager } from 'src/engine/twenty-orm/global-workspace-datasource/global-workspace-orm.manager';import { buildSystemAuthContext } from 'src/engine/twenty-orm/utils/build-system-auth-context.util';-import { isThrottled } from 'src/modules/connected-account/utils/is-throttled';-import { MessageChannelSyncStatusService } from 'src/modules/messaging/common/services/message-channel-sync-status.service';import { MessagingMessagesImportService } from 'src/modules/messaging/message-import-manager/services/messaging-messages-import.service';import { MessagingMonitoringService } from 'src/modules/messaging/monitoring/services/messaging-monitoring.service';import { MessageChannelEntity } from 'src/engine/metadata-modules/message-channel/entities/message-channel.entity';-const toIsoStringOrNull = (- value: string | Date | null | undefined,-): string | null => {- if (value == null) {- return null;- }-- return value instanceof Date ? value.toISOString() : value;-};-export type MessagingMessagesImportJobData = {messageChannelId: string;workspaceId: string;@@ -38,7 +26,6 @@ export class MessagingMessagesImportJob {constructor(private readonly messagingMessagesImportService: MessagingMessagesImportService,private readonly messagingMonitoringService: MessagingMonitoringService,- private readonly messageChannelSyncStatusService: MessageChannelSyncStatusService,private readonly globalWorkspaceOrmManager: GlobalWorkspaceOrmManager,@InjectRepository(MessageChannelEntity)private readonly messageChannelRepository: Repository<MessageChannelEntity>,@@ -86,22 +73,6 @@ export class MessagingMessagesImportJob {return;}- if (- isThrottled(- toIsoStringOrNull(messageChannel.syncStageStartedAt),- messageChannel.throttleFailureCount,- toIsoStringOrNull(messageChannel.throttleRetryAfter),- )- ) {- await this.messageChannelSyncStatusService.markAsMessagesImportPending(- [messageChannel.id],- workspaceId,- true,- );-- return;- }-await this.messagingMessagesImportService.processMessageBatchImport(messageChannel,messageChannel.connectedAccount,diff --git a/packages/twenty-server/src/modules/messaging/message-import-manager/jobs/messaging-ongoing-stale.job.ts b/packages/twenty-server/src/modules/messaging/message-import-manager/jobs/messaging-ongoing-stale.job.tsindex 7912a0e6883..25021a2d05a 100644--- a/packages/twenty-server/src/modules/messaging/message-import-manager/jobs/messaging-ongoing-stale.job.ts+++ b/packages/twenty-server/src/modules/messaging/message-import-manager/jobs/messaging-ongoing-stale.job.ts@@ -12,16 +12,7 @@ import { GlobalWorkspaceOrmManager } from 'src/engine/twenty-orm/global-workspacimport { buildSystemAuthContext } from 'src/engine/twenty-orm/utils/build-system-auth-context.util';import { MessageChannelSyncStatusService } from 'src/modules/messaging/common/services/message-channel-sync-status.service';import { isSyncStale } from 'src/modules/messaging/message-import-manager/utils/is-sync-stale.util';--const toIsoStringOrNull = (- value: string | Date | null | undefined,-): string | null => {- if (value == null) {- return null;- }-- return value instanceof Date ? value.toISOString() : value;-};+import { toIsoStringOrNull } from 'src/utils/date/toIsoStringOrNull';export type MessagingOngoingStaleJobData = {workspaceId: string;diff --git a/packages/twenty-server/src/utils/date/__tests__/toIsoStringOrNull.spec.ts b/packages/twenty-server/src/utils/date/__tests__/toIsoStringOrNull.spec.tsnew file mode 100644index 00000000000..59e26b720a8--- /dev/null+++ b/packages/twenty-server/src/utils/date/__tests__/toIsoStringOrNull.spec.ts@@ -0,0 +1,24 @@+import { toIsoStringOrNull } from 'src/utils/date/toIsoStringOrNull';++describe('toIsoStringOrNull', () => {+ it('should return null for null or undefined', () => {+ expect(toIsoStringOrNull(null)).toBeNull();+ expect(toIsoStringOrNull(undefined)).toBeNull();+ });++ it('should convert Date to ISO string', () => {+ const date = new Date('2024-01-15T10:30:00.000Z');++ expect(toIsoStringOrNull(date)).toBe('2024-01-15T10:30:00.000Z');+ });++ it('should pass through strings unchanged', () => {+ expect(toIsoStringOrNull('2024-01-15T10:30:00.000Z')).toBe(+ '2024-01-15T10:30:00.000Z',+ );+ });++ it('should throw on invalid Date', () => {+ expect(() => toIsoStringOrNull(new Date('invalid'))).toThrow(RangeError);+ });+});diff --git a/packages/twenty-server/src/utils/date/toIsoStringOrNull.ts b/packages/twenty-server/src/utils/date/toIsoStringOrNull.tsnew file mode 100644index 00000000000..ba92adaa071--- /dev/null+++ b/packages/twenty-server/src/utils/date/toIsoStringOrNull.ts@@ -0,0 +1,9 @@+export const toIsoStringOrNull = (+ value: string | Date | null | undefined,+): string | null => {+ if (value == null) {+ return null;+ }++ return value instanceof Date ? value.toISOString() : value;+};