The open alternative to Salesforce, designed for AI.
TypeScript crmcrm-systemcustomergood-first-issuegraphqlhacktoberfestjavascriptmarketingmonoreponestjsopen-sourcepostgresqlreactreactjssalestypescriptweb
Load sync cron channels with one query per tick (#26181)
neo773bec884cb51db
diff --git a/packages/twenty-server/src/modules/calendar/calendar-event-import-manager/crons/jobs/calendar-event-list-fetch.cron.job.ts b/packages/twenty-server/src/modules/calendar/calendar-event-import-manager/crons/jobs/calendar-event-list-fetch.cron.job.tsindex 00f846efcff..53331a98d69 100644--- a/packages/twenty-server/src/modules/calendar/calendar-event-import-manager/crons/jobs/calendar-event-list-fetch.cron.job.ts+++ b/packages/twenty-server/src/modules/calendar/calendar-event-import-manager/crons/jobs/calendar-event-list-fetch.cron.job.ts@@ -1,8 +1,9 @@+import groupBy from 'lodash.groupby';import { Logger } from '@nestjs/common';import { InjectRepository } from '@nestjs/typeorm';import { WorkspaceActivationStatus } from 'twenty-shared/workspace';-import { In, Repository } from 'typeorm';+import { In, IsNull, Repository } from 'typeorm';import {CalendarChannelSyncStage,@@ -15,7 +16,6 @@ import { Process } from 'src/engine/core-modules/message-queue/decorators/procesimport { Processor } from 'src/engine/core-modules/message-queue/decorators/processor.decorator';import { MessageQueue } from 'src/engine/core-modules/message-queue/message-queue.constants';import { MessageQueueService } from 'src/engine/core-modules/message-queue/services/message-queue.service';-import { WorkspaceEntity } from 'src/engine/core-modules/workspace/workspace.entity';import {CalendarEventListFetchJob,type CalendarEventListFetchJobData,@@ -34,8 +34,6 @@ export class CalendarEventListFetchCronJob {private readonly logger = new Logger(CalendarEventListFetchCronJob.name);constructor(- @InjectRepository(WorkspaceEntity)- private readonly workspaceRepository: Repository<WorkspaceEntity>,@InjectMessageQueue(MessageQueue.calendarQueue)private readonly messageQueueService: MessageQueueService,private readonly exceptionHandlerService: ExceptionHandlerService,@@ -49,24 +47,27 @@ export class CalendarEventListFetchCronJob {CALENDAR_EVENT_LIST_FETCH_CRON_PATTERN,)async handle(): Promise<void> {- const activeWorkspaces = await this.workspaceRepository.find({- where: {- activationStatus: WorkspaceActivationStatus.ACTIVE,- },- });+ const pendingCalendarChannelsAcrossWorkspaces =+ await this.calendarChannelRepository.find({+ where: {+ isSyncEnabled: true,+ syncStage: CalendarChannelSyncStage.CALENDAR_EVENT_LIST_FETCH_PENDING,+ workspace: {+ activationStatus: WorkspaceActivationStatus.ACTIVE,+ deletedAt: IsNull(),+ },+ },+ });- for (const activeWorkspace of activeWorkspaces) {- try {- const pendingCalendarChannels =- await this.calendarChannelRepository.find({- where: {- workspaceId: activeWorkspace.id,- isSyncEnabled: true,- syncStage:- CalendarChannelSyncStage.CALENDAR_EVENT_LIST_FETCH_PENDING,- },- });+ const pendingCalendarChannelsByWorkspaceId = groupBy(+ pendingCalendarChannelsAcrossWorkspaces,+ 'workspaceId',+ );+ for (const [workspaceId, pendingCalendarChannels] of Object.entries(+ pendingCalendarChannelsByWorkspaceId,+ )) {+ try {const calendarChannelsToSchedule = pendingCalendarChannels.filter((calendarChannel) =>!isThrottled(@@ -85,7 +86,7 @@ export class CalendarEventListFetchCronJob {if (throttledCount > 0) {this.logger.log(- `Skipped ${throttledCount} throttled calendar channels for workspace ${activeWorkspace.id}`,+ `Skipped ${throttledCount} throttled calendar channels for workspace ${workspaceId}`,);}@@ -107,7 +108,7 @@ export class CalendarEventListFetchCronJob {}).where({id: In(calendarChannelIds),- workspaceId: activeWorkspace.id,+ workspaceId,isSyncEnabled: true,syncStage:CalendarChannelSyncStage.CALENDAR_EVENT_LIST_FETCH_PENDING,@@ -124,14 +125,14 @@ export class CalendarEventListFetchCronJob {CalendarEventListFetchJob.name,{calendarChannelId,- workspaceId: activeWorkspace.id,+ workspaceId,},);}} catch (error) {this.exceptionHandlerService.captureExceptions([error], {workspace: {- id: activeWorkspace.id,+ id: workspaceId,},});}diff --git a/packages/twenty-server/src/modules/calendar/calendar-event-import-manager/crons/jobs/calendar-events-import.cron.job.ts b/packages/twenty-server/src/modules/calendar/calendar-event-import-manager/crons/jobs/calendar-events-import.cron.job.tsindex c97daa4075f..543e6f8350a 100644--- a/packages/twenty-server/src/modules/calendar/calendar-event-import-manager/crons/jobs/calendar-events-import.cron.job.ts+++ b/packages/twenty-server/src/modules/calendar/calendar-event-import-manager/crons/jobs/calendar-events-import.cron.job.ts@@ -1,8 +1,9 @@+import groupBy from 'lodash.groupby';import { Logger } from '@nestjs/common';import { InjectRepository } from '@nestjs/typeorm';import { WorkspaceActivationStatus } from 'twenty-shared/workspace';-import { In, Repository } from 'typeorm';+import { In, IsNull, Repository } from 'typeorm';import { CalendarChannelSyncStage } from 'twenty-shared/types';import { SentryCronMonitor } from 'src/engine/core-modules/cron/sentry-cron-monitor.decorator';@@ -12,7 +13,6 @@ import { Process } from 'src/engine/core-modules/message-queue/decorators/procesimport { Processor } from 'src/engine/core-modules/message-queue/decorators/processor.decorator';import { MessageQueue } from 'src/engine/core-modules/message-queue/message-queue.constants';import { MessageQueueService } from 'src/engine/core-modules/message-queue/services/message-queue.service';-import { WorkspaceEntity } from 'src/engine/core-modules/workspace/workspace.entity';import {CalendarEventsImportJob,type CalendarEventsImportJobData,@@ -30,8 +30,6 @@ export class CalendarEventsImportCronJob {private readonly logger = new Logger(CalendarEventsImportCronJob.name);constructor(- @InjectRepository(WorkspaceEntity)- private readonly workspaceRepository: Repository<WorkspaceEntity>,@InjectMessageQueue(MessageQueue.calendarQueue)private readonly messageQueueService: MessageQueueService,@InjectRepository(CalendarChannelEntity)@@ -45,24 +43,27 @@ export class CalendarEventsImportCronJob {CALENDAR_EVENTS_IMPORT_CRON_PATTERN,)async handle(): Promise<void> {- const activeWorkspaces = await this.workspaceRepository.find({- where: {- activationStatus: WorkspaceActivationStatus.ACTIVE,- },- });+ const pendingCalendarChannelsAcrossWorkspaces =+ await this.calendarChannelRepository.find({+ where: {+ isSyncEnabled: true,+ syncStage: CalendarChannelSyncStage.CALENDAR_EVENTS_IMPORT_PENDING,+ workspace: {+ activationStatus: WorkspaceActivationStatus.ACTIVE,+ deletedAt: IsNull(),+ },+ },+ });- for (const activeWorkspace of activeWorkspaces) {- try {- const pendingCalendarChannels =- await this.calendarChannelRepository.find({- where: {- workspaceId: activeWorkspace.id,- isSyncEnabled: true,- syncStage:- CalendarChannelSyncStage.CALENDAR_EVENTS_IMPORT_PENDING,- },- });+ const pendingCalendarChannelsByWorkspaceId = groupBy(+ pendingCalendarChannelsAcrossWorkspaces,+ 'workspaceId',+ );+ for (const [workspaceId, pendingCalendarChannels] of Object.entries(+ pendingCalendarChannelsByWorkspaceId,+ )) {+ try {const calendarChannelsToSchedule = pendingCalendarChannels.filter((calendarChannel) =>!isThrottled(@@ -76,7 +77,7 @@ export class CalendarEventsImportCronJob {if (throttledCount > 0) {this.logger.log(- `Skipped ${throttledCount} throttled calendar channels for workspace ${activeWorkspace.id}`,+ `Skipped ${throttledCount} throttled calendar channels for workspace ${workspaceId}`,);}@@ -98,7 +99,7 @@ export class CalendarEventsImportCronJob {}).where({id: In(calendarChannelIds),- workspaceId: activeWorkspace.id,+ workspaceId,isSyncEnabled: true,syncStage: CalendarChannelSyncStage.CALENDAR_EVENTS_IMPORT_PENDING,})@@ -114,14 +115,14 @@ export class CalendarEventsImportCronJob {CalendarEventsImportJob.name,{calendarChannelId,- workspaceId: activeWorkspace.id,+ workspaceId,},);}} catch (error) {this.exceptionHandlerService.captureExceptions([error], {workspace: {- id: activeWorkspace.id,+ id: workspaceId,},});}diff --git a/packages/twenty-server/src/modules/messaging/message-import-manager/crons/jobs/messaging-message-list-fetch.cron.job.ts b/packages/twenty-server/src/modules/messaging/message-import-manager/crons/jobs/messaging-message-list-fetch.cron.job.tsindex f4679da33ed..e00aa487a7b 100644--- a/packages/twenty-server/src/modules/messaging/message-import-manager/crons/jobs/messaging-message-list-fetch.cron.job.ts+++ b/packages/twenty-server/src/modules/messaging/message-import-manager/crons/jobs/messaging-message-list-fetch.cron.job.ts@@ -1,9 +1,10 @@import { Logger } from '@nestjs/common';import { InjectRepository } from '@nestjs/typeorm';+import groupBy from 'lodash.groupby';import { POLLED_MESSAGE_CHANNEL_TYPES } from 'twenty-shared/constants';import { WorkspaceActivationStatus } from 'twenty-shared/workspace';-import { In, Repository } from 'typeorm';+import { In, IsNull, Repository } from 'typeorm';import {MessageChannelSyncStage,@@ -16,7 +17,6 @@ import { Process } from 'src/engine/core-modules/message-queue/decorators/procesimport { Processor } from 'src/engine/core-modules/message-queue/decorators/processor.decorator';import { MessageQueue } from 'src/engine/core-modules/message-queue/message-queue.constants';import { MessageQueueService } from 'src/engine/core-modules/message-queue/services/message-queue.service';-import { WorkspaceEntity } from 'src/engine/core-modules/workspace/workspace.entity';import {MessagingMessageListFetchJob,type MessagingMessageListFetchJobData,@@ -33,8 +33,6 @@ export class MessagingMessageListFetchCronJob {private readonly logger = new Logger(MessagingMessageListFetchCronJob.name);constructor(- @InjectRepository(WorkspaceEntity)- private readonly workspaceRepository: Repository<WorkspaceEntity>,@InjectMessageQueue(MessageQueue.messagingQueue)private readonly messageQueueService: MessageQueueService,@InjectRepository(MessageChannelEntity)@@ -48,25 +46,28 @@ export class MessagingMessageListFetchCronJob {MESSAGING_MESSAGE_LIST_FETCH_CRON_PATTERN,)async handle(): Promise<void> {- const activeWorkspaces = await this.workspaceRepository.find({- where: {- activationStatus: WorkspaceActivationStatus.ACTIVE,- },- });-- for (const activeWorkspace of activeWorkspaces) {- try {- const pendingMessageChannels = await this.messageChannelRepository.find(- {- where: {- workspaceId: activeWorkspace.id,- isSyncEnabled: true,- syncStage: MessageChannelSyncStage.MESSAGE_LIST_FETCH_PENDING,- type: In([...POLLED_MESSAGE_CHANNEL_TYPES]),- },+ const pendingMessageChannelsAcrossWorkspaces =+ await this.messageChannelRepository.find({+ where: {+ isSyncEnabled: true,+ syncStage: MessageChannelSyncStage.MESSAGE_LIST_FETCH_PENDING,+ type: In([...POLLED_MESSAGE_CHANNEL_TYPES]),+ workspace: {+ activationStatus: WorkspaceActivationStatus.ACTIVE,+ deletedAt: IsNull(),},- );+ },+ });++ const pendingMessageChannelsByWorkspaceId = groupBy(+ pendingMessageChannelsAcrossWorkspaces,+ 'workspaceId',+ );+ for (const [workspaceId, pendingMessageChannels] of Object.entries(+ pendingMessageChannelsByWorkspaceId,+ )) {+ try {const messageChannelsToSchedule = pendingMessageChannels.filter((messageChannel) =>!isThrottled(@@ -86,7 +87,7 @@ export class MessagingMessageListFetchCronJob {if (throttledCount > 0) {this.logger.log(- `Skipped ${throttledCount} throttled message channels for workspace ${activeWorkspace.id}`,+ `Skipped ${throttledCount} throttled message channels for workspace ${workspaceId}`,);}@@ -107,7 +108,7 @@ export class MessagingMessageListFetchCronJob {}).where({id: In(messageChannelIdsToSchedule),- workspaceId: activeWorkspace.id,+ workspaceId,isSyncEnabled: true,syncStage: MessageChannelSyncStage.MESSAGE_LIST_FETCH_PENDING,})@@ -122,7 +123,7 @@ export class MessagingMessageListFetchCronJob {await this.messageQueueService.add<MessagingMessageListFetchJobData>(MessagingMessageListFetchJob.name,{- workspaceId: activeWorkspace.id,+ workspaceId,messageChannelId,},);@@ -130,7 +131,7 @@ export class MessagingMessageListFetchCronJob {} catch (error) {this.exceptionHandlerService.captureExceptions([error], {workspace: {- id: activeWorkspace.id,+ id: workspaceId,},});}diff --git a/packages/twenty-server/src/modules/messaging/message-import-manager/crons/jobs/messaging-messages-import.cron.job.ts b/packages/twenty-server/src/modules/messaging/message-import-manager/crons/jobs/messaging-messages-import.cron.job.tsindex 6861323883a..779e0d51739 100644--- a/packages/twenty-server/src/modules/messaging/message-import-manager/crons/jobs/messaging-messages-import.cron.job.ts+++ b/packages/twenty-server/src/modules/messaging/message-import-manager/crons/jobs/messaging-messages-import.cron.job.ts@@ -1,10 +1,11 @@+import groupBy from 'lodash.groupby';import { Logger } from '@nestjs/common';import { InjectRepository } from '@nestjs/typeorm';import { POLLED_MESSAGE_CHANNEL_TYPES } from 'twenty-shared/constants';import { isDefined } from 'twenty-shared/utils';import { WorkspaceActivationStatus } from 'twenty-shared/workspace';-import { In, Repository } from 'typeorm';+import { In, IsNull, Repository } from 'typeorm';import { MessageChannelSyncStage } from 'twenty-shared/types';import { SentryCronMonitor } from 'src/engine/core-modules/cron/sentry-cron-monitor.decorator';@@ -45,25 +46,28 @@ export class MessagingMessagesImportCronJob {MESSAGING_MESSAGES_IMPORT_CRON_PATTERN,)async handle(): Promise<void> {- const activeWorkspaces = await this.workspaceRepository.find({- where: {- activationStatus: WorkspaceActivationStatus.ACTIVE,- },- });-- for (const activeWorkspace of activeWorkspaces) {- try {- const pendingMessageChannels = await this.messageChannelRepository.find(- {- where: {- workspaceId: activeWorkspace.id,- isSyncEnabled: true,- syncStage: MessageChannelSyncStage.MESSAGES_IMPORT_PENDING,- type: In([...POLLED_MESSAGE_CHANNEL_TYPES]),- },+ const pendingMessageChannelsAcrossWorkspaces =+ await this.messageChannelRepository.find({+ where: {+ isSyncEnabled: true,+ syncStage: MessageChannelSyncStage.MESSAGES_IMPORT_PENDING,+ type: In([...POLLED_MESSAGE_CHANNEL_TYPES]),+ workspace: {+ activationStatus: WorkspaceActivationStatus.ACTIVE,+ deletedAt: IsNull(),},- );+ },+ });++ const pendingMessageChannelsByWorkspaceId = groupBy(+ pendingMessageChannelsAcrossWorkspaces,+ 'workspaceId',+ );+ for (const [workspaceId, pendingMessageChannels] of Object.entries(+ pendingMessageChannelsByWorkspaceId,+ )) {+ try {const messageChannelsToSchedule = pendingMessageChannels.filter((messageChannel) =>!isThrottled(@@ -78,7 +82,7 @@ export class MessagingMessagesImportCronJob {if (throttledCount > 0) {this.logger.log(- `Skipped ${throttledCount} throttled message channels for workspace ${activeWorkspace.id}`,+ `Skipped ${throttledCount} throttled message channels for workspace ${workspaceId}`,);}@@ -99,7 +103,7 @@ export class MessagingMessagesImportCronJob {}).where({id: In(messageChannelIdsToSchedule),- workspaceId: activeWorkspace.id,+ workspaceId,isSyncEnabled: true,syncStage: MessageChannelSyncStage.MESSAGES_IMPORT_PENDING,})@@ -114,7 +118,7 @@ export class MessagingMessagesImportCronJob {await this.messageQueueService.add<MessagingMessagesImportJobData>(MessagingMessagesImportJob.name,{- workspaceId: activeWorkspace.id,+ workspaceId,messageChannelId,},);@@ -125,13 +129,13 @@ export class MessagingMessagesImportCronJob {error.message.includes('messageChannel" does not exist')) {const refetchedWorkspace = await this.workspaceRepository.findOneBy({- id: activeWorkspace.id,+ id: workspaceId,});if (isDefined(refetchedWorkspace)) {this.exceptionHandlerService.captureExceptions([error], {workspace: {- id: activeWorkspace.id,+ id: workspaceId,},});throw new Error(@@ -141,7 +145,7 @@ export class MessagingMessagesImportCronJob {} else {this.exceptionHandlerService.captureExceptions([error], {workspace: {- id: activeWorkspace.id,+ id: workspaceId,},});}