Skip to content

The open alternative to Salesforce, designed for AI.

Load sync cron channels with one query per tick (#26181)

neo773bec884cb51db
diff --git a/packages/twenty-server/src/modules/calendar/calendar-event-import-manager/crons/jobs/calendar-event-list-fetch.cron.job.ts b/packages/twenty-server/src/modules/calendar/calendar-event-import-manager/crons/jobs/calendar-event-list-fetch.cron.job.ts
index 00f846efcff..53331a98d69 100644
--- a/packages/twenty-server/src/modules/calendar/calendar-event-import-manager/crons/jobs/calendar-event-list-fetch.cron.job.ts
+++ b/packages/twenty-server/src/modules/calendar/calendar-event-import-manager/crons/jobs/calendar-event-list-fetch.cron.job.ts
@@ -1,8 +1,9 @@
+import groupBy from 'lodash.groupby';
import { Logger } from '@nestjs/common';
import { InjectRepository } from '@nestjs/typeorm';
import { WorkspaceActivationStatus } from 'twenty-shared/workspace';
-import { In, Repository } from 'typeorm';
+import { In, IsNull, Repository } from 'typeorm';
import {
CalendarChannelSyncStage,
@@ -15,7 +16,6 @@ import { Process } from 'src/engine/core-modules/message-queue/decorators/proces
import { Processor } from 'src/engine/core-modules/message-queue/decorators/processor.decorator';
import { MessageQueue } from 'src/engine/core-modules/message-queue/message-queue.constants';
import { MessageQueueService } from 'src/engine/core-modules/message-queue/services/message-queue.service';
-import { WorkspaceEntity } from 'src/engine/core-modules/workspace/workspace.entity';
import {
CalendarEventListFetchJob,
type CalendarEventListFetchJobData,
@@ -34,8 +34,6 @@ export class CalendarEventListFetchCronJob {
private readonly logger = new Logger(CalendarEventListFetchCronJob.name);
constructor(
- @InjectRepository(WorkspaceEntity)
- private readonly workspaceRepository: Repository<WorkspaceEntity>,
@InjectMessageQueue(MessageQueue.calendarQueue)
private readonly messageQueueService: MessageQueueService,
private readonly exceptionHandlerService: ExceptionHandlerService,
@@ -49,24 +47,27 @@ export class CalendarEventListFetchCronJob {
CALENDAR_EVENT_LIST_FETCH_CRON_PATTERN,
)
async handle(): Promise<void> {
- const activeWorkspaces = await this.workspaceRepository.find({
- where: {
- activationStatus: WorkspaceActivationStatus.ACTIVE,
- },
- });
+ const pendingCalendarChannelsAcrossWorkspaces =
+ await this.calendarChannelRepository.find({
+ where: {
+ isSyncEnabled: true,
+ syncStage: CalendarChannelSyncStage.CALENDAR_EVENT_LIST_FETCH_PENDING,
+ workspace: {
+ activationStatus: WorkspaceActivationStatus.ACTIVE,
+ deletedAt: IsNull(),
+ },
+ },
+ });
- for (const activeWorkspace of activeWorkspaces) {
- try {
- const pendingCalendarChannels =
- await this.calendarChannelRepository.find({
- where: {
- workspaceId: activeWorkspace.id,
- isSyncEnabled: true,
- syncStage:
- CalendarChannelSyncStage.CALENDAR_EVENT_LIST_FETCH_PENDING,
- },
- });
+ const pendingCalendarChannelsByWorkspaceId = groupBy(
+ pendingCalendarChannelsAcrossWorkspaces,
+ 'workspaceId',
+ );
+ for (const [workspaceId, pendingCalendarChannels] of Object.entries(
+ pendingCalendarChannelsByWorkspaceId,
+ )) {
+ try {
const calendarChannelsToSchedule = pendingCalendarChannels.filter(
(calendarChannel) =>
!isThrottled(
@@ -85,7 +86,7 @@ export class CalendarEventListFetchCronJob {
if (throttledCount > 0) {
this.logger.log(
- `Skipped ${throttledCount} throttled calendar channels for workspace ${activeWorkspace.id}`,
+ `Skipped ${throttledCount} throttled calendar channels for workspace ${workspaceId}`,
);
}
@@ -107,7 +108,7 @@ export class CalendarEventListFetchCronJob {
})
.where({
id: In(calendarChannelIds),
- workspaceId: activeWorkspace.id,
+ workspaceId,
isSyncEnabled: true,
syncStage:
CalendarChannelSyncStage.CALENDAR_EVENT_LIST_FETCH_PENDING,
@@ -124,14 +125,14 @@ export class CalendarEventListFetchCronJob {
CalendarEventListFetchJob.name,
{
calendarChannelId,
- workspaceId: activeWorkspace.id,
+ workspaceId,
},
);
}
} catch (error) {
this.exceptionHandlerService.captureExceptions([error], {
workspace: {
- id: activeWorkspace.id,
+ id: workspaceId,
},
});
}
diff --git a/packages/twenty-server/src/modules/calendar/calendar-event-import-manager/crons/jobs/calendar-events-import.cron.job.ts b/packages/twenty-server/src/modules/calendar/calendar-event-import-manager/crons/jobs/calendar-events-import.cron.job.ts
index c97daa4075f..543e6f8350a 100644
--- a/packages/twenty-server/src/modules/calendar/calendar-event-import-manager/crons/jobs/calendar-events-import.cron.job.ts
+++ b/packages/twenty-server/src/modules/calendar/calendar-event-import-manager/crons/jobs/calendar-events-import.cron.job.ts
@@ -1,8 +1,9 @@
+import groupBy from 'lodash.groupby';
import { Logger } from '@nestjs/common';
import { InjectRepository } from '@nestjs/typeorm';
import { WorkspaceActivationStatus } from 'twenty-shared/workspace';
-import { In, Repository } from 'typeorm';
+import { In, IsNull, Repository } from 'typeorm';
import { CalendarChannelSyncStage } from 'twenty-shared/types';
import { SentryCronMonitor } from 'src/engine/core-modules/cron/sentry-cron-monitor.decorator';
@@ -12,7 +13,6 @@ import { Process } from 'src/engine/core-modules/message-queue/decorators/proces
import { Processor } from 'src/engine/core-modules/message-queue/decorators/processor.decorator';
import { MessageQueue } from 'src/engine/core-modules/message-queue/message-queue.constants';
import { MessageQueueService } from 'src/engine/core-modules/message-queue/services/message-queue.service';
-import { WorkspaceEntity } from 'src/engine/core-modules/workspace/workspace.entity';
import {
CalendarEventsImportJob,
type CalendarEventsImportJobData,
@@ -30,8 +30,6 @@ export class CalendarEventsImportCronJob {
private readonly logger = new Logger(CalendarEventsImportCronJob.name);
constructor(
- @InjectRepository(WorkspaceEntity)
- private readonly workspaceRepository: Repository<WorkspaceEntity>,
@InjectMessageQueue(MessageQueue.calendarQueue)
private readonly messageQueueService: MessageQueueService,
@InjectRepository(CalendarChannelEntity)
@@ -45,24 +43,27 @@ export class CalendarEventsImportCronJob {
CALENDAR_EVENTS_IMPORT_CRON_PATTERN,
)
async handle(): Promise<void> {
- const activeWorkspaces = await this.workspaceRepository.find({
- where: {
- activationStatus: WorkspaceActivationStatus.ACTIVE,
- },
- });
+ const pendingCalendarChannelsAcrossWorkspaces =
+ await this.calendarChannelRepository.find({
+ where: {
+ isSyncEnabled: true,
+ syncStage: CalendarChannelSyncStage.CALENDAR_EVENTS_IMPORT_PENDING,
+ workspace: {
+ activationStatus: WorkspaceActivationStatus.ACTIVE,
+ deletedAt: IsNull(),
+ },
+ },
+ });
- for (const activeWorkspace of activeWorkspaces) {
- try {
- const pendingCalendarChannels =
- await this.calendarChannelRepository.find({
- where: {
- workspaceId: activeWorkspace.id,
- isSyncEnabled: true,
- syncStage:
- CalendarChannelSyncStage.CALENDAR_EVENTS_IMPORT_PENDING,
- },
- });
+ const pendingCalendarChannelsByWorkspaceId = groupBy(
+ pendingCalendarChannelsAcrossWorkspaces,
+ 'workspaceId',
+ );
+ for (const [workspaceId, pendingCalendarChannels] of Object.entries(
+ pendingCalendarChannelsByWorkspaceId,
+ )) {
+ try {
const calendarChannelsToSchedule = pendingCalendarChannels.filter(
(calendarChannel) =>
!isThrottled(
@@ -76,7 +77,7 @@ export class CalendarEventsImportCronJob {
if (throttledCount > 0) {
this.logger.log(
- `Skipped ${throttledCount} throttled calendar channels for workspace ${activeWorkspace.id}`,
+ `Skipped ${throttledCount} throttled calendar channels for workspace ${workspaceId}`,
);
}
@@ -98,7 +99,7 @@ export class CalendarEventsImportCronJob {
})
.where({
id: In(calendarChannelIds),
- workspaceId: activeWorkspace.id,
+ workspaceId,
isSyncEnabled: true,
syncStage: CalendarChannelSyncStage.CALENDAR_EVENTS_IMPORT_PENDING,
})
@@ -114,14 +115,14 @@ export class CalendarEventsImportCronJob {
CalendarEventsImportJob.name,
{
calendarChannelId,
- workspaceId: activeWorkspace.id,
+ workspaceId,
},
);
}
} catch (error) {
this.exceptionHandlerService.captureExceptions([error], {
workspace: {
- id: activeWorkspace.id,
+ id: workspaceId,
},
});
}
diff --git a/packages/twenty-server/src/modules/messaging/message-import-manager/crons/jobs/messaging-message-list-fetch.cron.job.ts b/packages/twenty-server/src/modules/messaging/message-import-manager/crons/jobs/messaging-message-list-fetch.cron.job.ts
index f4679da33ed..e00aa487a7b 100644
--- a/packages/twenty-server/src/modules/messaging/message-import-manager/crons/jobs/messaging-message-list-fetch.cron.job.ts
+++ b/packages/twenty-server/src/modules/messaging/message-import-manager/crons/jobs/messaging-message-list-fetch.cron.job.ts
@@ -1,9 +1,10 @@
import { Logger } from '@nestjs/common';
import { InjectRepository } from '@nestjs/typeorm';
+import groupBy from 'lodash.groupby';
import { POLLED_MESSAGE_CHANNEL_TYPES } from 'twenty-shared/constants';
import { WorkspaceActivationStatus } from 'twenty-shared/workspace';
-import { In, Repository } from 'typeorm';
+import { In, IsNull, Repository } from 'typeorm';
import {
MessageChannelSyncStage,
@@ -16,7 +17,6 @@ import { Process } from 'src/engine/core-modules/message-queue/decorators/proces
import { Processor } from 'src/engine/core-modules/message-queue/decorators/processor.decorator';
import { MessageQueue } from 'src/engine/core-modules/message-queue/message-queue.constants';
import { MessageQueueService } from 'src/engine/core-modules/message-queue/services/message-queue.service';
-import { WorkspaceEntity } from 'src/engine/core-modules/workspace/workspace.entity';
import {
MessagingMessageListFetchJob,
type MessagingMessageListFetchJobData,
@@ -33,8 +33,6 @@ export class MessagingMessageListFetchCronJob {
private readonly logger = new Logger(MessagingMessageListFetchCronJob.name);
constructor(
- @InjectRepository(WorkspaceEntity)
- private readonly workspaceRepository: Repository<WorkspaceEntity>,
@InjectMessageQueue(MessageQueue.messagingQueue)
private readonly messageQueueService: MessageQueueService,
@InjectRepository(MessageChannelEntity)
@@ -48,25 +46,28 @@ export class MessagingMessageListFetchCronJob {
MESSAGING_MESSAGE_LIST_FETCH_CRON_PATTERN,
)
async handle(): Promise<void> {
- const activeWorkspaces = await this.workspaceRepository.find({
- where: {
- activationStatus: WorkspaceActivationStatus.ACTIVE,
- },
- });
-
- for (const activeWorkspace of activeWorkspaces) {
- try {
- const pendingMessageChannels = await this.messageChannelRepository.find(
- {
- where: {
- workspaceId: activeWorkspace.id,
- isSyncEnabled: true,
- syncStage: MessageChannelSyncStage.MESSAGE_LIST_FETCH_PENDING,
- type: In([...POLLED_MESSAGE_CHANNEL_TYPES]),
- },
+ const pendingMessageChannelsAcrossWorkspaces =
+ await this.messageChannelRepository.find({
+ where: {
+ isSyncEnabled: true,
+ syncStage: MessageChannelSyncStage.MESSAGE_LIST_FETCH_PENDING,
+ type: In([...POLLED_MESSAGE_CHANNEL_TYPES]),
+ workspace: {
+ activationStatus: WorkspaceActivationStatus.ACTIVE,
+ deletedAt: IsNull(),
},
- );
+ },
+ });
+
+ const pendingMessageChannelsByWorkspaceId = groupBy(
+ pendingMessageChannelsAcrossWorkspaces,
+ 'workspaceId',
+ );
+ for (const [workspaceId, pendingMessageChannels] of Object.entries(
+ pendingMessageChannelsByWorkspaceId,
+ )) {
+ try {
const messageChannelsToSchedule = pendingMessageChannels.filter(
(messageChannel) =>
!isThrottled(
@@ -86,7 +87,7 @@ export class MessagingMessageListFetchCronJob {
if (throttledCount > 0) {
this.logger.log(
- `Skipped ${throttledCount} throttled message channels for workspace ${activeWorkspace.id}`,
+ `Skipped ${throttledCount} throttled message channels for workspace ${workspaceId}`,
);
}
@@ -107,7 +108,7 @@ export class MessagingMessageListFetchCronJob {
})
.where({
id: In(messageChannelIdsToSchedule),
- workspaceId: activeWorkspace.id,
+ workspaceId,
isSyncEnabled: true,
syncStage: MessageChannelSyncStage.MESSAGE_LIST_FETCH_PENDING,
})
@@ -122,7 +123,7 @@ export class MessagingMessageListFetchCronJob {
await this.messageQueueService.add<MessagingMessageListFetchJobData>(
MessagingMessageListFetchJob.name,
{
- workspaceId: activeWorkspace.id,
+ workspaceId,
messageChannelId,
},
);
@@ -130,7 +131,7 @@ export class MessagingMessageListFetchCronJob {
} catch (error) {
this.exceptionHandlerService.captureExceptions([error], {
workspace: {
- id: activeWorkspace.id,
+ id: workspaceId,
},
});
}
diff --git a/packages/twenty-server/src/modules/messaging/message-import-manager/crons/jobs/messaging-messages-import.cron.job.ts b/packages/twenty-server/src/modules/messaging/message-import-manager/crons/jobs/messaging-messages-import.cron.job.ts
index 6861323883a..779e0d51739 100644
--- a/packages/twenty-server/src/modules/messaging/message-import-manager/crons/jobs/messaging-messages-import.cron.job.ts
+++ b/packages/twenty-server/src/modules/messaging/message-import-manager/crons/jobs/messaging-messages-import.cron.job.ts
@@ -1,10 +1,11 @@
+import groupBy from 'lodash.groupby';
import { Logger } from '@nestjs/common';
import { InjectRepository } from '@nestjs/typeorm';
import { POLLED_MESSAGE_CHANNEL_TYPES } from 'twenty-shared/constants';
import { isDefined } from 'twenty-shared/utils';
import { WorkspaceActivationStatus } from 'twenty-shared/workspace';
-import { In, Repository } from 'typeorm';
+import { In, IsNull, Repository } from 'typeorm';
import { MessageChannelSyncStage } from 'twenty-shared/types';
import { SentryCronMonitor } from 'src/engine/core-modules/cron/sentry-cron-monitor.decorator';
@@ -45,25 +46,28 @@ export class MessagingMessagesImportCronJob {
MESSAGING_MESSAGES_IMPORT_CRON_PATTERN,
)
async handle(): Promise<void> {
- const activeWorkspaces = await this.workspaceRepository.find({
- where: {
- activationStatus: WorkspaceActivationStatus.ACTIVE,
- },
- });
-
- for (const activeWorkspace of activeWorkspaces) {
- try {
- const pendingMessageChannels = await this.messageChannelRepository.find(
- {
- where: {
- workspaceId: activeWorkspace.id,
- isSyncEnabled: true,
- syncStage: MessageChannelSyncStage.MESSAGES_IMPORT_PENDING,
- type: In([...POLLED_MESSAGE_CHANNEL_TYPES]),
- },
+ const pendingMessageChannelsAcrossWorkspaces =
+ await this.messageChannelRepository.find({
+ where: {
+ isSyncEnabled: true,
+ syncStage: MessageChannelSyncStage.MESSAGES_IMPORT_PENDING,
+ type: In([...POLLED_MESSAGE_CHANNEL_TYPES]),
+ workspace: {
+ activationStatus: WorkspaceActivationStatus.ACTIVE,
+ deletedAt: IsNull(),
},
- );
+ },
+ });
+
+ const pendingMessageChannelsByWorkspaceId = groupBy(
+ pendingMessageChannelsAcrossWorkspaces,
+ 'workspaceId',
+ );
+ for (const [workspaceId, pendingMessageChannels] of Object.entries(
+ pendingMessageChannelsByWorkspaceId,
+ )) {
+ try {
const messageChannelsToSchedule = pendingMessageChannels.filter(
(messageChannel) =>
!isThrottled(
@@ -78,7 +82,7 @@ export class MessagingMessagesImportCronJob {
if (throttledCount > 0) {
this.logger.log(
- `Skipped ${throttledCount} throttled message channels for workspace ${activeWorkspace.id}`,
+ `Skipped ${throttledCount} throttled message channels for workspace ${workspaceId}`,
);
}
@@ -99,7 +103,7 @@ export class MessagingMessagesImportCronJob {
})
.where({
id: In(messageChannelIdsToSchedule),
- workspaceId: activeWorkspace.id,
+ workspaceId,
isSyncEnabled: true,
syncStage: MessageChannelSyncStage.MESSAGES_IMPORT_PENDING,
})
@@ -114,7 +118,7 @@ export class MessagingMessagesImportCronJob {
await this.messageQueueService.add<MessagingMessagesImportJobData>(
MessagingMessagesImportJob.name,
{
- workspaceId: activeWorkspace.id,
+ workspaceId,
messageChannelId,
},
);
@@ -125,13 +129,13 @@ export class MessagingMessagesImportCronJob {
error.message.includes('messageChannel" does not exist')
) {
const refetchedWorkspace = await this.workspaceRepository.findOneBy({
- id: activeWorkspace.id,
+ id: workspaceId,
});
if (isDefined(refetchedWorkspace)) {
this.exceptionHandlerService.captureExceptions([error], {
workspace: {
- id: activeWorkspace.id,
+ id: workspaceId,
},
});
throw new Error(
@@ -141,7 +145,7 @@ export class MessagingMessagesImportCronJob {
} else {
this.exceptionHandlerService.captureExceptions([error], {
workspace: {
- id: activeWorkspace.id,
+ id: workspaceId,
},
});
}