The open alternative to Salesforce, designed for AI.
TypeScript crmcrm-systemcustomergood-first-issuegraphqlhacktoberfestjavascriptmarketingmonoreponestjsopen-sourcepostgresqlreactreactjssalestypescriptweb
Debounce calendar webhook syncs instead of throwing when one is running (#25656)
neo77340e95c5cfe19
diff --git a/packages/twenty-server/src/modules/connected-account-sync-webhooks/calendar-event-webhook-sync/calendar-event-webhook-sync.exception.ts b/packages/twenty-server/src/modules/connected-account-sync-webhooks/calendar-event-webhook-sync/calendar-event-webhook-sync.exception.tsdeleted file mode 100644index eeba2f3b632..00000000000--- a/packages/twenty-server/src/modules/connected-account-sync-webhooks/calendar-event-webhook-sync/calendar-event-webhook-sync.exception.ts+++ /dev/null@@ -1,34 +0,0 @@-import { type MessageDescriptor } from '@lingui/core';-import { msg } from '@lingui/core/macro';-import { assertUnreachable } from 'twenty-shared/utils';--import { CustomException } from 'src/utils/custom-exception';--export enum CalendarEventWebhookSyncExceptionCode {- CALENDAR_CHANNEL_SYNC_ALREADY_IN_PROGRESS = 'CALENDAR_CHANNEL_SYNC_ALREADY_IN_PROGRESS',-}--const getCalendarEventWebhookSyncExceptionUserFriendlyMessage = (- code: CalendarEventWebhookSyncExceptionCode,-) => {- switch (code) {- case CalendarEventWebhookSyncExceptionCode.CALENDAR_CHANNEL_SYNC_ALREADY_IN_PROGRESS:- return msg`The calendar is already syncing.`;- default:- assertUnreachable(code);- }-};--export class CalendarEventWebhookSyncException extends CustomException<CalendarEventWebhookSyncExceptionCode> {- constructor(- message: string,- code: CalendarEventWebhookSyncExceptionCode,- { userFriendlyMessage }: { userFriendlyMessage?: MessageDescriptor } = {},- ) {- super(message, code, {- userFriendlyMessage:- userFriendlyMessage ??- getCalendarEventWebhookSyncExceptionUserFriendlyMessage(code),- });- }-}diff --git a/packages/twenty-server/src/modules/connected-account-sync-webhooks/calendar-event-webhook-sync/constants/calendar-event-webhook-sync-debounce-ms.constant.ts b/packages/twenty-server/src/modules/connected-account-sync-webhooks/calendar-event-webhook-sync/constants/calendar-event-webhook-sync-debounce-ms.constant.tsnew file mode 100644index 00000000000..7fc96b1652c--- /dev/null+++ b/packages/twenty-server/src/modules/connected-account-sync-webhooks/calendar-event-webhook-sync/constants/calendar-event-webhook-sync-debounce-ms.constant.ts@@ -0,0 +1 @@+export const CALENDAR_EVENT_WEBHOOK_SYNC_DEBOUNCE_MS = 30000;diff --git a/packages/twenty-server/src/modules/connected-account-sync-webhooks/calendar-event-webhook-sync/constants/calendar-event-webhook-sync-retry-jitter.constant.ts b/packages/twenty-server/src/modules/connected-account-sync-webhooks/calendar-event-webhook-sync/constants/calendar-event-webhook-sync-retry-jitter.constant.tsnew file mode 100644index 00000000000..ca28c4d2247--- /dev/null+++ b/packages/twenty-server/src/modules/connected-account-sync-webhooks/calendar-event-webhook-sync/constants/calendar-event-webhook-sync-retry-jitter.constant.ts@@ -0,0 +1 @@+export const CALENDAR_EVENT_WEBHOOK_SYNC_RETRY_JITTER = 0.5;diff --git a/packages/twenty-server/src/modules/connected-account-sync-webhooks/calendar-event-webhook-sync/jobs/calendar-event-webhook-sync.job.ts b/packages/twenty-server/src/modules/connected-account-sync-webhooks/calendar-event-webhook-sync/jobs/calendar-event-webhook-sync.job.tsindex da86d4b33b3..6c034652bed 100644--- a/packages/twenty-server/src/modules/connected-account-sync-webhooks/calendar-event-webhook-sync/jobs/calendar-event-webhook-sync.job.ts+++ b/packages/twenty-server/src/modules/connected-account-sync-webhooks/calendar-event-webhook-sync/jobs/calendar-event-webhook-sync.job.ts@@ -4,11 +4,7 @@ import { Process } from 'src/engine/core-modules/message-queue/decorators/procesimport { Processor } from 'src/engine/core-modules/message-queue/decorators/processor.decorator';import { MessageQueue } from 'src/engine/core-modules/message-queue/message-queue.constants';import { CalendarEventWebhookSyncService } from 'src/modules/connected-account-sync-webhooks/calendar-event-webhook-sync/services/calendar-event-webhook-sync.service';--export type CalendarEventWebhookSyncJobData = {- calendarChannelId: string;- workspaceId: string;-};+import { type CalendarEventWebhookSyncJobData } from 'src/modules/connected-account-sync-webhooks/calendar-event-webhook-sync/types/calendar-event-webhook-sync-job-data.type';@Processor({queueName: MessageQueue.connectedAccountSyncWebhookQueue,diff --git a/packages/twenty-server/src/modules/connected-account-sync-webhooks/calendar-event-webhook-sync/services/calendar-event-webhook-sync.service.ts b/packages/twenty-server/src/modules/connected-account-sync-webhooks/calendar-event-webhook-sync/services/calendar-event-webhook-sync.service.tsindex 0dd806e88b9..8ef52f44fcb 100644--- a/packages/twenty-server/src/modules/connected-account-sync-webhooks/calendar-event-webhook-sync/services/calendar-event-webhook-sync.service.ts+++ b/packages/twenty-server/src/modules/connected-account-sync-webhooks/calendar-event-webhook-sync/services/calendar-event-webhook-sync.service.ts@@ -11,10 +11,6 @@ import { CacheStorageNamespace } from 'src/engine/core-modules/cache-storage/typimport { CalendarChannelEntity } from 'src/engine/metadata-modules/calendar-channel/entities/calendar-channel.entity';import { CalendarEventsImportService } from 'src/modules/calendar/calendar-event-import-manager/services/calendar-events-import.service';import { CalendarFetchEventsService } from 'src/modules/calendar/calendar-event-import-manager/services/calendar-fetch-events.service';-import {- CalendarEventWebhookSyncException,- CalendarEventWebhookSyncExceptionCode,-} from 'src/modules/connected-account-sync-webhooks/calendar-event-webhook-sync/calendar-event-webhook-sync.exception';import { CALENDAR_EVENT_WEBHOOK_SYNC_INLINE_IMPORT_MAX_EVENTS } from 'src/modules/connected-account-sync-webhooks/calendar-event-webhook-sync/constants/calendar-event-webhook-sync-inline-import-max-events.constant';@Injectable()@@ -44,10 +40,11 @@ export class CalendarEventWebhookSyncService {});if (!isCalendarChannelScheduled) {- throw new CalendarEventWebhookSyncException(- `Calendar channel ${calendarChannelId} is not available for a webhook sync`,- CalendarEventWebhookSyncExceptionCode.CALENDAR_CHANNEL_SYNC_ALREADY_IN_PROGRESS,+ this.logger.log(+ `Skipping webhook sync for calendar channel ${calendarChannelId}, a sync is already in progress`,);++ return;}const calendarChannel = await this.findSyncEnabledCalendarChannel({diff --git a/packages/twenty-server/src/modules/connected-account-sync-webhooks/calendar-event-webhook-sync/types/calendar-event-webhook-sync-job-data.type.ts b/packages/twenty-server/src/modules/connected-account-sync-webhooks/calendar-event-webhook-sync/types/calendar-event-webhook-sync-job-data.type.tsnew file mode 100644index 00000000000..8033e966f13--- /dev/null+++ b/packages/twenty-server/src/modules/connected-account-sync-webhooks/calendar-event-webhook-sync/types/calendar-event-webhook-sync-job-data.type.ts@@ -0,0 +1,4 @@+export type CalendarEventWebhookSyncJobData = {+ calendarChannelId: string;+ workspaceId: string;+};diff --git a/packages/twenty-server/src/modules/connected-account/webhook-subscription-manager/services/webhook-sync-trigger.service.ts b/packages/twenty-server/src/modules/connected-account/webhook-subscription-manager/services/webhook-sync-trigger.service.tsindex 964c87cbf0f..08507a593c6 100644--- a/packages/twenty-server/src/modules/connected-account/webhook-subscription-manager/services/webhook-sync-trigger.service.ts+++ b/packages/twenty-server/src/modules/connected-account/webhook-subscription-manager/services/webhook-sync-trigger.service.ts@@ -4,15 +4,18 @@ import { InjectRepository } from '@nestjs/typeorm';import { MessageChannelSyncStage } from 'twenty-shared/types';import { Repository } from 'typeorm';+import { InjectCacheStorage } from 'src/engine/core-modules/cache-storage/decorators/cache-storage.decorator';+import { CacheStorageService } from 'src/engine/core-modules/cache-storage/services/cache-storage.service';+import { CacheStorageNamespace } from 'src/engine/core-modules/cache-storage/types/cache-storage-namespace.enum';import { InjectMessageQueue } from 'src/engine/core-modules/message-queue/decorators/message-queue.decorator';import { MessageQueue } from 'src/engine/core-modules/message-queue/message-queue.constants';import { MessageQueueService } from 'src/engine/core-modules/message-queue/services/message-queue.service';import { MessageChannelEntity } from 'src/engine/metadata-modules/message-channel/entities/message-channel.entity';-import {- CalendarEventWebhookSyncJob,- type CalendarEventWebhookSyncJobData,-} from 'src/modules/connected-account-sync-webhooks/calendar-event-webhook-sync/jobs/calendar-event-webhook-sync.job';+import { CalendarEventWebhookSyncJob } from 'src/modules/connected-account-sync-webhooks/calendar-event-webhook-sync/jobs/calendar-event-webhook-sync.job';+import { type CalendarEventWebhookSyncJobData } from 'src/modules/connected-account-sync-webhooks/calendar-event-webhook-sync/types/calendar-event-webhook-sync-job-data.type';+import { CALENDAR_EVENT_WEBHOOK_SYNC_DEBOUNCE_MS } from 'src/modules/connected-account-sync-webhooks/calendar-event-webhook-sync/constants/calendar-event-webhook-sync-debounce-ms.constant';import { CALENDAR_EVENT_WEBHOOK_SYNC_RETRY_INITIAL_DELAY_MS } from 'src/modules/connected-account-sync-webhooks/calendar-event-webhook-sync/constants/calendar-event-webhook-sync-retry-initial-delay-ms.constant';+import { CALENDAR_EVENT_WEBHOOK_SYNC_RETRY_JITTER } from 'src/modules/connected-account-sync-webhooks/calendar-event-webhook-sync/constants/calendar-event-webhook-sync-retry-jitter.constant';import { CALENDAR_EVENT_WEBHOOK_SYNC_RETRY_LIMIT } from 'src/modules/connected-account-sync-webhooks/calendar-event-webhook-sync/constants/calendar-event-webhook-sync-retry-limit.constant';import {MessagingMessageListFetchJob,@@ -26,6 +29,8 @@ export class WebhookSyncTriggerService {private readonly messagingQueueService: MessageQueueService,@InjectMessageQueue(MessageQueue.connectedAccountSyncWebhookQueue)private readonly connectedAccountSyncWebhookQueueService: MessageQueueService,+ @InjectCacheStorage(CacheStorageNamespace.ModuleCalendar)+ private readonly cacheStorage: CacheStorageService,@InjectRepository(MessageChannelEntity)private readonly messageChannelRepository: Repository<MessageChannelEntity>,) {}@@ -80,18 +85,37 @@ export class WebhookSyncTriggerService {calendarChannelId: string,workspaceId: string,): Promise<void> {- await this.connectedAccountSyncWebhookQueueService.add<CalendarEventWebhookSyncJobData>(- CalendarEventWebhookSyncJob.name,- { workspaceId, calendarChannelId },- {- id: `${CalendarEventWebhookSyncJob.name}-${calendarChannelId}`,- retryLimit: CALENDAR_EVENT_WEBHOOK_SYNC_RETRY_LIMIT,- backoff: {- strategy: 'exponential',- initialDelayMilliseconds:- CALENDAR_EVENT_WEBHOOK_SYNC_RETRY_INITIAL_DELAY_MS,- },- },+ const debounceCacheKey = `calendar-event-webhook-sync-debounce:${workspaceId}:${calendarChannelId}`;++ const hasOpenedDebounceWindow = await this.cacheStorage.setIfAbsent(+ debounceCacheKey,+ true,+ CALENDAR_EVENT_WEBHOOK_SYNC_DEBOUNCE_MS,);++ if (!hasOpenedDebounceWindow) {+ return;+ }++ try {+ await this.connectedAccountSyncWebhookQueueService.add<CalendarEventWebhookSyncJobData>(+ CalendarEventWebhookSyncJob.name,+ { workspaceId, calendarChannelId },+ {+ delay: CALENDAR_EVENT_WEBHOOK_SYNC_DEBOUNCE_MS,+ retryLimit: CALENDAR_EVENT_WEBHOOK_SYNC_RETRY_LIMIT,+ backoff: {+ strategy: 'exponential',+ initialDelayMilliseconds:+ CALENDAR_EVENT_WEBHOOK_SYNC_RETRY_INITIAL_DELAY_MS,+ jitter: CALENDAR_EVENT_WEBHOOK_SYNC_RETRY_JITTER,+ },+ },+ );+ } catch (error) {+ await this.cacheStorage.del(debounceCacheKey);++ throw error;+ }}}diff --git a/packages/twenty-server/test/integration/utils/wait-for-all-jobs-to-finish.util.ts b/packages/twenty-server/test/integration/utils/wait-for-all-jobs-to-finish.util.tsindex e53f07c5934..328e5c64ced 100644--- a/packages/twenty-server/test/integration/utils/wait-for-all-jobs-to-finish.util.ts+++ b/packages/twenty-server/test/integration/utils/wait-for-all-jobs-to-finish.util.ts@@ -6,6 +6,8 @@ import { MessageQueue } from 'src/engine/core-modules/message-queue/message-queuconst POLL_INTERVAL_MS = 25;const REQUIRED_CONSECUTIVE_QUIET_CHECKS = 2;const STALL_TIMEOUT_MS = 15_000;+const DELAYED_JOB_PAGE_SIZE = 50;+const MAX_PROMOTABLE_DELAY_MS = 60_000;const HARD_TIMEOUT_MS = 120_000;const PENDING_JOB_STATES = ['waiting',@@ -15,6 +17,11 @@ const PENDING_JOB_STATES = ['delayed',] as const;+type PendingJobCounts = {+ pending: number;+ delayed: number;+};+let redisConnection: IORedis | null = null;let queues: Queue[] | null = null;@@ -35,7 +42,7 @@ const getQueues = (): Queue[] => {};const getPendingJobCountsByQueue = async (): Promise<- Record<string, number>+ Record<string, PendingJobCounts>> => {const countsByQueue = await Promise.all(getQueues().map(async (queue) => {@@ -45,13 +52,44 @@ const getPendingJobCountsByQueue = async (): Promise<0,);- return [queue.name, pendingCount] as const;+ return [+ queue.name,+ { pending: pendingCount, delayed: jobCounts.delayed ?? 0 },+ ] as const;}),);return Object.fromEntries(- countsByQueue.filter(([, pendingCount]) => pendingCount > 0),+ countsByQueue.filter(([, { pending }]) => pending > 0),+ );+};++const promoteDelayedJobs = async (queueName: string): Promise<number> => {+ const queue = getQueues().find(({ name }) => name === queueName);++ if (!queue) {+ return 0;+ }++ const delayedJobs = await queue.getDelayed(0, DELAYED_JOB_PAGE_SIZE);+ const promotableJobs = delayedJobs.filter(+ (job) => (job.delay ?? 0) <= MAX_PROMOTABLE_DELAY_MS,);++ let promotedJobCount = 0;++ for (const promotableJob of promotableJobs) {+ const hasBeenPromoted = await promotableJob+ .promote()+ .then(() => true)+ .catch(() => false);++ if (hasBeenPromoted) {+ promotedJobCount += 1;+ }+ }++ return promotedJobCount;};const getActiveJobsFingerprint = async (@@ -79,9 +117,12 @@ export const waitForAllJobsToFinish = async (): Promise<void> => {while (consecutiveQuietChecks < REQUIRED_CONSECUTIVE_QUIET_CHECKS) {const pendingJobCountsByQueue = await getPendingJobCountsByQueue();const pendingTotal = Object.values(pendingJobCountsByQueue).reduce(- (sum, count) => sum + count,+ (sum, { pending }) => sum + pending,0,);+ const stalledQueueNames = Object.entries(pendingJobCountsByQueue)+ .filter(([, { pending, delayed }]) => pending === delayed)+ .map(([queueName]) => queueName);if (pendingTotal === 0) {consecutiveQuietChecks += 1;@@ -89,6 +130,24 @@ export const waitForAllJobsToFinish = async (): Promise<void> => {consecutiveQuietChecks = 0;const now = Date.now();+ if (now - startedAt > HARD_TIMEOUT_MS) {+ throw new Error(+ `Message queues still busy after ${HARD_TIMEOUT_MS}ms, pending jobs: ${JSON.stringify(pendingJobCountsByQueue)}`,+ );+ }++ let promotedJobCount = 0;++ for (const stalledQueueName of stalledQueueNames) {+ promotedJobCount += await promoteDelayedJobs(stalledQueueName);+ }++ if (promotedJobCount > 0) {+ lastProgressAt = now;+ await new Promise((resolve) => setTimeout(resolve, POLL_INTERVAL_MS));+ continue;+ }+const activeJobsFingerprint = await getActiveJobsFingerprint(Object.keys(pendingJobCountsByQueue),);@@ -104,12 +163,6 @@ export const waitForAllJobsToFinish = async (): Promise<void> => {`Message queues stalled, no progress for ${STALL_TIMEOUT_MS}ms, pending jobs: ${JSON.stringify(pendingJobCountsByQueue)}`,);}-- if (now - startedAt > HARD_TIMEOUT_MS) {- throw new Error(- `Message queues still busy after ${HARD_TIMEOUT_MS}ms, pending jobs: ${JSON.stringify(pendingJobCountsByQueue)}`,- );- }}if (consecutiveQuietChecks < REQUIRED_CONSECUTIVE_QUIET_CHECKS) {