Skip to content

The open alternative to Salesforce, designed for AI.

Debounce calendar webhook syncs instead of throwing when one is running (#25656)

neo77340e95c5cfe19
diff --git a/packages/twenty-server/src/modules/connected-account-sync-webhooks/calendar-event-webhook-sync/calendar-event-webhook-sync.exception.ts b/packages/twenty-server/src/modules/connected-account-sync-webhooks/calendar-event-webhook-sync/calendar-event-webhook-sync.exception.ts
deleted file mode 100644
index eeba2f3b632..00000000000
--- a/packages/twenty-server/src/modules/connected-account-sync-webhooks/calendar-event-webhook-sync/calendar-event-webhook-sync.exception.ts
+++ /dev/null
@@ -1,34 +0,0 @@
-import { type MessageDescriptor } from '@lingui/core';
-import { msg } from '@lingui/core/macro';
-import { assertUnreachable } from 'twenty-shared/utils';
-
-import { CustomException } from 'src/utils/custom-exception';
-
-export enum CalendarEventWebhookSyncExceptionCode {
- CALENDAR_CHANNEL_SYNC_ALREADY_IN_PROGRESS = 'CALENDAR_CHANNEL_SYNC_ALREADY_IN_PROGRESS',
-}
-
-const getCalendarEventWebhookSyncExceptionUserFriendlyMessage = (
- code: CalendarEventWebhookSyncExceptionCode,
-) => {
- switch (code) {
- case CalendarEventWebhookSyncExceptionCode.CALENDAR_CHANNEL_SYNC_ALREADY_IN_PROGRESS:
- return msg`The calendar is already syncing.`;
- default:
- assertUnreachable(code);
- }
-};
-
-export class CalendarEventWebhookSyncException extends CustomException<CalendarEventWebhookSyncExceptionCode> {
- constructor(
- message: string,
- code: CalendarEventWebhookSyncExceptionCode,
- { userFriendlyMessage }: { userFriendlyMessage?: MessageDescriptor } = {},
- ) {
- super(message, code, {
- userFriendlyMessage:
- userFriendlyMessage ??
- getCalendarEventWebhookSyncExceptionUserFriendlyMessage(code),
- });
- }
-}
diff --git a/packages/twenty-server/src/modules/connected-account-sync-webhooks/calendar-event-webhook-sync/constants/calendar-event-webhook-sync-debounce-ms.constant.ts b/packages/twenty-server/src/modules/connected-account-sync-webhooks/calendar-event-webhook-sync/constants/calendar-event-webhook-sync-debounce-ms.constant.ts
new file mode 100644
index 00000000000..7fc96b1652c
--- /dev/null
+++ b/packages/twenty-server/src/modules/connected-account-sync-webhooks/calendar-event-webhook-sync/constants/calendar-event-webhook-sync-debounce-ms.constant.ts
@@ -0,0 +1 @@
+export const CALENDAR_EVENT_WEBHOOK_SYNC_DEBOUNCE_MS = 30000;
diff --git a/packages/twenty-server/src/modules/connected-account-sync-webhooks/calendar-event-webhook-sync/constants/calendar-event-webhook-sync-retry-jitter.constant.ts b/packages/twenty-server/src/modules/connected-account-sync-webhooks/calendar-event-webhook-sync/constants/calendar-event-webhook-sync-retry-jitter.constant.ts
new file mode 100644
index 00000000000..ca28c4d2247
--- /dev/null
+++ b/packages/twenty-server/src/modules/connected-account-sync-webhooks/calendar-event-webhook-sync/constants/calendar-event-webhook-sync-retry-jitter.constant.ts
@@ -0,0 +1 @@
+export const CALENDAR_EVENT_WEBHOOK_SYNC_RETRY_JITTER = 0.5;
diff --git a/packages/twenty-server/src/modules/connected-account-sync-webhooks/calendar-event-webhook-sync/jobs/calendar-event-webhook-sync.job.ts b/packages/twenty-server/src/modules/connected-account-sync-webhooks/calendar-event-webhook-sync/jobs/calendar-event-webhook-sync.job.ts
index da86d4b33b3..6c034652bed 100644
--- a/packages/twenty-server/src/modules/connected-account-sync-webhooks/calendar-event-webhook-sync/jobs/calendar-event-webhook-sync.job.ts
+++ b/packages/twenty-server/src/modules/connected-account-sync-webhooks/calendar-event-webhook-sync/jobs/calendar-event-webhook-sync.job.ts
@@ -4,11 +4,7 @@ import { Process } from 'src/engine/core-modules/message-queue/decorators/proces
import { Processor } from 'src/engine/core-modules/message-queue/decorators/processor.decorator';
import { MessageQueue } from 'src/engine/core-modules/message-queue/message-queue.constants';
import { CalendarEventWebhookSyncService } from 'src/modules/connected-account-sync-webhooks/calendar-event-webhook-sync/services/calendar-event-webhook-sync.service';
-
-export type CalendarEventWebhookSyncJobData = {
- calendarChannelId: string;
- workspaceId: string;
-};
+import { type CalendarEventWebhookSyncJobData } from 'src/modules/connected-account-sync-webhooks/calendar-event-webhook-sync/types/calendar-event-webhook-sync-job-data.type';
@Processor({
queueName: MessageQueue.connectedAccountSyncWebhookQueue,
diff --git a/packages/twenty-server/src/modules/connected-account-sync-webhooks/calendar-event-webhook-sync/services/calendar-event-webhook-sync.service.ts b/packages/twenty-server/src/modules/connected-account-sync-webhooks/calendar-event-webhook-sync/services/calendar-event-webhook-sync.service.ts
index 0dd806e88b9..8ef52f44fcb 100644
--- a/packages/twenty-server/src/modules/connected-account-sync-webhooks/calendar-event-webhook-sync/services/calendar-event-webhook-sync.service.ts
+++ b/packages/twenty-server/src/modules/connected-account-sync-webhooks/calendar-event-webhook-sync/services/calendar-event-webhook-sync.service.ts
@@ -11,10 +11,6 @@ import { CacheStorageNamespace } from 'src/engine/core-modules/cache-storage/typ
import { CalendarChannelEntity } from 'src/engine/metadata-modules/calendar-channel/entities/calendar-channel.entity';
import { CalendarEventsImportService } from 'src/modules/calendar/calendar-event-import-manager/services/calendar-events-import.service';
import { CalendarFetchEventsService } from 'src/modules/calendar/calendar-event-import-manager/services/calendar-fetch-events.service';
-import {
- CalendarEventWebhookSyncException,
- CalendarEventWebhookSyncExceptionCode,
-} from 'src/modules/connected-account-sync-webhooks/calendar-event-webhook-sync/calendar-event-webhook-sync.exception';
import { CALENDAR_EVENT_WEBHOOK_SYNC_INLINE_IMPORT_MAX_EVENTS } from 'src/modules/connected-account-sync-webhooks/calendar-event-webhook-sync/constants/calendar-event-webhook-sync-inline-import-max-events.constant';
@Injectable()
@@ -44,10 +40,11 @@ export class CalendarEventWebhookSyncService {
});
if (!isCalendarChannelScheduled) {
- throw new CalendarEventWebhookSyncException(
- `Calendar channel ${calendarChannelId} is not available for a webhook sync`,
- CalendarEventWebhookSyncExceptionCode.CALENDAR_CHANNEL_SYNC_ALREADY_IN_PROGRESS,
+ this.logger.log(
+ `Skipping webhook sync for calendar channel ${calendarChannelId}, a sync is already in progress`,
);
+
+ return;
}
const calendarChannel = await this.findSyncEnabledCalendarChannel({
diff --git a/packages/twenty-server/src/modules/connected-account-sync-webhooks/calendar-event-webhook-sync/types/calendar-event-webhook-sync-job-data.type.ts b/packages/twenty-server/src/modules/connected-account-sync-webhooks/calendar-event-webhook-sync/types/calendar-event-webhook-sync-job-data.type.ts
new file mode 100644
index 00000000000..8033e966f13
--- /dev/null
+++ b/packages/twenty-server/src/modules/connected-account-sync-webhooks/calendar-event-webhook-sync/types/calendar-event-webhook-sync-job-data.type.ts
@@ -0,0 +1,4 @@
+export type CalendarEventWebhookSyncJobData = {
+ calendarChannelId: string;
+ workspaceId: string;
+};
diff --git a/packages/twenty-server/src/modules/connected-account/webhook-subscription-manager/services/webhook-sync-trigger.service.ts b/packages/twenty-server/src/modules/connected-account/webhook-subscription-manager/services/webhook-sync-trigger.service.ts
index 964c87cbf0f..08507a593c6 100644
--- a/packages/twenty-server/src/modules/connected-account/webhook-subscription-manager/services/webhook-sync-trigger.service.ts
+++ b/packages/twenty-server/src/modules/connected-account/webhook-subscription-manager/services/webhook-sync-trigger.service.ts
@@ -4,15 +4,18 @@ import { InjectRepository } from '@nestjs/typeorm';
import { MessageChannelSyncStage } from 'twenty-shared/types';
import { Repository } from 'typeorm';
+import { InjectCacheStorage } from 'src/engine/core-modules/cache-storage/decorators/cache-storage.decorator';
+import { CacheStorageService } from 'src/engine/core-modules/cache-storage/services/cache-storage.service';
+import { CacheStorageNamespace } from 'src/engine/core-modules/cache-storage/types/cache-storage-namespace.enum';
import { InjectMessageQueue } from 'src/engine/core-modules/message-queue/decorators/message-queue.decorator';
import { MessageQueue } from 'src/engine/core-modules/message-queue/message-queue.constants';
import { MessageQueueService } from 'src/engine/core-modules/message-queue/services/message-queue.service';
import { MessageChannelEntity } from 'src/engine/metadata-modules/message-channel/entities/message-channel.entity';
-import {
- CalendarEventWebhookSyncJob,
- type CalendarEventWebhookSyncJobData,
-} from 'src/modules/connected-account-sync-webhooks/calendar-event-webhook-sync/jobs/calendar-event-webhook-sync.job';
+import { CalendarEventWebhookSyncJob } from 'src/modules/connected-account-sync-webhooks/calendar-event-webhook-sync/jobs/calendar-event-webhook-sync.job';
+import { type CalendarEventWebhookSyncJobData } from 'src/modules/connected-account-sync-webhooks/calendar-event-webhook-sync/types/calendar-event-webhook-sync-job-data.type';
+import { CALENDAR_EVENT_WEBHOOK_SYNC_DEBOUNCE_MS } from 'src/modules/connected-account-sync-webhooks/calendar-event-webhook-sync/constants/calendar-event-webhook-sync-debounce-ms.constant';
import { CALENDAR_EVENT_WEBHOOK_SYNC_RETRY_INITIAL_DELAY_MS } from 'src/modules/connected-account-sync-webhooks/calendar-event-webhook-sync/constants/calendar-event-webhook-sync-retry-initial-delay-ms.constant';
+import { CALENDAR_EVENT_WEBHOOK_SYNC_RETRY_JITTER } from 'src/modules/connected-account-sync-webhooks/calendar-event-webhook-sync/constants/calendar-event-webhook-sync-retry-jitter.constant';
import { CALENDAR_EVENT_WEBHOOK_SYNC_RETRY_LIMIT } from 'src/modules/connected-account-sync-webhooks/calendar-event-webhook-sync/constants/calendar-event-webhook-sync-retry-limit.constant';
import {
MessagingMessageListFetchJob,
@@ -26,6 +29,8 @@ export class WebhookSyncTriggerService {
private readonly messagingQueueService: MessageQueueService,
@InjectMessageQueue(MessageQueue.connectedAccountSyncWebhookQueue)
private readonly connectedAccountSyncWebhookQueueService: MessageQueueService,
+ @InjectCacheStorage(CacheStorageNamespace.ModuleCalendar)
+ private readonly cacheStorage: CacheStorageService,
@InjectRepository(MessageChannelEntity)
private readonly messageChannelRepository: Repository<MessageChannelEntity>,
) {}
@@ -80,18 +85,37 @@ export class WebhookSyncTriggerService {
calendarChannelId: string,
workspaceId: string,
): Promise<void> {
- await this.connectedAccountSyncWebhookQueueService.add<CalendarEventWebhookSyncJobData>(
- CalendarEventWebhookSyncJob.name,
- { workspaceId, calendarChannelId },
- {
- id: `${CalendarEventWebhookSyncJob.name}-${calendarChannelId}`,
- retryLimit: CALENDAR_EVENT_WEBHOOK_SYNC_RETRY_LIMIT,
- backoff: {
- strategy: 'exponential',
- initialDelayMilliseconds:
- CALENDAR_EVENT_WEBHOOK_SYNC_RETRY_INITIAL_DELAY_MS,
- },
- },
+ const debounceCacheKey = `calendar-event-webhook-sync-debounce:${workspaceId}:${calendarChannelId}`;
+
+ const hasOpenedDebounceWindow = await this.cacheStorage.setIfAbsent(
+ debounceCacheKey,
+ true,
+ CALENDAR_EVENT_WEBHOOK_SYNC_DEBOUNCE_MS,
);
+
+ if (!hasOpenedDebounceWindow) {
+ return;
+ }
+
+ try {
+ await this.connectedAccountSyncWebhookQueueService.add<CalendarEventWebhookSyncJobData>(
+ CalendarEventWebhookSyncJob.name,
+ { workspaceId, calendarChannelId },
+ {
+ delay: CALENDAR_EVENT_WEBHOOK_SYNC_DEBOUNCE_MS,
+ retryLimit: CALENDAR_EVENT_WEBHOOK_SYNC_RETRY_LIMIT,
+ backoff: {
+ strategy: 'exponential',
+ initialDelayMilliseconds:
+ CALENDAR_EVENT_WEBHOOK_SYNC_RETRY_INITIAL_DELAY_MS,
+ jitter: CALENDAR_EVENT_WEBHOOK_SYNC_RETRY_JITTER,
+ },
+ },
+ );
+ } catch (error) {
+ await this.cacheStorage.del(debounceCacheKey);
+
+ throw error;
+ }
}
}
diff --git a/packages/twenty-server/test/integration/utils/wait-for-all-jobs-to-finish.util.ts b/packages/twenty-server/test/integration/utils/wait-for-all-jobs-to-finish.util.ts
index e53f07c5934..328e5c64ced 100644
--- a/packages/twenty-server/test/integration/utils/wait-for-all-jobs-to-finish.util.ts
+++ b/packages/twenty-server/test/integration/utils/wait-for-all-jobs-to-finish.util.ts
@@ -6,6 +6,8 @@ import { MessageQueue } from 'src/engine/core-modules/message-queue/message-queu
const POLL_INTERVAL_MS = 25;
const REQUIRED_CONSECUTIVE_QUIET_CHECKS = 2;
const STALL_TIMEOUT_MS = 15_000;
+const DELAYED_JOB_PAGE_SIZE = 50;
+const MAX_PROMOTABLE_DELAY_MS = 60_000;
const HARD_TIMEOUT_MS = 120_000;
const PENDING_JOB_STATES = [
'waiting',
@@ -15,6 +17,11 @@ const PENDING_JOB_STATES = [
'delayed',
] as const;
+type PendingJobCounts = {
+ pending: number;
+ delayed: number;
+};
+
let redisConnection: IORedis | null = null;
let queues: Queue[] | null = null;
@@ -35,7 +42,7 @@ const getQueues = (): Queue[] => {
};
const getPendingJobCountsByQueue = async (): Promise<
- Record<string, number>
+ Record<string, PendingJobCounts>
> => {
const countsByQueue = await Promise.all(
getQueues().map(async (queue) => {
@@ -45,13 +52,44 @@ const getPendingJobCountsByQueue = async (): Promise<
0,
);
- return [queue.name, pendingCount] as const;
+ return [
+ queue.name,
+ { pending: pendingCount, delayed: jobCounts.delayed ?? 0 },
+ ] as const;
}),
);
return Object.fromEntries(
- countsByQueue.filter(([, pendingCount]) => pendingCount > 0),
+ countsByQueue.filter(([, { pending }]) => pending > 0),
+ );
+};
+
+const promoteDelayedJobs = async (queueName: string): Promise<number> => {
+ const queue = getQueues().find(({ name }) => name === queueName);
+
+ if (!queue) {
+ return 0;
+ }
+
+ const delayedJobs = await queue.getDelayed(0, DELAYED_JOB_PAGE_SIZE);
+ const promotableJobs = delayedJobs.filter(
+ (job) => (job.delay ?? 0) <= MAX_PROMOTABLE_DELAY_MS,
);
+
+ let promotedJobCount = 0;
+
+ for (const promotableJob of promotableJobs) {
+ const hasBeenPromoted = await promotableJob
+ .promote()
+ .then(() => true)
+ .catch(() => false);
+
+ if (hasBeenPromoted) {
+ promotedJobCount += 1;
+ }
+ }
+
+ return promotedJobCount;
};
const getActiveJobsFingerprint = async (
@@ -79,9 +117,12 @@ export const waitForAllJobsToFinish = async (): Promise<void> => {
while (consecutiveQuietChecks < REQUIRED_CONSECUTIVE_QUIET_CHECKS) {
const pendingJobCountsByQueue = await getPendingJobCountsByQueue();
const pendingTotal = Object.values(pendingJobCountsByQueue).reduce(
- (sum, count) => sum + count,
+ (sum, { pending }) => sum + pending,
0,
);
+ const stalledQueueNames = Object.entries(pendingJobCountsByQueue)
+ .filter(([, { pending, delayed }]) => pending === delayed)
+ .map(([queueName]) => queueName);
if (pendingTotal === 0) {
consecutiveQuietChecks += 1;
@@ -89,6 +130,24 @@ export const waitForAllJobsToFinish = async (): Promise<void> => {
consecutiveQuietChecks = 0;
const now = Date.now();
+ if (now - startedAt > HARD_TIMEOUT_MS) {
+ throw new Error(
+ `Message queues still busy after ${HARD_TIMEOUT_MS}ms, pending jobs: ${JSON.stringify(pendingJobCountsByQueue)}`,
+ );
+ }
+
+ let promotedJobCount = 0;
+
+ for (const stalledQueueName of stalledQueueNames) {
+ promotedJobCount += await promoteDelayedJobs(stalledQueueName);
+ }
+
+ if (promotedJobCount > 0) {
+ lastProgressAt = now;
+ await new Promise((resolve) => setTimeout(resolve, POLL_INTERVAL_MS));
+ continue;
+ }
+
const activeJobsFingerprint = await getActiveJobsFingerprint(
Object.keys(pendingJobCountsByQueue),
);
@@ -104,12 +163,6 @@ export const waitForAllJobsToFinish = async (): Promise<void> => {
`Message queues stalled, no progress for ${STALL_TIMEOUT_MS}ms, pending jobs: ${JSON.stringify(pendingJobCountsByQueue)}`,
);
}
-
- if (now - startedAt > HARD_TIMEOUT_MS) {
- throw new Error(
- `Message queues still busy after ${HARD_TIMEOUT_MS}ms, pending jobs: ${JSON.stringify(pendingJobCountsByQueue)}`,
- );
- }
}
if (consecutiveQuietChecks < REQUIRED_CONSECUTIVE_QUIET_CHECKS) {