Skip to content

The open alternative to Salesforce, designed for AI.

Stagger daily enterprise license refresh by server ID (#29358)

Félix Malfaitb5b4c8e500b9
diff --git a/packages/twenty-server/src/engine/core-modules/cron/sentry-cron-monitor.decorator.ts b/packages/twenty-server/src/engine/core-modules/cron/sentry-cron-monitor.decorator.ts
index 465b809dc16..7e307e74cc4 100644
--- a/packages/twenty-server/src/engine/core-modules/cron/sentry-cron-monitor.decorator.ts
+++ b/packages/twenty-server/src/engine/core-modules/cron/sentry-cron-monitor.decorator.ts
@@ -2,9 +2,12 @@ import * as Sentry from '@sentry/node';
const DEFAULT_MAX_RUNTIME_IN_MINUTES = 5;
+// A schedule function receives the job's arguments, for schedules chosen at
+// registration time.
export function SentryCronMonitor(
monitorSlug: string,
- schedule: string,
+ // oxlint-disable-next-line typescript/no-explicit-any
+ schedule: string | ((...args: any[]) => string),
{
maxRuntimeInMinutes = DEFAULT_MAX_RUNTIME_IN_MINUTES,
}: { maxRuntimeInMinutes?: number } = {},
@@ -34,7 +37,8 @@ export function SentryCronMonitor(
{
schedule: {
type: 'crontab',
- value: schedule,
+ value:
+ typeof schedule === 'function' ? schedule(...args) : schedule,
},
checkinMargin: 1,
maxRuntime: maxRuntimeInMinutes,
diff --git a/packages/twenty-server/src/engine/core-modules/enterprise/cron/__tests__/organization-key-validation-schedule.spec.ts b/packages/twenty-server/src/engine/core-modules/enterprise/cron/__tests__/organization-key-validation-schedule.spec.ts
new file mode 100644
index 00000000000..7bfa95d6cef
--- /dev/null
+++ b/packages/twenty-server/src/engine/core-modules/enterprise/cron/__tests__/organization-key-validation-schedule.spec.ts
@@ -0,0 +1,129 @@
+import { Test } from '@nestjs/testing';
+import * as Sentry from '@sentry/node';
+
+import { EnterpriseKeyValidationCronCommand } from 'src/engine/core-modules/enterprise/cron/command/organization-key-validation.cron.command';
+import { EnterpriseKeyValidationCronJob } from 'src/engine/core-modules/enterprise/cron/jobs/organization-key-validation.cron.job';
+import { EnterprisePlanService } from 'src/engine/core-modules/enterprise/services/enterprise-plan.service';
+import { getOrganizationKeyValidationCronPattern } from 'src/engine/core-modules/enterprise/utils/get-organization-key-validation-cron-pattern.util';
+import { toMinuteLevelCronPattern } from 'src/engine/core-modules/enterprise/utils/to-minute-level-cron-pattern.util';
+import { MessageQueue } from 'src/engine/core-modules/message-queue/message-queue.constants';
+import { getQueueToken } from 'src/engine/core-modules/message-queue/utils/get-queue-token.util';
+
+jest.mock('@sentry/node', () => ({
+ isInitialized: jest.fn(() => true),
+ captureCheckIn: jest.fn(() => 'check-in-id'),
+}));
+
+jest.mock(
+ 'src/engine/core-modules/enterprise/services/enterprise-plan.service',
+ () => ({ EnterprisePlanService: class EnterprisePlanService {} }),
+);
+
+jest.mock(
+ 'src/engine/core-modules/message-queue/services/message-queue.service',
+ () => ({ MessageQueueService: class MessageQueueService {} }),
+);
+
+describe('organization key validation schedule', () => {
+ const enterprisePlanService = {
+ getOrCreateServerId: jest.fn(),
+ refreshValidityToken: jest.fn(),
+ getBillableSeatCount: jest.fn(),
+ reportSeats: jest.fn(),
+ };
+ const messageQueueService = { addCron: jest.fn() };
+ let command: EnterpriseKeyValidationCronCommand;
+ let job: EnterpriseKeyValidationCronJob;
+
+ beforeEach(async () => {
+ jest.clearAllMocks();
+ enterprisePlanService.getOrCreateServerId.mockResolvedValue(
+ 'server-example',
+ );
+ enterprisePlanService.refreshValidityToken.mockResolvedValue(true);
+ enterprisePlanService.getBillableSeatCount.mockResolvedValue(2);
+ enterprisePlanService.reportSeats.mockResolvedValue(true);
+
+ const module = await Test.createTestingModule({
+ providers: [
+ EnterpriseKeyValidationCronCommand,
+ EnterpriseKeyValidationCronJob,
+ { provide: EnterprisePlanService, useValue: enterprisePlanService },
+ {
+ provide: getQueueToken(MessageQueue.cronQueue),
+ useValue: messageQueueService,
+ },
+ ],
+ }).compile();
+
+ command = module.get(EnterpriseKeyValidationCronCommand);
+ job = module.get(EnterpriseKeyValidationCronJob);
+ });
+
+ const registeredJobData = () =>
+ messageQueueService.addCron.mock.lastCall?.[0].data;
+
+ const expectSentrySchedule = (value: string) =>
+ expect(Sentry.captureCheckIn).toHaveBeenCalledWith(
+ {
+ monitorSlug: EnterpriseKeyValidationCronJob.name,
+ status: 'in_progress',
+ },
+ expect.objectContaining({
+ schedule: { type: 'crontab', value },
+ timezone: 'UTC',
+ }),
+ );
+
+ it('registers a stable time and hands it to the job for Sentry', async () => {
+ await command.run();
+ await command.run();
+
+ const pattern = getOrganizationKeyValidationCronPattern('server-example');
+
+ for (const registration of messageQueueService.addCron.mock.calls) {
+ expect(registration[0]).toEqual({
+ jobName: EnterpriseKeyValidationCronJob.name,
+ data: { cronPattern: pattern },
+ options: { repeat: { pattern, tz: 'UTC' } },
+ });
+ }
+
+ await job.handle(registeredJobData());
+
+ expectSentrySchedule(toMinuteLevelCronPattern(pattern));
+ expect(enterprisePlanService.refreshValidityToken).toHaveBeenCalledTimes(1);
+ expect(enterprisePlanService.reportSeats).toHaveBeenCalledWith(2);
+ });
+
+ it('keeps Sentry on the registered fallback after the server ID becomes available', async () => {
+ enterprisePlanService.getOrCreateServerId.mockResolvedValueOnce(null);
+
+ await command.run();
+ await job.handle(registeredJobData());
+
+ expect(messageQueueService.addCron).toHaveBeenCalledWith(
+ expect.objectContaining({
+ options: { repeat: { pattern: '0 4 * * *', tz: 'UTC' } },
+ }),
+ );
+ expectSentrySchedule('0 4 * * *');
+ expect(enterprisePlanService.getOrCreateServerId).toHaveBeenCalledTimes(1);
+ });
+
+ it('assumes 04:00 for a scheduler registered by an older version', async () => {
+ await job.handle(undefined);
+
+ expectSentrySchedule('0 4 * * *');
+ });
+
+ it('still reports seats after a failed refresh', async () => {
+ enterprisePlanService.refreshValidityToken.mockRejectedValue(
+ new Error('Billing service temporarily unavailable'),
+ );
+
+ await expect(job.handle(undefined)).resolves.toBeUndefined();
+
+ expect(enterprisePlanService.reportSeats).toHaveBeenCalledWith(2);
+ });
+});
diff --git a/packages/twenty-server/src/engine/core-modules/enterprise/cron/command/organization-key-validation.cron.command.ts b/packages/twenty-server/src/engine/core-modules/enterprise/cron/command/organization-key-validation.cron.command.ts
index f8556ffc5f5..d415960ae29 100644
--- a/packages/twenty-server/src/engine/core-modules/enterprise/cron/command/organization-key-validation.cron.command.ts
+++ b/packages/twenty-server/src/engine/core-modules/enterprise/cron/command/organization-key-validation.cron.command.ts
@@ -2,8 +2,10 @@
import { Command, CommandRunner } from 'nest-commander';
-import { ENTERPRISE_KEY_VALIDATION_CRON_PATTERN } from 'src/engine/core-modules/enterprise/constants/organization-key-validation-cron-pattern.constant';
import { EnterpriseKeyValidationCronJob } from 'src/engine/core-modules/enterprise/cron/jobs/organization-key-validation.cron.job';
+import { EnterprisePlanService } from 'src/engine/core-modules/enterprise/services/enterprise-plan.service';
+import { type EnterpriseKeyValidationCronJobData } from 'src/engine/core-modules/enterprise/types/enterprise-key-validation-cron-job-data.type';
+import { getOrganizationKeyValidationCronPattern } from 'src/engine/core-modules/enterprise/utils/get-organization-key-validation-cron-pattern.util';
import { InjectMessageQueue } from 'src/engine/core-modules/message-queue/decorators/message-queue.decorator';
import { MessageQueue } from 'src/engine/core-modules/message-queue/message-queue.constants';
import { MessageQueueService } from 'src/engine/core-modules/message-queue/services/message-queue.service';
@@ -17,17 +19,22 @@ export class EnterpriseKeyValidationCronCommand extends CommandRunner {
constructor(
@InjectMessageQueue(MessageQueue.cronQueue)
private readonly messageQueueService: MessageQueueService,
+ private readonly enterprisePlanService: EnterprisePlanService,
) {
super();
}
async run(): Promise<void> {
- await this.messageQueueService.addCron<undefined>({
+ const serverId = await this.enterprisePlanService.getOrCreateServerId();
+ const cronPattern = getOrganizationKeyValidationCronPattern(serverId);
+
+ await this.messageQueueService.addCron<EnterpriseKeyValidationCronJobData>({
jobName: EnterpriseKeyValidationCronJob.name,
- data: undefined,
+ data: { cronPattern },
options: {
repeat: {
- pattern: ENTERPRISE_KEY_VALIDATION_CRON_PATTERN,
+ pattern: cronPattern,
+ tz: 'UTC',
},
},
});
diff --git a/packages/twenty-server/src/engine/core-modules/enterprise/cron/jobs/organization-key-validation.cron.job.ts b/packages/twenty-server/src/engine/core-modules/enterprise/cron/jobs/organization-key-validation.cron.job.ts
index 34cfcc708de..941f71598c6 100644
--- a/packages/twenty-server/src/engine/core-modules/enterprise/cron/jobs/organization-key-validation.cron.job.ts
+++ b/packages/twenty-server/src/engine/core-modules/enterprise/cron/jobs/organization-key-validation.cron.job.ts
@@ -5,6 +5,8 @@ import { Injectable, Logger } from '@nestjs/common';
import { SentryCronMonitor } from 'src/engine/core-modules/cron/sentry-cron-monitor.decorator';
import { ENTERPRISE_KEY_VALIDATION_CRON_PATTERN } from 'src/engine/core-modules/enterprise/constants/organization-key-validation-cron-pattern.constant';
import { EnterprisePlanService } from 'src/engine/core-modules/enterprise/services/enterprise-plan.service';
+import { type EnterpriseKeyValidationCronJobData } from 'src/engine/core-modules/enterprise/types/enterprise-key-validation-cron-job-data.type';
+import { toMinuteLevelCronPattern } from 'src/engine/core-modules/enterprise/utils/to-minute-level-cron-pattern.util';
import { Process } from 'src/engine/core-modules/message-queue/decorators/process.decorator';
import { Processor } from 'src/engine/core-modules/message-queue/decorators/processor.decorator';
import { MessageQueue } from 'src/engine/core-modules/message-queue/message-queue.constants';
@@ -17,11 +19,22 @@ export class EnterpriseKeyValidationCronJob {
constructor(private readonly enterprisePlanService: EnterprisePlanService) {}
@Process(EnterpriseKeyValidationCronJob.name)
+ // Sentry follows the pattern the queue registered, not one recomputed here.
+ // Schedulers registered by older versions carry no data and run at 04:00.
@SentryCronMonitor(
EnterpriseKeyValidationCronJob.name,
- ENTERPRISE_KEY_VALIDATION_CRON_PATTERN,
+ (data?: EnterpriseKeyValidationCronJobData) =>
+ toMinuteLevelCronPattern(
+ data?.cronPattern ?? ENTERPRISE_KEY_VALIDATION_CRON_PATTERN,
+ ),
)
- async handle(): Promise<void> {
+ async handle(
+ _data: EnterpriseKeyValidationCronJobData | undefined,
+ ): Promise<void> {
+ await this.refreshValidityAndReportSeats();
+ }
+
+ private async refreshValidityAndReportSeats(): Promise<void> {
this.logger.log(
'Starting enterprise validity token refresh and seat report...',
);
diff --git a/packages/twenty-server/src/engine/core-modules/enterprise/types/enterprise-key-validation-cron-job-data.type.ts b/packages/twenty-server/src/engine/core-modules/enterprise/types/enterprise-key-validation-cron-job-data.type.ts
new file mode 100644
index 00000000000..80bb9c5f82b
--- /dev/null
+++ b/packages/twenty-server/src/engine/core-modules/enterprise/types/enterprise-key-validation-cron-job-data.type.ts
@@ -0,0 +1,3 @@
+export type EnterpriseKeyValidationCronJobData = {
+ cronPattern: string;
+};
diff --git a/packages/twenty-server/src/engine/core-modules/enterprise/utils/__tests__/get-organization-key-validation-cron-pattern.util.spec.ts b/packages/twenty-server/src/engine/core-modules/enterprise/utils/__tests__/get-organization-key-validation-cron-pattern.util.spec.ts
new file mode 100644
index 00000000000..0bb8c79e1a1
--- /dev/null
+++ b/packages/twenty-server/src/engine/core-modules/enterprise/utils/__tests__/get-organization-key-validation-cron-pattern.util.spec.ts
@@ -0,0 +1,37 @@
+import { getOrganizationKeyValidationCronPattern } from 'src/engine/core-modules/enterprise/utils/get-organization-key-validation-cron-pattern.util';
+
+describe('getOrganizationKeyValidationCronPattern', () => {
+ it('keeps a server on the same daily time after a restart', () => {
+ expect(getOrganizationKeyValidationCronPattern('server-production')).toBe(
+ getOrganizationKeyValidationCronPattern('server-production'),
+ );
+ });
+
+ it('spreads servers across the whole day, down to the second', () => {
+ const hours = new Set<number>();
+ const seconds = new Set<number>();
+
+ for (let index = 0; index < 1000; index++) {
+ const pattern = getOrganizationKeyValidationCronPattern(
+ `server-${index}`,
+ );
+
+ expect(pattern).toMatch(/^\d{1,2} \d{1,2} \d{1,2} \* \* \*$/);
+
+ const [second, minute, hour] = pattern.split(' ').map(Number);
+
+ expect(second).toBeLessThan(60);
+ expect(minute).toBeLessThan(60);
+ expect(hour).toBeLessThan(24);
+ hours.add(hour);
+ seconds.add(second);
+ }
+
+ expect(hours.size).toBe(24);
+ expect(seconds.size).toBeGreaterThan(50);
+ });
+
+ it('keeps refreshing if a server ID cannot be persisted', () => {
+ expect(getOrganizationKeyValidationCronPattern(null)).toBe('0 4 * * *');
+ });
+});
diff --git a/packages/twenty-server/src/engine/core-modules/enterprise/utils/__tests__/to-minute-level-cron-pattern.util.spec.ts b/packages/twenty-server/src/engine/core-modules/enterprise/utils/__tests__/to-minute-level-cron-pattern.util.spec.ts
new file mode 100644
index 00000000000..c5a070f2b99
--- /dev/null
+++ b/packages/twenty-server/src/engine/core-modules/enterprise/utils/__tests__/to-minute-level-cron-pattern.util.spec.ts
@@ -0,0 +1,11 @@
+import { toMinuteLevelCronPattern } from 'src/engine/core-modules/enterprise/utils/to-minute-level-cron-pattern.util';
+
+describe('toMinuteLevelCronPattern', () => {
+ it('drops the seconds field of a six-field pattern', () => {
+ expect(toMinuteLevelCronPattern('37 12 9 * * *')).toBe('12 9 * * *');
+ });
+
+ it('keeps a five-field pattern as it is', () => {
+ expect(toMinuteLevelCronPattern('0 4 * * *')).toBe('0 4 * * *');
+ });
+});
diff --git a/packages/twenty-server/src/engine/core-modules/enterprise/utils/get-organization-key-validation-cron-pattern.util.ts b/packages/twenty-server/src/engine/core-modules/enterprise/utils/get-organization-key-validation-cron-pattern.util.ts
new file mode 100644
index 00000000000..55f0b6b689e
--- /dev/null
+++ b/packages/twenty-server/src/engine/core-modules/enterprise/utils/get-organization-key-validation-cron-pattern.util.ts
@@ -0,0 +1,27 @@
+import { createHash } from 'crypto';
+
+import { ENTERPRISE_KEY_VALIDATION_CRON_PATTERN } from 'src/engine/core-modules/enterprise/constants/organization-key-validation-cron-pattern.constant';
+
+const SECONDS_PER_MINUTE = 60;
+const SECONDS_PER_HOUR = 60 * SECONDS_PER_MINUTE;
+const SECONDS_PER_DAY = 24 * SECONDS_PER_HOUR;
+
+export const getOrganizationKeyValidationCronPattern = (
+ serverId: string | null,
+): string => {
+ if (serverId === null) {
+ return ENTERPRISE_KEY_VALIDATION_CRON_PATTERN;
+ }
+
+ const offsetInDay =
+ createHash('sha256').update(serverId).digest().readUInt32BE(0) %
+ SECONDS_PER_DAY;
+
+ const hour = Math.floor(offsetInDay / SECONDS_PER_HOUR);
+ const minute = Math.floor(
+ (offsetInDay % SECONDS_PER_HOUR) / SECONDS_PER_MINUTE,
+ );
+ const second = offsetInDay % SECONDS_PER_MINUTE;
+
+ return `${second} ${minute} ${hour} * * *`;
+};
diff --git a/packages/twenty-server/src/engine/core-modules/enterprise/utils/to-minute-level-cron-pattern.util.ts b/packages/twenty-server/src/engine/core-modules/enterprise/utils/to-minute-level-cron-pattern.util.ts
new file mode 100644
index 00000000000..24dc1a38f79
--- /dev/null
+++ b/packages/twenty-server/src/engine/core-modules/enterprise/utils/to-minute-level-cron-pattern.util.ts
@@ -0,0 +1,6 @@
+// Sentry crontab schedules have no seconds field; BullMQ patterns may have one.
+export const toMinuteLevelCronPattern = (cronPattern: string): string => {
+ const fields = cronPattern.split(' ');
+
+ return fields.length === 6 ? fields.slice(1).join(' ') : cronPattern;
+};
diff --git a/packages/twenty-server/src/engine/core-modules/message-queue/drivers/interfaces/job-options.interface.ts b/packages/twenty-server/src/engine/core-modules/message-queue/drivers/interfaces/job-options.interface.ts
index cb600a5a055..b86b4ad8b59 100644
--- a/packages/twenty-server/src/engine/core-modules/message-queue/drivers/interfaces/job-options.interface.ts
+++ b/packages/twenty-server/src/engine/core-modules/message-queue/drivers/interfaces/job-options.interface.ts
@@ -58,6 +58,7 @@ export type QueueCronJobOptions = Pick<QueueJobSettings, 'priority'> & {
repeat: {
every?: number;
pattern?: string;
+ tz?: string;
limit?: number;
};
};