The open alternative to Salesforce, designed for AI.
TypeScript crmcrm-systemcustomergood-first-issuegraphqlhacktoberfestjavascriptmarketingmonoreponestjsopen-sourcepostgresqlreactreactjssalestypescriptweb
Order worker jobs by tenant virtual clocks behind IS_WORKER_FAIRNESS_ENABLED (#28990)
martmull2a7901250ec4
diff --git a/packages/twenty-server/package.json b/packages/twenty-server/package.jsonindex b99ad696310..5f48be974fe 100644--- a/packages/twenty-server/package.json+++ b/packages/twenty-server/package.json@@ -195,6 +195,7 @@"@types/graphql-fields": "^1.3.6","@types/graphql-upload": "^16.0.7","@types/html-to-text": "^9.0.4",+ "@types/ioredis-mock": "8.2.8","@types/jest": "^30.0.0","@types/lodash.chunk": "^4.2.9","@types/lodash.compact": "^3.0.9",@@ -221,6 +222,7 @@"@types/supertest": "^2.0.11","@types/unzipper": "^0","@types/uuid": "^9.0.2",+ "ioredis-mock": "8.13.1","jest": "29.7.0","msw": "2.12.14","node-fetch": "^2.7.0",diff --git a/packages/twenty-server/src/engine/core-modules/application/application-job/services/application-job.service.ts b/packages/twenty-server/src/engine/core-modules/application/application-job/services/application-job.service.tsindex 52031d49b68..b82a765ef60 100644--- a/packages/twenty-server/src/engine/core-modules/application/application-job/services/application-job.service.ts+++ b/packages/twenty-server/src/engine/core-modules/application/application-job/services/application-job.service.ts@@ -8,7 +8,10 @@ import {import { isDefined } from 'twenty-shared/utils';import { v4 } from 'uuid';+import { findActiveFlatApplicationById } from 'src/engine/core-modules/application/utils/find-active-flat-application-by-id.util';import { buildJobStatus } from 'src/engine/core-modules/message-queue/utils/build-job-status.util';+import { buildApplicationGroupKey } from 'src/engine/core-modules/message-queue/utils/build-application-group-key.util';+import { buildWorkspaceGroupKey } from 'src/engine/core-modules/message-queue/utils/build-workspace-group-key.util';import { ENQUEUE_JOB_DEFAULT_RETRY_LIMIT } from 'src/engine/core-modules/application/application-job/constants/enqueue-job-default-retry-limit.constant';import { ENQUEUE_JOB_PRIORITY } from 'src/engine/core-modules/application/application-job/constants/enqueue-job-priority.constant';import { MAX_JOBS_PER_STATUS_READ } from 'src/engine/core-modules/application/application-job/constants/max-jobs-per-status-read.constant';@@ -87,9 +90,10 @@ export class ApplicationJobService {const jobItems = this.getJobItems(input);- const { flatLogicFunctionMaps } =+ const { flatLogicFunctionMaps, flatApplicationMaps } =await this.workspaceCacheService.getOrRecompute(workspaceId, ['flatLogicFunctionMaps',+ 'flatApplicationMaps',]);const flatLogicFunction = findFlatEntityByUniversalIdentifier({@@ -109,6 +113,10 @@ export class ApplicationJobService {}const jobIds = jobItems.map((jobItem) => jobItem.jobId ?? v4());+ const application = findActiveFlatApplicationById(+ flatApplicationMaps,+ applicationId,+ );await this.messageQueueService.bulkAdd<LogicFunctionTriggerJobData>(LogicFunctionTriggerJob.name,@@ -126,6 +134,12 @@ export class ApplicationJobService {retryLimit: input.retryLimit ?? ENQUEUE_JOB_DEFAULT_RETRY_LIMIT,backoff: LOGIC_FUNCTION_QUEUE_RETRY_BACKOFF,priority: ENQUEUE_JOB_PRIORITY,+ groupKeys: [+ buildWorkspaceGroupKey(workspaceId),+ ...(isDefined(application)+ ? [buildApplicationGroupKey(application.universalIdentifier)]+ : []),+ ],...(isDefined(input.delayMs) ? { delay: input.delayMs } : {}),},);diff --git a/packages/twenty-server/src/engine/core-modules/logic-function/logic-function-trigger/triggers/cron/cron-trigger.cron.job.ts b/packages/twenty-server/src/engine/core-modules/logic-function/logic-function-trigger/triggers/cron/cron-trigger.cron.job.tsindex 64d28285971..03a7951ba62 100644--- a/packages/twenty-server/src/engine/core-modules/logic-function/logic-function-trigger/triggers/cron/cron-trigger.cron.job.ts+++ b/packages/twenty-server/src/engine/core-modules/logic-function/logic-function-trigger/triggers/cron/cron-trigger.cron.job.ts@@ -19,6 +19,8 @@ import { MessageQueueService } from 'src/engine/core-modules/message-queue/serviimport { ThrottlerException } from 'src/engine/core-modules/throttler/throttler.exception';import { WorkspaceEntity } from 'src/engine/core-modules/workspace/workspace.entity';import { LOGIC_FUNCTION_QUEUE_RETRY_BACKOFF } from 'src/engine/core-modules/logic-function/logic-function-trigger/constants/logic-function-queue-retry-backoff.constant';+import { buildApplicationGroupKey } from 'src/engine/core-modules/message-queue/utils/build-application-group-key.util';+import { buildWorkspaceGroupKey } from 'src/engine/core-modules/message-queue/utils/build-workspace-group-key.util';import { buildApplicationJobThrottledLogEntry } from 'src/engine/core-modules/logic-function/logic-function-trigger/utils/build-application-job-throttled-log-entry.util';import {LogicFunctionTriggerJob,@@ -101,7 +103,10 @@ export class CronTriggerCronJob {const applicationRegistrationId =application?.applicationRegistrationId;- if (!isDefined(applicationRegistrationId)) {+ if (+ !isDefined(application) ||+ !isDefined(applicationRegistrationId)+ ) {continue;}@@ -115,14 +120,12 @@ export class CronTriggerCronJob {throw error;}- if (isDefined(application)) {- this.applicationRepeatedEventRecorderService.record({- entry: buildApplicationJobThrottledLogEntry({- workspaceId: activeWorkspace.id,- application,- }),- });- }+ this.applicationRepeatedEventRecorderService.record({+ entry: buildApplicationJobThrottledLogEntry({+ workspaceId: activeWorkspace.id,+ application,+ }),+ });continue;}@@ -143,6 +146,10 @@ export class CronTriggerCronJob {{retryLimit: 10,backoff: LOGIC_FUNCTION_QUEUE_RETRY_BACKOFF,+ groupKeys: [+ buildWorkspaceGroupKey(activeWorkspace.id),+ buildApplicationGroupKey(application.universalIdentifier),+ ],},);}diff --git a/packages/twenty-server/src/engine/core-modules/logic-function/logic-function-trigger/triggers/database-event/call-database-event-trigger-jobs.job.ts b/packages/twenty-server/src/engine/core-modules/logic-function/logic-function-trigger/triggers/database-event/call-database-event-trigger-jobs.job.tsindex f1a83127d1a..14a30ab159e 100644--- a/packages/twenty-server/src/engine/core-modules/logic-function/logic-function-trigger/triggers/database-event/call-database-event-trigger-jobs.job.ts+++ b/packages/twenty-server/src/engine/core-modules/logic-function/logic-function-trigger/triggers/database-event/call-database-event-trigger-jobs.job.ts@@ -15,6 +15,8 @@ import { ApplicationJobEnqueueThrottlerService } from 'src/engine/core-modules/mimport { MessageQueueService } from 'src/engine/core-modules/message-queue/services/message-queue.service';import { ThrottlerException } from 'src/engine/core-modules/throttler/throttler.exception';import { LOGIC_FUNCTION_QUEUE_RETRY_BACKOFF } from 'src/engine/core-modules/logic-function/logic-function-trigger/constants/logic-function-queue-retry-backoff.constant';+import { buildApplicationGroupKey } from 'src/engine/core-modules/message-queue/utils/build-application-group-key.util';+import { buildWorkspaceGroupKey } from 'src/engine/core-modules/message-queue/utils/build-workspace-group-key.util';import { buildApplicationJobThrottledLogEntry } from 'src/engine/core-modules/logic-function/logic-function-trigger/utils/build-application-job-throttled-log-entry.util';import { buildLogicFunctionTriggerJobIds } from 'src/engine/core-modules/logic-function/logic-function-trigger/triggers/database-event/utils/build-logic-function-trigger-job-ids';import { findLogicFunctionsTriggeredByEventName } from 'src/engine/core-modules/logic-function/logic-function-trigger/triggers/database-event/utils/find-logic-functions-triggered-by-event-name';@@ -181,6 +183,10 @@ export class CallDatabaseEventTriggerJobsJob {retryLimit: 3,backoff: LOGIC_FUNCTION_QUEUE_RETRY_BACKOFF,doNotCheckDuplicatesBasedOnPrefix: true,+ groupKeys: [+ buildWorkspaceGroupKey(workspaceEventBatch.workspaceId),+ buildApplicationGroupKey(application.universalIdentifier),+ ],},);}diff --git a/packages/twenty-server/src/engine/core-modules/message-queue/drivers/__tests__/bullmq.driver.spec.ts b/packages/twenty-server/src/engine/core-modules/message-queue/drivers/__tests__/bullmq.driver.spec.tsindex 89919f1188b..fad79ebfa27 100644--- a/packages/twenty-server/src/engine/core-modules/message-queue/drivers/__tests__/bullmq.driver.spec.ts+++ b/packages/twenty-server/src/engine/core-modules/message-queue/drivers/__tests__/bullmq.driver.spec.ts@@ -3,6 +3,7 @@ import { type Job, Worker } from 'bullmq';import { DEFAULT_JOB_PRIORITY } from 'src/engine/core-modules/message-queue/constants/default-job-priority.constant';import { QUEUE_JOB_CHANGED_EVENT } from 'src/engine/core-modules/message-queue/constants/queue-job-changed-event.constant';import { BullMQDriver } from 'src/engine/core-modules/message-queue/drivers/bullmq.driver';+import { FAIR_TICKET_OFFSET } from 'src/engine/core-modules/message-queue/drivers/fairness/fair-priority.constant';import { MessageQueue } from 'src/engine/core-modules/message-queue/message-queue.constants';import { JOB_LATENCY_MS_BUCKET_BOUNDARIES } from 'src/engine/core-modules/metrics/constants/job-latency-ms-bucket-boundaries.constant';import { MetricsKeys } from 'src/engine/core-modules/metrics/types/metrics-keys.type';@@ -15,6 +16,8 @@ const mockAddBulk = jest.fn();const mockSetGlobalConcurrency = jest.fn();const mockRemoveGlobalConcurrency = jest.fn();const mockGetJobCountByTypes = jest.fn();+const mockTwentyConfigService = { get: jest.fn().mockReturnValue(false) };+const mockAllocateTickets = jest.fn();jest.mock('bullmq', () => ({Queue: jest.fn().mockImplementation(() => ({@@ -29,17 +32,27 @@ jest.mock('bullmq', () => ({})),Worker: jest.fn().mockImplementation(() => ({ on: jest.fn() })),MetricsTime: { ONE_WEEK: 1 },+ PRIORITY_LIMIT: jest.requireActual('bullmq').PRIORITY_LIMIT,}));jest.mock('uuid', () => ({ v4: () => 'generated-uuid-000000000000000000000' }));+jest.mock(+ 'src/engine/core-modules/message-queue/drivers/fairness/fair-ticket-allocator',+ () => ({+ FairTicketAllocator: jest+ .fn()+ .mockImplementation(() => ({ allocateTickets: mockAllocateTickets })),+ }),+);+const WAITING_JOB_ID = 'sync-catalog-ws-1-5c98b035-5b09-4550-a4fb-b52056c494d1';describe('BullMQDriver deduplication', () => {const driver = new BullMQDriver({} as never,{} as never,- {} as never,+ mockTwentyConfigService as never,{} as never,);@@ -177,7 +190,7 @@ describe('BullMQDriver progress', () => {const driver = new BullMQDriver({} as never,{ recordHistogram: jest.fn() } as never,- {} as never,+ mockTwentyConfigService as never,{} as never,);@@ -237,7 +250,7 @@ describe('BullMQDriver job change events', () => {const driver = new BullMQDriver({} as never,{ recordHistogram: jest.fn() } as never,- {} as never,+ mockTwentyConfigService as never,eventEmitter as never,);@@ -314,7 +327,7 @@ describe('BullMQDriver queue wait metric', () => {const driver = new BullMQDriver({} as never,{ recordHistogram } as never,- {} as never,+ mockTwentyConfigService as never,{} as never,);@@ -414,7 +427,7 @@ describe('BullMQDriver queue job count gauges', () => {const driver = new BullMQDriver({} as never,{ createMultiObservableGauge } as never,- {} as never,+ mockTwentyConfigService as never,{} as never,);@@ -470,7 +483,7 @@ describe('BullMQDriver global concurrency', () => {const driver = new BullMQDriver({} as never,{} as never,- {} as never,+ mockTwentyConfigService as never,{} as never,);@@ -496,3 +509,114 @@ describe('BullMQDriver global concurrency', () => {expect(mockSetGlobalConcurrency).not.toHaveBeenCalled();});});++describe('BullMQDriver fair tickets', () => {+ const incrementCounterForEvent = jest.fn();+ const driver = new BullMQDriver(+ {} as never,+ { incrementCounterForEvent } as never,+ {+ get: jest.fn((key: string) => key === 'IS_WORKER_FAIRNESS_ENABLED'),+ } as never,+ {} as never,+ );++ driver.register(MessageQueue.workspaceQueue);++ beforeEach(() => {+ jest.clearAllMocks();+ mockGetRanges.mockResolvedValue([]);+ mockAdd.mockImplementation(async (_name, _data, opts) => ({+ id: opts.jobId,+ }));+ mockAddBulk.mockImplementation(async (jobs) =>+ jobs.map((job: { opts: { jobId?: string } }, index: number) => ({+ id: job.opts.jobId ?? `auto-${index}`,+ })),+ );+ });++ it('ranks a job by the ticket of its workspace', async () => {+ mockAllocateTickets.mockResolvedValue([{ tag: 7, weight: 1 }]);++ await driver.add(MessageQueue.workspaceQueue, 'job', {+ workspaceId: 'ws-1',+ });++ expect(mockAllocateTickets).toHaveBeenCalledWith({+ queueName: MessageQueue.workspaceQueue,+ requests: [{ groupKeys: ['workspace:ws-1'], jobCount: 1 }],+ });+ expect(mockAdd).toHaveBeenCalledWith(+ 'job',+ { workspaceId: 'ws-1' },+ expect.objectContaining({ priority: FAIR_TICKET_OFFSET + 7 }),+ );+ });++ it('allocates one ticket per group of a bulk and spaces its jobs by the weight', async () => {+ mockAllocateTickets.mockResolvedValue([+ { tag: 10, weight: 0.5 },+ { tag: 0, weight: 1 },+ ]);++ await driver.bulkAdd(MessageQueue.workspaceQueue, 'job', [+ { data: { workspaceId: 'ws-1' } },+ { data: { workspaceId: 'ws-2' } },+ { data: { workspaceId: 'ws-1' } },+ ]);++ expect(mockAllocateTickets).toHaveBeenCalledWith({+ queueName: MessageQueue.workspaceQueue,+ requests: [+ { groupKeys: ['workspace:ws-1'], jobCount: 2 },+ { groupKeys: ['workspace:ws-2'], jobCount: 1 },+ ],+ });+ expect(+ mockAddBulk.mock.calls[0][0].map(+ ({ opts }: { opts: { priority: number } }) => opts.priority,+ ),+ ).toEqual([+ FAIR_TICKET_OFFSET + 10,+ FAIR_TICKET_OFFSET,+ FAIR_TICKET_OFFSET + 12,+ ]);+ });++ it('keeps the static priority and counts a fallback when no ticket comes back', async () => {+ mockAllocateTickets.mockResolvedValue(undefined);++ await driver.add(+ MessageQueue.workspaceQueue,+ 'job',+ { workspaceId: 'ws-1' },+ { priority: 4 },+ );++ expect(mockAdd).toHaveBeenCalledWith(+ 'job',+ { workspaceId: 'ws-1' },+ expect.objectContaining({ priority: 4 }),+ );+ expect(incrementCounterForEvent).toHaveBeenCalledWith(+ expect.objectContaining({ key: MetricsKeys.JobFairTagFallback }),+ );+ });++ it('leaves a delayed job on its static priority without charging a clock', async () => {+ await driver.add(+ MessageQueue.workspaceQueue,+ 'job',+ { workspaceId: 'ws-1' },+ { delay: 1_000, priority: 4 },+ );++ expect(mockAllocateTickets).not.toHaveBeenCalled();+ expect(mockAdd).toHaveBeenCalledWith(+ 'job',+ { workspaceId: 'ws-1' },+ expect.objectContaining({ delay: 1_000, priority: 4 }),+ );+ });+});diff --git a/packages/twenty-server/src/engine/core-modules/message-queue/drivers/bullmq.driver.ts b/packages/twenty-server/src/engine/core-modules/message-queue/drivers/bullmq.driver.tsindex 15d4585bc50..5f687f16cf8 100644--- a/packages/twenty-server/src/engine/core-modules/message-queue/drivers/bullmq.driver.ts+++ b/packages/twenty-server/src/engine/core-modules/message-queue/drivers/bullmq.driver.ts@@ -39,6 +39,7 @@ import { type MessageQueueWorkerOptions } from 'src/engine/core-modules/message-import { DEFAULT_JOB_PRIORITY } from 'src/engine/core-modules/message-queue/constants/default-job-priority.constant';import { QUEUE_JOB_CHANGED_EVENT } from 'src/engine/core-modules/message-queue/constants/queue-job-changed-event.constant';import { QUEUE_RETENTION } from 'src/engine/core-modules/message-queue/constants/queue-retention.constants';+import { FairQueueScheduler } from 'src/engine/core-modules/message-queue/drivers/fairness/fair-queue-scheduler';import { type MessageQueue } from 'src/engine/core-modules/message-queue/message-queue.constants';import { type QueueJobChangedEvent } from 'src/engine/core-modules/message-queue/types/queue-job-changed-event.type';import { getJobKey } from 'src/engine/core-modules/message-queue/utils/get-job-key.util';@@ -74,13 +75,21 @@ export class BullMQDriverRecord<MessageQueue, MessageQueueWorkerOptions>> = {};private workersDrain: Promise<void> | undefined;+ private readonly fairQueueScheduler: FairQueueScheduler;constructor(private options: BullMQDriverOptions,private metricsService: MetricsService,private twentyConfigService: TwentyConfigService,private eventEmitter: EventEmitter2,- ) {}+ ) {+ this.fairQueueScheduler = new FairQueueScheduler({+ getClient: (queueName) => this.queueMap[queueName].client,+ keyPrefix: options.prefix ?? 'bull',+ metricsService,+ twentyConfigService,+ });+ }onModuleInit() {this.metricsService.createMultiObservableGauge({@@ -520,8 +529,17 @@ export class BullMQDriver}const queueOptions = this.buildJobsOptions(options);+ const [priority] = await this.fairQueueScheduler.computeJobPriorities({+ queueName,+ jobsData: [data],+ options,+ staticPriority: queueOptions.priority,+ });- const job = await this.queueMap[queueName].add(jobName, data, queueOptions);+ const job = await this.queueMap[queueName].add(jobName, data, {+ ...queueOptions,+ priority,+ });return job.id;}@@ -572,6 +590,12 @@ export class BullMQDriver}const queueOptions = this.buildJobsOptions(options);+ const priorities = await this.fairQueueScheduler.computeJobPriorities({+ queueName,+ jobsData: jobsToAdd.map(({ data }) => data),+ options,+ staticPriority: queueOptions.priority,+ });const addedJobs = await this.queueMap[queueName].addBulk(jobsToAdd.map(({ data, jobId }, index) => ({@@ -579,6 +603,7 @@ export class BullMQDriverdata,opts: {...queueOptions,+ priority: priorities[index],jobId:jobId ??(queueOptions.jobId ? `${queueOptions.jobId}-${index}` : undefined),diff --git a/packages/twenty-server/src/engine/core-modules/message-queue/drivers/fairness/__tests__/fair-ticket-allocator.spec.ts b/packages/twenty-server/src/engine/core-modules/message-queue/drivers/fairness/__tests__/fair-ticket-allocator.spec.tsnew file mode 100644index 00000000000..93bba9f8ded--- /dev/null+++ b/packages/twenty-server/src/engine/core-modules/message-queue/drivers/fairness/__tests__/fair-ticket-allocator.spec.ts@@ -0,0 +1,153 @@+import { Logger } from '@nestjs/common';++import { type Redis } from 'ioredis';+import RedisMock from 'ioredis-mock';++import { FairTicketAllocator } from 'src/engine/core-modules/message-queue/drivers/fairness/fair-ticket-allocator';+import { MessageQueue } from 'src/engine/core-modules/message-queue/message-queue.constants';++const QUEUE = MessageQueue.logicFunctionQueue;+const KEY_PREFIX = 'bull';+const QUEUE_KEY_PREFIX = `${KEY_PREFIX}:${QUEUE}:`;+const VIRTUAL_TIME_KEY = `${QUEUE_KEY_PREFIX}fair:vtime`;+const WEIGHTS_KEY = `${QUEUE_KEY_PREFIX}fair:weights`;++const createAllocator = async () => {+ const redis: Redis = new RedisMock();++ await redis.flushall();++ const client = {+ runCommand: async (name: string, args: (string | number)[]) => {+ const [script, numberOfKeys, ...scriptArguments] = args;++ if (name === 'evalsha') {+ return redis.evalsha(+ String(script),+ Number(numberOfKeys),+ ...scriptArguments,+ );+ }++ if (name !== 'eval') {+ throw new Error(`Unexpected Redis command ${name}`);+ }++ return redis.eval(+ String(script),+ Number(numberOfKeys),+ ...scriptArguments,+ );+ },+ };++ return {+ redis,+ allocator: new FairTicketAllocator({+ getClient: async () => client as never,+ keyPrefix: KEY_PREFIX,+ }),+ readClock: async (groupKey: string) =>+ Number(await redis.get(`${QUEUE_KEY_PREFIX}fair:clock:${groupKey}`)),+ };+};++const createBrokenAllocator = () =>+ new FairTicketAllocator({+ getClient: async () => {+ throw new Error('Redis connection lost');+ },+ keyPrefix: KEY_PREFIX,+ });++describe('FairTicketAllocator', () => {+ beforeAll(() => {+ jest.spyOn(Logger.prototype, 'warn').mockImplementation(() => undefined);+ });++ describe('allocateTickets', () => {+ it('tags a request at its latest clock and charges each key for its own batch only', async () => {+ const { allocator, readClock } = await createAllocator();++ expect(+ await allocator.allocateTickets({+ queueName: QUEUE,+ requests: [+ { groupKeys: ['workspace:X', 'application:H'], jobCount: 100 },+ { groupKeys: ['workspace:W', 'application:H'], jobCount: 1 },+ { groupKeys: ['workspace:W', 'application:Q'], jobCount: 1 },+ ],+ }),+ ).toEqual([+ { tag: 0, weight: 1 },+ { tag: 100, weight: 1 },+ { tag: 1, weight: 1 },+ ]);+ expect(await readClock('application:H')).toBe(101);+ expect(await readClock('workspace:W')).toBe(2);+ });++ it('tags a request at the virtual time when it is ahead of its clocks', async () => {+ const { redis, allocator, readClock } = await createAllocator();++ await redis.set(VIRTUAL_TIME_KEY, 5);++ expect(+ await allocator.allocateTickets({+ queueName: QUEUE,+ requests: [{ groupKeys: ['workspace:B'], jobCount: 2 }],+ }),+ ).toEqual([{ tag: 5, weight: 1 }]);+ expect(await readClock('workspace:B')).toBe(7);+ });++ it('weighs a request by the product of its key weights and ignores a malformed weight', async () => {+ const { redis, allocator } = await createAllocator();++ await redis.hset(WEIGHTS_KEY, 'application:P', '2');+ await redis.hset(WEIGHTS_KEY, 'workspace:W', 'not-a-number');+ await redis.hset(WEIGHTS_KEY, 'application:N', '-1');++ expect(+ await allocator.allocateTickets({+ queueName: QUEUE,+ requests: [+ { groupKeys: ['workspace:W', 'application:P'], jobCount: 2 },+ { groupKeys: ['workspace:W', 'application:N'], jobCount: 1 },+ ],+ }),+ ).toEqual([+ { tag: 0, weight: 2 },+ { tag: 1, weight: 1 },+ ]);+ });++ it('returns nothing when Redis is unavailable', async () => {+ expect(+ await createBrokenAllocator().allocateTickets({+ queueName: QUEUE,+ requests: [{ groupKeys: ['workspace:A'], jobCount: 1 }],+ }),+ ).toBeUndefined();+ });++ it('retries with EVAL only when Redis does not know the script', async () => {+ const runCommand = jest+ .fn()+ .mockRejectedValue(new Error('READONLY replica is read only'));+ const allocator = new FairTicketAllocator({+ getClient: async () => ({ runCommand }) as never,+ keyPrefix: KEY_PREFIX,+ });++ expect(+ await allocator.allocateTickets({+ queueName: QUEUE,+ requests: [{ groupKeys: ['workspace:A'], jobCount: 1 }],+ }),+ ).toBeUndefined();+ expect(runCommand).toHaveBeenCalledTimes(1);+ expect(runCommand).toHaveBeenCalledWith('evalsha', expect.any(Array));+ });+ });+});diff --git a/packages/twenty-server/src/engine/core-modules/message-queue/drivers/fairness/fair-priority.constant.ts b/packages/twenty-server/src/engine/core-modules/message-queue/drivers/fairness/fair-priority.constant.tsnew file mode 100644index 00000000000..5be5183a654--- /dev/null+++ b/packages/twenty-server/src/engine/core-modules/message-queue/drivers/fairness/fair-priority.constant.ts@@ -0,0 +1,3 @@+export const FAIR_TICKET_OFFSET = 2;++export const FAIR_CLOCK_TTL_MS = 24 * 60 * 60 * 1000;diff --git a/packages/twenty-server/src/engine/core-modules/message-queue/drivers/fairness/fair-queue-scheduler.ts b/packages/twenty-server/src/engine/core-modules/message-queue/drivers/fairness/fair-queue-scheduler.tsnew file mode 100644index 00000000000..24b0cc766ff--- /dev/null+++ b/packages/twenty-server/src/engine/core-modules/message-queue/drivers/fairness/fair-queue-scheduler.ts@@ -0,0 +1,105 @@+import { PRIORITY_LIMIT, type RedisClient } from 'bullmq';+import { isDefined, isNonEmptyArray } from 'twenty-shared/utils';++import { FAIR_TICKET_OFFSET } from 'src/engine/core-modules/message-queue/drivers/fairness/fair-priority.constant';+import { FairTicketAllocator } from 'src/engine/core-modules/message-queue/drivers/fairness/fair-ticket-allocator';+import { type QueueJobOptions } from 'src/engine/core-modules/message-queue/drivers/interfaces/job-options.interface';+import { type MessageQueue } from 'src/engine/core-modules/message-queue/message-queue.constants';+import { getDefaultGroupKeys } from 'src/engine/core-modules/message-queue/utils/get-default-group-keys.util';+import { type MetricsService } from 'src/engine/core-modules/metrics/metrics.service';+import { MetricsKeys } from 'src/engine/core-modules/metrics/types/metrics-keys.type';+import { type TwentyConfigService } from 'src/engine/core-modules/twenty-config/twenty-config.service';++type FairQueueSchedulerOptions = {+ getClient: (queueName: MessageQueue) => Promise<RedisClient>;+ keyPrefix: string;+ metricsService: MetricsService;+ twentyConfigService: TwentyConfigService;+};++export class FairQueueScheduler {+ private readonly fairTicketAllocator: FairTicketAllocator;++ constructor(private readonly options: FairQueueSchedulerOptions) {+ this.fairTicketAllocator = new FairTicketAllocator({+ getClient: options.getClient,+ keyPrefix: options.keyPrefix,+ });+ }++ async computeJobPriorities({+ queueName,+ jobsData,+ options,+ staticPriority,+ }: {+ queueName: MessageQueue;+ jobsData: unknown[];+ options?: Pick<QueueJobOptions, 'groupKeys' | 'delay'>;+ staticPriority: number | undefined;+ }): Promise<(number | undefined)[]> {+ const staticPriorities = jobsData.map(() => staticPriority);++ if (!this.isFairnessEnabled() || (options?.delay ?? 0) > 0) {+ return staticPriorities;+ }++ const requestByGroupKeys = new Map<+ string,+ { groupKeys: string[]; jobIndexes: number[] }+ >();++ jobsData.forEach((jobData, jobIndex) => {+ const groupKeys = isNonEmptyArray(options?.groupKeys)+ ? [...new Set(options.groupKeys)]+ : getDefaultGroupKeys(jobData);+ const signature = JSON.stringify(groupKeys);+ const request = requestByGroupKeys.get(signature) ?? {+ groupKeys,+ jobIndexes: [],+ };++ request.jobIndexes.push(jobIndex);+ requestByGroupKeys.set(signature, request);+ });++ const requests = [...requestByGroupKeys.values()];+ const tickets = await this.fairTicketAllocator.allocateTickets({+ queueName,+ requests: requests.map(({ groupKeys, jobIndexes }) => ({+ groupKeys,+ jobCount: jobIndexes.length,+ })),+ });++ if (!isDefined(tickets)) {+ void this.options.metricsService.incrementCounterForEvent({+ key: MetricsKeys.JobFairTagFallback,+ attributes: { queue: queueName },+ shouldStoreInCache: false,+ });++ return staticPriorities;+ }++ const priorities = [...staticPriorities];++ requests.forEach(({ jobIndexes }, requestIndex) => {+ const { tag, weight } = tickets[requestIndex];++ jobIndexes.forEach((jobIndex, position) => {+ const ticket = Math.floor(tag + position / weight);++ priorities[jobIndex] =+ FAIR_TICKET_OFFSET ++ Math.min(PRIORITY_LIMIT - FAIR_TICKET_OFFSET, Math.max(0, ticket));+ });+ });++ return priorities;+ }++ private isFairnessEnabled(): boolean {+ return this.options.twentyConfigService.get('IS_WORKER_FAIRNESS_ENABLED');+ }+}diff --git a/packages/twenty-server/src/engine/core-modules/message-queue/drivers/fairness/fair-tag-script.constant.ts b/packages/twenty-server/src/engine/core-modules/message-queue/drivers/fairness/fair-tag-script.constant.tsnew file mode 100644index 00000000000..fc25d3d1c0f--- /dev/null+++ b/packages/twenty-server/src/engine/core-modules/message-queue/drivers/fairness/fair-tag-script.constant.ts@@ -0,0 +1,70 @@+export const FAIR_TAG_SCRIPT = `+local virtualTimeKey = KEYS[1]+local weightsKey = KEYS[2]+local firstClockKeyIndex = 3+local clockTtlMs = ARGV[1]+local distinctKeyCount = tonumber(ARGV[2])+local firstGroupKeyArgIndex = 3+local requestCount = tonumber(ARGV[firstGroupKeyArgIndex + distinctKeyCount])+local firstRequestArgIndex = firstGroupKeyArgIndex + distinctKeyCount + 1++local virtualTime = tonumber(redis.call('GET', virtualTimeKey) or '0')+local clocks = {}+local weights = {}++for i = 1, distinctKeyCount do+ local clock = tonumber(redis.call('GET', KEYS[firstClockKeyIndex + i - 1]))++ clocks[i] = virtualTime++ if clock ~= nil and clock > virtualTime then+ clocks[i] = clock+ end++ local weight = tonumber(redis.call('HGET', weightsKey, ARGV[firstGroupKeyArgIndex + i - 1]))++ if weight == nil or weight <= 0 then+ weight = 1+ end++ weights[i] = weight+end++local tickets = {}+local ticketCount = 0+local position = firstRequestArgIndex++for _ = 1, requestCount do+ local jobCount = tonumber(ARGV[position])+ local keyCount = tonumber(ARGV[position + 1])+ local tag = virtualTime+ local weight = 1++ for k = 1, keyCount do+ local index = tonumber(ARGV[position + 1 + k])++ if clocks[index] > tag then+ tag = clocks[index]+ end++ weight = weight * weights[index]+ end++ for k = 1, keyCount do+ local index = tonumber(ARGV[position + 1 + k])++ clocks[index] = clocks[index] + jobCount / weight+ end++ tickets[ticketCount + 1] = tostring(tag)+ tickets[ticketCount + 2] = tostring(weight)+ ticketCount = ticketCount + 2+ position = position + 2 + keyCount+end++for i = 1, distinctKeyCount do+ redis.call('SET', KEYS[firstClockKeyIndex + i - 1], tostring(clocks[i]), 'PX', clockTtlMs)+end++return table.concat(tickets, ' ')+`;diff --git a/packages/twenty-server/src/engine/core-modules/message-queue/drivers/fairness/fair-ticket-allocator.ts b/packages/twenty-server/src/engine/core-modules/message-queue/drivers/fairness/fair-ticket-allocator.tsnew file mode 100644index 00000000000..4b50f1920b1--- /dev/null+++ b/packages/twenty-server/src/engine/core-modules/message-queue/drivers/fairness/fair-ticket-allocator.ts@@ -0,0 +1,107 @@+import { createHash } from 'node:crypto';++import { Logger } from '@nestjs/common';++import { type RedisClient } from 'bullmq';++import { FAIR_CLOCK_TTL_MS } from 'src/engine/core-modules/message-queue/drivers/fairness/fair-priority.constant';+import { FAIR_TAG_SCRIPT } from 'src/engine/core-modules/message-queue/drivers/fairness/fair-tag-script.constant';+import { type MessageQueue } from 'src/engine/core-modules/message-queue/message-queue.constants';++export type FairTicketRequest = { groupKeys: string[]; jobCount: number };++export type FairTicket = { tag: number; weight: number };++type FairTicketAllocatorOptions = {+ getClient: (queueName: MessageQueue) => Promise<RedisClient>;+ keyPrefix: string;+};++const FAIR_TAG_SCRIPT_SHA = createHash('sha1')+ .update(FAIR_TAG_SCRIPT)+ .digest('hex');++export class FairTicketAllocator {+ private readonly logger = new Logger(FairTicketAllocator.name);++ constructor(private readonly options: FairTicketAllocatorOptions) {}++ async allocateTickets({+ queueName,+ requests,+ }: {+ queueName: MessageQueue;+ requests: FairTicketRequest[];+ }): Promise<FairTicket[] | undefined> {+ const fairKeyPrefix = `${this.options.keyPrefix}:${queueName}:fair:`;+ const distinctGroupKeys = [+ ...new Set(requests.flatMap(({ groupKeys }) => groupKeys)),+ ];+ const groupKeyIndexes = new Map(+ distinctGroupKeys.map((groupKey, index) => [groupKey, index + 1]),+ );+ const keys = [+ `${fairKeyPrefix}vtime`,+ `${fairKeyPrefix}weights`,+ ...distinctGroupKeys.map(+ (groupKey) => `${fairKeyPrefix}clock:${groupKey}`,+ ),+ ];+ const args = [+ FAIR_CLOCK_TTL_MS,+ distinctGroupKeys.length,+ ...distinctGroupKeys,+ requests.length,+ ...requests.flatMap(({ groupKeys, jobCount }) => [+ jobCount,+ groupKeys.length,+ ...groupKeys.map((groupKey) => groupKeyIndexes.get(groupKey) ?? 0),+ ]),+ ];++ try {+ const client = await this.options.getClient(queueName);+ const rawTickets = await client+ .runCommand('evalsha', [+ FAIR_TAG_SCRIPT_SHA,+ keys.length,+ ...keys,+ ...args,+ ])+ .catch((error: unknown) => {+ if (+ !(error instanceof Error && error.message.startsWith('NOSCRIPT'))+ ) {+ throw error;+ }++ return client.runCommand('eval', [+ FAIR_TAG_SCRIPT,+ keys.length,+ ...keys,+ ...args,+ ]);+ });+ const values = String(rawTickets).split(' ').map(Number);+ const tickets = requests.map((_, index) => ({+ tag: values[2 * index],+ weight: values[2 * index + 1],+ }));+ const hasValidTickets =+ values.length === 2 * requests.length &&+ tickets.every(({ tag, weight }) => Number.isFinite(tag) && weight > 0);++ if (!hasValidTickets) {+ throw new Error(`unexpected tickets "${String(rawTickets)}"`);+ }++ return tickets;+ } catch (error) {+ this.logger.warn(+ `Failed to allocate the fair tickets on queue ${queueName}: ${error instanceof Error ? error.message : String(error)}`,+ );++ return undefined;+ }+ }+}diff --git a/packages/twenty-server/src/engine/core-modules/message-queue/drivers/interfaces/job-options.interface.ts b/packages/twenty-server/src/engine/core-modules/message-queue/drivers/interfaces/job-options.interface.tsindex 6388e3e8b3a..919d2b6a9f2 100644--- a/packages/twenty-server/src/engine/core-modules/message-queue/drivers/interfaces/job-options.interface.ts+++ b/packages/twenty-server/src/engine/core-modules/message-queue/drivers/interfaces/job-options.interface.ts@@ -23,6 +23,7 @@ export interface QueueJobOptions {delay?: number;deduplication?: QueueJobDeduplicationOptions;broadcastTo?: QueueJobRecipient;+ groupKeys?: string[];}export interface QueueCronJobOptions extends QueueJobOptions {diff --git a/packages/twenty-server/src/engine/core-modules/message-queue/utils/build-application-group-key.util.ts b/packages/twenty-server/src/engine/core-modules/message-queue/utils/build-application-group-key.util.tsnew file mode 100644index 00000000000..d3373010874--- /dev/null+++ b/packages/twenty-server/src/engine/core-modules/message-queue/utils/build-application-group-key.util.ts@@ -0,0 +1,3 @@+export const buildApplicationGroupKey = (+ applicationUniversalIdentifier: string,+): string => `application:${applicationUniversalIdentifier}`;diff --git a/packages/twenty-server/src/engine/core-modules/message-queue/utils/build-workspace-group-key.util.ts b/packages/twenty-server/src/engine/core-modules/message-queue/utils/build-workspace-group-key.util.tsnew file mode 100644index 00000000000..6b44ca9c9e0--- /dev/null+++ b/packages/twenty-server/src/engine/core-modules/message-queue/utils/build-workspace-group-key.util.ts@@ -0,0 +1,2 @@+export const buildWorkspaceGroupKey = (workspaceId: string): string =>+ `workspace:${workspaceId}`;diff --git a/packages/twenty-server/src/engine/core-modules/message-queue/utils/get-default-group-keys.util.ts b/packages/twenty-server/src/engine/core-modules/message-queue/utils/get-default-group-keys.util.tsnew file mode 100644index 00000000000..6d07e7cd698--- /dev/null+++ b/packages/twenty-server/src/engine/core-modules/message-queue/utils/get-default-group-keys.util.ts@@ -0,0 +1,14 @@+import { isNonEmptyString } from '@sniptt/guards';+import { isPlainObject } from 'twenty-shared/utils';++import { buildWorkspaceGroupKey } from 'src/engine/core-modules/message-queue/utils/build-workspace-group-key.util';++const SYSTEM_GROUP_KEY = 'system';++export const getDefaultGroupKeys = (jobData: unknown): string[] => {+ const workspaceId = isPlainObject(jobData) ? jobData.workspaceId : undefined;++ return isNonEmptyString(workspaceId)+ ? [buildWorkspaceGroupKey(workspaceId)]+ : [SYSTEM_GROUP_KEY];+};diff --git a/packages/twenty-server/src/engine/core-modules/metrics/types/metrics-keys.type.ts b/packages/twenty-server/src/engine/core-modules/metrics/types/metrics-keys.type.tsindex 850052c5358..4d361070ea0 100644--- a/packages/twenty-server/src/engine/core-modules/metrics/types/metrics-keys.type.ts+++ b/packages/twenty-server/src/engine/core-modules/metrics/types/metrics-keys.type.ts@@ -68,6 +68,7 @@ export enum MetricsKeys {JobWaiting = 'job/waiting',JobLatencyMs = 'job/latency-ms',JobDurationMs = 'job/duration-ms',+ JobFairTagFallback = 'job/fair-tag-fallback',AiChatTurnLatencyMs = 'ai-chat/turn-latency-ms',AiChatStepLatencyMs = 'ai-chat/step-latency-ms',AiChatTtftMs = 'ai-chat/ttft-ms',diff --git a/packages/twenty-server/src/engine/core-modules/twenty-config/config-variables.ts b/packages/twenty-server/src/engine/core-modules/twenty-config/config-variables.tsindex 1966efb76b2..404652cd104 100644--- a/packages/twenty-server/src/engine/core-modules/twenty-config/config-variables.ts+++ b/packages/twenty-server/src/engine/core-modules/twenty-config/config-variables.ts@@ -1617,6 +1617,15 @@ export class ConfigVariables {@IsOptional()WORKER_EXCLUDED_QUEUES: string[] = [];+ @ConfigVariablesMetadata({+ group: ConfigVariablesGroup.ADVANCED_SETTINGS,+ description:+ 'Order the jobs of each queue by tenant virtual clocks (workspace and application group keys) so that a burst from one tenant does not delay the others. Off: jobs of a queue are processed in FIFO order.',+ type: ConfigVariableType.BOOLEAN,+ })+ @IsOptional()+ IS_WORKER_FAIRNESS_ENABLED = false;+@ConfigVariablesMetadata({group: ConfigVariablesGroup.SERVER_CONFIG,description: 'Node environment (development, production, etc.)',diff --git a/packages/twenty-server/test/integration/message-queue/utils/with-queue-order-test-context.util.ts b/packages/twenty-server/test/integration/message-queue/utils/with-queue-order-test-context.util.tsindex 35ca123a0e7..fbe3d02026c 100644--- a/packages/twenty-server/test/integration/message-queue/utils/with-queue-order-test-context.util.ts+++ b/packages/twenty-server/test/integration/message-queue/utils/with-queue-order-test-context.util.ts@@ -30,7 +30,13 @@ export const withQueueOrderTestContext = async (incrementCounterForEvent: jest.fn(),},},- { provide: TwentyConfigService, useValue: {} },+ {+ provide: TwentyConfigService,+ useValue: {+ get: (key: string) =>+ key === 'IS_WORKER_FAIRNESS_ENABLED' ? false : undefined,+ },+ },{ provide: EventEmitter2, useValue: new EventEmitter2() },{provide: BullMQDriver,diff --git a/yarn.lock b/yarn.lockindex ee678dbd659..1838e152b60 100644--- a/yarn.lock+++ b/yarn.lock@@ -8743,6 +8743,13 @@ __metadata:languageName: nodelinkType: hard+"@ioredis/as-callback@npm:^3.0.0":+ version: 3.0.0+ resolution: "@ioredis/as-callback@npm:3.0.0"+ checksum: 10c0/cbeae0c4a8f8f8ea1987d105056fd595a5a1a781a73dfa3a13ac13027fd1de099af6f3590bd315fd20284efd8597430a7ba3b912570804d158dabbdb2071523b+ languageName: node+ linkType: hard+"@ioredis/commands@npm:1.5.1":version: 1.5.1resolution: "@ioredis/commands@npm:1.5.1"@@ -8750,6 +8757,13 @@ __metadata:languageName: nodelinkType: hard+"@ioredis/commands@npm:^1.4.0":+ version: 1.11.0+ resolution: "@ioredis/commands@npm:1.11.0"+ checksum: 10c0/fa8758f51ad1557e6c710f59890b83cd95c9c7fd419b6f7018268612d6479202d18566e16378097b52549e5dfafedc11a9a4703107a77d4b972684ee1757611a+ languageName: node+ linkType: hard+"@isaacs/cliui@npm:^8.0.2":version: 8.0.2resolution: "@isaacs/cliui@npm:8.0.2"@@ -19432,6 +19446,15 @@ __metadata:languageName: nodelinkType: hard+"@types/ioredis-mock@npm:8.2.8":+ version: 8.2.8+ resolution: "@types/ioredis-mock@npm:8.2.8"+ peerDependencies:+ ioredis: ^5+ checksum: 10c0/af7a6546b2f43f4c8af4ef1deb453d1e436d901bd9e25d23b0e100b6170712ae38e72ca022fd12fa3616b4abe8ceee5a5b483db8774a99b3403ee3d2097fb9c6+ languageName: node+ linkType: hard+"@types/istanbul-lib-coverage@npm:*, @types/istanbul-lib-coverage@npm:^2.0.0, @types/istanbul-lib-coverage@npm:^2.0.1, @types/istanbul-lib-coverage@npm:^2.0.4, @types/istanbul-lib-coverage@npm:^2.0.6":version: 2.0.6resolution: "@types/istanbul-lib-coverage@npm:2.0.6"@@ -29266,6 +29289,26 @@ __metadata:languageName: nodelinkType: hard+"fengari-interop@npm:^0.1.3":+ version: 0.1.4+ resolution: "fengari-interop@npm:0.1.4"+ peerDependencies:+ fengari: ^0.1.0+ checksum: 10c0/ba8aca9b741b603a7740343b3f99dbd60c4536aab115a8e02caa33030e16e0dd13a0d03f0c31050f7fa1ead7945358040cead48531df41c3b3147e61b50f383c+ languageName: node+ linkType: hard++"fengari@npm:^0.1.4":+ version: 0.1.5+ resolution: "fengari@npm:0.1.5"+ dependencies:+ readline-sync: "npm:^1.4.10"+ sprintf-js: "npm:^1.1.3"+ tmp: "npm:^0.2.5"+ checksum: 10c0/8688c3483e00028b0b09928f71452bb2c8b4db5f7380e0e8dd65619bb4ccb38438557ebdfd977c55a6d3fb5e7601cbac26c62873317320ffd20b59936f464244+ languageName: node+ linkType: hard+"fernet@npm:^0.3.3":version: 0.3.3resolution: "fernet@npm:0.3.3"@@ -32036,6 +32079,22 @@ __metadata:languageName: nodelinkType: hard+"ioredis-mock@npm:8.13.1":+ version: 8.13.1+ resolution: "ioredis-mock@npm:8.13.1"+ dependencies:+ "@ioredis/as-callback": "npm:^3.0.0"+ "@ioredis/commands": "npm:^1.4.0"+ fengari: "npm:^0.1.4"+ fengari-interop: "npm:^0.1.3"+ semver: "npm:^7.7.2"+ peerDependencies:+ "@types/ioredis-mock": ^8+ ioredis: ^5+ checksum: 10c0/47e17a307301da01a89d041a6d6101828baefd3a1c216a09fe8ff9e1bd7d7239de93544365280f6cda4ad98af1f302e7497f9563f36cb24b528d9c7b3164301f+ languageName: node+ linkType: hard+"ioredis@npm:5.10.1":version: 5.10.1resolution: "ioredis@npm:5.10.1"@@ -41401,7 +41460,7 @@ __metadata:languageName: nodelinkType: hard-"readline-sync@npm:^1.4.9":+"readline-sync@npm:^1.4.10, readline-sync@npm:^1.4.9":version: 1.4.10resolution: "readline-sync@npm:1.4.10"checksum: 10c0/0a4d0fe4ad501f8f005a3c9cbf3cc0ae6ca2ced93e9a1c7c46f226bdfcb6ef5d3f437ae7e9d2e1098ee13524a3739c830e4c8dbc7f543a693eecd293e41093a3@@ -43789,6 +43848,13 @@ __metadata:languageName: nodelinkType: hard+"sprintf-js@npm:^1.1.3":+ version: 1.1.3+ resolution: "sprintf-js@npm:1.1.3"+ checksum: 10c0/09270dc4f30d479e666aee820eacd9e464215cdff53848b443964202bf4051490538e5dd1b42e1a65cf7296916ca17640aebf63dae9812749c7542ee5f288dec+ languageName: node+ linkType: hard+"sprintf-js@npm:~1.0.2":version: 1.0.3resolution: "sprintf-js@npm:1.0.3"@@ -46101,6 +46167,7 @@ __metadata:"@types/graphql-fields": "npm:^1.3.6""@types/graphql-upload": "npm:^16.0.7""@types/html-to-text": "npm:^9.0.4"+ "@types/ioredis-mock": "npm:8.2.8""@types/jest": "npm:^30.0.0""@types/lodash.chunk": "npm:^4.2.9""@types/lodash.compact": "npm:^3.0.9"@@ -46168,6 +46235,7 @@ __metadata:ical-generator: "npm:^11.0.0"imapflow: "npm:2.0.2"ioredis: "npm:5.10.1"+ ioredis-mock: "npm:8.13.1"jest: "npm:29.7.0"jsdom: "npm:^30.0.1"json-schema: "npm:0.4.0"