The open alternative to Salesforce, designed for AI.
TypeScript crmcrm-systemcustomergood-first-issuegraphqlhacktoberfestjavascriptmarketingmonoreponestjsopen-sourcepostgresqlreactreactjssalestypescriptweb
Advance the queue's virtual time when a fair job is dispatched (#28992)
martmull7be5668f48f5
diff --git a/packages/twenty-server/src/engine/core-modules/message-queue/drivers/__tests__/bullmq.driver.spec.ts b/packages/twenty-server/src/engine/core-modules/message-queue/drivers/__tests__/bullmq.driver.spec.tsindex f9859eaa145..415948e19e1 100644--- a/packages/twenty-server/src/engine/core-modules/message-queue/drivers/__tests__/bullmq.driver.spec.ts+++ b/packages/twenty-server/src/engine/core-modules/message-queue/drivers/__tests__/bullmq.driver.spec.ts@@ -21,6 +21,7 @@ const mockRemoveGlobalConcurrency = jest.fn();const mockGetJobCountByTypes = jest.fn();const mockTwentyConfigService = { get: jest.fn().mockReturnValue(false) };const mockAllocateTickets = jest.fn();+const mockRecordDispatch = jest.fn();jest.mock('bullmq', () => ({Queue: jest.fn().mockImplementation(() => ({@@ -43,9 +44,10 @@ jest.mock('uuid', () => ({ v4: () => 'generated-uuid-000000000000000000000' }));jest.mock('src/engine/core-modules/message-queue/drivers/fairness/fair-ticket-allocator',() => ({- FairTicketAllocator: jest- .fn()- .mockImplementation(() => ({ allocateTickets: mockAllocateTickets })),+ FairTicketAllocator: jest.fn().mockImplementation(() => ({+ allocateTickets: mockAllocateTickets,+ recordDispatch: mockRecordDispatch,+ })),}),);@@ -641,3 +643,59 @@ describe('BullMQDriver fair tickets', () => {);});});++describe('BullMQDriver fair dispatch', () => {+ const processJob = async ({+ isFairnessEnabled,+ }: {+ isFairnessEnabled: boolean;+ }) => {+ const driver = new BullMQDriver(+ {} as never,+ { recordHistogram: jest.fn() } as never,+ {+ get: jest.fn(+ (key: string) =>+ key === 'IS_WORKER_FAIRNESS_ENABLED' && isFairnessEnabled,+ ),+ } as never,+ {} as never,+ );++ driver.register(MessageQueue.workflowQueue);+ jest.clearAllMocks();+ driver.work(MessageQueue.workflowQueue, jest.fn());++ const processor = jest.mocked(Worker).mock.calls[0][1];++ if (typeof processor !== 'function') {+ throw new Error('Worker processor was not registered');+ }++ await processor({+ id: 'job-id',+ name: 'job',+ data: {},+ opts: {},+ timestamp: Date.now(),+ attemptsStarted: 1,+ updateData: jest.fn(),+ updateProgress: jest.fn(),+ } as unknown as Job);+ };++ it('records the dispatch of a job when fairness is on', async () => {+ await processJob({ isFairnessEnabled: true });++ expect(mockRecordDispatch).toHaveBeenCalledWith({+ queueName: MessageQueue.workflowQueue,+ jobId: 'job-id',+ });+ });++ it('records nothing when fairness is off', async () => {+ await processJob({ isFairnessEnabled: false });++ expect(mockRecordDispatch).not.toHaveBeenCalled();+ });+});diff --git a/packages/twenty-server/src/engine/core-modules/message-queue/drivers/bullmq.driver.ts b/packages/twenty-server/src/engine/core-modules/message-queue/drivers/bullmq.driver.tsindex 5f687f16cf8..70da33203ae 100644--- a/packages/twenty-server/src/engine/core-modules/message-queue/drivers/bullmq.driver.ts+++ b/packages/twenty-server/src/engine/core-modules/message-queue/drivers/bullmq.driver.ts@@ -85,6 +85,8 @@ export class BullMQDriver) {this.fairQueueScheduler = new FairQueueScheduler({getClient: (queueName) => this.queueMap[queueName].client,+ getJobKey: ({ queueName, jobId }) =>+ this.queueMap[queueName].toKey(jobId),keyPrefix: options.prefix ?? 'bull',metricsService,twentyConfigService,@@ -288,6 +290,10 @@ export class BullMQDriverSentry.withIsolationScope(async () => {applyWorkspaceSentryContextFromJobData(job.data);+ if (isDefined(job.id)) {+ this.fairQueueScheduler.onJobStarted({ queueName, jobId: job.id });+ }+const queueWaitMs = computeJobQueueWaitMs({ job, now: Date.now() });if (isDefined(queueWaitMs)) {diff --git a/packages/twenty-server/src/engine/core-modules/message-queue/drivers/fairness/__tests__/fair-ticket-allocator.spec.ts b/packages/twenty-server/src/engine/core-modules/message-queue/drivers/fairness/__tests__/fair-ticket-allocator.spec.tsindex 93bba9f8ded..d51189abec2 100644--- a/packages/twenty-server/src/engine/core-modules/message-queue/drivers/fairness/__tests__/fair-ticket-allocator.spec.ts+++ b/packages/twenty-server/src/engine/core-modules/message-queue/drivers/fairness/__tests__/fair-ticket-allocator.spec.ts@@ -4,6 +4,10 @@ import { type Redis } from 'ioredis';import RedisMock from 'ioredis-mock';import { FairTicketAllocator } from 'src/engine/core-modules/message-queue/drivers/fairness/fair-ticket-allocator';+import {+ FAIR_TICKET_OFFSET,+ URGENT_FAIR_PRIORITY,+} from 'src/engine/core-modules/message-queue/drivers/fairness/fair-priority.constant';import { MessageQueue } from 'src/engine/core-modules/message-queue/message-queue.constants';const QUEUE = MessageQueue.logicFunctionQueue;@@ -12,6 +16,10 @@ const QUEUE_KEY_PREFIX = `${KEY_PREFIX}:${QUEUE}:`;const VIRTUAL_TIME_KEY = `${QUEUE_KEY_PREFIX}fair:vtime`;const WEIGHTS_KEY = `${QUEUE_KEY_PREFIX}fair:weights`;+const ticketOf = (tag: number) => FAIR_TICKET_OFFSET + tag;++const jobKeyOf = (jobId: string) => `${QUEUE_KEY_PREFIX}${jobId}`;+const createAllocator = async () => {const redis: Redis = new RedisMock();@@ -45,10 +53,14 @@ const createAllocator = async () => {redis,allocator: new FairTicketAllocator({getClient: async () => client as never,+ getJobKey: ({ jobId }) => jobKeyOf(jobId),keyPrefix: KEY_PREFIX,}),readClock: async (groupKey: string) =>Number(await redis.get(`${QUEUE_KEY_PREFIX}fair:clock:${groupKey}`)),+ readVirtualTime: async () => Number(await redis.get(VIRTUAL_TIME_KEY)),+ seedJob: async ({ jobId, priority }: { jobId: string; priority: number }) =>+ redis.hset(jobKeyOf(jobId), 'priority', priority),};};@@ -57,6 +69,7 @@ const createBrokenAllocator = () =>getClient: async () => {throw new Error('Redis connection lost');},+ getJobKey: ({ jobId }) => jobKeyOf(jobId),keyPrefix: KEY_PREFIX,});@@ -137,6 +150,7 @@ describe('FairTicketAllocator', () => {.mockRejectedValue(new Error('READONLY replica is read only'));const allocator = new FairTicketAllocator({getClient: async () => ({ runCommand }) as never,+ getJobKey: ({ jobId }) => jobKeyOf(jobId),keyPrefix: KEY_PREFIX,});@@ -150,4 +164,46 @@ describe('FairTicketAllocator', () => {expect(runCommand).toHaveBeenCalledWith('evalsha', expect.any(Array));});});++ describe('recordDispatch', () => {+ it('advances the virtual time from the dispatched job hash and leaves urgent and missing jobs alone', async () => {+ const { allocator, seedJob, readVirtualTime } = await createAllocator();++ await seedJob({ jobId: 'A1', priority: ticketOf(5) });+ await seedJob({ jobId: 'urgent', priority: URGENT_FAIR_PRIORITY });++ await allocator.recordDispatch({ queueName: QUEUE, jobId: 'A1' });+ await allocator.recordDispatch({ queueName: QUEUE, jobId: 'urgent' });+ await allocator.recordDispatch({ queueName: QUEUE, jobId: 'missing' });++ expect(await readVirtualTime()).toBe(5);+ expect(+ await allocator.allocateTickets({+ queueName: QUEUE,+ requests: [{ groupKeys: ['workspace:B'], jobCount: 1 }],+ }),+ ).toEqual([{ tag: 5, weight: 1 }]);+ });++ it('never moves the virtual time back', async () => {+ const { allocator, seedJob, readVirtualTime } = await createAllocator();++ await seedJob({ jobId: 'late', priority: ticketOf(9) });+ await seedJob({ jobId: 'early', priority: ticketOf(3) });++ await allocator.recordDispatch({ queueName: QUEUE, jobId: 'late' });+ await allocator.recordDispatch({ queueName: QUEUE, jobId: 'early' });++ expect(await readVirtualTime()).toBe(9);+ });++ it('swallows a Redis failure', async () => {+ await expect(+ createBrokenAllocator().recordDispatch({+ queueName: QUEUE,+ jobId: 'j',+ }),+ ).resolves.toBeUndefined();+ });+ });});diff --git a/packages/twenty-server/src/engine/core-modules/message-queue/drivers/fairness/fair-dispatch-script.constant.ts b/packages/twenty-server/src/engine/core-modules/message-queue/drivers/fairness/fair-dispatch-script.constant.tsnew file mode 100644index 00000000000..9ea92c6dbd1--- /dev/null+++ b/packages/twenty-server/src/engine/core-modules/message-queue/drivers/fairness/fair-dispatch-script.constant.ts@@ -0,0 +1,16 @@+export const FAIR_DISPATCH_SCRIPT = `+local ticketOffset = tonumber(ARGV[1])+local priority = tonumber(redis.call('HGET', KEYS[2], 'priority'))++if priority == nil or priority < ticketOffset then+ return 0+end++local ticket = priority - ticketOffset++if ticket > tonumber(redis.call('GET', KEYS[1]) or '0') then+ redis.call('SET', KEYS[1], ticket)+end++return 0+`;diff --git a/packages/twenty-server/src/engine/core-modules/message-queue/drivers/fairness/fair-queue-scheduler.ts b/packages/twenty-server/src/engine/core-modules/message-queue/drivers/fairness/fair-queue-scheduler.tsindex f54f8ecd67f..ba4f0a32363 100644--- a/packages/twenty-server/src/engine/core-modules/message-queue/drivers/fairness/fair-queue-scheduler.ts+++ b/packages/twenty-server/src/engine/core-modules/message-queue/drivers/fairness/fair-queue-scheduler.ts@@ -15,6 +15,7 @@ import { type TwentyConfigService } from 'src/engine/core-modules/twenty-config/type FairQueueSchedulerOptions = {getClient: (queueName: MessageQueue) => Promise<RedisClient>;+ getJobKey: (job: { queueName: MessageQueue; jobId: string }) => string;keyPrefix: string;metricsService: MetricsService;twentyConfigService: TwentyConfigService;@@ -26,6 +27,7 @@ export class FairQueueScheduler {constructor(private readonly options: FairQueueSchedulerOptions) {this.fairTicketAllocator = new FairTicketAllocator({getClient: options.getClient,+ getJobKey: options.getJobKey,keyPrefix: options.keyPrefix,});}@@ -106,6 +108,20 @@ export class FairQueueScheduler {return priorities;}+ onJobStarted({+ queueName,+ jobId,+ }: {+ queueName: MessageQueue;+ jobId: string;+ }): void {+ if (!this.isFairnessEnabled()) {+ return;+ }++ void this.fairTicketAllocator.recordDispatch({ queueName, jobId });+ }+private isFairnessEnabled(): boolean {return this.options.twentyConfigService.get('IS_WORKER_FAIRNESS_ENABLED');}diff --git a/packages/twenty-server/src/engine/core-modules/message-queue/drivers/fairness/fair-ticket-allocator.ts b/packages/twenty-server/src/engine/core-modules/message-queue/drivers/fairness/fair-ticket-allocator.tsindex 4b50f1920b1..41ac7caf3df 100644--- a/packages/twenty-server/src/engine/core-modules/message-queue/drivers/fairness/fair-ticket-allocator.ts+++ b/packages/twenty-server/src/engine/core-modules/message-queue/drivers/fairness/fair-ticket-allocator.ts@@ -3,8 +3,13 @@ import { createHash } from 'node:crypto';import { Logger } from '@nestjs/common';import { type RedisClient } from 'bullmq';+import { isDefined } from 'twenty-shared/utils';-import { FAIR_CLOCK_TTL_MS } from 'src/engine/core-modules/message-queue/drivers/fairness/fair-priority.constant';+import { FAIR_DISPATCH_SCRIPT } from 'src/engine/core-modules/message-queue/drivers/fairness/fair-dispatch-script.constant';+import {+ FAIR_CLOCK_TTL_MS,+ FAIR_TICKET_OFFSET,+} from 'src/engine/core-modules/message-queue/drivers/fairness/fair-priority.constant';import { FAIR_TAG_SCRIPT } from 'src/engine/core-modules/message-queue/drivers/fairness/fair-tag-script.constant';import { type MessageQueue } from 'src/engine/core-modules/message-queue/message-queue.constants';@@ -14,15 +19,13 @@ export type FairTicket = { tag: number; weight: number };type FairTicketAllocatorOptions = {getClient: (queueName: MessageQueue) => Promise<RedisClient>;+ getJobKey: (job: { queueName: MessageQueue; jobId: string }) => string;keyPrefix: string;};-const FAIR_TAG_SCRIPT_SHA = createHash('sha1')- .update(FAIR_TAG_SCRIPT)- .digest('hex');-export class FairTicketAllocator {private readonly logger = new Logger(FairTicketAllocator.name);+ private readonly scriptShas = new Map<string, string>();constructor(private readonly options: FairTicketAllocatorOptions) {}@@ -40,48 +43,30 @@ export class FairTicketAllocator {const groupKeyIndexes = new Map(distinctGroupKeys.map((groupKey, index) => [groupKey, index + 1]),);- const keys = [- `${fairKeyPrefix}vtime`,- `${fairKeyPrefix}weights`,- ...distinctGroupKeys.map(- (groupKey) => `${fairKeyPrefix}clock:${groupKey}`,- ),- ];- const args = [- FAIR_CLOCK_TTL_MS,- distinctGroupKeys.length,- ...distinctGroupKeys,- requests.length,- ...requests.flatMap(({ groupKeys, jobCount }) => [- jobCount,- groupKeys.length,- ...groupKeys.map((groupKey) => groupKeyIndexes.get(groupKey) ?? 0),- ]),- ];try {- const client = await this.options.getClient(queueName);- const rawTickets = await client- .runCommand('evalsha', [- FAIR_TAG_SCRIPT_SHA,- keys.length,- ...keys,- ...args,- ])- .catch((error: unknown) => {- if (- !(error instanceof Error && error.message.startsWith('NOSCRIPT'))- ) {- throw error;- }-- return client.runCommand('eval', [- FAIR_TAG_SCRIPT,- keys.length,- ...keys,- ...args,- ]);- });+ const rawTickets = await this.runScript({+ queueName,+ script: FAIR_TAG_SCRIPT,+ keys: [+ `${fairKeyPrefix}vtime`,+ `${fairKeyPrefix}weights`,+ ...distinctGroupKeys.map(+ (groupKey) => `${fairKeyPrefix}clock:${groupKey}`,+ ),+ ],+ args: [+ FAIR_CLOCK_TTL_MS,+ distinctGroupKeys.length,+ ...distinctGroupKeys,+ requests.length,+ ...requests.flatMap(({ groupKeys, jobCount }) => [+ jobCount,+ groupKeys.length,+ ...groupKeys.map((groupKey) => groupKeyIndexes.get(groupKey) ?? 0),+ ]),+ ],+ });const values = String(rawTickets).split(' ').map(Number);const tickets = requests.map((_, index) => ({tag: values[2 * index],@@ -104,4 +89,74 @@ export class FairTicketAllocator {return undefined;}}++ async recordDispatch({+ queueName,+ jobId,+ }: {+ queueName: MessageQueue;+ jobId: string;+ }): Promise<void> {+ const fairKeyPrefix = `${this.options.keyPrefix}:${queueName}:fair:`;++ try {+ await this.runScript({+ queueName,+ script: FAIR_DISPATCH_SCRIPT,+ keys: [+ `${fairKeyPrefix}vtime`,+ this.options.getJobKey({ queueName, jobId }),+ ],+ args: [FAIR_TICKET_OFFSET],+ });+ } catch (error) {+ this.logger.warn(+ `Failed to record the fair dispatch of job ${jobId} on queue ${queueName}: ${error instanceof Error ? error.message : String(error)}`,+ );+ }+ }++ private async runScript({+ queueName,+ script,+ keys,+ args,+ }: {+ queueName: MessageQueue;+ script: string;+ keys: string[];+ args: (string | number)[];+ }): Promise<unknown> {+ const client = await this.options.getClient(queueName);+ const scriptSha = this.getScriptSha(script);++ return client+ .runCommand('evalsha', [scriptSha, keys.length, ...keys, ...args])+ .catch((error: unknown) => {+ if (!(error instanceof Error && error.message.startsWith('NOSCRIPT'))) {+ throw error;+ }++ return client.runCommand('eval', [+ script,+ keys.length,+ ...keys,+ ...args,+ ]);+ });+ }++ private getScriptSha(script: string): string {+ const cachedScriptSha = this.scriptShas.get(script);++ if (isDefined(cachedScriptSha)) {+ return cachedScriptSha;+ }++ const scriptSha = createHash('sha1').update(script).digest('hex');++ this.scriptShas.set(script, scriptSha);++ return scriptSha;+ }}