Skip to content

The open alternative to Salesforce, designed for AI.

Advance the queue's virtual time when a fair job is dispatched (#28992)

martmull7be5668f48f5
diff --git a/packages/twenty-server/src/engine/core-modules/message-queue/drivers/__tests__/bullmq.driver.spec.ts b/packages/twenty-server/src/engine/core-modules/message-queue/drivers/__tests__/bullmq.driver.spec.ts
index f9859eaa145..415948e19e1 100644
--- a/packages/twenty-server/src/engine/core-modules/message-queue/drivers/__tests__/bullmq.driver.spec.ts
+++ b/packages/twenty-server/src/engine/core-modules/message-queue/drivers/__tests__/bullmq.driver.spec.ts
@@ -21,6 +21,7 @@ const mockRemoveGlobalConcurrency = jest.fn();
const mockGetJobCountByTypes = jest.fn();
const mockTwentyConfigService = { get: jest.fn().mockReturnValue(false) };
const mockAllocateTickets = jest.fn();
+const mockRecordDispatch = jest.fn();
jest.mock('bullmq', () => ({
Queue: jest.fn().mockImplementation(() => ({
@@ -43,9 +44,10 @@ jest.mock('uuid', () => ({ v4: () => 'generated-uuid-000000000000000000000' }));
jest.mock(
'src/engine/core-modules/message-queue/drivers/fairness/fair-ticket-allocator',
() => ({
- FairTicketAllocator: jest
- .fn()
- .mockImplementation(() => ({ allocateTickets: mockAllocateTickets })),
+ FairTicketAllocator: jest.fn().mockImplementation(() => ({
+ allocateTickets: mockAllocateTickets,
+ recordDispatch: mockRecordDispatch,
+ })),
}),
);
@@ -641,3 +643,59 @@ describe('BullMQDriver fair tickets', () => {
);
});
});
+
+describe('BullMQDriver fair dispatch', () => {
+ const processJob = async ({
+ isFairnessEnabled,
+ }: {
+ isFairnessEnabled: boolean;
+ }) => {
+ const driver = new BullMQDriver(
+ {} as never,
+ { recordHistogram: jest.fn() } as never,
+ {
+ get: jest.fn(
+ (key: string) =>
+ key === 'IS_WORKER_FAIRNESS_ENABLED' && isFairnessEnabled,
+ ),
+ } as never,
+ {} as never,
+ );
+
+ driver.register(MessageQueue.workflowQueue);
+ jest.clearAllMocks();
+ driver.work(MessageQueue.workflowQueue, jest.fn());
+
+ const processor = jest.mocked(Worker).mock.calls[0][1];
+
+ if (typeof processor !== 'function') {
+ throw new Error('Worker processor was not registered');
+ }
+
+ await processor({
+ id: 'job-id',
+ name: 'job',
+ data: {},
+ opts: {},
+ timestamp: Date.now(),
+ attemptsStarted: 1,
+ updateData: jest.fn(),
+ updateProgress: jest.fn(),
+ } as unknown as Job);
+ };
+
+ it('records the dispatch of a job when fairness is on', async () => {
+ await processJob({ isFairnessEnabled: true });
+
+ expect(mockRecordDispatch).toHaveBeenCalledWith({
+ queueName: MessageQueue.workflowQueue,
+ jobId: 'job-id',
+ });
+ });
+
+ it('records nothing when fairness is off', async () => {
+ await processJob({ isFairnessEnabled: false });
+
+ expect(mockRecordDispatch).not.toHaveBeenCalled();
+ });
+});
diff --git a/packages/twenty-server/src/engine/core-modules/message-queue/drivers/bullmq.driver.ts b/packages/twenty-server/src/engine/core-modules/message-queue/drivers/bullmq.driver.ts
index 5f687f16cf8..70da33203ae 100644
--- a/packages/twenty-server/src/engine/core-modules/message-queue/drivers/bullmq.driver.ts
+++ b/packages/twenty-server/src/engine/core-modules/message-queue/drivers/bullmq.driver.ts
@@ -85,6 +85,8 @@ export class BullMQDriver
) {
this.fairQueueScheduler = new FairQueueScheduler({
getClient: (queueName) => this.queueMap[queueName].client,
+ getJobKey: ({ queueName, jobId }) =>
+ this.queueMap[queueName].toKey(jobId),
keyPrefix: options.prefix ?? 'bull',
metricsService,
twentyConfigService,
@@ -288,6 +290,10 @@ export class BullMQDriver
Sentry.withIsolationScope(async () => {
applyWorkspaceSentryContextFromJobData(job.data);
+ if (isDefined(job.id)) {
+ this.fairQueueScheduler.onJobStarted({ queueName, jobId: job.id });
+ }
+
const queueWaitMs = computeJobQueueWaitMs({ job, now: Date.now() });
if (isDefined(queueWaitMs)) {
diff --git a/packages/twenty-server/src/engine/core-modules/message-queue/drivers/fairness/__tests__/fair-ticket-allocator.spec.ts b/packages/twenty-server/src/engine/core-modules/message-queue/drivers/fairness/__tests__/fair-ticket-allocator.spec.ts
index 93bba9f8ded..d51189abec2 100644
--- a/packages/twenty-server/src/engine/core-modules/message-queue/drivers/fairness/__tests__/fair-ticket-allocator.spec.ts
+++ b/packages/twenty-server/src/engine/core-modules/message-queue/drivers/fairness/__tests__/fair-ticket-allocator.spec.ts
@@ -4,6 +4,10 @@ import { type Redis } from 'ioredis';
import RedisMock from 'ioredis-mock';
import { FairTicketAllocator } from 'src/engine/core-modules/message-queue/drivers/fairness/fair-ticket-allocator';
+import {
+ FAIR_TICKET_OFFSET,
+ URGENT_FAIR_PRIORITY,
+} from 'src/engine/core-modules/message-queue/drivers/fairness/fair-priority.constant';
import { MessageQueue } from 'src/engine/core-modules/message-queue/message-queue.constants';
const QUEUE = MessageQueue.logicFunctionQueue;
@@ -12,6 +16,10 @@ const QUEUE_KEY_PREFIX = `${KEY_PREFIX}:${QUEUE}:`;
const VIRTUAL_TIME_KEY = `${QUEUE_KEY_PREFIX}fair:vtime`;
const WEIGHTS_KEY = `${QUEUE_KEY_PREFIX}fair:weights`;
+const ticketOf = (tag: number) => FAIR_TICKET_OFFSET + tag;
+
+const jobKeyOf = (jobId: string) => `${QUEUE_KEY_PREFIX}${jobId}`;
+
const createAllocator = async () => {
const redis: Redis = new RedisMock();
@@ -45,10 +53,14 @@ const createAllocator = async () => {
redis,
allocator: new FairTicketAllocator({
getClient: async () => client as never,
+ getJobKey: ({ jobId }) => jobKeyOf(jobId),
keyPrefix: KEY_PREFIX,
}),
readClock: async (groupKey: string) =>
Number(await redis.get(`${QUEUE_KEY_PREFIX}fair:clock:${groupKey}`)),
+ readVirtualTime: async () => Number(await redis.get(VIRTUAL_TIME_KEY)),
+ seedJob: async ({ jobId, priority }: { jobId: string; priority: number }) =>
+ redis.hset(jobKeyOf(jobId), 'priority', priority),
};
};
@@ -57,6 +69,7 @@ const createBrokenAllocator = () =>
getClient: async () => {
throw new Error('Redis connection lost');
},
+ getJobKey: ({ jobId }) => jobKeyOf(jobId),
keyPrefix: KEY_PREFIX,
});
@@ -137,6 +150,7 @@ describe('FairTicketAllocator', () => {
.mockRejectedValue(new Error('READONLY replica is read only'));
const allocator = new FairTicketAllocator({
getClient: async () => ({ runCommand }) as never,
+ getJobKey: ({ jobId }) => jobKeyOf(jobId),
keyPrefix: KEY_PREFIX,
});
@@ -150,4 +164,46 @@ describe('FairTicketAllocator', () => {
expect(runCommand).toHaveBeenCalledWith('evalsha', expect.any(Array));
});
});
+
+ describe('recordDispatch', () => {
+ it('advances the virtual time from the dispatched job hash and leaves urgent and missing jobs alone', async () => {
+ const { allocator, seedJob, readVirtualTime } = await createAllocator();
+
+ await seedJob({ jobId: 'A1', priority: ticketOf(5) });
+ await seedJob({ jobId: 'urgent', priority: URGENT_FAIR_PRIORITY });
+
+ await allocator.recordDispatch({ queueName: QUEUE, jobId: 'A1' });
+ await allocator.recordDispatch({ queueName: QUEUE, jobId: 'urgent' });
+ await allocator.recordDispatch({ queueName: QUEUE, jobId: 'missing' });
+
+ expect(await readVirtualTime()).toBe(5);
+ expect(
+ await allocator.allocateTickets({
+ queueName: QUEUE,
+ requests: [{ groupKeys: ['workspace:B'], jobCount: 1 }],
+ }),
+ ).toEqual([{ tag: 5, weight: 1 }]);
+ });
+
+ it('never moves the virtual time back', async () => {
+ const { allocator, seedJob, readVirtualTime } = await createAllocator();
+
+ await seedJob({ jobId: 'late', priority: ticketOf(9) });
+ await seedJob({ jobId: 'early', priority: ticketOf(3) });
+
+ await allocator.recordDispatch({ queueName: QUEUE, jobId: 'late' });
+ await allocator.recordDispatch({ queueName: QUEUE, jobId: 'early' });
+
+ expect(await readVirtualTime()).toBe(9);
+ });
+
+ it('swallows a Redis failure', async () => {
+ await expect(
+ createBrokenAllocator().recordDispatch({
+ queueName: QUEUE,
+ jobId: 'j',
+ }),
+ ).resolves.toBeUndefined();
+ });
+ });
});
diff --git a/packages/twenty-server/src/engine/core-modules/message-queue/drivers/fairness/fair-dispatch-script.constant.ts b/packages/twenty-server/src/engine/core-modules/message-queue/drivers/fairness/fair-dispatch-script.constant.ts
new file mode 100644
index 00000000000..9ea92c6dbd1
--- /dev/null
+++ b/packages/twenty-server/src/engine/core-modules/message-queue/drivers/fairness/fair-dispatch-script.constant.ts
@@ -0,0 +1,16 @@
+export const FAIR_DISPATCH_SCRIPT = `
+local ticketOffset = tonumber(ARGV[1])
+local priority = tonumber(redis.call('HGET', KEYS[2], 'priority'))
+
+if priority == nil or priority < ticketOffset then
+ return 0
+end
+
+local ticket = priority - ticketOffset
+
+if ticket > tonumber(redis.call('GET', KEYS[1]) or '0') then
+ redis.call('SET', KEYS[1], ticket)
+end
+
+return 0
+`;
diff --git a/packages/twenty-server/src/engine/core-modules/message-queue/drivers/fairness/fair-queue-scheduler.ts b/packages/twenty-server/src/engine/core-modules/message-queue/drivers/fairness/fair-queue-scheduler.ts
index f54f8ecd67f..ba4f0a32363 100644
--- a/packages/twenty-server/src/engine/core-modules/message-queue/drivers/fairness/fair-queue-scheduler.ts
+++ b/packages/twenty-server/src/engine/core-modules/message-queue/drivers/fairness/fair-queue-scheduler.ts
@@ -15,6 +15,7 @@ import { type TwentyConfigService } from 'src/engine/core-modules/twenty-config/
type FairQueueSchedulerOptions = {
getClient: (queueName: MessageQueue) => Promise<RedisClient>;
+ getJobKey: (job: { queueName: MessageQueue; jobId: string }) => string;
keyPrefix: string;
metricsService: MetricsService;
twentyConfigService: TwentyConfigService;
@@ -26,6 +27,7 @@ export class FairQueueScheduler {
constructor(private readonly options: FairQueueSchedulerOptions) {
this.fairTicketAllocator = new FairTicketAllocator({
getClient: options.getClient,
+ getJobKey: options.getJobKey,
keyPrefix: options.keyPrefix,
});
}
@@ -106,6 +108,20 @@ export class FairQueueScheduler {
return priorities;
}
+ onJobStarted({
+ queueName,
+ jobId,
+ }: {
+ queueName: MessageQueue;
+ jobId: string;
+ }): void {
+ if (!this.isFairnessEnabled()) {
+ return;
+ }
+
+ void this.fairTicketAllocator.recordDispatch({ queueName, jobId });
+ }
+
private isFairnessEnabled(): boolean {
return this.options.twentyConfigService.get('IS_WORKER_FAIRNESS_ENABLED');
}
diff --git a/packages/twenty-server/src/engine/core-modules/message-queue/drivers/fairness/fair-ticket-allocator.ts b/packages/twenty-server/src/engine/core-modules/message-queue/drivers/fairness/fair-ticket-allocator.ts
index 4b50f1920b1..41ac7caf3df 100644
--- a/packages/twenty-server/src/engine/core-modules/message-queue/drivers/fairness/fair-ticket-allocator.ts
+++ b/packages/twenty-server/src/engine/core-modules/message-queue/drivers/fairness/fair-ticket-allocator.ts
@@ -3,8 +3,13 @@ import { createHash } from 'node:crypto';
import { Logger } from '@nestjs/common';
import { type RedisClient } from 'bullmq';
+import { isDefined } from 'twenty-shared/utils';
-import { FAIR_CLOCK_TTL_MS } from 'src/engine/core-modules/message-queue/drivers/fairness/fair-priority.constant';
+import { FAIR_DISPATCH_SCRIPT } from 'src/engine/core-modules/message-queue/drivers/fairness/fair-dispatch-script.constant';
+import {
+ FAIR_CLOCK_TTL_MS,
+ FAIR_TICKET_OFFSET,
+} from 'src/engine/core-modules/message-queue/drivers/fairness/fair-priority.constant';
import { FAIR_TAG_SCRIPT } from 'src/engine/core-modules/message-queue/drivers/fairness/fair-tag-script.constant';
import { type MessageQueue } from 'src/engine/core-modules/message-queue/message-queue.constants';
@@ -14,15 +19,13 @@ export type FairTicket = { tag: number; weight: number };
type FairTicketAllocatorOptions = {
getClient: (queueName: MessageQueue) => Promise<RedisClient>;
+ getJobKey: (job: { queueName: MessageQueue; jobId: string }) => string;
keyPrefix: string;
};
-const FAIR_TAG_SCRIPT_SHA = createHash('sha1')
- .update(FAIR_TAG_SCRIPT)
- .digest('hex');
-
export class FairTicketAllocator {
private readonly logger = new Logger(FairTicketAllocator.name);
+ private readonly scriptShas = new Map<string, string>();
constructor(private readonly options: FairTicketAllocatorOptions) {}
@@ -40,48 +43,30 @@ export class FairTicketAllocator {
const groupKeyIndexes = new Map(
distinctGroupKeys.map((groupKey, index) => [groupKey, index + 1]),
);
- const keys = [
- `${fairKeyPrefix}vtime`,
- `${fairKeyPrefix}weights`,
- ...distinctGroupKeys.map(
- (groupKey) => `${fairKeyPrefix}clock:${groupKey}`,
- ),
- ];
- const args = [
- FAIR_CLOCK_TTL_MS,
- distinctGroupKeys.length,
- ...distinctGroupKeys,
- requests.length,
- ...requests.flatMap(({ groupKeys, jobCount }) => [
- jobCount,
- groupKeys.length,
- ...groupKeys.map((groupKey) => groupKeyIndexes.get(groupKey) ?? 0),
- ]),
- ];
try {
- const client = await this.options.getClient(queueName);
- const rawTickets = await client
- .runCommand('evalsha', [
- FAIR_TAG_SCRIPT_SHA,
- keys.length,
- ...keys,
- ...args,
- ])
- .catch((error: unknown) => {
- if (
- !(error instanceof Error && error.message.startsWith('NOSCRIPT'))
- ) {
- throw error;
- }
-
- return client.runCommand('eval', [
- FAIR_TAG_SCRIPT,
- keys.length,
- ...keys,
- ...args,
- ]);
- });
+ const rawTickets = await this.runScript({
+ queueName,
+ script: FAIR_TAG_SCRIPT,
+ keys: [
+ `${fairKeyPrefix}vtime`,
+ `${fairKeyPrefix}weights`,
+ ...distinctGroupKeys.map(
+ (groupKey) => `${fairKeyPrefix}clock:${groupKey}`,
+ ),
+ ],
+ args: [
+ FAIR_CLOCK_TTL_MS,
+ distinctGroupKeys.length,
+ ...distinctGroupKeys,
+ requests.length,
+ ...requests.flatMap(({ groupKeys, jobCount }) => [
+ jobCount,
+ groupKeys.length,
+ ...groupKeys.map((groupKey) => groupKeyIndexes.get(groupKey) ?? 0),
+ ]),
+ ],
+ });
const values = String(rawTickets).split(' ').map(Number);
const tickets = requests.map((_, index) => ({
tag: values[2 * index],
@@ -104,4 +89,74 @@ export class FairTicketAllocator {
return undefined;
}
}
+
+ async recordDispatch({
+ queueName,
+ jobId,
+ }: {
+ queueName: MessageQueue;
+ jobId: string;
+ }): Promise<void> {
+ const fairKeyPrefix = `${this.options.keyPrefix}:${queueName}:fair:`;
+
+ try {
+ await this.runScript({
+ queueName,
+ script: FAIR_DISPATCH_SCRIPT,
+ keys: [
+ `${fairKeyPrefix}vtime`,
+ this.options.getJobKey({ queueName, jobId }),
+ ],
+ args: [FAIR_TICKET_OFFSET],
+ });
+ } catch (error) {
+ this.logger.warn(
+ `Failed to record the fair dispatch of job ${jobId} on queue ${queueName}: ${error instanceof Error ? error.message : String(error)}`,
+ );
+ }
+ }
+
+ private async runScript({
+ queueName,
+ script,
+ keys,
+ args,
+ }: {
+ queueName: MessageQueue;
+ script: string;
+ keys: string[];
+ args: (string | number)[];
+ }): Promise<unknown> {
+ const client = await this.options.getClient(queueName);
+ const scriptSha = this.getScriptSha(script);
+
+ return client
+ .runCommand('evalsha', [scriptSha, keys.length, ...keys, ...args])
+ .catch((error: unknown) => {
+ if (!(error instanceof Error && error.message.startsWith('NOSCRIPT'))) {
+ throw error;
+ }
+
+ return client.runCommand('eval', [
+ script,
+ keys.length,
+ ...keys,
+ ...args,
+ ]);
+ });
+ }
+
+ private getScriptSha(script: string): string {
+ const cachedScriptSha = this.scriptShas.get(script);
+
+ if (isDefined(cachedScriptSha)) {
+ return cachedScriptSha;
+ }
+
+ const scriptSha = createHash('sha1').update(script).digest('hex');
+
+ this.scriptShas.set(script, scriptSha);
+
+ return scriptSha;
+ }
}