Skip to content

The open alternative to Salesforce, designed for AI.

Rebase the fair tickets in bounded chunks once the virtual time passes 2^20 (#28994)

martmull9c2ef10c7b89
diff --git a/packages/twenty-server/src/engine/core-modules/message-queue/drivers/__tests__/bullmq.driver.spec.ts b/packages/twenty-server/src/engine/core-modules/message-queue/drivers/__tests__/bullmq.driver.spec.ts
index 415948e19e1..d6f4065355b 100644
--- a/packages/twenty-server/src/engine/core-modules/message-queue/drivers/__tests__/bullmq.driver.spec.ts
+++ b/packages/twenty-server/src/engine/core-modules/message-queue/drivers/__tests__/bullmq.driver.spec.ts
@@ -4,6 +4,7 @@ import { DEFAULT_JOB_PRIORITY } from 'src/engine/core-modules/message-queue/cons
import { QUEUE_JOB_CHANGED_EVENT } from 'src/engine/core-modules/message-queue/constants/queue-job-changed-event.constant';
import { BullMQDriver } from 'src/engine/core-modules/message-queue/drivers/bullmq.driver';
import {
+ FAIR_REBASE_STEP,
FAIR_TICKET_OFFSET,
URGENT_FAIR_PRIORITY,
} from 'src/engine/core-modules/message-queue/drivers/fairness/fair-priority.constant';
@@ -542,7 +543,10 @@ describe('BullMQDriver fair tickets', () => {
});
it('ranks a job by the ticket of its workspace', async () => {
- mockAllocateTickets.mockResolvedValue([{ tag: 7, weight: 1 }]);
+ mockAllocateTickets.mockResolvedValue({
+ base: FAIR_REBASE_STEP,
+ tickets: [{ tag: 7, weight: 1 }],
+ });
await driver.add(MessageQueue.workspaceQueue, 'job', {
workspaceId: 'ws-1',
@@ -555,15 +559,21 @@ describe('BullMQDriver fair tickets', () => {
expect(mockAdd).toHaveBeenCalledWith(
'job',
{ workspaceId: 'ws-1' },
- expect.objectContaining({ priority: FAIR_TICKET_OFFSET + 7 }),
+ expect.objectContaining({
+ priority: FAIR_TICKET_OFFSET + 7,
+ fairBase: FAIR_REBASE_STEP,
+ }),
);
});
it('allocates one ticket per group of a bulk and spaces its jobs by the weight', async () => {
- mockAllocateTickets.mockResolvedValue([
- { tag: 10, weight: 0.5 },
- { tag: 0, weight: 1 },
- ]);
+ mockAllocateTickets.mockResolvedValue({
+ base: 0,
+ tickets: [
+ { tag: 10, weight: 0.5 },
+ { tag: 0, weight: 1 },
+ ],
+ });
await driver.bulkAdd(MessageQueue.workspaceQueue, 'job', [
{ data: { workspaceId: 'ws-1' } },
@@ -590,7 +600,10 @@ describe('BullMQDriver fair tickets', () => {
});
it('keeps priority 1 ahead of the fair tickets while still charging its group keys', async () => {
- mockAllocateTickets.mockResolvedValue([{ tag: 7, weight: 1 }]);
+ mockAllocateTickets.mockResolvedValue({
+ base: 0,
+ tickets: [{ tag: 7, weight: 1 }],
+ });
await driver.add(
MessageQueue.workspaceQueue,
diff --git a/packages/twenty-server/src/engine/core-modules/message-queue/drivers/bullmq.driver.ts b/packages/twenty-server/src/engine/core-modules/message-queue/drivers/bullmq.driver.ts
index 70da33203ae..fd5fc2bcc26 100644
--- a/packages/twenty-server/src/engine/core-modules/message-queue/drivers/bullmq.driver.ts
+++ b/packages/twenty-server/src/engine/core-modules/message-queue/drivers/bullmq.driver.ts
@@ -57,6 +57,7 @@ export type BullMQDriverOptions = QueueOptions;
type BullMQJobsOptions = JobsOptions & {
broadcastTo?: QueueJobRecipient;
+ fairBase?: number;
};
export class BullMQDriver
@@ -535,7 +536,7 @@ export class BullMQDriver
}
const queueOptions = this.buildJobsOptions(options);
- const [priority] = await this.fairQueueScheduler.computeJobPriorities({
+ const [jobOptions] = await this.fairQueueScheduler.computeJobOptions({
queueName,
jobsData: [data],
options,
@@ -544,7 +545,7 @@ export class BullMQDriver
const job = await this.queueMap[queueName].add(jobName, data, {
...queueOptions,
- priority,
+ ...jobOptions,
});
return job.id;
@@ -596,7 +597,7 @@ export class BullMQDriver
}
const queueOptions = this.buildJobsOptions(options);
- const priorities = await this.fairQueueScheduler.computeJobPriorities({
+ const jobOptions = await this.fairQueueScheduler.computeJobOptions({
queueName,
jobsData: jobsToAdd.map(({ data }) => data),
options,
@@ -609,7 +610,7 @@ export class BullMQDriver
data,
opts: {
...queueOptions,
- priority: priorities[index],
+ ...jobOptions[index],
jobId:
jobId ??
(queueOptions.jobId ? `${queueOptions.jobId}-${index}` : undefined),
diff --git a/packages/twenty-server/src/engine/core-modules/message-queue/drivers/fairness/__tests__/fair-ticket-allocator.spec.ts b/packages/twenty-server/src/engine/core-modules/message-queue/drivers/fairness/__tests__/fair-ticket-allocator.spec.ts
index d51189abec2..95a9917d286 100644
--- a/packages/twenty-server/src/engine/core-modules/message-queue/drivers/fairness/__tests__/fair-ticket-allocator.spec.ts
+++ b/packages/twenty-server/src/engine/core-modules/message-queue/drivers/fairness/__tests__/fair-ticket-allocator.spec.ts
@@ -1,10 +1,15 @@
+import { readFileSync } from 'node:fs';
+
import { Logger } from '@nestjs/common';
import { type Redis } from 'ioredis';
import RedisMock from 'ioredis-mock';
+import { isDefined } from 'twenty-shared/utils';
import { FairTicketAllocator } from 'src/engine/core-modules/message-queue/drivers/fairness/fair-ticket-allocator';
import {
+ FAIR_REBASE_STEP,
+ FAIR_REBASE_THRESHOLD,
FAIR_TICKET_OFFSET,
URGENT_FAIR_PRIORITY,
} from 'src/engine/core-modules/message-queue/drivers/fairness/fair-priority.constant';
@@ -14,13 +19,23 @@ const QUEUE = MessageQueue.logicFunctionQueue;
const KEY_PREFIX = 'bull';
const QUEUE_KEY_PREFIX = `${KEY_PREFIX}:${QUEUE}:`;
const VIRTUAL_TIME_KEY = `${QUEUE_KEY_PREFIX}fair:vtime`;
+const BASE_KEY = `${QUEUE_KEY_PREFIX}fair:base`;
+const REBASE_KEY = `${QUEUE_KEY_PREFIX}fair:rebase`;
const WEIGHTS_KEY = `${QUEUE_KEY_PREFIX}fair:weights`;
+const PRIORITIZED_KEY = `${QUEUE_KEY_PREFIX}prioritized`;
+const DELAYED_KEY = `${QUEUE_KEY_PREFIX}delayed`;
+const ACTIVE_KEY = `${QUEUE_KEY_PREFIX}active`;
+const PRIORITY_SCORE_BASE = 2 ** 32;
+const BULLMQ_PRIORITY_SCORE_LUA =
+ require.resolve('bullmq/dist/cjs/commands/includes/getPriorityScore.lua');
const ticketOf = (tag: number) => FAIR_TICKET_OFFSET + tag;
const jobKeyOf = (jobId: string) => `${QUEUE_KEY_PREFIX}${jobId}`;
-const createAllocator = async () => {
+const createAllocator = async ({
+ rebaseGraceMs = 0,
+}: { rebaseGraceMs?: number } = {}) => {
const redis: Redis = new RedisMock();
await redis.flushall();
@@ -48,6 +63,49 @@ const createAllocator = async () => {
);
},
};
+ const seedJob = async ({
+ jobId,
+ priority,
+ counter,
+ state,
+ base = 0,
+ stampedBase,
+ }: {
+ jobId: string;
+ priority: number;
+ counter: number;
+ state: 'prioritized' | 'delayed' | 'active';
+ base?: number | 'none';
+ stampedBase?: number;
+ }) => {
+ const jobKey = jobKeyOf(jobId);
+
+ await redis.hset(jobKey, 'priority', priority);
+
+ if (base !== 'none') {
+ await redis.hset(
+ jobKey,
+ 'opts',
+ JSON.stringify({ priority, fairBase: base }),
+ );
+ }
+
+ if (isDefined(stampedBase)) {
+ await redis.hset(jobKey, 'fairBase', stampedBase);
+ }
+
+ if (state === 'prioritized') {
+ await redis.zadd(
+ PRIORITIZED_KEY,
+ priority * PRIORITY_SCORE_BASE + counter,
+ jobId,
+ );
+ } else if (state === 'delayed') {
+ await redis.zadd(DELAYED_KEY, counter, jobId);
+ } else {
+ await redis.rpush(ACTIVE_KEY, jobId);
+ }
+ };
return {
redis,
@@ -55,12 +113,28 @@ const createAllocator = async () => {
getClient: async () => client as never,
getJobKey: ({ jobId }) => jobKeyOf(jobId),
keyPrefix: KEY_PREFIX,
+ rebaseGraceMs,
}),
- readClock: async (groupKey: string) =>
- Number(await redis.get(`${QUEUE_KEY_PREFIX}fair:clock:${groupKey}`)),
+ seedJob,
+ readClock: async (groupKey: string) => {
+ const raw = await redis.get(`${QUEUE_KEY_PREFIX}fair:clock:${groupKey}`);
+
+ return raw?.split(':').map(Number);
+ },
+ readPriority: async (jobId: string) =>
+ Number(await redis.hget(jobKeyOf(jobId), 'priority')),
+ readStampedBase: async (jobId: string) => {
+ const raw = await redis.hget(jobKeyOf(jobId), 'fairBase');
+
+ return isDefined(raw) ? Number(raw) : undefined;
+ },
readVirtualTime: async () => Number(await redis.get(VIRTUAL_TIME_KEY)),
- seedJob: async ({ jobId, priority }: { jobId: string; priority: number }) =>
- redis.hset(jobKeyOf(jobId), 'priority', priority),
+ readBase: async () => Number((await redis.get(BASE_KEY)) ?? 0),
+ isRebasing: async () => (await redis.exists(REBASE_KEY)) === 1,
+ readPrioritizedWithScores: async () =>
+ (await redis.zrange(PRIORITIZED_KEY, 0, -1, 'WITHSCORES')).map(
+ (value, index) => (index % 2 === 1 ? Number(value) : value),
+ ),
};
};
@@ -75,6 +149,8 @@ const createBrokenAllocator = () =>
describe('FairTicketAllocator', () => {
beforeAll(() => {
+ jest.useRealTimers();
+ jest.spyOn(Logger.prototype, 'log').mockImplementation(() => undefined);
jest.spyOn(Logger.prototype, 'warn').mockImplementation(() => undefined);
});
@@ -91,13 +167,16 @@ describe('FairTicketAllocator', () => {
{ groupKeys: ['workspace:W', 'application:Q'], jobCount: 1 },
],
}),
- ).toEqual([
- { tag: 0, weight: 1 },
- { tag: 100, weight: 1 },
- { tag: 1, weight: 1 },
- ]);
- expect(await readClock('application:H')).toBe(101);
- expect(await readClock('workspace:W')).toBe(2);
+ ).toEqual({
+ base: 0,
+ tickets: [
+ { tag: 0, weight: 1 },
+ { tag: 100, weight: 1 },
+ { tag: 1, weight: 1 },
+ ],
+ });
+ expect(await readClock('application:H')).toEqual([0, 101]);
+ expect(await readClock('workspace:W')).toEqual([0, 2]);
});
it('tags a request at the virtual time when it is ahead of its clocks', async () => {
@@ -110,8 +189,41 @@ describe('FairTicketAllocator', () => {
queueName: QUEUE,
requests: [{ groupKeys: ['workspace:B'], jobCount: 2 }],
}),
- ).toEqual([{ tag: 5, weight: 1 }]);
- expect(await readClock('workspace:B')).toBe(7);
+ ).toEqual({ base: 0, tickets: [{ tag: 5, weight: 1 }] });
+ expect(await readClock('workspace:B')).toEqual([0, 7]);
+ });
+
+ it('returns the base it allocated on and reads a clock stamped with an older base one step down', async () => {
+ const { redis, allocator, readClock } = await createAllocator();
+
+ await redis.set(VIRTUAL_TIME_KEY, 3);
+ await redis.set(BASE_KEY, FAIR_REBASE_STEP);
+ await redis.set(
+ `${QUEUE_KEY_PREFIX}fair:clock:workspace:S`,
+ `0:${FAIR_REBASE_STEP + 8}`,
+ );
+
+ expect(
+ await allocator.allocateTickets({
+ queueName: QUEUE,
+ requests: [{ groupKeys: ['workspace:S'], jobCount: 1 }],
+ }),
+ ).toEqual({ base: FAIR_REBASE_STEP, tickets: [{ tag: 8, weight: 1 }] });
+ expect(await readClock('workspace:S')).toEqual([FAIR_REBASE_STEP, 9]);
+ });
+
+ it('reads a clock written before the base existed on the current base', async () => {
+ const { redis, allocator, readClock } = await createAllocator();
+
+ await redis.set(`${QUEUE_KEY_PREFIX}fair:clock:workspace:L`, 7);
+
+ expect(
+ await allocator.allocateTickets({
+ queueName: QUEUE,
+ requests: [{ groupKeys: ['workspace:L'], jobCount: 1 }],
+ }),
+ ).toEqual({ base: 0, tickets: [{ tag: 7, weight: 1 }] });
+ expect(await readClock('workspace:L')).toEqual([0, 8]);
});
it('weighs a request by the product of its key weights and ignores a malformed weight', async () => {
@@ -129,10 +241,13 @@ describe('FairTicketAllocator', () => {
{ groupKeys: ['workspace:W', 'application:N'], jobCount: 1 },
],
}),
- ).toEqual([
- { tag: 0, weight: 2 },
- { tag: 1, weight: 1 },
- ]);
+ ).toEqual({
+ base: 0,
+ tickets: [
+ { tag: 0, weight: 2 },
+ { tag: 1, weight: 1 },
+ ],
+ });
});
it('returns nothing when Redis is unavailable', async () => {
@@ -166,11 +281,27 @@ describe('FairTicketAllocator', () => {
});
describe('recordDispatch', () => {
+ it('relies on the priority score layout of the installed BullMQ', () => {
+ expect(readFileSync(BULLMQ_PRIORITY_SCORE_LUA, 'utf8')).toContain(
+ 'priority * 0x100000000 + prioCounter % 0x100000000',
+ );
+ });
+
it('advances the virtual time from the dispatched job hash and leaves urgent and missing jobs alone', async () => {
const { allocator, seedJob, readVirtualTime } = await createAllocator();
- await seedJob({ jobId: 'A1', priority: ticketOf(5) });
- await seedJob({ jobId: 'urgent', priority: URGENT_FAIR_PRIORITY });
+ await seedJob({
+ jobId: 'A1',
+ priority: ticketOf(5),
+ counter: 1,
+ state: 'active',
+ });
+ await seedJob({
+ jobId: 'urgent',
+ priority: URGENT_FAIR_PRIORITY,
+ counter: 2,
+ state: 'active',
+ });
await allocator.recordDispatch({ queueName: QUEUE, jobId: 'A1' });
await allocator.recordDispatch({ queueName: QUEUE, jobId: 'urgent' });
@@ -182,14 +313,24 @@ describe('FairTicketAllocator', () => {
queueName: QUEUE,
requests: [{ groupKeys: ['workspace:B'], jobCount: 1 }],
}),
- ).toEqual([{ tag: 5, weight: 1 }]);
+ ).toEqual({ base: 0, tickets: [{ tag: 5, weight: 1 }] });
});
it('never moves the virtual time back', async () => {
const { allocator, seedJob, readVirtualTime } = await createAllocator();
- await seedJob({ jobId: 'late', priority: ticketOf(9) });
- await seedJob({ jobId: 'early', priority: ticketOf(3) });
+ await seedJob({
+ jobId: 'late',
+ priority: ticketOf(9),
+ counter: 1,
+ state: 'active',
+ });
+ await seedJob({
+ jobId: 'early',
+ priority: ticketOf(3),
+ counter: 2,
+ state: 'active',
+ });
await allocator.recordDispatch({ queueName: QUEUE, jobId: 'late' });
await allocator.recordDispatch({ queueName: QUEUE, jobId: 'early' });
@@ -197,6 +338,376 @@ describe('FairTicketAllocator', () => {
expect(await readVirtualTime()).toBe(9);
});
+ it('rewrites the waiting tickets in chunks once the threshold is passed, keeping the order and leaving urgent and static priorities alone', async () => {
+ const {
+ redis,
+ allocator,
+ seedJob,
+ readPriority,
+ readStampedBase,
+ readVirtualTime,
+ readBase,
+ isRebasing,
+ readPrioritizedWithScores,
+ readClock,
+ } = await createAllocator();
+ const fillerCount = 1200;
+
+ await redis.set(VIRTUAL_TIME_KEY, FAIR_REBASE_THRESHOLD - 1);
+ await redis.set(
+ `${QUEUE_KEY_PREFIX}fair:clock:workspace:A`,
+ `0:${FAIR_REBASE_THRESHOLD + 3}`,
+ );
+ await seedJob({
+ jobId: 'urgent',
+ priority: URGENT_FAIR_PRIORITY,
+ counter: 1,
+ state: 'prioritized',
+ });
+ await seedJob({
+ jobId: 'static',
+ priority: 4,
+ counter: 2,
+ state: 'prioritized',
+ base: 'none',
+ });
+ await seedJob({
+ jobId: 'A2',
+ priority: ticketOf(FAIR_REBASE_THRESHOLD + 1),
+ counter: 3,
+ state: 'prioritized',
+ });
+ await seedJob({
+ jobId: 'A3',
+ priority: ticketOf(FAIR_REBASE_THRESHOLD + 2),
+ counter: 4,
+ state: 'prioritized',
+ });
+ for (let index = 0; index < fillerCount; index += 1) {
+ await seedJob({
+ jobId: `F${index}`,
+ priority: ticketOf(FAIR_REBASE_THRESHOLD + 10 + index),
+ counter: 100 + index,
+ state: 'prioritized',
+ });
+ }
+ await seedJob({
+ jobId: 'retrying',
+ priority: ticketOf(FAIR_REBASE_THRESHOLD + 5),
+ counter: 5,
+ state: 'delayed',
+ });
+ await seedJob({
+ jobId: 'running',
+ priority: ticketOf(FAIR_REBASE_THRESHOLD),
+ counter: 6,
+ state: 'active',
+ });
+
+ await allocator.recordDispatch({ queueName: QUEUE, jobId: 'running' });
+ await allocator.rebase(QUEUE);
+
+ expect(await readVirtualTime()).toBe(
+ FAIR_REBASE_THRESHOLD - FAIR_REBASE_STEP,
+ );
+ expect(await readBase()).toBe(FAIR_REBASE_STEP);
+ expect(await isRebasing()).toBe(false);
+ expect((await readPrioritizedWithScores()).slice(0, 8)).toEqual([
+ 'urgent',
+ URGENT_FAIR_PRIORITY * PRIORITY_SCORE_BASE + 1,
+ 'static',
+ 4 * PRIORITY_SCORE_BASE + 2,
+ 'A2',
+ ticketOf(FAIR_REBASE_THRESHOLD + 1 - FAIR_REBASE_STEP) *
+ PRIORITY_SCORE_BASE +
+ 3,
+ 'A3',
+ ticketOf(FAIR_REBASE_THRESHOLD + 2 - FAIR_REBASE_STEP) *
+ PRIORITY_SCORE_BASE +
+ 4,
+ ]);
+ expect(await redis.zcard(PRIORITIZED_KEY)).toBe(fillerCount + 4);
+ expect(await readPriority(`F${fillerCount - 1}`)).toBe(
+ ticketOf(
+ FAIR_REBASE_THRESHOLD + 10 + fillerCount - 1 - FAIR_REBASE_STEP,
+ ),
+ );
+ expect(await readStampedBase('A2')).toBe(FAIR_REBASE_STEP);
+ expect(await readPriority('static')).toBe(4);
+ expect(await readStampedBase('static')).toBeUndefined();
+ expect(await readPriority('retrying')).toBe(
+ ticketOf(FAIR_REBASE_THRESHOLD + 5 - FAIR_REBASE_STEP),
+ );
+ expect(await readPriority('running')).toBe(
+ ticketOf(FAIR_REBASE_THRESHOLD - FAIR_REBASE_STEP),
+ );
+ expect(
+ await allocator.allocateTickets({
+ queueName: QUEUE,
+ requests: [{ groupKeys: ['workspace:A'], jobCount: 1 }],
+ }),
+ ).toEqual({
+ base: FAIR_REBASE_STEP,
+ tickets: [
+ { tag: FAIR_REBASE_THRESHOLD + 3 - FAIR_REBASE_STEP, weight: 1 },
+ ],
+ });
+ expect(await readClock('workspace:A')).toEqual([
+ FAIR_REBASE_STEP,
+ FAIR_REBASE_THRESHOLD + 4 - FAIR_REBASE_STEP,
+ ]);
+ });
+
+ it('shifts every ticket of a backlog deeper than a step exactly once', async () => {
+ const { redis, allocator, seedJob, readPrioritizedWithScores } =
+ await createAllocator();
+
+ await redis.set(VIRTUAL_TIME_KEY, FAIR_REBASE_THRESHOLD - 1);
+ for (const depth of [1, 2, 3]) {
+ await seedJob({
+ jobId: `deep${depth}`,
+ priority: ticketOf(FAIR_REBASE_THRESHOLD + FAIR_REBASE_STEP + depth),
+ counter: depth,
+ state: 'prioritized',
+ });
+ }
+ await seedJob({
+ jobId: 'running',
+ priority: ticketOf(FAIR_REBASE_THRESHOLD),
+ counter: 9,
+ state: 'active',
+ });
+
+ await allocator.recordDispatch({ queueName: QUEUE, jobId: 'running' });
+ await allocator.rebase(QUEUE);
+
+ expect(await readPrioritizedWithScores()).toEqual([
+ 'deep1',
+ ticketOf(FAIR_REBASE_THRESHOLD + 1) * PRIORITY_SCORE_BASE + 1,
+ 'deep2',
+ ticketOf(FAIR_REBASE_THRESHOLD + 2) * PRIORITY_SCORE_BASE + 2,
+ 'deep3',
+ ticketOf(FAIR_REBASE_THRESHOLD + 3) * PRIORITY_SCORE_BASE + 3,
+ ]);
+ });
+
+ it('shifts a delayed retry that sits below the old virtual time and floors an overdue one at zero, keeping their order', async () => {
+ const { redis, allocator, seedJob, readPriority, readStampedBase } =
+ await createAllocator();
+
+ await redis.set(VIRTUAL_TIME_KEY, FAIR_REBASE_THRESHOLD - 1);
+ await seedJob({
+ jobId: 'retrying',
+ priority: ticketOf(FAIR_REBASE_THRESHOLD - FAIR_REBASE_STEP + 100),
+ counter: 1,
+ state: 'delayed',
+ });
+ await seedJob({
+ jobId: 'overdue',
+ priority: ticketOf(100),
+ counter: 2,
+ state: 'delayed',
+ });
+ await seedJob({
+ jobId: 'running',
+ priority: ticketOf(FAIR_REBASE_THRESHOLD),
+ counter: 3,
+ state: 'active',
+ });
+
+ await allocator.recordDispatch({ queueName: QUEUE, jobId: 'running' });
+ await allocator.rebase(QUEUE);
+
+ expect(await readPriority('retrying')).toBe(ticketOf(100));
+ expect(await readStampedBase('retrying')).toBe(FAIR_REBASE_STEP);
+ expect(await readPriority('overdue')).toBe(ticketOf(0));
+ expect(await readStampedBase('overdue')).toBe(FAIR_REBASE_STEP);
+ });
+
+ it('leaves a job already on the new scale and a ticket allocated before the base existed alone', async () => {
+ const { redis, allocator, seedJob, readPriority, readStampedBase } =
+ await createAllocator();
+
+ await redis.set(VIRTUAL_TIME_KEY, FAIR_REBASE_THRESHOLD - 1);
+ await seedJob({
+ jobId: 'fresh',
+ priority: ticketOf(FAIR_REBASE_THRESHOLD - FAIR_REBASE_STEP + 5),
+ counter: 1,
+ state: 'prioritized',
+ base: FAIR_REBASE_STEP,
+ });
+ await seedJob({
+ jobId: 'stamped',
+ priority: ticketOf(FAIR_REBASE_THRESHOLD - FAIR_REBASE_STEP + 6),
+ counter: 2,
+ state: 'prioritized',
+ stampedBase: FAIR_REBASE_STEP,
+ });
+ await seedJob({
+ jobId: 'legacy',
+ priority: ticketOf(FAIR_REBASE_THRESHOLD + 2),
+ counter: 3,
+ state: 'prioritized',
+ base: 'none',
+ });
+ await seedJob({
+ jobId: 'old',
+ priority: ticketOf(FAIR_REBASE_THRESHOLD + 3),
+ counter: 4,
+ state: 'prioritized',
+ });
+ await seedJob({
+ jobId: 'running',
+ priority: ticketOf(FAIR_REBASE_THRESHOLD),
+ counter: 5,
+ state: 'active',
+ });
+
+ await allocator.recordDispatch({ queueName: QUEUE, jobId: 'running' });
+ await allocator.rebase(QUEUE);
+
+ expect(await readPriority('old')).toBe(
+ ticketOf(FAIR_REBASE_THRESHOLD + 3 - FAIR_REBASE_STEP),
+ );
+ expect(await readPriority('fresh')).toBe(
+ ticketOf(FAIR_REBASE_THRESHOLD - FAIR_REBASE_STEP + 5),
+ );
+ expect(await readPriority('stamped')).toBe(
+ ticketOf(FAIR_REBASE_THRESHOLD - FAIR_REBASE_STEP + 6),
+ );
+ expect(await readPriority('legacy')).toBe(
+ ticketOf(FAIR_REBASE_THRESHOLD + 2),
+ );
+ expect(await readStampedBase('legacy')).toBeUndefined();
+ });
+
+ it('dispatches a job fetched before the switch without switching the scale again', async () => {
+ const {
+ redis,
+ allocator,
+ seedJob,
+ readPriority,
+ readVirtualTime,
+ readBase,
+ isRebasing,
+ } = await createAllocator();
+
+ await redis.set(VIRTUAL_TIME_KEY, FAIR_REBASE_THRESHOLD - 1);
+ await seedJob({
+ jobId: 'j1',
+ priority: ticketOf(FAIR_REBASE_THRESHOLD),
+ counter: 1,
+ state: 'active',
+ });
+ await seedJob({
+ jobId: 'j2',
+ priority: ticketOf(FAIR_REBASE_THRESHOLD + 3),
+ counter: 2,
+ state: 'active',
+ });
+
+ await allocator.recordDispatch({ queueName: QUEUE, jobId: 'j1' });
+ await allocator.rebase(QUEUE);
+ await allocator.recordDispatch({ queueName: QUEUE, jobId: 'j2' });
+
+ expect(await readBase()).toBe(FAIR_REBASE_STEP);
+ expect(await readVirtualTime()).toBe(
+ FAIR_REBASE_THRESHOLD + 3 - FAIR_REBASE_STEP,
+ );
+ expect(await readPriority('j2')).toBe(
+ ticketOf(FAIR_REBASE_THRESHOLD + 3 - FAIR_REBASE_STEP),
+ );
+ expect(await isRebasing()).toBe(false);
+ });
+
+ it('normalizes a ticket allocated before the switch and inserted after it', async () => {
+ const {
+ redis,
+ allocator,
+ seedJob,
+ readPriority,
+ readStampedBase,
+ readPrioritizedWithScores,
+ isRebasing,
+ } = await createAllocator({ rebaseGraceMs: 50 });
+
+ await redis.set(VIRTUAL_TIME_KEY, FAIR_REBASE_THRESHOLD - 1);
+ await seedJob({
+ jobId: 'x',
+ priority: ticketOf(FAIR_REBASE_THRESHOLD),
+ counter: 1,
+ state: 'active',
+ });
+
+ const allocation = await allocator.allocateTickets({
+ queueName: QUEUE,
+ requests: [{ groupKeys: ['workspace:late'], jobCount: 1 }],
+ });
+
+ expect(allocation).toEqual({
+ base: 0,
+ tickets: [{ tag: FAIR_REBASE_THRESHOLD - 1, weight: 1 }],
+ });
+
+ await allocator.recordDispatch({ queueName: QUEUE, jobId: 'x' });
+ await seedJob({
+ jobId: 'late',
+ priority: ticketOf(FAIR_REBASE_THRESHOLD - 1),
+ counter: 9,
+ state: 'prioritized',
+ });
+ await allocator.rebase(QUEUE);
+
+ expect(await readPriority('late')).toBe(
+ ticketOf(FAIR_REBASE_THRESHOLD - 1 - FAIR_REBASE_STEP),
+ );
+ expect(await readStampedBase('late')).toBe(FAIR_REBASE_STEP);
+ expect(await readPrioritizedWithScores()).toEqual([
+ 'late',
+ ticketOf(FAIR_REBASE_THRESHOLD - 1 - FAIR_REBASE_STEP) *
+ PRIORITY_SCORE_BASE +
+ 9,
+ ]);
+ expect(await isRebasing()).toBe(false);
+ });
+
+ it('shifts a ticket allocated on an older scale at dispatch and stamps the job', async () => {
+ const {
+ redis,
+ allocator,
+ seedJob,
+ readPriority,
+ readStampedBase,
+ readVirtualTime,
+ readBase,
+ isRebasing,
+ } = await createAllocator();
+
+ await redis.set(
+ VIRTUAL_TIME_KEY,
+ FAIR_REBASE_THRESHOLD - FAIR_REBASE_STEP,
+ );
+ await redis.set(BASE_KEY, FAIR_REBASE_STEP);
+ await seedJob({
+ jobId: 'stray',
+ priority: ticketOf(FAIR_REBASE_THRESHOLD + 5),
+ counter: 1,
+ state: 'active',
+ });
+
+ await allocator.recordDispatch({ queueName: QUEUE, jobId: 'stray' });
+
+ expect(await readPriority('stray')).toBe(
+ ticketOf(FAIR_REBASE_THRESHOLD + 5 - FAIR_REBASE_STEP),
+ );
+ expect(await readStampedBase('stray')).toBe(FAIR_REBASE_STEP);
+ expect(await readVirtualTime()).toBe(
+ FAIR_REBASE_THRESHOLD + 5 - FAIR_REBASE_STEP,
+ );
+ expect(await readBase()).toBe(FAIR_REBASE_STEP);
+ expect(await isRebasing()).toBe(false);
+ });
+
it('swallows a Redis failure', async () => {
await expect(
createBrokenAllocator().recordDispatch({
diff --git a/packages/twenty-server/src/engine/core-modules/message-queue/drivers/fairness/fair-dispatch-script.constant.ts b/packages/twenty-server/src/engine/core-modules/message-queue/drivers/fairness/fair-dispatch-script.constant.ts
index 9ea92c6dbd1..ff878c2a90a 100644
--- a/packages/twenty-server/src/engine/core-modules/message-queue/drivers/fairness/fair-dispatch-script.constant.ts
+++ b/packages/twenty-server/src/engine/core-modules/message-queue/drivers/fairness/fair-dispatch-script.constant.ts
@@ -1,16 +1,40 @@
export const FAIR_DISPATCH_SCRIPT = `
-local ticketOffset = tonumber(ARGV[1])
-local priority = tonumber(redis.call('HGET', KEYS[2], 'priority'))
+local virtualTime = tonumber(redis.call('GET', KEYS[1]) or '0')
+local base = tonumber(redis.call('GET', KEYS[2]) or '0')
+local isRebasing = redis.call('EXISTS', KEYS[3]) == 1
+local threshold = tonumber(ARGV[1])
+local step = tonumber(ARGV[2])
+local ticketOffset = tonumber(ARGV[3])
+local job = redis.call('HMGET', KEYS[4], 'priority', 'fairBase', 'opts')
+local priority = tonumber(job[1])
if priority == nil or priority < ticketOffset then
- return 0
+ return isRebasing and 2 or 0
end
+local rawOptions = type(job[3]) == 'string' and job[3] or ''
+local jobBase = tonumber(job[2]) or tonumber(string.match(rawOptions, '"fairBase":(%d+)') or '') or base
+local shift = base - jobBase
local ticket = priority - ticketOffset
-if ticket > tonumber(redis.call('GET', KEYS[1]) or '0') then
- redis.call('SET', KEYS[1], ticket)
+if shift > 0 then
+ ticket = math.max(ticket - shift, 0)
+ redis.call('HSET', KEYS[4], 'priority', ticket + ticketOffset, 'fairBase', base)
end
-return 0
+if ticket > virtualTime then
+ virtualTime = ticket
+end
+
+if isRebasing or virtualTime < threshold then
+ redis.call('SET', KEYS[1], virtualTime)
+
+ return isRebasing and 2 or 0
+end
+
+redis.call('SET', KEYS[1], virtualTime - step)
+redis.call('INCRBY', KEYS[2], step)
+redis.call('HSET', KEYS[3], 'phase', 'p', 'cursor', '0', 'virtualTime', virtualTime)
+
+return 1
`;
diff --git a/packages/twenty-server/src/engine/core-modules/message-queue/drivers/fairness/fair-priority.constant.ts b/packages/twenty-server/src/engine/core-modules/message-queue/drivers/fairness/fair-priority.constant.ts
index bcdcdcf9172..5a027cecf5e 100644
--- a/packages/twenty-server/src/engine/core-modules/message-queue/drivers/fairness/fair-priority.constant.ts
+++ b/packages/twenty-server/src/engine/core-modules/message-queue/drivers/fairness/fair-priority.constant.ts
@@ -1,4 +1,10 @@
export const URGENT_FAIR_PRIORITY = 1;
export const FAIR_TICKET_OFFSET = 2;
+export const FAIR_REBASE_THRESHOLD = 2 ** 20;
+export const FAIR_REBASE_STEP = 2 ** 19;
+export const FAIR_REBASE_CHUNK_SIZE = 1000;
+export const FAIR_REBASE_GRACE_MS = 1000;
+export const FAIR_REBASE_MAX_CHUNKS = 100_000;
+
export const FAIR_CLOCK_TTL_MS = 24 * 60 * 60 * 1000;
diff --git a/packages/twenty-server/src/engine/core-modules/message-queue/drivers/fairness/fair-queue-scheduler.ts b/packages/twenty-server/src/engine/core-modules/message-queue/drivers/fairness/fair-queue-scheduler.ts
index ba4f0a32363..faaf8fdfaac 100644
--- a/packages/twenty-server/src/engine/core-modules/message-queue/drivers/fairness/fair-queue-scheduler.ts
+++ b/packages/twenty-server/src/engine/core-modules/message-queue/drivers/fairness/fair-queue-scheduler.ts
@@ -13,6 +13,8 @@ import { type MetricsService } from 'src/engine/core-modules/metrics/metrics.ser
import { MetricsKeys } from 'src/engine/core-modules/metrics/types/metrics-keys.type';
import { type TwentyConfigService } from 'src/engine/core-modules/twenty-config/twenty-config.service';
+export type FairJobOptions = { priority?: number; fairBase?: number };
+
type FairQueueSchedulerOptions = {
getClient: (queueName: MessageQueue) => Promise<RedisClient>;
getJobKey: (job: { queueName: MessageQueue; jobId: string }) => string;
@@ -32,7 +34,7 @@ export class FairQueueScheduler {
});
}
- async computeJobPriorities({
+ async computeJobOptions({
queueName,
jobsData,
options,
@@ -42,11 +44,13 @@ export class FairQueueScheduler {
jobsData: unknown[];
options?: Pick<QueueJobOptions, 'groupKeys' | 'delay'>;
staticPriority: number | undefined;
- }): Promise<(number | undefined)[]> {
- const staticPriorities = jobsData.map(() => staticPriority);
+ }): Promise<FairJobOptions[]> {
+ const staticJobOptions: FairJobOptions[] = jobsData.map(() => ({
+ priority: staticPriority,
+ }));
if (!this.isFairnessEnabled() || (options?.delay ?? 0) > 0) {
- return staticPriorities;
+ return staticJobOptions;
}
const requestByGroupKeys = new Map<
@@ -69,7 +73,7 @@ export class FairQueueScheduler {
});
const requests = [...requestByGroupKeys.values()];
- const tickets = await this.fairTicketAllocator.allocateTickets({
+ const fairTickets = await this.fairTicketAllocator.allocateTickets({
queueName,
requests: requests.map(({ groupKeys, jobIndexes }) => ({
groupKeys,
@@ -78,20 +82,21 @@ export class FairQueueScheduler {
});
if (staticPriority === URGENT_FAIR_PRIORITY) {
- return staticPriorities;
+ return staticJobOptions;
}
- if (!isDefined(tickets)) {
+ if (!isDefined(fairTickets)) {
void this.options.metricsService.incrementCounterForEvent({
key: MetricsKeys.JobFairTagFallback,
attributes: { queue: queueName },
shouldStoreInCache: false,
});
- return staticPriorities;
+ return staticJobOptions;
}
- const priorities = [...staticPriorities];
+ const { base, tickets } = fairTickets;
+ const fairJobOptions = [...staticJobOptions];
requests.forEach(({ jobIndexes }, requestIndex) => {
const { tag, weight } = tickets[requestIndex];
@@ -99,13 +104,16 @@ export class FairQueueScheduler {
jobIndexes.forEach((jobIndex, position) => {
const ticket = Math.floor(tag + position / weight);
- priorities[jobIndex] =
- FAIR_TICKET_OFFSET +
- Math.min(PRIORITY_LIMIT - FAIR_TICKET_OFFSET, Math.max(0, ticket));
+ fairJobOptions[jobIndex] = {
+ priority:
+ FAIR_TICKET_OFFSET +
+ Math.min(PRIORITY_LIMIT - FAIR_TICKET_OFFSET, Math.max(0, ticket)),
+ fairBase: base,
+ };
});
});
- return priorities;
+ return fairJobOptions;
}
onJobStarted({
diff --git a/packages/twenty-server/src/engine/core-modules/message-queue/drivers/fairness/fair-rebase-script.constant.ts b/packages/twenty-server/src/engine/core-modules/message-queue/drivers/fairness/fair-rebase-script.constant.ts
new file mode 100644
index 00000000000..8af7696410f
--- /dev/null
+++ b/packages/twenty-server/src/engine/core-modules/message-queue/drivers/fairness/fair-rebase-script.constant.ts
@@ -0,0 +1,116 @@
+export const FAIR_REBASE_SCRIPT = `
+local state = redis.call('HMGET', KEYS[1], 'phase', 'cursor', 'virtualTime')
+local phase = state[1]
+
+if not phase then
+ return {0, 0}
+end
+
+local cursor = tonumber(state[2])
+local oldVirtualTime = tonumber(state[3])
+local base = tonumber(redis.call('GET', KEYS[5]) or '0')
+local step = tonumber(ARGV[1])
+local chunkSize = tonumber(ARGV[2])
+local prefix = ARGV[3]
+local now = tonumber(ARGV[4])
+local graceMs = tonumber(ARGV[5])
+local ticketOffset = tonumber(ARGV[6])
+local scoreBase = 4294967296
+
+local function formatScore(value)
+ return string.format('%.0f', value)
+end
+
+local function rebaseJob(jobId)
+ local jobKey = prefix .. jobId
+ local job = redis.call('HMGET', jobKey, 'priority', 'fairBase', 'opts')
+ local priority = tonumber(job[1])
+
+ if priority == nil or priority < ticketOffset then
+ return 0
+ end
+
+ local rawOptions = type(job[3]) == 'string' and job[3] or ''
+ local jobBase = tonumber(job[2]) or tonumber(string.match(rawOptions, '"fairBase":(%d+)') or '') or base
+ local shift = base - jobBase
+
+ if shift <= 0 then
+ return 0
+ end
+
+ local ticket = priority - ticketOffset
+ local rebasedTicket = math.max(ticket - shift, 0)
+
+ redis.call('HSET', jobKey, 'priority', rebasedTicket + ticketOffset, 'fairBase', base)
+
+ return ticket - rebasedTicket
+end
+
+if phase == 'g' then
+ if now < cursor then
+ return {2, cursor - now}
+ end
+
+ redis.call('HSET', KEYS[1], 'phase', 'f', 'cursor', formatScore((oldVirtualTime - step / 2 + ticketOffset) * scoreBase - 1))
+
+ return {1, 0}
+end
+
+local processed = 0
+
+if phase == 'p' or phase == 'f' then
+ local members = redis.call('ZRANGEBYSCORE', KEYS[2], '(' .. formatScore(cursor), '+inf', 'WITHSCORES', 'LIMIT', 0, chunkSize)
+
+ for i = 1, #members, 2 do
+ local score = tonumber(members[i + 1])
+ local priorityDrop = rebaseJob(members[i])
+
+ if priorityDrop > 0 then
+ redis.call('ZADD', KEYS[2], formatScore(score - priorityDrop * scoreBase), members[i])
+ end
+
+ cursor = score
+ end
+
+ processed = #members / 2
+elseif phase == 'a' then
+ local jobIds = redis.call('LRANGE', KEYS[4], cursor, cursor + chunkSize - 1)
+
+ for _, jobId in ipairs(jobIds) do
+ rebaseJob(jobId)
+ end
+
+ cursor = cursor + #jobIds
+ processed = #jobIds
+elseif phase == 'd' then
+ local members = redis.call('ZRANGEBYSCORE', KEYS[3], '(' .. formatScore(cursor), '+inf', 'WITHSCORES', 'LIMIT', 0, chunkSize)
+
+ for i = 1, #members, 2 do
+ rebaseJob(members[i])
+ cursor = tonumber(members[i + 1])
+ end
+
+ processed = #members / 2
+end
+
+if processed < chunkSize then
+ if phase == 'p' then
+ phase = 'a'
+ cursor = 0
+ elseif phase == 'a' then
+ phase = 'd'
+ cursor = 0
+ elseif phase == 'd' then
+ phase = 'g'
+ cursor = now + graceMs
+ else
+ redis.call('DEL', KEYS[1])
+
+ return {0, 0}
+ end
+end
+
+redis.call('HSET', KEYS[1], 'phase', phase, 'cursor', formatScore(cursor))
+
+return {1, 0}
+`;
diff --git a/packages/twenty-server/src/engine/core-modules/message-queue/drivers/fairness/fair-tag-script.constant.ts b/packages/twenty-server/src/engine/core-modules/message-queue/drivers/fairness/fair-tag-script.constant.ts
index fc25d3d1c0f..aa29da80b9e 100644
--- a/packages/twenty-server/src/engine/core-modules/message-queue/drivers/fairness/fair-tag-script.constant.ts
+++ b/packages/twenty-server/src/engine/core-modules/message-queue/drivers/fairness/fair-tag-script.constant.ts
@@ -1,7 +1,8 @@
export const FAIR_TAG_SCRIPT = `
local virtualTimeKey = KEYS[1]
-local weightsKey = KEYS[2]
-local firstClockKeyIndex = 3
+local baseKey = KEYS[2]
+local weightsKey = KEYS[3]
+local firstClockKeyIndex = 4
local clockTtlMs = ARGV[1]
local distinctKeyCount = tonumber(ARGV[2])
local firstGroupKeyArgIndex = 3
@@ -9,16 +10,23 @@ local requestCount = tonumber(ARGV[firstGroupKeyArgIndex + distinctKeyCount])
local firstRequestArgIndex = firstGroupKeyArgIndex + distinctKeyCount + 1
local virtualTime = tonumber(redis.call('GET', virtualTimeKey) or '0')
+local base = tonumber(redis.call('GET', baseKey) or '0')
local clocks = {}
local weights = {}
for i = 1, distinctKeyCount do
- local clock = tonumber(redis.call('GET', KEYS[firstClockKeyIndex + i - 1]))
+ local raw = redis.call('GET', KEYS[firstClockKeyIndex + i - 1]) or ''
+ local clockBase, clock = string.match(raw, '^([^:]+):(.+)$')
+ local shiftedClock = tonumber(raw)
+
+ if clockBase ~= nil then
+ shiftedClock = tonumber(clock) - (base - tonumber(clockBase))
+ end
clocks[i] = virtualTime
- if clock ~= nil and clock > virtualTime then
- clocks[i] = clock
+ if shiftedClock ~= nil and shiftedClock > virtualTime then
+ clocks[i] = shiftedClock
end
local weight = tonumber(redis.call('HGET', weightsKey, ARGV[firstGroupKeyArgIndex + i - 1]))
@@ -30,8 +38,8 @@ for i = 1, distinctKeyCount do
weights[i] = weight
end
-local tickets = {}
-local ticketCount = 0
+local tickets = { string.format('%d', base) }
+local ticketCount = 1
local position = firstRequestArgIndex
for _ = 1, requestCount do
@@ -63,7 +71,7 @@ for _ = 1, requestCount do
end
for i = 1, distinctKeyCount do
- redis.call('SET', KEYS[firstClockKeyIndex + i - 1], tostring(clocks[i]), 'PX', clockTtlMs)
+ redis.call('SET', KEYS[firstClockKeyIndex + i - 1], string.format('%d', base) .. ':' .. clocks[i], 'PX', clockTtlMs)
end
return table.concat(tickets, ' ')
diff --git a/packages/twenty-server/src/engine/core-modules/message-queue/drivers/fairness/fair-ticket-allocator.ts b/packages/twenty-server/src/engine/core-modules/message-queue/drivers/fairness/fair-ticket-allocator.ts
index 41ac7caf3df..7b4db93a6a3 100644
--- a/packages/twenty-server/src/engine/core-modules/message-queue/drivers/fairness/fair-ticket-allocator.ts
+++ b/packages/twenty-server/src/engine/core-modules/message-queue/drivers/fairness/fair-ticket-allocator.ts
@@ -8,8 +8,14 @@ import { isDefined } from 'twenty-shared/utils';
import { FAIR_DISPATCH_SCRIPT } from 'src/engine/core-modules/message-queue/drivers/fairness/fair-dispatch-script.constant';
import {
FAIR_CLOCK_TTL_MS,
+ FAIR_REBASE_CHUNK_SIZE,
+ FAIR_REBASE_GRACE_MS,
+ FAIR_REBASE_MAX_CHUNKS,
+ FAIR_REBASE_STEP,
+ FAIR_REBASE_THRESHOLD,
FAIR_TICKET_OFFSET,
} from 'src/engine/core-modules/message-queue/drivers/fairness/fair-priority.constant';
+import { FAIR_REBASE_SCRIPT } from 'src/engine/core-modules/message-queue/drivers/fairness/fair-rebase-script.constant';
import { FAIR_TAG_SCRIPT } from 'src/engine/core-modules/message-queue/drivers/fairness/fair-tag-script.constant';
import { type MessageQueue } from 'src/engine/core-modules/message-queue/message-queue.constants';
@@ -17,14 +23,18 @@ export type FairTicketRequest = { groupKeys: string[]; jobCount: number };
export type FairTicket = { tag: number; weight: number };
+export type FairTickets = { base: number; tickets: FairTicket[] };
+
type FairTicketAllocatorOptions = {
getClient: (queueName: MessageQueue) => Promise<RedisClient>;
getJobKey: (job: { queueName: MessageQueue; jobId: string }) => string;
keyPrefix: string;
+ rebaseGraceMs?: number;
};
export class FairTicketAllocator {
private readonly logger = new Logger(FairTicketAllocator.name);
+ private readonly rebasesInProgress = new Map<MessageQueue, Promise<void>>();
private readonly scriptShas = new Map<string, string>();
constructor(private readonly options: FairTicketAllocatorOptions) {}
@@ -35,7 +45,7 @@ export class FairTicketAllocator {
}: {
queueName: MessageQueue;
requests: FairTicketRequest[];
- }): Promise<FairTicket[] | undefined> {
+ }): Promise<FairTickets | undefined> {
const fairKeyPrefix = `${this.options.keyPrefix}:${queueName}:fair:`;
const distinctGroupKeys = [
...new Set(requests.flatMap(({ groupKeys }) => groupKeys)),
@@ -50,6 +60,7 @@ export class FairTicketAllocator {
script: FAIR_TAG_SCRIPT,
keys: [
`${fairKeyPrefix}vtime`,
+ `${fairKeyPrefix}base`,
`${fairKeyPrefix}weights`,
...distinctGroupKeys.map(
(groupKey) => `${fairKeyPrefix}clock:${groupKey}`,
@@ -67,12 +78,14 @@ export class FairTicketAllocator {
]),
],
});
- const values = String(rawTickets).split(' ').map(Number);
+ const [base, ...values] = String(rawTickets).split(' ').map(Number);
const tickets = requests.map((_, index) => ({
tag: values[2 * index],
weight: values[2 * index + 1],
}));
const hasValidTickets =
+ Number.isInteger(base) &&
+ base >= 0 &&
values.length === 2 * requests.length &&
tickets.every(({ tag, weight }) => Number.isFinite(tag) && weight > 0);
@@ -80,7 +93,7 @@ export class FairTicketAllocator {
throw new Error(`unexpected tickets "${String(rawTickets)}"`);
}
- return tickets;
+ return { base, tickets };
} catch (error) {
this.logger.warn(
`Failed to allocate the fair tickets on queue ${queueName}: ${error instanceof Error ? error.message : String(error)}`,
@@ -100,15 +113,29 @@ export class FairTicketAllocator {
const fairKeyPrefix = `${this.options.keyPrefix}:${queueName}:fair:`;
try {
- await this.runScript({
- queueName,
- script: FAIR_DISPATCH_SCRIPT,
- keys: [
- `${fairKeyPrefix}vtime`,
- this.options.getJobKey({ queueName, jobId }),
- ],
- args: [FAIR_TICKET_OFFSET],
- });
+ const rebaseState = Number(
+ await this.runScript({
+ queueName,
+ script: FAIR_DISPATCH_SCRIPT,
+ keys: [
+ `${fairKeyPrefix}vtime`,
+ `${fairKeyPrefix}base`,
+ `${fairKeyPrefix}rebase`,
+ this.options.getJobKey({ queueName, jobId }),
+ ],
+ args: [FAIR_REBASE_THRESHOLD, FAIR_REBASE_STEP, FAIR_TICKET_OFFSET],
+ }),
+ );
+
+ if (rebaseState === 1) {
+ this.logger.log(
+ `Rebasing the fair tickets of queue ${queueName} by ${FAIR_REBASE_STEP}`,
+ );
+ }
+
+ if (rebaseState > 0) {
+ void this.rebase(queueName);
+ }
} catch (error) {
this.logger.warn(
`Failed to record the fair dispatch of job ${jobId} on queue ${queueName}: ${error instanceof Error ? error.message : String(error)}`,
@@ -116,6 +143,74 @@ export class FairTicketAllocator {
}
}
+ rebase(queueName: MessageQueue): Promise<void> {
+ const rebaseInProgress = this.rebasesInProgress.get(queueName);
+
+ if (isDefined(rebaseInProgress)) {
+ return rebaseInProgress;
+ }
+
+ const rebase = this.runRebase(queueName).finally(() => {
+ this.rebasesInProgress.delete(queueName);
+ });
+
+ this.rebasesInProgress.set(queueName, rebase);
+
+ return rebase;
+ }
+
+ private async runRebase(queueName: MessageQueue): Promise<void> {
+ const queueKeyPrefix = `${this.options.keyPrefix}:${queueName}:`;
+
+ try {
+ for (let chunk = 0; chunk < FAIR_REBASE_MAX_CHUNKS; chunk += 1) {
+ const rawResult = await this.runScript({
+ queueName,
+ script: FAIR_REBASE_SCRIPT,
+ keys: [
+ `${queueKeyPrefix}fair:rebase`,
+ `${queueKeyPrefix}prioritized`,
+ `${queueKeyPrefix}delayed`,
+ `${queueKeyPrefix}active`,
+ `${queueKeyPrefix}fair:base`,
+ ],
+ args: [
+ FAIR_REBASE_STEP,
+ FAIR_REBASE_CHUNK_SIZE,
+ queueKeyPrefix,
+ Date.now(),
+ this.options.rebaseGraceMs ?? FAIR_REBASE_GRACE_MS,
+ FAIR_TICKET_OFFSET,
+ ],
+ });
+
+ if (!Array.isArray(rawResult)) {
+ throw new Error(`unexpected rebase result "${String(rawResult)}"`);
+ }
+
+ const state = Number(rawResult[0]);
+
+ if (state === 0) {
+ return;
+ }
+
+ if (state === 2) {
+ const waitMs = Math.max(0, Number(rawResult[1]));
+
+ await new Promise<void>((resolve) => {
+ setTimeout(resolve, waitMs).unref();
+ });
+ }
+ }
+
+ throw new Error(`still pending after ${FAIR_REBASE_MAX_CHUNKS} chunks`);
+ } catch (error) {
+ this.logger.warn(
+ `Failed to rebase the fair tickets on queue ${queueName}: ${error instanceof Error ? error.message : String(error)}`,
+ );
+ }
+ }
+
private async runScript({
queueName,
script,