The open alternative to Salesforce, designed for AI.
TypeScript crmcrm-systemcustomergood-first-issuegraphqlhacktoberfestjavascriptmarketingmonoreponestjsopen-sourcepostgresqlreactreactjssalestypescriptweb
Rebase the fair tickets in bounded chunks once the virtual time passes 2^20 (#28994)
martmull9c2ef10c7b89
diff --git a/packages/twenty-server/src/engine/core-modules/message-queue/drivers/__tests__/bullmq.driver.spec.ts b/packages/twenty-server/src/engine/core-modules/message-queue/drivers/__tests__/bullmq.driver.spec.tsindex 415948e19e1..d6f4065355b 100644--- a/packages/twenty-server/src/engine/core-modules/message-queue/drivers/__tests__/bullmq.driver.spec.ts+++ b/packages/twenty-server/src/engine/core-modules/message-queue/drivers/__tests__/bullmq.driver.spec.ts@@ -4,6 +4,7 @@ import { DEFAULT_JOB_PRIORITY } from 'src/engine/core-modules/message-queue/consimport { QUEUE_JOB_CHANGED_EVENT } from 'src/engine/core-modules/message-queue/constants/queue-job-changed-event.constant';import { BullMQDriver } from 'src/engine/core-modules/message-queue/drivers/bullmq.driver';import {+ FAIR_REBASE_STEP,FAIR_TICKET_OFFSET,URGENT_FAIR_PRIORITY,} from 'src/engine/core-modules/message-queue/drivers/fairness/fair-priority.constant';@@ -542,7 +543,10 @@ describe('BullMQDriver fair tickets', () => {});it('ranks a job by the ticket of its workspace', async () => {- mockAllocateTickets.mockResolvedValue([{ tag: 7, weight: 1 }]);+ mockAllocateTickets.mockResolvedValue({+ base: FAIR_REBASE_STEP,+ tickets: [{ tag: 7, weight: 1 }],+ });await driver.add(MessageQueue.workspaceQueue, 'job', {workspaceId: 'ws-1',@@ -555,15 +559,21 @@ describe('BullMQDriver fair tickets', () => {expect(mockAdd).toHaveBeenCalledWith('job',{ workspaceId: 'ws-1' },- expect.objectContaining({ priority: FAIR_TICKET_OFFSET + 7 }),+ expect.objectContaining({+ priority: FAIR_TICKET_OFFSET + 7,+ fairBase: FAIR_REBASE_STEP,+ }),);});it('allocates one ticket per group of a bulk and spaces its jobs by the weight', async () => {- mockAllocateTickets.mockResolvedValue([- { tag: 10, weight: 0.5 },- { tag: 0, weight: 1 },- ]);+ mockAllocateTickets.mockResolvedValue({+ base: 0,+ tickets: [+ { tag: 10, weight: 0.5 },+ { tag: 0, weight: 1 },+ ],+ });await driver.bulkAdd(MessageQueue.workspaceQueue, 'job', [{ data: { workspaceId: 'ws-1' } },@@ -590,7 +600,10 @@ describe('BullMQDriver fair tickets', () => {});it('keeps priority 1 ahead of the fair tickets while still charging its group keys', async () => {- mockAllocateTickets.mockResolvedValue([{ tag: 7, weight: 1 }]);+ mockAllocateTickets.mockResolvedValue({+ base: 0,+ tickets: [{ tag: 7, weight: 1 }],+ });await driver.add(MessageQueue.workspaceQueue,diff --git a/packages/twenty-server/src/engine/core-modules/message-queue/drivers/bullmq.driver.ts b/packages/twenty-server/src/engine/core-modules/message-queue/drivers/bullmq.driver.tsindex 70da33203ae..fd5fc2bcc26 100644--- a/packages/twenty-server/src/engine/core-modules/message-queue/drivers/bullmq.driver.ts+++ b/packages/twenty-server/src/engine/core-modules/message-queue/drivers/bullmq.driver.ts@@ -57,6 +57,7 @@ export type BullMQDriverOptions = QueueOptions;type BullMQJobsOptions = JobsOptions & {broadcastTo?: QueueJobRecipient;+ fairBase?: number;};export class BullMQDriver@@ -535,7 +536,7 @@ export class BullMQDriver}const queueOptions = this.buildJobsOptions(options);- const [priority] = await this.fairQueueScheduler.computeJobPriorities({+ const [jobOptions] = await this.fairQueueScheduler.computeJobOptions({queueName,jobsData: [data],options,@@ -544,7 +545,7 @@ export class BullMQDriverconst job = await this.queueMap[queueName].add(jobName, data, {...queueOptions,- priority,+ ...jobOptions,});return job.id;@@ -596,7 +597,7 @@ export class BullMQDriver}const queueOptions = this.buildJobsOptions(options);- const priorities = await this.fairQueueScheduler.computeJobPriorities({+ const jobOptions = await this.fairQueueScheduler.computeJobOptions({queueName,jobsData: jobsToAdd.map(({ data }) => data),options,@@ -609,7 +610,7 @@ export class BullMQDriverdata,opts: {...queueOptions,- priority: priorities[index],+ ...jobOptions[index],jobId:jobId ??(queueOptions.jobId ? `${queueOptions.jobId}-${index}` : undefined),diff --git a/packages/twenty-server/src/engine/core-modules/message-queue/drivers/fairness/__tests__/fair-ticket-allocator.spec.ts b/packages/twenty-server/src/engine/core-modules/message-queue/drivers/fairness/__tests__/fair-ticket-allocator.spec.tsindex d51189abec2..95a9917d286 100644--- a/packages/twenty-server/src/engine/core-modules/message-queue/drivers/fairness/__tests__/fair-ticket-allocator.spec.ts+++ b/packages/twenty-server/src/engine/core-modules/message-queue/drivers/fairness/__tests__/fair-ticket-allocator.spec.ts@@ -1,10 +1,15 @@+import { readFileSync } from 'node:fs';+import { Logger } from '@nestjs/common';import { type Redis } from 'ioredis';import RedisMock from 'ioredis-mock';+import { isDefined } from 'twenty-shared/utils';import { FairTicketAllocator } from 'src/engine/core-modules/message-queue/drivers/fairness/fair-ticket-allocator';import {+ FAIR_REBASE_STEP,+ FAIR_REBASE_THRESHOLD,FAIR_TICKET_OFFSET,URGENT_FAIR_PRIORITY,} from 'src/engine/core-modules/message-queue/drivers/fairness/fair-priority.constant';@@ -14,13 +19,23 @@ const QUEUE = MessageQueue.logicFunctionQueue;const KEY_PREFIX = 'bull';const QUEUE_KEY_PREFIX = `${KEY_PREFIX}:${QUEUE}:`;const VIRTUAL_TIME_KEY = `${QUEUE_KEY_PREFIX}fair:vtime`;+const BASE_KEY = `${QUEUE_KEY_PREFIX}fair:base`;+const REBASE_KEY = `${QUEUE_KEY_PREFIX}fair:rebase`;const WEIGHTS_KEY = `${QUEUE_KEY_PREFIX}fair:weights`;+const PRIORITIZED_KEY = `${QUEUE_KEY_PREFIX}prioritized`;+const DELAYED_KEY = `${QUEUE_KEY_PREFIX}delayed`;+const ACTIVE_KEY = `${QUEUE_KEY_PREFIX}active`;+const PRIORITY_SCORE_BASE = 2 ** 32;+const BULLMQ_PRIORITY_SCORE_LUA =+ require.resolve('bullmq/dist/cjs/commands/includes/getPriorityScore.lua');const ticketOf = (tag: number) => FAIR_TICKET_OFFSET + tag;const jobKeyOf = (jobId: string) => `${QUEUE_KEY_PREFIX}${jobId}`;-const createAllocator = async () => {+const createAllocator = async ({+ rebaseGraceMs = 0,+}: { rebaseGraceMs?: number } = {}) => {const redis: Redis = new RedisMock();await redis.flushall();@@ -48,6 +63,49 @@ const createAllocator = async () => {);},};+ const seedJob = async ({+ jobId,+ priority,+ counter,+ state,+ base = 0,+ stampedBase,+ }: {+ jobId: string;+ priority: number;+ counter: number;+ state: 'prioritized' | 'delayed' | 'active';+ base?: number | 'none';+ stampedBase?: number;+ }) => {+ const jobKey = jobKeyOf(jobId);++ await redis.hset(jobKey, 'priority', priority);++ if (base !== 'none') {+ await redis.hset(+ jobKey,+ 'opts',+ JSON.stringify({ priority, fairBase: base }),+ );+ }++ if (isDefined(stampedBase)) {+ await redis.hset(jobKey, 'fairBase', stampedBase);+ }++ if (state === 'prioritized') {+ await redis.zadd(+ PRIORITIZED_KEY,+ priority * PRIORITY_SCORE_BASE + counter,+ jobId,+ );+ } else if (state === 'delayed') {+ await redis.zadd(DELAYED_KEY, counter, jobId);+ } else {+ await redis.rpush(ACTIVE_KEY, jobId);+ }+ };return {redis,@@ -55,12 +113,28 @@ const createAllocator = async () => {getClient: async () => client as never,getJobKey: ({ jobId }) => jobKeyOf(jobId),keyPrefix: KEY_PREFIX,+ rebaseGraceMs,}),- readClock: async (groupKey: string) =>- Number(await redis.get(`${QUEUE_KEY_PREFIX}fair:clock:${groupKey}`)),+ seedJob,+ readClock: async (groupKey: string) => {+ const raw = await redis.get(`${QUEUE_KEY_PREFIX}fair:clock:${groupKey}`);++ return raw?.split(':').map(Number);+ },+ readPriority: async (jobId: string) =>+ Number(await redis.hget(jobKeyOf(jobId), 'priority')),+ readStampedBase: async (jobId: string) => {+ const raw = await redis.hget(jobKeyOf(jobId), 'fairBase');++ return isDefined(raw) ? Number(raw) : undefined;+ },readVirtualTime: async () => Number(await redis.get(VIRTUAL_TIME_KEY)),- seedJob: async ({ jobId, priority }: { jobId: string; priority: number }) =>- redis.hset(jobKeyOf(jobId), 'priority', priority),+ readBase: async () => Number((await redis.get(BASE_KEY)) ?? 0),+ isRebasing: async () => (await redis.exists(REBASE_KEY)) === 1,+ readPrioritizedWithScores: async () =>+ (await redis.zrange(PRIORITIZED_KEY, 0, -1, 'WITHSCORES')).map(+ (value, index) => (index % 2 === 1 ? Number(value) : value),+ ),};};@@ -75,6 +149,8 @@ const createBrokenAllocator = () =>describe('FairTicketAllocator', () => {beforeAll(() => {+ jest.useRealTimers();+ jest.spyOn(Logger.prototype, 'log').mockImplementation(() => undefined);jest.spyOn(Logger.prototype, 'warn').mockImplementation(() => undefined);});@@ -91,13 +167,16 @@ describe('FairTicketAllocator', () => {{ groupKeys: ['workspace:W', 'application:Q'], jobCount: 1 },],}),- ).toEqual([- { tag: 0, weight: 1 },- { tag: 100, weight: 1 },- { tag: 1, weight: 1 },- ]);- expect(await readClock('application:H')).toBe(101);- expect(await readClock('workspace:W')).toBe(2);+ ).toEqual({+ base: 0,+ tickets: [+ { tag: 0, weight: 1 },+ { tag: 100, weight: 1 },+ { tag: 1, weight: 1 },+ ],+ });+ expect(await readClock('application:H')).toEqual([0, 101]);+ expect(await readClock('workspace:W')).toEqual([0, 2]);});it('tags a request at the virtual time when it is ahead of its clocks', async () => {@@ -110,8 +189,41 @@ describe('FairTicketAllocator', () => {queueName: QUEUE,requests: [{ groupKeys: ['workspace:B'], jobCount: 2 }],}),- ).toEqual([{ tag: 5, weight: 1 }]);- expect(await readClock('workspace:B')).toBe(7);+ ).toEqual({ base: 0, tickets: [{ tag: 5, weight: 1 }] });+ expect(await readClock('workspace:B')).toEqual([0, 7]);+ });++ it('returns the base it allocated on and reads a clock stamped with an older base one step down', async () => {+ const { redis, allocator, readClock } = await createAllocator();++ await redis.set(VIRTUAL_TIME_KEY, 3);+ await redis.set(BASE_KEY, FAIR_REBASE_STEP);+ await redis.set(+ `${QUEUE_KEY_PREFIX}fair:clock:workspace:S`,+ `0:${FAIR_REBASE_STEP + 8}`,+ );++ expect(+ await allocator.allocateTickets({+ queueName: QUEUE,+ requests: [{ groupKeys: ['workspace:S'], jobCount: 1 }],+ }),+ ).toEqual({ base: FAIR_REBASE_STEP, tickets: [{ tag: 8, weight: 1 }] });+ expect(await readClock('workspace:S')).toEqual([FAIR_REBASE_STEP, 9]);+ });++ it('reads a clock written before the base existed on the current base', async () => {+ const { redis, allocator, readClock } = await createAllocator();++ await redis.set(`${QUEUE_KEY_PREFIX}fair:clock:workspace:L`, 7);++ expect(+ await allocator.allocateTickets({+ queueName: QUEUE,+ requests: [{ groupKeys: ['workspace:L'], jobCount: 1 }],+ }),+ ).toEqual({ base: 0, tickets: [{ tag: 7, weight: 1 }] });+ expect(await readClock('workspace:L')).toEqual([0, 8]);});it('weighs a request by the product of its key weights and ignores a malformed weight', async () => {@@ -129,10 +241,13 @@ describe('FairTicketAllocator', () => {{ groupKeys: ['workspace:W', 'application:N'], jobCount: 1 },],}),- ).toEqual([- { tag: 0, weight: 2 },- { tag: 1, weight: 1 },- ]);+ ).toEqual({+ base: 0,+ tickets: [+ { tag: 0, weight: 2 },+ { tag: 1, weight: 1 },+ ],+ });});it('returns nothing when Redis is unavailable', async () => {@@ -166,11 +281,27 @@ describe('FairTicketAllocator', () => {});describe('recordDispatch', () => {+ it('relies on the priority score layout of the installed BullMQ', () => {+ expect(readFileSync(BULLMQ_PRIORITY_SCORE_LUA, 'utf8')).toContain(+ 'priority * 0x100000000 + prioCounter % 0x100000000',+ );+ });+it('advances the virtual time from the dispatched job hash and leaves urgent and missing jobs alone', async () => {const { allocator, seedJob, readVirtualTime } = await createAllocator();- await seedJob({ jobId: 'A1', priority: ticketOf(5) });- await seedJob({ jobId: 'urgent', priority: URGENT_FAIR_PRIORITY });+ await seedJob({+ jobId: 'A1',+ priority: ticketOf(5),+ counter: 1,+ state: 'active',+ });+ await seedJob({+ jobId: 'urgent',+ priority: URGENT_FAIR_PRIORITY,+ counter: 2,+ state: 'active',+ });await allocator.recordDispatch({ queueName: QUEUE, jobId: 'A1' });await allocator.recordDispatch({ queueName: QUEUE, jobId: 'urgent' });@@ -182,14 +313,24 @@ describe('FairTicketAllocator', () => {queueName: QUEUE,requests: [{ groupKeys: ['workspace:B'], jobCount: 1 }],}),- ).toEqual([{ tag: 5, weight: 1 }]);+ ).toEqual({ base: 0, tickets: [{ tag: 5, weight: 1 }] });});it('never moves the virtual time back', async () => {const { allocator, seedJob, readVirtualTime } = await createAllocator();- await seedJob({ jobId: 'late', priority: ticketOf(9) });- await seedJob({ jobId: 'early', priority: ticketOf(3) });+ await seedJob({+ jobId: 'late',+ priority: ticketOf(9),+ counter: 1,+ state: 'active',+ });+ await seedJob({+ jobId: 'early',+ priority: ticketOf(3),+ counter: 2,+ state: 'active',+ });await allocator.recordDispatch({ queueName: QUEUE, jobId: 'late' });await allocator.recordDispatch({ queueName: QUEUE, jobId: 'early' });@@ -197,6 +338,376 @@ describe('FairTicketAllocator', () => {expect(await readVirtualTime()).toBe(9);});+ it('rewrites the waiting tickets in chunks once the threshold is passed, keeping the order and leaving urgent and static priorities alone', async () => {+ const {+ redis,+ allocator,+ seedJob,+ readPriority,+ readStampedBase,+ readVirtualTime,+ readBase,+ isRebasing,+ readPrioritizedWithScores,+ readClock,+ } = await createAllocator();+ const fillerCount = 1200;++ await redis.set(VIRTUAL_TIME_KEY, FAIR_REBASE_THRESHOLD - 1);+ await redis.set(+ `${QUEUE_KEY_PREFIX}fair:clock:workspace:A`,+ `0:${FAIR_REBASE_THRESHOLD + 3}`,+ );+ await seedJob({+ jobId: 'urgent',+ priority: URGENT_FAIR_PRIORITY,+ counter: 1,+ state: 'prioritized',+ });+ await seedJob({+ jobId: 'static',+ priority: 4,+ counter: 2,+ state: 'prioritized',+ base: 'none',+ });+ await seedJob({+ jobId: 'A2',+ priority: ticketOf(FAIR_REBASE_THRESHOLD + 1),+ counter: 3,+ state: 'prioritized',+ });+ await seedJob({+ jobId: 'A3',+ priority: ticketOf(FAIR_REBASE_THRESHOLD + 2),+ counter: 4,+ state: 'prioritized',+ });+ for (let index = 0; index < fillerCount; index += 1) {+ await seedJob({+ jobId: `F${index}`,+ priority: ticketOf(FAIR_REBASE_THRESHOLD + 10 + index),+ counter: 100 + index,+ state: 'prioritized',+ });+ }+ await seedJob({+ jobId: 'retrying',+ priority: ticketOf(FAIR_REBASE_THRESHOLD + 5),+ counter: 5,+ state: 'delayed',+ });+ await seedJob({+ jobId: 'running',+ priority: ticketOf(FAIR_REBASE_THRESHOLD),+ counter: 6,+ state: 'active',+ });++ await allocator.recordDispatch({ queueName: QUEUE, jobId: 'running' });+ await allocator.rebase(QUEUE);++ expect(await readVirtualTime()).toBe(+ FAIR_REBASE_THRESHOLD - FAIR_REBASE_STEP,+ );+ expect(await readBase()).toBe(FAIR_REBASE_STEP);+ expect(await isRebasing()).toBe(false);+ expect((await readPrioritizedWithScores()).slice(0, 8)).toEqual([+ 'urgent',+ URGENT_FAIR_PRIORITY * PRIORITY_SCORE_BASE + 1,+ 'static',+ 4 * PRIORITY_SCORE_BASE + 2,+ 'A2',+ ticketOf(FAIR_REBASE_THRESHOLD + 1 - FAIR_REBASE_STEP) *+ PRIORITY_SCORE_BASE ++ 3,+ 'A3',+ ticketOf(FAIR_REBASE_THRESHOLD + 2 - FAIR_REBASE_STEP) *+ PRIORITY_SCORE_BASE ++ 4,+ ]);+ expect(await redis.zcard(PRIORITIZED_KEY)).toBe(fillerCount + 4);+ expect(await readPriority(`F${fillerCount - 1}`)).toBe(+ ticketOf(+ FAIR_REBASE_THRESHOLD + 10 + fillerCount - 1 - FAIR_REBASE_STEP,+ ),+ );+ expect(await readStampedBase('A2')).toBe(FAIR_REBASE_STEP);+ expect(await readPriority('static')).toBe(4);+ expect(await readStampedBase('static')).toBeUndefined();+ expect(await readPriority('retrying')).toBe(+ ticketOf(FAIR_REBASE_THRESHOLD + 5 - FAIR_REBASE_STEP),+ );+ expect(await readPriority('running')).toBe(+ ticketOf(FAIR_REBASE_THRESHOLD - FAIR_REBASE_STEP),+ );+ expect(+ await allocator.allocateTickets({+ queueName: QUEUE,+ requests: [{ groupKeys: ['workspace:A'], jobCount: 1 }],+ }),+ ).toEqual({+ base: FAIR_REBASE_STEP,+ tickets: [+ { tag: FAIR_REBASE_THRESHOLD + 3 - FAIR_REBASE_STEP, weight: 1 },+ ],+ });+ expect(await readClock('workspace:A')).toEqual([+ FAIR_REBASE_STEP,+ FAIR_REBASE_THRESHOLD + 4 - FAIR_REBASE_STEP,+ ]);+ });++ it('shifts every ticket of a backlog deeper than a step exactly once', async () => {+ const { redis, allocator, seedJob, readPrioritizedWithScores } =+ await createAllocator();++ await redis.set(VIRTUAL_TIME_KEY, FAIR_REBASE_THRESHOLD - 1);+ for (const depth of [1, 2, 3]) {+ await seedJob({+ jobId: `deep${depth}`,+ priority: ticketOf(FAIR_REBASE_THRESHOLD + FAIR_REBASE_STEP + depth),+ counter: depth,+ state: 'prioritized',+ });+ }+ await seedJob({+ jobId: 'running',+ priority: ticketOf(FAIR_REBASE_THRESHOLD),+ counter: 9,+ state: 'active',+ });++ await allocator.recordDispatch({ queueName: QUEUE, jobId: 'running' });+ await allocator.rebase(QUEUE);++ expect(await readPrioritizedWithScores()).toEqual([+ 'deep1',+ ticketOf(FAIR_REBASE_THRESHOLD + 1) * PRIORITY_SCORE_BASE + 1,+ 'deep2',+ ticketOf(FAIR_REBASE_THRESHOLD + 2) * PRIORITY_SCORE_BASE + 2,+ 'deep3',+ ticketOf(FAIR_REBASE_THRESHOLD + 3) * PRIORITY_SCORE_BASE + 3,+ ]);+ });++ it('shifts a delayed retry that sits below the old virtual time and floors an overdue one at zero, keeping their order', async () => {+ const { redis, allocator, seedJob, readPriority, readStampedBase } =+ await createAllocator();++ await redis.set(VIRTUAL_TIME_KEY, FAIR_REBASE_THRESHOLD - 1);+ await seedJob({+ jobId: 'retrying',+ priority: ticketOf(FAIR_REBASE_THRESHOLD - FAIR_REBASE_STEP + 100),+ counter: 1,+ state: 'delayed',+ });+ await seedJob({+ jobId: 'overdue',+ priority: ticketOf(100),+ counter: 2,+ state: 'delayed',+ });+ await seedJob({+ jobId: 'running',+ priority: ticketOf(FAIR_REBASE_THRESHOLD),+ counter: 3,+ state: 'active',+ });++ await allocator.recordDispatch({ queueName: QUEUE, jobId: 'running' });+ await allocator.rebase(QUEUE);++ expect(await readPriority('retrying')).toBe(ticketOf(100));+ expect(await readStampedBase('retrying')).toBe(FAIR_REBASE_STEP);+ expect(await readPriority('overdue')).toBe(ticketOf(0));+ expect(await readStampedBase('overdue')).toBe(FAIR_REBASE_STEP);+ });++ it('leaves a job already on the new scale and a ticket allocated before the base existed alone', async () => {+ const { redis, allocator, seedJob, readPriority, readStampedBase } =+ await createAllocator();++ await redis.set(VIRTUAL_TIME_KEY, FAIR_REBASE_THRESHOLD - 1);+ await seedJob({+ jobId: 'fresh',+ priority: ticketOf(FAIR_REBASE_THRESHOLD - FAIR_REBASE_STEP + 5),+ counter: 1,+ state: 'prioritized',+ base: FAIR_REBASE_STEP,+ });+ await seedJob({+ jobId: 'stamped',+ priority: ticketOf(FAIR_REBASE_THRESHOLD - FAIR_REBASE_STEP + 6),+ counter: 2,+ state: 'prioritized',+ stampedBase: FAIR_REBASE_STEP,+ });+ await seedJob({+ jobId: 'legacy',+ priority: ticketOf(FAIR_REBASE_THRESHOLD + 2),+ counter: 3,+ state: 'prioritized',+ base: 'none',+ });+ await seedJob({+ jobId: 'old',+ priority: ticketOf(FAIR_REBASE_THRESHOLD + 3),+ counter: 4,+ state: 'prioritized',+ });+ await seedJob({+ jobId: 'running',+ priority: ticketOf(FAIR_REBASE_THRESHOLD),+ counter: 5,+ state: 'active',+ });++ await allocator.recordDispatch({ queueName: QUEUE, jobId: 'running' });+ await allocator.rebase(QUEUE);++ expect(await readPriority('old')).toBe(+ ticketOf(FAIR_REBASE_THRESHOLD + 3 - FAIR_REBASE_STEP),+ );+ expect(await readPriority('fresh')).toBe(+ ticketOf(FAIR_REBASE_THRESHOLD - FAIR_REBASE_STEP + 5),+ );+ expect(await readPriority('stamped')).toBe(+ ticketOf(FAIR_REBASE_THRESHOLD - FAIR_REBASE_STEP + 6),+ );+ expect(await readPriority('legacy')).toBe(+ ticketOf(FAIR_REBASE_THRESHOLD + 2),+ );+ expect(await readStampedBase('legacy')).toBeUndefined();+ });++ it('dispatches a job fetched before the switch without switching the scale again', async () => {+ const {+ redis,+ allocator,+ seedJob,+ readPriority,+ readVirtualTime,+ readBase,+ isRebasing,+ } = await createAllocator();++ await redis.set(VIRTUAL_TIME_KEY, FAIR_REBASE_THRESHOLD - 1);+ await seedJob({+ jobId: 'j1',+ priority: ticketOf(FAIR_REBASE_THRESHOLD),+ counter: 1,+ state: 'active',+ });+ await seedJob({+ jobId: 'j2',+ priority: ticketOf(FAIR_REBASE_THRESHOLD + 3),+ counter: 2,+ state: 'active',+ });++ await allocator.recordDispatch({ queueName: QUEUE, jobId: 'j1' });+ await allocator.rebase(QUEUE);+ await allocator.recordDispatch({ queueName: QUEUE, jobId: 'j2' });++ expect(await readBase()).toBe(FAIR_REBASE_STEP);+ expect(await readVirtualTime()).toBe(+ FAIR_REBASE_THRESHOLD + 3 - FAIR_REBASE_STEP,+ );+ expect(await readPriority('j2')).toBe(+ ticketOf(FAIR_REBASE_THRESHOLD + 3 - FAIR_REBASE_STEP),+ );+ expect(await isRebasing()).toBe(false);+ });++ it('normalizes a ticket allocated before the switch and inserted after it', async () => {+ const {+ redis,+ allocator,+ seedJob,+ readPriority,+ readStampedBase,+ readPrioritizedWithScores,+ isRebasing,+ } = await createAllocator({ rebaseGraceMs: 50 });++ await redis.set(VIRTUAL_TIME_KEY, FAIR_REBASE_THRESHOLD - 1);+ await seedJob({+ jobId: 'x',+ priority: ticketOf(FAIR_REBASE_THRESHOLD),+ counter: 1,+ state: 'active',+ });++ const allocation = await allocator.allocateTickets({+ queueName: QUEUE,+ requests: [{ groupKeys: ['workspace:late'], jobCount: 1 }],+ });++ expect(allocation).toEqual({+ base: 0,+ tickets: [{ tag: FAIR_REBASE_THRESHOLD - 1, weight: 1 }],+ });++ await allocator.recordDispatch({ queueName: QUEUE, jobId: 'x' });+ await seedJob({+ jobId: 'late',+ priority: ticketOf(FAIR_REBASE_THRESHOLD - 1),+ counter: 9,+ state: 'prioritized',+ });+ await allocator.rebase(QUEUE);++ expect(await readPriority('late')).toBe(+ ticketOf(FAIR_REBASE_THRESHOLD - 1 - FAIR_REBASE_STEP),+ );+ expect(await readStampedBase('late')).toBe(FAIR_REBASE_STEP);+ expect(await readPrioritizedWithScores()).toEqual([+ 'late',+ ticketOf(FAIR_REBASE_THRESHOLD - 1 - FAIR_REBASE_STEP) *+ PRIORITY_SCORE_BASE ++ 9,+ ]);+ expect(await isRebasing()).toBe(false);+ });++ it('shifts a ticket allocated on an older scale at dispatch and stamps the job', async () => {+ const {+ redis,+ allocator,+ seedJob,+ readPriority,+ readStampedBase,+ readVirtualTime,+ readBase,+ isRebasing,+ } = await createAllocator();++ await redis.set(+ VIRTUAL_TIME_KEY,+ FAIR_REBASE_THRESHOLD - FAIR_REBASE_STEP,+ );+ await redis.set(BASE_KEY, FAIR_REBASE_STEP);+ await seedJob({+ jobId: 'stray',+ priority: ticketOf(FAIR_REBASE_THRESHOLD + 5),+ counter: 1,+ state: 'active',+ });++ await allocator.recordDispatch({ queueName: QUEUE, jobId: 'stray' });++ expect(await readPriority('stray')).toBe(+ ticketOf(FAIR_REBASE_THRESHOLD + 5 - FAIR_REBASE_STEP),+ );+ expect(await readStampedBase('stray')).toBe(FAIR_REBASE_STEP);+ expect(await readVirtualTime()).toBe(+ FAIR_REBASE_THRESHOLD + 5 - FAIR_REBASE_STEP,+ );+ expect(await readBase()).toBe(FAIR_REBASE_STEP);+ expect(await isRebasing()).toBe(false);+ });+it('swallows a Redis failure', async () => {await expect(createBrokenAllocator().recordDispatch({diff --git a/packages/twenty-server/src/engine/core-modules/message-queue/drivers/fairness/fair-dispatch-script.constant.ts b/packages/twenty-server/src/engine/core-modules/message-queue/drivers/fairness/fair-dispatch-script.constant.tsindex 9ea92c6dbd1..ff878c2a90a 100644--- a/packages/twenty-server/src/engine/core-modules/message-queue/drivers/fairness/fair-dispatch-script.constant.ts+++ b/packages/twenty-server/src/engine/core-modules/message-queue/drivers/fairness/fair-dispatch-script.constant.ts@@ -1,16 +1,40 @@export const FAIR_DISPATCH_SCRIPT = `-local ticketOffset = tonumber(ARGV[1])-local priority = tonumber(redis.call('HGET', KEYS[2], 'priority'))+local virtualTime = tonumber(redis.call('GET', KEYS[1]) or '0')+local base = tonumber(redis.call('GET', KEYS[2]) or '0')+local isRebasing = redis.call('EXISTS', KEYS[3]) == 1+local threshold = tonumber(ARGV[1])+local step = tonumber(ARGV[2])+local ticketOffset = tonumber(ARGV[3])+local job = redis.call('HMGET', KEYS[4], 'priority', 'fairBase', 'opts')+local priority = tonumber(job[1])if priority == nil or priority < ticketOffset then- return 0+ return isRebasing and 2 or 0end+local rawOptions = type(job[3]) == 'string' and job[3] or ''+local jobBase = tonumber(job[2]) or tonumber(string.match(rawOptions, '"fairBase":(%d+)') or '') or base+local shift = base - jobBaselocal ticket = priority - ticketOffset-if ticket > tonumber(redis.call('GET', KEYS[1]) or '0') then- redis.call('SET', KEYS[1], ticket)+if shift > 0 then+ ticket = math.max(ticket - shift, 0)+ redis.call('HSET', KEYS[4], 'priority', ticket + ticketOffset, 'fairBase', base)end-return 0+if ticket > virtualTime then+ virtualTime = ticket+end++if isRebasing or virtualTime < threshold then+ redis.call('SET', KEYS[1], virtualTime)++ return isRebasing and 2 or 0+end++redis.call('SET', KEYS[1], virtualTime - step)+redis.call('INCRBY', KEYS[2], step)+redis.call('HSET', KEYS[3], 'phase', 'p', 'cursor', '0', 'virtualTime', virtualTime)++return 1`;diff --git a/packages/twenty-server/src/engine/core-modules/message-queue/drivers/fairness/fair-priority.constant.ts b/packages/twenty-server/src/engine/core-modules/message-queue/drivers/fairness/fair-priority.constant.tsindex bcdcdcf9172..5a027cecf5e 100644--- a/packages/twenty-server/src/engine/core-modules/message-queue/drivers/fairness/fair-priority.constant.ts+++ b/packages/twenty-server/src/engine/core-modules/message-queue/drivers/fairness/fair-priority.constant.ts@@ -1,4 +1,10 @@export const URGENT_FAIR_PRIORITY = 1;export const FAIR_TICKET_OFFSET = 2;+export const FAIR_REBASE_THRESHOLD = 2 ** 20;+export const FAIR_REBASE_STEP = 2 ** 19;+export const FAIR_REBASE_CHUNK_SIZE = 1000;+export const FAIR_REBASE_GRACE_MS = 1000;+export const FAIR_REBASE_MAX_CHUNKS = 100_000;+export const FAIR_CLOCK_TTL_MS = 24 * 60 * 60 * 1000;diff --git a/packages/twenty-server/src/engine/core-modules/message-queue/drivers/fairness/fair-queue-scheduler.ts b/packages/twenty-server/src/engine/core-modules/message-queue/drivers/fairness/fair-queue-scheduler.tsindex ba4f0a32363..faaf8fdfaac 100644--- a/packages/twenty-server/src/engine/core-modules/message-queue/drivers/fairness/fair-queue-scheduler.ts+++ b/packages/twenty-server/src/engine/core-modules/message-queue/drivers/fairness/fair-queue-scheduler.ts@@ -13,6 +13,8 @@ import { type MetricsService } from 'src/engine/core-modules/metrics/metrics.serimport { MetricsKeys } from 'src/engine/core-modules/metrics/types/metrics-keys.type';import { type TwentyConfigService } from 'src/engine/core-modules/twenty-config/twenty-config.service';+export type FairJobOptions = { priority?: number; fairBase?: number };+type FairQueueSchedulerOptions = {getClient: (queueName: MessageQueue) => Promise<RedisClient>;getJobKey: (job: { queueName: MessageQueue; jobId: string }) => string;@@ -32,7 +34,7 @@ export class FairQueueScheduler {});}- async computeJobPriorities({+ async computeJobOptions({queueName,jobsData,options,@@ -42,11 +44,13 @@ export class FairQueueScheduler {jobsData: unknown[];options?: Pick<QueueJobOptions, 'groupKeys' | 'delay'>;staticPriority: number | undefined;- }): Promise<(number | undefined)[]> {- const staticPriorities = jobsData.map(() => staticPriority);+ }): Promise<FairJobOptions[]> {+ const staticJobOptions: FairJobOptions[] = jobsData.map(() => ({+ priority: staticPriority,+ }));if (!this.isFairnessEnabled() || (options?.delay ?? 0) > 0) {- return staticPriorities;+ return staticJobOptions;}const requestByGroupKeys = new Map<@@ -69,7 +73,7 @@ export class FairQueueScheduler {});const requests = [...requestByGroupKeys.values()];- const tickets = await this.fairTicketAllocator.allocateTickets({+ const fairTickets = await this.fairTicketAllocator.allocateTickets({queueName,requests: requests.map(({ groupKeys, jobIndexes }) => ({groupKeys,@@ -78,20 +82,21 @@ export class FairQueueScheduler {});if (staticPriority === URGENT_FAIR_PRIORITY) {- return staticPriorities;+ return staticJobOptions;}- if (!isDefined(tickets)) {+ if (!isDefined(fairTickets)) {void this.options.metricsService.incrementCounterForEvent({key: MetricsKeys.JobFairTagFallback,attributes: { queue: queueName },shouldStoreInCache: false,});- return staticPriorities;+ return staticJobOptions;}- const priorities = [...staticPriorities];+ const { base, tickets } = fairTickets;+ const fairJobOptions = [...staticJobOptions];requests.forEach(({ jobIndexes }, requestIndex) => {const { tag, weight } = tickets[requestIndex];@@ -99,13 +104,16 @@ export class FairQueueScheduler {jobIndexes.forEach((jobIndex, position) => {const ticket = Math.floor(tag + position / weight);- priorities[jobIndex] =- FAIR_TICKET_OFFSET +- Math.min(PRIORITY_LIMIT - FAIR_TICKET_OFFSET, Math.max(0, ticket));+ fairJobOptions[jobIndex] = {+ priority:+ FAIR_TICKET_OFFSET ++ Math.min(PRIORITY_LIMIT - FAIR_TICKET_OFFSET, Math.max(0, ticket)),+ fairBase: base,+ };});});- return priorities;+ return fairJobOptions;}onJobStarted({diff --git a/packages/twenty-server/src/engine/core-modules/message-queue/drivers/fairness/fair-rebase-script.constant.ts b/packages/twenty-server/src/engine/core-modules/message-queue/drivers/fairness/fair-rebase-script.constant.tsnew file mode 100644index 00000000000..8af7696410f--- /dev/null+++ b/packages/twenty-server/src/engine/core-modules/message-queue/drivers/fairness/fair-rebase-script.constant.ts@@ -0,0 +1,116 @@+export const FAIR_REBASE_SCRIPT = `+local state = redis.call('HMGET', KEYS[1], 'phase', 'cursor', 'virtualTime')+local phase = state[1]++if not phase then+ return {0, 0}+end++local cursor = tonumber(state[2])+local oldVirtualTime = tonumber(state[3])+local base = tonumber(redis.call('GET', KEYS[5]) or '0')+local step = tonumber(ARGV[1])+local chunkSize = tonumber(ARGV[2])+local prefix = ARGV[3]+local now = tonumber(ARGV[4])+local graceMs = tonumber(ARGV[5])+local ticketOffset = tonumber(ARGV[6])+local scoreBase = 4294967296++local function formatScore(value)+ return string.format('%.0f', value)+end++local function rebaseJob(jobId)+ local jobKey = prefix .. jobId+ local job = redis.call('HMGET', jobKey, 'priority', 'fairBase', 'opts')+ local priority = tonumber(job[1])++ if priority == nil or priority < ticketOffset then+ return 0+ end++ local rawOptions = type(job[3]) == 'string' and job[3] or ''+ local jobBase = tonumber(job[2]) or tonumber(string.match(rawOptions, '"fairBase":(%d+)') or '') or base+ local shift = base - jobBase++ if shift <= 0 then+ return 0+ end++ local ticket = priority - ticketOffset+ local rebasedTicket = math.max(ticket - shift, 0)++ redis.call('HSET', jobKey, 'priority', rebasedTicket + ticketOffset, 'fairBase', base)++ return ticket - rebasedTicket+end++if phase == 'g' then+ if now < cursor then+ return {2, cursor - now}+ end++ redis.call('HSET', KEYS[1], 'phase', 'f', 'cursor', formatScore((oldVirtualTime - step / 2 + ticketOffset) * scoreBase - 1))++ return {1, 0}+end++local processed = 0++if phase == 'p' or phase == 'f' then+ local members = redis.call('ZRANGEBYSCORE', KEYS[2], '(' .. formatScore(cursor), '+inf', 'WITHSCORES', 'LIMIT', 0, chunkSize)++ for i = 1, #members, 2 do+ local score = tonumber(members[i + 1])+ local priorityDrop = rebaseJob(members[i])++ if priorityDrop > 0 then+ redis.call('ZADD', KEYS[2], formatScore(score - priorityDrop * scoreBase), members[i])+ end++ cursor = score+ end++ processed = #members / 2+elseif phase == 'a' then+ local jobIds = redis.call('LRANGE', KEYS[4], cursor, cursor + chunkSize - 1)++ for _, jobId in ipairs(jobIds) do+ rebaseJob(jobId)+ end++ cursor = cursor + #jobIds+ processed = #jobIds+elseif phase == 'd' then+ local members = redis.call('ZRANGEBYSCORE', KEYS[3], '(' .. formatScore(cursor), '+inf', 'WITHSCORES', 'LIMIT', 0, chunkSize)++ for i = 1, #members, 2 do+ rebaseJob(members[i])+ cursor = tonumber(members[i + 1])+ end++ processed = #members / 2+end++if processed < chunkSize then+ if phase == 'p' then+ phase = 'a'+ cursor = 0+ elseif phase == 'a' then+ phase = 'd'+ cursor = 0+ elseif phase == 'd' then+ phase = 'g'+ cursor = now + graceMs+ else+ redis.call('DEL', KEYS[1])++ return {0, 0}+ end+end++redis.call('HSET', KEYS[1], 'phase', phase, 'cursor', formatScore(cursor))++return {1, 0}+`;diff --git a/packages/twenty-server/src/engine/core-modules/message-queue/drivers/fairness/fair-tag-script.constant.ts b/packages/twenty-server/src/engine/core-modules/message-queue/drivers/fairness/fair-tag-script.constant.tsindex fc25d3d1c0f..aa29da80b9e 100644--- a/packages/twenty-server/src/engine/core-modules/message-queue/drivers/fairness/fair-tag-script.constant.ts+++ b/packages/twenty-server/src/engine/core-modules/message-queue/drivers/fairness/fair-tag-script.constant.ts@@ -1,7 +1,8 @@export const FAIR_TAG_SCRIPT = `local virtualTimeKey = KEYS[1]-local weightsKey = KEYS[2]-local firstClockKeyIndex = 3+local baseKey = KEYS[2]+local weightsKey = KEYS[3]+local firstClockKeyIndex = 4local clockTtlMs = ARGV[1]local distinctKeyCount = tonumber(ARGV[2])local firstGroupKeyArgIndex = 3@@ -9,16 +10,23 @@ local requestCount = tonumber(ARGV[firstGroupKeyArgIndex + distinctKeyCount])local firstRequestArgIndex = firstGroupKeyArgIndex + distinctKeyCount + 1local virtualTime = tonumber(redis.call('GET', virtualTimeKey) or '0')+local base = tonumber(redis.call('GET', baseKey) or '0')local clocks = {}local weights = {}for i = 1, distinctKeyCount do- local clock = tonumber(redis.call('GET', KEYS[firstClockKeyIndex + i - 1]))+ local raw = redis.call('GET', KEYS[firstClockKeyIndex + i - 1]) or ''+ local clockBase, clock = string.match(raw, '^([^:]+):(.+)$')+ local shiftedClock = tonumber(raw)++ if clockBase ~= nil then+ shiftedClock = tonumber(clock) - (base - tonumber(clockBase))+ endclocks[i] = virtualTime- if clock ~= nil and clock > virtualTime then- clocks[i] = clock+ if shiftedClock ~= nil and shiftedClock > virtualTime then+ clocks[i] = shiftedClockendlocal weight = tonumber(redis.call('HGET', weightsKey, ARGV[firstGroupKeyArgIndex + i - 1]))@@ -30,8 +38,8 @@ for i = 1, distinctKeyCount doweights[i] = weightend-local tickets = {}-local ticketCount = 0+local tickets = { string.format('%d', base) }+local ticketCount = 1local position = firstRequestArgIndexfor _ = 1, requestCount do@@ -63,7 +71,7 @@ for _ = 1, requestCount doendfor i = 1, distinctKeyCount do- redis.call('SET', KEYS[firstClockKeyIndex + i - 1], tostring(clocks[i]), 'PX', clockTtlMs)+ redis.call('SET', KEYS[firstClockKeyIndex + i - 1], string.format('%d', base) .. ':' .. clocks[i], 'PX', clockTtlMs)endreturn table.concat(tickets, ' ')diff --git a/packages/twenty-server/src/engine/core-modules/message-queue/drivers/fairness/fair-ticket-allocator.ts b/packages/twenty-server/src/engine/core-modules/message-queue/drivers/fairness/fair-ticket-allocator.tsindex 41ac7caf3df..7b4db93a6a3 100644--- a/packages/twenty-server/src/engine/core-modules/message-queue/drivers/fairness/fair-ticket-allocator.ts+++ b/packages/twenty-server/src/engine/core-modules/message-queue/drivers/fairness/fair-ticket-allocator.ts@@ -8,8 +8,14 @@ import { isDefined } from 'twenty-shared/utils';import { FAIR_DISPATCH_SCRIPT } from 'src/engine/core-modules/message-queue/drivers/fairness/fair-dispatch-script.constant';import {FAIR_CLOCK_TTL_MS,+ FAIR_REBASE_CHUNK_SIZE,+ FAIR_REBASE_GRACE_MS,+ FAIR_REBASE_MAX_CHUNKS,+ FAIR_REBASE_STEP,+ FAIR_REBASE_THRESHOLD,FAIR_TICKET_OFFSET,} from 'src/engine/core-modules/message-queue/drivers/fairness/fair-priority.constant';+import { FAIR_REBASE_SCRIPT } from 'src/engine/core-modules/message-queue/drivers/fairness/fair-rebase-script.constant';import { FAIR_TAG_SCRIPT } from 'src/engine/core-modules/message-queue/drivers/fairness/fair-tag-script.constant';import { type MessageQueue } from 'src/engine/core-modules/message-queue/message-queue.constants';@@ -17,14 +23,18 @@ export type FairTicketRequest = { groupKeys: string[]; jobCount: number };export type FairTicket = { tag: number; weight: number };+export type FairTickets = { base: number; tickets: FairTicket[] };+type FairTicketAllocatorOptions = {getClient: (queueName: MessageQueue) => Promise<RedisClient>;getJobKey: (job: { queueName: MessageQueue; jobId: string }) => string;keyPrefix: string;+ rebaseGraceMs?: number;};export class FairTicketAllocator {private readonly logger = new Logger(FairTicketAllocator.name);+ private readonly rebasesInProgress = new Map<MessageQueue, Promise<void>>();private readonly scriptShas = new Map<string, string>();constructor(private readonly options: FairTicketAllocatorOptions) {}@@ -35,7 +45,7 @@ export class FairTicketAllocator {}: {queueName: MessageQueue;requests: FairTicketRequest[];- }): Promise<FairTicket[] | undefined> {+ }): Promise<FairTickets | undefined> {const fairKeyPrefix = `${this.options.keyPrefix}:${queueName}:fair:`;const distinctGroupKeys = [...new Set(requests.flatMap(({ groupKeys }) => groupKeys)),@@ -50,6 +60,7 @@ export class FairTicketAllocator {script: FAIR_TAG_SCRIPT,keys: [`${fairKeyPrefix}vtime`,+ `${fairKeyPrefix}base`,`${fairKeyPrefix}weights`,...distinctGroupKeys.map((groupKey) => `${fairKeyPrefix}clock:${groupKey}`,@@ -67,12 +78,14 @@ export class FairTicketAllocator {]),],});- const values = String(rawTickets).split(' ').map(Number);+ const [base, ...values] = String(rawTickets).split(' ').map(Number);const tickets = requests.map((_, index) => ({tag: values[2 * index],weight: values[2 * index + 1],}));const hasValidTickets =+ Number.isInteger(base) &&+ base >= 0 &&values.length === 2 * requests.length &&tickets.every(({ tag, weight }) => Number.isFinite(tag) && weight > 0);@@ -80,7 +93,7 @@ export class FairTicketAllocator {throw new Error(`unexpected tickets "${String(rawTickets)}"`);}- return tickets;+ return { base, tickets };} catch (error) {this.logger.warn(`Failed to allocate the fair tickets on queue ${queueName}: ${error instanceof Error ? error.message : String(error)}`,@@ -100,15 +113,29 @@ export class FairTicketAllocator {const fairKeyPrefix = `${this.options.keyPrefix}:${queueName}:fair:`;try {- await this.runScript({- queueName,- script: FAIR_DISPATCH_SCRIPT,- keys: [- `${fairKeyPrefix}vtime`,- this.options.getJobKey({ queueName, jobId }),- ],- args: [FAIR_TICKET_OFFSET],- });+ const rebaseState = Number(+ await this.runScript({+ queueName,+ script: FAIR_DISPATCH_SCRIPT,+ keys: [+ `${fairKeyPrefix}vtime`,+ `${fairKeyPrefix}base`,+ `${fairKeyPrefix}rebase`,+ this.options.getJobKey({ queueName, jobId }),+ ],+ args: [FAIR_REBASE_THRESHOLD, FAIR_REBASE_STEP, FAIR_TICKET_OFFSET],+ }),+ );++ if (rebaseState === 1) {+ this.logger.log(+ `Rebasing the fair tickets of queue ${queueName} by ${FAIR_REBASE_STEP}`,+ );+ }++ if (rebaseState > 0) {+ void this.rebase(queueName);+ }} catch (error) {this.logger.warn(`Failed to record the fair dispatch of job ${jobId} on queue ${queueName}: ${error instanceof Error ? error.message : String(error)}`,@@ -116,6 +143,74 @@ export class FairTicketAllocator {}}+ rebase(queueName: MessageQueue): Promise<void> {+ const rebaseInProgress = this.rebasesInProgress.get(queueName);++ if (isDefined(rebaseInProgress)) {+ return rebaseInProgress;+ }++ const rebase = this.runRebase(queueName).finally(() => {+ this.rebasesInProgress.delete(queueName);+ });++ this.rebasesInProgress.set(queueName, rebase);++ return rebase;+ }++ private async runRebase(queueName: MessageQueue): Promise<void> {+ const queueKeyPrefix = `${this.options.keyPrefix}:${queueName}:`;++ try {+ for (let chunk = 0; chunk < FAIR_REBASE_MAX_CHUNKS; chunk += 1) {+ const rawResult = await this.runScript({+ queueName,+ script: FAIR_REBASE_SCRIPT,+ keys: [+ `${queueKeyPrefix}fair:rebase`,+ `${queueKeyPrefix}prioritized`,+ `${queueKeyPrefix}delayed`,+ `${queueKeyPrefix}active`,+ `${queueKeyPrefix}fair:base`,+ ],+ args: [+ FAIR_REBASE_STEP,+ FAIR_REBASE_CHUNK_SIZE,+ queueKeyPrefix,+ Date.now(),+ this.options.rebaseGraceMs ?? FAIR_REBASE_GRACE_MS,+ FAIR_TICKET_OFFSET,+ ],+ });++ if (!Array.isArray(rawResult)) {+ throw new Error(`unexpected rebase result "${String(rawResult)}"`);+ }++ const state = Number(rawResult[0]);++ if (state === 0) {+ return;+ }++ if (state === 2) {+ const waitMs = Math.max(0, Number(rawResult[1]));++ await new Promise<void>((resolve) => {+ setTimeout(resolve, waitMs).unref();+ });+ }+ }++ throw new Error(`still pending after ${FAIR_REBASE_MAX_CHUNKS} chunks`);+ } catch (error) {+ this.logger.warn(+ `Failed to rebase the fair tickets on queue ${queueName}: ${error instanceof Error ? error.message : String(error)}`,+ );+ }+ }+private async runScript({queueName,script,