Skip to content

The open alternative to Salesforce, designed for AI.

Order worker jobs by tenant virtual clocks behind IS_WORKER_FAIRNESS_ENABLED (#28990)

martmull2a7901250ec4
diff --git a/packages/twenty-server/package.json b/packages/twenty-server/package.json
index b99ad696310..5f48be974fe 100644
--- a/packages/twenty-server/package.json
+++ b/packages/twenty-server/package.json
@@ -195,6 +195,7 @@
"@types/graphql-fields": "^1.3.6",
"@types/graphql-upload": "^16.0.7",
"@types/html-to-text": "^9.0.4",
+ "@types/ioredis-mock": "8.2.8",
"@types/jest": "^30.0.0",
"@types/lodash.chunk": "^4.2.9",
"@types/lodash.compact": "^3.0.9",
@@ -221,6 +222,7 @@
"@types/supertest": "^2.0.11",
"@types/unzipper": "^0",
"@types/uuid": "^9.0.2",
+ "ioredis-mock": "8.13.1",
"jest": "29.7.0",
"msw": "2.12.14",
"node-fetch": "^2.7.0",
diff --git a/packages/twenty-server/src/engine/core-modules/application/application-job/services/application-job.service.ts b/packages/twenty-server/src/engine/core-modules/application/application-job/services/application-job.service.ts
index 52031d49b68..b82a765ef60 100644
--- a/packages/twenty-server/src/engine/core-modules/application/application-job/services/application-job.service.ts
+++ b/packages/twenty-server/src/engine/core-modules/application/application-job/services/application-job.service.ts
@@ -8,7 +8,10 @@ import {
import { isDefined } from 'twenty-shared/utils';
import { v4 } from 'uuid';
+import { findActiveFlatApplicationById } from 'src/engine/core-modules/application/utils/find-active-flat-application-by-id.util';
import { buildJobStatus } from 'src/engine/core-modules/message-queue/utils/build-job-status.util';
+import { buildApplicationGroupKey } from 'src/engine/core-modules/message-queue/utils/build-application-group-key.util';
+import { buildWorkspaceGroupKey } from 'src/engine/core-modules/message-queue/utils/build-workspace-group-key.util';
import { ENQUEUE_JOB_DEFAULT_RETRY_LIMIT } from 'src/engine/core-modules/application/application-job/constants/enqueue-job-default-retry-limit.constant';
import { ENQUEUE_JOB_PRIORITY } from 'src/engine/core-modules/application/application-job/constants/enqueue-job-priority.constant';
import { MAX_JOBS_PER_STATUS_READ } from 'src/engine/core-modules/application/application-job/constants/max-jobs-per-status-read.constant';
@@ -87,9 +90,10 @@ export class ApplicationJobService {
const jobItems = this.getJobItems(input);
- const { flatLogicFunctionMaps } =
+ const { flatLogicFunctionMaps, flatApplicationMaps } =
await this.workspaceCacheService.getOrRecompute(workspaceId, [
'flatLogicFunctionMaps',
+ 'flatApplicationMaps',
]);
const flatLogicFunction = findFlatEntityByUniversalIdentifier({
@@ -109,6 +113,10 @@ export class ApplicationJobService {
}
const jobIds = jobItems.map((jobItem) => jobItem.jobId ?? v4());
+ const application = findActiveFlatApplicationById(
+ flatApplicationMaps,
+ applicationId,
+ );
await this.messageQueueService.bulkAdd<LogicFunctionTriggerJobData>(
LogicFunctionTriggerJob.name,
@@ -126,6 +134,12 @@ export class ApplicationJobService {
retryLimit: input.retryLimit ?? ENQUEUE_JOB_DEFAULT_RETRY_LIMIT,
backoff: LOGIC_FUNCTION_QUEUE_RETRY_BACKOFF,
priority: ENQUEUE_JOB_PRIORITY,
+ groupKeys: [
+ buildWorkspaceGroupKey(workspaceId),
+ ...(isDefined(application)
+ ? [buildApplicationGroupKey(application.universalIdentifier)]
+ : []),
+ ],
...(isDefined(input.delayMs) ? { delay: input.delayMs } : {}),
},
);
diff --git a/packages/twenty-server/src/engine/core-modules/logic-function/logic-function-trigger/triggers/cron/cron-trigger.cron.job.ts b/packages/twenty-server/src/engine/core-modules/logic-function/logic-function-trigger/triggers/cron/cron-trigger.cron.job.ts
index 64d28285971..03a7951ba62 100644
--- a/packages/twenty-server/src/engine/core-modules/logic-function/logic-function-trigger/triggers/cron/cron-trigger.cron.job.ts
+++ b/packages/twenty-server/src/engine/core-modules/logic-function/logic-function-trigger/triggers/cron/cron-trigger.cron.job.ts
@@ -19,6 +19,8 @@ import { MessageQueueService } from 'src/engine/core-modules/message-queue/servi
import { ThrottlerException } from 'src/engine/core-modules/throttler/throttler.exception';
import { WorkspaceEntity } from 'src/engine/core-modules/workspace/workspace.entity';
import { LOGIC_FUNCTION_QUEUE_RETRY_BACKOFF } from 'src/engine/core-modules/logic-function/logic-function-trigger/constants/logic-function-queue-retry-backoff.constant';
+import { buildApplicationGroupKey } from 'src/engine/core-modules/message-queue/utils/build-application-group-key.util';
+import { buildWorkspaceGroupKey } from 'src/engine/core-modules/message-queue/utils/build-workspace-group-key.util';
import { buildApplicationJobThrottledLogEntry } from 'src/engine/core-modules/logic-function/logic-function-trigger/utils/build-application-job-throttled-log-entry.util';
import {
LogicFunctionTriggerJob,
@@ -101,7 +103,10 @@ export class CronTriggerCronJob {
const applicationRegistrationId =
application?.applicationRegistrationId;
- if (!isDefined(applicationRegistrationId)) {
+ if (
+ !isDefined(application) ||
+ !isDefined(applicationRegistrationId)
+ ) {
continue;
}
@@ -115,14 +120,12 @@ export class CronTriggerCronJob {
throw error;
}
- if (isDefined(application)) {
- this.applicationRepeatedEventRecorderService.record({
- entry: buildApplicationJobThrottledLogEntry({
- workspaceId: activeWorkspace.id,
- application,
- }),
- });
- }
+ this.applicationRepeatedEventRecorderService.record({
+ entry: buildApplicationJobThrottledLogEntry({
+ workspaceId: activeWorkspace.id,
+ application,
+ }),
+ });
continue;
}
@@ -143,6 +146,10 @@ export class CronTriggerCronJob {
{
retryLimit: 10,
backoff: LOGIC_FUNCTION_QUEUE_RETRY_BACKOFF,
+ groupKeys: [
+ buildWorkspaceGroupKey(activeWorkspace.id),
+ buildApplicationGroupKey(application.universalIdentifier),
+ ],
},
);
}
diff --git a/packages/twenty-server/src/engine/core-modules/logic-function/logic-function-trigger/triggers/database-event/call-database-event-trigger-jobs.job.ts b/packages/twenty-server/src/engine/core-modules/logic-function/logic-function-trigger/triggers/database-event/call-database-event-trigger-jobs.job.ts
index f1a83127d1a..14a30ab159e 100644
--- a/packages/twenty-server/src/engine/core-modules/logic-function/logic-function-trigger/triggers/database-event/call-database-event-trigger-jobs.job.ts
+++ b/packages/twenty-server/src/engine/core-modules/logic-function/logic-function-trigger/triggers/database-event/call-database-event-trigger-jobs.job.ts
@@ -15,6 +15,8 @@ import { ApplicationJobEnqueueThrottlerService } from 'src/engine/core-modules/m
import { MessageQueueService } from 'src/engine/core-modules/message-queue/services/message-queue.service';
import { ThrottlerException } from 'src/engine/core-modules/throttler/throttler.exception';
import { LOGIC_FUNCTION_QUEUE_RETRY_BACKOFF } from 'src/engine/core-modules/logic-function/logic-function-trigger/constants/logic-function-queue-retry-backoff.constant';
+import { buildApplicationGroupKey } from 'src/engine/core-modules/message-queue/utils/build-application-group-key.util';
+import { buildWorkspaceGroupKey } from 'src/engine/core-modules/message-queue/utils/build-workspace-group-key.util';
import { buildApplicationJobThrottledLogEntry } from 'src/engine/core-modules/logic-function/logic-function-trigger/utils/build-application-job-throttled-log-entry.util';
import { buildLogicFunctionTriggerJobIds } from 'src/engine/core-modules/logic-function/logic-function-trigger/triggers/database-event/utils/build-logic-function-trigger-job-ids';
import { findLogicFunctionsTriggeredByEventName } from 'src/engine/core-modules/logic-function/logic-function-trigger/triggers/database-event/utils/find-logic-functions-triggered-by-event-name';
@@ -181,6 +183,10 @@ export class CallDatabaseEventTriggerJobsJob {
retryLimit: 3,
backoff: LOGIC_FUNCTION_QUEUE_RETRY_BACKOFF,
doNotCheckDuplicatesBasedOnPrefix: true,
+ groupKeys: [
+ buildWorkspaceGroupKey(workspaceEventBatch.workspaceId),
+ buildApplicationGroupKey(application.universalIdentifier),
+ ],
},
);
}
diff --git a/packages/twenty-server/src/engine/core-modules/message-queue/drivers/__tests__/bullmq.driver.spec.ts b/packages/twenty-server/src/engine/core-modules/message-queue/drivers/__tests__/bullmq.driver.spec.ts
index 89919f1188b..fad79ebfa27 100644
--- a/packages/twenty-server/src/engine/core-modules/message-queue/drivers/__tests__/bullmq.driver.spec.ts
+++ b/packages/twenty-server/src/engine/core-modules/message-queue/drivers/__tests__/bullmq.driver.spec.ts
@@ -3,6 +3,7 @@ import { type Job, Worker } from 'bullmq';
import { DEFAULT_JOB_PRIORITY } from 'src/engine/core-modules/message-queue/constants/default-job-priority.constant';
import { QUEUE_JOB_CHANGED_EVENT } from 'src/engine/core-modules/message-queue/constants/queue-job-changed-event.constant';
import { BullMQDriver } from 'src/engine/core-modules/message-queue/drivers/bullmq.driver';
+import { FAIR_TICKET_OFFSET } from 'src/engine/core-modules/message-queue/drivers/fairness/fair-priority.constant';
import { MessageQueue } from 'src/engine/core-modules/message-queue/message-queue.constants';
import { JOB_LATENCY_MS_BUCKET_BOUNDARIES } from 'src/engine/core-modules/metrics/constants/job-latency-ms-bucket-boundaries.constant';
import { MetricsKeys } from 'src/engine/core-modules/metrics/types/metrics-keys.type';
@@ -15,6 +16,8 @@ const mockAddBulk = jest.fn();
const mockSetGlobalConcurrency = jest.fn();
const mockRemoveGlobalConcurrency = jest.fn();
const mockGetJobCountByTypes = jest.fn();
+const mockTwentyConfigService = { get: jest.fn().mockReturnValue(false) };
+const mockAllocateTickets = jest.fn();
jest.mock('bullmq', () => ({
Queue: jest.fn().mockImplementation(() => ({
@@ -29,17 +32,27 @@ jest.mock('bullmq', () => ({
})),
Worker: jest.fn().mockImplementation(() => ({ on: jest.fn() })),
MetricsTime: { ONE_WEEK: 1 },
+ PRIORITY_LIMIT: jest.requireActual('bullmq').PRIORITY_LIMIT,
}));
jest.mock('uuid', () => ({ v4: () => 'generated-uuid-000000000000000000000' }));
+jest.mock(
+ 'src/engine/core-modules/message-queue/drivers/fairness/fair-ticket-allocator',
+ () => ({
+ FairTicketAllocator: jest
+ .fn()
+ .mockImplementation(() => ({ allocateTickets: mockAllocateTickets })),
+ }),
+);
+
const WAITING_JOB_ID = 'sync-catalog-ws-1-5c98b035-5b09-4550-a4fb-b52056c494d1';
describe('BullMQDriver deduplication', () => {
const driver = new BullMQDriver(
{} as never,
{} as never,
- {} as never,
+ mockTwentyConfigService as never,
{} as never,
);
@@ -177,7 +190,7 @@ describe('BullMQDriver progress', () => {
const driver = new BullMQDriver(
{} as never,
{ recordHistogram: jest.fn() } as never,
- {} as never,
+ mockTwentyConfigService as never,
{} as never,
);
@@ -237,7 +250,7 @@ describe('BullMQDriver job change events', () => {
const driver = new BullMQDriver(
{} as never,
{ recordHistogram: jest.fn() } as never,
- {} as never,
+ mockTwentyConfigService as never,
eventEmitter as never,
);
@@ -314,7 +327,7 @@ describe('BullMQDriver queue wait metric', () => {
const driver = new BullMQDriver(
{} as never,
{ recordHistogram } as never,
- {} as never,
+ mockTwentyConfigService as never,
{} as never,
);
@@ -414,7 +427,7 @@ describe('BullMQDriver queue job count gauges', () => {
const driver = new BullMQDriver(
{} as never,
{ createMultiObservableGauge } as never,
- {} as never,
+ mockTwentyConfigService as never,
{} as never,
);
@@ -470,7 +483,7 @@ describe('BullMQDriver global concurrency', () => {
const driver = new BullMQDriver(
{} as never,
{} as never,
- {} as never,
+ mockTwentyConfigService as never,
{} as never,
);
@@ -496,3 +509,114 @@ describe('BullMQDriver global concurrency', () => {
expect(mockSetGlobalConcurrency).not.toHaveBeenCalled();
});
});
+
+describe('BullMQDriver fair tickets', () => {
+ const incrementCounterForEvent = jest.fn();
+ const driver = new BullMQDriver(
+ {} as never,
+ { incrementCounterForEvent } as never,
+ {
+ get: jest.fn((key: string) => key === 'IS_WORKER_FAIRNESS_ENABLED'),
+ } as never,
+ {} as never,
+ );
+
+ driver.register(MessageQueue.workspaceQueue);
+
+ beforeEach(() => {
+ jest.clearAllMocks();
+ mockGetRanges.mockResolvedValue([]);
+ mockAdd.mockImplementation(async (_name, _data, opts) => ({
+ id: opts.jobId,
+ }));
+ mockAddBulk.mockImplementation(async (jobs) =>
+ jobs.map((job: { opts: { jobId?: string } }, index: number) => ({
+ id: job.opts.jobId ?? `auto-${index}`,
+ })),
+ );
+ });
+
+ it('ranks a job by the ticket of its workspace', async () => {
+ mockAllocateTickets.mockResolvedValue([{ tag: 7, weight: 1 }]);
+
+ await driver.add(MessageQueue.workspaceQueue, 'job', {
+ workspaceId: 'ws-1',
+ });
+
+ expect(mockAllocateTickets).toHaveBeenCalledWith({
+ queueName: MessageQueue.workspaceQueue,
+ requests: [{ groupKeys: ['workspace:ws-1'], jobCount: 1 }],
+ });
+ expect(mockAdd).toHaveBeenCalledWith(
+ 'job',
+ { workspaceId: 'ws-1' },
+ expect.objectContaining({ priority: FAIR_TICKET_OFFSET + 7 }),
+ );
+ });
+
+ it('allocates one ticket per group of a bulk and spaces its jobs by the weight', async () => {
+ mockAllocateTickets.mockResolvedValue([
+ { tag: 10, weight: 0.5 },
+ { tag: 0, weight: 1 },
+ ]);
+
+ await driver.bulkAdd(MessageQueue.workspaceQueue, 'job', [
+ { data: { workspaceId: 'ws-1' } },
+ { data: { workspaceId: 'ws-2' } },
+ { data: { workspaceId: 'ws-1' } },
+ ]);
+
+ expect(mockAllocateTickets).toHaveBeenCalledWith({
+ queueName: MessageQueue.workspaceQueue,
+ requests: [
+ { groupKeys: ['workspace:ws-1'], jobCount: 2 },
+ { groupKeys: ['workspace:ws-2'], jobCount: 1 },
+ ],
+ });
+ expect(
+ mockAddBulk.mock.calls[0][0].map(
+ ({ opts }: { opts: { priority: number } }) => opts.priority,
+ ),
+ ).toEqual([
+ FAIR_TICKET_OFFSET + 10,
+ FAIR_TICKET_OFFSET,
+ FAIR_TICKET_OFFSET + 12,
+ ]);
+ });
+
+ it('keeps the static priority and counts a fallback when no ticket comes back', async () => {
+ mockAllocateTickets.mockResolvedValue(undefined);
+
+ await driver.add(
+ MessageQueue.workspaceQueue,
+ 'job',
+ { workspaceId: 'ws-1' },
+ { priority: 4 },
+ );
+
+ expect(mockAdd).toHaveBeenCalledWith(
+ 'job',
+ { workspaceId: 'ws-1' },
+ expect.objectContaining({ priority: 4 }),
+ );
+ expect(incrementCounterForEvent).toHaveBeenCalledWith(
+ expect.objectContaining({ key: MetricsKeys.JobFairTagFallback }),
+ );
+ });
+
+ it('leaves a delayed job on its static priority without charging a clock', async () => {
+ await driver.add(
+ MessageQueue.workspaceQueue,
+ 'job',
+ { workspaceId: 'ws-1' },
+ { delay: 1_000, priority: 4 },
+ );
+
+ expect(mockAllocateTickets).not.toHaveBeenCalled();
+ expect(mockAdd).toHaveBeenCalledWith(
+ 'job',
+ { workspaceId: 'ws-1' },
+ expect.objectContaining({ delay: 1_000, priority: 4 }),
+ );
+ });
+});
diff --git a/packages/twenty-server/src/engine/core-modules/message-queue/drivers/bullmq.driver.ts b/packages/twenty-server/src/engine/core-modules/message-queue/drivers/bullmq.driver.ts
index 15d4585bc50..5f687f16cf8 100644
--- a/packages/twenty-server/src/engine/core-modules/message-queue/drivers/bullmq.driver.ts
+++ b/packages/twenty-server/src/engine/core-modules/message-queue/drivers/bullmq.driver.ts
@@ -39,6 +39,7 @@ import { type MessageQueueWorkerOptions } from 'src/engine/core-modules/message-
import { DEFAULT_JOB_PRIORITY } from 'src/engine/core-modules/message-queue/constants/default-job-priority.constant';
import { QUEUE_JOB_CHANGED_EVENT } from 'src/engine/core-modules/message-queue/constants/queue-job-changed-event.constant';
import { QUEUE_RETENTION } from 'src/engine/core-modules/message-queue/constants/queue-retention.constants';
+import { FairQueueScheduler } from 'src/engine/core-modules/message-queue/drivers/fairness/fair-queue-scheduler';
import { type MessageQueue } from 'src/engine/core-modules/message-queue/message-queue.constants';
import { type QueueJobChangedEvent } from 'src/engine/core-modules/message-queue/types/queue-job-changed-event.type';
import { getJobKey } from 'src/engine/core-modules/message-queue/utils/get-job-key.util';
@@ -74,13 +75,21 @@ export class BullMQDriver
Record<MessageQueue, MessageQueueWorkerOptions>
> = {};
private workersDrain: Promise<void> | undefined;
+ private readonly fairQueueScheduler: FairQueueScheduler;
constructor(
private options: BullMQDriverOptions,
private metricsService: MetricsService,
private twentyConfigService: TwentyConfigService,
private eventEmitter: EventEmitter2,
- ) {}
+ ) {
+ this.fairQueueScheduler = new FairQueueScheduler({
+ getClient: (queueName) => this.queueMap[queueName].client,
+ keyPrefix: options.prefix ?? 'bull',
+ metricsService,
+ twentyConfigService,
+ });
+ }
onModuleInit() {
this.metricsService.createMultiObservableGauge({
@@ -520,8 +529,17 @@ export class BullMQDriver
}
const queueOptions = this.buildJobsOptions(options);
+ const [priority] = await this.fairQueueScheduler.computeJobPriorities({
+ queueName,
+ jobsData: [data],
+ options,
+ staticPriority: queueOptions.priority,
+ });
- const job = await this.queueMap[queueName].add(jobName, data, queueOptions);
+ const job = await this.queueMap[queueName].add(jobName, data, {
+ ...queueOptions,
+ priority,
+ });
return job.id;
}
@@ -572,6 +590,12 @@ export class BullMQDriver
}
const queueOptions = this.buildJobsOptions(options);
+ const priorities = await this.fairQueueScheduler.computeJobPriorities({
+ queueName,
+ jobsData: jobsToAdd.map(({ data }) => data),
+ options,
+ staticPriority: queueOptions.priority,
+ });
const addedJobs = await this.queueMap[queueName].addBulk(
jobsToAdd.map(({ data, jobId }, index) => ({
@@ -579,6 +603,7 @@ export class BullMQDriver
data,
opts: {
...queueOptions,
+ priority: priorities[index],
jobId:
jobId ??
(queueOptions.jobId ? `${queueOptions.jobId}-${index}` : undefined),
diff --git a/packages/twenty-server/src/engine/core-modules/message-queue/drivers/fairness/__tests__/fair-ticket-allocator.spec.ts b/packages/twenty-server/src/engine/core-modules/message-queue/drivers/fairness/__tests__/fair-ticket-allocator.spec.ts
new file mode 100644
index 00000000000..93bba9f8ded
--- /dev/null
+++ b/packages/twenty-server/src/engine/core-modules/message-queue/drivers/fairness/__tests__/fair-ticket-allocator.spec.ts
@@ -0,0 +1,153 @@
+import { Logger } from '@nestjs/common';
+
+import { type Redis } from 'ioredis';
+import RedisMock from 'ioredis-mock';
+
+import { FairTicketAllocator } from 'src/engine/core-modules/message-queue/drivers/fairness/fair-ticket-allocator';
+import { MessageQueue } from 'src/engine/core-modules/message-queue/message-queue.constants';
+
+const QUEUE = MessageQueue.logicFunctionQueue;
+const KEY_PREFIX = 'bull';
+const QUEUE_KEY_PREFIX = `${KEY_PREFIX}:${QUEUE}:`;
+const VIRTUAL_TIME_KEY = `${QUEUE_KEY_PREFIX}fair:vtime`;
+const WEIGHTS_KEY = `${QUEUE_KEY_PREFIX}fair:weights`;
+
+const createAllocator = async () => {
+ const redis: Redis = new RedisMock();
+
+ await redis.flushall();
+
+ const client = {
+ runCommand: async (name: string, args: (string | number)[]) => {
+ const [script, numberOfKeys, ...scriptArguments] = args;
+
+ if (name === 'evalsha') {
+ return redis.evalsha(
+ String(script),
+ Number(numberOfKeys),
+ ...scriptArguments,
+ );
+ }
+
+ if (name !== 'eval') {
+ throw new Error(`Unexpected Redis command ${name}`);
+ }
+
+ return redis.eval(
+ String(script),
+ Number(numberOfKeys),
+ ...scriptArguments,
+ );
+ },
+ };
+
+ return {
+ redis,
+ allocator: new FairTicketAllocator({
+ getClient: async () => client as never,
+ keyPrefix: KEY_PREFIX,
+ }),
+ readClock: async (groupKey: string) =>
+ Number(await redis.get(`${QUEUE_KEY_PREFIX}fair:clock:${groupKey}`)),
+ };
+};
+
+const createBrokenAllocator = () =>
+ new FairTicketAllocator({
+ getClient: async () => {
+ throw new Error('Redis connection lost');
+ },
+ keyPrefix: KEY_PREFIX,
+ });
+
+describe('FairTicketAllocator', () => {
+ beforeAll(() => {
+ jest.spyOn(Logger.prototype, 'warn').mockImplementation(() => undefined);
+ });
+
+ describe('allocateTickets', () => {
+ it('tags a request at its latest clock and charges each key for its own batch only', async () => {
+ const { allocator, readClock } = await createAllocator();
+
+ expect(
+ await allocator.allocateTickets({
+ queueName: QUEUE,
+ requests: [
+ { groupKeys: ['workspace:X', 'application:H'], jobCount: 100 },
+ { groupKeys: ['workspace:W', 'application:H'], jobCount: 1 },
+ { groupKeys: ['workspace:W', 'application:Q'], jobCount: 1 },
+ ],
+ }),
+ ).toEqual([
+ { tag: 0, weight: 1 },
+ { tag: 100, weight: 1 },
+ { tag: 1, weight: 1 },
+ ]);
+ expect(await readClock('application:H')).toBe(101);
+ expect(await readClock('workspace:W')).toBe(2);
+ });
+
+ it('tags a request at the virtual time when it is ahead of its clocks', async () => {
+ const { redis, allocator, readClock } = await createAllocator();
+
+ await redis.set(VIRTUAL_TIME_KEY, 5);
+
+ expect(
+ await allocator.allocateTickets({
+ queueName: QUEUE,
+ requests: [{ groupKeys: ['workspace:B'], jobCount: 2 }],
+ }),
+ ).toEqual([{ tag: 5, weight: 1 }]);
+ expect(await readClock('workspace:B')).toBe(7);
+ });
+
+ it('weighs a request by the product of its key weights and ignores a malformed weight', async () => {
+ const { redis, allocator } = await createAllocator();
+
+ await redis.hset(WEIGHTS_KEY, 'application:P', '2');
+ await redis.hset(WEIGHTS_KEY, 'workspace:W', 'not-a-number');
+ await redis.hset(WEIGHTS_KEY, 'application:N', '-1');
+
+ expect(
+ await allocator.allocateTickets({
+ queueName: QUEUE,
+ requests: [
+ { groupKeys: ['workspace:W', 'application:P'], jobCount: 2 },
+ { groupKeys: ['workspace:W', 'application:N'], jobCount: 1 },
+ ],
+ }),
+ ).toEqual([
+ { tag: 0, weight: 2 },
+ { tag: 1, weight: 1 },
+ ]);
+ });
+
+ it('returns nothing when Redis is unavailable', async () => {
+ expect(
+ await createBrokenAllocator().allocateTickets({
+ queueName: QUEUE,
+ requests: [{ groupKeys: ['workspace:A'], jobCount: 1 }],
+ }),
+ ).toBeUndefined();
+ });
+
+ it('retries with EVAL only when Redis does not know the script', async () => {
+ const runCommand = jest
+ .fn()
+ .mockRejectedValue(new Error('READONLY replica is read only'));
+ const allocator = new FairTicketAllocator({
+ getClient: async () => ({ runCommand }) as never,
+ keyPrefix: KEY_PREFIX,
+ });
+
+ expect(
+ await allocator.allocateTickets({
+ queueName: QUEUE,
+ requests: [{ groupKeys: ['workspace:A'], jobCount: 1 }],
+ }),
+ ).toBeUndefined();
+ expect(runCommand).toHaveBeenCalledTimes(1);
+ expect(runCommand).toHaveBeenCalledWith('evalsha', expect.any(Array));
+ });
+ });
+});
diff --git a/packages/twenty-server/src/engine/core-modules/message-queue/drivers/fairness/fair-priority.constant.ts b/packages/twenty-server/src/engine/core-modules/message-queue/drivers/fairness/fair-priority.constant.ts
new file mode 100644
index 00000000000..5be5183a654
--- /dev/null
+++ b/packages/twenty-server/src/engine/core-modules/message-queue/drivers/fairness/fair-priority.constant.ts
@@ -0,0 +1,3 @@
+export const FAIR_TICKET_OFFSET = 2;
+
+export const FAIR_CLOCK_TTL_MS = 24 * 60 * 60 * 1000;
diff --git a/packages/twenty-server/src/engine/core-modules/message-queue/drivers/fairness/fair-queue-scheduler.ts b/packages/twenty-server/src/engine/core-modules/message-queue/drivers/fairness/fair-queue-scheduler.ts
new file mode 100644
index 00000000000..24b0cc766ff
--- /dev/null
+++ b/packages/twenty-server/src/engine/core-modules/message-queue/drivers/fairness/fair-queue-scheduler.ts
@@ -0,0 +1,105 @@
+import { PRIORITY_LIMIT, type RedisClient } from 'bullmq';
+import { isDefined, isNonEmptyArray } from 'twenty-shared/utils';
+
+import { FAIR_TICKET_OFFSET } from 'src/engine/core-modules/message-queue/drivers/fairness/fair-priority.constant';
+import { FairTicketAllocator } from 'src/engine/core-modules/message-queue/drivers/fairness/fair-ticket-allocator';
+import { type QueueJobOptions } from 'src/engine/core-modules/message-queue/drivers/interfaces/job-options.interface';
+import { type MessageQueue } from 'src/engine/core-modules/message-queue/message-queue.constants';
+import { getDefaultGroupKeys } from 'src/engine/core-modules/message-queue/utils/get-default-group-keys.util';
+import { type MetricsService } from 'src/engine/core-modules/metrics/metrics.service';
+import { MetricsKeys } from 'src/engine/core-modules/metrics/types/metrics-keys.type';
+import { type TwentyConfigService } from 'src/engine/core-modules/twenty-config/twenty-config.service';
+
+type FairQueueSchedulerOptions = {
+ getClient: (queueName: MessageQueue) => Promise<RedisClient>;
+ keyPrefix: string;
+ metricsService: MetricsService;
+ twentyConfigService: TwentyConfigService;
+};
+
+export class FairQueueScheduler {
+ private readonly fairTicketAllocator: FairTicketAllocator;
+
+ constructor(private readonly options: FairQueueSchedulerOptions) {
+ this.fairTicketAllocator = new FairTicketAllocator({
+ getClient: options.getClient,
+ keyPrefix: options.keyPrefix,
+ });
+ }
+
+ async computeJobPriorities({
+ queueName,
+ jobsData,
+ options,
+ staticPriority,
+ }: {
+ queueName: MessageQueue;
+ jobsData: unknown[];
+ options?: Pick<QueueJobOptions, 'groupKeys' | 'delay'>;
+ staticPriority: number | undefined;
+ }): Promise<(number | undefined)[]> {
+ const staticPriorities = jobsData.map(() => staticPriority);
+
+ if (!this.isFairnessEnabled() || (options?.delay ?? 0) > 0) {
+ return staticPriorities;
+ }
+
+ const requestByGroupKeys = new Map<
+ string,
+ { groupKeys: string[]; jobIndexes: number[] }
+ >();
+
+ jobsData.forEach((jobData, jobIndex) => {
+ const groupKeys = isNonEmptyArray(options?.groupKeys)
+ ? [...new Set(options.groupKeys)]
+ : getDefaultGroupKeys(jobData);
+ const signature = JSON.stringify(groupKeys);
+ const request = requestByGroupKeys.get(signature) ?? {
+ groupKeys,
+ jobIndexes: [],
+ };
+
+ request.jobIndexes.push(jobIndex);
+ requestByGroupKeys.set(signature, request);
+ });
+
+ const requests = [...requestByGroupKeys.values()];
+ const tickets = await this.fairTicketAllocator.allocateTickets({
+ queueName,
+ requests: requests.map(({ groupKeys, jobIndexes }) => ({
+ groupKeys,
+ jobCount: jobIndexes.length,
+ })),
+ });
+
+ if (!isDefined(tickets)) {
+ void this.options.metricsService.incrementCounterForEvent({
+ key: MetricsKeys.JobFairTagFallback,
+ attributes: { queue: queueName },
+ shouldStoreInCache: false,
+ });
+
+ return staticPriorities;
+ }
+
+ const priorities = [...staticPriorities];
+
+ requests.forEach(({ jobIndexes }, requestIndex) => {
+ const { tag, weight } = tickets[requestIndex];
+
+ jobIndexes.forEach((jobIndex, position) => {
+ const ticket = Math.floor(tag + position / weight);
+
+ priorities[jobIndex] =
+ FAIR_TICKET_OFFSET +
+ Math.min(PRIORITY_LIMIT - FAIR_TICKET_OFFSET, Math.max(0, ticket));
+ });
+ });
+
+ return priorities;
+ }
+
+ private isFairnessEnabled(): boolean {
+ return this.options.twentyConfigService.get('IS_WORKER_FAIRNESS_ENABLED');
+ }
+}
diff --git a/packages/twenty-server/src/engine/core-modules/message-queue/drivers/fairness/fair-tag-script.constant.ts b/packages/twenty-server/src/engine/core-modules/message-queue/drivers/fairness/fair-tag-script.constant.ts
new file mode 100644
index 00000000000..fc25d3d1c0f
--- /dev/null
+++ b/packages/twenty-server/src/engine/core-modules/message-queue/drivers/fairness/fair-tag-script.constant.ts
@@ -0,0 +1,70 @@
+export const FAIR_TAG_SCRIPT = `
+local virtualTimeKey = KEYS[1]
+local weightsKey = KEYS[2]
+local firstClockKeyIndex = 3
+local clockTtlMs = ARGV[1]
+local distinctKeyCount = tonumber(ARGV[2])
+local firstGroupKeyArgIndex = 3
+local requestCount = tonumber(ARGV[firstGroupKeyArgIndex + distinctKeyCount])
+local firstRequestArgIndex = firstGroupKeyArgIndex + distinctKeyCount + 1
+
+local virtualTime = tonumber(redis.call('GET', virtualTimeKey) or '0')
+local clocks = {}
+local weights = {}
+
+for i = 1, distinctKeyCount do
+ local clock = tonumber(redis.call('GET', KEYS[firstClockKeyIndex + i - 1]))
+
+ clocks[i] = virtualTime
+
+ if clock ~= nil and clock > virtualTime then
+ clocks[i] = clock
+ end
+
+ local weight = tonumber(redis.call('HGET', weightsKey, ARGV[firstGroupKeyArgIndex + i - 1]))
+
+ if weight == nil or weight <= 0 then
+ weight = 1
+ end
+
+ weights[i] = weight
+end
+
+local tickets = {}
+local ticketCount = 0
+local position = firstRequestArgIndex
+
+for _ = 1, requestCount do
+ local jobCount = tonumber(ARGV[position])
+ local keyCount = tonumber(ARGV[position + 1])
+ local tag = virtualTime
+ local weight = 1
+
+ for k = 1, keyCount do
+ local index = tonumber(ARGV[position + 1 + k])
+
+ if clocks[index] > tag then
+ tag = clocks[index]
+ end
+
+ weight = weight * weights[index]
+ end
+
+ for k = 1, keyCount do
+ local index = tonumber(ARGV[position + 1 + k])
+
+ clocks[index] = clocks[index] + jobCount / weight
+ end
+
+ tickets[ticketCount + 1] = tostring(tag)
+ tickets[ticketCount + 2] = tostring(weight)
+ ticketCount = ticketCount + 2
+ position = position + 2 + keyCount
+end
+
+for i = 1, distinctKeyCount do
+ redis.call('SET', KEYS[firstClockKeyIndex + i - 1], tostring(clocks[i]), 'PX', clockTtlMs)
+end
+
+return table.concat(tickets, ' ')
+`;
diff --git a/packages/twenty-server/src/engine/core-modules/message-queue/drivers/fairness/fair-ticket-allocator.ts b/packages/twenty-server/src/engine/core-modules/message-queue/drivers/fairness/fair-ticket-allocator.ts
new file mode 100644
index 00000000000..4b50f1920b1
--- /dev/null
+++ b/packages/twenty-server/src/engine/core-modules/message-queue/drivers/fairness/fair-ticket-allocator.ts
@@ -0,0 +1,107 @@
+import { createHash } from 'node:crypto';
+
+import { Logger } from '@nestjs/common';
+
+import { type RedisClient } from 'bullmq';
+
+import { FAIR_CLOCK_TTL_MS } from 'src/engine/core-modules/message-queue/drivers/fairness/fair-priority.constant';
+import { FAIR_TAG_SCRIPT } from 'src/engine/core-modules/message-queue/drivers/fairness/fair-tag-script.constant';
+import { type MessageQueue } from 'src/engine/core-modules/message-queue/message-queue.constants';
+
+export type FairTicketRequest = { groupKeys: string[]; jobCount: number };
+
+export type FairTicket = { tag: number; weight: number };
+
+type FairTicketAllocatorOptions = {
+ getClient: (queueName: MessageQueue) => Promise<RedisClient>;
+ keyPrefix: string;
+};
+
+const FAIR_TAG_SCRIPT_SHA = createHash('sha1')
+ .update(FAIR_TAG_SCRIPT)
+ .digest('hex');
+
+export class FairTicketAllocator {
+ private readonly logger = new Logger(FairTicketAllocator.name);
+
+ constructor(private readonly options: FairTicketAllocatorOptions) {}
+
+ async allocateTickets({
+ queueName,
+ requests,
+ }: {
+ queueName: MessageQueue;
+ requests: FairTicketRequest[];
+ }): Promise<FairTicket[] | undefined> {
+ const fairKeyPrefix = `${this.options.keyPrefix}:${queueName}:fair:`;
+ const distinctGroupKeys = [
+ ...new Set(requests.flatMap(({ groupKeys }) => groupKeys)),
+ ];
+ const groupKeyIndexes = new Map(
+ distinctGroupKeys.map((groupKey, index) => [groupKey, index + 1]),
+ );
+ const keys = [
+ `${fairKeyPrefix}vtime`,
+ `${fairKeyPrefix}weights`,
+ ...distinctGroupKeys.map(
+ (groupKey) => `${fairKeyPrefix}clock:${groupKey}`,
+ ),
+ ];
+ const args = [
+ FAIR_CLOCK_TTL_MS,
+ distinctGroupKeys.length,
+ ...distinctGroupKeys,
+ requests.length,
+ ...requests.flatMap(({ groupKeys, jobCount }) => [
+ jobCount,
+ groupKeys.length,
+ ...groupKeys.map((groupKey) => groupKeyIndexes.get(groupKey) ?? 0),
+ ]),
+ ];
+
+ try {
+ const client = await this.options.getClient(queueName);
+ const rawTickets = await client
+ .runCommand('evalsha', [
+ FAIR_TAG_SCRIPT_SHA,
+ keys.length,
+ ...keys,
+ ...args,
+ ])
+ .catch((error: unknown) => {
+ if (
+ !(error instanceof Error && error.message.startsWith('NOSCRIPT'))
+ ) {
+ throw error;
+ }
+
+ return client.runCommand('eval', [
+ FAIR_TAG_SCRIPT,
+ keys.length,
+ ...keys,
+ ...args,
+ ]);
+ });
+ const values = String(rawTickets).split(' ').map(Number);
+ const tickets = requests.map((_, index) => ({
+ tag: values[2 * index],
+ weight: values[2 * index + 1],
+ }));
+ const hasValidTickets =
+ values.length === 2 * requests.length &&
+ tickets.every(({ tag, weight }) => Number.isFinite(tag) && weight > 0);
+
+ if (!hasValidTickets) {
+ throw new Error(`unexpected tickets "${String(rawTickets)}"`);
+ }
+
+ return tickets;
+ } catch (error) {
+ this.logger.warn(
+ `Failed to allocate the fair tickets on queue ${queueName}: ${error instanceof Error ? error.message : String(error)}`,
+ );
+
+ return undefined;
+ }
+ }
+}
diff --git a/packages/twenty-server/src/engine/core-modules/message-queue/drivers/interfaces/job-options.interface.ts b/packages/twenty-server/src/engine/core-modules/message-queue/drivers/interfaces/job-options.interface.ts
index 6388e3e8b3a..919d2b6a9f2 100644
--- a/packages/twenty-server/src/engine/core-modules/message-queue/drivers/interfaces/job-options.interface.ts
+++ b/packages/twenty-server/src/engine/core-modules/message-queue/drivers/interfaces/job-options.interface.ts
@@ -23,6 +23,7 @@ export interface QueueJobOptions {
delay?: number;
deduplication?: QueueJobDeduplicationOptions;
broadcastTo?: QueueJobRecipient;
+ groupKeys?: string[];
}
export interface QueueCronJobOptions extends QueueJobOptions {
diff --git a/packages/twenty-server/src/engine/core-modules/message-queue/utils/build-application-group-key.util.ts b/packages/twenty-server/src/engine/core-modules/message-queue/utils/build-application-group-key.util.ts
new file mode 100644
index 00000000000..d3373010874
--- /dev/null
+++ b/packages/twenty-server/src/engine/core-modules/message-queue/utils/build-application-group-key.util.ts
@@ -0,0 +1,3 @@
+export const buildApplicationGroupKey = (
+ applicationUniversalIdentifier: string,
+): string => `application:${applicationUniversalIdentifier}`;
diff --git a/packages/twenty-server/src/engine/core-modules/message-queue/utils/build-workspace-group-key.util.ts b/packages/twenty-server/src/engine/core-modules/message-queue/utils/build-workspace-group-key.util.ts
new file mode 100644
index 00000000000..6b44ca9c9e0
--- /dev/null
+++ b/packages/twenty-server/src/engine/core-modules/message-queue/utils/build-workspace-group-key.util.ts
@@ -0,0 +1,2 @@
+export const buildWorkspaceGroupKey = (workspaceId: string): string =>
+ `workspace:${workspaceId}`;
diff --git a/packages/twenty-server/src/engine/core-modules/message-queue/utils/get-default-group-keys.util.ts b/packages/twenty-server/src/engine/core-modules/message-queue/utils/get-default-group-keys.util.ts
new file mode 100644
index 00000000000..6d07e7cd698
--- /dev/null
+++ b/packages/twenty-server/src/engine/core-modules/message-queue/utils/get-default-group-keys.util.ts
@@ -0,0 +1,14 @@
+import { isNonEmptyString } from '@sniptt/guards';
+import { isPlainObject } from 'twenty-shared/utils';
+
+import { buildWorkspaceGroupKey } from 'src/engine/core-modules/message-queue/utils/build-workspace-group-key.util';
+
+const SYSTEM_GROUP_KEY = 'system';
+
+export const getDefaultGroupKeys = (jobData: unknown): string[] => {
+ const workspaceId = isPlainObject(jobData) ? jobData.workspaceId : undefined;
+
+ return isNonEmptyString(workspaceId)
+ ? [buildWorkspaceGroupKey(workspaceId)]
+ : [SYSTEM_GROUP_KEY];
+};
diff --git a/packages/twenty-server/src/engine/core-modules/metrics/types/metrics-keys.type.ts b/packages/twenty-server/src/engine/core-modules/metrics/types/metrics-keys.type.ts
index 850052c5358..4d361070ea0 100644
--- a/packages/twenty-server/src/engine/core-modules/metrics/types/metrics-keys.type.ts
+++ b/packages/twenty-server/src/engine/core-modules/metrics/types/metrics-keys.type.ts
@@ -68,6 +68,7 @@ export enum MetricsKeys {
JobWaiting = 'job/waiting',
JobLatencyMs = 'job/latency-ms',
JobDurationMs = 'job/duration-ms',
+ JobFairTagFallback = 'job/fair-tag-fallback',
AiChatTurnLatencyMs = 'ai-chat/turn-latency-ms',
AiChatStepLatencyMs = 'ai-chat/step-latency-ms',
AiChatTtftMs = 'ai-chat/ttft-ms',
diff --git a/packages/twenty-server/src/engine/core-modules/twenty-config/config-variables.ts b/packages/twenty-server/src/engine/core-modules/twenty-config/config-variables.ts
index 1966efb76b2..404652cd104 100644
--- a/packages/twenty-server/src/engine/core-modules/twenty-config/config-variables.ts
+++ b/packages/twenty-server/src/engine/core-modules/twenty-config/config-variables.ts
@@ -1617,6 +1617,15 @@ export class ConfigVariables {
@IsOptional()
WORKER_EXCLUDED_QUEUES: string[] = [];
+ @ConfigVariablesMetadata({
+ group: ConfigVariablesGroup.ADVANCED_SETTINGS,
+ description:
+ 'Order the jobs of each queue by tenant virtual clocks (workspace and application group keys) so that a burst from one tenant does not delay the others. Off: jobs of a queue are processed in FIFO order.',
+ type: ConfigVariableType.BOOLEAN,
+ })
+ @IsOptional()
+ IS_WORKER_FAIRNESS_ENABLED = false;
+
@ConfigVariablesMetadata({
group: ConfigVariablesGroup.SERVER_CONFIG,
description: 'Node environment (development, production, etc.)',
diff --git a/packages/twenty-server/test/integration/message-queue/utils/with-queue-order-test-context.util.ts b/packages/twenty-server/test/integration/message-queue/utils/with-queue-order-test-context.util.ts
index 35ca123a0e7..fbe3d02026c 100644
--- a/packages/twenty-server/test/integration/message-queue/utils/with-queue-order-test-context.util.ts
+++ b/packages/twenty-server/test/integration/message-queue/utils/with-queue-order-test-context.util.ts
@@ -30,7 +30,13 @@ export const withQueueOrderTestContext = async (
incrementCounterForEvent: jest.fn(),
},
},
- { provide: TwentyConfigService, useValue: {} },
+ {
+ provide: TwentyConfigService,
+ useValue: {
+ get: (key: string) =>
+ key === 'IS_WORKER_FAIRNESS_ENABLED' ? false : undefined,
+ },
+ },
{ provide: EventEmitter2, useValue: new EventEmitter2() },
{
provide: BullMQDriver,
diff --git a/yarn.lock b/yarn.lock
index ee678dbd659..1838e152b60 100644
--- a/yarn.lock
+++ b/yarn.lock
@@ -8743,6 +8743,13 @@ __metadata:
languageName: node
linkType: hard
+"@ioredis/as-callback@npm:^3.0.0":
+ version: 3.0.0
+ resolution: "@ioredis/as-callback@npm:3.0.0"
+ checksum: 10c0/cbeae0c4a8f8f8ea1987d105056fd595a5a1a781a73dfa3a13ac13027fd1de099af6f3590bd315fd20284efd8597430a7ba3b912570804d158dabbdb2071523b
+ languageName: node
+ linkType: hard
+
"@ioredis/commands@npm:1.5.1":
version: 1.5.1
resolution: "@ioredis/commands@npm:1.5.1"
@@ -8750,6 +8757,13 @@ __metadata:
languageName: node
linkType: hard
+"@ioredis/commands@npm:^1.4.0":
+ version: 1.11.0
+ resolution: "@ioredis/commands@npm:1.11.0"
+ checksum: 10c0/fa8758f51ad1557e6c710f59890b83cd95c9c7fd419b6f7018268612d6479202d18566e16378097b52549e5dfafedc11a9a4703107a77d4b972684ee1757611a
+ languageName: node
+ linkType: hard
+
"@isaacs/cliui@npm:^8.0.2":
version: 8.0.2
resolution: "@isaacs/cliui@npm:8.0.2"
@@ -19432,6 +19446,15 @@ __metadata:
languageName: node
linkType: hard
+"@types/ioredis-mock@npm:8.2.8":
+ version: 8.2.8
+ resolution: "@types/ioredis-mock@npm:8.2.8"
+ peerDependencies:
+ ioredis: ^5
+ checksum: 10c0/af7a6546b2f43f4c8af4ef1deb453d1e436d901bd9e25d23b0e100b6170712ae38e72ca022fd12fa3616b4abe8ceee5a5b483db8774a99b3403ee3d2097fb9c6
+ languageName: node
+ linkType: hard
+
"@types/istanbul-lib-coverage@npm:*, @types/istanbul-lib-coverage@npm:^2.0.0, @types/istanbul-lib-coverage@npm:^2.0.1, @types/istanbul-lib-coverage@npm:^2.0.4, @types/istanbul-lib-coverage@npm:^2.0.6":
version: 2.0.6
resolution: "@types/istanbul-lib-coverage@npm:2.0.6"
@@ -29266,6 +29289,26 @@ __metadata:
languageName: node
linkType: hard
+"fengari-interop@npm:^0.1.3":
+ version: 0.1.4
+ resolution: "fengari-interop@npm:0.1.4"
+ peerDependencies:
+ fengari: ^0.1.0
+ checksum: 10c0/ba8aca9b741b603a7740343b3f99dbd60c4536aab115a8e02caa33030e16e0dd13a0d03f0c31050f7fa1ead7945358040cead48531df41c3b3147e61b50f383c
+ languageName: node
+ linkType: hard
+
+"fengari@npm:^0.1.4":
+ version: 0.1.5
+ resolution: "fengari@npm:0.1.5"
+ dependencies:
+ readline-sync: "npm:^1.4.10"
+ sprintf-js: "npm:^1.1.3"
+ tmp: "npm:^0.2.5"
+ checksum: 10c0/8688c3483e00028b0b09928f71452bb2c8b4db5f7380e0e8dd65619bb4ccb38438557ebdfd977c55a6d3fb5e7601cbac26c62873317320ffd20b59936f464244
+ languageName: node
+ linkType: hard
+
"fernet@npm:^0.3.3":
version: 0.3.3
resolution: "fernet@npm:0.3.3"
@@ -32036,6 +32079,22 @@ __metadata:
languageName: node
linkType: hard
+"ioredis-mock@npm:8.13.1":
+ version: 8.13.1
+ resolution: "ioredis-mock@npm:8.13.1"
+ dependencies:
+ "@ioredis/as-callback": "npm:^3.0.0"
+ "@ioredis/commands": "npm:^1.4.0"
+ fengari: "npm:^0.1.4"
+ fengari-interop: "npm:^0.1.3"
+ semver: "npm:^7.7.2"
+ peerDependencies:
+ "@types/ioredis-mock": ^8
+ ioredis: ^5
+ checksum: 10c0/47e17a307301da01a89d041a6d6101828baefd3a1c216a09fe8ff9e1bd7d7239de93544365280f6cda4ad98af1f302e7497f9563f36cb24b528d9c7b3164301f
+ languageName: node
+ linkType: hard
+
"ioredis@npm:5.10.1":
version: 5.10.1
resolution: "ioredis@npm:5.10.1"
@@ -41401,7 +41460,7 @@ __metadata:
languageName: node
linkType: hard
-"readline-sync@npm:^1.4.9":
+"readline-sync@npm:^1.4.10, readline-sync@npm:^1.4.9":
version: 1.4.10
resolution: "readline-sync@npm:1.4.10"
checksum: 10c0/0a4d0fe4ad501f8f005a3c9cbf3cc0ae6ca2ced93e9a1c7c46f226bdfcb6ef5d3f437ae7e9d2e1098ee13524a3739c830e4c8dbc7f543a693eecd293e41093a3
@@ -43789,6 +43848,13 @@ __metadata:
languageName: node
linkType: hard
+"sprintf-js@npm:^1.1.3":
+ version: 1.1.3
+ resolution: "sprintf-js@npm:1.1.3"
+ checksum: 10c0/09270dc4f30d479e666aee820eacd9e464215cdff53848b443964202bf4051490538e5dd1b42e1a65cf7296916ca17640aebf63dae9812749c7542ee5f288dec
+ languageName: node
+ linkType: hard
+
"sprintf-js@npm:~1.0.2":
version: 1.0.3
resolution: "sprintf-js@npm:1.0.3"
@@ -46101,6 +46167,7 @@ __metadata:
"@types/graphql-fields": "npm:^1.3.6"
"@types/graphql-upload": "npm:^16.0.7"
"@types/html-to-text": "npm:^9.0.4"
+ "@types/ioredis-mock": "npm:8.2.8"
"@types/jest": "npm:^30.0.0"
"@types/lodash.chunk": "npm:^4.2.9"
"@types/lodash.compact": "npm:^3.0.9"
@@ -46168,6 +46235,7 @@ __metadata:
ical-generator: "npm:^11.0.0"
imapflow: "npm:2.0.2"
ioredis: "npm:5.10.1"
+ ioredis-mock: "npm:8.13.1"
jest: "npm:29.7.0"
jsdom: "npm:^30.0.1"
json-schema: "npm:0.4.0"