The open alternative to Salesforce, designed for AI.
TypeScript crmcrm-systemcustomergood-first-issuegraphqlhacktoberfestjavascriptmarketingmonoreponestjsopen-sourcepostgresqlreactreactjssalestypescriptweb
Add ordering scenarios for the fair worker queue (#28848)
martmull965d6f49f5ea
diff --git a/packages/twenty-server/src/engine/core-modules/message-queue/drivers/__tests__/bullmq.driver.fairness.spec.ts b/packages/twenty-server/src/engine/core-modules/message-queue/drivers/__tests__/bullmq.driver.fairness.spec.tsnew file mode 100644index 00000000000..e8716a00393--- /dev/null+++ b/packages/twenty-server/src/engine/core-modules/message-queue/drivers/__tests__/bullmq.driver.fairness.spec.ts@@ -0,0 +1,960 @@+import { Logger } from '@nestjs/common';++import { type Job } from 'bullmq';+import { type Redis } from 'ioredis';+import RedisMock from 'ioredis-mock';+import { isDefined } from 'twenty-shared/utils';++import { ENQUEUE_JOB_PRIORITY } from 'src/engine/core-modules/application/application-job/constants/enqueue-job-priority.constant';+import { BullMQDriver } from 'src/engine/core-modules/message-queue/drivers/bullmq.driver';+import { FAIR_TICKET_OFFSET } from 'src/engine/core-modules/message-queue/drivers/fairness/fair-priority.constant';+import { type QueueJobOptions } from 'src/engine/core-modules/message-queue/drivers/interfaces/job-options.interface';+import { MessageQueue } from 'src/engine/core-modules/message-queue/message-queue.constants';+import { buildApplicationGroupKey } from 'src/engine/core-modules/message-queue/utils/build-application-group-key.util';+import { buildWorkspaceGroupKey } from 'src/engine/core-modules/message-queue/utils/build-workspace-group-key.util';+import { MetricsKeys } from 'src/engine/core-modules/metrics/types/metrics-keys.type';+import { SERVER_ROUTE_DISPATCH_JOB_PRIORITY } from 'src/engine/core-modules/server-route-trigger/constants/server-route-dispatch-job-priority.constant';++type ScenarioJobData = { label: string; workspaceId?: string };++type StoredJobOptions = { priority?: number };++type StoredJob = {+ id: string;+ name: string;+ data: ScenarioJobData;+ options: StoredJobOptions;+ timestamp: number;+ insertionIndex: number;+};++type JobToStore = {+ name: string;+ data: ScenarioJobData;+ options: StoredJobOptions;+};++type RedisClientDouble = {+ runCommand: (name: string, args: (string | number)[]) => Promise<unknown>;+};++type BullMQDouble = {+ jobs: StoredJob[];+ nextInsertionIndex: number;+ processor: ((job: Job) => Promise<void>) | undefined;+ client: RedisClientDouble | undefined;+ redis: Redis | undefined;+ store: (jobToStore: JobToStore) => Promise<{ id: string }>;+ takeNext: () => StoredJob | undefined;+ reset: (store: { redis: Redis; client: RedisClientDouble }) => void;+};++const mockBullMQ: BullMQDouble = {+ jobs: [],+ nextInsertionIndex: 0,+ processor: undefined,+ client: undefined,+ redis: undefined,+ store: async ({ name, data, options }) => {+ const job: StoredJob = {+ id: `job-${mockBullMQ.nextInsertionIndex}`,+ name,+ data,+ options,+ timestamp: Date.now(),+ insertionIndex: mockBullMQ.nextInsertionIndex,+ };++ mockBullMQ.nextInsertionIndex += 1;+ mockBullMQ.jobs.push(job);++ await mockBullMQ.redis?.hset(+ `${KEY_PREFIX}:${QUEUE}:${job.id}`,+ 'priority',+ options.priority ?? 0,+ );++ return { id: job.id };+ },+ takeNext: () => {+ const [next] = [...mockBullMQ.jobs].sort(+ (left, right) =>+ (left.options.priority ?? 0) - (right.options.priority ?? 0) ||+ left.insertionIndex - right.insertionIndex,+ );++ if (!isDefined(next)) {+ return undefined;+ }++ mockBullMQ.jobs = mockBullMQ.jobs.filter((job) => job !== next);++ return next;+ },+ reset: ({ redis, client }) => {+ mockBullMQ.jobs = [];+ mockBullMQ.nextInsertionIndex = 0;+ mockBullMQ.processor = undefined;+ mockBullMQ.client = client;+ mockBullMQ.redis = redis;+ },+};++jest.mock('bullmq', () => ({+ Queue: jest.fn().mockImplementation(() => ({+ get client() {+ return Promise.resolve(mockBullMQ.client);+ },+ toKey: (type: string) => `${KEY_PREFIX}:${QUEUE}:${type}`,+ add: async (+ name: string,+ data: ScenarioJobData,+ options: StoredJobOptions,+ ) => mockBullMQ.store({ name, data, options }),+ addBulk: async (+ jobs: { name: string; data: ScenarioJobData; opts: StoredJobOptions }[],+ ) =>+ Promise.all(+ jobs.map(({ name, data, opts }) =>+ mockBullMQ.store({ name, data, options: opts }),+ ),+ ),+ setGlobalConcurrency: async () => undefined,+ removeGlobalConcurrency: async () => undefined,+ })),+ Worker: jest+ .fn()+ .mockImplementation(+ (_queueName: string, processor: (job: Job) => Promise<void>) => {+ mockBullMQ.processor = processor;++ return { on: () => undefined };+ },+ ),+ MetricsTime: { ONE_WEEK: 1 },+ PRIORITY_LIMIT: 2 ** 21,+}));++const QUEUE = MessageQueue.logicFunctionQueue;+const KEY_PREFIX = 'bull';+const JOB_NAME = 'scenario-job';+const FAIRNESS_FLAG = 'IS_WORKER_FAIRNESS_ENABLED';++const flushAsyncWork = async () => {+ for (let round = 0; round < 30; round += 1) {+ await new Promise((resolve) => setImmediate(resolve));+ }+};++const buildLabels = ({+ prefix,+ count,+ from = 1,+}: {+ prefix: string;+ count: number;+ from?: number;+}) => Array.from({ length: count }, (_, index) => `${prefix}${from + index}`);++const ticketOf = (tag: number) => FAIR_TICKET_OFFSET + tag;++const createRedisClientDouble = (redis: Redis): RedisClientDouble => ({+ runCommand: async (name, args) => {+ const [script, numberOfKeys, ...scriptArguments] = args;++ if (name === 'evalsha') {+ return redis.evalsha(+ String(script),+ Number(numberOfKeys),+ ...scriptArguments,+ );+ }++ if (name !== 'eval') {+ throw new Error(`Unexpected Redis command ${name}`);+ }++ return redis.eval(String(script), Number(numberOfKeys), ...scriptArguments);+ },+});++type EnqueueInput = { label: string; workspaceId?: string } & Pick<+ QueueJobOptions,+ 'groupKeys' | 'priority'+>;++const createHarness = async ({+ isFairnessEnabled,+}: {+ isFairnessEnabled: boolean;+}) => {+ const redis = new RedisMock();++ await redis.flushall();+ mockBullMQ.reset({ redis, client: createRedisClientDouble(redis) });++ const metricsService = {+ recordHistogram: jest.fn(),+ incrementCounterForEvent: jest.fn().mockResolvedValue(undefined),+ };+ const driver = new BullMQDriver(+ { prefix: KEY_PREFIX, connection: {} },+ metricsService as never,+ {+ get: (key: string) =>+ key === FAIRNESS_FLAG ? isFairnessEnabled : undefined,+ } as never,+ { emit: jest.fn() } as never,+ );++ driver.register(QUEUE);++ const processedLabels: string[] = [];++ driver.work<ScenarioJobData>(QUEUE, async ({ data }) => {+ processedLabels.push(data.label);+ });+ await flushAsyncWork();++ const processOne = async (): Promise<boolean> => {+ const job = mockBullMQ.takeNext();++ if (!isDefined(job) || !isDefined(mockBullMQ.processor)) {+ return false;+ }++ await mockBullMQ.processor({+ ...job,+ opts: job.options,+ priority: job.options.priority ?? 0,+ attemptsStarted: 1,+ updateData: jest.fn(),+ updateProgress: jest.fn(),+ } as unknown as Job);+ await flushAsyncWork();++ return true;+ };++ const enqueue = async ({ label, workspaceId, ...options }: EnqueueInput) => {+ await driver.add<ScenarioJobData>(+ QUEUE,+ JOB_NAME,+ { label, workspaceId },+ options,+ );+ };++ return {+ metricsService,+ enqueue,+ enqueueMany: async ({+ labels,+ ...options+ }: { labels: string[] } & Omit<EnqueueInput, 'label'>) => {+ for (const label of labels) {+ await enqueue({ label, ...options });+ }+ },+ enqueueBulk: async ({+ labels,+ workspaceId,+ groupKeys,+ }: {+ labels: string[];+ workspaceId: string;+ groupKeys?: string[];+ }) => {+ await driver.bulkAdd<ScenarioJobData>(+ QUEUE,+ JOB_NAME,+ labels.map((label) => ({ data: { label, workspaceId } })),+ isDefined(groupKeys) ? { groupKeys } : undefined,+ );+ },+ processNext: async (count: number) => {+ const alreadyProcessed = processedLabels.length;++ for (let index = 0; index < count; index += 1) {+ if (!(await processOne())) {+ break;+ }+ }++ return processedLabels.slice(alreadyProcessed);+ },+ processAll: async () => {+ const alreadyProcessed = processedLabels.length;+ let hasProcessedJob = await processOne();++ while (hasProcessedJob) {+ hasProcessedJob = await processOne();+ }++ return processedLabels.slice(alreadyProcessed);+ },+ setWeight: async ({+ groupKey,+ weight,+ }: {+ groupKey: string;+ weight: number;+ }) => {+ await redis.hset(`${KEY_PREFIX}:${QUEUE}:fair:weights`, groupKey, weight);+ },+ readPriority: ({ label }: { label: string }) =>+ mockBullMQ.jobs.find((job) => job.data.label === label)?.options.priority,+ breakRedis: () => {+ const runCommand = jest+ .fn()+ .mockRejectedValue(new Error('Redis connection lost'));++ mockBullMQ.client = { runCommand };++ return runCommand;+ },+ };+};++type Harness = Awaited<ReturnType<typeof createHarness>>;++describe('BullMQDriver fairness scenarios', () => {+ beforeAll(() => {+ jest.useRealTimers();+ jest.spyOn(Logger.prototype, 'log').mockImplementation(() => undefined);+ jest.spyOn(Logger.prototype, 'warn').mockImplementation(() => undefined);+ });++ describe('S1: A enqueues a burst of 10 jobs, then B enqueues 1 job (group keys deduced from workspaceId)', () => {+ const enqueue = async (harness: Harness) => {+ await harness.enqueueMany({+ labels: buildLabels({ prefix: 'A', count: 10 }),+ workspaceId: 'A',+ });+ await harness.enqueue({ label: 'B1', workspaceId: 'B' });+ };++ it('flag off: B waits behind the whole burst of A', async () => {+ const harness = await createHarness({ isFairnessEnabled: false });++ await enqueue(harness);++ expect(await harness.processAll()).toEqual([+ ...buildLabels({ prefix: 'A', count: 10 }),+ 'B1',+ ]);+ });++ it('flag on: B runs right after the first job of A', async () => {+ const harness = await createHarness({ isFairnessEnabled: true });++ await enqueue(harness);++ expect(await harness.processAll()).toEqual([+ 'A1',+ 'B1',+ ...buildLabels({ prefix: 'A', count: 9, from: 2 }),+ ]);+ });+ });++ describe('S2: A and B each enqueue a bulk of 4 jobs', () => {+ const enqueue = async (harness: Harness) => {+ await harness.enqueueBulk({+ labels: buildLabels({ prefix: 'A', count: 4 }),+ workspaceId: 'A',+ });+ await harness.enqueueBulk({+ labels: buildLabels({ prefix: 'B', count: 4 }),+ workspaceId: 'B',+ });+ };++ it('flag off: the bulk of B waits behind the bulk of A', async () => {+ const harness = await createHarness({ isFairnessEnabled: false });++ await enqueue(harness);++ expect(await harness.processAll()).toEqual([+ ...buildLabels({ prefix: 'A', count: 4 }),+ ...buildLabels({ prefix: 'B', count: 4 }),+ ]);+ });++ it('flag on: the two bulks alternate, each one keeping its internal order', async () => {+ const harness = await createHarness({ isFairnessEnabled: true });++ await enqueue(harness);++ expect(await harness.processAll()).toEqual([+ 'A1',+ 'B1',+ 'A2',+ 'B2',+ 'A3',+ 'B3',+ 'A4',+ 'B4',+ ]);+ });+ });++ describe('S3: A enqueues 5 default jobs, B sends an urgent route dispatch and an app enqueueJobs job', () => {+ const enqueue = async (harness: Harness) => {+ await harness.enqueueMany({+ labels: buildLabels({ prefix: 'A', count: 5 }),+ workspaceId: 'A',+ });+ await harness.enqueue({+ label: 'R_B',+ workspaceId: 'B',+ priority: SERVER_ROUTE_DISPATCH_JOB_PRIORITY,+ });+ await harness.enqueue({+ label: 'E_B',+ workspaceId: 'B',+ priority: ENQUEUE_JOB_PRIORITY,+ });+ };++ it('flag off: the static priorities decide, the route dispatch first and the app job last', async () => {+ const harness = await createHarness({ isFairnessEnabled: false });++ await enqueue(harness);++ expect(await harness.processAll()).toEqual([+ 'R_B',+ ...buildLabels({ prefix: 'A', count: 5 }),+ 'E_B',+ ]);+ });++ it('flag on: the route dispatch still goes first and charges B, whose app job then takes a fair slot instead of the last one', async () => {+ const harness = await createHarness({ isFairnessEnabled: true });++ await enqueue(harness);++ expect(await harness.processAll()).toEqual([+ 'R_B',+ 'A1',+ 'A2',+ 'E_B',+ 'A3',+ 'A4',+ 'A5',+ ]);+ });+ });++ describe('S4: W1, W2 and W3 each enqueue 2 jobs of application H, then W4 enqueues 1 job of application X', () => {+ const enqueue = async (harness: Harness) => {+ await harness.enqueue({+ label: 'W1a',+ workspaceId: 'W1',+ groupKeys: [+ buildWorkspaceGroupKey('W1'),+ buildApplicationGroupKey('H'),+ ],+ });+ await harness.enqueue({+ label: 'W1b',+ workspaceId: 'W1',+ groupKeys: [+ buildWorkspaceGroupKey('W1'),+ buildApplicationGroupKey('H'),+ ],+ });+ await harness.enqueue({+ label: 'W2a',+ workspaceId: 'W2',+ groupKeys: [+ buildWorkspaceGroupKey('W2'),+ buildApplicationGroupKey('H'),+ ],+ });+ await harness.enqueue({+ label: 'W2b',+ workspaceId: 'W2',+ groupKeys: [+ buildWorkspaceGroupKey('W2'),+ buildApplicationGroupKey('H'),+ ],+ });+ await harness.enqueue({+ label: 'W3a',+ workspaceId: 'W3',+ groupKeys: [+ buildWorkspaceGroupKey('W3'),+ buildApplicationGroupKey('H'),+ ],+ });+ await harness.enqueue({+ label: 'W3b',+ workspaceId: 'W3',+ groupKeys: [+ buildWorkspaceGroupKey('W3'),+ buildApplicationGroupKey('H'),+ ],+ });+ await harness.enqueue({+ label: 'W4',+ workspaceId: 'W4',+ groupKeys: [+ buildWorkspaceGroupKey('W4'),+ buildApplicationGroupKey('X'),+ ],+ });+ };++ it('flag off: W4 waits behind the whole herd', async () => {+ const harness = await createHarness({ isFairnessEnabled: false });++ await enqueue(harness);++ expect(await harness.processAll()).toEqual([+ 'W1a',+ 'W1b',+ 'W2a',+ 'W2b',+ 'W3a',+ 'W3b',+ 'W4',+ ]);+ });++ it('flag on: the application key makes the herd share one clock, W4 runs second', async () => {+ const harness = await createHarness({ isFairnessEnabled: true });++ await enqueue(harness);++ expect(await harness.processAll()).toEqual([+ 'W1a',+ 'W4',+ 'W1b',+ 'W2a',+ 'W2b',+ 'W3a',+ 'W3b',+ ]);+ });+ });++ describe('S5: C keeps the queue busy with 12 jobs, A gets a burst of 6 processed, then B bursts 5 jobs and A enqueues 1 job', () => {+ const enqueueFirstPhase = async (harness: Harness) => {+ await harness.enqueueMany({+ labels: buildLabels({ prefix: 'C', count: 12 }),+ workspaceId: 'C',+ });+ await harness.enqueueMany({+ labels: buildLabels({ prefix: 'A', count: 6 }),+ workspaceId: 'A',+ });+ };+ const enqueueSecondPhase = async (harness: Harness) => {+ await harness.enqueueMany({+ labels: buildLabels({ prefix: 'B', count: 5 }),+ workspaceId: 'B',+ });+ await harness.enqueue({ label: 'A7', workspaceId: 'A' });+ };++ it('flag off: A7 runs last, behind the rest of C and the whole burst of B', async () => {+ const harness = await createHarness({ isFairnessEnabled: false });++ await enqueueFirstPhase(harness);++ expect(await harness.processNext(12)).toEqual(+ buildLabels({ prefix: 'C', count: 12 }),+ );++ await enqueueSecondPhase(harness);++ expect(await harness.processAll()).toEqual([+ ...buildLabels({ prefix: 'A', count: 6 }),+ ...buildLabels({ prefix: 'B', count: 5 }),+ 'A7',+ ]);+ });++ it('flag on: the processed burst of A is forgotten, A7 interleaves with the burst of B', async () => {+ const harness = await createHarness({ isFairnessEnabled: true });++ await enqueueFirstPhase(harness);++ expect(await harness.processNext(12)).toEqual([+ 'C1',+ 'A1',+ 'C2',+ 'A2',+ 'C3',+ 'A3',+ 'C4',+ 'A4',+ 'C5',+ 'A5',+ 'C6',+ 'A6',+ ]);++ await enqueueSecondPhase(harness);++ expect(await harness.processAll()).toEqual([+ 'B1',+ 'C7',+ 'B2',+ 'A7',+ 'C8',+ 'B3',+ 'C9',+ 'B4',+ 'C10',+ 'B5',+ 'C11',+ 'C12',+ ]);+ });+ });++ describe('S6: A enqueues 5 jobs that get fully processed, then B enqueues 4 jobs and A enqueues 2 jobs', () => {+ const drainFirstBurst = async (harness: Harness) => {+ await harness.enqueueMany({+ labels: buildLabels({ prefix: 'A', count: 5 }),+ workspaceId: 'A',+ });++ expect(await harness.processAll()).toEqual(+ buildLabels({ prefix: 'A', count: 5 }),+ );+ };+ const enqueueSecondPhase = async (harness: Harness) => {+ await harness.enqueueMany({+ labels: buildLabels({ prefix: 'B', count: 4 }),+ workspaceId: 'B',+ });+ await harness.enqueueMany({+ labels: buildLabels({ prefix: 'A', count: 2, from: 6 }),+ workspaceId: 'A',+ });+ };++ it('flag off: B first, then A, in arrival order', async () => {+ const harness = await createHarness({ isFairnessEnabled: false });++ await drainFirstBurst(harness);+ await enqueueSecondPhase(harness);++ expect(await harness.processAll()).toEqual([+ ...buildLabels({ prefix: 'B', count: 4 }),+ 'A6',+ 'A7',+ ]);+ });++ it('flag on: the virtual time caught up with A while its burst ran, A interleaves with B from the third slot', async () => {+ const harness = await createHarness({ isFairnessEnabled: true });++ await drainFirstBurst(harness);+ await enqueueSecondPhase(harness);++ expect(await harness.processAll()).toEqual([+ 'B1',+ 'B2',+ 'A6',+ 'B3',+ 'A7',+ 'B4',+ ]);+ });+ });++ describe('S7: application P weighs 2, application Q weighs 1, WP enqueues 4 jobs of P and WQ 4 jobs of Q, keyed by workspace and application', () => {+ const enqueue = async (harness: Harness) => {+ await harness.setWeight({+ groupKey: buildApplicationGroupKey('P'),+ weight: 2,+ });+ await harness.enqueueMany({+ labels: buildLabels({ prefix: 'P', count: 4 }),+ workspaceId: 'WP',+ groupKeys: [+ buildWorkspaceGroupKey('WP'),+ buildApplicationGroupKey('P'),+ ],+ });+ await harness.enqueueMany({+ labels: buildLabels({ prefix: 'Q', count: 4 }),+ workspaceId: 'WQ',+ groupKeys: [+ buildWorkspaceGroupKey('WQ'),+ buildApplicationGroupKey('Q'),+ ],+ });+ };++ it('flag off: the weight is ignored, Q waits behind P', async () => {+ const harness = await createHarness({ isFairnessEnabled: false });++ await enqueue(harness);++ expect(await harness.processAll()).toEqual([+ ...buildLabels({ prefix: 'P', count: 4 }),+ ...buildLabels({ prefix: 'Q', count: 4 }),+ ]);+ });++ it('flag on: P gets two slots for each slot of Q, the unweighted workspace key does not dilute the weight', async () => {+ const harness = await createHarness({ isFairnessEnabled: true });++ await enqueue(harness);++ expect(await harness.processAll()).toEqual([+ 'P1',+ 'P2',+ 'Q1',+ 'P3',+ 'P4',+ 'Q2',+ 'Q3',+ 'Q4',+ ]);+ });+ });++ describe('S7b: application P weighs 2; WP bulk-enqueues 4 jobs of P, then WP enqueues 1 more job of P and WQ enqueues 1 job of Q', () => {+ const groupKeysOfP = [+ buildWorkspaceGroupKey('WP'),+ buildApplicationGroupKey('P'),+ ];+ const enqueue = async (harness: Harness) => {+ await harness.setWeight({+ groupKey: buildApplicationGroupKey('P'),+ weight: 2,+ });+ await harness.enqueueBulk({+ labels: buildLabels({ prefix: 'P', count: 4 }),+ workspaceId: 'WP',+ groupKeys: groupKeysOfP,+ });+ await harness.enqueue({+ label: 'P5',+ workspaceId: 'WP',+ groupKeys: groupKeysOfP,+ });+ await harness.enqueue({+ label: 'Q1',+ workspaceId: 'WQ',+ groupKeys: [+ buildWorkspaceGroupKey('WQ'),+ buildApplicationGroupKey('Q'),+ ],+ });+ };++ it('flag off: FIFO', async () => {+ const harness = await createHarness({ isFairnessEnabled: false });++ await enqueue(harness);++ expect(await harness.processAll()).toEqual([+ ...buildLabels({ prefix: 'P', count: 5 }),+ 'Q1',+ ]);+ });++ it('flag on: the bulk is spaced like single jobs of the same weight, P keeps its order and its two slots per slot of Q', async () => {+ const harness = await createHarness({ isFairnessEnabled: true });++ await enqueue(harness);++ expect(await harness.processAll()).toEqual([+ 'P1',+ 'P2',+ 'Q1',+ 'P3',+ 'P4',+ 'P5',+ ]);+ });+ });++ describe('S7c: application H has 100 jobs waiting from workspaces X1 to X10, then workspace W enqueues 1 job of H and 1 job of application Q', () => {+ const herdLabels = buildLabels({ prefix: 'H', count: 100 });+ const enqueue = async (harness: Harness) => {+ for (const [index, label] of herdLabels.entries()) {+ const workspaceId = `X${(index % 10) + 1}`;++ await harness.enqueue({+ label,+ workspaceId,+ groupKeys: [+ buildWorkspaceGroupKey(workspaceId),+ buildApplicationGroupKey('H'),+ ],+ });+ }+ await harness.enqueue({+ label: 'W_H',+ workspaceId: 'W',+ groupKeys: [buildWorkspaceGroupKey('W'), buildApplicationGroupKey('H')],+ });+ await harness.enqueue({+ label: 'W_Q',+ workspaceId: 'W',+ groupKeys: [buildWorkspaceGroupKey('W'), buildApplicationGroupKey('Q')],+ });+ };++ it('flag off: W waits behind the whole backlog of H', async () => {+ const harness = await createHarness({ isFairnessEnabled: false });++ await enqueue(harness);++ expect(await harness.processAll()).toEqual([...herdLabels, 'W_H', 'W_Q']);+ });++ it('flag on: the job of H takes its turn in the backlog of H, the job of Q runs right away because W is only charged for its own jobs', async () => {+ const harness = await createHarness({ isFairnessEnabled: true });++ await enqueue(harness);++ expect(harness.readPriority({ label: 'W_H' })).toBe(ticketOf(100));+ expect(harness.readPriority({ label: 'W_Q' })).toBe(ticketOf(1));+ expect(await harness.processNext(3)).toEqual(['H1', 'H2', 'W_Q']);+ expect(await harness.processAll()).toEqual([+ ...herdLabels.slice(2),+ 'W_H',+ ]);+ });+ });++ describe('S7d: application P weighs 2; X enqueues 6 jobs of P, W bulk-enqueues 4 jobs of P then 1 job of application Q, X enqueues 1 more job of P', () => {+ const groupKeysOfP = (workspaceId: string) => [+ buildWorkspaceGroupKey(workspaceId),+ buildApplicationGroupKey('P'),+ ];+ const enqueue = async (harness: Harness) => {+ await harness.setWeight({+ groupKey: buildApplicationGroupKey('P'),+ weight: 2,+ });+ await harness.enqueueMany({+ labels: buildLabels({ prefix: 'P', count: 6 }),+ workspaceId: 'X',+ groupKeys: groupKeysOfP('X'),+ });+ await harness.enqueueBulk({+ labels: buildLabels({ prefix: 'B', count: 4 }),+ workspaceId: 'W',+ groupKeys: groupKeysOfP('W'),+ });+ await harness.enqueue({+ label: 'Q1',+ workspaceId: 'W',+ groupKeys: [buildWorkspaceGroupKey('W'), buildApplicationGroupKey('Q')],+ });+ await harness.enqueue({+ label: 'P7',+ workspaceId: 'X',+ groupKeys: groupKeysOfP('X'),+ });+ };++ it('flag off: FIFO', async () => {+ const harness = await createHarness({ isFairnessEnabled: false });++ await enqueue(harness);++ expect(await harness.processAll()).toEqual([+ ...buildLabels({ prefix: 'P', count: 6 }),+ ...buildLabels({ prefix: 'B', count: 4 }),+ 'Q1',+ 'P7',+ ]);+ });++ it('flag on: the bulk fills the half-slots it allocated on P, W is charged those half-slots only, so its job of Q (ticket 2, like P6) overtakes its bulk and the next job of P follows the bulk', async () => {+ const harness = await createHarness({ isFairnessEnabled: true });++ await enqueue(harness);++ expect(harness.readPriority({ label: 'B1' })).toBe(ticketOf(3));+ expect(harness.readPriority({ label: 'B4' })).toBe(ticketOf(4));+ expect(harness.readPriority({ label: 'Q1' })).toBe(ticketOf(2));+ expect(harness.readPriority({ label: 'P7' })).toBe(ticketOf(5));+ expect(await harness.processAll()).toEqual([+ ...buildLabels({ prefix: 'P', count: 6 }),+ 'Q1',+ ...buildLabels({ prefix: 'B', count: 4 }),+ 'P7',+ ]);+ });+ });++ describe('S8: Redis is unavailable, A enqueues 3 jobs and B enqueues 1 job', () => {+ const enqueue = async (harness: Harness) => {+ await harness.enqueueMany({+ labels: buildLabels({ prefix: 'A', count: 3 }),+ workspaceId: 'A',+ });+ await harness.enqueue({ label: 'B1', workspaceId: 'B' });+ };+ const expectedOrder = [...buildLabels({ prefix: 'A', count: 3 }), 'B1'];++ it('flag off: Redis is never called and the order is FIFO', async () => {+ const harness = await createHarness({ isFairnessEnabled: false });+ const runCommand = harness.breakRedis();++ await enqueue(harness);++ expect(await harness.processAll()).toEqual(expectedOrder);+ expect(runCommand).not.toHaveBeenCalled();+ expect(+ harness.metricsService.incrementCounterForEvent,+ ).not.toHaveBeenCalledWith(+ expect.objectContaining({ key: MetricsKeys.JobFairTagFallback }),+ );+ });++ it('flag on: every job falls back to FIFO and the fallback is counted', async () => {+ const harness = await createHarness({ isFairnessEnabled: true });++ harness.breakRedis();+ await enqueue(harness);++ expect(await harness.processAll()).toEqual(expectedOrder);+ expect(+ harness.metricsService.incrementCounterForEvent.mock.calls.filter(+ ([event]) => event.key === MetricsKeys.JobFairTagFallback,+ ),+ ).toHaveLength(4);+ });+ });++ describe('S9: 3 jobs without workspaceId, then 1 job of workspace W', () => {+ const enqueue = async (harness: Harness) => {+ await harness.enqueueMany({+ labels: buildLabels({ prefix: 'S', count: 3 }),+ });+ await harness.enqueue({ label: 'W1', workspaceId: 'W' });+ };++ it('flag off: the workspace job waits behind the system jobs', async () => {+ const harness = await createHarness({ isFairnessEnabled: false });++ await enqueue(harness);++ expect(await harness.processAll()).toEqual([+ ...buildLabels({ prefix: 'S', count: 3 }),+ 'W1',+ ]);+ });++ it('flag on: the system jobs form one group, the workspace job runs second', async () => {+ const harness = await createHarness({ isFairnessEnabled: true });++ await enqueue(harness);++ expect(await harness.processAll()).toEqual(['S1', 'W1', 'S2', 'S3']);+ });+ });+});