The open alternative to Salesforce, designed for AI.
TypeScript crmcrm-systemcustomergood-first-issuegraphqlhacktoberfestjavascriptmarketingmonoreponestjsopen-sourcepostgresqlreactreactjssalestypescriptweb
Measure real queue wait and report delayed jobs apart from waiting ones (#27329)
martmulld0d7c1aef145
diff --git a/packages/twenty-server/src/engine/core-modules/message-queue/drivers/__tests__/bullmq.driver.spec.ts b/packages/twenty-server/src/engine/core-modules/message-queue/drivers/__tests__/bullmq.driver.spec.tsindex b0cd26f9840..c188ea2c69e 100644--- a/packages/twenty-server/src/engine/core-modules/message-queue/drivers/__tests__/bullmq.driver.spec.ts+++ b/packages/twenty-server/src/engine/core-modules/message-queue/drivers/__tests__/bullmq.driver.spec.ts@@ -9,6 +9,7 @@ const mockAdd = jest.fn();const mockAddBulk = jest.fn();const mockSetGlobalConcurrency = jest.fn();const mockRemoveGlobalConcurrency = jest.fn();+const mockGetJobCountByTypes = jest.fn();jest.mock('bullmq', () => ({Queue: jest.fn().mockImplementation(() => ({@@ -18,6 +19,7 @@ jest.mock('bullmq', () => ({addBulk: mockAddBulk,setGlobalConcurrency: mockSetGlobalConcurrency,removeGlobalConcurrency: mockRemoveGlobalConcurrency,+ getJobCountByTypes: mockGetJobCountByTypes,})),Worker: jest.fn().mockImplementation(() => ({ on: jest.fn() })),MetricsTime: { ONE_WEEK: 1 },@@ -196,6 +198,138 @@ describe('BullMQDriver progress', () => {);});+describe('BullMQDriver queue wait metric', () => {+ const recordHistogram = jest.fn();+ const driver = new BullMQDriver(+ {} as never,+ { recordHistogram } as never,+ {} as never,+ {} as never,+ );++ driver.register(MessageQueue.workflowQueue);++ const processJob = async (+ job: Pick<Job, 'opts' | 'timestamp' | 'attemptsStarted'>,+ ) => {+ jest.clearAllMocks();+ driver.work(MessageQueue.workflowQueue, jest.fn());++ const processor = jest.mocked(Worker).mock.calls[0][1];++ if (typeof processor !== 'function') {+ throw new Error('Worker processor was not registered');+ }++ await processor({+ id: 'job-id',+ name: 'job',+ data: {},+ updateData: jest.fn(),+ updateProgress: jest.fn(),+ ...job,+ } as unknown as Job);+ };++ beforeAll(() => {+ jest.setSystemTime(1_700_000_000_000);+ });++ it('records the wait net of the scheduled delay', async () => {+ await processJob({+ opts: { delay: 60_000 },+ timestamp: Date.now() - 62_000,+ attemptsStarted: 1,+ });++ expect(recordHistogram).toHaveBeenCalledWith(+ expect.objectContaining({+ value: 2_000,+ attributes: { queue: MessageQueue.workflowQueue, job_name: 'job' },+ }),+ );+ });++ it('records no wait for a job picked up before its scheduled delay elapsed', async () => {+ await processJob({+ opts: { delay: 60_000 },+ timestamp: Date.now() - 30_000,+ attemptsStarted: 1,+ });++ expect(recordHistogram).toHaveBeenCalledWith(+ expect.objectContaining({ value: 0 }),+ );+ });++ it('does not sample the wait of a re-run after a retry or a stall', async () => {+ await processJob({+ opts: {},+ timestamp: Date.now() - 120_000,+ attemptsStarted: 2,+ });++ expect(recordHistogram).not.toHaveBeenCalled();+ });+});++describe('BullMQDriver queue job count gauges', () => {+ const createMultiObservableGauge = jest.fn();+ const driver = new BullMQDriver(+ {} as never,+ { createMultiObservableGauge } as never,+ {} as never,+ {} as never,+ );++ driver.register(MessageQueue.workflowQueue);++ const collectGauge = async (metricName: string) => {+ const gauge = createMultiObservableGauge.mock.calls+ .map(([options]) => options)+ .find((options) => options.metricName === metricName);++ if (!gauge) {+ throw new Error(`Gauge ${metricName} was not registered`);+ }++ return gauge.callback();+ };++ beforeEach(() => {+ jest.clearAllMocks();+ mockGetJobCountByTypes.mockImplementation(async (...types: string[]) =>+ types.includes('delayed') ? 7 : 3,+ );+ driver.onModuleInit();+ });++ it('reports the runnable backlog without the delayed jobs', async () => {+ await expect(+ collectGauge('twenty_queue_jobs_waiting_total'),+ ).resolves.toEqual([+ { value: 3, attributes: { queue: MessageQueue.workflowQueue } },+ ]);++ expect(mockGetJobCountByTypes).toHaveBeenCalledWith(+ 'waiting',+ 'prioritized',+ 'paused',+ 'waiting-children',+ );+ });++ it('reports the delayed jobs on their own gauge', async () => {+ await expect(+ collectGauge('twenty_queue_jobs_delayed_total'),+ ).resolves.toEqual([+ { value: 7, attributes: { queue: MessageQueue.workflowQueue } },+ ]);++ expect(mockGetJobCountByTypes).toHaveBeenCalledWith('delayed');+ });+});+describe('BullMQDriver global concurrency', () => {const driver = new BullMQDriver({} as never,diff --git a/packages/twenty-server/src/engine/core-modules/message-queue/drivers/bullmq.driver.ts b/packages/twenty-server/src/engine/core-modules/message-queue/drivers/bullmq.driver.tsindex 0f5c5c1a645..0ef97b56a42 100644--- a/packages/twenty-server/src/engine/core-modules/message-queue/drivers/bullmq.driver.ts+++ b/packages/twenty-server/src/engine/core-modules/message-queue/drivers/bullmq.driver.ts@@ -9,6 +9,7 @@ import * as Sentry from '@sentry/node';import {type Job,type JobsOptions,+ type JobType,MetricsTime,Queue,type QueueOptions,@@ -42,7 +43,9 @@ import { type MessageQueue } from 'src/engine/core-modules/message-queue/messageimport { type QueueJobChangedEvent } from 'src/engine/core-modules/message-queue/types/queue-job-changed-event.type';import { getJobKey } from 'src/engine/core-modules/message-queue/utils/get-job-key.util';import { buildQueueJobIdWithSuffix } from 'src/engine/core-modules/message-queue/utils/build-queue-job-id-with-suffix.util';+import { computeJobQueueWaitMs } from 'src/engine/core-modules/message-queue/utils/compute-job-queue-wait-ms.util';import { isQueueJobIdAlreadyWaiting } from 'src/engine/core-modules/message-queue/utils/is-queue-job-id-already-waiting.util';+import { JOB_LATENCY_MS_BUCKET_BOUNDARIES } from 'src/engine/core-modules/metrics/constants/job-latency-ms-bucket-boundaries.constant';import { type MetricsService } from 'src/engine/core-modules/metrics/metrics.service';import { MetricsKeys } from 'src/engine/core-modules/metrics/types/metrics-keys.type';import { applyWorkspaceSentryContextFromJobData } from 'src/engine/core-modules/sentry/utils/apply-workspace-sentry-context-from-job-data.util';@@ -80,34 +83,54 @@ export class BullMQDriveronModuleInit() {this.metricsService.createMultiObservableGauge({metricName: 'twenty_queue_jobs_waiting_total',- options: { description: 'Current number of jobs waiting in queue' },- callback: async () => {- const observations: Array<{- value: number;- attributes: { queue: string };- }> = [];-- for (const [queueName, queue] of Object.entries(this.queueMap)) {- try {- const waitingCount = await queue.count();-- observations.push({- value: waitingCount,- attributes: { queue: queueName },- });- } catch (error) {- this.logger.error(- `Failed to collect waiting jobs metrics for queue ${queueName}`,- error,- );- }- }+ options: {+ description:+ 'Current number of jobs waiting in queue, scheduled (delayed) jobs excluded',+ },+ callback: () =>+ this.collectJobCountsByQueue([+ 'waiting',+ 'prioritized',+ 'paused',+ 'waiting-children',+ ]),+ });- return observations;+ this.metricsService.createMultiObservableGauge({+ metricName: 'twenty_queue_jobs_delayed_total',+ options: {+ description:+ 'Current number of jobs waiting for their scheduled time (delay or retry backoff)',},+ callback: () => this.collectJobCountsByQueue(['delayed']),});}+ private async collectJobCountsByQueue(+ jobTypes: JobType[],+ ): Promise<Array<{ value: number; attributes: { queue: string } }>> {+ const observations: Array<{+ value: number;+ attributes: { queue: string };+ }> = [];++ for (const [queueName, queue] of Object.entries(this.queueMap)) {+ try {+ observations.push({+ value: await queue.getJobCountByTypes(...jobTypes),+ attributes: { queue: queueName },+ });+ } catch (error) {+ this.logger.error(+ `Failed to collect ${jobTypes.join('/')} jobs metrics for queue ${queueName}`,+ error,+ );+ }+ }++ return observations;+ }+register(queueName: MessageQueue): void {this.queueMap[queueName] = new Queue(queueName, this.options);}@@ -228,14 +251,17 @@ export class BullMQDriverSentry.withIsolationScope(async () => {applyWorkspaceSentryContextFromJobData(job.data);- const queueLatency = Math.max(0, Date.now() - job.timestamp);+ const queueWaitMs = computeJobQueueWaitMs({ job, now: Date.now() });- this.metricsService.recordHistogram({- key: MetricsKeys.JobLatencyMs,- value: queueLatency,- unit: 'ms',- attributes: { queue: queueName, job_name: job.name },- });+ if (isDefined(queueWaitMs)) {+ this.metricsService.recordHistogram({+ key: MetricsKeys.JobLatencyMs,+ value: queueWaitMs,+ unit: 'ms',+ attributes: { queue: queueName, job_name: job.name },+ bucketBoundaries: JOB_LATENCY_MS_BUCKET_BOUNDARIES,+ });+ }// TODO: Correctly support for job.idconst timeStart = performance.now();diff --git a/packages/twenty-server/src/engine/core-modules/message-queue/utils/__tests__/compute-job-queue-wait-ms.util.spec.ts b/packages/twenty-server/src/engine/core-modules/message-queue/utils/__tests__/compute-job-queue-wait-ms.util.spec.tsnew file mode 100644index 00000000000..ecd9f95e55a--- /dev/null+++ b/packages/twenty-server/src/engine/core-modules/message-queue/utils/__tests__/compute-job-queue-wait-ms.util.spec.ts@@ -0,0 +1,58 @@+import { computeJobQueueWaitMs } from 'src/engine/core-modules/message-queue/utils/compute-job-queue-wait-ms.util';++const NOW = 1_700_000_000_000;++describe('computeJobQueueWaitMs', () => {+ it('measures the time since creation for a job enqueued without delay', () => {+ expect(+ computeJobQueueWaitMs({+ job: { timestamp: NOW - 1_500, opts: {}, attemptsStarted: 1 },+ now: NOW,+ }),+ ).toBe(1_500);+ });++ it('subtracts the scheduled delay so an on-time delayed job reads as no wait', () => {+ expect(+ computeJobQueueWaitMs({+ job: {+ timestamp: NOW - 60_000,+ opts: { delay: 60_000 },+ attemptsStarted: 1,+ },+ now: NOW,+ }),+ ).toBe(0);+ });++ it('keeps only the wait beyond the scheduled delay', () => {+ expect(+ computeJobQueueWaitMs({+ job: {+ timestamp: NOW - 62_000,+ opts: { delay: 60_000 },+ attemptsStarted: 1,+ },+ now: NOW,+ }),+ ).toBe(2_000);+ });++ it('never goes negative when the worker runs ahead of the clock', () => {+ expect(+ computeJobQueueWaitMs({+ job: { timestamp: NOW + 10, opts: {}, attemptsStarted: 1 },+ now: NOW,+ }),+ ).toBe(0);+ });++ it('does not measure a re-run, whose creation timestamp predates its previous start', () => {+ expect(+ computeJobQueueWaitMs({+ job: { timestamp: NOW - 120_000, opts: {}, attemptsStarted: 2 },+ now: NOW,+ }),+ ).toBeUndefined();+ });+});diff --git a/packages/twenty-server/src/engine/core-modules/message-queue/utils/compute-job-queue-wait-ms.util.ts b/packages/twenty-server/src/engine/core-modules/message-queue/utils/compute-job-queue-wait-ms.util.tsnew file mode 100644index 00000000000..e9309c27346--- /dev/null+++ b/packages/twenty-server/src/engine/core-modules/message-queue/utils/compute-job-queue-wait-ms.util.ts@@ -0,0 +1,17 @@+import { type Job } from 'bullmq';++export const computeJobQueueWaitMs = ({+ job,+ now,+}: {+ job: Pick<Job, 'timestamp' | 'opts' | 'attemptsStarted'>;+ now: number;+}): number | undefined => {+ if (job.attemptsStarted > 1) {+ return undefined;+ }++ const expectedDelayMs = job.opts.delay ?? 0;++ return Math.max(0, now - job.timestamp - expectedDelayMs);+};diff --git a/packages/twenty-server/src/engine/core-modules/metrics/constants/job-latency-ms-bucket-boundaries.constant.ts b/packages/twenty-server/src/engine/core-modules/metrics/constants/job-latency-ms-bucket-boundaries.constant.tsnew file mode 100644index 00000000000..fa701503bde--- /dev/null+++ b/packages/twenty-server/src/engine/core-modules/metrics/constants/job-latency-ms-bucket-boundaries.constant.ts@@ -0,0 +1,4 @@+export const JOB_LATENCY_MS_BUCKET_BOUNDARIES = [+ 50, 100, 250, 500, 1000, 2500, 5000, 10000, 30000, 60000, 120000, 300000,+ 600000, 1800000, 3600000,+] as const;