Skip to content

The open alternative to Salesforce, designed for AI.

Measure real queue wait and report delayed jobs apart from waiting ones (#27329)

martmulld0d7c1aef145
diff --git a/packages/twenty-server/src/engine/core-modules/message-queue/drivers/__tests__/bullmq.driver.spec.ts b/packages/twenty-server/src/engine/core-modules/message-queue/drivers/__tests__/bullmq.driver.spec.ts
index b0cd26f9840..c188ea2c69e 100644
--- a/packages/twenty-server/src/engine/core-modules/message-queue/drivers/__tests__/bullmq.driver.spec.ts
+++ b/packages/twenty-server/src/engine/core-modules/message-queue/drivers/__tests__/bullmq.driver.spec.ts
@@ -9,6 +9,7 @@ const mockAdd = jest.fn();
const mockAddBulk = jest.fn();
const mockSetGlobalConcurrency = jest.fn();
const mockRemoveGlobalConcurrency = jest.fn();
+const mockGetJobCountByTypes = jest.fn();
jest.mock('bullmq', () => ({
Queue: jest.fn().mockImplementation(() => ({
@@ -18,6 +19,7 @@ jest.mock('bullmq', () => ({
addBulk: mockAddBulk,
setGlobalConcurrency: mockSetGlobalConcurrency,
removeGlobalConcurrency: mockRemoveGlobalConcurrency,
+ getJobCountByTypes: mockGetJobCountByTypes,
})),
Worker: jest.fn().mockImplementation(() => ({ on: jest.fn() })),
MetricsTime: { ONE_WEEK: 1 },
@@ -196,6 +198,138 @@ describe('BullMQDriver progress', () => {
);
});
+describe('BullMQDriver queue wait metric', () => {
+ const recordHistogram = jest.fn();
+ const driver = new BullMQDriver(
+ {} as never,
+ { recordHistogram } as never,
+ {} as never,
+ {} as never,
+ );
+
+ driver.register(MessageQueue.workflowQueue);
+
+ const processJob = async (
+ job: Pick<Job, 'opts' | 'timestamp' | 'attemptsStarted'>,
+ ) => {
+ jest.clearAllMocks();
+ driver.work(MessageQueue.workflowQueue, jest.fn());
+
+ const processor = jest.mocked(Worker).mock.calls[0][1];
+
+ if (typeof processor !== 'function') {
+ throw new Error('Worker processor was not registered');
+ }
+
+ await processor({
+ id: 'job-id',
+ name: 'job',
+ data: {},
+ updateData: jest.fn(),
+ updateProgress: jest.fn(),
+ ...job,
+ } as unknown as Job);
+ };
+
+ beforeAll(() => {
+ jest.setSystemTime(1_700_000_000_000);
+ });
+
+ it('records the wait net of the scheduled delay', async () => {
+ await processJob({
+ opts: { delay: 60_000 },
+ timestamp: Date.now() - 62_000,
+ attemptsStarted: 1,
+ });
+
+ expect(recordHistogram).toHaveBeenCalledWith(
+ expect.objectContaining({
+ value: 2_000,
+ attributes: { queue: MessageQueue.workflowQueue, job_name: 'job' },
+ }),
+ );
+ });
+
+ it('records no wait for a job picked up before its scheduled delay elapsed', async () => {
+ await processJob({
+ opts: { delay: 60_000 },
+ timestamp: Date.now() - 30_000,
+ attemptsStarted: 1,
+ });
+
+ expect(recordHistogram).toHaveBeenCalledWith(
+ expect.objectContaining({ value: 0 }),
+ );
+ });
+
+ it('does not sample the wait of a re-run after a retry or a stall', async () => {
+ await processJob({
+ opts: {},
+ timestamp: Date.now() - 120_000,
+ attemptsStarted: 2,
+ });
+
+ expect(recordHistogram).not.toHaveBeenCalled();
+ });
+});
+
+describe('BullMQDriver queue job count gauges', () => {
+ const createMultiObservableGauge = jest.fn();
+ const driver = new BullMQDriver(
+ {} as never,
+ { createMultiObservableGauge } as never,
+ {} as never,
+ {} as never,
+ );
+
+ driver.register(MessageQueue.workflowQueue);
+
+ const collectGauge = async (metricName: string) => {
+ const gauge = createMultiObservableGauge.mock.calls
+ .map(([options]) => options)
+ .find((options) => options.metricName === metricName);
+
+ if (!gauge) {
+ throw new Error(`Gauge ${metricName} was not registered`);
+ }
+
+ return gauge.callback();
+ };
+
+ beforeEach(() => {
+ jest.clearAllMocks();
+ mockGetJobCountByTypes.mockImplementation(async (...types: string[]) =>
+ types.includes('delayed') ? 7 : 3,
+ );
+ driver.onModuleInit();
+ });
+
+ it('reports the runnable backlog without the delayed jobs', async () => {
+ await expect(
+ collectGauge('twenty_queue_jobs_waiting_total'),
+ ).resolves.toEqual([
+ { value: 3, attributes: { queue: MessageQueue.workflowQueue } },
+ ]);
+
+ expect(mockGetJobCountByTypes).toHaveBeenCalledWith(
+ 'waiting',
+ 'prioritized',
+ 'paused',
+ 'waiting-children',
+ );
+ });
+
+ it('reports the delayed jobs on their own gauge', async () => {
+ await expect(
+ collectGauge('twenty_queue_jobs_delayed_total'),
+ ).resolves.toEqual([
+ { value: 7, attributes: { queue: MessageQueue.workflowQueue } },
+ ]);
+
+ expect(mockGetJobCountByTypes).toHaveBeenCalledWith('delayed');
+ });
+});
+
describe('BullMQDriver global concurrency', () => {
const driver = new BullMQDriver(
{} as never,
diff --git a/packages/twenty-server/src/engine/core-modules/message-queue/drivers/bullmq.driver.ts b/packages/twenty-server/src/engine/core-modules/message-queue/drivers/bullmq.driver.ts
index 0f5c5c1a645..0ef97b56a42 100644
--- a/packages/twenty-server/src/engine/core-modules/message-queue/drivers/bullmq.driver.ts
+++ b/packages/twenty-server/src/engine/core-modules/message-queue/drivers/bullmq.driver.ts
@@ -9,6 +9,7 @@ import * as Sentry from '@sentry/node';
import {
type Job,
type JobsOptions,
+ type JobType,
MetricsTime,
Queue,
type QueueOptions,
@@ -42,7 +43,9 @@ import { type MessageQueue } from 'src/engine/core-modules/message-queue/message
import { type QueueJobChangedEvent } from 'src/engine/core-modules/message-queue/types/queue-job-changed-event.type';
import { getJobKey } from 'src/engine/core-modules/message-queue/utils/get-job-key.util';
import { buildQueueJobIdWithSuffix } from 'src/engine/core-modules/message-queue/utils/build-queue-job-id-with-suffix.util';
+import { computeJobQueueWaitMs } from 'src/engine/core-modules/message-queue/utils/compute-job-queue-wait-ms.util';
import { isQueueJobIdAlreadyWaiting } from 'src/engine/core-modules/message-queue/utils/is-queue-job-id-already-waiting.util';
+import { JOB_LATENCY_MS_BUCKET_BOUNDARIES } from 'src/engine/core-modules/metrics/constants/job-latency-ms-bucket-boundaries.constant';
import { type MetricsService } from 'src/engine/core-modules/metrics/metrics.service';
import { MetricsKeys } from 'src/engine/core-modules/metrics/types/metrics-keys.type';
import { applyWorkspaceSentryContextFromJobData } from 'src/engine/core-modules/sentry/utils/apply-workspace-sentry-context-from-job-data.util';
@@ -80,34 +83,54 @@ export class BullMQDriver
onModuleInit() {
this.metricsService.createMultiObservableGauge({
metricName: 'twenty_queue_jobs_waiting_total',
- options: { description: 'Current number of jobs waiting in queue' },
- callback: async () => {
- const observations: Array<{
- value: number;
- attributes: { queue: string };
- }> = [];
-
- for (const [queueName, queue] of Object.entries(this.queueMap)) {
- try {
- const waitingCount = await queue.count();
-
- observations.push({
- value: waitingCount,
- attributes: { queue: queueName },
- });
- } catch (error) {
- this.logger.error(
- `Failed to collect waiting jobs metrics for queue ${queueName}`,
- error,
- );
- }
- }
+ options: {
+ description:
+ 'Current number of jobs waiting in queue, scheduled (delayed) jobs excluded',
+ },
+ callback: () =>
+ this.collectJobCountsByQueue([
+ 'waiting',
+ 'prioritized',
+ 'paused',
+ 'waiting-children',
+ ]),
+ });
- return observations;
+ this.metricsService.createMultiObservableGauge({
+ metricName: 'twenty_queue_jobs_delayed_total',
+ options: {
+ description:
+ 'Current number of jobs waiting for their scheduled time (delay or retry backoff)',
},
+ callback: () => this.collectJobCountsByQueue(['delayed']),
});
}
+ private async collectJobCountsByQueue(
+ jobTypes: JobType[],
+ ): Promise<Array<{ value: number; attributes: { queue: string } }>> {
+ const observations: Array<{
+ value: number;
+ attributes: { queue: string };
+ }> = [];
+
+ for (const [queueName, queue] of Object.entries(this.queueMap)) {
+ try {
+ observations.push({
+ value: await queue.getJobCountByTypes(...jobTypes),
+ attributes: { queue: queueName },
+ });
+ } catch (error) {
+ this.logger.error(
+ `Failed to collect ${jobTypes.join('/')} jobs metrics for queue ${queueName}`,
+ error,
+ );
+ }
+ }
+
+ return observations;
+ }
+
register(queueName: MessageQueue): void {
this.queueMap[queueName] = new Queue(queueName, this.options);
}
@@ -228,14 +251,17 @@ export class BullMQDriver
Sentry.withIsolationScope(async () => {
applyWorkspaceSentryContextFromJobData(job.data);
- const queueLatency = Math.max(0, Date.now() - job.timestamp);
+ const queueWaitMs = computeJobQueueWaitMs({ job, now: Date.now() });
- this.metricsService.recordHistogram({
- key: MetricsKeys.JobLatencyMs,
- value: queueLatency,
- unit: 'ms',
- attributes: { queue: queueName, job_name: job.name },
- });
+ if (isDefined(queueWaitMs)) {
+ this.metricsService.recordHistogram({
+ key: MetricsKeys.JobLatencyMs,
+ value: queueWaitMs,
+ unit: 'ms',
+ attributes: { queue: queueName, job_name: job.name },
+ bucketBoundaries: JOB_LATENCY_MS_BUCKET_BOUNDARIES,
+ });
+ }
// TODO: Correctly support for job.id
const timeStart = performance.now();
diff --git a/packages/twenty-server/src/engine/core-modules/message-queue/utils/__tests__/compute-job-queue-wait-ms.util.spec.ts b/packages/twenty-server/src/engine/core-modules/message-queue/utils/__tests__/compute-job-queue-wait-ms.util.spec.ts
new file mode 100644
index 00000000000..ecd9f95e55a
--- /dev/null
+++ b/packages/twenty-server/src/engine/core-modules/message-queue/utils/__tests__/compute-job-queue-wait-ms.util.spec.ts
@@ -0,0 +1,58 @@
+import { computeJobQueueWaitMs } from 'src/engine/core-modules/message-queue/utils/compute-job-queue-wait-ms.util';
+
+const NOW = 1_700_000_000_000;
+
+describe('computeJobQueueWaitMs', () => {
+ it('measures the time since creation for a job enqueued without delay', () => {
+ expect(
+ computeJobQueueWaitMs({
+ job: { timestamp: NOW - 1_500, opts: {}, attemptsStarted: 1 },
+ now: NOW,
+ }),
+ ).toBe(1_500);
+ });
+
+ it('subtracts the scheduled delay so an on-time delayed job reads as no wait', () => {
+ expect(
+ computeJobQueueWaitMs({
+ job: {
+ timestamp: NOW - 60_000,
+ opts: { delay: 60_000 },
+ attemptsStarted: 1,
+ },
+ now: NOW,
+ }),
+ ).toBe(0);
+ });
+
+ it('keeps only the wait beyond the scheduled delay', () => {
+ expect(
+ computeJobQueueWaitMs({
+ job: {
+ timestamp: NOW - 62_000,
+ opts: { delay: 60_000 },
+ attemptsStarted: 1,
+ },
+ now: NOW,
+ }),
+ ).toBe(2_000);
+ });
+
+ it('never goes negative when the worker runs ahead of the clock', () => {
+ expect(
+ computeJobQueueWaitMs({
+ job: { timestamp: NOW + 10, opts: {}, attemptsStarted: 1 },
+ now: NOW,
+ }),
+ ).toBe(0);
+ });
+
+ it('does not measure a re-run, whose creation timestamp predates its previous start', () => {
+ expect(
+ computeJobQueueWaitMs({
+ job: { timestamp: NOW - 120_000, opts: {}, attemptsStarted: 2 },
+ now: NOW,
+ }),
+ ).toBeUndefined();
+ });
+});
diff --git a/packages/twenty-server/src/engine/core-modules/message-queue/utils/compute-job-queue-wait-ms.util.ts b/packages/twenty-server/src/engine/core-modules/message-queue/utils/compute-job-queue-wait-ms.util.ts
new file mode 100644
index 00000000000..e9309c27346
--- /dev/null
+++ b/packages/twenty-server/src/engine/core-modules/message-queue/utils/compute-job-queue-wait-ms.util.ts
@@ -0,0 +1,17 @@
+import { type Job } from 'bullmq';
+
+export const computeJobQueueWaitMs = ({
+ job,
+ now,
+}: {
+ job: Pick<Job, 'timestamp' | 'opts' | 'attemptsStarted'>;
+ now: number;
+}): number | undefined => {
+ if (job.attemptsStarted > 1) {
+ return undefined;
+ }
+
+ const expectedDelayMs = job.opts.delay ?? 0;
+
+ return Math.max(0, now - job.timestamp - expectedDelayMs);
+};
diff --git a/packages/twenty-server/src/engine/core-modules/metrics/constants/job-latency-ms-bucket-boundaries.constant.ts b/packages/twenty-server/src/engine/core-modules/metrics/constants/job-latency-ms-bucket-boundaries.constant.ts
new file mode 100644
index 00000000000..fa701503bde
--- /dev/null
+++ b/packages/twenty-server/src/engine/core-modules/metrics/constants/job-latency-ms-bucket-boundaries.constant.ts
@@ -0,0 +1,4 @@
+export const JOB_LATENCY_MS_BUCKET_BOUNDARIES = [
+ 50, 100, 250, 500, 1000, 2500, 5000, 10000, 30000, 60000, 120000, 300000,
+ 600000, 1800000, 3600000,
+] as const;