Skip to content

The open alternative to Salesforce, designed for AI.

feat: expose queue job progress (#26157)

Weiko58e16b81cbc1
diff --git a/packages/twenty-server/src/engine/core-modules/message-queue/__tests__/message-queue.explorer.spec.ts b/packages/twenty-server/src/engine/core-modules/message-queue/__tests__/message-queue.explorer.spec.ts
new file mode 100644
index 00000000000..5a791e0921a
--- /dev/null
+++ b/packages/twenty-server/src/engine/core-modules/message-queue/__tests__/message-queue.explorer.spec.ts
@@ -0,0 +1,70 @@
+import { DiscoveryModule } from '@nestjs/core';
+import { Test } from '@nestjs/testing';
+
+import { ExceptionHandlerService } from 'src/engine/core-modules/exception-handler/exception-handler.service';
+import { Process } from 'src/engine/core-modules/message-queue/decorators/process.decorator';
+import { Processor } from 'src/engine/core-modules/message-queue/decorators/processor.decorator';
+import { SyncDriver } from 'src/engine/core-modules/message-queue/drivers/sync.driver';
+import { type MessageQueueJobProgressContext } from 'src/engine/core-modules/message-queue/interfaces/message-queue-job.interface';
+import { MessageQueueMetadataAccessor } from 'src/engine/core-modules/message-queue/message-queue-metadata.accessor';
+import { MessageQueue } from 'src/engine/core-modules/message-queue/message-queue.constants';
+import { MessageQueueExplorer } from 'src/engine/core-modules/message-queue/message-queue.explorer';
+import { EventLoopStallMonitorService } from 'src/engine/core-modules/message-queue/services/event-loop-stall-monitor.service';
+import { MessageQueueService } from 'src/engine/core-modules/message-queue/services/message-queue.service';
+import { getQueueToken } from 'src/engine/core-modules/message-queue/utils/get-queue-token.util';
+import { TwentyConfigService } from 'src/engine/core-modules/twenty-config/twenty-config.service';
+
+@Processor(MessageQueue.workspaceQueue)
+class ProgressProcessor {
+ @Process('progress-job')
+ async handle(
+ data: { total: number },
+ context: MessageQueueJobProgressContext,
+ ) {
+ await context.updateProgress({ completed: data.total, total: data.total });
+ }
+}
+
+describe('MessageQueueExplorer progress context', () => {
+ it('lets a decorated processor publish progress through its context', async () => {
+ const queue = new MessageQueueService(
+ new SyncDriver(),
+ MessageQueue.workspaceQueue,
+ );
+ const module = await Test.createTestingModule({
+ imports: [DiscoveryModule],
+ providers: [
+ ProgressProcessor,
+ MessageQueueExplorer,
+ MessageQueueMetadataAccessor,
+ {
+ provide: getQueueToken(MessageQueue.workspaceQueue),
+ useValue: queue,
+ },
+ { provide: TwentyConfigService, useValue: { get: () => [] } },
+ {
+ provide: ExceptionHandlerService,
+ useValue: { captureExceptions: jest.fn() },
+ },
+ {
+ provide: EventLoopStallMonitorService,
+ useValue: { registerJobStart: jest.fn(), registerJobEnd: jest.fn() },
+ },
+ ],
+ }).compile();
+
+ try {
+ await module.init();
+ const [id] = await queue.bulkAdd('progress-job', [
+ { data: { total: 10 } },
+ ]);
+
+ expect((await queue.getJobs([id]))[id]).toMatchObject({
+ state: 'completed',
+ progress: { completed: 10, total: 10 },
+ });
+ } finally {
+ await module.close();
+ }
+ });
+});
diff --git a/packages/twenty-server/src/engine/core-modules/message-queue/drivers/__tests__/bullmq.driver.spec.ts b/packages/twenty-server/src/engine/core-modules/message-queue/drivers/__tests__/bullmq.driver.spec.ts
index 11be5aa0102..8e30ebe4117 100644
--- a/packages/twenty-server/src/engine/core-modules/message-queue/drivers/__tests__/bullmq.driver.spec.ts
+++ b/packages/twenty-server/src/engine/core-modules/message-queue/drivers/__tests__/bullmq.driver.spec.ts
@@ -1,17 +1,21 @@
+import { type Job, Worker } from 'bullmq';
+
import { BullMQDriver } from 'src/engine/core-modules/message-queue/drivers/bullmq.driver';
import { MessageQueue } from 'src/engine/core-modules/message-queue/message-queue.constants';
const mockGetJobs = jest.fn();
+const mockGetJob = jest.fn();
const mockAdd = jest.fn();
const mockAddBulk = jest.fn();
jest.mock('bullmq', () => ({
Queue: jest.fn().mockImplementation(() => ({
getJobs: mockGetJobs,
+ getJob: mockGetJob,
add: mockAdd,
addBulk: mockAddBulk,
})),
- Worker: jest.fn(),
+ Worker: jest.fn().mockImplementation(() => ({ on: jest.fn() })),
MetricsTime: { ONE_WEEK: 1 },
}));
@@ -128,3 +132,62 @@ describe('BullMQDriver deduplication', () => {
});
});
});
+
+describe('BullMQDriver progress', () => {
+ const driver = new BullMQDriver(
+ {} as never,
+ { recordHistogram: jest.fn() } as never,
+ {} as never,
+ {} as never,
+ );
+
+ driver.register(MessageQueue.workspaceQueue);
+
+ it.each([0, 50, { completed: 5, total: 10 }])(
+ 'persists progress %p through BullMQ and exposes it in job snapshots',
+ async (progress) => {
+ const job: Pick<
+ Job,
+ | 'id'
+ | 'name'
+ | 'data'
+ | 'opts'
+ | 'timestamp'
+ | 'progress'
+ | 'updateProgress'
+ | 'getState'
+ > = {
+ id: 'job-id',
+ name: 'job',
+ data: {},
+ opts: {},
+ timestamp: Date.now(),
+ progress: 0,
+ async updateProgress(updatedProgress: Job['progress']) {
+ this.progress = updatedProgress;
+ },
+ getState: jest.fn().mockResolvedValue('active'),
+ };
+ const updateProgress = jest.spyOn(job, 'updateProgress');
+ mockGetJob.mockResolvedValue(job);
+ driver.work(MessageQueue.workspaceQueue, async (queueJob) => {
+ await queueJob.updateProgress(progress);
+ });
+
+ const processor = jest.mocked(Worker).mock.calls[0][1];
+
+ if (typeof processor !== 'function') {
+ throw new Error('Worker processor was not registered');
+ }
+
+ await processor(job as Job);
+
+ expect(updateProgress).toHaveBeenCalledWith(progress);
+ const jobs = await driver.getJobs(MessageQueue.workspaceQueue, [
+ 'job-id',
+ ]);
+
+ expect(jobs['job-id']).toMatchObject({ state: 'active', progress });
+ },
+ );
+});
diff --git a/packages/twenty-server/src/engine/core-modules/message-queue/drivers/__tests__/sync.driver.spec.ts b/packages/twenty-server/src/engine/core-modules/message-queue/drivers/__tests__/sync.driver.spec.ts
new file mode 100644
index 00000000000..6eb227c4cbd
--- /dev/null
+++ b/packages/twenty-server/src/engine/core-modules/message-queue/drivers/__tests__/sync.driver.spec.ts
@@ -0,0 +1,121 @@
+import { QUEUE_RETENTION } from 'src/engine/core-modules/message-queue/constants/queue-retention.constants';
+import { SyncDriver } from 'src/engine/core-modules/message-queue/drivers/sync.driver';
+import { MessageQueue } from 'src/engine/core-modules/message-queue/message-queue.constants';
+
+const QUEUE = MessageQueue.workspaceQueue;
+
+describe('SyncDriver job snapshots', () => {
+ it.each([50, { completed: 5, total: 10 }])(
+ 'retains progress %p and payload updates when work completes',
+ async (progress) => {
+ const driver = new SyncDriver();
+ driver.work(QUEUE, async (job) => {
+ expect((await driver.getJobs(QUEUE, [job.id]))[job.id]).toMatchObject({
+ state: 'active',
+ progress: 0,
+ });
+ await job.updateData({ cursor: 'next-page' });
+ await job.updateProgress(progress);
+ expect((await driver.getJobs(QUEUE, [job.id]))[job.id]).toMatchObject({
+ state: 'active',
+ data: { cursor: 'next-page' },
+ progress,
+ });
+ });
+ const [id] = await driver.bulkAdd(QUEUE, 'job', [{ data: {} }]);
+
+ expect((await driver.getJobs(QUEUE, [id]))[id]).toMatchObject({
+ state: 'completed',
+ data: { cursor: 'next-page' },
+ progress,
+ attemptsMade: 1,
+ finishedOn: expect.any(Number),
+ });
+ expect(await driver.getJobs(MessageQueue.emailQueue, [id])).toEqual({});
+ expect(await driver.getJobs(QUEUE, ['missing-job'])).toEqual({});
+ },
+ );
+
+ it('retains failed progress and still processes the remaining bulk jobs', async () => {
+ const driver = new SyncDriver();
+ const error = new Error('Storage failed');
+ driver.work(QUEUE, async (job) => {
+ await job.updateProgress(50);
+ if (job.id === 'failed-job') {
+ throw error;
+ }
+ await job.updateProgress(100);
+ });
+
+ await expect(
+ driver.bulkAdd(QUEUE, 'job', [
+ { jobId: 'failed-job', data: {} },
+ { jobId: 'completed-job', data: {} },
+ ]),
+ ).rejects.toBe(error);
+ const jobs = await driver.getJobs(QUEUE, ['failed-job', 'completed-job']);
+
+ expect(jobs['failed-job']).toMatchObject({
+ state: 'failed',
+ progress: 50,
+ failedReason: 'Storage failed',
+ attemptsMade: 1,
+ finishedOn: expect.any(Number),
+ });
+ expect(jobs['completed-job']).toMatchObject({
+ state: 'completed',
+ progress: 100,
+ });
+ });
+
+ it('bounds retained snapshots without evicting an active job', async () => {
+ const driver = new SyncDriver();
+ let finishActiveJob = () => {};
+ const activeJob = new Promise<void>((resolve) => {
+ finishActiveJob = resolve;
+ });
+ driver.work(QUEUE, async (job) => {
+ if (job.id === 'active-job') {
+ await job.updateProgress(25);
+ await activeJob;
+ }
+ });
+ const pendingAdd = driver.bulkAdd(QUEUE, 'job', [
+ { jobId: 'active-job', data: {} },
+ ]);
+ const jobs = Array.from(
+ { length: QUEUE_RETENTION.completedMaxCount + 1 },
+ (_, index) => ({
+ jobId: `job-${index}`,
+ data: {},
+ }),
+ );
+ const ids = await driver.bulkAdd(QUEUE, 'job', jobs);
+ const snapshots = await driver.getJobs(QUEUE, ['active-job', ...ids]);
+
+ expect(Object.keys(snapshots)).toHaveLength(
+ QUEUE_RETENTION.completedMaxCount,
+ );
+ expect(snapshots[ids[0]]).toBeUndefined();
+ expect(snapshots[ids[ids.length - 1]]?.state).toBe('completed');
+ expect(snapshots['active-job']).toMatchObject({
+ state: 'active',
+ progress: 25,
+ });
+
+ finishActiveJob();
+ await pendingAdd;
+ });
+
+ it('prunes old snapshots when another job finishes', async () => {
+ const driver = new SyncDriver();
+ driver.work(QUEUE, async () => {});
+ const [oldId] = await driver.bulkAdd(QUEUE, 'job', [{ data: {} }]);
+ jest.setSystemTime(Date.now() + QUEUE_RETENTION.completedMaxAge * 1000 + 1);
+ const [newId] = await driver.bulkAdd(QUEUE, 'job', [{ data: {} }]);
+ const jobs = await driver.getJobs(QUEUE, [oldId, newId]);
+
+ expect(jobs[oldId]).toBeUndefined();
+ expect(jobs[newId]?.state).toBe('completed');
+ });
+});
diff --git a/packages/twenty-server/src/engine/core-modules/message-queue/drivers/bullmq.driver.ts b/packages/twenty-server/src/engine/core-modules/message-queue/drivers/bullmq.driver.ts
index 8cff3468a51..d470466edcf 100644
--- a/packages/twenty-server/src/engine/core-modules/message-queue/drivers/bullmq.driver.ts
+++ b/packages/twenty-server/src/engine/core-modules/message-queue/drivers/bullmq.driver.ts
@@ -236,6 +236,7 @@ export class BullMQDriver
name: job.name,
retryLimit: Math.max(0, (job.opts.attempts ?? 1) - 1),
updateData: (data) => job.updateData(data),
+ updateProgress: (progress) => job.updateProgress(progress),
abortSignal,
});
const timeEnd = performance.now();
@@ -567,6 +568,7 @@ export class BullMQDriver
state,
attemptsMade: job.attemptsMade,
failedReason: job.failedReason,
+ progress: job.progress,
timestamp: job.timestamp,
processedOn: job.processedOn,
finishedOn: job.finishedOn,
diff --git a/packages/twenty-server/src/engine/core-modules/message-queue/drivers/interfaces/message-queue-driver.interface.ts b/packages/twenty-server/src/engine/core-modules/message-queue/drivers/interfaces/message-queue-driver.interface.ts
index 966a523ca2c..0a3d0474bb3 100644
--- a/packages/twenty-server/src/engine/core-modules/message-queue/drivers/interfaces/message-queue-driver.interface.ts
+++ b/packages/twenty-server/src/engine/core-modules/message-queue/drivers/interfaces/message-queue-driver.interface.ts
@@ -5,7 +5,10 @@ import {
type QueueJobOptions,
type QueueJobRecipient,
} from 'src/engine/core-modules/message-queue/drivers/interfaces/job-options.interface';
-import { type MessageQueueJobData } from 'src/engine/core-modules/message-queue/interfaces/message-queue-job.interface';
+import {
+ type MessageQueueJob,
+ type MessageQueueJobData,
+} from 'src/engine/core-modules/message-queue/interfaces/message-queue-job.interface';
import { type MessageQueueWorkerOptions } from 'src/engine/core-modules/message-queue/interfaces/message-queue-worker-options.interface';
import { type MessageQueue } from 'src/engine/core-modules/message-queue/message-queue.constants';
@@ -25,7 +28,7 @@ export interface MessageQueueDriver {
): Promise<string[]>;
work<T extends MessageQueueJobData>(
queueName: MessageQueue,
- handler: ({ data, id }: { data: T; id: string }) => Promise<void> | void,
+ handler: (job: MessageQueueJob<T>) => Promise<void> | void,
options?: MessageQueueWorkerOptions,
): void;
addCron<T extends MessageQueueJobData | undefined>({
@@ -71,6 +74,7 @@ export type QueueJobDetails<T extends MessageQueueJobData> = {
state: JobState;
attemptsMade: number;
failedReason?: string;
+ progress?: unknown;
timestamp: number;
processedOn?: number;
finishedOn?: number;
diff --git a/packages/twenty-server/src/engine/core-modules/message-queue/drivers/sync.driver.ts b/packages/twenty-server/src/engine/core-modules/message-queue/drivers/sync.driver.ts
index 804d62a7e79..b997293e601 100644
--- a/packages/twenty-server/src/engine/core-modules/message-queue/drivers/sync.driver.ts
+++ b/packages/twenty-server/src/engine/core-modules/message-queue/drivers/sync.driver.ts
@@ -3,9 +3,11 @@ import { Logger } from '@nestjs/common';
import { isDefined } from 'twenty-shared/utils';
import { v4 } from 'uuid';
+import { QUEUE_RETENTION } from 'src/engine/core-modules/message-queue/constants/queue-retention.constants';
import {
type MessageQueueDriver,
type QueueJobToAdd,
+ type QueueJobDetails,
} from 'src/engine/core-modules/message-queue/drivers/interfaces/message-queue-driver.interface';
import {
type MessageQueueJob,
@@ -21,7 +23,22 @@ export class SyncDriver implements MessageQueueDriver {
[queueName: string]: (job: MessageQueueJob) => Promise<void> | void;
} = {};
- constructor() {}
+ private readonly jobs = new Map<
+ string,
+ QueueJobDetails<MessageQueueJobData>
+ >();
+
+ async getJobs<TData extends MessageQueueJobData>(
+ queueName: MessageQueue,
+ jobIds: string[],
+ ): Promise<Partial<Record<string, QueueJobDetails<TData>>>> {
+ return Object.fromEntries(
+ jobIds
+ .map((id) => this.jobs.get(`${queueName}:${id}`))
+ .filter(isDefined)
+ .map((job) => [job.id, job]),
+ ) as Partial<Record<string, QueueJobDetails<TData>>>;
+ }
async add<T extends MessageQueueJobData>(
queueName: MessageQueue,
@@ -96,7 +113,45 @@ export class SyncDriver implements MessageQueueDriver {
const worker = this.workersMap[queueName];
if (worker) {
- await worker(job);
+ const details: QueueJobDetails<MessageQueueJobData> = {
+ id: job.id,
+ data: job.data ?? {},
+ state: 'active',
+ attemptsMade: 0,
+ progress: 0,
+ timestamp: Date.now(),
+ processedOn: Date.now(),
+ };
+ this.jobs.set(`${queueName}:${job.id}`, details);
+ job.updateData = async (data) => {
+ job.data = data;
+ details.data = data ?? {};
+ };
+ job.updateProgress = async (progress) => {
+ details.progress = progress;
+ };
+ try {
+ await worker(job);
+ details.state = 'completed';
+ } catch (error) {
+ details.state = 'failed';
+ details.failedReason =
+ error instanceof Error ? error.message : String(error);
+ throw error;
+ } finally {
+ details.finishedOn = Date.now();
+ details.attemptsMade++;
+ for (const [key, entry] of this.jobs) {
+ if (
+ entry.state !== 'active' &&
+ (this.jobs.size > QUEUE_RETENTION.completedMaxCount ||
+ Date.now() - (entry.finishedOn ?? entry.timestamp) >
+ QUEUE_RETENTION.completedMaxAge * 1000)
+ ) {
+ this.jobs.delete(key);
+ }
+ }
+ }
} else {
if (process.env.NODE_ENV !== 'test') {
this.logger.error(`No handler found for job: ${queueName}`);
@@ -114,6 +169,7 @@ export class SyncDriver implements MessageQueueDriver {
name,
data,
retryLimit: 0,
+ updateProgress: async () => {},
updateData: async (updatedData) => {
job.data = updatedData;
},
diff --git a/packages/twenty-server/src/engine/core-modules/message-queue/interfaces/message-queue-job.interface.ts b/packages/twenty-server/src/engine/core-modules/message-queue/interfaces/message-queue-job.interface.ts
index e947b12dd67..9d54389be03 100644
--- a/packages/twenty-server/src/engine/core-modules/message-queue/interfaces/message-queue-job.interface.ts
+++ b/packages/twenty-server/src/engine/core-modules/message-queue/interfaces/message-queue-job.interface.ts
@@ -5,6 +5,7 @@ export interface MessageQueueJob<T = any> {
data: T;
retryLimit: number;
updateData(data: T): Promise<void>;
+ updateProgress(progress: number | object): Promise<void>;
abortSignal?: AbortSignal;
}
@@ -12,6 +13,10 @@ export interface MessageQueueJobContext {
abortSignal?: AbortSignal;
}
+export interface MessageQueueJobProgressContext extends MessageQueueJobContext {
+ updateProgress(progress: number | object): Promise<void>;
+}
+
export interface MessageQueueJobRetryContext<
T = MessageQueueJobData,
> extends MessageQueueJobContext {
diff --git a/packages/twenty-server/src/engine/core-modules/message-queue/message-queue.explorer.ts b/packages/twenty-server/src/engine/core-modules/message-queue/message-queue.explorer.ts
index 8cdd1f3903e..67703de1f50 100644
--- a/packages/twenty-server/src/engine/core-modules/message-queue/message-queue.explorer.ts
+++ b/packages/twenty-server/src/engine/core-modules/message-queue/message-queue.explorer.ts
@@ -271,6 +271,7 @@ export class MessageQueueExplorer implements OnModuleInit {
abortSignal: job.abortSignal,
retryLimit: job.retryLimit,
updateData: job.updateData,
+ updateProgress: job.updateProgress,
});
} catch (err) {
if (shouldCaptureException(err)) {