The open alternative to Salesforce, designed for AI.
TypeScript crmcrm-systemcustomergood-first-issuegraphqlhacktoberfestjavascriptmarketingmonoreponestjsopen-sourcepostgresqlreactreactjssalestypescriptweb
feat: expose queue job progress (#26157)
Weiko58e16b81cbc1
diff --git a/packages/twenty-server/src/engine/core-modules/message-queue/__tests__/message-queue.explorer.spec.ts b/packages/twenty-server/src/engine/core-modules/message-queue/__tests__/message-queue.explorer.spec.tsnew file mode 100644index 00000000000..5a791e0921a--- /dev/null+++ b/packages/twenty-server/src/engine/core-modules/message-queue/__tests__/message-queue.explorer.spec.ts@@ -0,0 +1,70 @@+import { DiscoveryModule } from '@nestjs/core';+import { Test } from '@nestjs/testing';++import { ExceptionHandlerService } from 'src/engine/core-modules/exception-handler/exception-handler.service';+import { Process } from 'src/engine/core-modules/message-queue/decorators/process.decorator';+import { Processor } from 'src/engine/core-modules/message-queue/decorators/processor.decorator';+import { SyncDriver } from 'src/engine/core-modules/message-queue/drivers/sync.driver';+import { type MessageQueueJobProgressContext } from 'src/engine/core-modules/message-queue/interfaces/message-queue-job.interface';+import { MessageQueueMetadataAccessor } from 'src/engine/core-modules/message-queue/message-queue-metadata.accessor';+import { MessageQueue } from 'src/engine/core-modules/message-queue/message-queue.constants';+import { MessageQueueExplorer } from 'src/engine/core-modules/message-queue/message-queue.explorer';+import { EventLoopStallMonitorService } from 'src/engine/core-modules/message-queue/services/event-loop-stall-monitor.service';+import { MessageQueueService } from 'src/engine/core-modules/message-queue/services/message-queue.service';+import { getQueueToken } from 'src/engine/core-modules/message-queue/utils/get-queue-token.util';+import { TwentyConfigService } from 'src/engine/core-modules/twenty-config/twenty-config.service';++@Processor(MessageQueue.workspaceQueue)+class ProgressProcessor {+ @Process('progress-job')+ async handle(+ data: { total: number },+ context: MessageQueueJobProgressContext,+ ) {+ await context.updateProgress({ completed: data.total, total: data.total });+ }+}++describe('MessageQueueExplorer progress context', () => {+ it('lets a decorated processor publish progress through its context', async () => {+ const queue = new MessageQueueService(+ new SyncDriver(),+ MessageQueue.workspaceQueue,+ );+ const module = await Test.createTestingModule({+ imports: [DiscoveryModule],+ providers: [+ ProgressProcessor,+ MessageQueueExplorer,+ MessageQueueMetadataAccessor,+ {+ provide: getQueueToken(MessageQueue.workspaceQueue),+ useValue: queue,+ },+ { provide: TwentyConfigService, useValue: { get: () => [] } },+ {+ provide: ExceptionHandlerService,+ useValue: { captureExceptions: jest.fn() },+ },+ {+ provide: EventLoopStallMonitorService,+ useValue: { registerJobStart: jest.fn(), registerJobEnd: jest.fn() },+ },+ ],+ }).compile();++ try {+ await module.init();+ const [id] = await queue.bulkAdd('progress-job', [+ { data: { total: 10 } },+ ]);++ expect((await queue.getJobs([id]))[id]).toMatchObject({+ state: 'completed',+ progress: { completed: 10, total: 10 },+ });+ } finally {+ await module.close();+ }+ });+});diff --git a/packages/twenty-server/src/engine/core-modules/message-queue/drivers/__tests__/bullmq.driver.spec.ts b/packages/twenty-server/src/engine/core-modules/message-queue/drivers/__tests__/bullmq.driver.spec.tsindex 11be5aa0102..8e30ebe4117 100644--- a/packages/twenty-server/src/engine/core-modules/message-queue/drivers/__tests__/bullmq.driver.spec.ts+++ b/packages/twenty-server/src/engine/core-modules/message-queue/drivers/__tests__/bullmq.driver.spec.ts@@ -1,17 +1,21 @@+import { type Job, Worker } from 'bullmq';+import { BullMQDriver } from 'src/engine/core-modules/message-queue/drivers/bullmq.driver';import { MessageQueue } from 'src/engine/core-modules/message-queue/message-queue.constants';const mockGetJobs = jest.fn();+const mockGetJob = jest.fn();const mockAdd = jest.fn();const mockAddBulk = jest.fn();jest.mock('bullmq', () => ({Queue: jest.fn().mockImplementation(() => ({getJobs: mockGetJobs,+ getJob: mockGetJob,add: mockAdd,addBulk: mockAddBulk,})),- Worker: jest.fn(),+ Worker: jest.fn().mockImplementation(() => ({ on: jest.fn() })),MetricsTime: { ONE_WEEK: 1 },}));@@ -128,3 +132,62 @@ describe('BullMQDriver deduplication', () => {});});});++describe('BullMQDriver progress', () => {+ const driver = new BullMQDriver(+ {} as never,+ { recordHistogram: jest.fn() } as never,+ {} as never,+ {} as never,+ );++ driver.register(MessageQueue.workspaceQueue);++ it.each([0, 50, { completed: 5, total: 10 }])(+ 'persists progress %p through BullMQ and exposes it in job snapshots',+ async (progress) => {+ const job: Pick<+ Job,+ | 'id'+ | 'name'+ | 'data'+ | 'opts'+ | 'timestamp'+ | 'progress'+ | 'updateProgress'+ | 'getState'+ > = {+ id: 'job-id',+ name: 'job',+ data: {},+ opts: {},+ timestamp: Date.now(),+ progress: 0,+ async updateProgress(updatedProgress: Job['progress']) {+ this.progress = updatedProgress;+ },+ getState: jest.fn().mockResolvedValue('active'),+ };+ const updateProgress = jest.spyOn(job, 'updateProgress');+ mockGetJob.mockResolvedValue(job);+ driver.work(MessageQueue.workspaceQueue, async (queueJob) => {+ await queueJob.updateProgress(progress);+ });++ const processor = jest.mocked(Worker).mock.calls[0][1];++ if (typeof processor !== 'function') {+ throw new Error('Worker processor was not registered');+ }++ await processor(job as Job);++ expect(updateProgress).toHaveBeenCalledWith(progress);+ const jobs = await driver.getJobs(MessageQueue.workspaceQueue, [+ 'job-id',+ ]);++ expect(jobs['job-id']).toMatchObject({ state: 'active', progress });+ },+ );+});diff --git a/packages/twenty-server/src/engine/core-modules/message-queue/drivers/__tests__/sync.driver.spec.ts b/packages/twenty-server/src/engine/core-modules/message-queue/drivers/__tests__/sync.driver.spec.tsnew file mode 100644index 00000000000..6eb227c4cbd--- /dev/null+++ b/packages/twenty-server/src/engine/core-modules/message-queue/drivers/__tests__/sync.driver.spec.ts@@ -0,0 +1,121 @@+import { QUEUE_RETENTION } from 'src/engine/core-modules/message-queue/constants/queue-retention.constants';+import { SyncDriver } from 'src/engine/core-modules/message-queue/drivers/sync.driver';+import { MessageQueue } from 'src/engine/core-modules/message-queue/message-queue.constants';++const QUEUE = MessageQueue.workspaceQueue;++describe('SyncDriver job snapshots', () => {+ it.each([50, { completed: 5, total: 10 }])(+ 'retains progress %p and payload updates when work completes',+ async (progress) => {+ const driver = new SyncDriver();+ driver.work(QUEUE, async (job) => {+ expect((await driver.getJobs(QUEUE, [job.id]))[job.id]).toMatchObject({+ state: 'active',+ progress: 0,+ });+ await job.updateData({ cursor: 'next-page' });+ await job.updateProgress(progress);+ expect((await driver.getJobs(QUEUE, [job.id]))[job.id]).toMatchObject({+ state: 'active',+ data: { cursor: 'next-page' },+ progress,+ });+ });+ const [id] = await driver.bulkAdd(QUEUE, 'job', [{ data: {} }]);++ expect((await driver.getJobs(QUEUE, [id]))[id]).toMatchObject({+ state: 'completed',+ data: { cursor: 'next-page' },+ progress,+ attemptsMade: 1,+ finishedOn: expect.any(Number),+ });+ expect(await driver.getJobs(MessageQueue.emailQueue, [id])).toEqual({});+ expect(await driver.getJobs(QUEUE, ['missing-job'])).toEqual({});+ },+ );++ it('retains failed progress and still processes the remaining bulk jobs', async () => {+ const driver = new SyncDriver();+ const error = new Error('Storage failed');+ driver.work(QUEUE, async (job) => {+ await job.updateProgress(50);+ if (job.id === 'failed-job') {+ throw error;+ }+ await job.updateProgress(100);+ });++ await expect(+ driver.bulkAdd(QUEUE, 'job', [+ { jobId: 'failed-job', data: {} },+ { jobId: 'completed-job', data: {} },+ ]),+ ).rejects.toBe(error);+ const jobs = await driver.getJobs(QUEUE, ['failed-job', 'completed-job']);++ expect(jobs['failed-job']).toMatchObject({+ state: 'failed',+ progress: 50,+ failedReason: 'Storage failed',+ attemptsMade: 1,+ finishedOn: expect.any(Number),+ });+ expect(jobs['completed-job']).toMatchObject({+ state: 'completed',+ progress: 100,+ });+ });++ it('bounds retained snapshots without evicting an active job', async () => {+ const driver = new SyncDriver();+ let finishActiveJob = () => {};+ const activeJob = new Promise<void>((resolve) => {+ finishActiveJob = resolve;+ });+ driver.work(QUEUE, async (job) => {+ if (job.id === 'active-job') {+ await job.updateProgress(25);+ await activeJob;+ }+ });+ const pendingAdd = driver.bulkAdd(QUEUE, 'job', [+ { jobId: 'active-job', data: {} },+ ]);+ const jobs = Array.from(+ { length: QUEUE_RETENTION.completedMaxCount + 1 },+ (_, index) => ({+ jobId: `job-${index}`,+ data: {},+ }),+ );+ const ids = await driver.bulkAdd(QUEUE, 'job', jobs);+ const snapshots = await driver.getJobs(QUEUE, ['active-job', ...ids]);++ expect(Object.keys(snapshots)).toHaveLength(+ QUEUE_RETENTION.completedMaxCount,+ );+ expect(snapshots[ids[0]]).toBeUndefined();+ expect(snapshots[ids[ids.length - 1]]?.state).toBe('completed');+ expect(snapshots['active-job']).toMatchObject({+ state: 'active',+ progress: 25,+ });++ finishActiveJob();+ await pendingAdd;+ });++ it('prunes old snapshots when another job finishes', async () => {+ const driver = new SyncDriver();+ driver.work(QUEUE, async () => {});+ const [oldId] = await driver.bulkAdd(QUEUE, 'job', [{ data: {} }]);+ jest.setSystemTime(Date.now() + QUEUE_RETENTION.completedMaxAge * 1000 + 1);+ const [newId] = await driver.bulkAdd(QUEUE, 'job', [{ data: {} }]);+ const jobs = await driver.getJobs(QUEUE, [oldId, newId]);++ expect(jobs[oldId]).toBeUndefined();+ expect(jobs[newId]?.state).toBe('completed');+ });+});diff --git a/packages/twenty-server/src/engine/core-modules/message-queue/drivers/bullmq.driver.ts b/packages/twenty-server/src/engine/core-modules/message-queue/drivers/bullmq.driver.tsindex 8cff3468a51..d470466edcf 100644--- a/packages/twenty-server/src/engine/core-modules/message-queue/drivers/bullmq.driver.ts+++ b/packages/twenty-server/src/engine/core-modules/message-queue/drivers/bullmq.driver.ts@@ -236,6 +236,7 @@ export class BullMQDrivername: job.name,retryLimit: Math.max(0, (job.opts.attempts ?? 1) - 1),updateData: (data) => job.updateData(data),+ updateProgress: (progress) => job.updateProgress(progress),abortSignal,});const timeEnd = performance.now();@@ -567,6 +568,7 @@ export class BullMQDriverstate,attemptsMade: job.attemptsMade,failedReason: job.failedReason,+ progress: job.progress,timestamp: job.timestamp,processedOn: job.processedOn,finishedOn: job.finishedOn,diff --git a/packages/twenty-server/src/engine/core-modules/message-queue/drivers/interfaces/message-queue-driver.interface.ts b/packages/twenty-server/src/engine/core-modules/message-queue/drivers/interfaces/message-queue-driver.interface.tsindex 966a523ca2c..0a3d0474bb3 100644--- a/packages/twenty-server/src/engine/core-modules/message-queue/drivers/interfaces/message-queue-driver.interface.ts+++ b/packages/twenty-server/src/engine/core-modules/message-queue/drivers/interfaces/message-queue-driver.interface.ts@@ -5,7 +5,10 @@ import {type QueueJobOptions,type QueueJobRecipient,} from 'src/engine/core-modules/message-queue/drivers/interfaces/job-options.interface';-import { type MessageQueueJobData } from 'src/engine/core-modules/message-queue/interfaces/message-queue-job.interface';+import {+ type MessageQueueJob,+ type MessageQueueJobData,+} from 'src/engine/core-modules/message-queue/interfaces/message-queue-job.interface';import { type MessageQueueWorkerOptions } from 'src/engine/core-modules/message-queue/interfaces/message-queue-worker-options.interface';import { type MessageQueue } from 'src/engine/core-modules/message-queue/message-queue.constants';@@ -25,7 +28,7 @@ export interface MessageQueueDriver {): Promise<string[]>;work<T extends MessageQueueJobData>(queueName: MessageQueue,- handler: ({ data, id }: { data: T; id: string }) => Promise<void> | void,+ handler: (job: MessageQueueJob<T>) => Promise<void> | void,options?: MessageQueueWorkerOptions,): void;addCron<T extends MessageQueueJobData | undefined>({@@ -71,6 +74,7 @@ export type QueueJobDetails<T extends MessageQueueJobData> = {state: JobState;attemptsMade: number;failedReason?: string;+ progress?: unknown;timestamp: number;processedOn?: number;finishedOn?: number;diff --git a/packages/twenty-server/src/engine/core-modules/message-queue/drivers/sync.driver.ts b/packages/twenty-server/src/engine/core-modules/message-queue/drivers/sync.driver.tsindex 804d62a7e79..b997293e601 100644--- a/packages/twenty-server/src/engine/core-modules/message-queue/drivers/sync.driver.ts+++ b/packages/twenty-server/src/engine/core-modules/message-queue/drivers/sync.driver.ts@@ -3,9 +3,11 @@ import { Logger } from '@nestjs/common';import { isDefined } from 'twenty-shared/utils';import { v4 } from 'uuid';+import { QUEUE_RETENTION } from 'src/engine/core-modules/message-queue/constants/queue-retention.constants';import {type MessageQueueDriver,type QueueJobToAdd,+ type QueueJobDetails,} from 'src/engine/core-modules/message-queue/drivers/interfaces/message-queue-driver.interface';import {type MessageQueueJob,@@ -21,7 +23,22 @@ export class SyncDriver implements MessageQueueDriver {[queueName: string]: (job: MessageQueueJob) => Promise<void> | void;} = {};- constructor() {}+ private readonly jobs = new Map<+ string,+ QueueJobDetails<MessageQueueJobData>+ >();++ async getJobs<TData extends MessageQueueJobData>(+ queueName: MessageQueue,+ jobIds: string[],+ ): Promise<Partial<Record<string, QueueJobDetails<TData>>>> {+ return Object.fromEntries(+ jobIds+ .map((id) => this.jobs.get(`${queueName}:${id}`))+ .filter(isDefined)+ .map((job) => [job.id, job]),+ ) as Partial<Record<string, QueueJobDetails<TData>>>;+ }async add<T extends MessageQueueJobData>(queueName: MessageQueue,@@ -96,7 +113,45 @@ export class SyncDriver implements MessageQueueDriver {const worker = this.workersMap[queueName];if (worker) {- await worker(job);+ const details: QueueJobDetails<MessageQueueJobData> = {+ id: job.id,+ data: job.data ?? {},+ state: 'active',+ attemptsMade: 0,+ progress: 0,+ timestamp: Date.now(),+ processedOn: Date.now(),+ };+ this.jobs.set(`${queueName}:${job.id}`, details);+ job.updateData = async (data) => {+ job.data = data;+ details.data = data ?? {};+ };+ job.updateProgress = async (progress) => {+ details.progress = progress;+ };+ try {+ await worker(job);+ details.state = 'completed';+ } catch (error) {+ details.state = 'failed';+ details.failedReason =+ error instanceof Error ? error.message : String(error);+ throw error;+ } finally {+ details.finishedOn = Date.now();+ details.attemptsMade++;+ for (const [key, entry] of this.jobs) {+ if (+ entry.state !== 'active' &&+ (this.jobs.size > QUEUE_RETENTION.completedMaxCount ||+ Date.now() - (entry.finishedOn ?? entry.timestamp) >+ QUEUE_RETENTION.completedMaxAge * 1000)+ ) {+ this.jobs.delete(key);+ }+ }+ }} else {if (process.env.NODE_ENV !== 'test') {this.logger.error(`No handler found for job: ${queueName}`);@@ -114,6 +169,7 @@ export class SyncDriver implements MessageQueueDriver {name,data,retryLimit: 0,+ updateProgress: async () => {},updateData: async (updatedData) => {job.data = updatedData;},diff --git a/packages/twenty-server/src/engine/core-modules/message-queue/interfaces/message-queue-job.interface.ts b/packages/twenty-server/src/engine/core-modules/message-queue/interfaces/message-queue-job.interface.tsindex e947b12dd67..9d54389be03 100644--- a/packages/twenty-server/src/engine/core-modules/message-queue/interfaces/message-queue-job.interface.ts+++ b/packages/twenty-server/src/engine/core-modules/message-queue/interfaces/message-queue-job.interface.ts@@ -5,6 +5,7 @@ export interface MessageQueueJob<T = any> {data: T;retryLimit: number;updateData(data: T): Promise<void>;+ updateProgress(progress: number | object): Promise<void>;abortSignal?: AbortSignal;}@@ -12,6 +13,10 @@ export interface MessageQueueJobContext {abortSignal?: AbortSignal;}+export interface MessageQueueJobProgressContext extends MessageQueueJobContext {+ updateProgress(progress: number | object): Promise<void>;+}+export interface MessageQueueJobRetryContext<T = MessageQueueJobData,> extends MessageQueueJobContext {diff --git a/packages/twenty-server/src/engine/core-modules/message-queue/message-queue.explorer.ts b/packages/twenty-server/src/engine/core-modules/message-queue/message-queue.explorer.tsindex 8cdd1f3903e..67703de1f50 100644--- a/packages/twenty-server/src/engine/core-modules/message-queue/message-queue.explorer.ts+++ b/packages/twenty-server/src/engine/core-modules/message-queue/message-queue.explorer.ts@@ -271,6 +271,7 @@ export class MessageQueueExplorer implements OnModuleInit {abortSignal: job.abortSignal,retryLimit: job.retryLimit,updateData: job.updateData,+ updateProgress: job.updateProgress,});} catch (err) {if (shouldCaptureException(err)) {