The open alternative to Salesforce, designed for AI.
TypeScript crmcrm-systemcustomergood-first-issuegraphqlhacktoberfestjavascriptmarketingmonoreponestjsopen-sourcepostgresqlreactreactjssalestypescriptweb
Remove DATABASE_EVENT_JOBS_CHUNK_SIZE and Promise.all from logic function trigger jobs (#23205)
martmull4d09c400a452
diff --git a/packages/twenty-server/src/engine/core-modules/application/application-install/application-install.service.ts b/packages/twenty-server/src/engine/core-modules/application/application-install/application-install.service.tsindex 51e3a442f9f..76a40d354ac 100644--- a/packages/twenty-server/src/engine/core-modules/application/application-install/application-install.service.ts+++ b/packages/twenty-server/src/engine/core-modules/application/application-install/application-install.service.ts@@ -557,15 +557,13 @@ export class ApplicationInstallService {);if (!shouldRunSynchronously) {- await this.messageQueueService.add<LogicFunctionTriggerJobData[]>(+ await this.messageQueueService.add<LogicFunctionTriggerJobData>(LogicFunctionTriggerJob.name,- [- {- logicFunctionId: flatLogicFunction.id,- workspaceId,- payload,- },- ],+ {+ logicFunctionId: flatLogicFunction.id,+ workspaceId,+ payload,+ },{ retryLimit: 3 },);return;diff --git a/packages/twenty-server/src/engine/core-modules/application/connection-provider/__tests__/connection-provider-oauth-flow.service.spec.ts b/packages/twenty-server/src/engine/core-modules/application/connection-provider/__tests__/connection-provider-oauth-flow.service.spec.tsindex 05c4d3ea488..2d1956493f7 100644--- a/packages/twenty-server/src/engine/core-modules/application/connection-provider/__tests__/connection-provider-oauth-flow.service.spec.ts+++ b/packages/twenty-server/src/engine/core-modules/application/connection-provider/__tests__/connection-provider-oauth-flow.service.spec.ts@@ -501,17 +501,15 @@ describe('ConnectionProviderOAuthFlowService', () => {);expect(messageQueueService.add).toHaveBeenCalledWith('LogicFunctionTriggerJob',- [- {- logicFunctionId: 'logic-function-1',- workspaceId: 'workspace-1',- payload: {- connectionProviderId: 'provider-1',- connectionProviderName: 'linear',- connectedAccountId: result.connectedAccountId,- },+ {+ logicFunctionId: 'logic-function-1',+ workspaceId: 'workspace-1',+ payload: {+ connectionProviderId: 'provider-1',+ connectionProviderName: 'linear',+ connectedAccountId: result.connectedAccountId,},- ],+ },{ retryLimit: 3 },);});diff --git a/packages/twenty-server/src/engine/core-modules/application/connection-provider/connection-provider-oauth-flow.service.ts b/packages/twenty-server/src/engine/core-modules/application/connection-provider/connection-provider-oauth-flow.service.tsindex 48a548a9528..efce22153ed 100644--- a/packages/twenty-server/src/engine/core-modules/application/connection-provider/connection-provider-oauth-flow.service.ts+++ b/packages/twenty-server/src/engine/core-modules/application/connection-provider/connection-provider-oauth-flow.service.ts@@ -252,19 +252,17 @@ export class ConnectionProviderOAuthFlowService {);}- await this.messageQueueService.add<LogicFunctionTriggerJobData[]>(+ await this.messageQueueService.add<LogicFunctionTriggerJobData>(LogicFunctionTriggerJob.name,- [- {- logicFunctionId: flatLogicFunction.id,- workspaceId,- payload: {- connectionProviderId: provider.id,- connectionProviderName: provider.name,- connectedAccountId,- },+ {+ logicFunctionId: flatLogicFunction.id,+ workspaceId,+ payload: {+ connectionProviderId: provider.id,+ connectionProviderName: provider.name,+ connectedAccountId,},- ],+ },{ retryLimit: 3 },);} catch (error) {diff --git a/packages/twenty-server/src/engine/core-modules/logic-function/logic-function-trigger/jobs/logic-function-trigger.job.ts b/packages/twenty-server/src/engine/core-modules/logic-function/logic-function-trigger/jobs/logic-function-trigger.job.tsindex 61f5d855a3a..f3890cb2a98 100644--- a/packages/twenty-server/src/engine/core-modules/logic-function/logic-function-trigger/jobs/logic-function-trigger.job.ts+++ b/packages/twenty-server/src/engine/core-modules/logic-function/logic-function-trigger/jobs/logic-function-trigger.job.ts@@ -27,30 +27,33 @@ export class LogicFunctionTriggerJob {) {}@Process(LogicFunctionTriggerJob.name)- async handle(logicFunctionPayloads: LogicFunctionTriggerJobData[]) {- await Promise.all(- logicFunctionPayloads.map(async (logicFunctionPayload) => {- try {- await this.logicFunctionExecutorService.execute({- logicFunctionId: logicFunctionPayload.logicFunctionId,- workspaceId: logicFunctionPayload.workspaceId,- payload: logicFunctionPayload.payload ?? {},- userId: logicFunctionPayload.userId,- userWorkspaceId: logicFunctionPayload.userWorkspaceId,- });- } catch (error) {- // A stopped application must not fail the job: failing would make- // the queue retry an execution that is intentionally blocked.- if (- error instanceof LogicFunctionException &&- error.code === LogicFunctionExceptionCode.LOGIC_FUNCTION_DISABLED- ) {- return;- }+ async handle(+ jobData: LogicFunctionTriggerJobData | LogicFunctionTriggerJobData[],+ ) {+ // Jobs enqueued in version <=2.24.x carry arrays, remove this case once those jobs are drained+ const logicFunctionPayloads = Array.isArray(jobData) ? jobData : [jobData];- throw error;+ for (const logicFunctionPayload of logicFunctionPayloads) {+ try {+ await this.logicFunctionExecutorService.execute({+ logicFunctionId: logicFunctionPayload.logicFunctionId,+ workspaceId: logicFunctionPayload.workspaceId,+ payload: logicFunctionPayload.payload ?? {},+ userId: logicFunctionPayload.userId,+ userWorkspaceId: logicFunctionPayload.userWorkspaceId,+ });+ } catch (error) {+ // A stopped application must not fail the job: failing would make+ // the queue retry an execution that is intentionally blocked.+ if (+ error instanceof LogicFunctionException &&+ error.code === LogicFunctionExceptionCode.LOGIC_FUNCTION_DISABLED+ ) {+ continue;}- }),- );++ throw error;+ }+ }}}diff --git a/packages/twenty-server/src/engine/core-modules/logic-function/logic-function-trigger/triggers/cron/cron-trigger.cron.job.ts b/packages/twenty-server/src/engine/core-modules/logic-function/logic-function-trigger/triggers/cron/cron-trigger.cron.job.tsindex 37adbad8d60..f9a891e06be 100644--- a/packages/twenty-server/src/engine/core-modules/logic-function/logic-function-trigger/triggers/cron/cron-trigger.cron.job.ts+++ b/packages/twenty-server/src/engine/core-modules/logic-function/logic-function-trigger/triggers/cron/cron-trigger.cron.job.ts@@ -85,15 +85,13 @@ export class CronTriggerCronJob {continue;}- await this.messageQueueService.add<LogicFunctionTriggerJobData[]>(+ await this.messageQueueService.add<LogicFunctionTriggerJobData>(LogicFunctionTriggerJob.name,- [- {- logicFunctionId: logicFunction.id,- workspaceId: activeWorkspace.id,- payload: {},- },- ],+ {+ logicFunctionId: logicFunction.id,+ workspaceId: activeWorkspace.id,+ payload: {},+ },{ retryLimit: 10 },);}diff --git a/packages/twenty-server/src/engine/core-modules/logic-function/logic-function-trigger/triggers/database-event/call-database-event-trigger-jobs.job.ts b/packages/twenty-server/src/engine/core-modules/logic-function/logic-function-trigger/triggers/database-event/call-database-event-trigger-jobs.job.tsindex 7c340de4f25..dbdd5d4439c 100644--- a/packages/twenty-server/src/engine/core-modules/logic-function/logic-function-trigger/triggers/database-event/call-database-event-trigger-jobs.job.ts+++ b/packages/twenty-server/src/engine/core-modules/logic-function/logic-function-trigger/triggers/database-event/call-database-event-trigger-jobs.job.ts@@ -1,4 +1,3 @@-import chunk from 'lodash.chunk';import { isDefined } from 'twenty-shared/utils';import type { ObjectRecordEvent } from 'twenty-shared/database-events';@@ -16,8 +15,6 @@ import {import { WorkspaceCacheService } from 'src/engine/workspace-cache/services/workspace-cache.service';import { WorkspaceEventBatch } from 'src/engine/workspace-event-emitter/types/workspace-event-batch.type';-const DATABASE_EVENT_JOBS_CHUNK_SIZE = 20;-@Processor(MessageQueue.triggerQueue)export class CallDatabaseEventTriggerJobsJob {constructor(@@ -59,22 +56,11 @@ export class CallDatabaseEventTriggerJobsJob {workspaceEventBatch,});- if (logicFunctionPayloads.length === 0) {- return;- }-- const logicFunctionPayloadsChunks = chunk(+ await this.messageQueueService.bulkAdd<LogicFunctionTriggerJobData>(+ LogicFunctionTriggerJob.name,logicFunctionPayloads,- DATABASE_EVENT_JOBS_CHUNK_SIZE,+ { retryLimit: 3 },);-- for (const logicFunctionPayloadsChunk of logicFunctionPayloadsChunks) {- await this.messageQueueService.add<LogicFunctionTriggerJobData[]>(- LogicFunctionTriggerJob.name,- logicFunctionPayloadsChunk,- { retryLimit: 3 },- );- }}private shouldTriggerJob({diff --git a/packages/twenty-server/src/engine/core-modules/message-queue/drivers/bullmq.driver.ts b/packages/twenty-server/src/engine/core-modules/message-queue/drivers/bullmq.driver.tsindex 177e33e042c..0aefd7278d4 100644--- a/packages/twenty-server/src/engine/core-modules/message-queue/drivers/bullmq.driver.ts+++ b/packages/twenty-server/src/engine/core-modules/message-queue/drivers/bullmq.driver.ts@@ -327,6 +327,30 @@ export class BullMQDriver);}+ private buildJobsOptions({+ queueName,+ options,+ }: {+ queueName: MessageQueue;+ options?: QueueJobOptions;+ }): JobsOptions {+ return {+ // We suffix the id with V4() to make sure ids are unique so we can add a waiting job when a job related with the same option.id is running+ jobId: options?.id ? `${options.id}-${v4()}` : undefined,+ priority: options?.priority ?? MESSAGE_QUEUE_PRIORITY[queueName],+ attempts: 1 + (options?.retryLimit || 0),+ removeOnComplete: {+ age: QUEUE_RETENTION.completedMaxAge,+ count: QUEUE_RETENTION.completedMaxCount,+ },+ removeOnFail: {+ age: QUEUE_RETENTION.failedMaxAge,+ count: QUEUE_RETENTION.failedMaxCount,+ },+ delay: options?.delay,+ };+ }+async add<T>(queueName: MessageQueue,jobName: string,@@ -352,24 +376,43 @@ export class BullMQDriver}}- const queueOptions: JobsOptions = {- jobId: options?.id ? `${options.id}-${v4()}` : undefined, // We add V4() to id to make sure ids are uniques so we can add a waiting job when a job related with the same option.id is running- priority: options?.priority ?? MESSAGE_QUEUE_PRIORITY[queueName],- attempts: 1 + (options?.retryLimit || 0),- removeOnComplete: {- age: QUEUE_RETENTION.completedMaxAge,- count: QUEUE_RETENTION.completedMaxCount,- },- removeOnFail: {- age: QUEUE_RETENTION.failedMaxAge,- count: QUEUE_RETENTION.failedMaxCount,- },- delay: options?.delay,- };+ const queueOptions = this.buildJobsOptions({ queueName, options });await this.queueMap[queueName].add(jobName, data, queueOptions);}+ async bulkAdd<T>(+ queueName: MessageQueue,+ jobName: string,+ dataItems: T[],+ options?: QueueJobOptions,+ ): Promise<void> {+ if (!this.queueMap[queueName]) {+ throw new Error(+ `Queue ${queueName} is not registered, make sure you have added it as a queue provider`,+ );+ }++ if (dataItems.length === 0) {+ return;+ }++ const queueOptions = this.buildJobsOptions({ queueName, options });++ await this.queueMap[queueName].addBulk(+ dataItems.map((data, index) => ({+ name: jobName,+ data,+ opts: {+ ...queueOptions,+ jobId: queueOptions.jobId+ ? `${queueOptions.jobId}-${index}`+ : undefined,+ },+ })),+ );+ }+async getInFlightJobs<T extends MessageQueueJobData>(queueName: MessageQueue,): Promise<InFlightQueueJob<T>[]> {diff --git a/packages/twenty-server/src/engine/core-modules/message-queue/drivers/interfaces/message-queue-driver.interface.ts b/packages/twenty-server/src/engine/core-modules/message-queue/drivers/interfaces/message-queue-driver.interface.tsindex 8107f31c68e..1ee089e4802 100644--- a/packages/twenty-server/src/engine/core-modules/message-queue/drivers/interfaces/message-queue-driver.interface.ts+++ b/packages/twenty-server/src/engine/core-modules/message-queue/drivers/interfaces/message-queue-driver.interface.ts@@ -14,6 +14,12 @@ export interface MessageQueueDriver {data: T,options?: QueueJobOptions,): Promise<void>;+ bulkAdd<T extends MessageQueueJobData>(+ queueName: MessageQueue,+ jobName: string,+ dataItems: T[],+ options?: QueueJobOptions,+ ): Promise<void>;work<T extends MessageQueueJobData>(queueName: MessageQueue,handler: ({ data, id }: { data: T; id: string }) => Promise<void> | void,diff --git a/packages/twenty-server/src/engine/core-modules/message-queue/drivers/sync.driver.ts b/packages/twenty-server/src/engine/core-modules/message-queue/drivers/sync.driver.tsindex 7e2447f621f..47537815d6d 100644--- a/packages/twenty-server/src/engine/core-modules/message-queue/drivers/sync.driver.ts+++ b/packages/twenty-server/src/engine/core-modules/message-queue/drivers/sync.driver.ts@@ -1,5 +1,7 @@import { Logger } from '@nestjs/common';+import { isDefined } from 'twenty-shared/utils';+import { type MessageQueueDriver } from 'src/engine/core-modules/message-queue/drivers/interfaces/message-queue-driver.interface';import {type MessageQueueJob,@@ -25,6 +27,28 @@ export class SyncDriver implements MessageQueueDriver {await this.processJob(queueName, { id: '', name: jobName, data });}+ async bulkAdd<T extends MessageQueueJobData>(+ queueName: MessageQueue,+ jobName: string,+ dataItems: T[],+ ): Promise<void> {+ let firstError: unknown = undefined;++ // Each payload is an independent job in BullMQ, so a failing one must not+ // prevent the others from being processed+ for (const data of dataItems) {+ try {+ await this.processJob(queueName, { id: '', name: jobName, data });+ } catch (error) {+ firstError = firstError ?? error;+ }+ }++ if (isDefined(firstError)) {+ throw firstError;+ }+ }+async addCron<T extends MessageQueueJobData | undefined>({queueName,jobName,diff --git a/packages/twenty-server/src/engine/core-modules/message-queue/services/message-queue.service.ts b/packages/twenty-server/src/engine/core-modules/message-queue/services/message-queue.service.tsindex 71bc29801d6..1f36a63c492 100644--- a/packages/twenty-server/src/engine/core-modules/message-queue/services/message-queue.service.ts+++ b/packages/twenty-server/src/engine/core-modules/message-queue/services/message-queue.service.ts@@ -38,6 +38,14 @@ export class MessageQueueService {return this.driver.add(this.queueName, jobName, data, options);}+ bulkAdd<T extends MessageQueueJobData>(+ jobName: string,+ dataItems: T[],+ options?: QueueJobOptions,+ ): Promise<void> {+ return this.driver.bulkAdd(this.queueName, jobName, dataItems, options);+ }+getInFlightJobs<T extends MessageQueueJobData>(): Promise<InFlightQueueJob<T>[]> {diff --git a/packages/twenty-server/src/engine/core-modules/server-route-trigger/__tests__/server-route-trigger.service.spec.ts b/packages/twenty-server/src/engine/core-modules/server-route-trigger/__tests__/server-route-trigger.service.spec.tsindex 529ba9f28fd..75593f1e57d 100644--- a/packages/twenty-server/src/engine/core-modules/server-route-trigger/__tests__/server-route-trigger.service.spec.ts+++ b/packages/twenty-server/src/engine/core-modules/server-route-trigger/__tests__/server-route-trigger.service.spec.ts@@ -146,13 +146,11 @@ describe('ServerRouteTriggerService', () => {);expect(messageQueueService.add).toHaveBeenCalledWith(LogicFunctionTriggerJob.name,- [- {- logicFunctionId: 'target-id',- workspaceId: 'target-ws',- payload: { from: 'resolver' },- },- ],+ {+ logicFunctionId: 'target-id',+ workspaceId: 'target-ws',+ payload: { from: 'resolver' },+ },{ retryLimit: 3 },);expect(result).toEqual(diff --git a/packages/twenty-server/src/engine/core-modules/server-route-trigger/server-route-trigger.service.ts b/packages/twenty-server/src/engine/core-modules/server-route-trigger/server-route-trigger.service.tsindex 33ecaafd20b..d65ab3c5ded 100644--- a/packages/twenty-server/src/engine/core-modules/server-route-trigger/server-route-trigger.service.ts+++ b/packages/twenty-server/src/engine/core-modules/server-route-trigger/server-route-trigger.service.ts@@ -189,15 +189,13 @@ export class ServerRouteTriggerService {applicationRegistrationId,});- await this.messageQueueService.add<LogicFunctionTriggerJobData[]>(+ await this.messageQueueService.add<LogicFunctionTriggerJobData>(LogicFunctionTriggerJob.name,- [- {- logicFunctionId: logicFunction.id,- workspaceId,- payload,- },- ],+ {+ logicFunctionId: logicFunction.id,+ workspaceId,+ payload,+ },{ retryLimit: QUEUED_TARGET_RETRY_LIMIT },);