The open alternative to Salesforce, designed for AI.
TypeScript crmcrm-systemcustomergood-first-issuegraphqlhacktoberfestjavascriptmarketingmonoreponestjsopen-sourcepostgresqlreactreactjssalestypescriptweb
Make logic function trigger job fan-out idempotent across retries (#27953)
Charles Bochet658f309ce627
diff --git a/packages/twenty-server/src/engine/core-modules/application/application-install/jobs/__tests__/trigger-install-application.job.spec.ts b/packages/twenty-server/src/engine/core-modules/application/application-install/jobs/__tests__/trigger-install-application.job.spec.tsindex 07e63056dd4..915d9e8496c 100644--- a/packages/twenty-server/src/engine/core-modules/application/application-install/jobs/__tests__/trigger-install-application.job.spec.ts+++ b/packages/twenty-server/src/engine/core-modules/application/application-install/jobs/__tests__/trigger-install-application.job.spec.ts@@ -18,6 +18,7 @@ describe('TriggerInstallApplicationJob', () => {};const jobContext: MessageQueueJobProgressContext = {+ jobId: 'job-id',updateProgress: jest.fn(),};diff --git a/packages/twenty-server/src/engine/core-modules/application/application-install/jobs/__tests__/trigger-uninstall-application.job.spec.ts b/packages/twenty-server/src/engine/core-modules/application/application-install/jobs/__tests__/trigger-uninstall-application.job.spec.tsindex bf1a71123ec..b9537f01cda 100644--- a/packages/twenty-server/src/engine/core-modules/application/application-install/jobs/__tests__/trigger-uninstall-application.job.spec.ts+++ b/packages/twenty-server/src/engine/core-modules/application/application-install/jobs/__tests__/trigger-uninstall-application.job.spec.ts@@ -20,6 +20,7 @@ describe('TriggerUninstallApplicationJob', () => {};const jobContext: MessageQueueJobProgressContext = {+ jobId: 'job-id',updateProgress: jest.fn(),};diff --git a/packages/twenty-server/src/engine/core-modules/application/application-upgrade/jobs/__tests__/trigger-upgrade-application.job.spec.ts b/packages/twenty-server/src/engine/core-modules/application/application-upgrade/jobs/__tests__/trigger-upgrade-application.job.spec.tsindex 3fe1f5b5e67..376a0033966 100644--- a/packages/twenty-server/src/engine/core-modules/application/application-upgrade/jobs/__tests__/trigger-upgrade-application.job.spec.ts+++ b/packages/twenty-server/src/engine/core-modules/application/application-upgrade/jobs/__tests__/trigger-upgrade-application.job.spec.ts@@ -19,6 +19,7 @@ describe('TriggerUpgradeApplicationJob', () => {};const jobContext: MessageQueueJobProgressContext = {+ jobId: 'job-id',updateProgress: jest.fn(),};diff --git a/packages/twenty-server/src/engine/core-modules/logic-function/logic-function-trigger/triggers/database-event/__tests__/call-database-event-trigger-jobs.job.spec.ts b/packages/twenty-server/src/engine/core-modules/logic-function/logic-function-trigger/triggers/database-event/__tests__/call-database-event-trigger-jobs.job.spec.tsindex 72f0bf74b9e..ebc5e26c1ec 100644--- a/packages/twenty-server/src/engine/core-modules/logic-function/logic-function-trigger/triggers/database-event/__tests__/call-database-event-trigger-jobs.job.spec.ts+++ b/packages/twenty-server/src/engine/core-modules/logic-function/logic-function-trigger/triggers/database-event/__tests__/call-database-event-trigger-jobs.job.spec.ts@@ -33,6 +33,7 @@ const CITY_FIELD_ID = 'city-field-id';const APPLICATION_ID = 'application-id';const APPLICATION_ROLE_ID = 'application-role-id';const LOGIC_FUNCTION_ID = 'logic-function-id';+const TRIGGER_JOB_ID = 'trigger-job-id';const buildMaps = (entities: ({ id: string; universalIdentifier: string } & Record<@@ -229,7 +230,7 @@ describe('CallDatabaseEventTriggerJobsJob', () => {it('should enqueue the events of an object the application role reads', async () => {const event = buildEvent('record-1', { name: 'New', salary: 10 });- await job.handle(buildBatch([event]));+ await job.handle(buildBatch([event]), { jobId: TRIGGER_JOB_ID });expect(messageQueueService.bulkAdd).toHaveBeenCalledWith(LogicFunctionTriggerJob.name,@@ -242,12 +243,33 @@ describe('CallDatabaseEventTriggerJobsJob', () => {userWorkspaceId: 'user-workspace-id',payload: expect.objectContaining(event),}),+ jobId: `${TRIGGER_JOB_ID}.${LOGIC_FUNCTION_ID}.0`,},],expect.anything(),);});+ it('should enqueue the same job ids when the trigger job is retried', async () => {+ const batch = buildBatch([+ buildEvent('record-1', { name: 'New', salary: 10 }),+ ]);++ await job.handle(batch, { jobId: TRIGGER_JOB_ID });+ await job.handle(batch, { jobId: TRIGGER_JOB_ID });++ const [firstAttemptJobIds, retryJobIds] =+ messageQueueService.bulkAdd.mock.calls.map(([, jobs]) =>+ (jobs as { jobId: string }[]).map((enqueuedJob) => enqueuedJob.jobId),+ );++ expect(firstAttemptJobIds).toEqual([expect.any(String)]);+ expect(retryJobIds).toEqual(firstAttemptJobIds);+ expect(messageQueueService.bulkAdd.mock.calls[1][2]).toMatchObject({+ doNotCheckDuplicatesBasedOnPrefix: true,+ });+ });+it('should not enqueue events of an object the application role cannot read', async () => {cacheData.rolesPermissions = buildRolesPermissions({canReadObjectRecords: false,@@ -255,6 +277,7 @@ describe('CallDatabaseEventTriggerJobsJob', () => {await job.handle(buildBatch([buildEvent('record-1', { name: 'New', salary: 10 })]),+ { jobId: TRIGGER_JOB_ID },);expect(messageQueueService.bulkAdd).not.toHaveBeenCalled();@@ -269,6 +292,7 @@ describe('CallDatabaseEventTriggerJobsJob', () => {await job.handle(buildBatch([buildEvent('record-1', { name: 'New', salary: 10 })]),+ { jobId: TRIGGER_JOB_ID },);expect(messageQueueService.bulkAdd).not.toHaveBeenCalled();@@ -279,6 +303,7 @@ describe('CallDatabaseEventTriggerJobsJob', () => {await job.handle(buildBatch([buildEvent('record-1', { name: 'New', salary: 10 })]),+ { jobId: TRIGGER_JOB_ID },);expect(messageQueueService.bulkAdd).not.toHaveBeenCalled();@@ -308,6 +333,7 @@ describe('CallDatabaseEventTriggerJobsJob', () => {buildEvent('record-paris', { name: 'A', salary: 1, city: 'Paris' }),buildEvent('record-berlin', { name: 'B', salary: 2, city: 'Berlin' }),]),+ { jobId: TRIGGER_JOB_ID },);expect(enqueuedPayloads().map((payload) => payload.recordId)).toEqual([@@ -324,6 +350,7 @@ describe('CallDatabaseEventTriggerJobsJob', () => {await job.handle(buildBatch([buildEvent('record-1', { name: 'New', salary: 10 })]),+ { jobId: TRIGGER_JOB_ID },);expect(enqueuedPayloads()[0].properties).toEqual({@@ -345,6 +372,7 @@ describe('CallDatabaseEventTriggerJobsJob', () => {buildBatch([buildEvent('record-1', { name: 'Same', salary: 10 }, ['salary']),]),+ { jobId: TRIGGER_JOB_ID },);expect(messageQueueService.bulkAdd).not.toHaveBeenCalled();@@ -372,6 +400,7 @@ describe('CallDatabaseEventTriggerJobsJob', () => {],MetadataReadability.PRIVATE,),+ { jobId: TRIGGER_JOB_ID },);expect(enqueuedPayloads().map((payload) => payload.recordId)).toEqual([diff --git a/packages/twenty-server/src/engine/core-modules/logic-function/logic-function-trigger/triggers/database-event/call-database-event-trigger-jobs.job.ts b/packages/twenty-server/src/engine/core-modules/logic-function/logic-function-trigger/triggers/database-event/call-database-event-trigger-jobs.job.tsindex a397a17ff5f..622ed7d1cc6 100644--- a/packages/twenty-server/src/engine/core-modules/logic-function/logic-function-trigger/triggers/database-event/call-database-event-trigger-jobs.job.ts+++ b/packages/twenty-server/src/engine/core-modules/logic-function/logic-function-trigger/triggers/database-event/call-database-event-trigger-jobs.job.ts@@ -9,10 +9,12 @@ import { InjectMessageQueue } from 'src/engine/core-modules/message-queue/decoraimport { Process } from 'src/engine/core-modules/message-queue/decorators/process.decorator';import { Processor } from 'src/engine/core-modules/message-queue/decorators/processor.decorator';import { MessageQueue } from 'src/engine/core-modules/message-queue/message-queue.constants';+import { type MessageQueueJobContext } from 'src/engine/core-modules/message-queue/interfaces/message-queue-job.interface';import { ApplicationJobEnqueueThrottlerService } from 'src/engine/core-modules/message-queue/services/application-job-enqueue-throttler.service';import { MessageQueueService } from 'src/engine/core-modules/message-queue/services/message-queue.service';import { ThrottlerException } from 'src/engine/core-modules/throttler/throttler.exception';import { LOGIC_FUNCTION_QUEUE_RETRY_BACKOFF } from 'src/engine/core-modules/logic-function/logic-function-trigger/constants/logic-function-queue-retry-backoff.constant';+import { buildLogicFunctionTriggerJobIds } from 'src/engine/core-modules/logic-function/logic-function-trigger/triggers/database-event/utils/build-logic-function-trigger-job-ids';import { findLogicFunctionsTriggeredByEventName } from 'src/engine/core-modules/logic-function/logic-function-trigger/triggers/database-event/utils/find-logic-functions-triggered-by-event-name';import { transformEventBatchToEventPayloads } from 'src/engine/core-modules/logic-function/logic-function-trigger/triggers/database-event/utils/transform-event-batch-to-event-payloads';import {@@ -37,7 +39,10 @@ export class CallDatabaseEventTriggerJobsJob {) {}@Process(CallDatabaseEventTriggerJobsJob.name)- async handle(workspaceEventBatch: WorkspaceEventBatch<ObjectRecordEvent>) {+ async handle(+ workspaceEventBatch: WorkspaceEventBatch<ObjectRecordEvent>,+ { jobId }: MessageQueueJobContext,+ ) {const {flatLogicFunctionMaps,flatApplicationMaps,@@ -147,14 +152,21 @@ export class CallDatabaseEventTriggerJobsJob {throw error;}+ const logicFunctionTriggerJobIds = buildLogicFunctionTriggerJobIds({+ triggerJobId: jobId,+ logicFunctionPayloads,+ });+await this.messageQueueService.bulkAdd<LogicFunctionTriggerJobData>(LogicFunctionTriggerJob.name,- logicFunctionPayloads.map((logicFunctionPayload) => ({+ logicFunctionPayloads.map((logicFunctionPayload, index) => ({data: logicFunctionPayload,+ jobId: logicFunctionTriggerJobIds[index],})),{retryLimit: 3,backoff: LOGIC_FUNCTION_QUEUE_RETRY_BACKOFF,+ doNotCheckDuplicatesBasedOnPrefix: true,},);}diff --git a/packages/twenty-server/src/engine/core-modules/logic-function/logic-function-trigger/triggers/database-event/utils/__tests__/build-logic-function-trigger-job-ids.spec.ts b/packages/twenty-server/src/engine/core-modules/logic-function/logic-function-trigger/triggers/database-event/utils/__tests__/build-logic-function-trigger-job-ids.spec.tsnew file mode 100644index 00000000000..c1babaf5ad7--- /dev/null+++ b/packages/twenty-server/src/engine/core-modules/logic-function/logic-function-trigger/triggers/database-event/utils/__tests__/build-logic-function-trigger-job-ids.spec.ts@@ -0,0 +1,47 @@+import { buildLogicFunctionTriggerJobIds } from 'src/engine/core-modules/logic-function/logic-function-trigger/triggers/database-event/utils/build-logic-function-trigger-job-ids';++describe('buildLogicFunctionTriggerJobIds', () => {+ it('should number the chunks of each logic function separately', () => {+ expect(+ buildLogicFunctionTriggerJobIds({+ triggerJobId: 'trigger-job-id',+ logicFunctionPayloads: [+ { logicFunctionId: 'logic-function-a' },+ { logicFunctionId: 'logic-function-b' },+ { logicFunctionId: 'logic-function-a' },+ ],+ }),+ ).toEqual([+ 'trigger-job-id.logic-function-a.0',+ 'trigger-job-id.logic-function-b.0',+ 'trigger-job-id.logic-function-a.1',+ ]);+ });++ it('should give each chunk the same id whatever the order of logic functions', () => {+ const buildJobIdSet = (logicFunctionIds: string[]) => {+ const jobIds = buildLogicFunctionTriggerJobIds({+ triggerJobId: 'trigger-job-id',+ logicFunctionPayloads: logicFunctionIds.map((logicFunctionId) => ({+ logicFunctionId,+ })),+ });++ return new Set(jobIds);+ };++ expect(+ buildJobIdSet([+ 'logic-function-a',+ 'logic-function-a',+ 'logic-function-b',+ ]),+ ).toEqual(+ buildJobIdSet([+ 'logic-function-b',+ 'logic-function-a',+ 'logic-function-a',+ ]),+ );+ });+});diff --git a/packages/twenty-server/src/engine/core-modules/logic-function/logic-function-trigger/triggers/database-event/utils/build-logic-function-trigger-job-ids.ts b/packages/twenty-server/src/engine/core-modules/logic-function/logic-function-trigger/triggers/database-event/utils/build-logic-function-trigger-job-ids.tsnew file mode 100644index 00000000000..a104b7cc09a--- /dev/null+++ b/packages/twenty-server/src/engine/core-modules/logic-function/logic-function-trigger/triggers/database-event/utils/build-logic-function-trigger-job-ids.ts@@ -0,0 +1,19 @@+import { type LogicFunctionTriggerJobData } from 'src/engine/core-modules/logic-function/logic-function-trigger/jobs/logic-function-trigger.job';++export const buildLogicFunctionTriggerJobIds = ({+ triggerJobId,+ logicFunctionPayloads,+}: {+ triggerJobId: string;+ logicFunctionPayloads: Pick<LogicFunctionTriggerJobData, 'logicFunctionId'>[];+}): string[] => {+ const chunkCountByLogicFunctionId = new Map<string, number>();++ return logicFunctionPayloads.map(({ logicFunctionId }) => {+ const chunkIndex = chunkCountByLogicFunctionId.get(logicFunctionId) ?? 0;++ chunkCountByLogicFunctionId.set(logicFunctionId, chunkIndex + 1);++ return `${triggerJobId}.${logicFunctionId}.${chunkIndex}`;+ });+};diff --git a/packages/twenty-server/src/engine/core-modules/message-queue/drivers/__tests__/bullmq.driver.spec.ts b/packages/twenty-server/src/engine/core-modules/message-queue/drivers/__tests__/bullmq.driver.spec.tsindex 8cf14193450..e4730d53597 100644--- a/packages/twenty-server/src/engine/core-modules/message-queue/drivers/__tests__/bullmq.driver.spec.ts+++ b/packages/twenty-server/src/engine/core-modules/message-queue/drivers/__tests__/bullmq.driver.spec.ts@@ -130,12 +130,12 @@ describe('BullMQDriver deduplication', () => {expect(mockAddBulk).not.toHaveBeenCalled();});- it('skips the deduplication when duplicated prefixes are allowed', async () => {+ it('skips the deduplication when told not to check duplicates based on prefix', async () => {await driver.bulkAdd(MessageQueue.workspaceQueue,'job',[{ data: {}, jobId: 'ws-1.exact-job-id' }],- { allowDuplicatedPrefixes: true },+ { doNotCheckDuplicatesBasedOnPrefix: true },);expect(mockGetRanges).not.toHaveBeenCalled();diff --git a/packages/twenty-server/src/engine/core-modules/message-queue/drivers/bullmq.driver.ts b/packages/twenty-server/src/engine/core-modules/message-queue/drivers/bullmq.driver.tsindex f99f7783aee..cb97ceedb3c 100644--- a/packages/twenty-server/src/engine/core-modules/message-queue/drivers/bullmq.driver.ts+++ b/packages/twenty-server/src/engine/core-modules/message-queue/drivers/bullmq.driver.ts@@ -496,7 +496,7 @@ export class BullMQDriver);}- if (options?.id && !options?.allowDuplicatedPrefixes) {+ if (options?.id && !options?.doNotCheckDuplicatesBasedOnPrefix) {const waitingJobIds = await this.getWaitingJobIds(queueName);if (@@ -550,7 +550,7 @@ export class BullMQDriver);}- const jobsToAdd = options?.allowDuplicatedPrefixes+ const jobsToAdd = options?.doNotCheckDuplicatesBasedOnPrefix? jobs: await this.filterOutAlreadyWaitingJobs(queueName, jobs);diff --git a/packages/twenty-server/src/engine/core-modules/message-queue/drivers/interfaces/job-options.interface.ts b/packages/twenty-server/src/engine/core-modules/message-queue/drivers/interfaces/job-options.interface.tsindex 23656443a3c..6388e3e8b3a 100644--- a/packages/twenty-server/src/engine/core-modules/message-queue/drivers/interfaces/job-options.interface.ts+++ b/packages/twenty-server/src/engine/core-modules/message-queue/drivers/interfaces/job-options.interface.ts@@ -16,7 +16,7 @@ export type QueueJobRecipient = {export interface QueueJobOptions {id?: string;- allowDuplicatedPrefixes?: boolean;+ doNotCheckDuplicatesBasedOnPrefix?: boolean;priority?: number;retryLimit?: number;backoff?: QueueJobBackoffOptions;diff --git a/packages/twenty-server/src/engine/core-modules/message-queue/interfaces/message-queue-job.interface.ts b/packages/twenty-server/src/engine/core-modules/message-queue/interfaces/message-queue-job.interface.tsindex 9d54389be03..a27e027c8e6 100644--- a/packages/twenty-server/src/engine/core-modules/message-queue/interfaces/message-queue-job.interface.ts+++ b/packages/twenty-server/src/engine/core-modules/message-queue/interfaces/message-queue-job.interface.ts@@ -10,6 +10,7 @@ export interface MessageQueueJob<T = any> {}export interface MessageQueueJobContext {+ jobId: string;abortSignal?: AbortSignal;}diff --git a/packages/twenty-server/src/engine/core-modules/message-queue/message-queue.explorer.ts b/packages/twenty-server/src/engine/core-modules/message-queue/message-queue.explorer.tsindex fc1a8b78512..6577b5aecad 100644--- a/packages/twenty-server/src/engine/core-modules/message-queue/message-queue.explorer.ts+++ b/packages/twenty-server/src/engine/core-modules/message-queue/message-queue.explorer.ts@@ -266,6 +266,7 @@ export class MessageQueueExplorer implements OnModuleInit {try {// @ts-expect-error legacy noImplicitAnyawait instance[processMethodName].call(instance, job.data, {+ jobId: job.id,abortSignal: job.abortSignal,retryLimit: job.retryLimit,updateData: job.updateData,diff --git a/packages/twenty-server/src/engine/metadata-modules/ai/ai-chat/jobs/__tests__/stream-agent-chat.job.spec.ts b/packages/twenty-server/src/engine/metadata-modules/ai/ai-chat/jobs/__tests__/stream-agent-chat.job.spec.tsindex 006b4d30756..3fc6a221abc 100644--- a/packages/twenty-server/src/engine/metadata-modules/ai/ai-chat/jobs/__tests__/stream-agent-chat.job.spec.ts+++ b/packages/twenty-server/src/engine/metadata-modules/ai/ai-chat/jobs/__tests__/stream-agent-chat.job.spec.ts@@ -827,7 +827,10 @@ describe('StreamAgentChatJob', () => {triggerShutdown = () => shutdownController.abort();await expect(- job.handle(jobData, { abortSignal: shutdownController.signal }),+ job.handle(jobData, {+ jobId: 'job-id',+ abortSignal: shutdownController.signal,+ }),).rejects.toMatchObject({ code: AiExceptionCode.STREAM_INTERRUPTED });const eventTypes = publishedEvents.map((event) => event.type);@@ -865,7 +868,10 @@ describe('StreamAgentChatJob', () => {const shutdownController = new AbortController();- await job.handle(jobData, { abortSignal: shutdownController.signal });+ await job.handle(jobData, {+ jobId: 'job-id',+ abortSignal: shutdownController.signal,+ });expect(publishedEvents.map((event) => event.type)).not.toContain('stream-error',diff --git a/packages/twenty-server/src/modules/messaging-webhooks/handlers/outbound-delivery-event-handler.service.ts b/packages/twenty-server/src/modules/messaging-webhooks/handlers/outbound-delivery-event-handler.service.tsindex b0c5c8def36..7f50e0fe0cb 100644--- a/packages/twenty-server/src/modules/messaging-webhooks/handlers/outbound-delivery-event-handler.service.ts+++ b/packages/twenty-server/src/modules/messaging-webhooks/handlers/outbound-delivery-event-handler.service.ts@@ -22,7 +22,7 @@ export class OutboundDeliveryEventHandlerService {event,{id: event.dedupeKey,- allowDuplicatedPrefixes: true,+ doNotCheckDuplicatesBasedOnPrefix: true,retryLimit: DELIVERY_EVENT_RETRY_LIMIT,backoff: CAMPAIGN_SEND_RETRY_BACKOFF,},diff --git a/packages/twenty-server/src/modules/workflow/workflow-runner/utils/build-run-workflow-job-options.util.ts b/packages/twenty-server/src/modules/workflow/workflow-runner/utils/build-run-workflow-job-options.util.tsindex 52d20c0a12f..5517bad1343 100644--- a/packages/twenty-server/src/modules/workflow/workflow-runner/utils/build-run-workflow-job-options.util.ts+++ b/packages/twenty-server/src/modules/workflow/workflow-runner/utils/build-run-workflow-job-options.util.ts@@ -4,5 +4,5 @@ export const buildRunWorkflowJobOptions = (workflowRunId: string,): QueueJobOptions => ({id: workflowRunId,- allowDuplicatedPrefixes: true,+ doNotCheckDuplicatesBasedOnPrefix: true,});