Skip to content

The open alternative to Salesforce, designed for AI.

Make logic function trigger job fan-out idempotent across retries (#27953)

Charles Bochet658f309ce627
diff --git a/packages/twenty-server/src/engine/core-modules/application/application-install/jobs/__tests__/trigger-install-application.job.spec.ts b/packages/twenty-server/src/engine/core-modules/application/application-install/jobs/__tests__/trigger-install-application.job.spec.ts
index 07e63056dd4..915d9e8496c 100644
--- a/packages/twenty-server/src/engine/core-modules/application/application-install/jobs/__tests__/trigger-install-application.job.spec.ts
+++ b/packages/twenty-server/src/engine/core-modules/application/application-install/jobs/__tests__/trigger-install-application.job.spec.ts
@@ -18,6 +18,7 @@ describe('TriggerInstallApplicationJob', () => {
};
const jobContext: MessageQueueJobProgressContext = {
+ jobId: 'job-id',
updateProgress: jest.fn(),
};
diff --git a/packages/twenty-server/src/engine/core-modules/application/application-install/jobs/__tests__/trigger-uninstall-application.job.spec.ts b/packages/twenty-server/src/engine/core-modules/application/application-install/jobs/__tests__/trigger-uninstall-application.job.spec.ts
index bf1a71123ec..b9537f01cda 100644
--- a/packages/twenty-server/src/engine/core-modules/application/application-install/jobs/__tests__/trigger-uninstall-application.job.spec.ts
+++ b/packages/twenty-server/src/engine/core-modules/application/application-install/jobs/__tests__/trigger-uninstall-application.job.spec.ts
@@ -20,6 +20,7 @@ describe('TriggerUninstallApplicationJob', () => {
};
const jobContext: MessageQueueJobProgressContext = {
+ jobId: 'job-id',
updateProgress: jest.fn(),
};
diff --git a/packages/twenty-server/src/engine/core-modules/application/application-upgrade/jobs/__tests__/trigger-upgrade-application.job.spec.ts b/packages/twenty-server/src/engine/core-modules/application/application-upgrade/jobs/__tests__/trigger-upgrade-application.job.spec.ts
index 3fe1f5b5e67..376a0033966 100644
--- a/packages/twenty-server/src/engine/core-modules/application/application-upgrade/jobs/__tests__/trigger-upgrade-application.job.spec.ts
+++ b/packages/twenty-server/src/engine/core-modules/application/application-upgrade/jobs/__tests__/trigger-upgrade-application.job.spec.ts
@@ -19,6 +19,7 @@ describe('TriggerUpgradeApplicationJob', () => {
};
const jobContext: MessageQueueJobProgressContext = {
+ jobId: 'job-id',
updateProgress: jest.fn(),
};
diff --git a/packages/twenty-server/src/engine/core-modules/logic-function/logic-function-trigger/triggers/database-event/__tests__/call-database-event-trigger-jobs.job.spec.ts b/packages/twenty-server/src/engine/core-modules/logic-function/logic-function-trigger/triggers/database-event/__tests__/call-database-event-trigger-jobs.job.spec.ts
index 72f0bf74b9e..ebc5e26c1ec 100644
--- a/packages/twenty-server/src/engine/core-modules/logic-function/logic-function-trigger/triggers/database-event/__tests__/call-database-event-trigger-jobs.job.spec.ts
+++ b/packages/twenty-server/src/engine/core-modules/logic-function/logic-function-trigger/triggers/database-event/__tests__/call-database-event-trigger-jobs.job.spec.ts
@@ -33,6 +33,7 @@ const CITY_FIELD_ID = 'city-field-id';
const APPLICATION_ID = 'application-id';
const APPLICATION_ROLE_ID = 'application-role-id';
const LOGIC_FUNCTION_ID = 'logic-function-id';
+const TRIGGER_JOB_ID = 'trigger-job-id';
const buildMaps = (
entities: ({ id: string; universalIdentifier: string } & Record<
@@ -229,7 +230,7 @@ describe('CallDatabaseEventTriggerJobsJob', () => {
it('should enqueue the events of an object the application role reads', async () => {
const event = buildEvent('record-1', { name: 'New', salary: 10 });
- await job.handle(buildBatch([event]));
+ await job.handle(buildBatch([event]), { jobId: TRIGGER_JOB_ID });
expect(messageQueueService.bulkAdd).toHaveBeenCalledWith(
LogicFunctionTriggerJob.name,
@@ -242,12 +243,33 @@ describe('CallDatabaseEventTriggerJobsJob', () => {
userWorkspaceId: 'user-workspace-id',
payload: expect.objectContaining(event),
}),
+ jobId: `${TRIGGER_JOB_ID}.${LOGIC_FUNCTION_ID}.0`,
},
],
expect.anything(),
);
});
+ it('should enqueue the same job ids when the trigger job is retried', async () => {
+ const batch = buildBatch([
+ buildEvent('record-1', { name: 'New', salary: 10 }),
+ ]);
+
+ await job.handle(batch, { jobId: TRIGGER_JOB_ID });
+ await job.handle(batch, { jobId: TRIGGER_JOB_ID });
+
+ const [firstAttemptJobIds, retryJobIds] =
+ messageQueueService.bulkAdd.mock.calls.map(([, jobs]) =>
+ (jobs as { jobId: string }[]).map((enqueuedJob) => enqueuedJob.jobId),
+ );
+
+ expect(firstAttemptJobIds).toEqual([expect.any(String)]);
+ expect(retryJobIds).toEqual(firstAttemptJobIds);
+ expect(messageQueueService.bulkAdd.mock.calls[1][2]).toMatchObject({
+ doNotCheckDuplicatesBasedOnPrefix: true,
+ });
+ });
+
it('should not enqueue events of an object the application role cannot read', async () => {
cacheData.rolesPermissions = buildRolesPermissions({
canReadObjectRecords: false,
@@ -255,6 +277,7 @@ describe('CallDatabaseEventTriggerJobsJob', () => {
await job.handle(
buildBatch([buildEvent('record-1', { name: 'New', salary: 10 })]),
+ { jobId: TRIGGER_JOB_ID },
);
expect(messageQueueService.bulkAdd).not.toHaveBeenCalled();
@@ -269,6 +292,7 @@ describe('CallDatabaseEventTriggerJobsJob', () => {
await job.handle(
buildBatch([buildEvent('record-1', { name: 'New', salary: 10 })]),
+ { jobId: TRIGGER_JOB_ID },
);
expect(messageQueueService.bulkAdd).not.toHaveBeenCalled();
@@ -279,6 +303,7 @@ describe('CallDatabaseEventTriggerJobsJob', () => {
await job.handle(
buildBatch([buildEvent('record-1', { name: 'New', salary: 10 })]),
+ { jobId: TRIGGER_JOB_ID },
);
expect(messageQueueService.bulkAdd).not.toHaveBeenCalled();
@@ -308,6 +333,7 @@ describe('CallDatabaseEventTriggerJobsJob', () => {
buildEvent('record-paris', { name: 'A', salary: 1, city: 'Paris' }),
buildEvent('record-berlin', { name: 'B', salary: 2, city: 'Berlin' }),
]),
+ { jobId: TRIGGER_JOB_ID },
);
expect(enqueuedPayloads().map((payload) => payload.recordId)).toEqual([
@@ -324,6 +350,7 @@ describe('CallDatabaseEventTriggerJobsJob', () => {
await job.handle(
buildBatch([buildEvent('record-1', { name: 'New', salary: 10 })]),
+ { jobId: TRIGGER_JOB_ID },
);
expect(enqueuedPayloads()[0].properties).toEqual({
@@ -345,6 +372,7 @@ describe('CallDatabaseEventTriggerJobsJob', () => {
buildBatch([
buildEvent('record-1', { name: 'Same', salary: 10 }, ['salary']),
]),
+ { jobId: TRIGGER_JOB_ID },
);
expect(messageQueueService.bulkAdd).not.toHaveBeenCalled();
@@ -372,6 +400,7 @@ describe('CallDatabaseEventTriggerJobsJob', () => {
],
MetadataReadability.PRIVATE,
),
+ { jobId: TRIGGER_JOB_ID },
);
expect(enqueuedPayloads().map((payload) => payload.recordId)).toEqual([
diff --git a/packages/twenty-server/src/engine/core-modules/logic-function/logic-function-trigger/triggers/database-event/call-database-event-trigger-jobs.job.ts b/packages/twenty-server/src/engine/core-modules/logic-function/logic-function-trigger/triggers/database-event/call-database-event-trigger-jobs.job.ts
index a397a17ff5f..622ed7d1cc6 100644
--- a/packages/twenty-server/src/engine/core-modules/logic-function/logic-function-trigger/triggers/database-event/call-database-event-trigger-jobs.job.ts
+++ b/packages/twenty-server/src/engine/core-modules/logic-function/logic-function-trigger/triggers/database-event/call-database-event-trigger-jobs.job.ts
@@ -9,10 +9,12 @@ import { InjectMessageQueue } from 'src/engine/core-modules/message-queue/decora
import { Process } from 'src/engine/core-modules/message-queue/decorators/process.decorator';
import { Processor } from 'src/engine/core-modules/message-queue/decorators/processor.decorator';
import { MessageQueue } from 'src/engine/core-modules/message-queue/message-queue.constants';
+import { type MessageQueueJobContext } from 'src/engine/core-modules/message-queue/interfaces/message-queue-job.interface';
import { ApplicationJobEnqueueThrottlerService } from 'src/engine/core-modules/message-queue/services/application-job-enqueue-throttler.service';
import { MessageQueueService } from 'src/engine/core-modules/message-queue/services/message-queue.service';
import { ThrottlerException } from 'src/engine/core-modules/throttler/throttler.exception';
import { LOGIC_FUNCTION_QUEUE_RETRY_BACKOFF } from 'src/engine/core-modules/logic-function/logic-function-trigger/constants/logic-function-queue-retry-backoff.constant';
+import { buildLogicFunctionTriggerJobIds } from 'src/engine/core-modules/logic-function/logic-function-trigger/triggers/database-event/utils/build-logic-function-trigger-job-ids';
import { findLogicFunctionsTriggeredByEventName } from 'src/engine/core-modules/logic-function/logic-function-trigger/triggers/database-event/utils/find-logic-functions-triggered-by-event-name';
import { transformEventBatchToEventPayloads } from 'src/engine/core-modules/logic-function/logic-function-trigger/triggers/database-event/utils/transform-event-batch-to-event-payloads';
import {
@@ -37,7 +39,10 @@ export class CallDatabaseEventTriggerJobsJob {
) {}
@Process(CallDatabaseEventTriggerJobsJob.name)
- async handle(workspaceEventBatch: WorkspaceEventBatch<ObjectRecordEvent>) {
+ async handle(
+ workspaceEventBatch: WorkspaceEventBatch<ObjectRecordEvent>,
+ { jobId }: MessageQueueJobContext,
+ ) {
const {
flatLogicFunctionMaps,
flatApplicationMaps,
@@ -147,14 +152,21 @@ export class CallDatabaseEventTriggerJobsJob {
throw error;
}
+ const logicFunctionTriggerJobIds = buildLogicFunctionTriggerJobIds({
+ triggerJobId: jobId,
+ logicFunctionPayloads,
+ });
+
await this.messageQueueService.bulkAdd<LogicFunctionTriggerJobData>(
LogicFunctionTriggerJob.name,
- logicFunctionPayloads.map((logicFunctionPayload) => ({
+ logicFunctionPayloads.map((logicFunctionPayload, index) => ({
data: logicFunctionPayload,
+ jobId: logicFunctionTriggerJobIds[index],
})),
{
retryLimit: 3,
backoff: LOGIC_FUNCTION_QUEUE_RETRY_BACKOFF,
+ doNotCheckDuplicatesBasedOnPrefix: true,
},
);
}
diff --git a/packages/twenty-server/src/engine/core-modules/logic-function/logic-function-trigger/triggers/database-event/utils/__tests__/build-logic-function-trigger-job-ids.spec.ts b/packages/twenty-server/src/engine/core-modules/logic-function/logic-function-trigger/triggers/database-event/utils/__tests__/build-logic-function-trigger-job-ids.spec.ts
new file mode 100644
index 00000000000..c1babaf5ad7
--- /dev/null
+++ b/packages/twenty-server/src/engine/core-modules/logic-function/logic-function-trigger/triggers/database-event/utils/__tests__/build-logic-function-trigger-job-ids.spec.ts
@@ -0,0 +1,47 @@
+import { buildLogicFunctionTriggerJobIds } from 'src/engine/core-modules/logic-function/logic-function-trigger/triggers/database-event/utils/build-logic-function-trigger-job-ids';
+
+describe('buildLogicFunctionTriggerJobIds', () => {
+ it('should number the chunks of each logic function separately', () => {
+ expect(
+ buildLogicFunctionTriggerJobIds({
+ triggerJobId: 'trigger-job-id',
+ logicFunctionPayloads: [
+ { logicFunctionId: 'logic-function-a' },
+ { logicFunctionId: 'logic-function-b' },
+ { logicFunctionId: 'logic-function-a' },
+ ],
+ }),
+ ).toEqual([
+ 'trigger-job-id.logic-function-a.0',
+ 'trigger-job-id.logic-function-b.0',
+ 'trigger-job-id.logic-function-a.1',
+ ]);
+ });
+
+ it('should give each chunk the same id whatever the order of logic functions', () => {
+ const buildJobIdSet = (logicFunctionIds: string[]) => {
+ const jobIds = buildLogicFunctionTriggerJobIds({
+ triggerJobId: 'trigger-job-id',
+ logicFunctionPayloads: logicFunctionIds.map((logicFunctionId) => ({
+ logicFunctionId,
+ })),
+ });
+
+ return new Set(jobIds);
+ };
+
+ expect(
+ buildJobIdSet([
+ 'logic-function-a',
+ 'logic-function-a',
+ 'logic-function-b',
+ ]),
+ ).toEqual(
+ buildJobIdSet([
+ 'logic-function-b',
+ 'logic-function-a',
+ 'logic-function-a',
+ ]),
+ );
+ });
+});
diff --git a/packages/twenty-server/src/engine/core-modules/logic-function/logic-function-trigger/triggers/database-event/utils/build-logic-function-trigger-job-ids.ts b/packages/twenty-server/src/engine/core-modules/logic-function/logic-function-trigger/triggers/database-event/utils/build-logic-function-trigger-job-ids.ts
new file mode 100644
index 00000000000..a104b7cc09a
--- /dev/null
+++ b/packages/twenty-server/src/engine/core-modules/logic-function/logic-function-trigger/triggers/database-event/utils/build-logic-function-trigger-job-ids.ts
@@ -0,0 +1,19 @@
+import { type LogicFunctionTriggerJobData } from 'src/engine/core-modules/logic-function/logic-function-trigger/jobs/logic-function-trigger.job';
+
+export const buildLogicFunctionTriggerJobIds = ({
+ triggerJobId,
+ logicFunctionPayloads,
+}: {
+ triggerJobId: string;
+ logicFunctionPayloads: Pick<LogicFunctionTriggerJobData, 'logicFunctionId'>[];
+}): string[] => {
+ const chunkCountByLogicFunctionId = new Map<string, number>();
+
+ return logicFunctionPayloads.map(({ logicFunctionId }) => {
+ const chunkIndex = chunkCountByLogicFunctionId.get(logicFunctionId) ?? 0;
+
+ chunkCountByLogicFunctionId.set(logicFunctionId, chunkIndex + 1);
+
+ return `${triggerJobId}.${logicFunctionId}.${chunkIndex}`;
+ });
+};
diff --git a/packages/twenty-server/src/engine/core-modules/message-queue/drivers/__tests__/bullmq.driver.spec.ts b/packages/twenty-server/src/engine/core-modules/message-queue/drivers/__tests__/bullmq.driver.spec.ts
index 8cf14193450..e4730d53597 100644
--- a/packages/twenty-server/src/engine/core-modules/message-queue/drivers/__tests__/bullmq.driver.spec.ts
+++ b/packages/twenty-server/src/engine/core-modules/message-queue/drivers/__tests__/bullmq.driver.spec.ts
@@ -130,12 +130,12 @@ describe('BullMQDriver deduplication', () => {
expect(mockAddBulk).not.toHaveBeenCalled();
});
- it('skips the deduplication when duplicated prefixes are allowed', async () => {
+ it('skips the deduplication when told not to check duplicates based on prefix', async () => {
await driver.bulkAdd(
MessageQueue.workspaceQueue,
'job',
[{ data: {}, jobId: 'ws-1.exact-job-id' }],
- { allowDuplicatedPrefixes: true },
+ { doNotCheckDuplicatesBasedOnPrefix: true },
);
expect(mockGetRanges).not.toHaveBeenCalled();
diff --git a/packages/twenty-server/src/engine/core-modules/message-queue/drivers/bullmq.driver.ts b/packages/twenty-server/src/engine/core-modules/message-queue/drivers/bullmq.driver.ts
index f99f7783aee..cb97ceedb3c 100644
--- a/packages/twenty-server/src/engine/core-modules/message-queue/drivers/bullmq.driver.ts
+++ b/packages/twenty-server/src/engine/core-modules/message-queue/drivers/bullmq.driver.ts
@@ -496,7 +496,7 @@ export class BullMQDriver
);
}
- if (options?.id && !options?.allowDuplicatedPrefixes) {
+ if (options?.id && !options?.doNotCheckDuplicatesBasedOnPrefix) {
const waitingJobIds = await this.getWaitingJobIds(queueName);
if (
@@ -550,7 +550,7 @@ export class BullMQDriver
);
}
- const jobsToAdd = options?.allowDuplicatedPrefixes
+ const jobsToAdd = options?.doNotCheckDuplicatesBasedOnPrefix
? jobs
: await this.filterOutAlreadyWaitingJobs(queueName, jobs);
diff --git a/packages/twenty-server/src/engine/core-modules/message-queue/drivers/interfaces/job-options.interface.ts b/packages/twenty-server/src/engine/core-modules/message-queue/drivers/interfaces/job-options.interface.ts
index 23656443a3c..6388e3e8b3a 100644
--- a/packages/twenty-server/src/engine/core-modules/message-queue/drivers/interfaces/job-options.interface.ts
+++ b/packages/twenty-server/src/engine/core-modules/message-queue/drivers/interfaces/job-options.interface.ts
@@ -16,7 +16,7 @@ export type QueueJobRecipient = {
export interface QueueJobOptions {
id?: string;
- allowDuplicatedPrefixes?: boolean;
+ doNotCheckDuplicatesBasedOnPrefix?: boolean;
priority?: number;
retryLimit?: number;
backoff?: QueueJobBackoffOptions;
diff --git a/packages/twenty-server/src/engine/core-modules/message-queue/interfaces/message-queue-job.interface.ts b/packages/twenty-server/src/engine/core-modules/message-queue/interfaces/message-queue-job.interface.ts
index 9d54389be03..a27e027c8e6 100644
--- a/packages/twenty-server/src/engine/core-modules/message-queue/interfaces/message-queue-job.interface.ts
+++ b/packages/twenty-server/src/engine/core-modules/message-queue/interfaces/message-queue-job.interface.ts
@@ -10,6 +10,7 @@ export interface MessageQueueJob<T = any> {
}
export interface MessageQueueJobContext {
+ jobId: string;
abortSignal?: AbortSignal;
}
diff --git a/packages/twenty-server/src/engine/core-modules/message-queue/message-queue.explorer.ts b/packages/twenty-server/src/engine/core-modules/message-queue/message-queue.explorer.ts
index fc1a8b78512..6577b5aecad 100644
--- a/packages/twenty-server/src/engine/core-modules/message-queue/message-queue.explorer.ts
+++ b/packages/twenty-server/src/engine/core-modules/message-queue/message-queue.explorer.ts
@@ -266,6 +266,7 @@ export class MessageQueueExplorer implements OnModuleInit {
try {
// @ts-expect-error legacy noImplicitAny
await instance[processMethodName].call(instance, job.data, {
+ jobId: job.id,
abortSignal: job.abortSignal,
retryLimit: job.retryLimit,
updateData: job.updateData,
diff --git a/packages/twenty-server/src/engine/metadata-modules/ai/ai-chat/jobs/__tests__/stream-agent-chat.job.spec.ts b/packages/twenty-server/src/engine/metadata-modules/ai/ai-chat/jobs/__tests__/stream-agent-chat.job.spec.ts
index 006b4d30756..3fc6a221abc 100644
--- a/packages/twenty-server/src/engine/metadata-modules/ai/ai-chat/jobs/__tests__/stream-agent-chat.job.spec.ts
+++ b/packages/twenty-server/src/engine/metadata-modules/ai/ai-chat/jobs/__tests__/stream-agent-chat.job.spec.ts
@@ -827,7 +827,10 @@ describe('StreamAgentChatJob', () => {
triggerShutdown = () => shutdownController.abort();
await expect(
- job.handle(jobData, { abortSignal: shutdownController.signal }),
+ job.handle(jobData, {
+ jobId: 'job-id',
+ abortSignal: shutdownController.signal,
+ }),
).rejects.toMatchObject({ code: AiExceptionCode.STREAM_INTERRUPTED });
const eventTypes = publishedEvents.map((event) => event.type);
@@ -865,7 +868,10 @@ describe('StreamAgentChatJob', () => {
const shutdownController = new AbortController();
- await job.handle(jobData, { abortSignal: shutdownController.signal });
+ await job.handle(jobData, {
+ jobId: 'job-id',
+ abortSignal: shutdownController.signal,
+ });
expect(publishedEvents.map((event) => event.type)).not.toContain(
'stream-error',
diff --git a/packages/twenty-server/src/modules/messaging-webhooks/handlers/outbound-delivery-event-handler.service.ts b/packages/twenty-server/src/modules/messaging-webhooks/handlers/outbound-delivery-event-handler.service.ts
index b0c5c8def36..7f50e0fe0cb 100644
--- a/packages/twenty-server/src/modules/messaging-webhooks/handlers/outbound-delivery-event-handler.service.ts
+++ b/packages/twenty-server/src/modules/messaging-webhooks/handlers/outbound-delivery-event-handler.service.ts
@@ -22,7 +22,7 @@ export class OutboundDeliveryEventHandlerService {
event,
{
id: event.dedupeKey,
- allowDuplicatedPrefixes: true,
+ doNotCheckDuplicatesBasedOnPrefix: true,
retryLimit: DELIVERY_EVENT_RETRY_LIMIT,
backoff: CAMPAIGN_SEND_RETRY_BACKOFF,
},
diff --git a/packages/twenty-server/src/modules/workflow/workflow-runner/utils/build-run-workflow-job-options.util.ts b/packages/twenty-server/src/modules/workflow/workflow-runner/utils/build-run-workflow-job-options.util.ts
index 52d20c0a12f..5517bad1343 100644
--- a/packages/twenty-server/src/modules/workflow/workflow-runner/utils/build-run-workflow-job-options.util.ts
+++ b/packages/twenty-server/src/modules/workflow/workflow-runner/utils/build-run-workflow-job-options.util.ts
@@ -4,5 +4,5 @@ export const buildRunWorkflowJobOptions = (
workflowRunId: string,
): QueueJobOptions => ({
id: workflowRunId,
- allowDuplicatedPrefixes: true,
+ doNotCheckDuplicatesBasedOnPrefix: true,
});