The open alternative to Salesforce, designed for AI.
TypeScript crmcrm-systemcustomergood-first-issuegraphqlhacktoberfestjavascriptmarketingmonoreponestjsopen-sourcepostgresqlreactreactjssalestypescriptweb
Type every queued job as a workspace job or a system job (#29025)
martmull7c45221a37e8
diff --git a/packages/twenty-server/src/database/commands/upgrade-version-command/2-46/2-46-workspace-command-1791398841984-close-pending-ask-questions-calls.command.ts b/packages/twenty-server/src/database/commands/upgrade-version-command/2-46/2-46-workspace-command-1791398841984-close-pending-ask-questions-calls.command.tsindex c92245f5185..1d53a3da178 100644--- a/packages/twenty-server/src/database/commands/upgrade-version-command/2-46/2-46-workspace-command-1791398841984-close-pending-ask-questions-calls.command.ts+++ b/packages/twenty-server/src/database/commands/upgrade-version-command/2-46/2-46-workspace-command-1791398841984-close-pending-ask-questions-calls.command.ts@@ -175,6 +175,10 @@ export class ClosePendingAskQuestionsCallsCommand extends ProvisionedWorkspaceCoawait this.messageQueueService.add<ResumePendingWakeUpJobData>(RESUME_PENDING_WAKE_UP_JOB_NAME,{ workspaceId, wakeUpId: wakeUp.id, answer: { result: {} } },+ {+ workspaceId,+ isWorkspaceJob: true,+ },);}diff --git a/packages/twenty-server/src/engine/api/graphql/workspace-query-runner/listeners/__tests__/entity-events-to-db.listener.spec.ts b/packages/twenty-server/src/engine/api/graphql/workspace-query-runner/listeners/__tests__/entity-events-to-db.listener.spec.tsindex d9424676b11..54116860072 100644--- a/packages/twenty-server/src/engine/api/graphql/workspace-query-runner/listeners/__tests__/entity-events-to-db.listener.spec.ts+++ b/packages/twenty-server/src/engine/api/graphql/workspace-query-runner/listeners/__tests__/entity-events-to-db.listener.spec.ts@@ -72,7 +72,11 @@ describe('EntityEventsToDbListener', () => {expect(triggerQueueService.add).toHaveBeenCalledWith(CallAgentDatabaseEventTriggersJob.name,UPSERT_BATCH_EVENT,- { retryLimit: 3 },+ {+ retryLimit: 3,+ workspaceId: UPSERT_BATCH_EVENT.workspaceId,+ isWorkspaceJob: true,+ },);expect(otherQueueService.add).not.toHaveBeenCalled();});diff --git a/packages/twenty-server/src/engine/api/graphql/workspace-query-runner/listeners/entity-events-to-db.listener.ts b/packages/twenty-server/src/engine/api/graphql/workspace-query-runner/listeners/entity-events-to-db.listener.tsindex 9368b679da1..898c1934ab0 100644--- a/packages/twenty-server/src/engine/api/graphql/workspace-query-runner/listeners/entity-events-to-db.listener.ts+++ b/packages/twenty-server/src/engine/api/graphql/workspace-query-runner/listeners/entity-events-to-db.listener.ts@@ -135,7 +135,11 @@ export class EntityEventsToDbListener {>(UpsertTimelineActivityFromInternalEvent.name,batchEvent as WorkspaceEventBatch<ObjectRecordNonDestructiveEvent>,- { retryLimit: 1 },+ {+ retryLimit: 1,+ workspaceId: batchEvent.workspaceId,+ isWorkspaceJob: true,+ },),);}@@ -145,7 +149,11 @@ export class EntityEventsToDbListener {this.eventLogQueueService.add<WorkspaceEventBatch<T>>(CreateEventLogFromInternalEvent.name,batchEvent,- { retryLimit: 1 },+ {+ retryLimit: 1,+ workspaceId: batchEvent.workspaceId,+ isWorkspaceJob: true,+ },),);}@@ -174,7 +182,11 @@ export class EntityEventsToDbListener {await this.webhookQueueService.add<WorkspaceEventBatchForWebhook<T>>(CallWebhookJobsJob.name,batchEventForWebhook,- { retryLimit: 3 },+ {+ retryLimit: 3,+ workspaceId: batchEventForWebhook.workspaceId,+ isWorkspaceJob: true,+ },);}@@ -199,7 +211,11 @@ export class EntityEventsToDbListener {await this.triggerQueueService.add<WorkspaceEventBatch<T>>(CallDatabaseEventTriggerJobsJob.name,batchEvent,- { retryLimit: 3 },+ {+ retryLimit: 3,+ workspaceId: batchEvent.workspaceId,+ isWorkspaceJob: true,+ },);}@@ -224,7 +240,11 @@ export class EntityEventsToDbListener {await this.triggerQueueService.add<WorkspaceEventBatch<T>>(CallAgentDatabaseEventTriggersJob.name,batchEvent,- { retryLimit: 3 },+ {+ retryLimit: 3,+ workspaceId: batchEvent.workspaceId,+ isWorkspaceJob: true,+ },);}}diff --git a/packages/twenty-server/src/engine/core-modules/admin-panel/admin-panel.resolver.ts b/packages/twenty-server/src/engine/core-modules/admin-panel/admin-panel.resolver.tsindex 7c66dbc3ea7..80eb1988471 100644--- a/packages/twenty-server/src/engine/core-modules/admin-panel/admin-panel.resolver.ts+++ b/packages/twenty-server/src/engine/core-modules/admin-panel/admin-panel.resolver.ts@@ -573,7 +573,7 @@ export class AdminPanelResolver {await this.cronQueueService.add(MarketplaceCatalogSyncCronJob.name,{},- { id: 'marketplace-catalog-sync' }, // Avoids triggering multiple pending jobs+ { id: 'marketplace-catalog-sync', isSystemJob: true }, // Avoids triggering multiple pending jobs);return true;diff --git a/packages/twenty-server/src/engine/core-modules/application/application-install/application-install.service.ts b/packages/twenty-server/src/engine/core-modules/application/application-install/application-install.service.tsindex 731040d6090..c00149f3955 100644--- a/packages/twenty-server/src/engine/core-modules/application/application-install/application-install.service.ts+++ b/packages/twenty-server/src/engine/core-modules/application/application-install/application-install.service.ts@@ -545,7 +545,7 @@ export class ApplicationInstallService {previousVersion,newVersion,isVersionUpgrade,- universalIdentifier,+ applicationUniversalIdentifier: application.universalIdentifier,});if (isPostInstallHookSynchronous) {@@ -570,6 +570,7 @@ export class ApplicationInstallService {await this.enqueueLogicFunctionWarmUp({workspaceId: params.workspaceId,applicationId: application.id,+ applicationUniversalIdentifier: application.universalIdentifier,logicFunctionUniversalIdentifiers:findLogicFunctionUniversalIdentifiersToWarmUp(workspaceMigration),});@@ -690,10 +691,12 @@ export class ApplicationInstallService {private async enqueueLogicFunctionWarmUp({workspaceId,applicationId,+ applicationUniversalIdentifier,logicFunctionUniversalIdentifiers,}: {workspaceId: string;applicationId: string;+ applicationUniversalIdentifier: string;logicFunctionUniversalIdentifiers: string[];}): Promise<void> {if (logicFunctionUniversalIdentifiers.length === 0) {@@ -710,6 +713,8 @@ export class ApplicationInstallService {applicationId,logicFunctionUniversalIdentifier,},+ workspaceId,+ applicationUniversalIdentifier,}),),WARM_UP_LOGIC_FUNCTION_JOB_OPTIONS,@@ -744,7 +749,7 @@ export class ApplicationInstallService {previousVersion?: string;newVersion: string;isVersionUpgrade: boolean;- universalIdentifier: string;+ applicationUniversalIdentifier: string;}): Promise<void> {const {manifest,@@ -752,7 +757,7 @@ export class ApplicationInstallService {previousVersion,newVersion,isVersionUpgrade,- universalIdentifier,+ applicationUniversalIdentifier,} = params;if (!isDefined(manifest.application.postInstallLogicFunction)) {@@ -767,7 +772,7 @@ export class ApplicationInstallService {if (isVersionUpgrade && !shouldRunOnVersionUpgrade) {this.logger.log(- `Skipping post-install hook for app ${universalIdentifier}: version upgrade and shouldRunOnVersionUpgrade is false`,+ `Skipping post-install hook for app ${applicationUniversalIdentifier}: version upgrade and shouldRunOnVersionUpgrade is false`,);return;@@ -785,7 +790,7 @@ export class ApplicationInstallService {if (!isDefined(flatLogicFunction)) {throw new ApplicationException(- `Post-install logic function "${postInstallLogicFunctionUniversalIdentifier}" not found for application "${universalIdentifier}" after sync. Manifest may reference a stale identifier.`,+ `Post-install logic function "${postInstallLogicFunctionUniversalIdentifier}" not found for application "${applicationUniversalIdentifier}" after sync. Manifest may reference a stale identifier.`,ApplicationExceptionCode.ENTITY_NOT_FOUND,);}@@ -793,7 +798,7 @@ export class ApplicationInstallService {const payload = { previousVersion, newVersion };this.logger.log(- `Enqueuing post-install hook for app ${universalIdentifier} with payload:`,+ `Enqueuing post-install hook for app ${applicationUniversalIdentifier} with payload:`,JSON.stringify(payload),);@@ -806,6 +811,8 @@ export class ApplicationInstallService {payload,},{+ workspaceId,+ applicationUniversalIdentifier,retryLimit: 3,backoff: LOGIC_FUNCTION_QUEUE_RETRY_BACKOFF,},diff --git a/packages/twenty-server/src/engine/core-modules/application/application-install/services/__tests__/application-lifecycle-job.service.spec.ts b/packages/twenty-server/src/engine/core-modules/application/application-install/services/__tests__/application-lifecycle-job.service.spec.tsindex acd72737aed..837d9390b9a 100644--- a/packages/twenty-server/src/engine/core-modules/application/application-install/services/__tests__/application-lifecycle-job.service.spec.ts+++ b/packages/twenty-server/src/engine/core-modules/application/application-install/services/__tests__/application-lifecycle-job.service.spec.ts@@ -77,6 +77,8 @@ describe('ApplicationLifecycleJobService', () => {{id: `upgrade-application.${WORKSPACE_ID}.${UNIVERSAL_IDENTIFIER}`,broadcastTo: { workspaceId: WORKSPACE_ID },+ workspaceId: WORKSPACE_ID,+ applicationUniversalIdentifier: UNIVERSAL_IDENTIFIER,},);});diff --git a/packages/twenty-server/src/engine/core-modules/application/application-install/services/application-lifecycle-job.service.ts b/packages/twenty-server/src/engine/core-modules/application/application-install/services/application-lifecycle-job.service.tsindex 1508760b786..8cc3f001d9c 100644--- a/packages/twenty-server/src/engine/core-modules/application/application-install/services/application-lifecycle-job.service.ts+++ b/packages/twenty-server/src/engine/core-modules/application/application-install/services/application-lifecycle-job.service.ts@@ -256,6 +256,8 @@ export class ApplicationLifecycleJobService {(await this.workspaceQueueService.add<TData>(jobName, data, {id: jobIdPrefix,broadcastTo: { workspaceId },+ workspaceId,+ applicationUniversalIdentifier: universalIdentifier,})) ?? (await this.findInFlightJobId(jobIdPrefix));if (!isDefined(jobId)) {diff --git a/packages/twenty-server/src/engine/core-modules/application/application-job/services/application-job.service.ts b/packages/twenty-server/src/engine/core-modules/application/application-job/services/application-job.service.tsindex b82a765ef60..9642abca8e0 100644--- a/packages/twenty-server/src/engine/core-modules/application/application-job/services/application-job.service.ts+++ b/packages/twenty-server/src/engine/core-modules/application/application-job/services/application-job.service.ts@@ -10,8 +10,6 @@ import { v4 } from 'uuid';import { findActiveFlatApplicationById } from 'src/engine/core-modules/application/utils/find-active-flat-application-by-id.util';import { buildJobStatus } from 'src/engine/core-modules/message-queue/utils/build-job-status.util';-import { buildApplicationGroupKey } from 'src/engine/core-modules/message-queue/utils/build-application-group-key.util';-import { buildWorkspaceGroupKey } from 'src/engine/core-modules/message-queue/utils/build-workspace-group-key.util';import { ENQUEUE_JOB_DEFAULT_RETRY_LIMIT } from 'src/engine/core-modules/application/application-job/constants/enqueue-job-default-retry-limit.constant';import { ENQUEUE_JOB_PRIORITY } from 'src/engine/core-modules/application/application-job/constants/enqueue-job-priority.constant';import { MAX_JOBS_PER_STATUS_READ } from 'src/engine/core-modules/application/application-job/constants/max-jobs-per-status-read.constant';@@ -101,6 +99,18 @@ export class ApplicationJobService {universalIdentifier: logicFunctionUniversalIdentifier,});+ const application = findActiveFlatApplicationById(+ flatApplicationMaps,+ applicationId,+ );++ if (!isDefined(application)) {+ throw new ApplicationException(+ `Application ${applicationId} not found`,+ ApplicationExceptionCode.APPLICATION_NOT_FOUND,+ );+ }+if (!isDefined(flatLogicFunction) ||isDefined(flatLogicFunction.deletedAt) ||@@ -113,10 +123,6 @@ export class ApplicationJobService {}const jobIds = jobItems.map((jobItem) => jobItem.jobId ?? v4());- const application = findActiveFlatApplicationById(- flatApplicationMaps,- applicationId,- );await this.messageQueueService.bulkAdd<LogicFunctionTriggerJobData>(LogicFunctionTriggerJob.name,@@ -129,17 +135,13 @@ export class ApplicationJobService {...(isDefined(userWorkspaceId) ? { userWorkspaceId } : {}),},jobId: buildQueueJobId({ workspaceId, jobId: jobIds[index] }),+ workspaceId,+ applicationUniversalIdentifier: application.universalIdentifier,})),{retryLimit: input.retryLimit ?? ENQUEUE_JOB_DEFAULT_RETRY_LIMIT,backoff: LOGIC_FUNCTION_QUEUE_RETRY_BACKOFF,priority: ENQUEUE_JOB_PRIORITY,- groupKeys: [- buildWorkspaceGroupKey(workspaceId),- ...(isDefined(application)- ? [buildApplicationGroupKey(application.universalIdentifier)]- : []),- ],...(isDefined(input.delayMs) ? { delay: input.delayMs } : {}),},);diff --git a/packages/twenty-server/src/engine/core-modules/application/application-marketplace/crons/commands/marketplace-catalog-sync.cron.command.ts b/packages/twenty-server/src/engine/core-modules/application/application-marketplace/crons/commands/marketplace-catalog-sync.cron.command.tsindex cd7ffd38a88..25d993aa4a5 100644--- a/packages/twenty-server/src/engine/core-modules/application/application-marketplace/crons/commands/marketplace-catalog-sync.cron.command.ts+++ b/packages/twenty-server/src/engine/core-modules/application/application-marketplace/crons/commands/marketplace-catalog-sync.cron.command.ts@@ -20,7 +20,11 @@ export class MarketplaceCatalogSyncCronCommand extends CommandRunner {}async run(): Promise<void> {- await this.messageQueueService.add(MarketplaceCatalogSyncCronJob.name, {});+ await this.messageQueueService.add(+ MarketplaceCatalogSyncCronJob.name,+ {},+ { isSystemJob: true },+ );await this.messageQueueService.addCron<undefined>({jobName: MarketplaceCatalogSyncCronJob.name,diff --git a/packages/twenty-server/src/engine/core-modules/application/application-marketplace/marketplace.resolver.ts b/packages/twenty-server/src/engine/core-modules/application/application-marketplace/marketplace.resolver.tsindex 493b296a887..e89cbce2dde 100644--- a/packages/twenty-server/src/engine/core-modules/application/application-marketplace/marketplace.resolver.ts+++ b/packages/twenty-server/src/engine/core-modules/application/application-marketplace/marketplace.resolver.ts@@ -86,7 +86,7 @@ export class MarketplaceResolver {await this.messageQueueService.add(MarketplaceCatalogSyncCronJob.name,{},- { id: 'marketplace-catalog-sync' }, // Avoids triggering multiple pending jobs+ { id: 'marketplace-catalog-sync', isSystemJob: true }, // Avoids triggering multiple pending jobs);return true;diff --git a/packages/twenty-server/src/engine/core-modules/application/application-registration/application-registration.service.ts b/packages/twenty-server/src/engine/core-modules/application/application-registration/application-registration.service.tsindex b74e87193de..d751c067179 100644--- a/packages/twenty-server/src/engine/core-modules/application/application-registration/application-registration.service.ts+++ b/packages/twenty-server/src/engine/core-modules/application/application-registration/application-registration.service.ts@@ -236,6 +236,7 @@ export class ApplicationRegistrationService {await this.applicationUpgradeQueueService.add<UpgradeApplicationsJobData>(UPGRADE_APPLICATIONS_JOB_NAME,{ applicationRegistrationId, onlyAutoUpgrade: true },+ { isSystemJob: true },);} catch (error) {this.logger.error(diff --git a/packages/twenty-server/src/engine/core-modules/application/application-upgrade/application-upgrade.service.ts b/packages/twenty-server/src/engine/core-modules/application/application-upgrade/application-upgrade.service.tsindex a6d1eb103d5..17f0afbf404 100644--- a/packages/twenty-server/src/engine/core-modules/application/application-upgrade/application-upgrade.service.ts+++ b/packages/twenty-server/src/engine/core-modules/application/application-upgrade/application-upgrade.service.ts@@ -177,6 +177,8 @@ export class ApplicationUpgradeService {workspaceId: application.workspaceId,onlyAutoUpgrade,},+ workspaceId: application.workspaceId,+ applicationUniversalIdentifier: application.universalIdentifier,})),UPGRADE_WORKSPACE_APPLICATION_JOB_OPTIONS,);diff --git a/packages/twenty-server/src/engine/core-modules/application/connection-provider/connection-provider-lifecycle-hook.service.ts b/packages/twenty-server/src/engine/core-modules/application/connection-provider/connection-provider-lifecycle-hook.service.tsindex efeb53d7655..039160b52d7 100644--- a/packages/twenty-server/src/engine/core-modules/application/connection-provider/connection-provider-lifecycle-hook.service.ts+++ b/packages/twenty-server/src/engine/core-modules/application/connection-provider/connection-provider-lifecycle-hook.service.ts@@ -30,6 +30,7 @@ import {import { InjectMessageQueue } from 'src/engine/core-modules/message-queue/decorators/message-queue.decorator';import { MessageQueue } from 'src/engine/core-modules/message-queue/message-queue.constants';import { MessageQueueService } from 'src/engine/core-modules/message-queue/services/message-queue.service';+import { type FlatLogicFunction } from 'src/engine/metadata-modules/logic-function/types/flat-logic-function.type';import { findApplicationFlatLogicFunction } from 'src/engine/metadata-modules/logic-function/utils/find-application-flat-logic-function.util';import { WorkspaceCacheService } from 'src/engine/workspace-cache/services/workspace-cache.service';@@ -115,7 +116,7 @@ export class ConnectionProviderLifecycleHookService {connectedAccountId: string;}): Promise<void> {await this.captureFailures(workspaceId, async () => {- const logicFunctionId = await this.resolveLogicFunctionId({+ const logicFunction = await this.resolveLogicFunction({hook: 'onConnect',logicFunctionUniversalIdentifier:provider.onConnectLogicFunctionUniversalIdentifier,@@ -123,18 +124,21 @@ export class ConnectionProviderLifecycleHookService {workspaceId,});- if (!isDefined(logicFunctionId)) {+ if (!isDefined(logicFunction)) {return;}await this.messageQueueService.add<LogicFunctionTriggerJobData>(LogicFunctionTriggerJob.name,{- logicFunctionId,+ logicFunctionId: logicFunction.id,workspaceId,payload: this.buildPayload({ provider, connectedAccountId }),},{+ workspaceId,+ applicationUniversalIdentifier:+ logicFunction.applicationUniversalIdentifier,retryLimit: 3,backoff: LOGIC_FUNCTION_QUEUE_RETRY_BACKOFF,},@@ -158,7 +162,7 @@ export class ConnectionProviderLifecycleHookService {workspaceId: string;accessToken: string;}): Promise<IdentifiedAccount | null> {- const logicFunctionId = await this.resolveLogicFunctionId({+ const logicFunction = await this.resolveLogicFunction({hook: 'identifyAccount',logicFunctionUniversalIdentifier:provider.identifyAccountLogicFunctionUniversalIdentifier,@@ -166,7 +170,7 @@ export class ConnectionProviderLifecycleHookService {workspaceId,});- if (!isDefined(logicFunctionId)) {+ if (!isDefined(logicFunction)) {return null;}@@ -179,7 +183,7 @@ export class ConnectionProviderLifecycleHookService {const data = await this.executeConnectionFlowHook({hook: 'identifyAccount',provider,- logicFunctionId,+ logicFunctionId: logicFunction.id,workspaceId,payload,});@@ -209,7 +213,7 @@ export class ConnectionProviderLifecycleHookService {workspaceId: string;callbackUrl: string;}): Promise<string> {- const logicFunctionId = await this.resolveLogicFunctionIdOrThrow({+ const logicFunction = await this.resolveLogicFunctionOrThrow({hook: 'authorize',logicFunctionUniversalIdentifier:provider.customConfig.authorizeLogicFunctionUniversalIdentifier,@@ -226,7 +230,7 @@ export class ConnectionProviderLifecycleHookService {const data = await this.executeConnectionFlowHook({hook: 'authorize',provider,- logicFunctionId,+ logicFunctionId: logicFunction.id,workspaceId,payload,});@@ -259,7 +263,7 @@ export class ConnectionProviderLifecycleHookService {'httpMethod' | 'queryStringParameters' | 'headers' | 'body' | 'rawBody'>;}): Promise<ParsedConnectedAccountFromCallback | null> {- const logicFunctionId = await this.resolveLogicFunctionIdOrThrow({+ const logicFunction = await this.resolveLogicFunctionOrThrow({hook: 'callback',logicFunctionUniversalIdentifier:provider.customConfig.callbackLogicFunctionUniversalIdentifier,@@ -276,7 +280,7 @@ export class ConnectionProviderLifecycleHookService {const data = await this.executeConnectionFlowHook({hook: 'callback',provider,- logicFunctionId,+ logicFunctionId: logicFunction.id,workspaceId,payload,});@@ -310,7 +314,7 @@ export class ConnectionProviderLifecycleHookService {id: connectionProviderId,workspaceId,});- const logicFunctionId = await this.resolveLogicFunctionId({+ const logicFunction = await this.resolveLogicFunction({hook: 'onDisconnect',logicFunctionUniversalIdentifier:provider.onDisconnectLogicFunctionUniversalIdentifier,@@ -318,12 +322,12 @@ export class ConnectionProviderLifecycleHookService {workspaceId,});- if (!isDefined(logicFunctionId)) {+ if (!isDefined(logicFunction)) {return;}const executionResult = await this.logicFunctionExecutorService.execute({- logicFunctionId,+ logicFunctionId: logicFunction.id,workspaceId,payload: this.buildPayload({ provider, connectedAccountId }),shouldEnforceUsageLimits: false,@@ -393,7 +397,7 @@ export class ConnectionProviderLifecycleHookService {};}- private async resolveLogicFunctionId({+ private async resolveLogicFunction({logicFunctionUniversalIdentifier,...logicFunctionLookup}: {@@ -401,18 +405,18 @@ export class ConnectionProviderLifecycleHookService {logicFunctionUniversalIdentifier: string | null;provider: ConnectionLifecycleHookProvider;workspaceId: string;- }): Promise<string | undefined> {+ }): Promise<FlatLogicFunction | undefined> {if (!isDefined(logicFunctionUniversalIdentifier)) {return undefined;}- return this.resolveLogicFunctionIdOrThrow({+ return this.resolveLogicFunctionOrThrow({...logicFunctionLookup,logicFunctionUniversalIdentifier,});}- private async resolveLogicFunctionIdOrThrow({+ private async resolveLogicFunctionOrThrow({hook,logicFunctionUniversalIdentifier,provider,@@ -422,7 +426,7 @@ export class ConnectionProviderLifecycleHookService {logicFunctionUniversalIdentifier: string;provider: Pick<ConnectionLifecycleHookProvider, 'id' | 'applicationId'>;workspaceId: string;- }): Promise<string> {+ }): Promise<FlatLogicFunction> {const { flatLogicFunctionMaps } =await this.workspaceCacheService.getOrRecompute(workspaceId, ['flatLogicFunctionMaps',@@ -441,6 +445,6 @@ export class ConnectionProviderLifecycleHookService {);}- return flatLogicFunction.id;+ return flatLogicFunction;}}diff --git a/packages/twenty-server/src/engine/core-modules/application/jobs/upgrade-workspace-application.job-constants.ts b/packages/twenty-server/src/engine/core-modules/application/jobs/upgrade-workspace-application.job-constants.tsindex fcebc30a42d..4e084c2fd57 100644--- a/packages/twenty-server/src/engine/core-modules/application/jobs/upgrade-workspace-application.job-constants.ts+++ b/packages/twenty-server/src/engine/core-modules/application/jobs/upgrade-workspace-application.job-constants.ts@@ -1,11 +1,11 @@-import { type QueueJobOptions } from 'src/engine/core-modules/message-queue/drivers/interfaces/job-options.interface';+import { type QueueJobSettings } from 'src/engine/core-modules/message-queue/drivers/interfaces/job-options.interface';export const UPGRADE_WORKSPACE_APPLICATION_JOB_NAME ='UpgradeWorkspaceApplicationJob';export const UPGRADE_WORKSPACE_APPLICATION_JOB_ENQUEUE_BATCH_SIZE = 500;-export const UPGRADE_WORKSPACE_APPLICATION_JOB_OPTIONS: QueueJobOptions = {+export const UPGRADE_WORKSPACE_APPLICATION_JOB_OPTIONS: QueueJobSettings = {retryLimit: 2,backoff: {strategy: 'exponential',diff --git a/packages/twenty-server/src/engine/core-modules/application/pre-installed-apps/pre-installed-apps.service.ts b/packages/twenty-server/src/engine/core-modules/application/pre-installed-apps/pre-installed-apps.service.tsindex 9242a1d4097..3b1aaf9647e 100644--- a/packages/twenty-server/src/engine/core-modules/application/pre-installed-apps/pre-installed-apps.service.ts+++ b/packages/twenty-server/src/engine/core-modules/application/pre-installed-apps/pre-installed-apps.service.ts@@ -35,7 +35,11 @@ export class PreInstalledAppsService {await this.messageQueueService.add<InstallPreInstalledAppsJobData>(INSTALL_PRE_INSTALLED_APPS_JOB_NAME,{ workspaceId },- { id: `${INSTALL_PRE_INSTALLED_APPS_JOB_NAME}-${workspaceId}` },+ {+ id: `${INSTALL_PRE_INSTALLED_APPS_JOB_NAME}-${workspaceId}`,+ workspaceId,+ isWorkspaceJob: true,+ },);}diff --git a/packages/twenty-server/src/engine/core-modules/application/utils/build-application-queue-job-scope.util.ts b/packages/twenty-server/src/engine/core-modules/application/utils/build-application-queue-job-scope.util.tsnew file mode 100644index 00000000000..d389029050f--- /dev/null+++ b/packages/twenty-server/src/engine/core-modules/application/utils/build-application-queue-job-scope.util.ts@@ -0,0 +1,24 @@+import { isDefined } from 'twenty-shared/utils';++import { type FlatApplicationCacheMaps } from 'src/engine/core-modules/application/types/flat-application-cache-maps.type';+import { findActiveFlatApplicationById } from 'src/engine/core-modules/application/utils/find-active-flat-application-by-id.util';+import { type QueueJobScope } from 'src/engine/core-modules/message-queue/drivers/interfaces/job-options.interface';++export const buildApplicationQueueJobScope = ({+ workspaceId,+ applicationId,+ flatApplicationMaps,+}: {+ workspaceId: string;+ applicationId: string | null | undefined;+ flatApplicationMaps: FlatApplicationCacheMaps;+}): QueueJobScope => {+ const applicationUniversalIdentifier = isDefined(applicationId)+ ? findActiveFlatApplicationById(flatApplicationMaps, applicationId)+ ?.universalIdentifier+ : undefined;++ return isDefined(applicationUniversalIdentifier)+ ? { workspaceId, applicationUniversalIdentifier }+ : { workspaceId, isWorkspaceJob: true };+};diff --git a/packages/twenty-server/src/engine/core-modules/auth/services/google-apis.service.ts b/packages/twenty-server/src/engine/core-modules/auth/services/google-apis.service.tsindex 5f6638c29ea..15ab49b11f9 100644--- a/packages/twenty-server/src/engine/core-modules/auth/services/google-apis.service.ts+++ b/packages/twenty-server/src/engine/core-modules/auth/services/google-apis.service.ts@@ -350,6 +350,10 @@ export class GoogleAPIsService {await this.messageQueueService.add<MessagingMessageListFetchJobData>(MessagingMessageListFetchJob.name,{ workspaceId, messageChannelId: messageChannel.id },+ {+ workspaceId,+ isWorkspaceJob: true,+ },);await this.enqueueWebhookSubscriptionCreation({channelType: WebhookSubscriptionChannelType.MESSAGING,@@ -396,6 +400,10 @@ export class GoogleAPIsService {await this.calendarQueueService.add<CalendarEventListFetchJobData>(CalendarEventListFetchJob.name,{ workspaceId, calendarChannelId: calendarChannel.id },+ {+ workspaceId,+ isWorkspaceJob: true,+ },);await this.enqueueWebhookSubscriptionCreation({channelType: WebhookSubscriptionChannelType.CALENDAR,@@ -436,7 +444,11 @@ export class GoogleAPIsService {await this.webhookQueueService.add<CreateWebhookSubscriptionJobData>(CreateWebhookSubscriptionJob.name,{ channelType, channelId, workspaceId },- { retryLimit: WEBHOOK_SUBSCRIPTION_JOB_RETRY_LIMIT },+ {+ retryLimit: WEBHOOK_SUBSCRIPTION_JOB_RETRY_LIMIT,+ workspaceId,+ isWorkspaceJob: true,+ },);} catch (error) {this.logger.warn(diff --git a/packages/twenty-server/src/engine/core-modules/auth/services/microsoft-apis.service.ts b/packages/twenty-server/src/engine/core-modules/auth/services/microsoft-apis.service.tsindex 9ad91e606c7..a726bc37731 100644--- a/packages/twenty-server/src/engine/core-modules/auth/services/microsoft-apis.service.ts+++ b/packages/twenty-server/src/engine/core-modules/auth/services/microsoft-apis.service.ts@@ -310,6 +310,10 @@ export class MicrosoftAPIsService {workspaceId,messageChannelId: messageChannel.id,},+ {+ workspaceId,+ isWorkspaceJob: true,+ },);await this.enqueueWebhookSubscriptionCreation({channelType: WebhookSubscriptionChannelType.MESSAGING,@@ -348,6 +352,10 @@ export class MicrosoftAPIsService {calendarChannelId: calendarChannel.id,workspaceId,},+ {+ workspaceId,+ isWorkspaceJob: true,+ },);await this.enqueueWebhookSubscriptionCreation({channelType: WebhookSubscriptionChannelType.CALENDAR,@@ -386,7 +394,11 @@ export class MicrosoftAPIsService {await this.webhookQueueService.add<CreateWebhookSubscriptionJobData>(CreateWebhookSubscriptionJob.name,{ channelType, channelId, workspaceId },- { retryLimit: WEBHOOK_SUBSCRIPTION_JOB_RETRY_LIMIT },+ {+ retryLimit: WEBHOOK_SUBSCRIPTION_JOB_RETRY_LIMIT,+ workspaceId,+ isWorkspaceJob: true,+ },);} catch (error) {this.logger.warn(diff --git a/packages/twenty-server/src/engine/core-modules/billing-webhook/services/billing-webhook-subscription.service.ts b/packages/twenty-server/src/engine/core-modules/billing-webhook/services/billing-webhook-subscription.service.tsindex f8ba1d0abd5..884ad32e779 100644--- a/packages/twenty-server/src/engine/core-modules/billing-webhook/services/billing-webhook-subscription.service.ts+++ b/packages/twenty-server/src/engine/core-modules/billing-webhook/services/billing-webhook-subscription.service.ts@@ -263,6 +263,10 @@ export class BillingWebhookSubscriptionService {await this.messageQueueService.add<CleanWorkspaceDeletionWarningUserVarsJobData>(CleanWorkspaceDeletionWarningUserVarsJob.name,{ workspaceId },+ {+ workspaceId,+ isWorkspaceJob: true,+ },);}}diff --git a/packages/twenty-server/src/engine/core-modules/billing/listeners/billing-workspace-member.listener.ts b/packages/twenty-server/src/engine/core-modules/billing/listeners/billing-workspace-member.listener.tsindex efd74b66a75..e0642fef217 100644--- a/packages/twenty-server/src/engine/core-modules/billing/listeners/billing-workspace-member.listener.ts+++ b/packages/twenty-server/src/engine/core-modules/billing/listeners/billing-workspace-member.listener.ts@@ -45,6 +45,8 @@ export class BillingWorkspaceMemberListener {{ workspaceId: payload.workspaceId },{delay: UPDATE_SUBSCRIPTION_QUANTITY_JOB_DELAY_MS,+ workspaceId: payload.workspaceId,+ isWorkspaceJob: true,},);}diff --git a/packages/twenty-server/src/engine/core-modules/email/email.service.ts b/packages/twenty-server/src/engine/core-modules/email/email.service.tsindex 8a5bfdf54b8..69d617206fc 100644--- a/packages/twenty-server/src/engine/core-modules/email/email.service.ts+++ b/packages/twenty-server/src/engine/core-modules/email/email.service.ts@@ -19,7 +19,7 @@ export class EmailService {await this.messageQueueService.add<SendMailOptions>(EmailSenderJob.name,sendMailOptions,- { retryLimit: 3, backoff: EMAIL_SEND_RETRY_BACKOFF },+ { retryLimit: 3, backoff: EMAIL_SEND_RETRY_BACKOFF, isSystemJob: true },);}}diff --git a/packages/twenty-server/src/engine/core-modules/event-logs/ingest/create-event-log-from-internal-event.spec.ts b/packages/twenty-server/src/engine/core-modules/event-logs/ingest/create-event-log-from-internal-event.spec.tsindex 6e2dfc0884f..c357635a903 100644--- a/packages/twenty-server/src/engine/core-modules/event-logs/ingest/create-event-log-from-internal-event.spec.ts+++ b/packages/twenty-server/src/engine/core-modules/event-logs/ingest/create-event-log-from-internal-event.spec.ts@@ -83,6 +83,10 @@ describe('CreateEventLogFromInternalEvent', () => {expect(addToQueue).toHaveBeenCalledWith(CreateEventLogFromInternalEvent.name,expect.objectContaining({ transientErrorRetryCount: 1 }),+ {+ workspaceId: 'workspace-1',+ isWorkspaceJob: true,+ },);});diff --git a/packages/twenty-server/src/engine/core-modules/event-logs/ingest/create-event-log-from-internal-event.ts b/packages/twenty-server/src/engine/core-modules/event-logs/ingest/create-event-log-from-internal-event.tsindex f0f5f627ed0..74eb00b2912 100644--- a/packages/twenty-server/src/engine/core-modules/event-logs/ingest/create-event-log-from-internal-event.ts+++ b/packages/twenty-server/src/engine/core-modules/event-logs/ingest/create-event-log-from-internal-event.ts@@ -95,6 +95,10 @@ export class CreateEventLogFromInternalEvent {await this.messageQueueService.add<CreateEventLogFromInternalEventData>(CreateEventLogFromInternalEvent.name,{ ...batch, transientErrorRetryCount: transientErrorRetryCount + 1 },+ {+ workspaceId: batch.workspaceId,+ isWorkspaceJob: true,+ },);}diff --git a/packages/twenty-server/src/engine/core-modules/event-logs/ingest/forward-event-log-from-entity-events-to-db-queue.ts b/packages/twenty-server/src/engine/core-modules/event-logs/ingest/forward-event-log-from-entity-events-to-db-queue.tsindex 238b975e3c2..b6ad1e8483f 100644--- a/packages/twenty-server/src/engine/core-modules/event-logs/ingest/forward-event-log-from-entity-events-to-db-queue.ts+++ b/packages/twenty-server/src/engine/core-modules/event-logs/ingest/forward-event-log-from-entity-events-to-db-queue.ts@@ -20,7 +20,11 @@ export class ForwardEventLogFromEntityEventsToDbQueue {await this.eventLogQueueService.add<CreateEventLogFromInternalEventData>(CreateEventLogFromInternalEvent.name,batch,- { retryLimit: 1 },+ {+ retryLimit: 1,+ workspaceId: batch.workspaceId,+ isWorkspaceJob: true,+ },);}}diff --git a/packages/twenty-server/src/engine/core-modules/file/files-field/listeners/files-field-deletion.listener.ts b/packages/twenty-server/src/engine/core-modules/file/files-field/listeners/files-field-deletion.listener.tsindex 58dbae12ae0..bb44324198c 100644--- a/packages/twenty-server/src/engine/core-modules/file/files-field/listeners/files-field-deletion.listener.ts+++ b/packages/twenty-server/src/engine/core-modules/file/files-field/listeners/files-field-deletion.listener.ts@@ -111,6 +111,10 @@ export class FilesFieldDeletionListener {workspaceId,fileIds: Array.from(fileIds),},+ {+ workspaceId,+ isWorkspaceJob: true,+ },);}}diff --git a/packages/twenty-server/src/engine/core-modules/logic-function/logic-function-prebuilt-warm-up/jobs/warm-up-logic-function.job-constants.ts b/packages/twenty-server/src/engine/core-modules/logic-function/logic-function-prebuilt-warm-up/jobs/warm-up-logic-function.job-constants.tsindex 9d97db1761e..b01e9fed29b 100644--- a/packages/twenty-server/src/engine/core-modules/logic-function/logic-function-prebuilt-warm-up/jobs/warm-up-logic-function.job-constants.ts+++ b/packages/twenty-server/src/engine/core-modules/logic-function/logic-function-prebuilt-warm-up/jobs/warm-up-logic-function.job-constants.ts@@ -1,8 +1,8 @@-import { type QueueJobOptions } from 'src/engine/core-modules/message-queue/drivers/interfaces/job-options.interface';+import { type QueueJobSettings } from 'src/engine/core-modules/message-queue/drivers/interfaces/job-options.interface';export const WARM_UP_LOGIC_FUNCTION_JOB_NAME = 'WarmUpLogicFunctionJob';-export const WARM_UP_LOGIC_FUNCTION_JOB_OPTIONS: QueueJobOptions = {+export const WARM_UP_LOGIC_FUNCTION_JOB_OPTIONS: QueueJobSettings = {retryLimit: 5,backoff: { strategy: 'exponential', initialDelayMilliseconds: 30_000 },};diff --git a/packages/twenty-server/src/engine/core-modules/logic-function/logic-function-trigger/triggers/cron/cron-trigger.cron.job.ts b/packages/twenty-server/src/engine/core-modules/logic-function/logic-function-trigger/triggers/cron/cron-trigger.cron.job.tsindex 03a7951ba62..e83b3c1186f 100644--- a/packages/twenty-server/src/engine/core-modules/logic-function/logic-function-trigger/triggers/cron/cron-trigger.cron.job.ts+++ b/packages/twenty-server/src/engine/core-modules/logic-function/logic-function-trigger/triggers/cron/cron-trigger.cron.job.ts@@ -19,8 +19,6 @@ import { MessageQueueService } from 'src/engine/core-modules/message-queue/serviimport { ThrottlerException } from 'src/engine/core-modules/throttler/throttler.exception';import { WorkspaceEntity } from 'src/engine/core-modules/workspace/workspace.entity';import { LOGIC_FUNCTION_QUEUE_RETRY_BACKOFF } from 'src/engine/core-modules/logic-function/logic-function-trigger/constants/logic-function-queue-retry-backoff.constant';-import { buildApplicationGroupKey } from 'src/engine/core-modules/message-queue/utils/build-application-group-key.util';-import { buildWorkspaceGroupKey } from 'src/engine/core-modules/message-queue/utils/build-workspace-group-key.util';import { buildApplicationJobThrottledLogEntry } from 'src/engine/core-modules/logic-function/logic-function-trigger/utils/build-application-job-throttled-log-entry.util';import {LogicFunctionTriggerJob,@@ -146,10 +144,8 @@ export class CronTriggerCronJob {{retryLimit: 10,backoff: LOGIC_FUNCTION_QUEUE_RETRY_BACKOFF,- groupKeys: [- buildWorkspaceGroupKey(activeWorkspace.id),- buildApplicationGroupKey(application.universalIdentifier),- ],+ workspaceId: activeWorkspace.id,+ applicationUniversalIdentifier: application.universalIdentifier,},);}diff --git a/packages/twenty-server/src/engine/core-modules/logic-function/logic-function-trigger/triggers/database-event/__tests__/call-database-event-trigger-jobs.job.spec.ts b/packages/twenty-server/src/engine/core-modules/logic-function/logic-function-trigger/triggers/database-event/__tests__/call-database-event-trigger-jobs.job.spec.tsindex fbb19c952f7..bcadd75c972 100644--- a/packages/twenty-server/src/engine/core-modules/logic-function/logic-function-trigger/triggers/database-event/__tests__/call-database-event-trigger-jobs.job.spec.ts+++ b/packages/twenty-server/src/engine/core-modules/logic-function/logic-function-trigger/triggers/database-event/__tests__/call-database-event-trigger-jobs.job.spec.ts@@ -251,6 +251,8 @@ describe('CallDatabaseEventTriggerJobsJob', () => {payload: expect.objectContaining(event),}),jobId: `${TRIGGER_JOB_ID}.${LOGIC_FUNCTION_ID}.0`,+ workspaceId: WORKSPACE_ID,+ applicationUniversalIdentifier: 'application-universal-identifier',},],expect.anything(),diff --git a/packages/twenty-server/src/engine/core-modules/logic-function/logic-function-trigger/triggers/database-event/call-database-event-trigger-jobs.job.ts b/packages/twenty-server/src/engine/core-modules/logic-function/logic-function-trigger/triggers/database-event/call-database-event-trigger-jobs.job.tsindex 14a30ab159e..77e475aac99 100644--- a/packages/twenty-server/src/engine/core-modules/logic-function/logic-function-trigger/triggers/database-event/call-database-event-trigger-jobs.job.ts+++ b/packages/twenty-server/src/engine/core-modules/logic-function/logic-function-trigger/triggers/database-event/call-database-event-trigger-jobs.job.ts@@ -15,8 +15,6 @@ import { ApplicationJobEnqueueThrottlerService } from 'src/engine/core-modules/mimport { MessageQueueService } from 'src/engine/core-modules/message-queue/services/message-queue.service';import { ThrottlerException } from 'src/engine/core-modules/throttler/throttler.exception';import { LOGIC_FUNCTION_QUEUE_RETRY_BACKOFF } from 'src/engine/core-modules/logic-function/logic-function-trigger/constants/logic-function-queue-retry-backoff.constant';-import { buildApplicationGroupKey } from 'src/engine/core-modules/message-queue/utils/build-application-group-key.util';-import { buildWorkspaceGroupKey } from 'src/engine/core-modules/message-queue/utils/build-workspace-group-key.util';import { buildApplicationJobThrottledLogEntry } from 'src/engine/core-modules/logic-function/logic-function-trigger/utils/build-application-job-throttled-log-entry.util';import { buildLogicFunctionTriggerJobIds } from 'src/engine/core-modules/logic-function/logic-function-trigger/triggers/database-event/utils/build-logic-function-trigger-job-ids';import { findLogicFunctionsTriggeredByEventName } from 'src/engine/core-modules/logic-function/logic-function-trigger/triggers/database-event/utils/find-logic-functions-triggered-by-event-name';@@ -178,15 +176,13 @@ export class CallDatabaseEventTriggerJobsJob {applicationUniversalIdentifier: application.universalIdentifier,},jobId: logicFunctionTriggerJobIds[index],+ workspaceId: workspaceEventBatch.workspaceId,+ applicationUniversalIdentifier: application.universalIdentifier,})),{retryLimit: 3,backoff: LOGIC_FUNCTION_QUEUE_RETRY_BACKOFF,doNotCheckDuplicatesBasedOnPrefix: true,- groupKeys: [- buildWorkspaceGroupKey(workspaceEventBatch.workspaceId),- buildApplicationGroupKey(application.universalIdentifier),- ],},);}diff --git a/packages/twenty-server/src/engine/core-modules/message-queue/__tests__/message-queue.explorer.spec.ts b/packages/twenty-server/src/engine/core-modules/message-queue/__tests__/message-queue.explorer.spec.tsindex 5a791e0921a..2c76424fbc8 100644--- a/packages/twenty-server/src/engine/core-modules/message-queue/__tests__/message-queue.explorer.spec.ts+++ b/packages/twenty-server/src/engine/core-modules/message-queue/__tests__/message-queue.explorer.spec.ts@@ -56,7 +56,7 @@ describe('MessageQueueExplorer progress context', () => {try {await module.init();const [id] = await queue.bulkAdd('progress-job', [- { data: { total: 10 } },+ { data: { total: 10 }, isSystemJob: true },]);expect((await queue.getJobs([id]))[id]).toMatchObject({diff --git a/packages/twenty-server/src/engine/core-modules/message-queue/drivers/__tests__/bullmq.driver.fairness.spec.ts b/packages/twenty-server/src/engine/core-modules/message-queue/drivers/__tests__/bullmq.driver.fairness.spec.tsindex 21a7bee3b85..6478628d14a 100644--- a/packages/twenty-server/src/engine/core-modules/message-queue/drivers/__tests__/bullmq.driver.fairness.spec.ts+++ b/packages/twenty-server/src/engine/core-modules/message-queue/drivers/__tests__/bullmq.driver.fairness.spec.ts@@ -14,7 +14,7 @@ import { SERVER_ROUTE_DISPATCH_JOB_PRIORITY } from 'src/engine/core-modules/servdescribe('BullMQDriver fairness scenarios', () => {beforeAll(silenceDriverLogs);- describe('S1: A enqueues a burst of 10 jobs, then B enqueues 1 job (group keys deduced from workspaceId)', () => {+ describe('S1: A enqueues a burst of 10 jobs, then B enqueues 1 job (core jobs, keyed by workspace)', () => {const enqueue = async (harness: FairnessHarness) => {await harness.enqueueMany({labels: buildLabels({ prefix: 'A', count: 10 }),@@ -140,58 +140,37 @@ describe('BullMQDriver fairness scenarios', () => {await harness.enqueue({label: 'W1a',workspaceId: 'W1',- groupKeys: [- buildWorkspaceGroupKey('W1'),- buildApplicationGroupKey('H'),- ],+ applicationUniversalIdentifier: 'H',});await harness.enqueue({label: 'W1b',workspaceId: 'W1',- groupKeys: [- buildWorkspaceGroupKey('W1'),- buildApplicationGroupKey('H'),- ],+ applicationUniversalIdentifier: 'H',});await harness.enqueue({label: 'W2a',workspaceId: 'W2',- groupKeys: [- buildWorkspaceGroupKey('W2'),- buildApplicationGroupKey('H'),- ],+ applicationUniversalIdentifier: 'H',});await harness.enqueue({label: 'W2b',workspaceId: 'W2',- groupKeys: [- buildWorkspaceGroupKey('W2'),- buildApplicationGroupKey('H'),- ],+ applicationUniversalIdentifier: 'H',});await harness.enqueue({label: 'W3a',workspaceId: 'W3',- groupKeys: [- buildWorkspaceGroupKey('W3'),- buildApplicationGroupKey('H'),- ],+ applicationUniversalIdentifier: 'H',});await harness.enqueue({label: 'W3b',workspaceId: 'W3',- groupKeys: [- buildWorkspaceGroupKey('W3'),- buildApplicationGroupKey('H'),- ],+ applicationUniversalIdentifier: 'H',});await harness.enqueue({label: 'W4',workspaceId: 'W4',- groupKeys: [- buildWorkspaceGroupKey('W4'),- buildApplicationGroupKey('X'),- ],+ applicationUniversalIdentifier: 'X',});};@@ -365,18 +344,12 @@ describe('BullMQDriver fairness scenarios', () => {await harness.enqueueMany({labels: buildLabels({ prefix: 'P', count: 4 }),workspaceId: 'WP',- groupKeys: [- buildWorkspaceGroupKey('WP'),- buildApplicationGroupKey('P'),- ],+ applicationUniversalIdentifier: 'P',});await harness.enqueueMany({labels: buildLabels({ prefix: 'Q', count: 4 }),workspaceId: 'WQ',- groupKeys: [- buildWorkspaceGroupKey('WQ'),- buildApplicationGroupKey('Q'),- ],+ applicationUniversalIdentifier: 'Q',});};@@ -410,10 +383,6 @@ describe('BullMQDriver fairness scenarios', () => {});describe('S7b: application P weighs 2; WP bulk-enqueues 4 jobs of P, then WP enqueues 1 more job of P and WQ enqueues 1 job of Q', () => {- const groupKeysOfP = [- buildWorkspaceGroupKey('WP'),- buildApplicationGroupKey('P'),- ];const enqueue = async (harness: FairnessHarness) => {await harness.setWeight({groupKey: buildApplicationGroupKey('P'),@@ -422,20 +391,17 @@ describe('BullMQDriver fairness scenarios', () => {await harness.enqueueBulk({labels: buildLabels({ prefix: 'P', count: 4 }),workspaceId: 'WP',- groupKeys: groupKeysOfP,+ applicationUniversalIdentifier: 'P',});await harness.enqueue({label: 'P5',workspaceId: 'WP',- groupKeys: groupKeysOfP,+ applicationUniversalIdentifier: 'P',});await harness.enqueue({label: 'Q1',workspaceId: 'WQ',- groupKeys: [- buildWorkspaceGroupKey('WQ'),- buildApplicationGroupKey('Q'),- ],+ applicationUniversalIdentifier: 'Q',});};@@ -475,21 +441,18 @@ describe('BullMQDriver fairness scenarios', () => {await harness.enqueue({label,workspaceId,- groupKeys: [- buildWorkspaceGroupKey(workspaceId),- buildApplicationGroupKey('H'),- ],+ applicationUniversalIdentifier: 'H',});}await harness.enqueue({label: 'W_H',workspaceId: 'W',- groupKeys: [buildWorkspaceGroupKey('W'), buildApplicationGroupKey('H')],+ applicationUniversalIdentifier: 'H',});await harness.enqueue({label: 'W_Q',workspaceId: 'W',- groupKeys: [buildWorkspaceGroupKey('W'), buildApplicationGroupKey('Q')],+ applicationUniversalIdentifier: 'Q',});};@@ -517,10 +480,6 @@ describe('BullMQDriver fairness scenarios', () => {});describe('S7d: application P weighs 2; X enqueues 6 jobs of P, W bulk-enqueues 4 jobs of P then 1 job of application Q, X enqueues 1 more job of P', () => {- const groupKeysOfP = (workspaceId: string) => [- buildWorkspaceGroupKey(workspaceId),- buildApplicationGroupKey('P'),- ];const enqueue = async (harness: FairnessHarness) => {await harness.setWeight({groupKey: buildApplicationGroupKey('P'),@@ -529,22 +488,22 @@ describe('BullMQDriver fairness scenarios', () => {await harness.enqueueMany({labels: buildLabels({ prefix: 'P', count: 6 }),workspaceId: 'X',- groupKeys: groupKeysOfP('X'),+ applicationUniversalIdentifier: 'P',});await harness.enqueueBulk({labels: buildLabels({ prefix: 'B', count: 4 }),workspaceId: 'W',- groupKeys: groupKeysOfP('W'),+ applicationUniversalIdentifier: 'P',});await harness.enqueue({label: 'Q1',workspaceId: 'W',- groupKeys: [buildWorkspaceGroupKey('W'), buildApplicationGroupKey('Q')],+ applicationUniversalIdentifier: 'Q',});await harness.enqueue({label: 'P7',workspaceId: 'X',- groupKeys: groupKeysOfP('X'),+ applicationUniversalIdentifier: 'P',});};diff --git a/packages/twenty-server/src/engine/core-modules/message-queue/drivers/__tests__/bullmq.driver.spec.ts b/packages/twenty-server/src/engine/core-modules/message-queue/drivers/__tests__/bullmq.driver.spec.tsindex d6f4065355b..84b1e9f14c5 100644--- a/packages/twenty-server/src/engine/core-modules/message-queue/drivers/__tests__/bullmq.driver.spec.ts+++ b/packages/twenty-server/src/engine/core-modules/message-queue/drivers/__tests__/bullmq.driver.spec.ts@@ -1,5 +1,7 @@import { type Job, Worker } from 'bullmq';+import { TWENTY_STANDARD_APPLICATION_UNIVERSAL_IDENTIFIER } from 'twenty-shared/application';+import { type WorkspaceQueueJobScope } from 'src/engine/core-modules/message-queue/drivers/interfaces/job-options.interface';import { DEFAULT_JOB_PRIORITY } from 'src/engine/core-modules/message-queue/constants/default-job-priority.constant';import { QUEUE_JOB_CHANGED_EVENT } from 'src/engine/core-modules/message-queue/constants/queue-job-changed-event.constant';import { BullMQDriver } from 'src/engine/core-modules/message-queue/drivers/bullmq.driver';@@ -54,6 +56,15 @@ jest.mock(const WAITING_JOB_ID = 'sync-catalog-ws-1-5c98b035-5b09-4550-a4fb-b52056c494d1';+const SYSTEM_JOB = { isSystemJob: true } as const;++const buildWorkspaceJobScope = (+ workspaceId: string,+): WorkspaceQueueJobScope => ({+ workspaceId,+ isWorkspaceJob: true,+});+describe('BullMQDriver deduplication', () => {const driver = new BullMQDriver({} as never,@@ -79,7 +90,12 @@ describe('BullMQDriver deduplication', () => {describe.each([undefined, 0, 2])('job priority %s', (priority) => {it('uses the caller priority or the shared default to add', async () => {- await driver.add(MessageQueue.workspaceQueue, 'job', {}, { priority });+ await driver.add(+ MessageQueue.workspaceQueue,+ 'job',+ {},+ { ...SYSTEM_JOB, priority },+ );expect(mockAdd).toHaveBeenCalledWith('job',@@ -89,9 +105,12 @@ describe('BullMQDriver deduplication', () => {});it('uses the caller priority or the shared default to bulkAdd', async () => {- await driver.bulkAdd(MessageQueue.workspaceQueue, 'job', [{ data: {} }], {- priority,- });+ await driver.bulkAdd(+ MessageQueue.workspaceQueue,+ 'job',+ [{ data: {}, ...SYSTEM_JOB }],+ { priority },+ );expect(mockAddBulk).toHaveBeenCalledWith([expect.objectContaining({@@ -109,7 +128,7 @@ describe('BullMQDriver deduplication', () => {MessageQueue.workspaceQueue,'job',{},- { id: 'sync-catalog-ws-1' },+ { ...SYSTEM_JOB, id: 'sync-catalog-ws-1' },);expect(jobId).toBeUndefined();@@ -121,7 +140,7 @@ describe('BullMQDriver deduplication', () => {MessageQueue.workspaceQueue,'job',{},- { id: 'install-application-ws-1-app-1' },+ { ...SYSTEM_JOB, id: 'install-application-ws-1-app-1' },);expect(jobId).toBe(@@ -133,8 +152,8 @@ describe('BullMQDriver deduplication', () => {describe('bulkAdd', () => {it('does not read the waiting jobs when no job carries an id', async () => {await driver.bulkAdd(MessageQueue.workspaceQueue, 'job', [- { data: {} },- { data: {} },+ { data: {}, ...SYSTEM_JOB },+ { data: {}, ...SYSTEM_JOB },]);expect(mockGetRanges).not.toHaveBeenCalled();@@ -146,10 +165,10 @@ describe('BullMQDriver deduplication', () => {mockGetRanges.mockResolvedValue([WAITING_JOB_ID, 'ws-1.exact-job-id']);const jobIds = await driver.bulkAdd(MessageQueue.workspaceQueue, 'job', [- { data: {}, jobId: 'ws-1.exact-job-id' },- { data: {}, jobId: 'sync-catalog-ws-1' },- { data: {}, jobId: 'ws-1.new-job-id' },- { data: {} },+ { data: {}, jobId: 'ws-1.exact-job-id', ...SYSTEM_JOB },+ { data: {}, jobId: 'sync-catalog-ws-1', ...SYSTEM_JOB },+ { data: {}, jobId: 'ws-1.new-job-id', ...SYSTEM_JOB },+ { data: {}, ...SYSTEM_JOB },]);expect(mockGetRanges).toHaveBeenCalledTimes(1);@@ -171,7 +190,7 @@ describe('BullMQDriver deduplication', () => {mockGetRanges.mockResolvedValue(['ws-1.exact-job-id']);const jobIds = await driver.bulkAdd(MessageQueue.workspaceQueue, 'job', [- { data: {}, jobId: 'ws-1.exact-job-id' },+ { data: {}, jobId: 'ws-1.exact-job-id', ...SYSTEM_JOB },]);expect(jobIds).toEqual([]);@@ -182,7 +201,7 @@ describe('BullMQDriver deduplication', () => {await driver.bulkAdd(MessageQueue.workspaceQueue,'job',- [{ data: {}, jobId: 'ws-1.exact-job-id' }],+ [{ data: {}, jobId: 'ws-1.exact-job-id', ...SYSTEM_JOB }],{ doNotCheckDuplicatesBasedOnPrefix: true },);@@ -548,9 +567,12 @@ describe('BullMQDriver fair tickets', () => {tickets: [{ tag: 7, weight: 1 }],});- await driver.add(MessageQueue.workspaceQueue, 'job', {- workspaceId: 'ws-1',- });+ await driver.add(+ MessageQueue.workspaceQueue,+ 'job',+ {},+ buildWorkspaceJobScope('ws-1'),+ );expect(mockAllocateTickets).toHaveBeenCalledWith({queueName: MessageQueue.workspaceQueue,@@ -558,7 +580,7 @@ describe('BullMQDriver fair tickets', () => {});expect(mockAdd).toHaveBeenCalledWith('job',- { workspaceId: 'ws-1' },+ {},expect.objectContaining({priority: FAIR_TICKET_OFFSET + 7,fairBase: FAIR_REBASE_STEP,@@ -566,6 +588,93 @@ describe('BullMQDriver fair tickets', () => {);});+ it('adds the application group key for a job of an installed application', async () => {+ mockAllocateTickets.mockResolvedValue({+ base: 0,+ tickets: [{ tag: 0, weight: 1 }],+ });++ await driver.add(+ MessageQueue.workspaceQueue,+ 'job',+ {},+ { workspaceId: 'ws-1', applicationUniversalIdentifier: 'app-1' },+ );++ expect(mockAllocateTickets).toHaveBeenCalledWith({+ queueName: MessageQueue.workspaceQueue,+ requests: [+ { groupKeys: ['workspace:ws-1', 'application:app-1'], jobCount: 1 },+ ],+ });+ });++ it('adds the application group key for a job of the standard application', async () => {+ mockAllocateTickets.mockResolvedValue({+ base: 0,+ tickets: [{ tag: 0, weight: 1 }],+ });++ await driver.add(+ MessageQueue.workspaceQueue,+ 'job',+ {},+ {+ workspaceId: 'ws-1',+ applicationUniversalIdentifier:+ TWENTY_STANDARD_APPLICATION_UNIVERSAL_IDENTIFIER,+ },+ );++ expect(mockAllocateTickets).toHaveBeenCalledWith({+ queueName: MessageQueue.workspaceQueue,+ requests: [+ {+ groupKeys: [+ 'workspace:ws-1',+ `application:${TWENTY_STANDARD_APPLICATION_UNIVERSAL_IDENTIFIER}`,+ ],+ jobCount: 1,+ },+ ],+ });+ });++ it('ranks a system job on the system group', async () => {+ mockAllocateTickets.mockResolvedValue({+ base: 0,+ tickets: [{ tag: 0, weight: 1 }],+ });++ await driver.add(MessageQueue.workspaceQueue, 'job', {}, SYSTEM_JOB);++ expect(mockAllocateTickets).toHaveBeenCalledWith({+ queueName: MessageQueue.workspaceQueue,+ requests: [{ groupKeys: ['system'], jobCount: 1 }],+ });+ });++ it('appends the additional group keys to the keys of the scope', async () => {+ mockAllocateTickets.mockResolvedValue({+ base: 0,+ tickets: [{ tag: 0, weight: 1 }],+ });++ await driver.add(+ MessageQueue.workspaceQueue,+ 'job',+ {},+ { ...buildWorkspaceJobScope('ws-1'), groupKeys: ['integration:slack'] },+ );++ expect(mockAllocateTickets).toHaveBeenCalledWith({+ queueName: MessageQueue.workspaceQueue,+ requests: [+ { groupKeys: ['workspace:ws-1', 'integration:slack'], jobCount: 1 },+ ],+ });+ });+it('allocates one ticket per group of a bulk and spaces its jobs by the weight', async () => {mockAllocateTickets.mockResolvedValue({base: 0,@@ -576,9 +685,9 @@ describe('BullMQDriver fair tickets', () => {});await driver.bulkAdd(MessageQueue.workspaceQueue, 'job', [- { data: { workspaceId: 'ws-1' } },- { data: { workspaceId: 'ws-2' } },- { data: { workspaceId: 'ws-1' } },+ { data: {}, ...buildWorkspaceJobScope('ws-1') },+ { data: {}, ...buildWorkspaceJobScope('ws-2') },+ { data: {}, ...buildWorkspaceJobScope('ws-1') },]);expect(mockAllocateTickets).toHaveBeenCalledWith({@@ -609,7 +718,7 @@ describe('BullMQDriver fair tickets', () => {MessageQueue.workspaceQueue,'job',{ workspaceId: 'ws-1' },- { priority: URGENT_FAIR_PRIORITY },+ { ...buildWorkspaceJobScope('ws-1'), priority: URGENT_FAIR_PRIORITY },);expect(mockAllocateTickets).toHaveBeenCalledTimes(1);@@ -626,13 +735,13 @@ describe('BullMQDriver fair tickets', () => {await driver.add(MessageQueue.workspaceQueue,'job',- { workspaceId: 'ws-1' },- { priority: 4 },+ {},+ { ...buildWorkspaceJobScope('ws-1'), priority: 4 },);expect(mockAdd).toHaveBeenCalledWith('job',- { workspaceId: 'ws-1' },+ {},expect.objectContaining({ priority: 4 }),);expect(incrementCounterForEvent).toHaveBeenCalledWith(@@ -644,14 +753,14 @@ describe('BullMQDriver fair tickets', () => {await driver.add(MessageQueue.workspaceQueue,'job',- { workspaceId: 'ws-1' },- { delay: 1_000, priority: 4 },+ {},+ { ...buildWorkspaceJobScope('ws-1'), delay: 1_000, priority: 4 },);expect(mockAllocateTickets).not.toHaveBeenCalled();expect(mockAdd).toHaveBeenCalledWith('job',- { workspaceId: 'ws-1' },+ {},expect.objectContaining({ delay: 1_000, priority: 4 }),);});diff --git a/packages/twenty-server/src/engine/core-modules/message-queue/drivers/__tests__/fairness-harness.test-util.ts b/packages/twenty-server/src/engine/core-modules/message-queue/drivers/__tests__/fairness-harness.test-util.tsindex e3922a7a546..ee5ade00ced 100644--- a/packages/twenty-server/src/engine/core-modules/message-queue/drivers/__tests__/fairness-harness.test-util.ts+++ b/packages/twenty-server/src/engine/core-modules/message-queue/drivers/__tests__/fairness-harness.test-util.ts@@ -7,7 +7,10 @@ import { isDefined } from 'twenty-shared/utils';import { BullMQDriver } from 'src/engine/core-modules/message-queue/drivers/bullmq.driver';import { FAIR_TICKET_OFFSET } from 'src/engine/core-modules/message-queue/drivers/fairness/fair-priority.constant';-import { type QueueJobOptions } from 'src/engine/core-modules/message-queue/drivers/interfaces/job-options.interface';+import {+ type QueueJobScope,+ type QueueJobSettings,+} from 'src/engine/core-modules/message-queue/drivers/interfaces/job-options.interface';import { MessageQueue } from 'src/engine/core-modules/message-queue/message-queue.constants';export type ScenarioJobData = { label: string; workspaceId?: string };@@ -305,10 +308,31 @@ const createRedisClientDouble = (redis: Redis): RedisClientDouble => ({},});-type EnqueueInput = { label: string; workspaceId?: string } & Pick<- QueueJobOptions,- 'groupKeys' | 'priority' | 'delay' | 'id' | 'retryLimit' | 'backoff'->;+type ScenarioJobScope = {+ workspaceId?: string;+ applicationUniversalIdentifier?: string;+};++type EnqueueInput = { label: string } & ScenarioJobScope &+ Pick<+ QueueJobSettings,+ 'priority' | 'delay' | 'id' | 'retryLimit' | 'backoff'+ >;++const buildScenarioJobScope = ({+ workspaceId,+ applicationUniversalIdentifier,+}: ScenarioJobScope): QueueJobScope => {+ if (!isDefined(workspaceId)) {+ return { isSystemJob: true };+ }++ if (!isDefined(applicationUniversalIdentifier)) {+ return { workspaceId, isWorkspaceJob: true };+ }++ return { workspaceId, applicationUniversalIdentifier };+};export const createFairnessHarness = async ({isFairnessEnabled: initialFairnessEnabled,@@ -394,12 +418,23 @@ export const createFairnessHarness = async ({return true;};- const enqueue = async ({ label, workspaceId, ...options }: EnqueueInput) => {+ const enqueue = async ({+ label,+ workspaceId,+ applicationUniversalIdentifier,+ ...options+ }: EnqueueInput) => {await driver.add<ScenarioJobData>(QUEUE,JOB_NAME,{ label, workspaceId },- options,+ {+ ...options,+ ...buildScenarioJobScope({+ workspaceId,+ applicationUniversalIdentifier,+ }),+ },);};@@ -417,17 +452,22 @@ export const createFairnessHarness = async ({enqueueBulk: async ({labels,workspaceId,- groupKeys,+ applicationUniversalIdentifier,}: {labels: string[];workspaceId: string;- groupKeys?: string[];+ applicationUniversalIdentifier?: string;}) => {await driver.bulkAdd<ScenarioJobData>(QUEUE,JOB_NAME,- labels.map((label) => ({ data: { label, workspaceId } })),- isDefined(groupKeys) ? { groupKeys } : undefined,+ labels.map((label) => ({+ data: { label, workspaceId },+ ...buildScenarioJobScope({+ workspaceId,+ applicationUniversalIdentifier,+ }),+ })),);},scheduleCron: async ({diff --git a/packages/twenty-server/src/engine/core-modules/message-queue/drivers/__tests__/sync.driver.spec.ts b/packages/twenty-server/src/engine/core-modules/message-queue/drivers/__tests__/sync.driver.spec.tsindex 6eb227c4cbd..2997084511b 100644--- a/packages/twenty-server/src/engine/core-modules/message-queue/drivers/__tests__/sync.driver.spec.ts+++ b/packages/twenty-server/src/engine/core-modules/message-queue/drivers/__tests__/sync.driver.spec.ts@@ -3,6 +3,7 @@ import { SyncDriver } from 'src/engine/core-modules/message-queue/drivers/sync.dimport { MessageQueue } from 'src/engine/core-modules/message-queue/message-queue.constants';const QUEUE = MessageQueue.workspaceQueue;+const SYSTEM_JOB = { isSystemJob: true } as const;describe('SyncDriver job snapshots', () => {it.each([50, { completed: 5, total: 10 }])(@@ -22,7 +23,9 @@ describe('SyncDriver job snapshots', () => {progress,});});- const [id] = await driver.bulkAdd(QUEUE, 'job', [{ data: {} }]);+ const [id] = await driver.bulkAdd(QUEUE, 'job', [+ { data: {}, ...SYSTEM_JOB },+ ]);expect((await driver.getJobs(QUEUE, [id]))[id]).toMatchObject({state: 'completed',@@ -49,8 +52,8 @@ describe('SyncDriver job snapshots', () => {await expect(driver.bulkAdd(QUEUE, 'job', [- { jobId: 'failed-job', data: {} },- { jobId: 'completed-job', data: {} },+ { jobId: 'failed-job', data: {}, ...SYSTEM_JOB },+ { jobId: 'completed-job', data: {}, ...SYSTEM_JOB },]),).rejects.toBe(error);const jobs = await driver.getJobs(QUEUE, ['failed-job', 'completed-job']);@@ -81,13 +84,14 @@ describe('SyncDriver job snapshots', () => {}});const pendingAdd = driver.bulkAdd(QUEUE, 'job', [- { jobId: 'active-job', data: {} },+ { jobId: 'active-job', data: {}, ...SYSTEM_JOB },]);const jobs = Array.from({ length: QUEUE_RETENTION.completedMaxCount + 1 },(_, index) => ({jobId: `job-${index}`,data: {},+ ...SYSTEM_JOB,}),);const ids = await driver.bulkAdd(QUEUE, 'job', jobs);@@ -110,9 +114,13 @@ describe('SyncDriver job snapshots', () => {it('prunes old snapshots when another job finishes', async () => {const driver = new SyncDriver();driver.work(QUEUE, async () => {});- const [oldId] = await driver.bulkAdd(QUEUE, 'job', [{ data: {} }]);+ const [oldId] = await driver.bulkAdd(QUEUE, 'job', [+ { data: {}, ...SYSTEM_JOB },+ ]);jest.setSystemTime(Date.now() + QUEUE_RETENTION.completedMaxAge * 1000 + 1);- const [newId] = await driver.bulkAdd(QUEUE, 'job', [{ data: {} }]);+ const [newId] = await driver.bulkAdd(QUEUE, 'job', [+ { data: {}, ...SYSTEM_JOB },+ ]);const jobs = await driver.getJobs(QUEUE, [oldId, newId]);expect(jobs[oldId]).toBeUndefined();diff --git a/packages/twenty-server/src/engine/core-modules/message-queue/drivers/bullmq.driver.ts b/packages/twenty-server/src/engine/core-modules/message-queue/drivers/bullmq.driver.tsindex fd5fc2bcc26..938e6924d1d 100644--- a/packages/twenty-server/src/engine/core-modules/message-queue/drivers/bullmq.driver.ts+++ b/packages/twenty-server/src/engine/core-modules/message-queue/drivers/bullmq.driver.ts@@ -23,6 +23,7 @@ import {type QueueCronJobOptions,type QueueJobOptions,type QueueJobRecipient,+ type QueueJobSettings,} from 'src/engine/core-modules/message-queue/drivers/interfaces/job-options.interface';import {type InFlightQueueJob,@@ -485,7 +486,7 @@ export class BullMQDriverreturn broadcastTo;}- private buildJobsOptions(options?: QueueJobOptions): BullMQJobsOptions {+ private buildJobsOptions(options?: QueueJobSettings): BullMQJobsOptions {return {jobId: options?.id? buildQueueJobIdWithSuffix({ jobIdPrefix: options.id, suffix: v4() })@@ -517,7 +518,7 @@ export class BullMQDriverqueueName: MessageQueue,jobName: string,data: T,- options?: QueueJobOptions,+ options: QueueJobOptions,): Promise<string | undefined> {if (!this.queueMap[queueName]) {throw new Error(@@ -525,7 +526,7 @@ export class BullMQDriver);}- if (options?.id && !options?.doNotCheckDuplicatesBasedOnPrefix) {+ if (options.id && !options.doNotCheckDuplicatesBasedOnPrefix) {const waitingJobIds = await this.getWaitingJobIds(queueName);if (@@ -538,8 +539,9 @@ export class BullMQDriverconst queueOptions = this.buildJobsOptions(options);const [jobOptions] = await this.fairQueueScheduler.computeJobOptions({queueName,- jobsData: [data],- options,+ scopes: [options],+ additionalGroupKeys: options.groupKeys,+ delay: options.delay,staticPriority: queueOptions.priority,});@@ -580,7 +582,7 @@ export class BullMQDriverqueueName: MessageQueue,jobName: string,jobs: QueueJobToAdd<T>[],- options?: QueueJobOptions,+ options?: QueueJobSettings,): Promise<string[]> {if (!this.queueMap[queueName]) {throw new Error(@@ -599,8 +601,9 @@ export class BullMQDriverconst queueOptions = this.buildJobsOptions(options);const jobOptions = await this.fairQueueScheduler.computeJobOptions({queueName,- jobsData: jobsToAdd.map(({ data }) => data),- options,+ scopes: jobsToAdd,+ additionalGroupKeys: options?.groupKeys,+ delay: options?.delay,staticPriority: queueOptions.priority,});diff --git a/packages/twenty-server/src/engine/core-modules/message-queue/drivers/fairness/fair-queue-scheduler.ts b/packages/twenty-server/src/engine/core-modules/message-queue/drivers/fairness/fair-queue-scheduler.tsindex faaf8fdfaac..86ceee48ff9 100644--- a/packages/twenty-server/src/engine/core-modules/message-queue/drivers/fairness/fair-queue-scheduler.ts+++ b/packages/twenty-server/src/engine/core-modules/message-queue/drivers/fairness/fair-queue-scheduler.ts@@ -1,14 +1,14 @@import { PRIORITY_LIMIT, type RedisClient } from 'bullmq';-import { isDefined, isNonEmptyArray } from 'twenty-shared/utils';+import { isDefined } from 'twenty-shared/utils';import {FAIR_TICKET_OFFSET,URGENT_FAIR_PRIORITY,} from 'src/engine/core-modules/message-queue/drivers/fairness/fair-priority.constant';import { FairTicketAllocator } from 'src/engine/core-modules/message-queue/drivers/fairness/fair-ticket-allocator';-import { type QueueJobOptions } from 'src/engine/core-modules/message-queue/drivers/interfaces/job-options.interface';+import { type QueueJobScope } from 'src/engine/core-modules/message-queue/drivers/interfaces/job-options.interface';import { type MessageQueue } from 'src/engine/core-modules/message-queue/message-queue.constants';-import { getDefaultGroupKeys } from 'src/engine/core-modules/message-queue/utils/get-default-group-keys.util';+import { buildQueueJobGroupKeys } from 'src/engine/core-modules/message-queue/utils/build-queue-job-group-keys.util';import { type MetricsService } from 'src/engine/core-modules/metrics/metrics.service';import { MetricsKeys } from 'src/engine/core-modules/metrics/types/metrics-keys.type';import { type TwentyConfigService } from 'src/engine/core-modules/twenty-config/twenty-config.service';@@ -36,20 +36,22 @@ export class FairQueueScheduler {async computeJobOptions({queueName,- jobsData,- options,+ scopes,+ additionalGroupKeys,+ delay,staticPriority,}: {queueName: MessageQueue;- jobsData: unknown[];- options?: Pick<QueueJobOptions, 'groupKeys' | 'delay'>;+ scopes: QueueJobScope[];+ additionalGroupKeys?: string[];+ delay: number | undefined;staticPriority: number | undefined;}): Promise<FairJobOptions[]> {- const staticJobOptions: FairJobOptions[] = jobsData.map(() => ({+ const staticJobOptions: FairJobOptions[] = scopes.map(() => ({priority: staticPriority,}));- if (!this.isFairnessEnabled() || (options?.delay ?? 0) > 0) {+ if (!this.isFairnessEnabled() || (delay ?? 0) > 0) {return staticJobOptions;}@@ -58,10 +60,8 @@ export class FairQueueScheduler {{ groupKeys: string[]; jobIndexes: number[] }>();- jobsData.forEach((jobData, jobIndex) => {- const groupKeys = isNonEmptyArray(options?.groupKeys)- ? [...new Set(options.groupKeys)]- : getDefaultGroupKeys(jobData);+ scopes.forEach((scope, jobIndex) => {+ const groupKeys = buildQueueJobGroupKeys({ scope, additionalGroupKeys });const signature = JSON.stringify(groupKeys);const request = requestByGroupKeys.get(signature) ?? {groupKeys,diff --git a/packages/twenty-server/src/engine/core-modules/message-queue/drivers/interfaces/job-options.interface.ts b/packages/twenty-server/src/engine/core-modules/message-queue/drivers/interfaces/job-options.interface.tsindex 919d2b6a9f2..cb600a5a055 100644--- a/packages/twenty-server/src/engine/core-modules/message-queue/drivers/interfaces/job-options.interface.ts+++ b/packages/twenty-server/src/engine/core-modules/message-queue/drivers/interfaces/job-options.interface.ts@@ -14,7 +14,33 @@ export type QueueJobRecipient = {userWorkspaceId?: string;};-export interface QueueJobOptions {+export type SystemQueueJobScope = {+ isSystemJob: true;+ isWorkspaceJob?: never;+ workspaceId?: never;+ applicationUniversalIdentifier?: never;+};++export type WorkspaceQueueJobScope = {+ isSystemJob?: never;+ isWorkspaceJob: true;+ workspaceId: string;+ applicationUniversalIdentifier?: never;+};++export type ApplicationQueueJobScope = {+ isSystemJob?: never;+ isWorkspaceJob?: never;+ workspaceId: string;+ applicationUniversalIdentifier: string;+};++export type QueueJobScope =+ | SystemQueueJobScope+ | WorkspaceQueueJobScope+ | ApplicationQueueJobScope;++export type QueueJobSettings = {id?: string;doNotCheckDuplicatesBasedOnPrefix?: boolean;priority?: number;@@ -24,12 +50,14 @@ export interface QueueJobOptions {deduplication?: QueueJobDeduplicationOptions;broadcastTo?: QueueJobRecipient;groupKeys?: string[];-}+};++export type QueueJobOptions = QueueJobSettings & QueueJobScope;-export interface QueueCronJobOptions extends QueueJobOptions {+export type QueueCronJobOptions = Pick<QueueJobSettings, 'priority'> & {repeat: {every?: number;pattern?: string;limit?: number;};-}+};diff --git a/packages/twenty-server/src/engine/core-modules/message-queue/drivers/interfaces/message-queue-driver.interface.ts b/packages/twenty-server/src/engine/core-modules/message-queue/drivers/interfaces/message-queue-driver.interface.tsindex 73b873677d2..159272cf738 100644--- a/packages/twenty-server/src/engine/core-modules/message-queue/drivers/interfaces/message-queue-driver.interface.ts+++ b/packages/twenty-server/src/engine/core-modules/message-queue/drivers/interfaces/message-queue-driver.interface.ts@@ -4,6 +4,8 @@ import {type QueueCronJobOptions,type QueueJobOptions,type QueueJobRecipient,+ type QueueJobScope,+ type QueueJobSettings,} from 'src/engine/core-modules/message-queue/drivers/interfaces/job-options.interface';import {type MessageQueueJob,@@ -18,13 +20,13 @@ export interface MessageQueueDriver {queueName: MessageQueue,jobName: string,data: T,- options?: QueueJobOptions,+ options: QueueJobOptions,): Promise<string | undefined>;bulkAdd<T extends MessageQueueJobData>(queueName: MessageQueue,jobName: string,jobs: QueueJobToAdd<T>[],- options?: QueueJobOptions,+ options?: QueueJobSettings,): Promise<string[]>;work<T extends MessageQueueJobData>(queueName: MessageQueue,@@ -67,7 +69,7 @@ export interface MessageQueueDriver {export type QueueJobToAdd<T extends MessageQueueJobData> = {data: T;jobId?: string;-};+} & QueueJobScope;export type QueueJobDetails<T extends MessageQueueJobData> = {id: string;diff --git a/packages/twenty-server/src/engine/core-modules/message-queue/services/message-queue.service.ts b/packages/twenty-server/src/engine/core-modules/message-queue/services/message-queue.service.tsindex b77d092dceb..faee36a7ced 100644--- a/packages/twenty-server/src/engine/core-modules/message-queue/services/message-queue.service.ts+++ b/packages/twenty-server/src/engine/core-modules/message-queue/services/message-queue.service.ts@@ -3,6 +3,7 @@ import { Inject, Injectable } from '@nestjs/common';import {type QueueCronJobOptions,type QueueJobOptions,+ type QueueJobSettings,} from 'src/engine/core-modules/message-queue/drivers/interfaces/job-options.interface';import {type InFlightQueueJob,@@ -35,7 +36,7 @@ export class MessageQueueService {add<T extends MessageQueueJobData>(jobName: string,data: T,- options?: QueueJobOptions,+ options: QueueJobOptions,): Promise<string | undefined> {return this.driver.add(this.queueName, jobName, data, options);}@@ -43,7 +44,7 @@ export class MessageQueueService {bulkAdd<T extends MessageQueueJobData>(jobName: string,jobs: QueueJobToAdd<T>[],- options?: QueueJobOptions,+ options?: QueueJobSettings,): Promise<string[]> {return this.driver.bulkAdd(this.queueName, jobName, jobs, options);}diff --git a/packages/twenty-server/src/engine/core-modules/message-queue/utils/build-queue-job-group-keys.util.ts b/packages/twenty-server/src/engine/core-modules/message-queue/utils/build-queue-job-group-keys.util.tsnew file mode 100644index 00000000000..5cd79eec9fd--- /dev/null+++ b/packages/twenty-server/src/engine/core-modules/message-queue/utils/build-queue-job-group-keys.util.ts@@ -0,0 +1,30 @@+import { type QueueJobScope } from 'src/engine/core-modules/message-queue/drivers/interfaces/job-options.interface';+import { buildApplicationGroupKey } from 'src/engine/core-modules/message-queue/utils/build-application-group-key.util';+import { buildWorkspaceGroupKey } from 'src/engine/core-modules/message-queue/utils/build-workspace-group-key.util';++const SYSTEM_GROUP_KEY = 'system';++const buildScopeGroupKeys = (scope: QueueJobScope): string[] => {+ if (scope.isSystemJob) {+ return [SYSTEM_GROUP_KEY];+ }++ if (scope.isWorkspaceJob) {+ return [buildWorkspaceGroupKey(scope.workspaceId)];+ }++ return [+ buildWorkspaceGroupKey(scope.workspaceId),+ buildApplicationGroupKey(scope.applicationUniversalIdentifier),+ ];+};++export const buildQueueJobGroupKeys = ({+ scope,+ additionalGroupKeys = [],+}: {+ scope: QueueJobScope;+ additionalGroupKeys?: string[];+}): string[] => [+ ...new Set([...buildScopeGroupKeys(scope), ...additionalGroupKeys]),+];diff --git a/packages/twenty-server/src/engine/core-modules/message-queue/utils/get-default-group-keys.util.ts b/packages/twenty-server/src/engine/core-modules/message-queue/utils/get-default-group-keys.util.tsdeleted file mode 100644index 6d07e7cd698..00000000000--- a/packages/twenty-server/src/engine/core-modules/message-queue/utils/get-default-group-keys.util.ts+++ /dev/null@@ -1,14 +0,0 @@-import { isNonEmptyString } from '@sniptt/guards';-import { isPlainObject } from 'twenty-shared/utils';--import { buildWorkspaceGroupKey } from 'src/engine/core-modules/message-queue/utils/build-workspace-group-key.util';--const SYSTEM_GROUP_KEY = 'system';--export const getDefaultGroupKeys = (jobData: unknown): string[] => {- const workspaceId = isPlainObject(jobData) ? jobData.workspaceId : undefined;-- return isNonEmptyString(workspaceId)- ? [buildWorkspaceGroupKey(workspaceId)]- : [SYSTEM_GROUP_KEY];-};diff --git a/packages/twenty-server/src/engine/core-modules/pending-wake-up/services/pending-wake-up.service.ts b/packages/twenty-server/src/engine/core-modules/pending-wake-up/services/pending-wake-up.service.tsindex fbfaa8dd985..6e272220edc 100644--- a/packages/twenty-server/src/engine/core-modules/pending-wake-up/services/pending-wake-up.service.ts+++ b/packages/twenty-server/src/engine/core-modules/pending-wake-up/services/pending-wake-up.service.ts@@ -143,9 +143,12 @@ export class PendingWakeUpService {recordReadAttempt,},{- ...this.pendingWakeUpOwnerHandlerRegistryService+ ...((await this.pendingWakeUpOwnerHandlerRegistryService.getHandlerOrThrow(wakeUp.ownerType)- .buildResumeJobOptions?.(wakeUp.ownerId),+ .buildResumeJobOptions?.(wakeUp)) ?? {+ workspaceId: wakeUp.workspaceId,+ isWorkspaceJob: true,+ }),delay: Math.max(delayMs, 0),...(isDefined(deduplicationId)? { deduplication: { id: deduplicationId } }diff --git a/packages/twenty-server/src/engine/core-modules/pending-wake-up/types/pending-wake-up-owner-handler.type.ts b/packages/twenty-server/src/engine/core-modules/pending-wake-up/types/pending-wake-up-owner-handler.type.tsindex 16d4d9f297f..797c0b07a5c 100644--- a/packages/twenty-server/src/engine/core-modules/pending-wake-up/types/pending-wake-up-owner-handler.type.ts+++ b/packages/twenty-server/src/engine/core-modules/pending-wake-up/types/pending-wake-up-owner-handler.type.ts@@ -6,7 +6,9 @@ import { type PendingWakeUpOwnerState } from 'src/engine/core-modules/pending-waimport { type RolePermissionConfig } from 'src/engine/twenty-orm/types/role-permission-config.type';export type PendingWakeUpOwnerHandler<TOwner = unknown> = {- buildResumeJobOptions?(ownerId: string): QueueJobOptions;+ buildResumeJobOptions?(+ wakeUp: Pick<PendingWakeUpEntity, 'ownerId' | 'workspaceId'>,+ ): Promise<QueueJobOptions>;getOwnerState(wakeUp: PendingWakeUpEntity,diff --git a/packages/twenty-server/src/engine/core-modules/record-export/services/record-export.workspace-service.ts b/packages/twenty-server/src/engine/core-modules/record-export/services/record-export.workspace-service.tsindex 9d181a766cb..28a142b6457 100644--- a/packages/twenty-server/src/engine/core-modules/record-export/services/record-export.workspace-service.ts+++ b/packages/twenty-server/src/engine/core-modules/record-export/services/record-export.workspace-service.ts@@ -37,6 +37,7 @@ import { JwtWrapperService } from 'src/engine/core-modules/jwt/services/jwt-wrapimport { InjectMessageQueue } from 'src/engine/core-modules/message-queue/decorators/message-queue.decorator';import { MessageQueue } from 'src/engine/core-modules/message-queue/message-queue.constants';import { MessageQueueService } from 'src/engine/core-modules/message-queue/services/message-queue.service';+import { buildApplicationQueueJobScope } from 'src/engine/core-modules/application/utils/build-application-queue-job-scope.util';import {RECORD_EXPORT_CONNECTION_TTL_MS,RECORD_EXPORT_DOWNLOAD_TOKEN_TTL_SECONDS,@@ -224,10 +225,22 @@ export class RecordExportWorkspaceService {}async enqueue(recordExport: RecordExport): Promise<string> {+ const { workspaceId, applicationId } = recordExport;+ const { flatApplicationMaps } =+ await this.workspaceCacheService.getOrRecompute(workspaceId, [+ 'flatApplicationMaps',+ ]);const jobId = await this.messageQueueService.add('GenerateRecordExportJob',recordExport,- { id: recordExport.id },+ {+ id: recordExport.id,+ ...buildApplicationQueueJobScope({+ workspaceId,+ applicationId,+ flatApplicationMaps,+ }),+ },);if (!isDefined(jobId)) {throw new RecordExportException(diff --git a/packages/twenty-server/src/engine/core-modules/sdk-client/sdk-client-generation.service.ts b/packages/twenty-server/src/engine/core-modules/sdk-client/sdk-client-generation.service.tsindex cd4055f16fb..1b1e76624c6 100644--- a/packages/twenty-server/src/engine/core-modules/sdk-client/sdk-client-generation.service.ts+++ b/packages/twenty-server/src/engine/core-modules/sdk-client/sdk-client-generation.service.ts@@ -99,6 +99,8 @@ export class SdkClientGenerationService {{id: `sdk-client:${workspaceId}:${applicationId}`,retryLimit: SDK_CLIENT_GENERATION_RETRY_LIMIT,+ workspaceId,+ applicationUniversalIdentifier,},);}diff --git a/packages/twenty-server/src/engine/core-modules/server-route-trigger/server-route-trigger.service.ts b/packages/twenty-server/src/engine/core-modules/server-route-trigger/server-route-trigger.service.tsindex c936e09e305..2c78eb608a8 100644--- a/packages/twenty-server/src/engine/core-modules/server-route-trigger/server-route-trigger.service.ts+++ b/packages/twenty-server/src/engine/core-modules/server-route-trigger/server-route-trigger.service.ts@@ -349,6 +349,9 @@ export class ServerRouteTriggerService {retryLimit: QUEUED_TARGET_RETRY_LIMIT,backoff: LOGIC_FUNCTION_QUEUE_RETRY_BACKOFF,priority: SERVER_ROUTE_DISPATCH_JOB_PRIORITY,+ workspaceId,+ applicationUniversalIdentifier:+ logicFunction.application.universalIdentifier,},);diff --git a/packages/twenty-server/src/engine/core-modules/user/services/user.service.ts b/packages/twenty-server/src/engine/core-modules/user/services/user.service.tsindex eac6575d3c4..03aeca5fdf4 100644--- a/packages/twenty-server/src/engine/core-modules/user/services/user.service.ts+++ b/packages/twenty-server/src/engine/core-modules/user/services/user.service.ts@@ -614,6 +614,7 @@ export class UserService {data,{retryLimit: 2,+ isSystemJob: true,},);}diff --git a/packages/twenty-server/src/engine/core-modules/workspace-data-export/services/workspace-data-export-step.service.ts b/packages/twenty-server/src/engine/core-modules/workspace-data-export/services/workspace-data-export-step.service.tsindex 4983cd55309..93bd98c9ce6 100644--- a/packages/twenty-server/src/engine/core-modules/workspace-data-export/services/workspace-data-export-step.service.ts+++ b/packages/twenty-server/src/engine/core-modules/workspace-data-export/services/workspace-data-export-step.service.ts@@ -44,12 +44,17 @@ export class WorkspaceDataExportStepService {delay,...jobData}: WorkspaceDataExportJobData & { delay?: number }): Promise<void> {- const { exportId, cursor } = jobData;+ const { workspaceId, exportId, cursor } = jobData;await this.messageQueueService.add(WORKSPACE_DATA_EXPORT_STEP_JOB_NAME,jobData,- { id: `${exportId}:${cursor.stepIndex}:${cursor.attempt}`, delay },+ {+ id: `${exportId}:${cursor.stepIndex}:${cursor.attempt}`,+ delay,+ workspaceId,+ isWorkspaceJob: true,+ },);}diff --git a/packages/twenty-server/src/engine/core-modules/workspace/services/workspace.service.ts b/packages/twenty-server/src/engine/core-modules/workspace/services/workspace.service.tsindex 137e0ff3f04..25dd22b6f88 100644--- a/packages/twenty-server/src/engine/core-modules/workspace/services/workspace.service.ts+++ b/packages/twenty-server/src/engine/core-modules/workspace/services/workspace.service.ts@@ -814,6 +814,10 @@ export class WorkspaceService {await this.deleteCascadeMessageQueueService.add<FileWorkspaceFolderDeletionJobData>(FileWorkspaceFolderDeletionJob.name,{ workspaceId: workspace.id },+ {+ workspaceId: workspace.id,+ isWorkspaceJob: true,+ },);const emailingDomains = await this.coreDataSource@@ -828,6 +832,10 @@ export class WorkspaceService {(emailingDomain) => emailingDomain.domain,),},+ {+ workspaceId: workspace.id,+ isWorkspaceJob: true,+ },);if (workspace.customDomain) {@@ -869,6 +877,8 @@ export class WorkspaceService {id: `${WorkspaceDeletionApplicationUninstallJob.name}-${workspaceId}`,retryLimit: WORKSPACE_APPLICATION_UNINSTALL_RETRY_LIMIT,backoff: LOGIC_FUNCTION_QUEUE_RETRY_BACKOFF,+ workspaceId,+ isWorkspaceJob: true,},);}diff --git a/packages/twenty-server/src/engine/metadata-modules/ai/ai-agent-execution/ai-agent-execution.module.ts b/packages/twenty-server/src/engine/metadata-modules/ai/ai-agent-execution/ai-agent-execution.module.tsindex 58aa23d851a..93dafe4b6de 100644--- a/packages/twenty-server/src/engine/metadata-modules/ai/ai-agent-execution/ai-agent-execution.module.ts+++ b/packages/twenty-server/src/engine/metadata-modules/ai/ai-agent-execution/ai-agent-execution.module.ts@@ -11,6 +11,7 @@ import { PendingWakeUpModule } from 'src/engine/core-modules/pending-wake-up/penimport { ToolProviderModule } from 'src/engine/core-modules/tool-provider/tool-provider.module';import { UserWorkspaceModule } from 'src/engine/core-modules/user-workspace/user-workspace.module';import { WorkspaceEntity } from 'src/engine/core-modules/workspace/workspace.entity';+import { WorkspaceCacheModule } from 'src/engine/workspace-cache/workspace-cache.module';import { ContinueAgentRunJob } from 'src/engine/metadata-modules/ai/ai-agent-execution/jobs/continue-agent-run.job';import { DeliverAgentRunOutcomeJob } from 'src/engine/metadata-modules/ai/ai-agent-execution/jobs/deliver-agent-run-outcome.job';import { RunAgentJob } from 'src/engine/metadata-modules/ai/ai-agent-execution/jobs/run-agent.job';@@ -20,6 +21,7 @@ import { AgentRunResolver } from 'src/engine/metadata-modules/ai/ai-agent-executimport { AgentActorContextService } from 'src/engine/metadata-modules/ai/ai-agent-execution/services/agent-actor-context.service';import { AgentCallerConversationService } from 'src/engine/metadata-modules/ai/ai-agent-execution/services/agent-caller-conversation.service';import { AgentRunCallerHandlerRegistryService } from 'src/engine/metadata-modules/ai/ai-agent-execution/services/agent-run-caller-handler-registry.service';+import { AgentRunJobScopeService } from 'src/engine/metadata-modules/ai/ai-agent-execution/services/agent-run-job-scope.service';import { AgentRunConversationService } from 'src/engine/metadata-modules/ai/ai-agent-execution/services/agent-run-conversation.service';import { AgentRunPendingWakeUpHandlerService } from 'src/engine/metadata-modules/ai/ai-agent-execution/services/agent-run-pending-wake-up-handler.service';import { AgentRunService } from 'src/engine/metadata-modules/ai/ai-agent-execution/services/agent-run.service';@@ -63,6 +65,7 @@ import { provideWorkspaceScopedRepository } from 'src/engine/twenty-orm/workspacPermissionsModule,SkillModule,ToolProviderModule,+ WorkspaceCacheModule,TypeOrmModule.forFeature([AgentEntity,FileEntity,@@ -75,6 +78,7 @@ import { provideWorkspaceScopedRepository } from 'src/engine/twenty-orm/workspacAgentCallerConversationService,AgentRunCallerHandlerRegistryService,AgentRunConversationService,+ AgentRunJobScopeService,AgentRunSuspensionService,AgentMessagePartResolver,AgentMessageResolver,diff --git a/packages/twenty-server/src/engine/metadata-modules/ai/ai-agent-execution/services/__tests__/agent-run-job-scope.service.spec.ts b/packages/twenty-server/src/engine/metadata-modules/ai/ai-agent-execution/services/__tests__/agent-run-job-scope.service.spec.tsnew file mode 100644index 00000000000..2c4da2fa7d6--- /dev/null+++ b/packages/twenty-server/src/engine/metadata-modules/ai/ai-agent-execution/services/__tests__/agent-run-job-scope.service.spec.ts@@ -0,0 +1,56 @@+import { AgentRunJobScopeService } from 'src/engine/metadata-modules/ai/ai-agent-execution/services/agent-run-job-scope.service';+import { createEmptyFlatEntityMaps } from 'src/engine/metadata-modules/flat-entity/constant/create-empty-flat-entity-maps.constant';+import { addFlatEntityToFlatEntityMapsOrThrow } from 'src/engine/metadata-modules/flat-entity/utils/add-flat-entity-to-flat-entity-maps-or-throw.util';++const WORKSPACE_ID = 'workspace-id';++const buildFlatAgent = (id: string, applicationUniversalIdentifier: string) =>+ ({+ id,+ universalIdentifier: id,+ applicationUniversalIdentifier,+ workspaceId: WORKSPACE_ID,+ }) as never;++const buildService = () => {+ const flatAgentMaps = [+ buildFlatAgent('agent-a', 'application-a'),+ buildFlatAgent('agent-b', 'application-b'),+ ].reduce(+ (flatEntityMaps, flatEntity) =>+ addFlatEntityToFlatEntityMapsOrThrow({ flatEntity, flatEntityMaps }),+ createEmptyFlatEntityMaps(),+ );++ return new AgentRunJobScopeService({+ getOrRecompute: jest.fn().mockResolvedValue({ flatAgentMaps }),+ } as never);+};++describe('AgentRunJobScopeService', () => {+ it('scopes runs of agents owned by different applications to their own application', async () => {+ const service = buildService();++ await expect(+ service.resolve({ workspaceId: WORKSPACE_ID, agentId: 'agent-a' }),+ ).resolves.toEqual({+ workspaceId: WORKSPACE_ID,+ applicationUniversalIdentifier: 'application-a',+ });+ await expect(+ service.resolve({ workspaceId: WORKSPACE_ID, agentId: 'agent-b' }),+ ).resolves.toEqual({+ workspaceId: WORKSPACE_ID,+ applicationUniversalIdentifier: 'application-b',+ });+ });++ it('falls back to a workspace scope for an agent that no longer exists', async () => {+ await expect(+ buildService().resolve({+ workspaceId: WORKSPACE_ID,+ agentId: 'deleted-agent',+ }),+ ).resolves.toEqual({ workspaceId: WORKSPACE_ID, isWorkspaceJob: true });+ });+});diff --git a/packages/twenty-server/src/engine/metadata-modules/ai/ai-agent-execution/services/__tests__/agent-run-suspension.service.deliver-outcome.spec.ts b/packages/twenty-server/src/engine/metadata-modules/ai/ai-agent-execution/services/__tests__/agent-run-suspension.service.deliver-outcome.spec.tsindex 8fcadeb7d54..9a979f5ee81 100644--- a/packages/twenty-server/src/engine/metadata-modules/ai/ai-agent-execution/services/__tests__/agent-run-suspension.service.deliver-outcome.spec.ts+++ b/packages/twenty-server/src/engine/metadata-modules/ai/ai-agent-execution/services/__tests__/agent-run-suspension.service.deliver-outcome.spec.ts@@ -17,6 +17,7 @@ const buildService = (waitingState: 'NOT_READY' | 'WAITING') => {{} as never,{} as never,{ getHandlerOrThrow: () => handler } as never,+ {} as never,messageQueueService as never,);@@ -41,7 +42,11 @@ describe('AgentRunSuspensionService deliverOutcome', () => {expect(messageQueueService.add).toHaveBeenCalledWith(DELIVER_AGENT_RUN_OUTCOME_JOB_NAME,expect.objectContaining({ outcome: delivery.outcome, attempt: 501 }),- { delay: 60_000 },+ {+ delay: 60_000,+ workspaceId: 'workspace-id',+ isWorkspaceJob: true,+ },);});diff --git a/packages/twenty-server/src/engine/metadata-modules/ai/ai-agent-execution/services/__tests__/agent-run-suspension.service.find-conversation-wait.spec.ts b/packages/twenty-server/src/engine/metadata-modules/ai/ai-agent-execution/services/__tests__/agent-run-suspension.service.find-conversation-wait.spec.tsindex 811dbce4ea8..831c3eebe8c 100644--- a/packages/twenty-server/src/engine/metadata-modules/ai/ai-agent-execution/services/__tests__/agent-run-suspension.service.find-conversation-wait.spec.ts+++ b/packages/twenty-server/src/engine/metadata-modules/ai/ai-agent-execution/services/__tests__/agent-run-suspension.service.find-conversation-wait.spec.ts@@ -17,6 +17,7 @@ const buildService = ({{} as never,{} as never,{} as never,+ {} as never,);const buildThread = (pendingQuestionMessageId: string | null = null) => ({diff --git a/packages/twenty-server/src/engine/metadata-modules/ai/ai-agent-execution/services/__tests__/agent-run-suspension.service.schedule-continuation.spec.ts b/packages/twenty-server/src/engine/metadata-modules/ai/ai-agent-execution/services/__tests__/agent-run-suspension.service.schedule-continuation.spec.tsnew file mode 100644index 00000000000..cb7e3235da5--- /dev/null+++ b/packages/twenty-server/src/engine/metadata-modules/ai/ai-agent-execution/services/__tests__/agent-run-suspension.service.schedule-continuation.spec.ts@@ -0,0 +1,71 @@+import { CONTINUE_AGENT_RUN_JOB_NAME } from 'src/engine/metadata-modules/ai/ai-agent-execution/constants/continue-agent-run-job-name.constant';+import { AgentRunSuspensionService } from 'src/engine/metadata-modules/ai/ai-agent-execution/services/agent-run-suspension.service';++const WORKSPACE_ID = 'workspace-id';++const buildService = (wakeUp: object | null) => {+ const messageQueueService = { add: jest.fn().mockResolvedValue(undefined) };+ const agentRunJobScopeService = {+ resolve: jest.fn(async ({ agentId }: { agentId: string }) => ({+ workspaceId: WORKSPACE_ID,+ applicationUniversalIdentifier: `application-of-${agentId}`,+ })),+ };++ const service = new AgentRunSuspensionService(+ {} as never,+ {} as never,+ {} as never,+ {} as never,+ {} as never,+ { find: jest.fn().mockResolvedValue(wakeUp) } as never,+ {} as never,+ agentRunJobScopeService as never,+ messageQueueService as never,+ );++ return { service, messageQueueService };+};++const jobData = {+ workspaceId: WORKSPACE_ID,+ threadId: 'thread-id',+ wakeUpId: 'wake-up-id',+ outcome: { type: 'MESSAGE_RECEIVED' as const },+};++describe('AgentRunSuspensionService scheduleContinuation', () => {+ it("keeps the suspended agent's application scope on the continuation", async () => {+ const { service, messageQueueService } = buildService({+ payload: { runSpec: { agentId: 'agent-a' } },+ });++ await service.scheduleContinuation(jobData);++ expect(messageQueueService.add).toHaveBeenCalledWith(+ CONTINUE_AGENT_RUN_JOB_NAME,+ jobData,+ {+ deduplication: { id: 'continue-agent-run-wake-up-id' },+ workspaceId: WORKSPACE_ID,+ applicationUniversalIdentifier: 'application-of-agent-a',+ },+ );+ });++ it('uses a workspace scope once the wake-up is gone', async () => {+ const { service, messageQueueService } = buildService(null);++ await service.scheduleContinuation(jobData);++ expect(messageQueueService.add).toHaveBeenCalledWith(+ CONTINUE_AGENT_RUN_JOB_NAME,+ jobData,+ {+ deduplication: { id: 'continue-agent-run-wake-up-id' },+ workspaceId: WORKSPACE_ID,+ isWorkspaceJob: true,+ },+ );+ });+});diff --git a/packages/twenty-server/src/engine/metadata-modules/ai/ai-agent-execution/services/agent-run-job-scope.service.ts b/packages/twenty-server/src/engine/metadata-modules/ai/ai-agent-execution/services/agent-run-job-scope.service.tsnew file mode 100644index 00000000000..e39c48a4c81--- /dev/null+++ b/packages/twenty-server/src/engine/metadata-modules/ai/ai-agent-execution/services/agent-run-job-scope.service.ts@@ -0,0 +1,38 @@+import { Injectable } from '@nestjs/common';++import { isDefined } from 'twenty-shared/utils';++import { type QueueJobScope } from 'src/engine/core-modules/message-queue/drivers/interfaces/job-options.interface';+import { findFlatEntityByIdInFlatEntityMaps } from 'src/engine/metadata-modules/flat-entity/utils/find-flat-entity-by-id-in-flat-entity-maps.util';+import { WorkspaceCacheService } from 'src/engine/workspace-cache/services/workspace-cache.service';++@Injectable()+export class AgentRunJobScopeService {+ constructor(private readonly workspaceCacheService: WorkspaceCacheService) {}++ async resolve({+ workspaceId,+ agentId,+ }: {+ workspaceId: string;+ agentId: string;+ }): Promise<QueueJobScope> {+ const { flatAgentMaps } = await this.workspaceCacheService.getOrRecompute(+ workspaceId,+ ['flatAgentMaps'],+ );++ const flatAgent = findFlatEntityByIdInFlatEntityMaps({+ flatEntityId: agentId,+ flatEntityMaps: flatAgentMaps,+ });++ return isDefined(flatAgent)+ ? {+ workspaceId,+ applicationUniversalIdentifier:+ flatAgent.applicationUniversalIdentifier,+ }+ : { workspaceId, isWorkspaceJob: true };+ }+}diff --git a/packages/twenty-server/src/engine/metadata-modules/ai/ai-agent-execution/services/agent-run-suspension.service.ts b/packages/twenty-server/src/engine/metadata-modules/ai/ai-agent-execution/services/agent-run-suspension.service.tsindex 124450cc9c9..11b27259db7 100644--- a/packages/twenty-server/src/engine/metadata-modules/ai/ai-agent-execution/services/agent-run-suspension.service.ts+++ b/packages/twenty-server/src/engine/metadata-modules/ai/ai-agent-execution/services/agent-run-suspension.service.ts@@ -7,6 +7,7 @@ import { IsNull, Not, Raw } from 'typeorm';import { formatErrorWithCause } from 'src/engine/metadata-modules/ai/ai-chat/utils/format-error-with-cause.util';import { InjectMessageQueue } from 'src/engine/core-modules/message-queue/decorators/message-queue.decorator';+import { type QueueJobScope } from 'src/engine/core-modules/message-queue/drivers/interfaces/job-options.interface';import { MessageQueue } from 'src/engine/core-modules/message-queue/message-queue.constants';import { MessageQueueService } from 'src/engine/core-modules/message-queue/services/message-queue.service';import { PendingWakeUpEntity } from 'src/engine/core-modules/pending-wake-up/entities/pending-wake-up.entity';@@ -16,6 +17,7 @@ import { CONTINUE_AGENT_RUN_JOB_NAME } from 'src/engine/metadata-modules/ai/ai-aimport { DELIVER_AGENT_RUN_OUTCOME_JOB_NAME } from 'src/engine/metadata-modules/ai/ai-agent-execution/constants/deliver-agent-run-outcome-job-name.constant';import { AGENT_WAIT_TOOL_NAMES } from 'src/engine/metadata-modules/ai/ai-agent-execution/pausing-tools/constants/agent-wait-tool-names.constant';import { buildWaitOutcomeToolOutput } from 'src/engine/metadata-modules/ai/ai-agent-execution/pausing-tools/wait-tools/build-wait-outcome-tool-output.util';+import { AgentRunJobScopeService } from 'src/engine/metadata-modules/ai/ai-agent-execution/services/agent-run-job-scope.service';import { AgentRunCallerHandlerRegistryService } from 'src/engine/metadata-modules/ai/ai-agent-execution/services/agent-run-caller-handler-registry.service';import { type AgentConversationWait } from 'src/engine/metadata-modules/ai/ai-agent-execution/types/agent-conversation-wait.type';import { type AgentRunCaller } from 'src/engine/metadata-modules/ai/ai-agent-execution/types/agent-run-caller.type';@@ -70,6 +72,7 @@ export class AgentRunSuspensionService {private readonly turnRecorderService: AgentTurnRecorderService,private readonly pendingWakeUpService: PendingWakeUpService,private readonly callerHandlerRegistry: AgentRunCallerHandlerRegistryService,+ private readonly agentRunJobScopeService: AgentRunJobScopeService,@InjectMessageQueue(MessageQueue.aiQueue)private readonly messageQueueService: MessageQueueService,) {}@@ -146,10 +149,34 @@ export class AgentRunSuspensionService {await this.messageQueueService.add<ContinueAgentRunJobData>(CONTINUE_AGENT_RUN_JOB_NAME,jobData,- { deduplication: { id: `continue-agent-run-${jobData.wakeUpId}` } },+ {+ deduplication: { id: `continue-agent-run-${jobData.wakeUpId}` },+ ...(await this.resolveContinuationJobScope(jobData)),+ },);}+ private async resolveContinuationJobScope({+ workspaceId,+ wakeUpId,+ }: ContinueAgentRunJobData): Promise<QueueJobScope> {+ const wakeUp = await this.pendingWakeUpService.find({+ workspaceId,+ wakeUpId,+ });++ if (!isDefined(wakeUp)) {+ return { workspaceId, isWorkspaceJob: true };+ }++ const suspension = wakeUp.payload as AgentRunSuspension;++ return this.agentRunJobScopeService.resolve({+ workspaceId,+ agentId: suspension.runSpec.agentId,+ });+ }+// A run that ended, or could not go on, leaves nothing to wait on, and its caller gets the outcomeasync settle({workspaceId,@@ -220,7 +247,11 @@ export class AgentRunSuspensionService {await this.messageQueueService.add<AgentRunOutcomeDelivery>(DELIVER_AGENT_RUN_OUTCOME_JOB_NAME,{ ...delivery, attempt: attempt + 1 },- { delay: computePendingWakeUpRetryDelayMs(attempt) },+ {+ delay: computePendingWakeUpRetryDelayMs(attempt),+ workspaceId,+ isWorkspaceJob: true,+ },);}diff --git a/packages/twenty-server/src/engine/metadata-modules/ai/ai-agent-execution/services/agent-runner.service.ts b/packages/twenty-server/src/engine/metadata-modules/ai/ai-agent-execution/services/agent-runner.service.tsindex 69149b50802..292d9143fde 100644--- a/packages/twenty-server/src/engine/metadata-modules/ai/ai-agent-execution/services/agent-runner.service.ts+++ b/packages/twenty-server/src/engine/metadata-modules/ai/ai-agent-execution/services/agent-runner.service.ts@@ -17,6 +17,7 @@ import { AGENT_WAIT_PROMPT } from 'src/engine/metadata-modules/ai/ai-agent-execuimport { createAgentWaitTools } from 'src/engine/metadata-modules/ai/ai-agent-execution/pausing-tools/wait-tools/create-agent-wait-tools.util';import { findAgentRunWait } from 'src/engine/metadata-modules/ai/ai-agent-execution/pausing-tools/wait-tools/find-agent-run-wait.util';import { AgentRunCallerHandlerRegistryService } from 'src/engine/metadata-modules/ai/ai-agent-execution/services/agent-run-caller-handler-registry.service';+import { AgentRunJobScopeService } from 'src/engine/metadata-modules/ai/ai-agent-execution/services/agent-run-job-scope.service';import { AgentRunConversationService } from 'src/engine/metadata-modules/ai/ai-agent-execution/services/agent-run-conversation.service';import { AgentRunSuspensionService } from 'src/engine/metadata-modules/ai/ai-agent-execution/services/agent-run-suspension.service';import { AgentRunTurnService } from 'src/engine/metadata-modules/ai/ai-agent-execution/services/agent-run-turn.service';@@ -81,6 +82,7 @@ export class AgentRunnerService {private readonly streamRecoveryService: AgentChatStreamRecoveryService,private readonly eventPublisherService: AgentChatEventPublisherService,private readonly waitingRunService: AgentChatWaitingRunService,+ private readonly agentRunJobScopeService: AgentRunJobScopeService,@InjectMessageQueue(MessageQueue.aiQueue)private readonly messageQueueService: MessageQueueService,@InjectWorkspaceScopedRepository(AgentEntity)@@ -91,6 +93,10 @@ export class AgentRunnerService {await this.messageQueueService.add<RunAgentJobData>(RUN_AGENT_JOB_NAME,input,+ await this.agentRunJobScopeService.resolve({+ workspaceId: input.workspaceId,+ agentId: input.spec.agentId,+ }),);}@@ -137,6 +143,10 @@ export class AgentRunnerService {run: {jobName: RUN_AGENT_JOB_NAME,jobData: { ...jobData, parkId: undefined },+ scope: await this.agentRunJobScopeService.resolve({+ workspaceId,+ agentId: jobData.spec.agentId,+ }),},});@@ -282,6 +292,10 @@ export class AgentRunnerService {run: {jobName: CONTINUE_AGENT_RUN_JOB_NAME,jobData: { ...jobData, parkId: undefined },+ scope: await this.agentRunJobScopeService.resolve({+ workspaceId,+ agentId: suspension.runSpec.agentId,+ }),},});diff --git a/packages/twenty-server/src/engine/metadata-modules/ai/ai-agent-trigger/crons/__tests__/agent-cron-trigger.cron.job.spec.ts b/packages/twenty-server/src/engine/metadata-modules/ai/ai-agent-trigger/crons/__tests__/agent-cron-trigger.cron.job.spec.tsindex d2910a4960a..6e03e1cdb82 100644--- a/packages/twenty-server/src/engine/metadata-modules/ai/ai-agent-trigger/crons/__tests__/agent-cron-trigger.cron.job.spec.ts+++ b/packages/twenty-server/src/engine/metadata-modules/ai/ai-agent-trigger/crons/__tests__/agent-cron-trigger.cron.job.spec.ts@@ -19,6 +19,7 @@ import { WorkspaceCacheService } from 'src/engine/workspace-cache/services/worksconst WORKSPACE_ID = 'workspace-id';const AGENT_ID = 'agent-id';const TRIGGER_ID = '0d2b1a8c-77a4-4e2e-8f0c-3a8e9f6b4c22';+const APPLICATION_UNIVERSAL_IDENTIFIER = 'agent-application';const buildCronTrigger = (isActive = true): AgentTrigger => ({id: TRIGGER_ID,@@ -39,6 +40,7 @@ const buildFlatAgentMaps = ({flatEntity: {id: AGENT_ID,universalIdentifier: AGENT_ID,+ applicationUniversalIdentifier: APPLICATION_UNIVERSAL_IDENTIFIER,workspaceId: WORKSPACE_ID,triggers,isActive: true,@@ -120,6 +122,10 @@ describe('AgentCronTriggerCronJob', () => {triggerId: TRIGGER_ID,payload: { type: 'CRON', firedAt: expect.any(String) },}),+ {+ workspaceId: WORKSPACE_ID,+ applicationUniversalIdentifier: APPLICATION_UNIVERSAL_IDENTIFIER,+ },);});diff --git a/packages/twenty-server/src/engine/metadata-modules/ai/ai-agent-trigger/crons/agent-cron-trigger.cron.job.ts b/packages/twenty-server/src/engine/metadata-modules/ai/ai-agent-trigger/crons/agent-cron-trigger.cron.job.tsindex b6c0c3907a1..720479ccd66 100644--- a/packages/twenty-server/src/engine/metadata-modules/ai/ai-agent-trigger/crons/agent-cron-trigger.cron.job.ts+++ b/packages/twenty-server/src/engine/metadata-modules/ai/ai-agent-trigger/crons/agent-cron-trigger.cron.job.ts@@ -118,6 +118,11 @@ export class AgentCronTriggerCronJob {triggerId: trigger.id,payload: { type: 'CRON', firedAt: now.toISOString() },},+ {+ workspaceId,+ applicationUniversalIdentifier:+ flatAgent.applicationUniversalIdentifier,+ },);}}diff --git a/packages/twenty-server/src/engine/metadata-modules/ai/ai-agent-trigger/jobs/call-agent-database-event-triggers.job.ts b/packages/twenty-server/src/engine/metadata-modules/ai/ai-agent-trigger/jobs/call-agent-database-event-triggers.job.tsindex 7f0d57bb085..add271b81bd 100644--- a/packages/twenty-server/src/engine/metadata-modules/ai/ai-agent-trigger/jobs/call-agent-database-event-triggers.job.ts+++ b/packages/twenty-server/src/engine/metadata-modules/ai/ai-agent-trigger/jobs/call-agent-database-event-triggers.job.ts@@ -172,6 +172,9 @@ export class CallAgentDatabaseEventTriggersJob {events,},},+ workspaceId,+ applicationUniversalIdentifier:+ flatAgent.applicationUniversalIdentifier,})),);}diff --git a/packages/twenty-server/src/engine/metadata-modules/ai/ai-chat/services/__tests__/agent-chat-waiting-run.service.spec.ts b/packages/twenty-server/src/engine/metadata-modules/ai/ai-chat/services/__tests__/agent-chat-waiting-run.service.spec.tsnew file mode 100644index 00000000000..2b634f875ef--- /dev/null+++ b/packages/twenty-server/src/engine/metadata-modules/ai/ai-chat/services/__tests__/agent-chat-waiting-run.service.spec.ts@@ -0,0 +1,65 @@+import { AgentChatWaitingRunService } from 'src/engine/metadata-modules/ai/ai-chat/services/agent-chat-waiting-run.service';++const WORKSPACE_ID = 'workspace-id';++const buildService = (payload: object) => {+ const messageQueueService = { add: jest.fn().mockResolvedValue('job-id') };++ const service = new AgentChatWaitingRunService(+ {+ findOne: jest+ .fn()+ .mockResolvedValue({ id: 'thread-id', activeStreamId: null }),+ } as never,+ {} as never,+ { claim: jest.fn().mockResolvedValue({ payload }) } as never,+ {} as never,+ messageQueueService as never,+ {} as never,+ );++ return { service, messageQueueService };+};++const wakeUp = {+ id: 'wake-up-id',+ workspaceId: WORKSPACE_ID,+ ownerId: 'thread-id',+};++describe('AgentChatWaitingRunService resolve', () => {+ it('requeues a waiting run with the scope it was parked with', async () => {+ const scope = {+ workspaceId: WORKSPACE_ID,+ applicationUniversalIdentifier: 'application-a',+ };+ const { service, messageQueueService } = buildService({+ jobName: 'RunAgentJob',+ jobData: { threadId: 'thread-id' },+ scope,+ });++ await service.resolve({ wakeUp: wakeUp as never });++ expect(messageQueueService.add).toHaveBeenCalledWith(+ 'RunAgentJob',+ { threadId: 'thread-id' },+ scope,+ );+ });++ it('requeues a run parked without a scope as a workspace job', async () => {+ const { service, messageQueueService } = buildService({+ jobName: 'RunAgentJob',+ jobData: { threadId: 'thread-id' },+ });++ await service.resolve({ wakeUp: wakeUp as never });++ expect(messageQueueService.add).toHaveBeenCalledWith(+ 'RunAgentJob',+ { threadId: 'thread-id' },+ { workspaceId: WORKSPACE_ID, isWorkspaceJob: true },+ );+ });+});diff --git a/packages/twenty-server/src/engine/metadata-modules/ai/ai-chat/services/agent-chat-channel-grant.service.ts b/packages/twenty-server/src/engine/metadata-modules/ai/ai-chat/services/agent-chat-channel-grant.service.tsindex 88383966883..696acbde97f 100644--- a/packages/twenty-server/src/engine/metadata-modules/ai/ai-chat/services/agent-chat-channel-grant.service.ts+++ b/packages/twenty-server/src/engine/metadata-modules/ai/ai-chat/services/agent-chat-channel-grant.service.ts@@ -39,7 +39,11 @@ export class AgentChatChannelGrantService {await this.workspaceQueueService.add<RewriteAgentChatChannelGrantsJobData>(REWRITE_AGENT_CHAT_CHANNEL_GRANTS_JOB_NAME,data,- AGENT_CHAT_CHANNEL_GRANTS_REWRITE_JOB_OPTIONS,+ {+ ...AGENT_CHAT_CHANNEL_GRANTS_REWRITE_JOB_OPTIONS,+ workspaceId: data.workspaceId,+ isWorkspaceJob: true,+ },);}diff --git a/packages/twenty-server/src/engine/metadata-modules/ai/ai-chat/services/agent-chat-stream-recovery.service.ts b/packages/twenty-server/src/engine/metadata-modules/ai/ai-chat/services/agent-chat-stream-recovery.service.tsindex 6eac4fa9c56..b75790ec96b 100644--- a/packages/twenty-server/src/engine/metadata-modules/ai/ai-chat/services/agent-chat-stream-recovery.service.ts+++ b/packages/twenty-server/src/engine/metadata-modules/ai/ai-chat/services/agent-chat-stream-recovery.service.ts@@ -205,6 +205,10 @@ export class AgentChatStreamRecoveryService {await this.chatMessageQueueService.add<FlushQueuedAgentChatMessagesJobData>(FLUSH_QUEUED_AGENT_CHAT_MESSAGES_JOB_NAME,{ workspaceId, threadId },+ {+ workspaceId,+ isWorkspaceJob: true,+ },);} else if (isDefined(nextRun)) {await this.pendingWakeUpService.scheduleResolution({ wakeUp: nextRun });diff --git a/packages/twenty-server/src/engine/metadata-modules/ai/ai-chat/services/agent-chat-streaming.service.ts b/packages/twenty-server/src/engine/metadata-modules/ai/ai-chat/services/agent-chat-streaming.service.tsindex 5cfa396b349..5a7fdafeb35 100644--- a/packages/twenty-server/src/engine/metadata-modules/ai/ai-chat/services/agent-chat-streaming.service.ts+++ b/packages/twenty-server/src/engine/metadata-modules/ai/ai-chat/services/agent-chat-streaming.service.ts@@ -479,6 +479,10 @@ export class AgentChatStreamingService {hasTitle: isNonEmptyString(thread.title),conversationSizeTokens: thread.conversationSize,},+ {+ workspaceId,+ isWorkspaceJob: true,+ },);}diff --git a/packages/twenty-server/src/engine/metadata-modules/ai/ai-chat/services/agent-chat-thread-participant.service.ts b/packages/twenty-server/src/engine/metadata-modules/ai/ai-chat/services/agent-chat-thread-participant.service.tsindex af5d3e35501..58eb847d129 100644--- a/packages/twenty-server/src/engine/metadata-modules/ai/ai-chat/services/agent-chat-thread-participant.service.ts+++ b/packages/twenty-server/src/engine/metadata-modules/ai/ai-chat/services/agent-chat-thread-participant.service.ts@@ -546,7 +546,12 @@ export class AgentChatThreadParticipantService {await this.delayedJobsQueueService.add<EndAgentChatThreadSnoozeJobData>(END_AGENT_CHAT_THREAD_SNOOZE_JOB_NAME,data,- { delay, ...AGENT_CHAT_THREAD_SNOOZE_END_JOB_RETRY_OPTIONS },+ {+ delay,+ ...AGENT_CHAT_THREAD_SNOOZE_END_JOB_RETRY_OPTIONS,+ workspaceId: data.workspaceId,+ isWorkspaceJob: true,+ },);}diff --git a/packages/twenty-server/src/engine/metadata-modules/ai/ai-chat/services/agent-chat-thread-triage.service.ts b/packages/twenty-server/src/engine/metadata-modules/ai/ai-chat/services/agent-chat-thread-triage.service.tsindex c962645f929..8fb9b70c643 100644--- a/packages/twenty-server/src/engine/metadata-modules/ai/ai-chat/services/agent-chat-thread-triage.service.ts+++ b/packages/twenty-server/src/engine/metadata-modules/ai/ai-chat/services/agent-chat-thread-triage.service.ts@@ -349,7 +349,12 @@ export class AgentChatThreadTriageService {await this.delayedJobsQueueService.add<EndAgentChatChannelSnoozeJobData>(END_AGENT_CHAT_CHANNEL_SNOOZE_JOB_NAME,data,- { delay, ...AGENT_CHAT_THREAD_SNOOZE_END_JOB_RETRY_OPTIONS },+ {+ delay,+ ...AGENT_CHAT_THREAD_SNOOZE_END_JOB_RETRY_OPTIONS,+ workspaceId: data.workspaceId,+ isWorkspaceJob: true,+ },);}}diff --git a/packages/twenty-server/src/engine/metadata-modules/ai/ai-chat/services/agent-chat-waiting-run.service.ts b/packages/twenty-server/src/engine/metadata-modules/ai/ai-chat/services/agent-chat-waiting-run.service.tsindex 5e0106897ab..3e54c96dd4f 100644--- a/packages/twenty-server/src/engine/metadata-modules/ai/ai-chat/services/agent-chat-waiting-run.service.ts+++ b/packages/twenty-server/src/engine/metadata-modules/ai/ai-chat/services/agent-chat-waiting-run.service.ts@@ -6,6 +6,7 @@ import { IsNull } from 'typeorm';import { v4 } from 'uuid';import { InjectMessageQueue } from 'src/engine/core-modules/message-queue/decorators/message-queue.decorator';+import { type QueueJobScope } from 'src/engine/core-modules/message-queue/drivers/interfaces/job-options.interface';import { MessageQueue } from 'src/engine/core-modules/message-queue/message-queue.constants';import { MessageQueueService } from 'src/engine/core-modules/message-queue/services/message-queue.service';import { type PendingWakeUpEntity } from 'src/engine/core-modules/pending-wake-up/entities/pending-wake-up.entity';@@ -20,7 +21,11 @@ import { AgentHistoryRepository } from 'src/engine/metadata-modules/ai/ai-historimport { InjectAgentHistoryRepository } from 'src/engine/metadata-modules/ai/ai-history/repositories/inject-agent-history-repository.decorator';import { type AgentChatThreadWorkspaceEntity } from 'src/engine/metadata-modules/ai/ai-history/standard-objects/agent-chat-thread.workspace-entity';-type AgentChatWaitingRun = { jobName: string; jobData: object };+type AgentChatWaitingRun = {+ jobName: string;+ jobData: object;+ scope?: QueueJobScope;+};// the recheck reaps a holder whose worker died, which nothing else releasesconst buildRecheckAt = () =>@@ -140,7 +145,11 @@ export class AgentChatWaitingRunServiceconst run = claimedWakeUp.payload as AgentChatWaitingRun;try {- await this.messageQueueService.add(run.jobName, run.jobData);+ await this.messageQueueService.add(+ run.jobName,+ run.jobData,+ run.scope ?? { workspaceId, isWorkspaceJob: true },+ );} catch (error) {await this.wait({ workspaceId, threadId, run });diff --git a/packages/twenty-server/src/engine/metadata-modules/navigation-menu-item/listeners/navigation-menu-item-deletion.listener.ts b/packages/twenty-server/src/engine/metadata-modules/navigation-menu-item/listeners/navigation-menu-item-deletion.listener.tsindex 3b72c88e58e..53a7e956d29 100644--- a/packages/twenty-server/src/engine/metadata-modules/navigation-menu-item/listeners/navigation-menu-item-deletion.listener.ts+++ b/packages/twenty-server/src/engine/metadata-modules/navigation-menu-item/listeners/navigation-menu-item-deletion.listener.ts@@ -36,6 +36,10 @@ export class NavigationMenuItemDeletionListener {workspaceId: payload.workspaceId,deletedRecordIds,},+ {+ workspaceId: payload.workspaceId,+ isWorkspaceJob: true,+ },);}}diff --git a/packages/twenty-server/src/engine/metadata-modules/webhook/jobs/call-webhook-jobs-for-metadata.job.ts b/packages/twenty-server/src/engine/metadata-modules/webhook/jobs/call-webhook-jobs-for-metadata.job.tsindex 46f8057570a..d54dadb95d5 100644--- a/packages/twenty-server/src/engine/metadata-modules/webhook/jobs/call-webhook-jobs-for-metadata.job.ts+++ b/packages/twenty-server/src/engine/metadata-modules/webhook/jobs/call-webhook-jobs-for-metadata.job.ts@@ -75,7 +75,11 @@ export class CallWebhookJobsForMetadataJob {await this.messageQueueService.add<CallMetadataWebhookJobData[]>(CallWebhookJob.name,webhookEventsChunk,- { retryLimit: 3 },+ {+ retryLimit: 3,+ workspaceId: metadataEventBatch.workspaceId,+ isWorkspaceJob: true,+ },);}}diff --git a/packages/twenty-server/src/engine/metadata-modules/webhook/jobs/call-webhook-jobs.job.ts b/packages/twenty-server/src/engine/metadata-modules/webhook/jobs/call-webhook-jobs.job.tsindex 6339d460a61..97da56ca153 100644--- a/packages/twenty-server/src/engine/metadata-modules/webhook/jobs/call-webhook-jobs.job.ts+++ b/packages/twenty-server/src/engine/metadata-modules/webhook/jobs/call-webhook-jobs.job.ts@@ -97,7 +97,11 @@ export class CallWebhookJobsJob {await this.messageQueueService.add<CallWebhookJobData[]>(CallWebhookJob.name,webhookEventsChunk,- { retryLimit: 3 },+ {+ retryLimit: 3,+ workspaceId: workspaceEventBatch.workspaceId,+ isWorkspaceJob: true,+ },);}}diff --git a/packages/twenty-server/src/engine/subscriptions/metadata-event/metadata-events-to-db.listener.ts b/packages/twenty-server/src/engine/subscriptions/metadata-event/metadata-events-to-db.listener.tsindex c27c7384f4b..8dbc34f46c9 100644--- a/packages/twenty-server/src/engine/subscriptions/metadata-event/metadata-events-to-db.listener.ts+++ b/packages/twenty-server/src/engine/subscriptions/metadata-event/metadata-events-to-db.listener.ts@@ -54,6 +54,8 @@ export class MetadataEventsToDbListener {MetadataEventBatch<AllMetadataName, AllMetadataEventType>>(CallWebhookJobsForMetadataJob.name, metadataEventBatch, {retryLimit: 3,+ workspaceId: metadataEventBatch.workspaceId,+ isWorkspaceJob: true,});if (metadataEventBatch.events.length > 0) {diff --git a/packages/twenty-server/src/engine/trash-cleanup/crons/trash-cleanup.cron.job.ts b/packages/twenty-server/src/engine/trash-cleanup/crons/trash-cleanup.cron.job.tsindex 6a9d026f7d9..4b2f25695aa 100644--- a/packages/twenty-server/src/engine/trash-cleanup/crons/trash-cleanup.cron.job.ts+++ b/packages/twenty-server/src/engine/trash-cleanup/crons/trash-cleanup.cron.job.ts@@ -54,6 +54,10 @@ export class TrashCleanupCronJob {workspaceId: workspace.id,trashRetentionDays: workspace.trashRetentionDays,},+ {+ workspaceId: workspace.id,+ isWorkspaceJob: true,+ },);} catch (error) {this.exceptionHandlerService.captureExceptions([error], {diff --git a/packages/twenty-server/src/engine/workspace-manager/workspace-cleaner/services/cleaner.workspace-service.ts b/packages/twenty-server/src/engine/workspace-manager/workspace-cleaner/services/cleaner.workspace-service.tsindex 1099cc49af2..b91584c9cb1 100644--- a/packages/twenty-server/src/engine/workspace-manager/workspace-cleaner/services/cleaner.workspace-service.ts+++ b/packages/twenty-server/src/engine/workspace-manager/workspace-cleaner/services/cleaner.workspace-service.ts@@ -110,7 +110,11 @@ export class CleanerWorkspaceService {return this.workspaceDestroyQueueService.add<DestroySoftDeletedWorkspaceJobData>(DestroySoftDeletedWorkspaceJob.name,{ workspaceId },- { id: `destroy-soft-deleted-workspace-${workspaceId}` },+ {+ id: `destroy-soft-deleted-workspace-${workspaceId}`,+ workspaceId,+ isWorkspaceJob: true,+ },);}diff --git a/packages/twenty-server/src/engine/workspace-manager/workspace-migration/workspace-migration-runner/services/deferred-workspace-migration-action-runner.service.ts b/packages/twenty-server/src/engine/workspace-manager/workspace-migration/workspace-migration-runner/services/deferred-workspace-migration-action-runner.service.tsindex f0c39d98e9f..baae79921bf 100644--- a/packages/twenty-server/src/engine/workspace-manager/workspace-migration/workspace-migration-runner/services/deferred-workspace-migration-action-runner.service.ts+++ b/packages/twenty-server/src/engine/workspace-manager/workspace-migration/workspace-migration-runner/services/deferred-workspace-migration-action-runner.service.ts@@ -164,6 +164,8 @@ export class DeferredWorkspaceMigrationActionRunnerService {retryLimit: DEFERRED_WORKSPACE_MIGRATION_ACTION_MAX_ATTEMPTS - 1,backoff: DEFERRED_WORKSPACE_MIGRATION_ACTION_RETRY_BACKOFF,priority: DEFERRED_WORKSPACE_MIGRATION_ACTION_JOB_PRIORITY,+ workspaceId,+ isWorkspaceJob: true,},);} catch (error) {diff --git a/packages/twenty-server/src/modules/calendar/blocklist-manager/listeners/calendar-blocklist.listener.ts b/packages/twenty-server/src/modules/calendar/blocklist-manager/listeners/calendar-blocklist.listener.tsindex 62df51593cc..5bb6182af76 100644--- a/packages/twenty-server/src/modules/calendar/blocklist-manager/listeners/calendar-blocklist.listener.ts+++ b/packages/twenty-server/src/modules/calendar/blocklist-manager/listeners/calendar-blocklist.listener.ts@@ -38,6 +38,10 @@ export class CalendarBlocklistListener {await this.messageQueueService.add<BlocklistItemDeleteCalendarEventsJobData>(BlocklistItemDeleteCalendarEventsJob.name,payload,+ {+ workspaceId: payload.workspaceId,+ isWorkspaceJob: true,+ },);}@@ -50,6 +54,10 @@ export class CalendarBlocklistListener {await this.messageQueueService.add<BlocklistReimportCalendarEventsJobData>(BlocklistReimportCalendarEventsJob.name,payload,+ {+ workspaceId: payload.workspaceId,+ isWorkspaceJob: true,+ },);}@@ -62,11 +70,19 @@ export class CalendarBlocklistListener {await this.messageQueueService.add<BlocklistItemDeleteCalendarEventsJobData>(BlocklistItemDeleteCalendarEventsJob.name,payload,+ {+ workspaceId: payload.workspaceId,+ isWorkspaceJob: true,+ },);await this.messageQueueService.add<BlocklistReimportCalendarEventsJobData>(BlocklistReimportCalendarEventsJob.name,payload,+ {+ workspaceId: payload.workspaceId,+ isWorkspaceJob: true,+ },);}}diff --git a/packages/twenty-server/src/modules/calendar/calendar-event-cleaner/listeners/calendar-event-cleaner-calendar-channel.listener.ts b/packages/twenty-server/src/modules/calendar/calendar-event-cleaner/listeners/calendar-event-cleaner-calendar-channel.listener.tsindex e1dfdbbdb2a..f55379c692a 100644--- a/packages/twenty-server/src/modules/calendar/calendar-event-cleaner/listeners/calendar-event-cleaner-calendar-channel.listener.ts+++ b/packages/twenty-server/src/modules/calendar/calendar-event-cleaner/listeners/calendar-event-cleaner-calendar-channel.listener.ts@@ -39,6 +39,10 @@ export class CalendarEventCleanerCalendarChannelListener {workspaceId,calendarChannelId: event.calendarChannelId,},+ {+ workspaceId,+ isWorkspaceJob: true,+ },),),);diff --git a/packages/twenty-server/src/modules/calendar/calendar-event-cleaner/listeners/calendar-event-cleaner-connected-account.listener.ts b/packages/twenty-server/src/modules/calendar/calendar-event-cleaner/listeners/calendar-event-cleaner-connected-account.listener.tsindex 1dccf65762f..bc85dd38095 100644--- a/packages/twenty-server/src/modules/calendar/calendar-event-cleaner/listeners/calendar-event-cleaner-connected-account.listener.ts+++ b/packages/twenty-server/src/modules/calendar/calendar-event-cleaner/listeners/calendar-event-cleaner-connected-account.listener.ts@@ -39,6 +39,10 @@ export class CalendarEventCleanerConnectedAccountListener {workspaceId,connectedAccountId: event.connectedAccountId,},+ {+ workspaceId,+ isWorkspaceJob: true,+ },),),);diff --git a/packages/twenty-server/src/modules/calendar/calendar-event-import-manager/commands/calendar-trigger-event-list-fetch.command.ts b/packages/twenty-server/src/modules/calendar/calendar-event-import-manager/commands/calendar-trigger-event-list-fetch.command.tsindex 2649821fde5..9b6804ce937 100644--- a/packages/twenty-server/src/modules/calendar/calendar-event-import-manager/commands/calendar-trigger-event-list-fetch.command.ts+++ b/packages/twenty-server/src/modules/calendar/calendar-event-import-manager/commands/calendar-trigger-event-list-fetch.command.ts@@ -93,6 +93,10 @@ export class CalendarTriggerEventListFetchCommand extends CommandRunner {calendarChannelId: calendarChannel.id,workspaceId,},+ {+ workspaceId,+ isWorkspaceJob: true,+ },);this.logger.log(diff --git a/packages/twenty-server/src/modules/calendar/calendar-event-import-manager/crons/jobs/__tests__/calendar-ongoing-stale.cron.job.spec.ts b/packages/twenty-server/src/modules/calendar/calendar-event-import-manager/crons/jobs/__tests__/calendar-ongoing-stale.cron.job.spec.tsindex 359948eba0e..0836e7ae020 100644--- a/packages/twenty-server/src/modules/calendar/calendar-event-import-manager/crons/jobs/__tests__/calendar-ongoing-stale.cron.job.spec.ts+++ b/packages/twenty-server/src/modules/calendar/calendar-event-import-manager/crons/jobs/__tests__/calendar-ongoing-stale.cron.job.spec.ts@@ -41,6 +41,10 @@ describe('CalendarOngoingStaleCronJob', () => {{workspaceId: 'workspace-1',},+ {+ workspaceId: 'workspace-1',+ isWorkspaceJob: true,+ },);expect(messageQueueService.add).toHaveBeenNthCalledWith(2,@@ -48,6 +52,10 @@ describe('CalendarOngoingStaleCronJob', () => {{workspaceId: 'workspace-2',},+ {+ workspaceId: 'workspace-2',+ isWorkspaceJob: true,+ },);});diff --git a/packages/twenty-server/src/modules/calendar/calendar-event-import-manager/crons/jobs/calendar-event-list-fetch.cron.job.ts b/packages/twenty-server/src/modules/calendar/calendar-event-import-manager/crons/jobs/calendar-event-list-fetch.cron.job.tsindex 53331a98d69..2faaf4fb43c 100644--- a/packages/twenty-server/src/modules/calendar/calendar-event-import-manager/crons/jobs/calendar-event-list-fetch.cron.job.ts+++ b/packages/twenty-server/src/modules/calendar/calendar-event-import-manager/crons/jobs/calendar-event-list-fetch.cron.job.ts@@ -127,6 +127,10 @@ export class CalendarEventListFetchCronJob {calendarChannelId,workspaceId,},+ {+ workspaceId,+ isWorkspaceJob: true,+ },);}} catch (error) {diff --git a/packages/twenty-server/src/modules/calendar/calendar-event-import-manager/crons/jobs/calendar-events-import.cron.job.ts b/packages/twenty-server/src/modules/calendar/calendar-event-import-manager/crons/jobs/calendar-events-import.cron.job.tsindex 543e6f8350a..4034aebcfa5 100644--- a/packages/twenty-server/src/modules/calendar/calendar-event-import-manager/crons/jobs/calendar-events-import.cron.job.ts+++ b/packages/twenty-server/src/modules/calendar/calendar-event-import-manager/crons/jobs/calendar-events-import.cron.job.ts@@ -117,6 +117,10 @@ export class CalendarEventsImportCronJob {calendarChannelId,workspaceId,},+ {+ workspaceId,+ isWorkspaceJob: true,+ },);}} catch (error) {diff --git a/packages/twenty-server/src/modules/calendar/calendar-event-import-manager/crons/jobs/calendar-ongoing-stale.cron.job.ts b/packages/twenty-server/src/modules/calendar/calendar-event-import-manager/crons/jobs/calendar-ongoing-stale.cron.job.tsindex dbb9fc8d4c7..66257b8d865 100644--- a/packages/twenty-server/src/modules/calendar/calendar-event-import-manager/crons/jobs/calendar-ongoing-stale.cron.job.ts+++ b/packages/twenty-server/src/modules/calendar/calendar-event-import-manager/crons/jobs/calendar-ongoing-stale.cron.job.ts@@ -45,6 +45,10 @@ export class CalendarOngoingStaleCronJob {{workspaceId,},+ {+ workspaceId,+ isWorkspaceJob: true,+ },);} catch (error) {this.exceptionHandlerService.captureExceptions([error], {diff --git a/packages/twenty-server/src/modules/calendar/calendar-event-import-manager/crons/jobs/calendar-relaunch-failed-calendar-channels.cron.job.ts b/packages/twenty-server/src/modules/calendar/calendar-event-import-manager/crons/jobs/calendar-relaunch-failed-calendar-channels.cron.job.tsindex 34924c57ab3..8b8fd5f90c2 100644--- a/packages/twenty-server/src/modules/calendar/calendar-event-import-manager/crons/jobs/calendar-relaunch-failed-calendar-channels.cron.job.ts+++ b/packages/twenty-server/src/modules/calendar/calendar-event-import-manager/crons/jobs/calendar-relaunch-failed-calendar-channels.cron.job.ts@@ -78,6 +78,10 @@ export class CalendarRelaunchFailedCalendarChannelsCronJob {workspaceId: calendarChannel.workspaceId,calendarChannelId: calendarChannel.id,},+ {+ workspaceId: calendarChannel.workspaceId,+ isWorkspaceJob: true,+ },);} catch (error) {this.exceptionHandlerService.captureExceptions([error], {diff --git a/packages/twenty-server/src/modules/calendar/calendar-event-participant-manager/listeners/calendar-event-participant-person.listener.ts b/packages/twenty-server/src/modules/calendar/calendar-event-participant-manager/listeners/calendar-event-participant-person.listener.tsindex 5a358f6735b..2d933b02245 100644--- a/packages/twenty-server/src/modules/calendar/calendar-event-participant-manager/listeners/calendar-event-participant-person.listener.ts+++ b/packages/twenty-server/src/modules/calendar/calendar-event-participant-manager/listeners/calendar-event-participant-person.listener.ts@@ -55,6 +55,10 @@ export class CalendarEventParticipantPersonListener {workspaceMemberIds: [],},},+ {+ workspaceId: payload.workspaceId,+ isWorkspaceJob: true,+ },);}@@ -89,6 +93,10 @@ export class CalendarEventParticipantPersonListener {workspaceMemberIds: [],},},+ {+ workspaceId: payload.workspaceId,+ isWorkspaceJob: true,+ },);}@@ -115,6 +123,10 @@ export class CalendarEventParticipantPersonListener {workspaceMemberIds: [],},},+ {+ workspaceId: payload.workspaceId,+ isWorkspaceJob: true,+ },);}}diff --git a/packages/twenty-server/src/modules/calendar/calendar-event-participant-manager/listeners/calendar-event-participant-workspace-member.listener.ts b/packages/twenty-server/src/modules/calendar/calendar-event-participant-manager/listeners/calendar-event-participant-workspace-member.listener.tsindex 73242bfa05f..83735f928c4 100644--- a/packages/twenty-server/src/modules/calendar/calendar-event-participant-manager/listeners/calendar-event-participant-workspace-member.listener.ts+++ b/packages/twenty-server/src/modules/calendar/calendar-event-participant-manager/listeners/calendar-event-participant-workspace-member.listener.ts@@ -45,6 +45,10 @@ export class CalendarEventParticipantWorkspaceMemberListener {workspaceMemberIds: [eventPayload.recordId],},},+ {+ workspaceId: payload.workspaceId,+ isWorkspaceJob: true,+ },);}}@@ -71,6 +75,10 @@ export class CalendarEventParticipantWorkspaceMemberListener {workspaceMemberIds: [eventPayload.recordId],},},+ {+ workspaceId: payload.workspaceId,+ isWorkspaceJob: true,+ },);}}diff --git a/packages/twenty-server/src/modules/calendar/calendar-event-participant-manager/services/calendar-event-participant.service.ts b/packages/twenty-server/src/modules/calendar/calendar-event-participant-manager/services/calendar-event-participant.service.tsindex bd7ed8d1679..b864ec59b1a 100644--- a/packages/twenty-server/src/modules/calendar/calendar-event-participant-manager/services/calendar-event-participant.service.ts+++ b/packages/twenty-server/src/modules/calendar/calendar-event-participant-manager/services/calendar-event-participant.service.ts@@ -136,6 +136,10 @@ export class CalendarEventParticipantService {})),source: FieldActorSource.CALENDAR,},+ {+ workspaceId,+ isWorkspaceJob: true,+ },);}diff --git a/packages/twenty-server/src/modules/connected-account/channel-sync/services/channel-sync.service.ts b/packages/twenty-server/src/modules/connected-account/channel-sync/services/channel-sync.service.tsindex a35c760e128..f41bce07f82 100644--- a/packages/twenty-server/src/modules/connected-account/channel-sync/services/channel-sync.service.ts+++ b/packages/twenty-server/src/modules/connected-account/channel-sync/services/channel-sync.service.ts@@ -93,6 +93,10 @@ export class ChannelSyncService {workspaceId,messageChannelId: messageChannel.id,},+ {+ workspaceId,+ isWorkspaceJob: true,+ },);if (@@ -111,7 +115,11 @@ export class ChannelSyncService {channelId: messageChannel.id,workspaceId,},- { retryLimit: WEBHOOK_SUBSCRIPTION_JOB_RETRY_LIMIT },+ {+ retryLimit: WEBHOOK_SUBSCRIPTION_JOB_RETRY_LIMIT,+ workspaceId,+ isWorkspaceJob: true,+ },);} catch (error) {this.logger.warn(@@ -154,6 +162,10 @@ export class ChannelSyncService {workspaceId,calendarChannelId: calendarChannel.id,},+ {+ workspaceId,+ isWorkspaceJob: true,+ },);if (@@ -172,7 +184,11 @@ export class ChannelSyncService {channelId: calendarChannel.id,workspaceId,},- { retryLimit: WEBHOOK_SUBSCRIPTION_JOB_RETRY_LIMIT },+ {+ retryLimit: WEBHOOK_SUBSCRIPTION_JOB_RETRY_LIMIT,+ workspaceId,+ isWorkspaceJob: true,+ },);} catch (error) {this.logger.warn(diff --git a/packages/twenty-server/src/modules/connected-account/services/imap-smtp-caldav-apis.service.ts b/packages/twenty-server/src/modules/connected-account/services/imap-smtp-caldav-apis.service.tsindex 8c077902108..2d1b6fe700a 100644--- a/packages/twenty-server/src/modules/connected-account/services/imap-smtp-caldav-apis.service.ts+++ b/packages/twenty-server/src/modules/connected-account/services/imap-smtp-caldav-apis.service.ts@@ -227,6 +227,10 @@ export class ImapSmtpCalDavApiService {await this.messageQueueService.add<MessagingMessageListFetchJobData>(MessagingMessageListFetchJob.name,{ workspaceId, messageChannelId: existingMessageChannel.id },+ {+ workspaceId,+ isWorkspaceJob: true,+ },);}@@ -244,6 +248,10 @@ export class ImapSmtpCalDavApiService {await this.calendarQueueService.add<CalendarEventListFetchJobData>(CalendarEventListFetchJob.name,{ workspaceId, calendarChannelId: existingCalendarChannel.id },+ {+ workspaceId,+ isWorkspaceJob: true,+ },);}diff --git a/packages/twenty-server/src/modules/connected-account/webhook-subscription-manager/commands/create-webhook-subscription-for-connected-account.command.ts b/packages/twenty-server/src/modules/connected-account/webhook-subscription-manager/commands/create-webhook-subscription-for-connected-account.command.tsindex 2071bf7cee1..7fce53b032d 100644--- a/packages/twenty-server/src/modules/connected-account/webhook-subscription-manager/commands/create-webhook-subscription-for-connected-account.command.ts+++ b/packages/twenty-server/src/modules/connected-account/webhook-subscription-manager/commands/create-webhook-subscription-for-connected-account.command.ts@@ -121,6 +121,8 @@ export class CreateWebhookSubscriptionForConnectedAccountCommand extends Provisi{delay: this.enqueueCursorMs,retryLimit: WEBHOOK_SUBSCRIPTION_JOB_RETRY_LIMIT,+ workspaceId,+ isWorkspaceJob: true,},);diff --git a/packages/twenty-server/src/modules/connected-account/webhook-subscription-manager/crons/jobs/webhook-subscription-renewal.cron.job.ts b/packages/twenty-server/src/modules/connected-account/webhook-subscription-manager/crons/jobs/webhook-subscription-renewal.cron.job.tsindex 8ba341185ab..ab0a138b53c 100644--- a/packages/twenty-server/src/modules/connected-account/webhook-subscription-manager/crons/jobs/webhook-subscription-renewal.cron.job.ts+++ b/packages/twenty-server/src/modules/connected-account/webhook-subscription-manager/crons/jobs/webhook-subscription-renewal.cron.job.ts@@ -279,6 +279,8 @@ export class WebhookSubscriptionRenewalCronJob {{retryLimit: WEBHOOK_SUBSCRIPTION_JOB_RETRY_LIMIT,delay: Math.round(index * spacingMs),+ workspaceId: channel.workspaceId,+ isWorkspaceJob: true,},);}diff --git a/packages/twenty-server/src/modules/connected-account/webhook-subscription-manager/listeners/webhook-subscription-workspace-activation.listener.ts b/packages/twenty-server/src/modules/connected-account/webhook-subscription-manager/listeners/webhook-subscription-workspace-activation.listener.tsindex ae4ae6b0881..07446788e6c 100644--- a/packages/twenty-server/src/modules/connected-account/webhook-subscription-manager/listeners/webhook-subscription-workspace-activation.listener.ts+++ b/packages/twenty-server/src/modules/connected-account/webhook-subscription-manager/listeners/webhook-subscription-workspace-activation.listener.ts@@ -53,7 +53,11 @@ export class WebhookSubscriptionWorkspaceActivationListener {.add<SyncWorkspaceWebhookSubscriptionsJobData>(SyncWorkspaceWebhookSubscriptionsJob.name,{ workspaceId, action },- { retryLimit: WEBHOOK_SUBSCRIPTION_JOB_RETRY_LIMIT },+ {+ retryLimit: WEBHOOK_SUBSCRIPTION_JOB_RETRY_LIMIT,+ workspaceId,+ isWorkspaceJob: true,+ },).catch((error) =>this.exceptionHandlerService.captureExceptions([error], {diff --git a/packages/twenty-server/src/modules/connected-account/webhook-subscription-manager/services/webhook-subscription-workspace-sync.service.ts b/packages/twenty-server/src/modules/connected-account/webhook-subscription-manager/services/webhook-subscription-workspace-sync.service.tsindex 1864326f11f..089c385a870 100644--- a/packages/twenty-server/src/modules/connected-account/webhook-subscription-manager/services/webhook-subscription-workspace-sync.service.ts+++ b/packages/twenty-server/src/modules/connected-account/webhook-subscription-manager/services/webhook-subscription-workspace-sync.service.ts@@ -102,7 +102,11 @@ export class WebhookSubscriptionWorkspaceSyncService {): Promise<void> {await this.webhookQueueService.bulkAdd(jobName,- channels.map((data) => ({ data })),+ channels.map((data) => ({+ data,+ workspaceId: data.workspaceId,+ isWorkspaceJob: true,+ })),{ retryLimit: WEBHOOK_SUBSCRIPTION_JOB_RETRY_LIMIT },);}diff --git a/packages/twenty-server/src/modules/connected-account/webhook-subscription-manager/services/webhook-sync-trigger.service.ts b/packages/twenty-server/src/modules/connected-account/webhook-subscription-manager/services/webhook-sync-trigger.service.tsindex 31f4dc0c7cd..843626c0ab1 100644--- a/packages/twenty-server/src/modules/connected-account/webhook-subscription-manager/services/webhook-sync-trigger.service.ts+++ b/packages/twenty-server/src/modules/connected-account/webhook-subscription-manager/services/webhook-sync-trigger.service.ts@@ -38,6 +38,8 @@ export class WebhookSyncTriggerService {id: `messaging-message-webhook-sync:${workspaceId}:${messageChannelId}`,keepLastIfActive: true,},+ workspaceId,+ isWorkspaceJob: true,},);}@@ -63,6 +65,8 @@ export class WebhookSyncTriggerService {id: `calendar-event-webhook-sync:${workspaceId}:${calendarChannelId}`,keepLastIfActive: true,},+ workspaceId,+ isWorkspaceJob: true,},);}diff --git a/packages/twenty-server/src/modules/emailing/services/add-people-to-message-list-job.service.ts b/packages/twenty-server/src/modules/emailing/services/add-people-to-message-list-job.service.tsindex d0cddf8df27..44c597c4902 100644--- a/packages/twenty-server/src/modules/emailing/services/add-people-to-message-list-job.service.ts+++ b/packages/twenty-server/src/modules/emailing/services/add-people-to-message-list-job.service.ts@@ -7,6 +7,7 @@ import { isUserAuthContext } from 'src/engine/core-modules/auth/guards/is-user-aimport { type WorkspaceAuthContext } from 'src/engine/core-modules/auth/types/workspace-auth-context.type';import { CacheLockService } from 'src/engine/core-modules/cache-lock/cache-lock.service';import { InjectMessageQueue } from 'src/engine/core-modules/message-queue/decorators/message-queue.decorator';+import { type QueueJobScope } from 'src/engine/core-modules/message-queue/drivers/interfaces/job-options.interface';import { MessageQueue } from 'src/engine/core-modules/message-queue/message-queue.constants';import { MessageQueueService } from 'src/engine/core-modules/message-queue/services/message-queue.service';import { findInFlightQueueJobIdByPrefix } from 'src/engine/core-modules/message-queue/utils/find-in-flight-queue-job-id-by-prefix.util';@@ -65,11 +66,20 @@ export class AddPeopleToMessageListJobService {return this.cacheLockService.withLock(() =>this.enqueueAddPeopleToMessageListJob({- workspaceId,- userWorkspaceId: authContext.userWorkspaceId,- applicationId: authContext.application?.id,- messageListId,- personFilter,+ data: {+ workspaceId,+ userWorkspaceId: authContext.userWorkspaceId,+ applicationId: authContext.application?.id,+ messageListId,+ personFilter,+ },+ scope: isDefined(authContext.application)+ ? {+ workspaceId,+ applicationUniversalIdentifier:+ authContext.application.universalIdentifier,+ }+ : { workspaceId, isWorkspaceJob: true },}),buildAddPeopleToMessageListLockKey({ workspaceId, messageListId }),);@@ -101,9 +111,13 @@ export class AddPeopleToMessageListJobService {}}- private async enqueueAddPeopleToMessageListJob(- data: AddPeopleToMessageListJobData,- ): Promise<{ jobId: string }> {+ private async enqueueAddPeopleToMessageListJob({+ data,+ scope,+ }: {+ data: AddPeopleToMessageListJobData;+ scope: QueueJobScope;+ }): Promise<{ jobId: string }> {const jobIdPrefix = buildAddPeopleToMessageListJobId(data);const inFlightJobId = findInFlightQueueJobIdByPrefix({inFlightJobs: await this.campaignQueueService.getInFlightJobs(),@@ -125,6 +139,7 @@ export class AddPeopleToMessageListJobService {id: jobIdPrefix,delay: ADD_PEOPLE_TO_MESSAGE_LIST_JOB_START_DELAY_MS,broadcastTo: { workspaceId: data.workspaceId },+ ...scope,},);diff --git a/packages/twenty-server/src/modules/emailing/services/campaign-engagement-capture.service.ts b/packages/twenty-server/src/modules/emailing/services/campaign-engagement-capture.service.tsindex 7bf0c08a1cc..85188dbb4ae 100644--- a/packages/twenty-server/src/modules/emailing/services/campaign-engagement-capture.service.ts+++ b/packages/twenty-server/src/modules/emailing/services/campaign-engagement-capture.service.ts@@ -84,6 +84,8 @@ export class CampaignEngagementCaptureService {initialDelayMilliseconds: 5_000,jitter: 0.5,},+ workspaceId: observation.workspaceId,+ isWorkspaceJob: true,},);} catch (error) {diff --git a/packages/twenty-server/src/modules/emailing/services/campaign-sending-workspace.service.ts b/packages/twenty-server/src/modules/emailing/services/campaign-sending-workspace.service.tsindex 73e8fe60e06..92efa450362 100644--- a/packages/twenty-server/src/modules/emailing/services/campaign-sending-workspace.service.ts+++ b/packages/twenty-server/src/modules/emailing/services/campaign-sending-workspace.service.ts@@ -33,7 +33,11 @@ export class CampaignSendingWorkspaceService {await this.messageQueueService.bulkAdd<CampaignWorkspaceJobData>(jobName,- workspaceIdsBatch.map((workspaceId) => ({ data: { workspaceId } })),+ workspaceIdsBatch.map((workspaceId) => ({+ data: { workspaceId },+ workspaceId,+ isWorkspaceJob: true,+ })),{ retryLimit: CAMPAIGN_JOB_RETRY_LIMIT },).catch((error) => {diff --git a/packages/twenty-server/src/modules/emailing/services/message-campaign-batch-delivery.service.ts b/packages/twenty-server/src/modules/emailing/services/message-campaign-batch-delivery.service.tsindex 8fd66864cf7..37c898e7142 100644--- a/packages/twenty-server/src/modules/emailing/services/message-campaign-batch-delivery.service.ts+++ b/packages/twenty-server/src/modules/emailing/services/message-campaign-batch-delivery.service.ts@@ -287,6 +287,8 @@ export class MessageCampaignBatchDeliveryService {recipients: chunk,rateLimitedAttemptCount: requeuedAttemptCount,},+ workspaceId: data.workspaceId,+ isWorkspaceJob: true,})),{delay,diff --git a/packages/twenty-server/src/modules/emailing/services/message-campaign-delivery.service.ts b/packages/twenty-server/src/modules/emailing/services/message-campaign-delivery.service.tsindex 9abd3a492b8..a5a5033d88d 100644--- a/packages/twenty-server/src/modules/emailing/services/message-campaign-delivery.service.ts+++ b/packages/twenty-server/src/modules/emailing/services/message-campaign-delivery.service.ts@@ -175,6 +175,8 @@ export class MessageCampaignDeliveryService {}),retryLimit: CAMPAIGN_SEND_RETRY_LIMIT,backoff: CAMPAIGN_SEND_RETRY_BACKOFF,+ workspaceId: data.workspaceId,+ isWorkspaceJob: true,},);}diff --git a/packages/twenty-server/src/modules/emailing/services/message-campaign-materialization.service.ts b/packages/twenty-server/src/modules/emailing/services/message-campaign-materialization.service.tsindex dbd629f207a..4d3566b6fc1 100644--- a/packages/twenty-server/src/modules/emailing/services/message-campaign-materialization.service.ts+++ b/packages/twenty-server/src/modules/emailing/services/message-campaign-materialization.service.ts@@ -185,7 +185,11 @@ export class MessageCampaignMaterializationService {await this.messageQueueService.bulkAdd<MaterializeCampaignChunkJobData>(MATERIALIZE_CAMPAIGN_CHUNK_JOB,- chunks.map((chunk) => ({ data: chunk })),+ chunks.map((chunk) => ({+ data: chunk,+ workspaceId,+ isWorkspaceJob: true,+ })),{ retryLimit: 3, backoff: CAMPAIGN_SEND_RETRY_BACKOFF },);@@ -363,6 +367,8 @@ export class MessageCampaignMaterializationService {email: recipient.email,})),},+ workspaceId,+ isWorkspaceJob: true,})),{retryLimit: CAMPAIGN_SEND_RETRY_LIMIT,diff --git a/packages/twenty-server/src/modules/emailing/services/message-campaign-schedule.service.ts b/packages/twenty-server/src/modules/emailing/services/message-campaign-schedule.service.tsindex eb5d0700fa7..e4675619153 100644--- a/packages/twenty-server/src/modules/emailing/services/message-campaign-schedule.service.ts+++ b/packages/twenty-server/src/modules/emailing/services/message-campaign-schedule.service.ts@@ -139,6 +139,8 @@ export class MessageCampaignScheduleService {delay: Math.max(scheduledAt.getTime() - Date.now(), 0),retryLimit: CAMPAIGN_SEND_RETRY_LIMIT,backoff: CAMPAIGN_SEND_RETRY_BACKOFF,+ workspaceId,+ isWorkspaceJob: true,},).catch(async (error) => {diff --git a/packages/twenty-server/src/modules/emailing/services/message-campaign-statistics.service.ts b/packages/twenty-server/src/modules/emailing/services/message-campaign-statistics.service.tsindex 5b660613718..2a740cd4190 100644--- a/packages/twenty-server/src/modules/emailing/services/message-campaign-statistics.service.ts+++ b/packages/twenty-server/src/modules/emailing/services/message-campaign-statistics.service.ts@@ -53,6 +53,8 @@ export class MessageCampaignStatisticsService {id: `campaign-stats-refresh:${workspaceId}:${campaignId}`,keepLastIfActive: true,},+ workspaceId,+ isWorkspaceJob: true,},);}diff --git a/packages/twenty-server/src/modules/emailing/services/message-campaign.service.ts b/packages/twenty-server/src/modules/emailing/services/message-campaign.service.tsindex 2cbc3bdd3c8..6e0eb6ece56 100644--- a/packages/twenty-server/src/modules/emailing/services/message-campaign.service.ts+++ b/packages/twenty-server/src/modules/emailing/services/message-campaign.service.ts@@ -358,6 +358,8 @@ export class MessageCampaignService {{retryLimit: CAMPAIGN_SEND_RETRY_LIMIT,backoff: CAMPAIGN_SEND_RETRY_BACKOFF,+ workspaceId,+ isWorkspaceJob: true,},).catch(async (error) => {diff --git a/packages/twenty-server/src/modules/match-participant/listeners/participant-target-person.listener.ts b/packages/twenty-server/src/modules/match-participant/listeners/participant-target-person.listener.tsindex fc5116fc6da..c44c0ff2ac1 100644--- a/packages/twenty-server/src/modules/match-participant/listeners/participant-target-person.listener.ts+++ b/packages/twenty-server/src/modules/match-participant/listeners/participant-target-person.listener.ts@@ -44,6 +44,10 @@ export class ParticipantTargetPersonListener {await this.messageQueueService.add<ReconcilePersonCompanyTargetsJobData>(ReconcilePersonCompanyTargetsJob.name,{ workspaceId: payload.workspaceId, personIds },+ {+ workspaceId: payload.workspaceId,+ isWorkspaceJob: true,+ },);}}diff --git a/packages/twenty-server/src/modules/messaging-webhooks/handlers/inbound-mail-handler.service.ts b/packages/twenty-server/src/modules/messaging-webhooks/handlers/inbound-mail-handler.service.tsindex 12a64ca000b..6f15ebb19d9 100644--- a/packages/twenty-server/src/modules/messaging-webhooks/handlers/inbound-mail-handler.service.ts+++ b/packages/twenty-server/src/modules/messaging-webhooks/handlers/inbound-mail-handler.service.ts@@ -45,7 +45,7 @@ export class InboundMailHandlerService {reference: notification.message.reference,envelopeRecipients: notification.recipients,},- { id: notification.dedupeKey },+ { id: notification.dedupeKey, isSystemJob: true },);}diff --git a/packages/twenty-server/src/modules/messaging-webhooks/handlers/outbound-delivery-event-handler.service.ts b/packages/twenty-server/src/modules/messaging-webhooks/handlers/outbound-delivery-event-handler.service.tsindex 7f50e0fe0cb..35fdf284f66 100644--- a/packages/twenty-server/src/modules/messaging-webhooks/handlers/outbound-delivery-event-handler.service.ts+++ b/packages/twenty-server/src/modules/messaging-webhooks/handlers/outbound-delivery-event-handler.service.ts@@ -25,6 +25,8 @@ export class OutboundDeliveryEventHandlerService {doNotCheckDuplicatesBasedOnPrefix: true,retryLimit: DELIVERY_EVENT_RETRY_LIMIT,backoff: CAMPAIGN_SEND_RETRY_BACKOFF,+ workspaceId: event.workspaceId,+ isWorkspaceJob: true,},);}diff --git a/packages/twenty-server/src/modules/messaging/blocklist-manager/listeners/messaging-blocklist.listener.ts b/packages/twenty-server/src/modules/messaging/blocklist-manager/listeners/messaging-blocklist.listener.tsindex ef787c4b907..9dfc75f4e78 100644--- a/packages/twenty-server/src/modules/messaging/blocklist-manager/listeners/messaging-blocklist.listener.ts+++ b/packages/twenty-server/src/modules/messaging/blocklist-manager/listeners/messaging-blocklist.listener.ts@@ -38,6 +38,10 @@ export class MessagingBlocklistListener {await this.messageQueueService.add<BlocklistItemDeleteMessagesJobData>(BlocklistItemDeleteMessagesJob.name,payload,+ {+ workspaceId: payload.workspaceId,+ isWorkspaceJob: true,+ },);}@@ -50,6 +54,10 @@ export class MessagingBlocklistListener {await this.messageQueueService.add<BlocklistReimportMessagesJobData>(BlocklistReimportMessagesJob.name,payload,+ {+ workspaceId: payload.workspaceId,+ isWorkspaceJob: true,+ },);}@@ -62,11 +70,19 @@ export class MessagingBlocklistListener {await this.messageQueueService.add<BlocklistItemDeleteMessagesJobData>(BlocklistItemDeleteMessagesJob.name,payload,+ {+ workspaceId: payload.workspaceId,+ isWorkspaceJob: true,+ },);await this.messageQueueService.add<BlocklistReimportMessagesJobData>(BlocklistReimportMessagesJob.name,payload,+ {+ workspaceId: payload.workspaceId,+ isWorkspaceJob: true,+ },);}}diff --git a/packages/twenty-server/src/modules/messaging/message-cleaner/listeners/messaging-message-cleaner-connected-account.listener.ts b/packages/twenty-server/src/modules/messaging/message-cleaner/listeners/messaging-message-cleaner-connected-account.listener.tsindex 49248f4ac90..b5734138032 100644--- a/packages/twenty-server/src/modules/messaging/message-cleaner/listeners/messaging-message-cleaner-connected-account.listener.ts+++ b/packages/twenty-server/src/modules/messaging/message-cleaner/listeners/messaging-message-cleaner-connected-account.listener.ts@@ -39,6 +39,10 @@ export class MessagingMessageCleanerConnectedAccountListener {workspaceId,connectedAccountId: event.connectedAccountId,},+ {+ workspaceId,+ isWorkspaceJob: true,+ },),),);diff --git a/packages/twenty-server/src/modules/messaging/message-cleaner/listeners/messaging-message-cleaner-message-channel.listener.ts b/packages/twenty-server/src/modules/messaging/message-cleaner/listeners/messaging-message-cleaner-message-channel.listener.tsindex 218f1802f51..e972bc2c87c 100644--- a/packages/twenty-server/src/modules/messaging/message-cleaner/listeners/messaging-message-cleaner-message-channel.listener.ts+++ b/packages/twenty-server/src/modules/messaging/message-cleaner/listeners/messaging-message-cleaner-message-channel.listener.ts@@ -39,6 +39,10 @@ export class MessagingMessageCleanerMessageChannelListener {workspaceId,messageChannelId: event.messageChannelId,},+ {+ workspaceId,+ isWorkspaceJob: true,+ },),),);diff --git a/packages/twenty-server/src/modules/messaging/message-import-manager/commands/messaging-trigger-message-list-fetch.command.ts b/packages/twenty-server/src/modules/messaging/message-import-manager/commands/messaging-trigger-message-list-fetch.command.tsindex 3391b9ccc18..928f5e4a593 100644--- a/packages/twenty-server/src/modules/messaging/message-import-manager/commands/messaging-trigger-message-list-fetch.command.ts+++ b/packages/twenty-server/src/modules/messaging/message-import-manager/commands/messaging-trigger-message-list-fetch.command.ts@@ -91,6 +91,10 @@ export class MessagingTriggerMessageListFetchCommand extends CommandRunner {messageChannelId: messageChannel.id,workspaceId,},+ {+ workspaceId,+ isWorkspaceJob: true,+ },);this.logger.log(diff --git a/packages/twenty-server/src/modules/messaging/message-import-manager/crons/jobs/__tests__/messaging-ongoing-stale.cron.job.spec.ts b/packages/twenty-server/src/modules/messaging/message-import-manager/crons/jobs/__tests__/messaging-ongoing-stale.cron.job.spec.tsindex 6f8728d3f17..6613dd432d1 100644--- a/packages/twenty-server/src/modules/messaging/message-import-manager/crons/jobs/__tests__/messaging-ongoing-stale.cron.job.spec.ts+++ b/packages/twenty-server/src/modules/messaging/message-import-manager/crons/jobs/__tests__/messaging-ongoing-stale.cron.job.spec.ts@@ -41,6 +41,10 @@ describe('MessagingOngoingStaleCronJob', () => {{workspaceId: 'workspace-1',},+ {+ workspaceId: 'workspace-1',+ isWorkspaceJob: true,+ },);expect(messageQueueService.add).toHaveBeenNthCalledWith(2,@@ -48,6 +52,10 @@ describe('MessagingOngoingStaleCronJob', () => {{workspaceId: 'workspace-2',},+ {+ workspaceId: 'workspace-2',+ isWorkspaceJob: true,+ },);});diff --git a/packages/twenty-server/src/modules/messaging/message-import-manager/crons/jobs/messaging-message-list-fetch.cron.job.ts b/packages/twenty-server/src/modules/messaging/message-import-manager/crons/jobs/messaging-message-list-fetch.cron.job.tsindex 3cb40f71e4a..d20e53cc228 100644--- a/packages/twenty-server/src/modules/messaging/message-import-manager/crons/jobs/messaging-message-list-fetch.cron.job.ts+++ b/packages/twenty-server/src/modules/messaging/message-import-manager/crons/jobs/messaging-message-list-fetch.cron.job.ts@@ -114,6 +114,10 @@ export class MessagingMessageListFetchCronJob {workspaceId,messageChannelId,},+ {+ workspaceId,+ isWorkspaceJob: true,+ },);}} catch (error) {diff --git a/packages/twenty-server/src/modules/messaging/message-import-manager/crons/jobs/messaging-messages-import.cron.job.ts b/packages/twenty-server/src/modules/messaging/message-import-manager/crons/jobs/messaging-messages-import.cron.job.tsindex 779e0d51739..6a8f2e3daa5 100644--- a/packages/twenty-server/src/modules/messaging/message-import-manager/crons/jobs/messaging-messages-import.cron.job.ts+++ b/packages/twenty-server/src/modules/messaging/message-import-manager/crons/jobs/messaging-messages-import.cron.job.ts@@ -121,6 +121,10 @@ export class MessagingMessagesImportCronJob {workspaceId,messageChannelId,},+ {+ workspaceId,+ isWorkspaceJob: true,+ },);}} catch (error) {diff --git a/packages/twenty-server/src/modules/messaging/message-import-manager/crons/jobs/messaging-ongoing-stale.cron.job.ts b/packages/twenty-server/src/modules/messaging/message-import-manager/crons/jobs/messaging-ongoing-stale.cron.job.tsindex 02203147c2e..398488b0de9 100644--- a/packages/twenty-server/src/modules/messaging/message-import-manager/crons/jobs/messaging-ongoing-stale.cron.job.ts+++ b/packages/twenty-server/src/modules/messaging/message-import-manager/crons/jobs/messaging-ongoing-stale.cron.job.ts@@ -45,6 +45,10 @@ export class MessagingOngoingStaleCronJob {{workspaceId,},+ {+ workspaceId,+ isWorkspaceJob: true,+ },);} catch (error) {this.exceptionHandlerService.captureExceptions([error], {diff --git a/packages/twenty-server/src/modules/messaging/message-import-manager/crons/jobs/messaging-relaunch-failed-message-channels.cron.job.ts b/packages/twenty-server/src/modules/messaging/message-import-manager/crons/jobs/messaging-relaunch-failed-message-channels.cron.job.tsindex 6d3329f1db4..9f46a556352 100644--- a/packages/twenty-server/src/modules/messaging/message-import-manager/crons/jobs/messaging-relaunch-failed-message-channels.cron.job.ts+++ b/packages/twenty-server/src/modules/messaging/message-import-manager/crons/jobs/messaging-relaunch-failed-message-channels.cron.job.ts@@ -80,6 +80,10 @@ export class MessagingRelaunchFailedMessageChannelsCronJob {workspaceId: messageChannel.workspaceId,messageChannelId: messageChannel.id,},+ {+ workspaceId: messageChannel.workspaceId,+ isWorkspaceJob: true,+ },);} catch (error) {this.exceptionHandlerService.captureExceptions([error], {diff --git a/packages/twenty-server/src/modules/messaging/message-import-manager/services/messaging-save-messages-and-enqueue-contact-creation.service.ts b/packages/twenty-server/src/modules/messaging/message-import-manager/services/messaging-save-messages-and-enqueue-contact-creation.service.tsindex 6ad379ee539..e524447b629 100644--- a/packages/twenty-server/src/modules/messaging/message-import-manager/services/messaging-save-messages-and-enqueue-contact-creation.service.ts+++ b/packages/twenty-server/src/modules/messaging/message-import-manager/services/messaging-save-messages-and-enqueue-contact-creation.service.ts@@ -204,6 +204,10 @@ export class MessagingSaveMessagesAndEnqueueContactCreationService {contactsToCreate,source: FieldActorSource.EMAIL,},+ {+ workspaceId,+ isWorkspaceJob: true,+ },);}diff --git a/packages/twenty-server/src/modules/messaging/message-participant-manager/listeners/message-participant-person.listener.ts b/packages/twenty-server/src/modules/messaging/message-participant-manager/listeners/message-participant-person.listener.tsindex 60cb5c202dd..80448e9badb 100644--- a/packages/twenty-server/src/modules/messaging/message-participant-manager/listeners/message-participant-person.listener.ts+++ b/packages/twenty-server/src/modules/messaging/message-participant-manager/listeners/message-participant-person.listener.ts@@ -121,6 +121,10 @@ export class MessageParticipantPersonListener {await this.messageQueueService.add<MessageParticipantMatchParticipantJobData>(MessageParticipantMatchParticipantJob.name,data,+ {+ workspaceId: data.workspaceId,+ isWorkspaceJob: true,+ },);}diff --git a/packages/twenty-server/src/modules/messaging/message-participant-manager/listeners/message-participant-workspace-member.listener.ts b/packages/twenty-server/src/modules/messaging/message-participant-manager/listeners/message-participant-workspace-member.listener.tsindex 888b6e416da..1c2c710a644 100644--- a/packages/twenty-server/src/modules/messaging/message-participant-manager/listeners/message-participant-workspace-member.listener.ts+++ b/packages/twenty-server/src/modules/messaging/message-participant-manager/listeners/message-participant-workspace-member.listener.ts@@ -62,6 +62,10 @@ export class MessageParticipantWorkspaceMemberListener {workspaceMemberIds: [eventPayload.recordId],},},+ {+ workspaceId: payload.workspaceId,+ isWorkspaceJob: true,+ },);}}@@ -88,6 +92,10 @@ export class MessageParticipantWorkspaceMemberListener {workspaceMemberIds: [eventPayload.recordId],},},+ {+ workspaceId: payload.workspaceId,+ isWorkspaceJob: true,+ },);}}diff --git a/packages/twenty-server/src/modules/workflow/workflow-executor/workflow-actions/delay/jobs/resume-delayed-workflow.job.ts b/packages/twenty-server/src/modules/workflow/workflow-executor/workflow-actions/delay/jobs/resume-delayed-workflow.job.tsindex fa3a41c2ca9..0452f612a12 100644--- a/packages/twenty-server/src/modules/workflow/workflow-executor/workflow-actions/delay/jobs/resume-delayed-workflow.job.ts+++ b/packages/twenty-server/src/modules/workflow/workflow-executor/workflow-actions/delay/jobs/resume-delayed-workflow.job.ts@@ -22,6 +22,7 @@ import { RunWorkflowJob } from 'src/modules/workflow/workflow-runner/jobs/run-woimport { type RunWorkflowJobData } from 'src/modules/workflow/workflow-runner/types/run-workflow-job-data.type';import { buildRunWorkflowJobOptions } from 'src/modules/workflow/workflow-runner/utils/build-run-workflow-job-options.util';import { isWorkflowRunNotFoundError } from 'src/modules/workflow/workflow-runner/utils/is-workflow-run-not-found-error.util';+import { WorkflowRunJobScopeWorkspaceService } from 'src/modules/workflow/workflow-runner/workflow-run/workflow-run-job-scope.workspace-service';import { WorkflowRunWorkspaceService } from 'src/modules/workflow/workflow-runner/workflow-run/workflow-run.workspace-service';// Drains delays 2.45 and earlier queued as delayed jobs; new ones are TIME waits resumed through@@ -36,6 +37,7 @@ export class ResumeDelayedWorkflowJob {private readonly messageQueueService: MessageQueueService,private readonly workflowRunWorkspaceService: WorkflowRunWorkspaceService,private readonly workspaceOrmManager: WorkspaceOrmManager,+ private readonly workflowRunJobScopeWorkspaceService: WorkflowRunJobScopeWorkspaceService,) {}@Process(RESUME_DELAYED_WORKFLOW_JOB_NAME)@@ -102,7 +104,13 @@ export class ResumeDelayedWorkflowJob {workflowRunId,lastExecutedStepId: stepId,},- buildRunWorkflowJobOptions(workflowRunId),+ buildRunWorkflowJobOptions({+ workflowRunId,+ scope:+ await this.workflowRunJobScopeWorkspaceService.resolveForCoreWorkflow(+ { workspaceId, coreWorkflowId: workflowRun.coreWorkflowId },+ ),+ }),);} catch (error) {if (isWorkflowRunNotFoundError(error)) {diff --git a/packages/twenty-server/src/modules/workflow/workflow-executor/workspace-services/workflow-executor.workspace-service.ts b/packages/twenty-server/src/modules/workflow/workflow-executor/workspace-services/workflow-executor.workspace-service.tsindex f8b64b26fac..33e1ef28d3b 100644--- a/packages/twenty-server/src/modules/workflow/workflow-executor/workspace-services/workflow-executor.workspace-service.ts+++ b/packages/twenty-server/src/modules/workflow/workflow-executor/workspace-services/workflow-executor.workspace-service.ts@@ -55,6 +55,7 @@ import { ApplicationWorkflowStepPermissionService } from 'src/modules/workflow/wimport { RUN_WORKFLOW_JOB_NAME } from 'src/modules/workflow/workflow-runner/constants/run-workflow-job-name';import { type RunWorkflowJobData } from 'src/modules/workflow/workflow-runner/types/run-workflow-job-data.type';import { buildRunWorkflowJobOptions } from 'src/modules/workflow/workflow-runner/utils/build-run-workflow-job-options.util';+import { WorkflowRunJobScopeWorkspaceService } from 'src/modules/workflow/workflow-runner/workflow-run/workflow-run-job-scope.workspace-service';import { WorkflowRunWorkspaceService } from 'src/modules/workflow/workflow-runner/workflow-run/workflow-run.workspace-service';import { WorkflowStepWaitWorkspaceService } from 'src/modules/workflow/workflow-wait/services/workflow-step-wait.workspace-service';@@ -80,6 +81,7 @@ export class WorkflowExecutorWorkspaceService {@InjectMessageQueue(MessageQueue.workflowQueue)private readonly messageQueueService: MessageQueueService,private readonly applicationWorkflowStepPermissionService: ApplicationWorkflowStepPermissionService,+ private readonly workflowRunJobScopeWorkspaceService: WorkflowRunJobScopeWorkspaceService,) {}async executeFromSteps({@@ -196,6 +198,7 @@ export class WorkflowExecutorWorkspaceService {retryDelayMs: getStepRetryDelayMs({ stepInfo: stepInfos[stepId] }),workflowRunId,workspaceId,+ applicationId: workflow.applicationId,});return;@@ -271,6 +274,7 @@ export class WorkflowExecutorWorkspaceService {lastExecutedStepId: stepId,workflowRunId,workspaceId,+ applicationId: workflow.applicationId,});return;@@ -758,12 +762,14 @@ export class WorkflowExecutorWorkspaceService {retryDelayMs,workflowRunId,workspaceId,+ applicationId,}: {stepId: string;error: string;retryDelayMs: number;workflowRunId: string;workspaceId: string;+ applicationId: string;}) {await this.workflowRunWorkspaceService.moveStepToRetry({stepId,@@ -780,7 +786,16 @@ export class WorkflowExecutorWorkspaceService {workflowRunId,stepIdsToRetry: [stepId],},- { ...buildRunWorkflowJobOptions(workflowRunId), delay: retryDelayMs },+ {+ ...buildRunWorkflowJobOptions({+ workflowRunId,+ scope:+ await this.workflowRunJobScopeWorkspaceService.resolveForApplication(+ { workspaceId, applicationId },+ ),+ }),+ delay: retryDelayMs,+ },);} catch (enqueueError) {await this.workflowRunWorkspaceService.updateWorkflowRunStepInfos({@@ -797,10 +812,12 @@ export class WorkflowExecutorWorkspaceService {lastExecutedStepId,workflowRunId,workspaceId,+ applicationId,}: {lastExecutedStepId: string;workflowRunId: string;workspaceId: string;+ applicationId: string;}) {await this.messageQueueService.add<RunWorkflowJobData>(RUN_WORKFLOW_JOB_NAME,@@ -809,7 +826,14 @@ export class WorkflowExecutorWorkspaceService {workflowRunId,lastExecutedStepId,},- buildRunWorkflowJobOptions(workflowRunId),+ buildRunWorkflowJobOptions({+ workflowRunId,+ scope:+ await this.workflowRunJobScopeWorkspaceService.resolveForApplication({+ workspaceId,+ applicationId,+ }),+ }),);}}diff --git a/packages/twenty-server/src/modules/workflow/workflow-runner/services/__tests__/core-workflow-runner.service.spec.ts b/packages/twenty-server/src/modules/workflow/workflow-runner/services/__tests__/core-workflow-runner.service.spec.tsnew file mode 100644index 00000000000..5a8192adece--- /dev/null+++ b/packages/twenty-server/src/modules/workflow/workflow-runner/services/__tests__/core-workflow-runner.service.spec.ts@@ -0,0 +1,162 @@+import { FieldActorSource } from 'twenty-shared/types';++import { RUN_WORKFLOW_JOB_NAME } from 'src/modules/workflow/workflow-runner/constants/run-workflow-job-name';+import { CoreWorkflowRunnerService } from 'src/modules/workflow/workflow-runner/services/core-workflow-runner.service';+import { WorkflowRunEnqueueJob } from 'src/modules/workflow/workflow-runner/workflow-run-queue/jobs/workflow-run-enqueue.job';+import { WorkflowRunJobScopeWorkspaceService } from 'src/modules/workflow/workflow-runner/workflow-run/workflow-run-job-scope.workspace-service';+import { WorkflowTriggerType } from 'src/modules/workflow/workflow-trigger/types/workflow-trigger.type';++const WORKSPACE_ID = 'workspace-id';++const WORKFLOWS = {+ 'workflow-a': { id: 'workflow-a', applicationId: 'application-a' },+ 'workflow-b': { id: 'workflow-b', applicationId: 'application-b' },+};++const buildService = (triggerType: WorkflowTriggerType) => {+ const messageQueueService = { add: jest.fn() };+ const workflowRunWorkspaceService = {+ createCoreWorkflowRun: jest+ .fn()+ .mockImplementation(+ async ({ coreWorkflowId }: { coreWorkflowId: string }) =>+ `run-of-${coreWorkflowId}`,+ ),+ };+ const workspaceCacheService = {+ getOrRecompute: jest.fn().mockResolvedValue({+ flatApplicationMaps: {+ byId: {+ 'application-a': {+ id: 'application-a',+ universalIdentifier: 'application-a-universal-id',+ },+ 'application-b': {+ id: 'application-b',+ universalIdentifier: 'application-b-universal-id',+ },+ },+ idByUniversalIdentifier: {},+ },+ }),+ };++ const service = new CoreWorkflowRunnerService(+ {+ findCoreWorkflowById: jest+ .fn()+ .mockImplementation(+ async (_workspaceId, coreWorkflowId: keyof typeof WORKFLOWS) => ({+ ...WORKFLOWS[coreWorkflowId],+ workspaceWorkflowId: null,+ name: coreWorkflowId,+ }),+ ),+ } as never,+ {+ findCoreVersionById: jest+ .fn()+ .mockImplementation(async (_workspaceId, versionId: string) => ({+ id: versionId,+ coreWorkflowId: versionId.replace('version-of-', ''),+ workflowId: null,+ workspaceWorkflowVersionId: null,+ triggers: [{ type: triggerType }],+ steps: [],+ })),+ } as never,+ workflowRunWorkspaceService as never,+ {+ getSubscriptionInactiveReason: jest.fn().mockResolvedValue(undefined),+ } as never,+ {+ throttleOrThrowIfHardLimitReached: jest.fn(),+ increaseWorkflowRunNotStartedCount: jest.fn(),+ } as never,+ {} as never,+ {+ findWorkspaceTwentyStandardAndCustomApplicationOrThrow: jest+ .fn()+ .mockResolvedValue({+ workspaceCustomFlatApplication: { id: 'workspace-custom-app' },+ }),+ } as never,+ new WorkflowRunJobScopeWorkspaceService(+ workspaceCacheService as never,+ {} as never,+ ),+ messageQueueService as never,+ );++ const runBothWorkflows = async () => {+ for (const coreWorkflowId of Object.keys(WORKFLOWS)) {+ await service.run({+ workspaceId: WORKSPACE_ID,+ coreWorkflowVersionId: `version-of-${coreWorkflowId}`,+ payload: {},+ source: {+ source: FieldActorSource.MANUAL,+ name: 'Tim',+ workspaceMemberId: null,+ context: {},+ },+ });+ }+ };++ return { runBothWorkflows, messageQueueService };+};++describe('CoreWorkflowRunnerService', () => {+ it('scopes the run job of each workflow to the application owning it', async () => {+ const { runBothWorkflows, messageQueueService } = buildService(+ WorkflowTriggerType.MANUAL,+ );++ await runBothWorkflows();++ expect(messageQueueService.add.mock.calls).toEqual([+ [+ RUN_WORKFLOW_JOB_NAME,+ { workspaceId: WORKSPACE_ID, workflowRunId: 'run-of-workflow-a' },+ {+ id: 'run-of-workflow-a',+ doNotCheckDuplicatesBasedOnPrefix: true,+ workspaceId: WORKSPACE_ID,+ applicationUniversalIdentifier: 'application-a-universal-id',+ },+ ],+ [+ RUN_WORKFLOW_JOB_NAME,+ { workspaceId: WORKSPACE_ID, workflowRunId: 'run-of-workflow-b' },+ {+ id: 'run-of-workflow-b',+ doNotCheckDuplicatesBasedOnPrefix: true,+ workspaceId: WORKSPACE_ID,+ applicationUniversalIdentifier: 'application-b-universal-id',+ },+ ],+ ]);+ });++ it('keeps the workspace dispatcher of queued runs out of any application scope', async () => {+ const { runBothWorkflows, messageQueueService } = buildService(+ WorkflowTriggerType.DATABASE_EVENT,+ );++ await runBothWorkflows();++ expect(messageQueueService.add.mock.calls).toEqual([+ [+ WorkflowRunEnqueueJob.name,+ { workspaceId: WORKSPACE_ID, isCacheMode: true },+ { workspaceId: WORKSPACE_ID, isWorkspaceJob: true },+ ],+ [+ WorkflowRunEnqueueJob.name,+ { workspaceId: WORKSPACE_ID, isCacheMode: true },+ { workspaceId: WORKSPACE_ID, isWorkspaceJob: true },+ ],+ ]);+ });+});diff --git a/packages/twenty-server/src/modules/workflow/workflow-runner/services/core-workflow-runner.service.ts b/packages/twenty-server/src/modules/workflow/workflow-runner/services/core-workflow-runner.service.tsindex 098d760ffae..21e40c5efec 100644--- a/packages/twenty-server/src/modules/workflow/workflow-runner/services/core-workflow-runner.service.ts+++ b/packages/twenty-server/src/modules/workflow/workflow-runner/services/core-workflow-runner.service.ts@@ -26,6 +26,7 @@ import {type WorkflowRunEnqueueJobData,} from 'src/modules/workflow/workflow-runner/workflow-run-queue/jobs/workflow-run-enqueue.job';import { WorkflowThrottlingWorkspaceService } from 'src/modules/workflow/workflow-runner/workflow-run-queue/workspace-services/workflow-throttling.workspace-service';+import { WorkflowRunJobScopeWorkspaceService } from 'src/modules/workflow/workflow-runner/workflow-run/workflow-run-job-scope.workspace-service';import { WorkflowRunWorkspaceService } from 'src/modules/workflow/workflow-runner/workflow-run/workflow-run.workspace-service';import { WorkflowTriggerType } from 'src/modules/workflow/workflow-trigger/types/workflow-trigger.type';@@ -41,6 +42,7 @@ export class CoreWorkflowRunnerService {private readonly workflowThrottlingWorkspaceService: WorkflowThrottlingWorkspaceService,private readonly metricsService: MetricsService,private readonly applicationService: ApplicationService,+ private readonly workflowRunJobScopeWorkspaceService: WorkflowRunJobScopeWorkspaceService,@InjectMessageQueue(MessageQueue.workflowQueue)private readonly messageQueueService: MessageQueueService,) {}@@ -151,7 +153,13 @@ export class CoreWorkflowRunnerService {await this.messageQueueService.add<RunWorkflowJobData>(RunWorkflowJob.name,{ workspaceId, workflowRunId: createdWorkflowRunId },- buildRunWorkflowJobOptions(createdWorkflowRunId),+ buildRunWorkflowJobOptions({+ workflowRunId: createdWorkflowRunId,+ scope:+ await this.workflowRunJobScopeWorkspaceService.resolveForApplication(+ { workspaceId, applicationId: workflow.applicationId },+ ),+ }),);} else {await this.workflowThrottlingWorkspaceService.increaseWorkflowRunNotStartedCount(@@ -160,6 +168,10 @@ export class CoreWorkflowRunnerService {await this.messageQueueService.add<WorkflowRunEnqueueJobData>(WorkflowRunEnqueueJob.name,{ workspaceId, isCacheMode: true },+ {+ workspaceId,+ isWorkspaceJob: true,+ },);}diff --git a/packages/twenty-server/src/modules/workflow/workflow-runner/utils/build-run-workflow-job-options.util.ts b/packages/twenty-server/src/modules/workflow/workflow-runner/utils/build-run-workflow-job-options.util.tsindex 5517bad1343..59438e0d366 100644--- a/packages/twenty-server/src/modules/workflow/workflow-runner/utils/build-run-workflow-job-options.util.ts+++ b/packages/twenty-server/src/modules/workflow/workflow-runner/utils/build-run-workflow-job-options.util.ts@@ -1,8 +1,16 @@-import { type QueueJobOptions } from 'src/engine/core-modules/message-queue/drivers/interfaces/job-options.interface';+import {+ type QueueJobOptions,+ type QueueJobScope,+} from 'src/engine/core-modules/message-queue/drivers/interfaces/job-options.interface';-export const buildRunWorkflowJobOptions = (- workflowRunId: string,-): QueueJobOptions => ({+export const buildRunWorkflowJobOptions = ({+ workflowRunId,+ scope,+}: {+ workflowRunId: string;+ scope: QueueJobScope;+}): QueueJobOptions => ({id: workflowRunId,doNotCheckDuplicatesBasedOnPrefix: true,+ ...scope,});diff --git a/packages/twenty-server/src/modules/workflow/workflow-runner/workflow-run-queue/cron/jobs/workflow-clean-workflow-runs.cron.job.ts b/packages/twenty-server/src/modules/workflow/workflow-runner/workflow-run-queue/cron/jobs/workflow-clean-workflow-runs.cron.job.tsindex 82f93f39b9d..78791e015a5 100644--- a/packages/twenty-server/src/modules/workflow/workflow-runner/workflow-run-queue/cron/jobs/workflow-clean-workflow-runs.cron.job.ts+++ b/packages/twenty-server/src/modules/workflow/workflow-runner/workflow-run-queue/cron/jobs/workflow-clean-workflow-runs.cron.job.ts@@ -111,6 +111,10 @@ export class WorkflowCleanWorkflowRunsCronJob {await this.messageQueueService.add<WorkflowCleanWorkflowRunsJobData>(WorkflowCleanWorkflowRunsJob.name,{ workspaceId },+ {+ workspaceId,+ isWorkspaceJob: true,+ },);return true;diff --git a/packages/twenty-server/src/modules/workflow/workflow-runner/workflow-run-queue/cron/jobs/workflow-handle-staled-runs.cron.job.ts b/packages/twenty-server/src/modules/workflow/workflow-runner/workflow-run-queue/cron/jobs/workflow-handle-staled-runs.cron.job.tsindex e8aa33329a6..88350259c24 100644--- a/packages/twenty-server/src/modules/workflow/workflow-runner/workflow-run-queue/cron/jobs/workflow-handle-staled-runs.cron.job.ts+++ b/packages/twenty-server/src/modules/workflow/workflow-runner/workflow-run-queue/cron/jobs/workflow-handle-staled-runs.cron.job.ts@@ -125,6 +125,10 @@ export class WorkflowHandleStaledRunsCronJob {await this.messageQueueService.add<WorkflowHandleStaledRunsJobData>(WorkflowHandleStaledRunsJob.name,{ workspaceId },+ {+ workspaceId,+ isWorkspaceJob: true,+ },);return true;diff --git a/packages/twenty-server/src/modules/workflow/workflow-runner/workflow-run-queue/cron/jobs/workflow-run-enqueue.cron.job.ts b/packages/twenty-server/src/modules/workflow/workflow-runner/workflow-run-queue/cron/jobs/workflow-run-enqueue.cron.job.tsindex d4a22c1ee79..efb5989f0f2 100644--- a/packages/twenty-server/src/modules/workflow/workflow-runner/workflow-run-queue/cron/jobs/workflow-run-enqueue.cron.job.ts+++ b/packages/twenty-server/src/modules/workflow/workflow-runner/workflow-run-queue/cron/jobs/workflow-run-enqueue.cron.job.ts@@ -105,6 +105,10 @@ export class WorkflowRunEnqueueCronJob {await this.messageQueueService.add<WorkflowRunEnqueueJobData>(WorkflowRunEnqueueJob.name,{ workspaceId, isCacheMode: false },+ {+ workspaceId,+ isWorkspaceJob: true,+ },);return true;diff --git a/packages/twenty-server/src/modules/workflow/workflow-runner/workflow-run-queue/workspace-services/__tests__/workflow-run-enqueue.workspace-service.spec.ts b/packages/twenty-server/src/modules/workflow/workflow-runner/workflow-run-queue/workspace-services/__tests__/workflow-run-enqueue.workspace-service.spec.tsnew file mode 100644index 00000000000..6fd1a8f1406--- /dev/null+++ b/packages/twenty-server/src/modules/workflow/workflow-runner/workflow-run-queue/workspace-services/__tests__/workflow-run-enqueue.workspace-service.spec.ts@@ -0,0 +1,139 @@+import { RUN_WORKFLOW_JOB_NAME } from 'src/modules/workflow/workflow-runner/constants/run-workflow-job-name';+import { WorkflowRunEnqueueWorkspaceService } from 'src/modules/workflow/workflow-runner/workflow-run-queue/workspace-services/workflow-run-enqueue.workspace-service';+import { WorkflowRunJobScopeWorkspaceService } from 'src/modules/workflow/workflow-runner/workflow-run/workflow-run-job-scope.workspace-service';++const WORKSPACE_ID = 'workspace-id';++const buildFlatWorkflowMaps = (+ workflows: { id: string; applicationId: string }[],+) => ({+ byUniversalIdentifier: Object.fromEntries(+ workflows.map((workflow) => [`${workflow.id}-universal-id`, workflow]),+ ),+ universalIdentifierById: Object.fromEntries(+ workflows.map((workflow) => [workflow.id, `${workflow.id}-universal-id`]),+ ),+ universalIdentifiersByApplicationId: {},+});++const buildService = (+ runs: { id: string; coreWorkflowId: string | null }[],+) => {+ const workspaceCacheService = {+ getOrRecompute: jest.fn().mockResolvedValue({+ flatWorkflowMaps: buildFlatWorkflowMaps([+ { id: 'workflow-a', applicationId: 'application-a' },+ { id: 'workflow-b', applicationId: 'application-b' },+ ]),+ flatApplicationMaps: {+ byId: {+ 'application-a': {+ id: 'application-a',+ universalIdentifier: 'application-a-universal-id',+ },+ 'application-b': {+ id: 'application-b',+ universalIdentifier: 'application-b-universal-id',+ },+ },+ idByUniversalIdentifier: {},+ },+ }),+ };+ const workflowRunRepository = {+ find: jest.fn().mockResolvedValueOnce(runs).mockResolvedValue([]),+ update: jest.fn(),+ };+ const workflowThrottlingWorkspaceService = {+ acquireWorkflowEnqueueLock: jest.fn().mockResolvedValue('lock-token'),+ releaseWorkflowEnqueueLock: jest.fn(),+ getNotStartedRunsCountFromCache: jest.fn().mockResolvedValue(runs.length),+ consumeRemainingRunsToEnqueueCount: jest+ .fn()+ .mockImplementation(async (_workspaceId, count: number) => count),+ decreaseWorkflowRunNotStartedCount: jest.fn(),+ };+ const messageQueueService = { add: jest.fn() };++ const service = new WorkflowRunEnqueueWorkspaceService(+ workflowThrottlingWorkspaceService as never,+ {+ executeInWorkspaceContext: jest.fn().mockImplementation((work) => work()),+ getRepository: jest.fn().mockReturnValue(workflowRunRepository),+ } as never,+ messageQueueService as never,+ { incrementCounterForEvent: jest.fn() } as never,+ new WorkflowRunJobScopeWorkspaceService(+ workspaceCacheService as never,+ {} as never,+ ),+ );++ return { service, messageQueueService, workspaceCacheService };+};++describe('WorkflowRunEnqueueWorkspaceService', () => {+ it('scopes each run job to the application owning its workflow', async () => {+ const { service, messageQueueService, workspaceCacheService } =+ buildService([+ { id: 'run-a', coreWorkflowId: 'workflow-a' },+ { id: 'run-b', coreWorkflowId: 'workflow-b' },+ ]);++ await service.enqueueRunsForWorkspace({+ workspaceId: WORKSPACE_ID,+ isCacheMode: true,+ });++ expect(workspaceCacheService.getOrRecompute).toHaveBeenCalledTimes(1);+ expect(messageQueueService.add).toHaveBeenCalledWith(+ RUN_WORKFLOW_JOB_NAME,+ { workflowRunId: 'run-a', workspaceId: WORKSPACE_ID },+ {+ id: 'run-a',+ doNotCheckDuplicatesBasedOnPrefix: true,+ workspaceId: WORKSPACE_ID,+ applicationUniversalIdentifier: 'application-a-universal-id',+ },+ );+ expect(messageQueueService.add).toHaveBeenCalledWith(+ RUN_WORKFLOW_JOB_NAME,+ { workflowRunId: 'run-b', workspaceId: WORKSPACE_ID },+ {+ id: 'run-b',+ doNotCheckDuplicatesBasedOnPrefix: true,+ workspaceId: WORKSPACE_ID,+ applicationUniversalIdentifier: 'application-b-universal-id',+ },+ );+ });++ it('falls back to the workspace scope for a run whose workflow cannot be resolved', async () => {+ const { service, messageQueueService } = buildService([+ { id: 'run-without-workflow', coreWorkflowId: null },+ { id: 'run-of-deleted-workflow', coreWorkflowId: 'deleted-workflow' },+ ]);++ await service.enqueueRunsForWorkspace({+ workspaceId: WORKSPACE_ID,+ isCacheMode: true,+ });++ expect(+ messageQueueService.add.mock.calls.map(([, , options]) => options),+ ).toEqual([+ {+ id: 'run-without-workflow',+ doNotCheckDuplicatesBasedOnPrefix: true,+ workspaceId: WORKSPACE_ID,+ isWorkspaceJob: true,+ },+ {+ id: 'run-of-deleted-workflow',+ doNotCheckDuplicatesBasedOnPrefix: true,+ workspaceId: WORKSPACE_ID,+ isWorkspaceJob: true,+ },+ ]);+ });+});diff --git a/packages/twenty-server/src/modules/workflow/workflow-runner/workflow-run-queue/workspace-services/workflow-run-enqueue.workspace-service.ts b/packages/twenty-server/src/modules/workflow/workflow-runner/workflow-run-queue/workspace-services/workflow-run-enqueue.workspace-service.tsindex 374c5c7e003..36c427a9064 100644--- a/packages/twenty-server/src/modules/workflow/workflow-runner/workflow-run-queue/workspace-services/workflow-run-enqueue.workspace-service.ts+++ b/packages/twenty-server/src/modules/workflow/workflow-runner/workflow-run-queue/workspace-services/workflow-run-enqueue.workspace-service.ts@@ -19,6 +19,7 @@ import { type RunWorkflowJobData } from 'src/modules/workflow/workflow-runner/tyimport { buildRunWorkflowJobOptions } from 'src/modules/workflow/workflow-runner/utils/build-run-workflow-job-options.util';import { NOT_STARTED_RUNS_FIND_OPTIONS } from 'src/modules/workflow/workflow-runner/workflow-run-queue/constants/not-started-runs-find-options';import { WorkflowThrottlingWorkspaceService } from 'src/modules/workflow/workflow-runner/workflow-run-queue/workspace-services/workflow-throttling.workspace-service';+import { WorkflowRunJobScopeWorkspaceService } from 'src/modules/workflow/workflow-runner/workflow-run/workflow-run-job-scope.workspace-service';@Injectable()export class WorkflowRunEnqueueWorkspaceService {@@ -29,6 +30,7 @@ export class WorkflowRunEnqueueWorkspaceService {@InjectMessageQueue(MessageQueue.workflowQueue)private readonly messageQueueService: MessageQueueService,private readonly metricsService: MetricsService,+ private readonly workflowRunJobScopeWorkspaceService: WorkflowRunJobScopeWorkspaceService,) {}async enqueueRunsForWorkspace({@@ -72,6 +74,11 @@ export class WorkflowRunEnqueueWorkspaceService {return;}+ const resolveJobScope =+ await this.workflowRunJobScopeWorkspaceService.getCoreWorkflowJobScopeResolver(+ workspaceId,+ );+let totalEnqueuedCount = 0;let isSoftThrottled = false;@@ -80,6 +87,7 @@ export class WorkflowRunEnqueueWorkspaceService {where: NOT_STARTED_RUNS_FIND_OPTIONS,select: {id: true,+ coreWorkflowId: true,},order: {createdAt: 'ASC',@@ -104,23 +112,27 @@ export class WorkflowRunEnqueueWorkspaceService {isSoftThrottled = admittedRunCount < batchRuns.length;- const batchIds = batchRuns- .slice(0, admittedRunCount)- .map((workflowRun: WorkflowRunWorkspaceEntity) => workflowRun.id);+ const admittedRuns = batchRuns.slice(0, admittedRunCount);+ const batchIds = admittedRuns.map(+ (workflowRun: WorkflowRunWorkspaceEntity) => workflowRun.id,+ );await workflowRunRepository.update(batchIds, {enqueuedAt: new Date().toISOString(),status: WorkflowRunStatus.ENQUEUED,});- for (const workflowRunId of batchIds) {+ for (const { id: workflowRunId, coreWorkflowId } of admittedRuns) {await this.messageQueueService.add<RunWorkflowJobData>(RunWorkflowJob.name,{workflowRunId,workspaceId,},- buildRunWorkflowJobOptions(workflowRunId),+ buildRunWorkflowJobOptions({+ workflowRunId,+ scope: resolveJobScope(coreWorkflowId),+ }),);}diff --git a/packages/twenty-server/src/modules/workflow/workflow-runner/workflow-run/__tests__/workflow-run-job-scope.workspace-service.spec.ts b/packages/twenty-server/src/modules/workflow/workflow-runner/workflow-run/__tests__/workflow-run-job-scope.workspace-service.spec.tsnew file mode 100644index 00000000000..1ce11038e97--- /dev/null+++ b/packages/twenty-server/src/modules/workflow/workflow-runner/workflow-run/__tests__/workflow-run-job-scope.workspace-service.spec.ts@@ -0,0 +1,92 @@+import { WorkflowRunJobScopeWorkspaceService } from 'src/modules/workflow/workflow-runner/workflow-run/workflow-run-job-scope.workspace-service';++const WORKSPACE_ID = 'workspace-id';++const buildService = () => {+ const workspaceCacheService = {+ getOrRecompute: jest.fn().mockResolvedValue({+ flatWorkflowMaps: {+ byUniversalIdentifier: {+ 'workflow-a-universal-id': {+ id: 'workflow-a',+ applicationId: 'application-a',+ },+ 'workflow-of-uninstalled-application-universal-id': {+ id: 'workflow-of-uninstalled-application',+ applicationId: 'uninstalled-application',+ },+ },+ universalIdentifierById: {+ 'workflow-a': 'workflow-a-universal-id',+ 'workflow-of-uninstalled-application':+ 'workflow-of-uninstalled-application-universal-id',+ },+ universalIdentifiersByApplicationId: {},+ },+ flatApplicationMaps: {+ byId: {+ 'application-a': {+ id: 'application-a',+ universalIdentifier: 'application-a-universal-id',+ },+ 'uninstalled-application': {+ id: 'uninstalled-application',+ universalIdentifier: 'uninstalled-application-universal-id',+ deletedAt: '2026-01-01T00:00:00.000Z',+ },+ },+ idByUniversalIdentifier: {},+ },+ }),+ };+ const workflowRunWorkspaceService = {+ getWorkflowRunCoreWorkflowId: jest.fn().mockResolvedValue('workflow-a'),+ };++ return new WorkflowRunJobScopeWorkspaceService(+ workspaceCacheService as never,+ workflowRunWorkspaceService as never,+ );+};++describe('WorkflowRunJobScopeWorkspaceService', () => {+ it('scopes a run to the application owning its workflow', async () => {+ await expect(+ buildService().resolveForWorkflowRun({+ workspaceId: WORKSPACE_ID,+ workflowRunId: 'run-a',+ }),+ ).resolves.toEqual({+ workspaceId: WORKSPACE_ID,+ applicationUniversalIdentifier: 'application-a-universal-id',+ });+ });++ it.each([+ ['no core workflow', null],+ ['a workflow missing from the cache', 'unknown-workflow'],+ [+ 'a workflow of an uninstalled application',+ 'workflow-of-uninstalled-application',+ ],+ ])(+ 'falls back to the workspace scope for %s',+ async (_case, coreWorkflowId) => {+ await expect(+ buildService().resolveForCoreWorkflow({+ workspaceId: WORKSPACE_ID,+ coreWorkflowId,+ }),+ ).resolves.toEqual({ workspaceId: WORKSPACE_ID, isWorkspaceJob: true });+ },+ );++ it('falls back to the workspace scope for an unknown application', async () => {+ await expect(+ buildService().resolveForApplication({+ workspaceId: WORKSPACE_ID,+ applicationId: 'unknown-application',+ }),+ ).resolves.toEqual({ workspaceId: WORKSPACE_ID, isWorkspaceJob: true });+ });+});diff --git a/packages/twenty-server/src/modules/workflow/workflow-runner/workflow-run/workflow-run-job-scope.workspace-service.ts b/packages/twenty-server/src/modules/workflow/workflow-runner/workflow-run/workflow-run-job-scope.workspace-service.tsnew file mode 100644index 00000000000..35820da06ad--- /dev/null+++ b/packages/twenty-server/src/modules/workflow/workflow-runner/workflow-run/workflow-run-job-scope.workspace-service.ts@@ -0,0 +1,88 @@+import { Injectable } from '@nestjs/common';++import { isDefined } from 'twenty-shared/utils';++import { type QueueJobScope } from 'src/engine/core-modules/message-queue/drivers/interfaces/job-options.interface';+import { findFlatEntityByIdInFlatEntityMaps } from 'src/engine/metadata-modules/flat-entity/utils/find-flat-entity-by-id-in-flat-entity-maps.util';+import { WorkspaceCacheService } from 'src/engine/workspace-cache/services/workspace-cache.service';+import { buildApplicationQueueJobScope } from 'src/engine/core-modules/application/utils/build-application-queue-job-scope.util';+import { WorkflowRunWorkspaceService } from 'src/modules/workflow/workflow-runner/workflow-run/workflow-run.workspace-service';++@Injectable()+export class WorkflowRunJobScopeWorkspaceService {+ constructor(+ private readonly workspaceCacheService: WorkspaceCacheService,+ private readonly workflowRunWorkspaceService: WorkflowRunWorkspaceService,+ ) {}++ async resolveForApplication({+ workspaceId,+ applicationId,+ }: {+ workspaceId: string;+ applicationId: string | null | undefined;+ }): Promise<QueueJobScope> {+ const { flatApplicationMaps } =+ await this.workspaceCacheService.getOrRecompute(workspaceId, [+ 'flatApplicationMaps',+ ]);++ return buildApplicationQueueJobScope({+ workspaceId,+ applicationId,+ flatApplicationMaps,+ });+ }++ async getCoreWorkflowJobScopeResolver(+ workspaceId: string,+ ): Promise<(coreWorkflowId: string | null) => QueueJobScope> {+ const { flatWorkflowMaps, flatApplicationMaps } =+ await this.workspaceCacheService.getOrRecompute(workspaceId, [+ 'flatWorkflowMaps',+ 'flatApplicationMaps',+ ]);++ return (coreWorkflowId) =>+ buildApplicationQueueJobScope({+ workspaceId,+ applicationId: isDefined(coreWorkflowId)+ ? findFlatEntityByIdInFlatEntityMaps({+ flatEntityId: coreWorkflowId,+ flatEntityMaps: flatWorkflowMaps,+ })?.applicationId+ : undefined,+ flatApplicationMaps,+ });+ }++ async resolveForCoreWorkflow({+ workspaceId,+ coreWorkflowId,+ }: {+ workspaceId: string;+ coreWorkflowId: string | null;+ }): Promise<QueueJobScope> {+ const resolveJobScope =+ await this.getCoreWorkflowJobScopeResolver(workspaceId);++ return resolveJobScope(coreWorkflowId);+ }++ async resolveForWorkflowRun({+ workspaceId,+ workflowRunId,+ }: {+ workspaceId: string;+ workflowRunId: string;+ }): Promise<QueueJobScope> {+ return this.resolveForCoreWorkflow({+ workspaceId,+ coreWorkflowId:+ await this.workflowRunWorkspaceService.getWorkflowRunCoreWorkflowId({+ workflowRunId,+ workspaceId,+ }),+ });+ }+}diff --git a/packages/twenty-server/src/modules/workflow/workflow-runner/workflow-run/workflow-run.module.ts b/packages/twenty-server/src/modules/workflow/workflow-runner/workflow-run/workflow-run.module.tsindex 0043e9b1f37..a103843bad2 100644--- a/packages/twenty-server/src/modules/workflow/workflow-runner/workflow-run/workflow-run.module.ts+++ b/packages/twenty-server/src/modules/workflow/workflow-runner/workflow-run/workflow-run.module.ts@@ -8,8 +8,10 @@ import { AiAgentExecutionModule } from 'src/engine/metadata-modules/ai/ai-agent-import { WorkflowRunRecordShareModule } from 'src/engine/core-modules/workflow/workflow-run-record-share.module';import { DeleteWorkflowRunsCommand } from 'src/modules/workflow/workflow-runner/workflow-run/command/delete-workflow-runs.command';import { WorkflowRunStepLogWorkspaceService } from 'src/modules/workflow/workflow-runner/workflow-run/workflow-run-step-log.workspace-service';+import { WorkflowRunJobScopeWorkspaceService } from 'src/modules/workflow/workflow-runner/workflow-run/workflow-run-job-scope.workspace-service';import { WorkflowRunWorkspaceService } from 'src/modules/workflow/workflow-runner/workflow-run/workflow-run.workspace-service';import { WorkflowStepWaitModule } from 'src/modules/workflow/workflow-wait/workflow-step-wait.module';+import { WorkspaceCacheModule } from 'src/engine/workspace-cache/workspace-cache.module';@Module({imports: [@@ -20,12 +22,18 @@ import { WorkflowStepWaitModule } from 'src/modules/workflow/workflow-wait/workfWorkflowRunRecordShareModule,AiAgentExecutionModule,WorkflowStepWaitModule,+ WorkspaceCacheModule,],providers: [WorkflowRunWorkspaceService,WorkflowRunStepLogWorkspaceService,+ WorkflowRunJobScopeWorkspaceService,DeleteWorkflowRunsCommand,],- exports: [WorkflowRunWorkspaceService, WorkflowRunStepLogWorkspaceService],+ exports: [+ WorkflowRunWorkspaceService,+ WorkflowRunStepLogWorkspaceService,+ WorkflowRunJobScopeWorkspaceService,+ ],})export class WorkflowRunModule {}diff --git a/packages/twenty-server/src/modules/workflow/workflow-runner/workflow-run/workflow-run.workspace-service.ts b/packages/twenty-server/src/modules/workflow/workflow-runner/workflow-run/workflow-run.workspace-service.tsindex 3656c8ccbb4..8b03941dcf5 100644--- a/packages/twenty-server/src/modules/workflow/workflow-runner/workflow-run/workflow-run.workspace-service.ts+++ b/packages/twenty-server/src/modules/workflow/workflow-runner/workflow-run/workflow-run.workspace-service.ts@@ -522,6 +522,32 @@ export class WorkflowRunWorkspaceService {}, authContext);}+ async getWorkflowRunCoreWorkflowId({+ workflowRunId,+ workspaceId,+ }: {+ workflowRunId: string;+ workspaceId: string;+ }): Promise<string | null> {+ const authContext = buildSystemAuthContext(workspaceId);++ const workflowRun =+ await this.workspaceOrmManager.executeInWorkspaceContext(+ async () =>+ this.workspaceOrmManager+ .getRepository<WorkflowRunWorkspaceEntity>('workflowRun', {+ shouldBypassPermissionChecks: true,+ })+ .findOne({+ where: { id: workflowRunId },+ select: ['id', 'coreWorkflowId'],+ }),+ authContext,+ );++ return workflowRun?.coreWorkflowId ?? null;+ }+// read with the requester's own permissions, which follow the visibility of the run's workflowasync isWorkflowRunReadableByRequester(workflowRunId: string,diff --git a/packages/twenty-server/src/modules/workflow/workflow-runner/workspace-services/__tests__/workflow-waiting-step.workspace-service.spec.ts b/packages/twenty-server/src/modules/workflow/workflow-runner/workspace-services/__tests__/workflow-waiting-step.workspace-service.spec.tsindex 39c3109febb..138f8593d11 100644--- a/packages/twenty-server/src/modules/workflow/workflow-runner/workspace-services/__tests__/workflow-waiting-step.workspace-service.spec.ts+++ b/packages/twenty-server/src/modules/workflow/workflow-runner/workspace-services/__tests__/workflow-waiting-step.workspace-service.spec.ts@@ -11,6 +11,7 @@ const WORKSPACE_ID = 'workspace-id';const WORKFLOW_RUN_ID = 'workflow-run-id';const STEP_ID = 'step-id';const WAIT_ID = 'wait-id';+const APPLICATION_UNIVERSAL_IDENTIFIER = 'workflow-application-universal-id';const EVENT = {eventName: 'company.updated',@@ -77,6 +78,12 @@ const buildService = ({),};const messageQueueService = { add: jest.fn() };+ const workflowRunJobScopeWorkspaceService = {+ resolveForWorkflowRun: jest.fn().mockResolvedValue({+ workspaceId: WORKSPACE_ID,+ applicationUniversalIdentifier: APPLICATION_UNIVERSAL_IDENTIFIER,+ }),+ };const registry = new PendingWakeUpOwnerHandlerRegistryService();@@ -88,6 +95,7 @@ const buildService = ({{} as never,workflowExecutionContextService as never,{} as never,+ workflowRunJobScopeWorkspaceService as never,messageQueueService as never,).onModuleInit();@@ -138,7 +146,12 @@ describe('WorkflowWaitingStepWorkspaceService as a wake-up owner', () => {},},},- expect.anything(),+ {+ id: WORKFLOW_RUN_ID,+ doNotCheckDuplicatesBasedOnPrefix: true,+ workspaceId: WORKSPACE_ID,+ applicationUniversalIdentifier: APPLICATION_UNIVERSAL_IDENTIFIER,+ },);});diff --git a/packages/twenty-server/src/modules/workflow/workflow-runner/workspace-services/workflow-runner.workspace-service.ts b/packages/twenty-server/src/modules/workflow/workflow-runner/workspace-services/workflow-runner.workspace-service.tsindex 67333559622..c65c69e3307 100644--- a/packages/twenty-server/src/modules/workflow/workflow-runner/workspace-services/workflow-runner.workspace-service.ts+++ b/packages/twenty-server/src/modules/workflow/workflow-runner/workspace-services/workflow-runner.workspace-service.ts@@ -33,6 +33,7 @@ import { buildRetryStepInfos } from 'src/modules/workflow/workflow-runner/utils/import { buildRunWorkflowJobOptions } from 'src/modules/workflow/workflow-runner/utils/build-run-workflow-job-options.util';import { getRunnableStepIds } from 'src/modules/workflow/workflow-runner/utils/get-runnable-step-ids.util';import { WorkflowThrottlingWorkspaceService } from 'src/modules/workflow/workflow-runner/workflow-run-queue/workspace-services/workflow-throttling.workspace-service';+import { WorkflowRunJobScopeWorkspaceService } from 'src/modules/workflow/workflow-runner/workflow-run/workflow-run-job-scope.workspace-service';import { WorkflowRunWorkspaceService } from 'src/modules/workflow/workflow-runner/workflow-run/workflow-run.workspace-service';@Injectable()@@ -44,6 +45,7 @@ export class WorkflowRunnerWorkspaceService {private readonly workflowVersionStepOperationsWorkspaceService: WorkflowVersionStepOperationsWorkspaceService,private readonly workflowThrottlingWorkspaceService: WorkflowThrottlingWorkspaceService,private readonly workflowExecutionContextService: WorkflowExecutionContextService,+ private readonly workflowRunJobScopeWorkspaceService: WorkflowRunJobScopeWorkspaceService,) {}async resume({@@ -62,7 +64,14 @@ export class WorkflowRunnerWorkspaceService {workflowRunId,lastExecutedStepId,},- buildRunWorkflowJobOptions(workflowRunId),+ buildRunWorkflowJobOptions({+ workflowRunId,+ scope:+ await this.workflowRunJobScopeWorkspaceService.resolveForWorkflowRun({+ workspaceId,+ workflowRunId,+ }),+ }),);}@@ -335,6 +344,12 @@ export class WorkflowRunnerWorkspaceService {};}+ const jobScope =+ await this.workflowRunJobScopeWorkspaceService.resolveForCoreWorkflow({+ workspaceId,+ coreWorkflowId: workflowRun.coreWorkflowId,+ });+await this.workflowRunWorkspaceService.updateWorkflowRun({workflowRunId,workspaceId,@@ -357,7 +372,7 @@ export class WorkflowRunnerWorkspaceService {workflowRunId,stepIdsToRetry: stepIdsToRun,},- buildRunWorkflowJobOptions(workflowRunId),+ buildRunWorkflowJobOptions({ workflowRunId, scope: jobScope }),);} catch (error) {// Revert so the run isn't left RUNNING without a worker jobdiff --git a/packages/twenty-server/src/modules/workflow/workflow-runner/workspace-services/workflow-waiting-step.workspace-service.ts b/packages/twenty-server/src/modules/workflow/workflow-runner/workspace-services/workflow-waiting-step.workspace-service.tsindex c3ed8dcaba7..5f613260b06 100644--- a/packages/twenty-server/src/modules/workflow/workflow-runner/workspace-services/workflow-waiting-step.workspace-service.ts+++ b/packages/twenty-server/src/modules/workflow/workflow-runner/workspace-services/workflow-waiting-step.workspace-service.ts@@ -28,6 +28,7 @@ import { RUN_WORKFLOW_JOB_NAME } from 'src/modules/workflow/workflow-runner/consimport { type RunWorkflowJobData } from 'src/modules/workflow/workflow-runner/types/run-workflow-job-data.type';import { buildRunWorkflowJobOptions } from 'src/modules/workflow/workflow-runner/utils/build-run-workflow-job-options.util';import { WorkflowRunStepLogWorkspaceService } from 'src/modules/workflow/workflow-runner/workflow-run/workflow-run-step-log.workspace-service';+import { WorkflowRunJobScopeWorkspaceService } from 'src/modules/workflow/workflow-runner/workflow-run/workflow-run-job-scope.workspace-service';import { WorkflowRunWorkspaceService } from 'src/modules/workflow/workflow-runner/workflow-run/workflow-run.workspace-service';import { buildDefaultWaitResult } from 'src/modules/workflow/workflow-wait/utils/build-default-wait-result.util';@@ -53,6 +54,7 @@ export class WorkflowWaitingStepWorkspaceServiceprivate readonly workflowRunStepLogService: WorkflowRunStepLogWorkspaceService,private readonly workflowExecutionContextService: WorkflowExecutionContextService,private readonly workflowAgentConversationService: WorkflowAgentConversationWorkspaceService,+ private readonly workflowRunJobScopeWorkspaceService: WorkflowRunJobScopeWorkspaceService,@InjectMessageQueue(MessageQueue.workflowQueue)private readonly messageQueueService: MessageQueueService,) {}@@ -168,7 +170,10 @@ export class WorkflowWaitingStepWorkspaceServiceworkflowRunId,awaitedStepOutput: { stepId, actionOutput },},- buildRunWorkflowJobOptions(workflowRunId),+ await this.buildResumeJobOptions({+ ownerId: workflowRunId,+ workspaceId,+ }),);} catch (error) {await this.workflowRunWorkspaceService.endWorkflowRun({@@ -182,8 +187,21 @@ export class WorkflowWaitingStepWorkspaceService}}- buildResumeJobOptions(workflowRunId: string): QueueJobOptions {- return buildRunWorkflowJobOptions(workflowRunId);+ async buildResumeJobOptions({+ ownerId: workflowRunId,+ workspaceId,+ }: Pick<+ PendingWakeUpEntity,+ 'ownerId' | 'workspaceId'+ >): Promise<QueueJobOptions> {+ return buildRunWorkflowJobOptions({+ workflowRunId,+ scope:+ await this.workflowRunJobScopeWorkspaceService.resolveForWorkflowRun({+ workspaceId,+ workflowRunId,+ }),+ });}// the wake-up's own run and step are all the step needs to resume@@ -249,7 +267,10 @@ export class WorkflowWaitingStepWorkspaceService: { result: buildDefaultWaitResult(outcome) },},},- buildRunWorkflowJobOptions(workflowRunId),+ await this.buildResumeJobOptions({+ ownerId: workflowRunId,+ workspaceId,+ }),);} catch (error) {await this.workflowRunWorkspaceService.endWorkflowRun({diff --git a/packages/twenty-server/src/modules/workflow/workflow-trigger/automated-trigger/crons/jobs/__tests__/workflow-cron-trigger-cron.job.spec.ts b/packages/twenty-server/src/modules/workflow/workflow-trigger/automated-trigger/crons/jobs/__tests__/workflow-cron-trigger-cron.job.spec.tsindex 1571a499f48..4f7221bfee2 100644--- a/packages/twenty-server/src/modules/workflow/workflow-trigger/automated-trigger/crons/jobs/__tests__/workflow-cron-trigger-cron.job.spec.ts+++ b/packages/twenty-server/src/modules/workflow/workflow-trigger/automated-trigger/crons/jobs/__tests__/workflow-cron-trigger-cron.job.spec.ts@@ -59,6 +59,8 @@ const buildCronTrigger = (index: number) => ({settings: { pattern: '* * * * *' },});+const APPLICATION_UNIVERSAL_IDENTIFIER = 'application-universal-id';+const mockWorkspaceCaches = (triggersByWorkspaceId: Record<string, Record<string, unknown> | Error>,applicationVersion: string | null = '1.0.1',@@ -70,7 +72,13 @@ const mockWorkspaceCaches = (if (keys.includes('flatApplicationMaps')) {return {flatApplicationMaps: {- byId: { 'application-id': { version: applicationVersion } },+ byId: {+ 'application-id': {+ id: 'application-id',+ universalIdentifier: APPLICATION_UNIVERSAL_IDENTIFIER,+ version: applicationVersion,+ },+ },idByUniversalIdentifier: {},},};@@ -206,7 +214,11 @@ describe('WorkflowCronTriggerCronJob', () => {workspaceWorkflowVersionId: 'workspace-version-1',payload: {},},- { retryLimit: 3 },+ {+ retryLimit: 3,+ workspaceId: WORKSPACE_1,+ applicationUniversalIdentifier: APPLICATION_UNIVERSAL_IDENTIFIER,+ },);});@@ -232,7 +244,37 @@ describe('WorkflowCronTriggerCronJob', () => {workspaceWorkflowVersionId: 'workspace-version-1',payload: {},},- { retryLimit: 3 },+ {+ retryLimit: 3,+ workspaceId: WORKSPACE_1,+ applicationUniversalIdentifier: APPLICATION_UNIVERSAL_IDENTIFIER,+ },+ );+ });++ it('should fall back to the workspace scope when the owning application cannot be resolved', async () => {+ mockWorkflowCoreSyncService.findCoreWorkflowByIdOrWorkspaceWorkflowId.mockResolvedValue(+ {+ id: 'workflow-1',+ applicationId: 'unknown-application-id',+ workspaceWorkflowId: 'workflow-1',+ lastPublishedCoreWorkflowVersionId: 'core-version-1',+ },+ );+ mockCacheStorageService.hashGetValues.mockResolvedValue([+ JSON.stringify({+ workspaceId: WORKSPACE_1,+ workflowId: 'workflow-1',+ pattern: '* * * * *',+ }),+ ]);++ await job.handle();++ expect(mockMessageQueueService.add).toHaveBeenCalledWith(+ WorkflowTriggerJob.name,+ expect.anything(),+ { retryLimit: 3, workspaceId: WORKSPACE_1, isWorkspaceJob: true },);});@@ -440,7 +482,11 @@ describe('WorkflowCronTriggerCronJob', () => {workspaceWorkflowVersionId: 'workspace-version-1',payload: {},},- { retryLimit: 3 },+ {+ retryLimit: 3,+ workspaceId: WORKSPACE_2,+ applicationUniversalIdentifier: APPLICATION_UNIVERSAL_IDENTIFIER,+ },);});@@ -473,7 +519,11 @@ describe('WorkflowCronTriggerCronJob', () => {workspaceWorkflowVersionId: 'workspace-version-1',payload: {},},- { retryLimit: 3 },+ {+ retryLimit: 3,+ workspaceId: WORKSPACE_2,+ applicationUniversalIdentifier: APPLICATION_UNIVERSAL_IDENTIFIER,+ },);});});diff --git a/packages/twenty-server/src/modules/workflow/workflow-trigger/automated-trigger/crons/jobs/workflow-cron-trigger-cron.job.ts b/packages/twenty-server/src/modules/workflow/workflow-trigger/automated-trigger/crons/jobs/workflow-cron-trigger-cron.job.tsindex f80ad9b0e6e..0591c9d2c30 100644--- a/packages/twenty-server/src/modules/workflow/workflow-trigger/automated-trigger/crons/jobs/workflow-cron-trigger-cron.job.ts+++ b/packages/twenty-server/src/modules/workflow/workflow-trigger/automated-trigger/crons/jobs/workflow-cron-trigger-cron.job.ts@@ -20,12 +20,14 @@ import { Process } from 'src/engine/core-modules/message-queue/decorators/procesimport { Processor } from 'src/engine/core-modules/message-queue/decorators/processor.decorator';import { MessageQueue } from 'src/engine/core-modules/message-queue/message-queue.constants';import { MessageQueueService } from 'src/engine/core-modules/message-queue/services/message-queue.service';+import { type QueueJobScope } from 'src/engine/core-modules/message-queue/drivers/interfaces/job-options.interface';import { WorkspaceEntity } from 'src/engine/core-modules/workspace/workspace.entity';import { isApplicationInstallPending } from 'src/engine/core-modules/application/utils/is-application-install-pending.util';import { isCachedCronTrigger } from 'src/engine/core-modules/workflow/utils/cached-workflow-automated-trigger.util';import { WorkflowCoreSyncService } from 'src/engine/core-modules/workflow/services/workflow-core-sync.service';import { WorkspaceCacheService } from 'src/engine/workspace-cache/services/workspace-cache.service';import { buildCoreDispatchIds } from 'src/engine/core-modules/workflow/utils/build-core-dispatch-ids.util';+import { buildApplicationQueueJobScope } from 'src/engine/core-modules/application/utils/build-application-queue-job-scope.util';import { type QueuedWorkflowTriggerDispatchIds } from 'src/modules/workflow/workflow-trigger/utils/resolve-workflow-trigger-dispatch-mode.util';import { WORKFLOW_CRON_TRIGGER_CACHE_KEY } from 'src/modules/workflow/workflow-trigger/automated-trigger/crons/constants/workflow-cron-trigger-cache-key.constant';import { WORKFLOW_CRON_TRIGGER_CACHE_TTL_MS } from 'src/modules/workflow/workflow-trigger/automated-trigger/crons/constants/workflow-cron-trigger-cache-ttl.constant';@@ -96,7 +98,7 @@ export class WorkflowCronTriggerCronJob {continue;}- const { trigger } = normalizedTrigger;+ const { trigger, jobScope } = normalizedTrigger;const shouldDispatch = await this.shouldDispatch({ trigger, now });@@ -116,7 +118,7 @@ export class WorkflowCronTriggerCronJob {...buildCoreDispatchIds(trigger),payload: {},},- { retryLimit: 3 },+ { retryLimit: 3, ...jobScope },);} catch (error) {this.logger.error(`Error processing cached trigger: ${error}`);@@ -197,8 +199,11 @@ export class WorkflowCronTriggerCronJob {continue;}- const { trigger: cachedTrigger, isApplicationInstalled } =- normalizedTrigger;+ const {+ trigger: cachedTrigger,+ isApplicationInstalled,+ jobScope,+ } = normalizedTrigger;triggersToCache.push(cachedTrigger);@@ -225,7 +230,7 @@ export class WorkflowCronTriggerCronJob {...buildCoreDispatchIds(cachedTrigger),payload: {},},- { retryLimit: 3 },+ { retryLimit: 3, ...jobScope },);}}@@ -268,6 +273,7 @@ export class WorkflowCronTriggerCronJob {private async normalizeCachedTrigger(trigger: CachedCronTrigger): Promise<{trigger: CachedCronTrigger;isApplicationInstalled: boolean;+ jobScope: QueueJobScope;} | null> {const workflow =await this.workflowCoreSyncService.findCoreWorkflowByIdOrWorkspaceWorkflowId(@@ -315,6 +321,11 @@ export class WorkflowCronTriggerCronJob {flatApplicationMaps,workflow.applicationId,),+ jobScope: buildApplicationQueueJobScope({+ workspaceId: trigger.workspaceId,+ applicationId: workflow.applicationId,+ flatApplicationMaps,+ }),};}diff --git a/packages/twenty-server/src/modules/workflow/workflow-trigger/automated-trigger/listeners/__tests__/workflow-database-event-trigger.listener.spec.ts b/packages/twenty-server/src/modules/workflow/workflow-trigger/automated-trigger/listeners/__tests__/workflow-database-event-trigger.listener.spec.tsindex 04697745c8a..3f7c800c201 100644--- a/packages/twenty-server/src/modules/workflow/workflow-trigger/automated-trigger/listeners/__tests__/workflow-database-event-trigger.listener.spec.ts+++ b/packages/twenty-server/src/modules/workflow/workflow-trigger/automated-trigger/listeners/__tests__/workflow-database-event-trigger.listener.spec.ts@@ -54,9 +54,11 @@ describe('WorkflowDatabaseEventTriggerListener', () => {{applicationVersion = '1.0.1',applicationRole,+ applicationUniversalIdentifierById = {},}: {applicationVersion?: string | null;applicationRole?: { canReadObjectRecords: boolean } | null;+ applicationUniversalIdentifierById?: Record<string, string>;} = {},) => {const isApplicationWorkflow = applicationRole !== undefined;@@ -65,13 +67,26 @@ describe('WorkflowDatabaseEventTriggerListener', () => {flatObjectMetadataMaps: { byUniversalIdentifier: {} },featureFlagsMap: {},flatApplicationMaps: {- byId: {- 'application-id': {- id: 'application-id',- version: applicationVersion,- defaultRoleId: applicationRole ? 'application-role-id' : null,- },- },+ byId: Object.fromEntries(+ [+ ...new Set([+ 'application-id',+ ...Object.keys(applicationUniversalIdentifierById),+ ]),+ ].map((applicationId) => [+ applicationId,+ {+ id: applicationId,+ universalIdentifier:+ applicationUniversalIdentifierById[applicationId],+ version: applicationVersion,+ defaultRoleId:+ applicationId === 'application-id' && applicationRole+ ? 'application-role-id'+ : null,+ },+ ]),+ ),idByUniversalIdentifier: {},},...(isApplicationWorkflow@@ -291,7 +306,11 @@ describe('WorkflowDatabaseEventTriggerListener', () => {workspaceWorkflowVersionId: `workspace-version-${workflowId}`,payload: mockPayload.events[0],},- { retryLimit: 3 },+ {+ retryLimit: 3,+ workspaceId,+ isWorkspaceJob: true,+ },);});@@ -428,7 +447,11 @@ describe('WorkflowDatabaseEventTriggerListener', () => {workspaceWorkflowVersionId: `workspace-version-${workflowId}`,payload: createPayload.events[0],},- { retryLimit: 3 },+ {+ retryLimit: 3,+ workspaceId,+ isWorkspaceJob: true,+ },);});@@ -474,7 +497,11 @@ describe('WorkflowDatabaseEventTriggerListener', () => {properties: { after: record },},},- { retryLimit: 3 },+ {+ retryLimit: 3,+ workspaceId,+ isWorkspaceJob: true,+ },);beforeEach(() => {@@ -562,6 +589,53 @@ describe('WorkflowDatabaseEventTriggerListener', () => {});});+ it('should scope each trigger job to the application owning its workflow', async () => {+ const workspaceWorkflowId = 'workspace-workflow';+ const applicationWorkflowId = 'application-workflow';++ setTriggerMap(+ [+ {+ ...mockEventListeners[0],+ workflowId: workspaceWorkflowId,+ applicationId: WORKSPACE_CUSTOM_APPLICATION_ID,+ },+ { ...mockEventListeners[0], workflowId: applicationWorkflowId },+ ],+ {+ applicationRole: { canReadObjectRecords: true },+ applicationUniversalIdentifierById: {+ [WORKSPACE_CUSTOM_APPLICATION_ID]:+ 'workspace-custom-application-universal-id',+ 'application-id': 'application-universal-id',+ },+ },+ );++ await listener.handleObjectRecordUpdateEvent(mockPayload);++ expect(messageQueueService.add).toHaveBeenCalledTimes(2);+ expect(messageQueueService.add).toHaveBeenCalledWith(+ WorkflowTriggerJob.name,+ expect.objectContaining({ workflowId: workspaceWorkflowId }),+ {+ retryLimit: 3,+ workspaceId,+ applicationUniversalIdentifier:+ 'workspace-custom-application-universal-id',+ },+ );+ expect(messageQueueService.add).toHaveBeenCalledWith(+ WorkflowTriggerJob.name,+ expect.objectContaining({ workflowId: applicationWorkflowId }),+ {+ retryLimit: 3,+ workspaceId,+ applicationUniversalIdentifier: 'application-universal-id',+ },+ );+ });+it('should handle delete events correctly', async () => {const deletePayload: WorkspaceEventBatch<any> = {...mockPayload,@@ -597,7 +671,11 @@ describe('WorkflowDatabaseEventTriggerListener', () => {workspaceWorkflowVersionId: `workspace-version-${workflowId}`,payload: deletePayload.events[0],},- { retryLimit: 3 },+ {+ retryLimit: 3,+ workspaceId,+ isWorkspaceJob: true,+ },);});@@ -636,7 +714,11 @@ describe('WorkflowDatabaseEventTriggerListener', () => {workspaceWorkflowVersionId: `workspace-version-${workflowId}`,payload: destroyPayload.events[0],},- { retryLimit: 3 },+ {+ retryLimit: 3,+ workspaceId,+ isWorkspaceJob: true,+ },);});@@ -726,7 +808,11 @@ describe('WorkflowDatabaseEventTriggerListener', () => {expect(messageQueueService.add).toHaveBeenCalledWith(WorkflowTriggerJob.name,expect.objectContaining({ payload: privatePayload.events[1] }),- { retryLimit: 3 },+ {+ retryLimit: 3,+ workspaceId,+ isWorkspaceJob: true,+ },);});@@ -771,7 +857,11 @@ describe('WorkflowDatabaseEventTriggerListener', () => {workspaceWorkflowVersionId: `workspace-version-${workflowId}`,payload: batchPayload.events[0],},- { retryLimit: 3 },+ {+ retryLimit: 3,+ workspaceId,+ isWorkspaceJob: true,+ },);expect(messageQueueService.add).toHaveBeenNthCalledWith(2,@@ -783,7 +873,11 @@ describe('WorkflowDatabaseEventTriggerListener', () => {workspaceWorkflowVersionId: `workspace-version-${workflowId}`,payload: batchPayload.events[1],},- { retryLimit: 3 },+ {+ retryLimit: 3,+ workspaceId,+ isWorkspaceJob: true,+ },);});@@ -823,7 +917,11 @@ describe('WorkflowDatabaseEventTriggerListener', () => {workspaceWorkflowVersionId: `workspace-version-${workflowId}`,payload: positionOnlyPayload.events[0],},- { retryLimit: 3 },+ {+ retryLimit: 3,+ workspaceId,+ isWorkspaceJob: true,+ },);});diff --git a/packages/twenty-server/src/modules/workflow/workflow-trigger/automated-trigger/listeners/workflow-database-event-trigger.listener.ts b/packages/twenty-server/src/modules/workflow/workflow-trigger/automated-trigger/listeners/workflow-database-event-trigger.listener.tsindex 5c1310d57dd..4455d06b3bd 100644--- a/packages/twenty-server/src/modules/workflow/workflow-trigger/automated-trigger/listeners/workflow-database-event-trigger.listener.ts+++ b/packages/twenty-server/src/modules/workflow/workflow-trigger/automated-trigger/listeners/workflow-database-event-trigger.listener.ts@@ -29,12 +29,14 @@ import { RecordAccessPolicyService } from 'src/engine/core-modules/record-share/import { WorkspaceOrmManager } from 'src/engine/twenty-orm/workspace-orm.manager';import { buildSystemAuthContext } from 'src/engine/twenty-orm/utils/build-system-auth-context.util';import { ApplicationService } from 'src/engine/core-modules/application/application.service';+import { type FlatApplicationCacheMaps } from 'src/engine/core-modules/application/types/flat-application-cache-maps.type';import { CoreEntityCacheService } from 'src/engine/core-entity-cache/services/core-entity-cache.service';import { findActiveFlatApplicationById } from 'src/engine/core-modules/application/utils/find-active-flat-application-by-id.util';import { isApplicationInstallPending } from 'src/engine/core-modules/application/utils/is-application-install-pending.util';import { isCachedDatabaseEventTrigger } from 'src/engine/core-modules/workflow/utils/cached-workflow-automated-trigger.util';import { WorkspaceCacheService } from 'src/engine/workspace-cache/services/workspace-cache.service';import { type WorkspaceEventBatch } from 'src/engine/workspace-event-emitter/types/workspace-event-batch.type';+import { buildApplicationQueueJobScope } from 'src/engine/core-modules/application/utils/build-application-queue-job-scope.util';import { WorkflowCommonWorkspaceService } from 'src/modules/workflow/common/workspace-services/workflow-common.workspace-service';import { evaluateStepFilters } from 'src/modules/workflow/workflow-executor/workflow-actions/filter/utils/evaluate-step-filters.util';import {@@ -356,10 +358,8 @@ export class WorkflowDatabaseEventTriggerListener {const workspaceId = payload.workspaceId;const databaseEventName = payload.name;- const eventListeners = await this.getDatabaseEventListeners(- workspaceId,- databaseEventName,- );+ const { eventListeners, flatApplicationMaps } =+ await this.getDatabaseEventListeners(workspaceId, databaseEventName);if (eventListeners.length === 0) {return;@@ -429,7 +429,14 @@ export class WorkflowDatabaseEventTriggerListener {...buildCoreDispatchIds(eventListener),payload: eventPayload,},- { retryLimit: 3 },+ {+ retryLimit: 3,+ ...buildApplicationQueueJobScope({+ workspaceId,+ applicationId,+ flatApplicationMaps,+ }),+ },);}}@@ -439,14 +446,17 @@ export class WorkflowDatabaseEventTriggerListener {private async getDatabaseEventListeners(workspaceId: string,databaseEventName: string,- ): Promise<DatabaseEventTriggerListener[]> {+ ): Promise<{+ eventListeners: DatabaseEventTriggerListener[];+ flatApplicationMaps: FlatApplicationCacheMaps;+ }> {const { workflowAutomatedTriggerMaps, flatApplicationMaps } =await this.workspaceCacheService.getOrRecompute(workspaceId, ['workflowAutomatedTriggerMaps','flatApplicationMaps',]);- return Object.values(+ const eventListeners = Object.values(workflowAutomatedTriggerMaps.byWorkflowId,).flatMap<DatabaseEventTriggerListener>((trigger) => {if (@@ -459,6 +469,8 @@ export class WorkflowDatabaseEventTriggerListener {return [trigger];});++ return { eventListeners, flatApplicationMaps };}private async buildEnrichedPayload<TEvent extends ObjectRecordEvent>({diff --git a/packages/twenty-server/test/integration/google/messaging/channel-deletion-cleanup.integration-spec.ts b/packages/twenty-server/test/integration/google/messaging/channel-deletion-cleanup.integration-spec.tsindex a090745b327..baee2881ebe 100644--- a/packages/twenty-server/test/integration/google/messaging/channel-deletion-cleanup.integration-spec.ts+++ b/packages/twenty-server/test/integration/google/messaging/channel-deletion-cleanup.integration-spec.ts@@ -69,11 +69,15 @@ describe('Channel deletion cleanup (integration)', () => {expectedSubject,]);- await enqueueJobAndDrain<MessagingMessageChannelDeletionCleanupJobData>(- MessageQueue.messagingQueue,- MessagingMessageChannelDeletionCleanupJob.name,- { workspaceId, messageChannelId: channel.channelId },- );+ await enqueueJobAndDrain<MessagingMessageChannelDeletionCleanupJobData>({+ queue: MessageQueue.messagingQueue,+ jobName: MessagingMessageChannelDeletionCleanupJob.name,+ data: { workspaceId, messageChannelId: channel.channelId },+ scope: {+ workspaceId,+ isWorkspaceJob: true,+ },+ });expect(await findImportedMessageSubjects([expectedSubject])).toEqual([]);}, 120000);@@ -83,11 +87,15 @@ describe('Channel deletion cleanup (integration)', () => {eventTitle,]);- await enqueueJobAndDrain<CalendarChannelDeletionCleanupJobData>(- MessageQueue.calendarQueue,- CalendarChannelDeletionCleanupJob.name,- { workspaceId, calendarChannelId: channel.calendarChannelId },- );+ await enqueueJobAndDrain<CalendarChannelDeletionCleanupJobData>({+ queue: MessageQueue.calendarQueue,+ jobName: CalendarChannelDeletionCleanupJob.name,+ data: { workspaceId, calendarChannelId: channel.calendarChannelId },+ scope: {+ workspaceId,+ isWorkspaceJob: true,+ },+ });expect(await findImportedCalendarEventTitles([eventTitle])).toEqual([]);}, 120000);diff --git a/packages/twenty-server/test/integration/graphql/suites/ai-chat-answer-tool-call.integration-spec.ts b/packages/twenty-server/test/integration/graphql/suites/ai-chat-answer-tool-call.integration-spec.tsindex 388435ca43a..9fcefe55b28 100644--- a/packages/twenty-server/test/integration/graphql/suites/ai-chat-answer-tool-call.integration-spec.ts+++ b/packages/twenty-server/test/integration/graphql/suites/ai-chat-answer-tool-call.integration-spec.ts@@ -173,6 +173,7 @@ describe('Answering a chat tool call', () => {streamId: response.body.data.answerToolCall.streamId,userWorkspaceId,}),+ expect.anything(),);await stopStream();@@ -209,6 +210,7 @@ describe('Answering a chat tool call', () => {threadId,streamId: last.body.data.answerToolCall.streamId,}),+ expect.anything(),);});diff --git a/packages/twenty-server/test/integration/graphql/suites/ai-chat-inbox-message.integration-spec.ts b/packages/twenty-server/test/integration/graphql/suites/ai-chat-inbox-message.integration-spec.tsindex 6724dacdf6c..a6daee129e9 100644--- a/packages/twenty-server/test/integration/graphql/suites/ai-chat-inbox-message.integration-spec.ts+++ b/packages/twenty-server/test/integration/graphql/suites/ai-chat-inbox-message.integration-spec.ts@@ -405,6 +405,7 @@ describe('Sending an inbox message as an application', () => {threadId,userWorkspaceId: USER_WORKSPACE_DATA_SEED_IDS.JANE,}),+ expect.anything(),);});});diff --git a/packages/twenty-server/test/integration/graphql/suites/auth/jwt-key-rotation/rotate-signing-keys-cron-job.integration-spec.ts b/packages/twenty-server/test/integration/graphql/suites/auth/jwt-key-rotation/rotate-signing-keys-cron-job.integration-spec.tsindex 7fc775fea8e..f5814f6a8b8 100644--- a/packages/twenty-server/test/integration/graphql/suites/auth/jwt-key-rotation/rotate-signing-keys-cron-job.integration-spec.ts+++ b/packages/twenty-server/test/integration/graphql/suites/auth/jwt-key-rotation/rotate-signing-keys-cron-job.integration-spec.ts@@ -62,7 +62,11 @@ describe('RotateSigningKeysCronJob (integration)', () => {input: { key: SIGNING_KEY_ROTATION_DAYS_KEY, value: 0 },});- await cronQueue.add(RotateSigningKeysCronJob.name, {});+ await cronQueue.add(+ RotateSigningKeysCronJob.name,+ {},+ { isSystemJob: true },+ );let rotatedApiKeyToken = '';diff --git a/packages/twenty-server/test/integration/graphql/suites/workflow/core-workflow-execution.integration-spec.ts b/packages/twenty-server/test/integration/graphql/suites/workflow/core-workflow-execution.integration-spec.tsindex d8702a140d2..062c2da642f 100644--- a/packages/twenty-server/test/integration/graphql/suites/workflow/core-workflow-execution.integration-spec.ts+++ b/packages/twenty-server/test/integration/graphql/suites/workflow/core-workflow-execution.integration-spec.ts@@ -430,17 +430,24 @@ describe('core workflow execution and queue compatibility (e2e)', () => {getQueueToken(MessageQueue.workflowQueue),);- await queue.add('WorkflowTriggerJob', {- workspaceId,- workflowId: fixture.workflowId as string,- payload: { marker: envelope },- ...(envelope === 'workspace-version' || envelope === 'backfilled-paired'- ? { workspaceWorkflowVersionId: fixture.workflowVersionId }- : {}),- ...(envelope === 'core-version' || envelope === 'backfilled-paired'- ? { coreWorkflowVersionId: fixture.coreWorkflowVersionId }- : {}),- });+ await queue.add(+ 'WorkflowTriggerJob',+ {+ workspaceId,+ workflowId: fixture.workflowId as string,+ payload: { marker: envelope },+ ...(envelope === 'workspace-version' || envelope === 'backfilled-paired'+ ? { workspaceWorkflowVersionId: fixture.workflowVersionId }+ : {}),+ ...(envelope === 'core-version' || envelope === 'backfilled-paired'+ ? { coreWorkflowVersionId: fixture.coreWorkflowVersionId }+ : {}),+ },+ {+ workspaceId,+ isWorkspaceJob: true,+ },+ );for (let attempt = 0; attempt < 100; attempt++) {const [run] = await global.testDataSource.query(`SELECT id FROM "${schema}"."workflowRun" WHERE "coreWorkflowId" = $1`,@@ -1947,7 +1954,10 @@ describe('core workflow execution and queue compatibility (e2e)', () => {workflowRunId: runId,lastExecutedStepId: parallelBranch.id,},- buildRunWorkflowJobOptions(runId),+ buildRunWorkflowJobOptions({+ workflowRunId: runId,+ scope: { workspaceId, isWorkspaceJob: true },+ }),);await new Promise((resolve) => setTimeout(resolve, 1000));@@ -2130,7 +2140,14 @@ describe('core workflow execution and queue compatibility (e2e)', () => {capacity.mockRestore();await global.app.get<MessageQueueService>(getQueueToken(MessageQueue.workflowQueue))- .add('WorkflowRunEnqueueJob', { workspaceId, isCacheMode: false });+ .add(+ 'WorkflowRunEnqueueJob',+ { workspaceId, isCacheMode: false },+ {+ workspaceId,+ isWorkspaceJob: true,+ },+ );await waitForRun(runId, 'COMPLETED');});diff --git a/packages/twenty-server/test/integration/message-queue/bullmq-driver.integration-spec.ts b/packages/twenty-server/test/integration/message-queue/bullmq-driver.integration-spec.tsindex 9942c06184c..25699c5d0e7 100644--- a/packages/twenty-server/test/integration/message-queue/bullmq-driver.integration-spec.ts+++ b/packages/twenty-server/test/integration/message-queue/bullmq-driver.integration-spec.ts@@ -51,6 +51,7 @@ describe('BullMQDriver retry backoff (integration)', () => {RETRY_BACKOFF_TEST_JOB_NAME,{ marker: testJobMarker },{+ isSystemJob: true,id: testJobMarker,delay: TEST_JOB_DELAY_MILLISECONDS,retryLimit: 3,diff --git a/packages/twenty-server/test/integration/message-queue/bullmq-queue-order.integration-spec.ts b/packages/twenty-server/test/integration/message-queue/bullmq-queue-order.integration-spec.tsindex cc49fab5c86..6d761bed5f2 100644--- a/packages/twenty-server/test/integration/message-queue/bullmq-queue-order.integration-spec.ts+++ b/packages/twenty-server/test/integration/message-queue/bullmq-queue-order.integration-spec.ts@@ -29,7 +29,12 @@ describe('BullMQDriver queue ordering (integration)', () => {await withQueueOrderTestContext(async (driver) => {for (const job of jobs) {- await driver.add(job.queue, job.name, {}, { priority: job.priority });+ await driver.add(+ job.queue,+ job.name,+ {},+ { isSystemJob: true, priority: job.priority },+ );}try {@@ -89,7 +94,12 @@ describe('BullMQDriver queue ordering (integration)', () => {await withQueueOrderTestContext(async (driver) => {for (const job of jobs) {- await driver.add(queue1, job.name, {}, { priority: job.priority });+ await driver.add(+ queue1,+ job.name,+ {},+ { isSystemJob: true, priority: job.priority },+ );}driver.work(queue1,diff --git a/packages/twenty-server/test/integration/upgrade/suites/2-46-workspace-command-1791398841984-close-pending-ask-questions-calls.integration-spec.ts b/packages/twenty-server/test/integration/upgrade/suites/2-46-workspace-command-1791398841984-close-pending-ask-questions-calls.integration-spec.tsindex cde2d5b9ea9..17e8f01b9c8 100644--- a/packages/twenty-server/test/integration/upgrade/suites/2-46-workspace-command-1791398841984-close-pending-ask-questions-calls.integration-spec.ts+++ b/packages/twenty-server/test/integration/upgrade/suites/2-46-workspace-command-1791398841984-close-pending-ask-questions-calls.integration-spec.ts@@ -211,11 +211,18 @@ describe('2-46 workspace command - close pending ask_questions calls (integratioit('resolves the wake-up of the agent run suspended on the closed call', async () => {expect(addJob).toHaveBeenCalledTimes(1);- expect(addJob).toHaveBeenCalledWith(RESUME_PENDING_WAKE_UP_JOB_NAME, {- workspaceId: SEED_APPLE_WORKSPACE_ID,- wakeUpId,- answer: { result: {} },- });+ expect(addJob).toHaveBeenCalledWith(+ RESUME_PENDING_WAKE_UP_JOB_NAME,+ {+ workspaceId: SEED_APPLE_WORKSPACE_ID,+ wakeUpId,+ answer: { result: {} },+ },+ {+ workspaceId: SEED_APPLE_WORKSPACE_ID,+ isWorkspaceJob: true,+ },+ );});it('resolves the wake-up again on a rerun, as long as it was not claimed', async () => {@@ -224,11 +231,18 @@ describe('2-46 workspace command - close pending ask_questions calls (integratioawait runCommand();expect(addJob).toHaveBeenCalledTimes(1);- expect(addJob).toHaveBeenCalledWith(RESUME_PENDING_WAKE_UP_JOB_NAME, {- workspaceId: SEED_APPLE_WORKSPACE_ID,- wakeUpId,- answer: { result: {} },- });+ expect(addJob).toHaveBeenCalledWith(+ RESUME_PENDING_WAKE_UP_JOB_NAME,+ {+ workspaceId: SEED_APPLE_WORKSPACE_ID,+ wakeUpId,+ answer: { result: {} },+ },+ {+ workspaceId: SEED_APPLE_WORKSPACE_ID,+ isWorkspaceJob: true,+ },+ );expect(await readThread(chatThread)).toEqual({pendingQuestionMessageId: null,turnStatus: 'completed',diff --git a/packages/twenty-server/test/integration/utils/enqueue-job-and-drain.util.ts b/packages/twenty-server/test/integration/utils/enqueue-job-and-drain.util.tsindex e1aa8624dc3..00d6d8b01db 100644--- a/packages/twenty-server/test/integration/utils/enqueue-job-and-drain.util.ts+++ b/packages/twenty-server/test/integration/utils/enqueue-job-and-drain.util.ts@@ -1,3 +1,4 @@+import { type QueueJobScope } from 'src/engine/core-modules/message-queue/drivers/interfaces/job-options.interface';import { type MessageQueueJobData } from 'src/engine/core-modules/message-queue/interfaces/message-queue-job.interface';import { type MessageQueue } from 'src/engine/core-modules/message-queue/message-queue.constants';@@ -10,16 +11,22 @@ import { waitForAllJobsToFinish } from 'test/integration/utils/wait-for-all-jobs// follow-up jobs the worker chains (imports, contact creation, ...) are also// done when this resolves. Failure-path suites rely on the job being allowed// to fail: they assert the resulting channel state afterwards.-export const enqueueJobAndDrain = async <TData extends MessageQueueJobData>(- queue: MessageQueue,- jobName: string,- data: TData,-): Promise<void> => {+export const enqueueJobAndDrain = async <TData extends MessageQueueJobData>({+ queue,+ jobName,+ data,+ scope,+}: {+ queue: MessageQueue;+ jobName: string;+ data: TData;+ scope: QueueJobScope;+}): Promise<void> => {const messageQueueService = global.app.get<MessageQueueService>(getQueueToken(queue),{ strict: false },);- await messageQueueService.add(jobName, data);+ await messageQueueService.add(jobName, data, scope);await waitForAllJobsToFinish();};diff --git a/packages/twenty-server/test/integration/utils/run-calendar-channel-events-import.util.ts b/packages/twenty-server/test/integration/utils/run-calendar-channel-events-import.util.tsindex 356a4cd8c72..ba52257ce2e 100644--- a/packages/twenty-server/test/integration/utils/run-calendar-channel-events-import.util.ts+++ b/packages/twenty-server/test/integration/utils/run-calendar-channel-events-import.util.ts@@ -16,9 +16,13 @@ export const runCalendarChannelEventsImport = async (CalendarChannelSyncStage.CALENDAR_EVENTS_IMPORT_SCHEDULED,);- await enqueueJobAndDrain(- MessageQueue.calendarQueue,- CalendarEventsImportJob.name,- { workspaceId, calendarChannelId },- );+ await enqueueJobAndDrain({+ queue: MessageQueue.calendarQueue,+ jobName: CalendarEventsImportJob.name,+ data: { workspaceId, calendarChannelId },+ scope: {+ workspaceId,+ isWorkspaceJob: true,+ },+ });};diff --git a/packages/twenty-server/test/integration/utils/run-calendar-channel-list-fetch.util.ts b/packages/twenty-server/test/integration/utils/run-calendar-channel-list-fetch.util.tsindex 0cd6fe2f000..18ef8c881e7 100644--- a/packages/twenty-server/test/integration/utils/run-calendar-channel-list-fetch.util.ts+++ b/packages/twenty-server/test/integration/utils/run-calendar-channel-list-fetch.util.ts@@ -16,9 +16,13 @@ export const runCalendarChannelListFetch = async (CalendarChannelSyncStage.CALENDAR_EVENT_LIST_FETCH_SCHEDULED,);- await enqueueJobAndDrain(- MessageQueue.calendarQueue,- CalendarEventListFetchJob.name,- { workspaceId, calendarChannelId },- );+ await enqueueJobAndDrain({+ queue: MessageQueue.calendarQueue,+ jobName: CalendarEventListFetchJob.name,+ data: { workspaceId, calendarChannelId },+ scope: {+ workspaceId,+ isWorkspaceJob: true,+ },+ });};diff --git a/packages/twenty-server/test/integration/utils/run-calendar-channel-webhook-sync.util.ts b/packages/twenty-server/test/integration/utils/run-calendar-channel-webhook-sync.util.tsindex d715077909b..1b5b4583556 100644--- a/packages/twenty-server/test/integration/utils/run-calendar-channel-webhook-sync.util.ts+++ b/packages/twenty-server/test/integration/utils/run-calendar-channel-webhook-sync.util.ts@@ -17,9 +17,13 @@ export const runCalendarChannelWebhookSync = async (startingStage,);- await enqueueJobAndDrain(- MessageQueue.connectedAccountSyncWebhookQueue,- CalendarEventWebhookSyncJob.name,- { workspaceId, calendarChannelId },- );+ await enqueueJobAndDrain({+ queue: MessageQueue.connectedAccountSyncWebhookQueue,+ jobName: CalendarEventWebhookSyncJob.name,+ data: { workspaceId, calendarChannelId },+ scope: {+ workspaceId,+ isWorkspaceJob: true,+ },+ });};diff --git a/packages/twenty-server/test/integration/utils/run-message-channel-sync.util.ts b/packages/twenty-server/test/integration/utils/run-message-channel-sync.util.tsindex 40eaf39b4f2..e78a4886e71 100644--- a/packages/twenty-server/test/integration/utils/run-message-channel-sync.util.ts+++ b/packages/twenty-server/test/integration/utils/run-message-channel-sync.util.ts@@ -16,9 +16,13 @@ export const runMessageChannelSync = async (MessageChannelSyncStage.MESSAGE_LIST_FETCH_SCHEDULED,);- await enqueueJobAndDrain(- MessageQueue.messagingQueue,- MessagingMessageListFetchJob.name,- { workspaceId, messageChannelId },- );+ await enqueueJobAndDrain({+ queue: MessageQueue.messagingQueue,+ jobName: MessagingMessageListFetchJob.name,+ data: { workspaceId, messageChannelId },+ scope: {+ workspaceId,+ isWorkspaceJob: true,+ },+ });};diff --git a/packages/twenty-server/test/integration/utils/run-message-channel-webhook-sync.util.ts b/packages/twenty-server/test/integration/utils/run-message-channel-webhook-sync.util.tsindex 50ed33a8303..6e3cb1cce04 100644--- a/packages/twenty-server/test/integration/utils/run-message-channel-webhook-sync.util.ts+++ b/packages/twenty-server/test/integration/utils/run-message-channel-webhook-sync.util.ts@@ -17,9 +17,13 @@ export const runMessageChannelWebhookSync = async (startingStage,);- await enqueueJobAndDrain(- MessageQueue.connectedAccountSyncWebhookQueue,- MessagingMessageWebhookSyncJob.name,- { workspaceId, messageChannelId },- );+ await enqueueJobAndDrain({+ queue: MessageQueue.connectedAccountSyncWebhookQueue,+ jobName: MessagingMessageWebhookSyncJob.name,+ data: { workspaceId, messageChannelId },+ scope: {+ workspaceId,+ isWorkspaceJob: true,+ },+ });};diff --git a/packages/twenty-server/test/integration/utils/run-sync-cron.util.ts b/packages/twenty-server/test/integration/utils/run-sync-cron.util.tsindex c91598d6410..b1d0534736f 100644--- a/packages/twenty-server/test/integration/utils/run-sync-cron.util.ts+++ b/packages/twenty-server/test/integration/utils/run-sync-cron.util.ts@@ -3,5 +3,12 @@ import { MessageQueue } from 'src/engine/core-modules/message-queue/message-queuimport { enqueueJobAndDrain } from 'test/integration/utils/enqueue-job-and-drain.util';export const runSyncCron = async (cronJob: { name: string }): Promise<void> => {- await enqueueJobAndDrain(MessageQueue.cronQueue, cronJob.name, {});+ await enqueueJobAndDrain({+ queue: MessageQueue.cronQueue,+ jobName: cronJob.name,+ data: {},+ scope: {+ isSystemJob: true,+ },+ });};