Skip to content

The open alternative to Salesforce, designed for AI.

fix(workspace-cleaner): serialize workspace destruction on a dedicated queue (#26911)

Etienne35cc140c91f4
diff --git a/packages/twenty-server/src/engine/core-modules/message-queue/drivers/bullmq.driver.ts b/packages/twenty-server/src/engine/core-modules/message-queue/drivers/bullmq.driver.ts
index a8e0713d7ee..b401c503c3c 100644
--- a/packages/twenty-server/src/engine/core-modules/message-queue/drivers/bullmq.driver.ts
+++ b/packages/twenty-server/src/engine/core-modules/message-queue/drivers/bullmq.driver.ts
@@ -187,20 +187,13 @@ export class BullMQDriver
): Promise<void> {
const queue = this.queueMap[queueName];
- try {
- if (isDefined(globalConcurrency)) {
- await queue.setGlobalConcurrency(globalConcurrency);
+ if (isDefined(globalConcurrency)) {
+ await queue.setGlobalConcurrency(globalConcurrency);
- return;
- }
-
- await queue.removeGlobalConcurrency();
- } catch (error) {
- this.logger.error(
- `Failed to write global concurrency for queue ${queueName}`,
- error,
- );
+ return;
}
+
+ await queue.removeGlobalConcurrency();
}
work<T>(
diff --git a/packages/twenty-server/src/engine/core-modules/message-queue/message-queue-worker-config.constant.ts b/packages/twenty-server/src/engine/core-modules/message-queue/message-queue-worker-config.constant.ts
index 46416ebdf8e..a457c845cd2 100644
--- a/packages/twenty-server/src/engine/core-modules/message-queue/message-queue-worker-config.constant.ts
+++ b/packages/twenty-server/src/engine/core-modules/message-queue/message-queue-worker-config.constant.ts
@@ -154,6 +154,16 @@ export const MESSAGE_QUEUE_WORKER_CONFIG: Record<
boundedShutdownDrain: false,
},
},
+ [MessageQueue.workspaceDestroyQueue]: {
+ priority: 5,
+ workerOptions: {
+ concurrency: 1,
+ globalConcurrency: 1,
+ lockDuration: 30_000,
+ maxStalledCount: 1,
+ boundedShutdownDrain: false,
+ },
+ },
[MessageQueue.entityEventsToDbQueue]: {
priority: 1,
workerOptions: {
diff --git a/packages/twenty-server/src/engine/core-modules/message-queue/message-queue.constants.ts b/packages/twenty-server/src/engine/core-modules/message-queue/message-queue.constants.ts
index e7b64c1849a..bed18e85a5d 100644
--- a/packages/twenty-server/src/engine/core-modules/message-queue/message-queue.constants.ts
+++ b/packages/twenty-server/src/engine/core-modules/message-queue/message-queue.constants.ts
@@ -17,6 +17,7 @@ export enum MessageQueue {
contactCreationQueue = 'contact-creation-queue',
billingQueue = 'billing-queue',
workspaceQueue = 'workspace-queue',
+ workspaceDestroyQueue = 'workspace-destroy-queue',
entityEventsToDbQueue = 'entity-events-to-db-queue',
eventLogQueue = 'event-log-queue',
workflowQueue = 'workflow-queue',
diff --git a/packages/twenty-server/src/engine/workspace-manager/workspace-cleaner/jobs/destroy-soft-deleted-workspace.job.ts b/packages/twenty-server/src/engine/workspace-manager/workspace-cleaner/jobs/destroy-soft-deleted-workspace.job.ts
index a0d07fe39d5..8975cadc2b5 100644
--- a/packages/twenty-server/src/engine/workspace-manager/workspace-cleaner/jobs/destroy-soft-deleted-workspace.job.ts
+++ b/packages/twenty-server/src/engine/workspace-manager/workspace-cleaner/jobs/destroy-soft-deleted-workspace.job.ts
@@ -16,7 +16,7 @@ export type DestroySoftDeletedWorkspaceJobData = {
};
@Injectable()
-@Processor(MessageQueue.workspaceQueue)
+@Processor(MessageQueue.workspaceDestroyQueue)
export class DestroySoftDeletedWorkspaceJob {
private readonly logger = new Logger(DestroySoftDeletedWorkspaceJob.name);
diff --git a/packages/twenty-server/src/engine/workspace-manager/workspace-cleaner/services/cleaner.workspace-service.ts b/packages/twenty-server/src/engine/workspace-manager/workspace-cleaner/services/cleaner.workspace-service.ts
index 37f5d6b0417..8261bec3f5f 100644
--- a/packages/twenty-server/src/engine/workspace-manager/workspace-cleaner/services/cleaner.workspace-service.ts
+++ b/packages/twenty-server/src/engine/workspace-manager/workspace-cleaner/services/cleaner.workspace-service.ts
@@ -81,8 +81,8 @@ export class CleanerWorkspaceService {
private readonly userWorkspaceRepository: Repository<UserWorkspaceEntity>,
private readonly i18nService: I18nService,
private readonly workspaceDomainsService: WorkspaceDomainsService,
- @InjectMessageQueue(MessageQueue.workspaceQueue)
- private readonly messageQueueService: MessageQueueService,
+ @InjectMessageQueue(MessageQueue.workspaceDestroyQueue)
+ private readonly workspaceDestroyQueueService: MessageQueueService,
) {
this.inactiveDaysBeforeSoftDelete = this.twentyConfigService.get(
'WORKSPACE_INACTIVE_DAYS_BEFORE_SOFT_DELETION',
@@ -109,6 +109,16 @@ export class CleanerWorkspaceService {
return null;
}
+ private async enqueueWorkspaceDestruction(
+ workspaceId: string,
+ ): Promise<string | undefined> {
+ return this.workspaceDestroyQueueService.add<DestroySoftDeletedWorkspaceJobData>(
+ DestroySoftDeletedWorkspaceJob.name,
+ { workspaceId },
+ { id: `destroy-soft-deleted-workspace-${workspaceId}` },
+ );
+ }
+
async sendWarningEmail(
workspaceMember: WorkspaceMemberWorkspaceEntity,
workspaceDisplayName: string | undefined,
@@ -317,18 +327,12 @@ export class CleanerWorkspaceService {
await this.workspaceService.deleteWorkspace(workspace.id, true);
}
} else {
- if (this.twentyConfigService.get('IS_BILLING_ENABLED')) {
- await this.billingSubscriptionService.assertSubscriptionCanceledOrNone(
- workspace.id,
- );
- }
-
this.logger.log(
- `${dryRun ? 'DRY RUN - ' : ''}Hard deleting onboarding workspace ${workspace.id}`,
+ `${dryRun ? 'DRY RUN - ' : ''}Enqueuing destruction of onboarding workspace ${workspace.id}`,
);
if (!dryRun) {
- await this.workspaceService.deleteWorkspace(workspace.id);
+ await this.enqueueWorkspaceDestruction(workspace.id);
}
}
}
@@ -402,12 +406,9 @@ export class CleanerWorkspaceService {
);
if (!dryRun) {
- const jobId =
- await this.messageQueueService.add<DestroySoftDeletedWorkspaceJobData>(
- DestroySoftDeletedWorkspaceJob.name,
- { workspaceId: workspace.id },
- { id: `destroy-soft-deleted-workspace-${workspace.id}` },
- );
+ const jobId = await this.enqueueWorkspaceDestruction(
+ workspace.id,
+ );
if (isDefined(jobId)) {
enqueuedWorkspacesCount++;