The open alternative to Salesforce, designed for AI.
TypeScript crmcrm-systemcustomergood-first-issuegraphqlhacktoberfestjavascriptmarketingmonoreponestjsopen-sourcepostgresqlreactreactjssalestypescriptweb
Prevent overlapping workspace cleanup executions (#23522)
Weikoe4e1d24731f9
diff --git a/packages/twenty-server/src/database/typeorm/postgres-advisory-lock.service.spec.ts b/packages/twenty-server/src/database/typeorm/postgres-advisory-lock.service.spec.tsnew file mode 100644index 00000000000..488ff620c5b--- /dev/null+++ b/packages/twenty-server/src/database/typeorm/postgres-advisory-lock.service.spec.ts@@ -0,0 +1,212 @@+import { Logger } from '@nestjs/common';++import { type PoolClient } from 'pg';+import { type DataSource } from 'typeorm';+import { type PostgresDriver } from 'typeorm/driver/postgres/PostgresDriver';++import { PostgresAdvisoryLockService } from 'src/database/typeorm/postgres-advisory-lock.service';++describe('PostgresAdvisoryLockService', () => {+ const obtainMasterConnection = jest.fn();+ const dataSource = {+ driver: {+ obtainMasterConnection,+ } as unknown as PostgresDriver,+ } as unknown as DataSource;++ const createMockConnection = (isLockAcquired: boolean) => {+ let connectionErrorHandler: ((error: Error) => void) | undefined;+ const query = jest+ .fn()+ .mockResolvedValueOnce({+ rows: [{ acquired: isLockAcquired }],+ })+ .mockResolvedValueOnce({+ rows: [{ released: true }],+ });+ const connection = {+ query,+ on: jest.fn((_eventName: string, handler: (error: Error) => void) => {+ connectionErrorHandler = handler;+ }),+ removeListener: jest.fn(),+ } as unknown as PoolClient;+ const release = jest.fn();++ return {+ connection,+ emitConnectionError: (error: Error) => connectionErrorHandler?.(error),+ query,+ release,+ };+ };++ let loggerWarn: jest.SpyInstance;++ beforeEach(() => {+ jest.clearAllMocks();+ loggerWarn = jest.spyOn(Logger.prototype, 'warn').mockImplementation();+ });++ afterEach(() => {+ loggerWarn.mockRestore();+ });++ it('runs the callback while holding the lock', async () => {+ const { connection, query, release } = createMockConnection(true);+ const callback = jest.fn().mockResolvedValue('result');++ obtainMasterConnection.mockResolvedValue([connection, release]);++ const result = await new PostgresAdvisoryLockService(+ dataSource,+ ).tryWithLock('lock-name', callback);++ expect(result).toEqual({+ acquired: true,+ value: 'result',+ });+ expect(callback).toHaveBeenCalledTimes(1);+ expect(query).toHaveBeenCalledTimes(2);+ expect(query).toHaveBeenNthCalledWith(+ 2,+ expect.stringContaining('pg_advisory_unlock'),+ ['lock-name'],+ );+ expect(query.mock.invocationCallOrder[1]).toBeLessThan(+ release.mock.invocationCallOrder[0],+ );+ });++ it('does not run the callback when the lock is held', async () => {+ const { connection, query, release } = createMockConnection(false);+ const callback = jest.fn();++ obtainMasterConnection.mockResolvedValue([connection, release]);++ await expect(+ new PostgresAdvisoryLockService(dataSource).tryWithLock(+ 'lock-name',+ callback,+ ),+ ).resolves.toEqual({+ acquired: false,+ });++ expect(callback).not.toHaveBeenCalled();+ expect(query).toHaveBeenCalledTimes(1);+ expect(release).toHaveBeenCalledTimes(1);+ });++ it('discards the connection when acquiring the lock fails', async () => {+ const { connection, query, release } = createMockConnection(true);+ const callback = jest.fn();+ const acquisitionError = new Error('acquisition failed');++ query.mockReset().mockRejectedValueOnce(acquisitionError);+ obtainMasterConnection.mockResolvedValue([connection, release]);++ await expect(+ new PostgresAdvisoryLockService(dataSource).tryWithLock(+ 'lock-name',+ callback,+ ),+ ).rejects.toBe(acquisitionError);++ expect(callback).not.toHaveBeenCalled();+ expect(release).toHaveBeenCalledWith(acquisitionError);+ });++ it('releases the lock when the callback fails', async () => {+ const { connection, query, release } = createMockConnection(true);+ const callbackError = new Error('callback failed');++ obtainMasterConnection.mockResolvedValue([connection, release]);++ await expect(+ new PostgresAdvisoryLockService(dataSource).tryWithLock(+ 'lock-name',+ async () => {+ throw callbackError;+ },+ ),+ ).rejects.toBe(callbackError);++ expect(query).toHaveBeenCalledTimes(2);+ expect(release).toHaveBeenCalledWith();+ });++ it('discards the connection when releasing the lock fails', async () => {+ const { connection, query, release } = createMockConnection(true);+ const releaseError = new Error('release failed');++ query+ .mockReset()+ .mockResolvedValueOnce({+ rows: [{ acquired: true }],+ })+ .mockRejectedValueOnce(releaseError);+ obtainMasterConnection.mockResolvedValue([connection, release]);++ await expect(+ new PostgresAdvisoryLockService(dataSource).tryWithLock(+ 'lock-name',+ async () => undefined,+ ),+ ).rejects.toBe(releaseError);++ expect(release).toHaveBeenCalledWith(releaseError);+ });++ it('preserves the callback error when the connection fails', async () => {+ const { connection, emitConnectionError, release } =+ createMockConnection(true);+ const callbackError = new Error('callback failed');+ const connectionError = new Error('connection failed');++ obtainMasterConnection.mockResolvedValue([connection, release]);++ await expect(+ new PostgresAdvisoryLockService(dataSource).tryWithLock(+ 'lock-name',+ async () => {+ emitConnectionError(connectionError);+ throw callbackError;+ },+ ),+ ).rejects.toBe(callbackError);++ expect(release).toHaveBeenCalledWith(connectionError);+ expect(loggerWarn).toHaveBeenCalledWith(+ expect.stringContaining(connectionError.message),+ );+ });++ it('preserves the callback error when releasing the lock fails', async () => {+ const { connection, query, release } = createMockConnection(true);+ const callbackError = new Error('callback failed');+ const releaseError = new Error('release failed');++ query+ .mockReset()+ .mockResolvedValueOnce({+ rows: [{ acquired: true }],+ })+ .mockRejectedValueOnce(releaseError);+ obtainMasterConnection.mockResolvedValue([connection, release]);++ await expect(+ new PostgresAdvisoryLockService(dataSource).tryWithLock(+ 'lock-name',+ async () => {+ throw callbackError;+ },+ ),+ ).rejects.toBe(callbackError);++ expect(release).toHaveBeenCalledWith(releaseError);+ expect(loggerWarn).toHaveBeenCalledWith(+ expect.stringContaining(releaseError.message),+ );+ });+});diff --git a/packages/twenty-server/src/database/typeorm/postgres-advisory-lock.service.ts b/packages/twenty-server/src/database/typeorm/postgres-advisory-lock.service.tsnew file mode 100644index 00000000000..60ac61b972e--- /dev/null+++ b/packages/twenty-server/src/database/typeorm/postgres-advisory-lock.service.ts@@ -0,0 +1,184 @@+import { Injectable, Logger } from '@nestjs/common';+import { InjectDataSource } from '@nestjs/typeorm';++import { type PoolClient } from 'pg';+import { DataSource } from 'typeorm';+import { type PostgresDriver } from 'typeorm/driver/postgres/PostgresDriver';++const toError = (error: unknown): Error =>+ error instanceof Error ? error : new Error(String(error));++export type PostgresAdvisoryLockResult<T> =+ | {+ acquired: false;+ }+ | {+ acquired: true;+ value: T;+ };++@Injectable()+export class PostgresAdvisoryLockService {+ private readonly logger = new Logger(PostgresAdvisoryLockService.name);++ constructor(+ @InjectDataSource()+ private readonly coreDataSource: DataSource,+ ) {}++ async tryWithLock<T>(+ lockName: string,+ callback: () => Promise<T>,+ ): Promise<PostgresAdvisoryLockResult<T>> {+ const lockSession = await PostgresAdvisoryLockSession.tryAcquire(+ this.coreDataSource,+ lockName,+ );++ if (!lockSession) {+ return {+ acquired: false,+ };+ }++ const callbackResult = await this.captureCallbackResult(callback);+ const cleanupError = await lockSession.unlockAndClose();++ if (callbackResult.status === 'rejected') {+ if (cleanupError) {+ this.logger.warn(+ `Secondary PostgreSQL advisory lock error for "${lockName}" after callback failure: ${cleanupError}`,+ );+ }++ throw callbackResult.reason;+ }++ if (cleanupError) {+ throw cleanupError;+ }++ return {+ acquired: true,+ value: callbackResult.value,+ };+ }++ private async captureCallbackResult<T>(+ callback: () => Promise<T>,+ ): Promise<PromiseSettledResult<T>> {+ try {+ return {+ status: 'fulfilled',+ value: await callback(),+ };+ } catch (reason) {+ return {+ status: 'rejected',+ reason,+ };+ }+ }+}++class PostgresAdvisoryLockSession {+ private connectionError: Error | undefined;++ private readonly handleConnectionError = (error: Error) => {+ this.connectionError ??= error;+ };++ private constructor(+ private readonly connection: PoolClient,+ private readonly releaseConnection: (error?: Error) => void,+ private readonly lockName: string,+ ) {+ this.connection.on('error', this.handleConnectionError);+ }++ static async tryAcquire(+ dataSource: DataSource,+ lockName: string,+ ): Promise<PostgresAdvisoryLockSession | undefined> {+ const [connection, releaseConnection] = (await (+ dataSource.driver as PostgresDriver+ ).obtainMasterConnection()) as [PoolClient, (error?: Error) => void];+ const lockSession = new PostgresAdvisoryLockSession(+ connection,+ releaseConnection,+ lockName,+ );++ let isLockAcquired: boolean;++ try {+ isLockAcquired = await lockSession.tryAcquireLock();+ } catch (error) {+ lockSession.close(toError(error));+ throw error;+ }++ if (!isLockAcquired) {+ const closeError = lockSession.close();++ if (closeError) {+ throw closeError;+ }++ return undefined;+ }++ return lockSession;+ }++ async unlockAndClose(): Promise<Error | undefined> {+ let unlockError: Error | undefined;++ try {+ await this.unlock();+ } catch (error) {+ unlockError = toError(error);+ }++ return this.close(unlockError);+ }++ private async tryAcquireLock(): Promise<boolean> {+ const {+ rows: [result],+ } = await this.connection.query<{ acquired: boolean }>(+ `SELECT pg_try_advisory_lock(hashtextextended($1, 0)) AS "acquired"`,+ [this.lockName],+ );++ return result?.acquired === true;+ }++ private async unlock(): Promise<void> {+ const {+ rows: [result],+ } = await this.connection.query<{ released: boolean }>(+ `SELECT pg_advisory_unlock(hashtextextended($1, 0)) AS "released"`,+ [this.lockName],+ );++ if (result?.released !== true) {+ throw new Error(+ `Could not release PostgreSQL advisory lock ${this.lockName}`,+ );+ }+ }++ private close(error?: Error): Error | undefined {+ this.connection.removeListener('error', this.handleConnectionError);+ const errorToRelease = error ?? this.connectionError;++ if (errorToRelease) {+ this.releaseConnection(errorToRelease);+ } else {+ this.releaseConnection();+ }++ return errorToRelease;+ }+}diff --git a/packages/twenty-server/src/database/typeorm/typeorm.module.ts b/packages/twenty-server/src/database/typeorm/typeorm.module.tsindex 3838a07541f..a06d5003cb1 100644--- a/packages/twenty-server/src/database/typeorm/typeorm.module.ts+++ b/packages/twenty-server/src/database/typeorm/typeorm.module.ts@@ -6,6 +6,7 @@ import { DataSource, type DataSourceOptions } from 'typeorm';import { typeORMCoreModuleOptions } from 'src/database/typeorm/core/core.datasource';import { DatabaseGaugeService } from 'src/database/typeorm/database-gauge.service';import { DatabasePoolMetricsService } from 'src/database/typeorm/database-pool-metrics.service';+import { PostgresAdvisoryLockService } from 'src/database/typeorm/postgres-advisory-lock.service';import { MetricsModule } from 'src/engine/core-modules/metrics/metrics.module';import { installUpgradeAwareRepositoryProxy } from 'src/engine/twenty-orm/upgrade-aware/install-upgrade-aware-repository-proxy';@@ -24,7 +25,11 @@ import { installUpgradeAwareRepositoryProxy } from 'src/engine/twenty-orm/upgrad}),MetricsModule,],- providers: [DatabasePoolMetricsService, DatabaseGaugeService],- exports: [DatabasePoolMetricsService],+ providers: [+ DatabasePoolMetricsService,+ DatabaseGaugeService,+ PostgresAdvisoryLockService,+ ],+ exports: [DatabasePoolMetricsService, PostgresAdvisoryLockService],})export class TypeORMModule {}diff --git a/packages/twenty-server/src/engine/core-modules/workspace/services/__tests__/workspace.service.spec.ts b/packages/twenty-server/src/engine/core-modules/workspace/services/__tests__/workspace.service.spec.tsindex e1301ecbd05..8d2636c3bd2 100644--- a/packages/twenty-server/src/engine/core-modules/workspace/services/__tests__/workspace.service.spec.ts+++ b/packages/twenty-server/src/engine/core-modules/workspace/services/__tests__/workspace.service.spec.ts@@ -2,7 +2,7 @@ import { Test, type TestingModule } from '@nestjs/testing';import { getDataSourceToken, getRepositoryToken } from '@nestjs/typeorm';import { WorkspaceActivationStatus } from 'twenty-shared/workspace';-import { IsNull, Not, type Repository } from 'typeorm';+import { IsNull, Not, type QueryRunner, type Repository } from 'typeorm';import { BillingSubscriptionService } from 'src/engine/core-modules/billing/services/billing-subscription.service';import { BillingService } from 'src/engine/core-modules/billing/services/billing.service';@@ -47,8 +47,21 @@ describe('WorkspaceService', () => {let dnsManagerService: DnsManagerService;let billingSubscriptionService: BillingSubscriptionService;let userWorkspaceService: UserWorkspaceService;+ let flatEntityMapsCacheService: WorkspaceManyOrAllFlatEntityMapsCacheService;+ let queryRunner: QueryRunner;beforeEach(async () => {+ queryRunner = {+ connect: jest.fn(),+ startTransaction: jest.fn(),+ commitTransaction: jest.fn(),+ rollbackTransaction: jest.fn(),+ release: jest.fn(),+ manager: {+ delete: jest.fn().mockResolvedValue({ affected: 0 }),+ },+ } as unknown as QueryRunner;+const module: TestingModule = await Test.createTestingModule({providers: [WorkspaceService,@@ -156,16 +169,7 @@ describe('WorkspaceService', () => {{provide: getDataSourceToken(),useValue: {- createQueryRunner: jest.fn().mockReturnValue({- connect: jest.fn(),- startTransaction: jest.fn(),- commitTransaction: jest.fn(),- rollbackTransaction: jest.fn(),- release: jest.fn(),- manager: {- delete: jest.fn().mockResolvedValue({ affected: 0 }),- },- }),+ createQueryRunner: jest.fn().mockReturnValue(queryRunner),getRepository: jest.fn().mockReturnValue({find: jest.fn().mockResolvedValue([]),}),@@ -197,6 +201,10 @@ describe('WorkspaceService', () => {);userWorkspaceService =module.get<UserWorkspaceService>(UserWorkspaceService);+ flatEntityMapsCacheService =+ module.get<WorkspaceManyOrAllFlatEntityMapsCacheService>(+ WorkspaceManyOrAllFlatEntityMapsCacheService,+ );});afterEach(() => {@@ -312,6 +320,29 @@ describe('WorkspaceService', () => {expect(workspaceRepository.softDelete).not.toHaveBeenCalled();});+ it('should retrieve field metadata before starting the deletion transaction', async () => {+ const mockWorkspace = {+ id: 'workspace-id',+ metadataVersion: 0,+ } as WorkspaceEntity;++ jest+ .spyOn(workspaceRepository, 'findOne')+ .mockResolvedValue(mockWorkspace);+ jest.spyOn(userWorkspaceRepository, 'find').mockResolvedValue([]);++ await service.deleteWorkspace(mockWorkspace.id, false);++ const cacheRetrieval = jest.mocked(+ flatEntityMapsCacheService.getOrRecomputeManyOrAllFlatEntityMaps,+ );++ expect(cacheRetrieval).toHaveBeenCalledTimes(1);+ expect(cacheRetrieval.mock.invocationCallOrder[0]).toBeLessThan(+ (queryRunner.startTransaction as jest.Mock).mock.invocationCallOrder[0],+ );+ });+it('should soft delete the workspace', async () => {const mockWorkspace = {id: 'workspace-id',diff --git a/packages/twenty-server/src/engine/core-modules/workspace/services/workspace.service.ts b/packages/twenty-server/src/engine/core-modules/workspace/services/workspace.service.tsindex 05d175db044..fa0ab9bdbc3 100644--- a/packages/twenty-server/src/engine/core-modules/workspace/services/workspace.service.ts+++ b/packages/twenty-server/src/engine/core-modules/workspace/services/workspace.service.ts@@ -631,6 +631,9 @@ export class WorkspaceService {private async deleteWorkspaceSyncableMetadataEntities(workspace: WorkspaceEntity,): Promise<void> {+ const fieldMetadataIdChunks = await this.getFieldMetadataIdChunks(+ workspace.id,+ );const queryRunner = this.coreDataSource.createQueryRunner();await queryRunner.connect();@@ -643,6 +646,7 @@ export class WorkspaceService {const deletedCount = await this.deleteFieldMetadataInChunks(queryRunner,workspace.id,+ fieldMetadataIdChunks,);if (deletedCount > 0) {@@ -679,12 +683,10 @@ export class WorkspaceService {// FieldMetadataEntity has a self-referencing FK (relationTargetFieldMetadataId)// Related fields must be deleted together to avoid constraint violations- private async deleteFieldMetadataInChunks(- queryRunner: QueryRunner,+ private async getFieldMetadataIdChunks(workspaceId: string,- ): Promise<number> {+ ): Promise<string[][]> {const CHUNK_SIZE = 50;- let totalDeleted = 0;const { flatFieldMetadataMaps } =await this.flatEntityMapsCacheService.getOrRecomputeManyOrAllFlatEntityMaps(@@ -699,7 +701,7 @@ export class WorkspaceService {).filter(isDefined);if (fields.length === 0) {- return 0;+ return [];}const processedIds = new Set<string>();@@ -734,7 +736,17 @@ export class WorkspaceService {chunks.push(currentChunk);}- for (const [index, chunk] of chunks.entries()) {+ return chunks;+ }++ private async deleteFieldMetadataInChunks(+ queryRunner: QueryRunner,+ workspaceId: string,+ fieldMetadataIdChunks: string[][],+ ): Promise<number> {+ let totalDeleted = 0;++ for (const [index, chunk] of fieldMetadataIdChunks.entries()) {const result = await queryRunner.manager.createQueryBuilder().delete()@@ -747,7 +759,7 @@ export class WorkspaceService {totalDeleted += deletedInChunk;this.logger.log(- `workspace ${workspaceId}: fieldMetadata chunk ${index + 1}/${chunks.length} - deleted ${deletedInChunk} record(s)`,+ `workspace ${workspaceId}: fieldMetadata chunk ${index + 1}/${fieldMetadataIdChunks.length} - deleted ${deletedInChunk} record(s)`,);}diff --git a/packages/twenty-server/src/engine/workspace-manager/workspace-cleaner/crons/__tests__/clean-suspended-workspaces.job.spec.ts b/packages/twenty-server/src/engine/workspace-manager/workspace-cleaner/crons/__tests__/clean-suspended-workspaces.job.spec.tsnew file mode 100644index 00000000000..43363cf1518--- /dev/null+++ b/packages/twenty-server/src/engine/workspace-manager/workspace-cleaner/crons/__tests__/clean-suspended-workspaces.job.spec.ts@@ -0,0 +1,74 @@+import { type Repository } from 'typeorm';++import { type PostgresAdvisoryLockService } from 'src/database/typeorm/postgres-advisory-lock.service';+import { WorkspaceEntity } from 'src/engine/core-modules/workspace/workspace.entity';+import { CleanSuspendedWorkspacesJob } from 'src/engine/workspace-manager/workspace-cleaner/crons/clean-suspended-workspaces.job';+import { type CleanerWorkspaceService } from 'src/engine/workspace-manager/workspace-cleaner/services/cleaner.workspace-service';++jest.mock(+ 'src/engine/workspace-manager/workspace-cleaner/services/cleaner.workspace-service',+ () => ({+ CleanerWorkspaceService: class {},+ }),+);++describe('CleanSuspendedWorkspacesJob', () => {+ const workspaceRepository = {+ find: jest.fn(),+ };+ const cleanerWorkspaceService = {+ batchWarnOrCleanSuspendedWorkspaces: jest.fn(),+ };+ const postgresAdvisoryLockService = {+ tryWithLock: jest.fn(),+ };++ const createJob = () =>+ new CleanSuspendedWorkspacesJob(+ cleanerWorkspaceService as unknown as CleanerWorkspaceService,+ workspaceRepository as unknown as Repository<WorkspaceEntity>,+ postgresAdvisoryLockService as unknown as PostgresAdvisoryLockService,+ );++ beforeEach(() => {+ jest.clearAllMocks();+ workspaceRepository.find.mockResolvedValue([{ id: 'workspace-id' }]);+ });++ it('skips cleanup when another execution holds the lock', async () => {+ postgresAdvisoryLockService.tryWithLock.mockResolvedValue({+ acquired: false,+ });++ await createJob().handle();++ expect(workspaceRepository.find).not.toHaveBeenCalled();+ expect(+ cleanerWorkspaceService.batchWarnOrCleanSuspendedWorkspaces,+ ).not.toHaveBeenCalled();+ });++ it('cleans suspended workspaces while holding the lock', async () => {+ postgresAdvisoryLockService.tryWithLock.mockImplementation(+ async (_lockName, callback) => ({+ acquired: true,+ value: await callback(),+ }),+ );++ await createJob().handle();++ expect(workspaceRepository.find).toHaveBeenCalledWith({+ select: ['id'],+ where: {+ activationStatus: 'SUSPENDED',+ },+ withDeleted: true,+ });+ expect(+ cleanerWorkspaceService.batchWarnOrCleanSuspendedWorkspaces,+ ).toHaveBeenCalledWith({+ workspaceIds: ['workspace-id'],+ });+ });+});diff --git a/packages/twenty-server/src/engine/workspace-manager/workspace-cleaner/crons/clean-suspended-workspaces.job.ts b/packages/twenty-server/src/engine/workspace-manager/workspace-cleaner/crons/clean-suspended-workspaces.job.tsindex 3fcdde8788c..ad5f3efcf69 100644--- a/packages/twenty-server/src/engine/workspace-manager/workspace-cleaner/crons/clean-suspended-workspaces.job.ts+++ b/packages/twenty-server/src/engine/workspace-manager/workspace-cleaner/crons/clean-suspended-workspaces.job.ts@@ -1,8 +1,10 @@+import { Logger } from '@nestjs/common';import { InjectRepository } from '@nestjs/typeorm';import { Repository } from 'typeorm';import { WorkspaceActivationStatus } from 'twenty-shared/workspace';+import { PostgresAdvisoryLockService } from 'src/database/typeorm/postgres-advisory-lock.service';import { SentryCronMonitor } from 'src/engine/core-modules/cron/sentry-cron-monitor.decorator';import { Process } from 'src/engine/core-modules/message-queue/decorators/process.decorator';import { Processor } from 'src/engine/core-modules/message-queue/decorators/processor.decorator';@@ -11,12 +13,17 @@ import { WorkspaceEntity } from 'src/engine/core-modules/workspace/workspace.entimport { cleanSuspendedWorkspaceCronPattern } from 'src/engine/workspace-manager/workspace-cleaner/crons/clean-suspended-workspaces.cron.pattern';import { CleanerWorkspaceService } from 'src/engine/workspace-manager/workspace-cleaner/services/cleaner.workspace-service';+const CLEAN_SUSPENDED_WORKSPACES_LOCK_NAME = 'clean-suspended-workspaces-job';+@Processor(MessageQueue.cronQueue)export class CleanSuspendedWorkspacesJob {+ private readonly logger = new Logger(CleanSuspendedWorkspacesJob.name);+constructor(private readonly cleanerWorkspaceService: CleanerWorkspaceService,@InjectRepository(WorkspaceEntity)private readonly workspaceRepository: Repository<WorkspaceEntity>,+ private readonly postgresAdvisoryLockService: PostgresAdvisoryLockService,) {}@Process(CleanSuspendedWorkspacesJob.name)@@ -25,16 +32,27 @@ export class CleanSuspendedWorkspacesJob {cleanSuspendedWorkspaceCronPattern,)async handle(): Promise<void> {- const suspendedWorkspaceIds = await this.workspaceRepository.find({- select: ['id'],- where: {- activationStatus: WorkspaceActivationStatus.SUSPENDED,+ const result = await this.postgresAdvisoryLockService.tryWithLock(+ CLEAN_SUSPENDED_WORKSPACES_LOCK_NAME,+ async () => {+ const suspendedWorkspaceIds = await this.workspaceRepository.find({+ select: ['id'],+ where: {+ activationStatus: WorkspaceActivationStatus.SUSPENDED,+ },+ withDeleted: true,+ });++ await this.cleanerWorkspaceService.batchWarnOrCleanSuspendedWorkspaces({+ workspaceIds: suspendedWorkspaceIds.map((workspace) => workspace.id),+ });},- withDeleted: true,- });+ );- await this.cleanerWorkspaceService.batchWarnOrCleanSuspendedWorkspaces({- workspaceIds: suspendedWorkspaceIds.map((workspace) => workspace.id),- });+ if (!result.acquired) {+ this.logger.log(+ 'Skipping suspended workspace cleanup because another execution is running',+ );+ }}}