The open alternative to Salesforce, designed for AI.
TypeScript crmcrm-systemcustomergood-first-issuegraphqlhacktoberfestjavascriptmarketingmonoreponestjsopen-sourcepostgresqlreactreactjssalestypescriptweb
Improve workflow crons efficiency (#19381)
Thomas Trompettee048d03872cb
diff --git a/packages/twenty-server/src/engine/core-modules/twenty-config/config-variables.ts b/packages/twenty-server/src/engine/core-modules/twenty-config/config-variables.tsindex 537de397700..346517abbb7 100644--- a/packages/twenty-server/src/engine/core-modules/twenty-config/config-variables.ts+++ b/packages/twenty-server/src/engine/core-modules/twenty-config/config-variables.ts@@ -21,9 +21,6 @@ import { CaptchaDriverType } from 'src/engine/core-modules/captcha/interfaces';import { CodeInterpreterDriverType } from 'src/engine/core-modules/code-interpreter/code-interpreter.interface';import { WebSearchDriverType } from 'src/engine/core-modules/web-search/web-search.interface';import { EmailDriver } from 'src/engine/core-modules/email/enums/email-driver.enum';-import { type AiModelPreferences } from 'src/engine/metadata-modules/ai/ai-models/types/ai-model-preferences.type';-import { type AiProvidersConfig } from 'src/engine/metadata-modules/ai/ai-models/types/ai-providers-config.type';-import { loadDefaultModelPreferences } from 'src/engine/metadata-modules/ai/ai-models/utils/load-default-model-preferences.util';import { ExceptionHandlerDriver } from 'src/engine/core-modules/exception-handler/interfaces';import { StorageDriverType } from 'src/engine/core-modules/file-storage/interfaces';import { LoggerDriverType } from 'src/engine/core-modules/logger/interfaces';@@ -45,6 +42,9 @@ import {ConfigVariableException,ConfigVariableExceptionCode,} from 'src/engine/core-modules/twenty-config/twenty-config.exception';+import { type AiModelPreferences } from 'src/engine/metadata-modules/ai/ai-models/types/ai-model-preferences.type';+import { type AiProvidersConfig } from 'src/engine/metadata-modules/ai/ai-models/types/ai-providers-config.type';+import { loadDefaultModelPreferences } from 'src/engine/metadata-modules/ai/ai-models/utils/load-default-model-preferences.util';export class ConfigVariables {@ConfigVariablesMetadata({diff --git a/packages/twenty-server/src/modules/workflow/workflow-runner/workflow-run-queue/cron/jobs/workflow-clean-workflow-runs.cron.job.ts b/packages/twenty-server/src/modules/workflow/workflow-runner/workflow-run-queue/cron/jobs/workflow-clean-workflow-runs.cron.job.tsindex d4e893268f2..70da1ccbb98 100644--- a/packages/twenty-server/src/modules/workflow/workflow-runner/workflow-run-queue/cron/jobs/workflow-clean-workflow-runs.cron.job.ts+++ b/packages/twenty-server/src/modules/workflow/workflow-runner/workflow-run-queue/cron/jobs/workflow-clean-workflow-runs.cron.job.ts@@ -4,6 +4,9 @@ import { InjectRepository } from '@nestjs/typeorm';import { WorkspaceActivationStatus } from 'twenty-shared/workspace';import { In, Repository } from 'typeorm';+import { InjectCacheStorage } from 'src/engine/core-modules/cache-storage/decorators/cache-storage.decorator';+import { CacheStorageService } from 'src/engine/core-modules/cache-storage/services/cache-storage.service';+import { CacheStorageNamespace } from 'src/engine/core-modules/cache-storage/types/cache-storage-namespace.enum';import { SentryCronMonitor } from 'src/engine/core-modules/cron/sentry-cron-monitor.decorator';import { ExceptionHandlerService } from 'src/engine/core-modules/exception-handler/exception-handler.service';import { InjectMessageQueue } from 'src/engine/core-modules/message-queue/decorators/message-queue.decorator';@@ -25,9 +28,11 @@ import {} from 'src/modules/workflow/workflow-runner/workflow-run-queue/jobs/workflow-clean-workflow-runs.job';import { getRunsToCleanFindOptions } from 'src/modules/workflow/workflow-runner/workflow-run-queue/utils/get-runs-to-clean-find-options.util';-export const CLEAN_WORKFLOW_RUN_CRON_PATTERN = '0 0 * * *';+export const CLEAN_WORKFLOW_RUN_CRON_PATTERN = '0 */3 * * *';-const WORKSPACE_BATCH_SIZE = 50;+const LAST_PARTITION_CACHE_KEY = 'workflow-clean-workflow-runs:last-partition';+const NUMBER_OF_PARTITIONS = 10;+const WORKSPACE_BATCH_SIZE = 10;@Processor(MessageQueue.cronQueue)export class WorkflowCleanWorkflowRunsCronJob {@@ -40,6 +45,8 @@ export class WorkflowCleanWorkflowRunsCronJob {private readonly messageQueueService: MessageQueueService,private readonly globalWorkspaceOrmManager: GlobalWorkspaceOrmManager,private readonly exceptionHandlerService: ExceptionHandlerService,+ @InjectCacheStorage(CacheStorageNamespace.ModuleWorkflow)+ private readonly cacheStorageService: CacheStorageService,) {}@Process(WorkflowCleanWorkflowRunsCronJob.name)@@ -50,21 +57,27 @@ export class WorkflowCleanWorkflowRunsCronJob {async handle() {this.logger.log('Starting WorkflowCleanWorkflowRunsCronJob cron');- const activeWorkspaces = await this.workspaceRepository.find({+ const allActiveWorkspaces = await this.workspaceRepository.find({where: {activationStatus: WorkspaceActivationStatus.ACTIVE,},select: ['id'],+ order: { id: 'ASC' },});+ const partition = await this.getAndIncrementPartition();+ const workspacesForThisRun = allActiveWorkspaces.filter(+ (_, index) => index % NUMBER_OF_PARTITIONS === partition,+ );+let enqueuedCount = 0;for (let workspaceIndex = 0;- workspaceIndex < activeWorkspaces.length;+ workspaceIndex < workspacesForThisRun.length;workspaceIndex += WORKSPACE_BATCH_SIZE) {- const batch = activeWorkspaces.slice(+ const batch = workspacesForThisRun.slice(workspaceIndex,workspaceIndex + WORKSPACE_BATCH_SIZE,);@@ -87,7 +100,7 @@ export class WorkflowCleanWorkflowRunsCronJob {}this.logger.log(- `Completed WorkflowCleanWorkflowRunsCronJob cron, enqueued ${enqueuedCount} jobs`,+ `Completed WorkflowCleanWorkflowRunsCronJob cron (partition ${partition}/${NUMBER_OF_PARTITIONS}), enqueued ${enqueuedCount} jobs`,);}@@ -106,6 +119,21 @@ export class WorkflowCleanWorkflowRunsCronJob {return false;}+ private async getAndIncrementPartition(): Promise<number> {+ const lastPartition = await this.cacheStorageService.get<number>(+ LAST_PARTITION_CACHE_KEY,+ );++ const partition =+ lastPartition !== undefined+ ? (lastPartition + 1) % NUMBER_OF_PARTITIONS+ : 0;++ await this.cacheStorageService.set(LAST_PARTITION_CACHE_KEY, partition);++ return partition;+ }+private async hasRunsToClean(workspaceId: string): Promise<boolean> {const authContext = buildSystemAuthContext(workspaceId);diff --git a/packages/twenty-server/src/modules/workflow/workflow-runner/workflow-run-queue/cron/jobs/workflow-handle-staled-runs.cron.job.ts b/packages/twenty-server/src/modules/workflow/workflow-runner/workflow-run-queue/cron/jobs/workflow-handle-staled-runs.cron.job.tsindex 91d26a65a03..923fd6d3ecb 100644--- a/packages/twenty-server/src/modules/workflow/workflow-runner/workflow-run-queue/cron/jobs/workflow-handle-staled-runs.cron.job.ts+++ b/packages/twenty-server/src/modules/workflow/workflow-runner/workflow-run-queue/cron/jobs/workflow-handle-staled-runs.cron.job.ts@@ -1,9 +1,12 @@import { Logger } from '@nestjs/common';-import { InjectRepository } from '@nestjs/typeorm';+import { InjectDataSource, InjectRepository } from '@nestjs/typeorm';import { WorkspaceActivationStatus } from 'twenty-shared/workspace';-import { Repository } from 'typeorm';+import { DataSource, Repository } from 'typeorm';+import { InjectCacheStorage } from 'src/engine/core-modules/cache-storage/decorators/cache-storage.decorator';+import { CacheStorageService } from 'src/engine/core-modules/cache-storage/services/cache-storage.service';+import { CacheStorageNamespace } from 'src/engine/core-modules/cache-storage/types/cache-storage-namespace.enum';import { SentryCronMonitor } from 'src/engine/core-modules/cron/sentry-cron-monitor.decorator';import { ExceptionHandlerService } from 'src/engine/core-modules/exception-handler/exception-handler.service';import { InjectMessageQueue } from 'src/engine/core-modules/message-queue/decorators/message-queue.decorator';@@ -12,17 +15,17 @@ import { Processor } from 'src/engine/core-modules/message-queue/decorators/procimport { MessageQueue } from 'src/engine/core-modules/message-queue/message-queue.constants';import { MessageQueueService } from 'src/engine/core-modules/message-queue/services/message-queue.service';import { WorkspaceEntity } from 'src/engine/core-modules/workspace/workspace.entity';-import { GlobalWorkspaceOrmManager } from 'src/engine/twenty-orm/global-workspace-datasource/global-workspace-orm.manager';-import { buildSystemAuthContext } from 'src/engine/twenty-orm/utils/build-system-auth-context.util';-import { WorkflowRunWorkspaceEntity } from 'src/modules/workflow/common/standard-objects/workflow-run.workspace-entity';+import { getWorkspaceSchemaName } from 'src/engine/workspace-datasource/utils/get-workspace-schema-name.util';+import { WorkflowRunStatus } from 'src/modules/workflow/common/standard-objects/workflow-run.workspace-entity';+import { STALED_RUNS_THRESHOLD_MS } from 'src/modules/workflow/workflow-runner/workflow-run-queue/constants/staled-runs-threshold';import {WorkflowHandleStaledRunsJob,WorkflowHandleStaledRunsJobData,} from 'src/modules/workflow/workflow-runner/workflow-run-queue/jobs/workflow-handle-staled-runs.job';-import { getStaledRunsFindOptions } from 'src/modules/workflow/workflow-runner/workflow-run-queue/utils/get-staled-runs-find-options.util';--export const WORKFLOW_HANDLE_STALED_RUNS_CRON_PATTERN = '0 * * * *';+export const WORKFLOW_HANDLE_STALED_RUNS_CRON_PATTERN = '*/10 * * * *';+const LAST_PARTITION_CACHE_KEY = 'workflow-handle-staled-runs:last-partition';+const NUMBER_OF_PARTITIONS = 10;const WORKSPACE_BATCH_SIZE = 50;@Processor(MessageQueue.cronQueue)@@ -30,12 +33,15 @@ export class WorkflowHandleStaledRunsCronJob {private readonly logger = new Logger(WorkflowHandleStaledRunsCronJob.name);constructor(+ @InjectDataSource()+ private readonly coreDataSource: DataSource,@InjectRepository(WorkspaceEntity)private readonly workspaceRepository: Repository<WorkspaceEntity>,@InjectMessageQueue(MessageQueue.workflowQueue)private readonly messageQueueService: MessageQueueService,- private readonly globalWorkspaceOrmManager: GlobalWorkspaceOrmManager,private readonly exceptionHandlerService: ExceptionHandlerService,+ @InjectCacheStorage(CacheStorageNamespace.ModuleWorkflow)+ private readonly cacheStorageService: CacheStorageService,) {}@Process(WorkflowHandleStaledRunsCronJob.name)@@ -46,21 +52,27 @@ export class WorkflowHandleStaledRunsCronJob {async handle() {this.logger.log('Starting WorkflowHandleStaledRunsCronJob cron');- const activeWorkspaces = await this.workspaceRepository.find({+ const allActiveWorkspaces = await this.workspaceRepository.find({where: {activationStatus: WorkspaceActivationStatus.ACTIVE,},select: ['id'],+ order: { id: 'ASC' },});+ const partition = await this.getAndIncrementPartition();+ const workspacesForThisRun = allActiveWorkspaces.filter(+ (_, index) => index % NUMBER_OF_PARTITIONS === partition,+ );+let enqueuedCount = 0;for (let workspaceIndex = 0;- workspaceIndex < activeWorkspaces.length;+ workspaceIndex < workspacesForThisRun.length;workspaceIndex += WORKSPACE_BATCH_SIZE) {- const batch = activeWorkspaces.slice(+ const batch = workspacesForThisRun.slice(workspaceIndex,workspaceIndex + WORKSPACE_BATCH_SIZE,);@@ -83,7 +95,7 @@ export class WorkflowHandleStaledRunsCronJob {}this.logger.log(- `Completed WorkflowHandleStaledRunsCronJob cron, enqueued ${enqueuedCount} jobs`,+ `Completed WorkflowHandleStaledRunsCronJob cron (partition ${partition}/${NUMBER_OF_PARTITIONS}), enqueued ${enqueuedCount} jobs`,);}@@ -102,24 +114,30 @@ export class WorkflowHandleStaledRunsCronJob {return false;}+ private async getAndIncrementPartition(): Promise<number> {+ const lastPartition = await this.cacheStorageService.get<number>(+ LAST_PARTITION_CACHE_KEY,+ );++ const partition =+ lastPartition !== undefined+ ? (lastPartition + 1) % NUMBER_OF_PARTITIONS+ : 0;++ await this.cacheStorageService.set(LAST_PARTITION_CACHE_KEY, partition);++ return partition;+ }+private async hasStaledRuns(workspaceId: string): Promise<boolean> {- const authContext = buildSystemAuthContext(workspaceId);-- return this.globalWorkspaceOrmManager.executeInWorkspaceContext(- async () => {- const workflowRunRepository =- await this.globalWorkspaceOrmManager.getRepository(- workspaceId,- WorkflowRunWorkspaceEntity,- { shouldBypassPermissionChecks: true },- );-- return workflowRunRepository.exists({- where: getStaledRunsFindOptions(),- });- },- authContext,- { lite: true },+ const schemaName = getWorkspaceSchemaName(workspaceId);+ const thresholdDate = new Date(Date.now() - STALED_RUNS_THRESHOLD_MS);++ const result = await this.coreDataSource.query(+ `SELECT 1 FROM ${schemaName}."workflowRun" WHERE "status" = $1 AND ("enqueuedAt" < $2 OR "enqueuedAt" IS NULL) LIMIT 1`,+ [WorkflowRunStatus.ENQUEUED, thresholdDate],);++ return result.length > 0;}}diff --git a/packages/twenty-server/src/modules/workflow/workflow-runner/workflow-run-queue/cron/jobs/workflow-run-enqueue.cron.job.ts b/packages/twenty-server/src/modules/workflow/workflow-runner/workflow-run-queue/cron/jobs/workflow-run-enqueue.cron.job.tsindex 1f8bf00ae13..d4a22c1ee79 100644--- a/packages/twenty-server/src/modules/workflow/workflow-runner/workflow-run-queue/cron/jobs/workflow-run-enqueue.cron.job.ts+++ b/packages/twenty-server/src/modules/workflow/workflow-runner/workflow-run-queue/cron/jobs/workflow-run-enqueue.cron.job.ts@@ -1,9 +1,12 @@import { Logger } from '@nestjs/common';-import { InjectRepository } from '@nestjs/typeorm';+import { InjectDataSource, InjectRepository } from '@nestjs/typeorm';import { WorkspaceActivationStatus } from 'twenty-shared/workspace';-import { Repository } from 'typeorm';+import { DataSource, Repository } from 'typeorm';+import { InjectCacheStorage } from 'src/engine/core-modules/cache-storage/decorators/cache-storage.decorator';+import { CacheStorageService } from 'src/engine/core-modules/cache-storage/services/cache-storage.service';+import { CacheStorageNamespace } from 'src/engine/core-modules/cache-storage/types/cache-storage-namespace.enum';import { SentryCronMonitor } from 'src/engine/core-modules/cron/sentry-cron-monitor.decorator';import { ExceptionHandlerService } from 'src/engine/core-modules/exception-handler/exception-handler.service';import { InjectMessageQueue } from 'src/engine/core-modules/message-queue/decorators/message-queue.decorator';@@ -12,17 +15,16 @@ import { Processor } from 'src/engine/core-modules/message-queue/decorators/procimport { MessageQueue } from 'src/engine/core-modules/message-queue/message-queue.constants';import { MessageQueueService } from 'src/engine/core-modules/message-queue/services/message-queue.service';import { WorkspaceEntity } from 'src/engine/core-modules/workspace/workspace.entity';-import { GlobalWorkspaceOrmManager } from 'src/engine/twenty-orm/global-workspace-datasource/global-workspace-orm.manager';-import { buildSystemAuthContext } from 'src/engine/twenty-orm/utils/build-system-auth-context.util';-import { WorkflowRunWorkspaceEntity } from 'src/modules/workflow/common/standard-objects/workflow-run.workspace-entity';-import { NOT_STARTED_RUNS_FIND_OPTIONS } from 'src/modules/workflow/workflow-runner/workflow-run-queue/constants/not-started-runs-find-options';+import { getWorkspaceSchemaName } from 'src/engine/workspace-datasource/utils/get-workspace-schema-name.util';+import { WorkflowRunStatus } from 'src/modules/workflow/common/standard-objects/workflow-run.workspace-entity';import {WorkflowRunEnqueueJob,WorkflowRunEnqueueJobData,} from 'src/modules/workflow/workflow-runner/workflow-run-queue/jobs/workflow-run-enqueue.job';+export const WORKFLOW_RUN_ENQUEUE_CRON_PATTERN = '* * * * *';-export const WORKFLOW_RUN_ENQUEUE_CRON_PATTERN = '*/5 * * * *';-+const LAST_PARTITION_CACHE_KEY = 'workflow-run-enqueue:last-partition';+const NUMBER_OF_PARTITIONS = 10;const WORKSPACE_BATCH_SIZE = 10;@Processor(MessageQueue.cronQueue)@@ -30,12 +32,15 @@ export class WorkflowRunEnqueueCronJob {private readonly logger = new Logger(WorkflowRunEnqueueCronJob.name);constructor(+ @InjectDataSource()+ private readonly coreDataSource: DataSource,@InjectRepository(WorkspaceEntity)private readonly workspaceRepository: Repository<WorkspaceEntity>,@InjectMessageQueue(MessageQueue.workflowQueue)private readonly messageQueueService: MessageQueueService,- private readonly globalWorkspaceOrmManager: GlobalWorkspaceOrmManager,private readonly exceptionHandlerService: ExceptionHandlerService,+ @InjectCacheStorage(CacheStorageNamespace.ModuleWorkflow)+ private readonly cacheStorageService: CacheStorageService,) {}@Process(WorkflowRunEnqueueCronJob.name)@@ -46,21 +51,27 @@ export class WorkflowRunEnqueueCronJob {async handle() {this.logger.log('Starting WorkflowRunEnqueueCronJob cron');- const activeWorkspaces = await this.workspaceRepository.find({+ const allActiveWorkspaces = await this.workspaceRepository.find({where: {activationStatus: WorkspaceActivationStatus.ACTIVE,},select: ['id'],+ order: { id: 'ASC' },});+ const partition = await this.getAndIncrementPartition();+ const workspacesForThisRun = allActiveWorkspaces.filter(+ (_, index) => index % NUMBER_OF_PARTITIONS === partition,+ );+let enqueuedCount = 0;for (let workspaceIndex = 0;- workspaceIndex < activeWorkspaces.length;+ workspaceIndex < workspacesForThisRun.length;workspaceIndex += WORKSPACE_BATCH_SIZE) {- const batch = activeWorkspaces.slice(+ const batch = workspacesForThisRun.slice(workspaceIndex,workspaceIndex + WORKSPACE_BATCH_SIZE,);@@ -83,7 +94,7 @@ export class WorkflowRunEnqueueCronJob {}this.logger.log(- `Completed WorkflowRunEnqueueCronJob cron, enqueued ${enqueuedCount} jobs`,+ `Completed WorkflowRunEnqueueCronJob cron (partition ${partition}/${NUMBER_OF_PARTITIONS}), enqueued ${enqueuedCount} jobs`,);}@@ -102,24 +113,29 @@ export class WorkflowRunEnqueueCronJob {return false;}+ private async getAndIncrementPartition(): Promise<number> {+ const lastPartition = await this.cacheStorageService.get<number>(+ LAST_PARTITION_CACHE_KEY,+ );++ const partition =+ lastPartition !== undefined+ ? (lastPartition + 1) % NUMBER_OF_PARTITIONS+ : 0;++ await this.cacheStorageService.set(LAST_PARTITION_CACHE_KEY, partition);++ return partition;+ }+private async hasNotStartedRuns(workspaceId: string): Promise<boolean> {- const authContext = buildSystemAuthContext(workspaceId);-- return this.globalWorkspaceOrmManager.executeInWorkspaceContext(- async () => {- const workflowRunRepository =- await this.globalWorkspaceOrmManager.getRepository(- workspaceId,- WorkflowRunWorkspaceEntity,- { shouldBypassPermissionChecks: true },- );-- return workflowRunRepository.exists({- where: NOT_STARTED_RUNS_FIND_OPTIONS,- });- },- authContext,- { lite: true },+ const schemaName = getWorkspaceSchemaName(workspaceId);++ const result = await this.coreDataSource.query(+ `SELECT 1 FROM ${schemaName}."workflowRun" WHERE "status" = $1 LIMIT 1`,+ [WorkflowRunStatus.NOT_STARTED],);++ return result.length > 0;}}