The open alternative to Salesforce, designed for AI.
TypeScript crmcrm-systemcustomergood-first-issuegraphqlhacktoberfestjavascriptmarketingmonoreponestjsopen-sourcepostgresqlreactreactjssalestypescriptweb
fix(workflow): batch staled run reset to avoid Postgres param limit (#22654)
Thomas Trompette6bc4d8efac9d
diff --git a/packages/twenty-server/src/modules/workflow/workflow-runner/workflow-run-queue/workspace-services/__tests__/workflow-handle-staled-runs.workspace-service.spec.ts b/packages/twenty-server/src/modules/workflow/workflow-runner/workflow-run-queue/workspace-services/__tests__/workflow-handle-staled-runs.workspace-service.spec.tsnew file mode 100644index 00000000000..17cb8789e08--- /dev/null+++ b/packages/twenty-server/src/modules/workflow/workflow-runner/workflow-run-queue/workspace-services/__tests__/workflow-handle-staled-runs.workspace-service.spec.ts@@ -0,0 +1,138 @@+import { Test, type TestingModule } from '@nestjs/testing';++import { GlobalWorkspaceOrmManager } from 'src/engine/twenty-orm/global-workspace-datasource/global-workspace-orm.manager';+import { WorkflowRunStatus } from 'src/modules/workflow/common/standard-objects/workflow-run.workspace-entity';+import { WorkflowHandleStaledRunsWorkspaceService } from 'src/modules/workflow/workflow-runner/workflow-run-queue/workspace-services/workflow-handle-staled-runs.workspace-service';+import { WorkflowThrottlingWorkspaceService } from 'src/modules/workflow/workflow-runner/workflow-run-queue/workspace-services/workflow-throttling.workspace-service';++const mockRepository = {+ count: jest.fn(),+ find: jest.fn(),+ update: jest.fn().mockResolvedValue(undefined),+};++const mockGlobalWorkspaceOrmManager = {+ getRepository: jest.fn().mockResolvedValue(mockRepository),+ executeInWorkspaceContext: jest+ .fn()+ // oxlint-disable-next-line typescript/no-explicit-any+ .mockImplementation((fn: () => any) => fn()),+};++const mockWorkflowThrottlingWorkspaceService = {+ recomputeWorkflowRunNotStartedCount: jest.fn().mockResolvedValue(undefined),+};++// Mirrors QUERY_MAX_RECORDS, the per-batch cap the service applies. Kept as a+// local literal rather than imported from twenty-shared/constants to avoid a+// jest circular-init issue with config-variables loading the same barrel.+const QUERY_MAX_RECORDS = 200;++const buildStaledRuns = (count: number) =>+ Array.from({ length: count }, (_, index) => ({ id: `run-${index}` }));++describe('WorkflowHandleStaledRunsWorkspaceService', () => {+ let service: WorkflowHandleStaledRunsWorkspaceService;++ const workspaceId = 'workspace-1';++ beforeEach(async () => {+ jest.clearAllMocks();++ const module: TestingModule = await Test.createTestingModule({+ providers: [+ WorkflowHandleStaledRunsWorkspaceService,+ {+ provide: GlobalWorkspaceOrmManager,+ useValue: mockGlobalWorkspaceOrmManager,+ },+ {+ provide: WorkflowThrottlingWorkspaceService,+ useValue: mockWorkflowThrottlingWorkspaceService,+ },+ ],+ }).compile();++ service = module.get<WorkflowHandleStaledRunsWorkspaceService>(+ WorkflowHandleStaledRunsWorkspaceService,+ );+ });++ it('should do nothing when there are no staled runs', async () => {+ mockRepository.count.mockResolvedValueOnce(0);++ await service.handleStaledRunsForWorkspace(workspaceId);++ expect(mockRepository.find).not.toHaveBeenCalled();+ expect(mockRepository.update).not.toHaveBeenCalled();+ expect(+ mockWorkflowThrottlingWorkspaceService.recomputeWorkflowRunNotStartedCount,+ ).not.toHaveBeenCalled();+ });++ it('should reset a single batch and recompute the not-started count once', async () => {+ const staledRuns = buildStaledRuns(3);++ mockRepository.count.mockResolvedValueOnce(3);+ mockRepository.find.mockResolvedValueOnce(staledRuns);++ await service.handleStaledRunsForWorkspace(workspaceId);++ expect(mockRepository.update).toHaveBeenCalledTimes(1);+ expect(mockRepository.update).toHaveBeenCalledWith(+ ['run-0', 'run-1', 'run-2'],+ {+ enqueuedAt: null,+ status: WorkflowRunStatus.NOT_STARTED,+ },+ );+ expect(+ mockWorkflowThrottlingWorkspaceService.recomputeWorkflowRunNotStartedCount,+ ).toHaveBeenCalledTimes(1);+ expect(+ mockWorkflowThrottlingWorkspaceService.recomputeWorkflowRunNotStartedCount,+ ).toHaveBeenCalledWith(workspaceId);+ });++ it('should never fetch more than QUERY_MAX_RECORDS ids per update', async () => {+ mockRepository.count.mockResolvedValueOnce(QUERY_MAX_RECORDS);+ mockRepository.find.mockResolvedValueOnce(+ buildStaledRuns(QUERY_MAX_RECORDS),+ );++ await service.handleStaledRunsForWorkspace(workspaceId);++ expect(mockRepository.find).toHaveBeenCalledWith(+ expect.objectContaining({ take: QUERY_MAX_RECORDS }),+ );+ expect(mockRepository.update.mock.calls[0][0]).toHaveLength(+ QUERY_MAX_RECORDS,+ );+ });++ it('should drain a multi-batch backlog based on the initial count', async () => {+ // 450 staled runs => 3 batches (200, 200, 50)+ mockRepository.count.mockResolvedValueOnce(450);+ mockRepository.find+ .mockResolvedValueOnce(buildStaledRuns(QUERY_MAX_RECORDS))+ .mockResolvedValueOnce(buildStaledRuns(QUERY_MAX_RECORDS))+ .mockResolvedValueOnce(buildStaledRuns(50));++ await service.handleStaledRunsForWorkspace(workspaceId);++ expect(mockRepository.find).toHaveBeenCalledTimes(3);+ expect(mockRepository.update).toHaveBeenCalledTimes(3);+ expect(mockRepository.update.mock.calls[0][0]).toHaveLength(+ QUERY_MAX_RECORDS,+ );+ expect(mockRepository.update.mock.calls[1][0]).toHaveLength(+ QUERY_MAX_RECORDS,+ );+ expect(mockRepository.update.mock.calls[2][0]).toHaveLength(50);++ // Recompute runs exactly once, after the whole backlog is drained+ expect(+ mockWorkflowThrottlingWorkspaceService.recomputeWorkflowRunNotStartedCount,+ ).toHaveBeenCalledTimes(1);+ });+});diff --git a/packages/twenty-server/src/modules/workflow/workflow-runner/workflow-run-queue/workspace-services/workflow-handle-staled-runs.workspace-service.ts b/packages/twenty-server/src/modules/workflow/workflow-runner/workflow-run-queue/workspace-services/workflow-handle-staled-runs.workspace-service.tsindex 35192a88f18..b4424ad3155 100644--- a/packages/twenty-server/src/modules/workflow/workflow-runner/workflow-run-queue/workspace-services/workflow-handle-staled-runs.workspace-service.ts+++ b/packages/twenty-server/src/modules/workflow/workflow-runner/workflow-run-queue/workspace-services/workflow-handle-staled-runs.workspace-service.ts@@ -1,5 +1,7 @@import { Injectable, Logger } from '@nestjs/common';+import { QUERY_MAX_RECORDS } from 'twenty-shared/constants';+import { GlobalWorkspaceOrmManager } from 'src/engine/twenty-orm/global-workspace-datasource/global-workspace-orm.manager';import { buildSystemAuthContext } from 'src/engine/twenty-orm/utils/build-system-auth-context.util';import {@@ -30,21 +32,40 @@ export class WorkflowHandleStaledRunsWorkspaceService {{ shouldBypassPermissionChecks: true },);- const staledWorkflowRuns = await workflowRunRepository.find({+ const staledRunsCount = await workflowRunRepository.count({where: getStaledRunsFindOptions(),});- if (staledWorkflowRuns.length <= 0) {+ if (staledRunsCount <= 0) {return;}- await workflowRunRepository.update(- staledWorkflowRuns.map((workflowRun) => workflowRun.id),- {- enqueuedAt: null,- status: WorkflowRunStatus.NOT_STARTED,- },- );+ const batchCount = Math.ceil(staledRunsCount / QUERY_MAX_RECORDS);++ for (let batchIndex = 0; batchIndex < batchCount; batchIndex++) {+ const staledWorkflowRuns = await workflowRunRepository.find({+ where: getStaledRunsFindOptions(),+ select: {+ id: true,+ },+ order: {+ createdAt: 'ASC',+ },+ take: QUERY_MAX_RECORDS,+ });++ if (staledWorkflowRuns.length <= 0) {+ break;+ }++ await workflowRunRepository.update(+ staledWorkflowRuns.map((workflowRun) => workflowRun.id),+ {+ enqueuedAt: null,+ status: WorkflowRunStatus.NOT_STARTED,+ },+ );+ }await this.workflowThrottlingWorkspaceService.recomputeWorkflowRunNotStartedCount(workspaceId,