The open alternative to Salesforce, designed for AI.
TypeScript crmcrm-systemcustomergood-first-issuegraphqlhacktoberfestjavascriptmarketingmonoreponestjsopen-sourcepostgresqlreactreactjssalestypescriptweb
fix(server): finalize workflow runs stuck in STOPPING (#22900)
Thomas Trompettef4b2968a7495
diff --git a/packages/twenty-server/src/modules/workflow/workflow-runner/workflow-run-queue/command/workflow-handle-staled-runs.command.ts b/packages/twenty-server/src/modules/workflow/workflow-runner/workflow-run-queue/command/workflow-handle-staled-runs.command.tsindex f55b7215c70..36e1b0f2c15 100644--- a/packages/twenty-server/src/modules/workflow/workflow-runner/workflow-run-queue/command/workflow-handle-staled-runs.command.ts+++ b/packages/twenty-server/src/modules/workflow/workflow-runner/workflow-run-queue/command/workflow-handle-staled-runs.command.ts@@ -49,6 +49,10 @@ export class WorkflowHandleStaledRunsCommand extends CommandRunner {await this.workflowHandleStaledRunsWorkspaceService.handleStaledRunsForWorkspace(workspaceId,);++ await this.workflowHandleStaledRunsWorkspaceService.handleStuckStoppingRunsForWorkspace(+ workspaceId,+ );} catch (error) {this.logger.error(`Failed to handle staled runs for workspace ${workspaceId}`,diff --git a/packages/twenty-server/src/modules/workflow/workflow-runner/workflow-run-queue/constants/stuck-stopping-runs-threshold.ts b/packages/twenty-server/src/modules/workflow/workflow-runner/workflow-run-queue/constants/stuck-stopping-runs-threshold.tsnew file mode 100644index 00000000000..b26b193bfe1--- /dev/null+++ b/packages/twenty-server/src/modules/workflow/workflow-runner/workflow-run-queue/constants/stuck-stopping-runs-threshold.ts@@ -0,0 +1 @@+export const STUCK_STOPPING_RUNS_THRESHOLD_MS = 60 * 60 * 1000; // 1 hourdiff --git a/packages/twenty-server/src/modules/workflow/workflow-runner/workflow-run-queue/cron/jobs/workflow-handle-staled-runs.cron.job.ts b/packages/twenty-server/src/modules/workflow/workflow-runner/workflow-run-queue/cron/jobs/workflow-handle-staled-runs.cron.job.tsindex 923fd6d3ecb..366839d5d8a 100644--- a/packages/twenty-server/src/modules/workflow/workflow-runner/workflow-run-queue/cron/jobs/workflow-handle-staled-runs.cron.job.ts+++ b/packages/twenty-server/src/modules/workflow/workflow-runner/workflow-run-queue/cron/jobs/workflow-handle-staled-runs.cron.job.ts@@ -18,6 +18,7 @@ import { WorkspaceEntity } from 'src/engine/core-modules/workspace/workspace.entimport { getWorkspaceSchemaName } from 'src/engine/workspace-datasource/utils/get-workspace-schema-name.util';import { WorkflowRunStatus } from 'src/modules/workflow/common/standard-objects/workflow-run.workspace-entity';import { STALED_RUNS_THRESHOLD_MS } from 'src/modules/workflow/workflow-runner/workflow-run-queue/constants/staled-runs-threshold';+import { STUCK_STOPPING_RUNS_THRESHOLD_MS } from 'src/modules/workflow/workflow-runner/workflow-run-queue/constants/stuck-stopping-runs-threshold';import {WorkflowHandleStaledRunsJob,WorkflowHandleStaledRunsJobData,@@ -100,9 +101,12 @@ export class WorkflowHandleStaledRunsCronJob {}private async checkAndEnqueue(workspaceId: string): Promise<boolean> {- const hasStaledRuns = await this.hasStaledRuns(workspaceId);+ const [hasStaledRuns, hasStuckStoppingRuns] = await Promise.all([+ this.hasStaledRuns(workspaceId),+ this.hasStuckStoppingRuns(workspaceId),+ ]);- if (hasStaledRuns) {+ if (hasStaledRuns || hasStuckStoppingRuns) {await this.messageQueueService.add<WorkflowHandleStaledRunsJobData>(WorkflowHandleStaledRunsJob.name,{ workspaceId },@@ -140,4 +144,18 @@ export class WorkflowHandleStaledRunsCronJob {return result.length > 0;}++ private async hasStuckStoppingRuns(workspaceId: string): Promise<boolean> {+ const schemaName = getWorkspaceSchemaName(workspaceId);+ const thresholdDate = new Date(+ Date.now() - STUCK_STOPPING_RUNS_THRESHOLD_MS,+ );++ const result = await this.coreDataSource.query(+ `SELECT 1 FROM ${schemaName}."workflowRun" WHERE "status" = $1 AND "updatedAt" < $2 LIMIT 1`,+ [WorkflowRunStatus.STOPPING, thresholdDate],+ );++ return result.length > 0;+ }}diff --git a/packages/twenty-server/src/modules/workflow/workflow-runner/workflow-run-queue/jobs/workflow-handle-staled-runs.job.ts b/packages/twenty-server/src/modules/workflow/workflow-runner/workflow-run-queue/jobs/workflow-handle-staled-runs.job.tsindex ee8f7c3ab8f..02aea3dcc3d 100644--- a/packages/twenty-server/src/modules/workflow/workflow-runner/workflow-run-queue/jobs/workflow-handle-staled-runs.job.ts+++ b/packages/twenty-server/src/modules/workflow/workflow-runner/workflow-run-queue/jobs/workflow-handle-staled-runs.job.ts@@ -19,8 +19,13 @@ export class WorkflowHandleStaledRunsJob {async handle({workspaceId,}: WorkflowHandleStaledRunsJobData): Promise<void> {- await this.workflowHandleStaledRunsWorkspaceService.handleStaledRunsForWorkspace(- workspaceId,- );+ await Promise.all([+ this.workflowHandleStaledRunsWorkspaceService.handleStaledRunsForWorkspace(+ workspaceId,+ ),+ this.workflowHandleStaledRunsWorkspaceService.handleStuckStoppingRunsForWorkspace(+ workspaceId,+ ),+ ]);}}diff --git a/packages/twenty-server/src/modules/workflow/workflow-runner/workflow-run-queue/utils/__tests__/get-stuck-stopping-runs-find-options.util.spec.ts b/packages/twenty-server/src/modules/workflow/workflow-runner/workflow-run-queue/utils/__tests__/get-stuck-stopping-runs-find-options.util.spec.tsnew file mode 100644index 00000000000..3e5315d1a06--- /dev/null+++ b/packages/twenty-server/src/modules/workflow/workflow-runner/workflow-run-queue/utils/__tests__/get-stuck-stopping-runs-find-options.util.spec.ts@@ -0,0 +1,13 @@+import { FindOperator } from 'typeorm';++import { WorkflowRunStatus } from 'src/modules/workflow/common/standard-objects/workflow-run.workspace-entity';+import { getStuckStoppingRunsFindOptions } from 'src/modules/workflow/workflow-runner/workflow-run-queue/utils/get-stuck-stopping-runs-find-options.util';++describe('getStuckStoppingRunsFindOptions', () => {+ it('should match STOPPING runs older than the threshold', () => {+ const where = getStuckStoppingRunsFindOptions();++ expect(where.status).toBe(WorkflowRunStatus.STOPPING);+ expect(where.updatedAt).toBeInstanceOf(FindOperator);+ });+});diff --git a/packages/twenty-server/src/modules/workflow/workflow-runner/workflow-run-queue/utils/get-stuck-stopping-runs-find-options.util.ts b/packages/twenty-server/src/modules/workflow/workflow-runner/workflow-run-queue/utils/get-stuck-stopping-runs-find-options.util.tsnew file mode 100644index 00000000000..9fbd8fce8fd--- /dev/null+++ b/packages/twenty-server/src/modules/workflow/workflow-runner/workflow-run-queue/utils/get-stuck-stopping-runs-find-options.util.ts@@ -0,0 +1,19 @@+import { type FindOptionsWhere, LessThan } from 'typeorm';++import {+ WorkflowRunStatus,+ type WorkflowRunWorkspaceEntity,+} from 'src/modules/workflow/common/standard-objects/workflow-run.workspace-entity';+import { STUCK_STOPPING_RUNS_THRESHOLD_MS } from 'src/modules/workflow/workflow-runner/workflow-run-queue/constants/stuck-stopping-runs-threshold';++export const getStuckStoppingRunsFindOptions =+ (): FindOptionsWhere<WorkflowRunWorkspaceEntity> => {+ const thresholdDate = new Date(+ Date.now() - STUCK_STOPPING_RUNS_THRESHOLD_MS,+ );++ return {+ status: WorkflowRunStatus.STOPPING,+ updatedAt: LessThan(thresholdDate.toISOString()),+ };+ };diff --git a/packages/twenty-server/src/modules/workflow/workflow-runner/workflow-run-queue/workflow-run-queue.module.ts b/packages/twenty-server/src/modules/workflow/workflow-runner/workflow-run-queue/workflow-run-queue.module.tsindex ce214144ce8..d666b1773f6 100644--- a/packages/twenty-server/src/modules/workflow/workflow-runner/workflow-run-queue/workflow-run-queue.module.ts+++ b/packages/twenty-server/src/modules/workflow/workflow-runner/workflow-run-queue/workflow-run-queue.module.ts@@ -7,6 +7,7 @@ import { MetricsModule } from 'src/engine/core-modules/metrics/metrics.module';import { ThrottlerModule } from 'src/engine/core-modules/throttler/throttler.module';import { WorkspaceEntity } from 'src/engine/core-modules/workspace/workspace.entity';import { WorkspaceDataSourceModule } from 'src/engine/workspace-datasource/workspace-datasource.module';+import { WorkflowRunModule } from 'src/modules/workflow/workflow-runner/workflow-run/workflow-run.module';import { WorkflowHandleStaledRunsCommand } from 'src/modules/workflow/workflow-runner/workflow-run-queue/command/workflow-handle-staled-runs.command';import { WorkflowCleanWorkflowRunsCronCommand } from 'src/modules/workflow/workflow-runner/workflow-run-queue/cron/command/workflow-clean-workflow-runs.cron.command';import { WorkflowHandleStaledRunsCronCommand } from 'src/modules/workflow/workflow-runner/workflow-run-queue/cron/command/workflow-handle-staled-runs.cron.command';@@ -29,6 +30,7 @@ import { WorkflowThrottlingWorkspaceService } from 'src/modules/workflow/workfloWorkspaceDataSourceModule,MetricsModule,ThrottlerModule,+ WorkflowRunModule,],providers: [WorkflowThrottlingWorkspaceService,diff --git a/packages/twenty-server/src/modules/workflow/workflow-runner/workflow-run-queue/workspace-services/__tests__/workflow-handle-staled-runs.workspace-service.spec.ts b/packages/twenty-server/src/modules/workflow/workflow-runner/workflow-run-queue/workspace-services/__tests__/workflow-handle-staled-runs.workspace-service.spec.tsindex 17cb8789e08..29492e1dd92 100644--- a/packages/twenty-server/src/modules/workflow/workflow-runner/workflow-run-queue/workspace-services/__tests__/workflow-handle-staled-runs.workspace-service.spec.ts+++ b/packages/twenty-server/src/modules/workflow/workflow-runner/workflow-run-queue/workspace-services/__tests__/workflow-handle-staled-runs.workspace-service.spec.ts@@ -4,6 +4,7 @@ import { GlobalWorkspaceOrmManager } from 'src/engine/twenty-orm/global-workspacimport { WorkflowRunStatus } from 'src/modules/workflow/common/standard-objects/workflow-run.workspace-entity';import { WorkflowHandleStaledRunsWorkspaceService } from 'src/modules/workflow/workflow-runner/workflow-run-queue/workspace-services/workflow-handle-staled-runs.workspace-service';import { WorkflowThrottlingWorkspaceService } from 'src/modules/workflow/workflow-runner/workflow-run-queue/workspace-services/workflow-throttling.workspace-service';+import { WorkflowRunWorkspaceService } from 'src/modules/workflow/workflow-runner/workflow-run/workflow-run.workspace-service';const mockRepository = {count: jest.fn(),@@ -23,13 +24,19 @@ const mockWorkflowThrottlingWorkspaceService = {recomputeWorkflowRunNotStartedCount: jest.fn().mockResolvedValue(undefined),};+const mockWorkflowRunWorkspaceService = {+ endWorkflowRun: jest.fn().mockResolvedValue(undefined),+};+// Mirrors QUERY_MAX_RECORDS, the per-batch cap the service applies. Kept as a// local literal rather than imported from twenty-shared/constants to avoid a// jest circular-init issue with config-variables loading the same barrel.const QUERY_MAX_RECORDS = 200;-const buildStaledRuns = (count: number) =>- Array.from({ length: count }, (_, index) => ({ id: `run-${index}` }));+const buildStaledRuns = (count: number, startIndex = 0) =>+ Array.from({ length: count }, (_, index) => ({+ id: `run-${startIndex + index}`,+ }));describe('WorkflowHandleStaledRunsWorkspaceService', () => {let service: WorkflowHandleStaledRunsWorkspaceService;@@ -50,6 +57,10 @@ describe('WorkflowHandleStaledRunsWorkspaceService', () => {provide: WorkflowThrottlingWorkspaceService,useValue: mockWorkflowThrottlingWorkspaceService,},+ {+ provide: WorkflowRunWorkspaceService,+ useValue: mockWorkflowRunWorkspaceService,+ },],}).compile();@@ -135,4 +146,95 @@ describe('WorkflowHandleStaledRunsWorkspaceService', () => {mockWorkflowThrottlingWorkspaceService.recomputeWorkflowRunNotStartedCount,).toHaveBeenCalledTimes(1);});++ describe('handleStuckStoppingRunsForWorkspace', () => {+ it('should do nothing when there are no stuck stopping runs', async () => {+ mockRepository.find.mockResolvedValueOnce([]);++ await service.handleStuckStoppingRunsForWorkspace(workspaceId);++ expect(+ mockWorkflowRunWorkspaceService.endWorkflowRun,+ ).not.toHaveBeenCalled();+ });++ it('should finalize each stuck stopping run to STOPPED', async () => {+ mockRepository.find.mockResolvedValueOnce(buildStaledRuns(3));++ await service.handleStuckStoppingRunsForWorkspace(workspaceId);++ expect(+ mockWorkflowRunWorkspaceService.endWorkflowRun,+ ).toHaveBeenCalledTimes(3);+ expect(+ mockWorkflowRunWorkspaceService.endWorkflowRun,+ ).toHaveBeenCalledWith({+ workflowRunId: 'run-0',+ workspaceId,+ status: WorkflowRunStatus.STOPPED,+ });+ });++ it('should page through a multi-page backlog until a short page', async () => {+ mockRepository.find+ .mockResolvedValueOnce(buildStaledRuns(QUERY_MAX_RECORDS))+ .mockResolvedValueOnce(+ buildStaledRuns(QUERY_MAX_RECORDS, QUERY_MAX_RECORDS),+ )+ .mockResolvedValueOnce(buildStaledRuns(50, 2 * QUERY_MAX_RECORDS));++ await service.handleStuckStoppingRunsForWorkspace(workspaceId);++ expect(mockRepository.find).toHaveBeenCalledTimes(3);+ expect(mockRepository.find).toHaveBeenCalledWith(+ expect.objectContaining({ take: QUERY_MAX_RECORDS }),+ );+ expect(+ mockWorkflowRunWorkspaceService.endWorkflowRun,+ ).toHaveBeenCalledTimes(2 * QUERY_MAX_RECORDS + 50);+ });++ it('should keep finalizing remaining runs when one fails', async () => {+ mockRepository.find.mockResolvedValueOnce(buildStaledRuns(3));+ mockWorkflowRunWorkspaceService.endWorkflowRun+ .mockRejectedValueOnce(new Error('boom'))+ .mockResolvedValue(undefined);++ await service.handleStuckStoppingRunsForWorkspace(workspaceId);++ expect(+ mockWorkflowRunWorkspaceService.endWorkflowRun,+ ).toHaveBeenCalledTimes(3);+ });++ it('should advance past a fully failed page instead of starving later runs', async () => {+ mockRepository.find+ .mockResolvedValueOnce(buildStaledRuns(QUERY_MAX_RECORDS))+ .mockResolvedValueOnce(buildStaledRuns(50, QUERY_MAX_RECORDS));+ // Whole first page fails, later runs still get finalized+ mockWorkflowRunWorkspaceService.endWorkflowRun.mockImplementation(+ ({ workflowRunId }: { workflowRunId: string }) => {+ const index = Number(workflowRunId.replace('run-', ''));++ return index < QUERY_MAX_RECORDS+ ? Promise.reject(new Error('boom'))+ : Promise.resolve(undefined);+ },+ );++ await service.handleStuckStoppingRunsForWorkspace(workspaceId);++ expect(mockRepository.find).toHaveBeenCalledTimes(2);+ expect(+ mockWorkflowRunWorkspaceService.endWorkflowRun,+ ).toHaveBeenCalledTimes(QUERY_MAX_RECORDS + 50);+ expect(+ mockWorkflowRunWorkspaceService.endWorkflowRun,+ ).toHaveBeenCalledWith({+ workflowRunId: `run-${QUERY_MAX_RECORDS}`,+ workspaceId,+ status: WorkflowRunStatus.STOPPED,+ });+ });+ });});diff --git a/packages/twenty-server/src/modules/workflow/workflow-runner/workflow-run-queue/workspace-services/workflow-handle-staled-runs.workspace-service.ts b/packages/twenty-server/src/modules/workflow/workflow-runner/workflow-run-queue/workspace-services/workflow-handle-staled-runs.workspace-service.tsindex b4424ad3155..d43c3be2562 100644--- a/packages/twenty-server/src/modules/workflow/workflow-runner/workflow-run-queue/workspace-services/workflow-handle-staled-runs.workspace-service.ts+++ b/packages/twenty-server/src/modules/workflow/workflow-runner/workflow-run-queue/workspace-services/workflow-handle-staled-runs.workspace-service.ts@@ -9,7 +9,9 @@ import {WorkflowRunWorkspaceEntity,} from 'src/modules/workflow/common/standard-objects/workflow-run.workspace-entity';import { getStaledRunsFindOptions } from 'src/modules/workflow/workflow-runner/workflow-run-queue/utils/get-staled-runs-find-options.util';+import { getStuckStoppingRunsFindOptions } from 'src/modules/workflow/workflow-runner/workflow-run-queue/utils/get-stuck-stopping-runs-find-options.util';import { WorkflowThrottlingWorkspaceService } from 'src/modules/workflow/workflow-runner/workflow-run-queue/workspace-services/workflow-throttling.workspace-service';+import { WorkflowRunWorkspaceService } from 'src/modules/workflow/workflow-runner/workflow-run/workflow-run.workspace-service';@Injectable()export class WorkflowHandleStaledRunsWorkspaceService {@@ -19,6 +21,7 @@ export class WorkflowHandleStaledRunsWorkspaceService {constructor(private readonly globalWorkspaceOrmManager: GlobalWorkspaceOrmManager,private readonly workflowThrottlingWorkspaceService: WorkflowThrottlingWorkspaceService,+ private readonly workflowRunWorkspaceService: WorkflowRunWorkspaceService,) {}async handleStaledRunsForWorkspace(workspaceId: string) {@@ -72,4 +75,55 @@ export class WorkflowHandleStaledRunsWorkspaceService {);}, authContext);}++ async handleStuckStoppingRunsForWorkspace(workspaceId: string) {+ const authContext = buildSystemAuthContext(workspaceId);++ const stuckStoppingRunIds =+ await this.globalWorkspaceOrmManager.executeInWorkspaceContext(+ async () => {+ const workflowRunRepository =+ await this.globalWorkspaceOrmManager.getRepository(+ workspaceId,+ WorkflowRunWorkspaceEntity,+ { shouldBypassPermissionChecks: true },+ );++ const findOptions = getStuckStoppingRunsFindOptions();++ const runIds: string[] = [];+ let page: WorkflowRunWorkspaceEntity[];++ do {+ page = await workflowRunRepository.find({+ where: findOptions,+ select: { id: true },+ order: { createdAt: 'ASC', id: 'ASC' },+ take: QUERY_MAX_RECORDS,+ skip: runIds.length,+ });++ runIds.push(...page.map((workflowRun) => workflowRun.id));+ } while (page.length === QUERY_MAX_RECORDS);++ return runIds;+ },+ authContext,+ );++ for (const workflowRunId of stuckStoppingRunIds) {+ try {+ await this.workflowRunWorkspaceService.endWorkflowRun({+ workflowRunId,+ workspaceId,+ status: WorkflowRunStatus.STOPPED,+ });+ } catch (error) {+ this.logger.error(+ `Failed to finalize stuck stopping workflow run ${workflowRunId} for workspace ${workspaceId}`,+ error,+ );+ }+ }+ }}