Skip to content

The open alternative to Salesforce, designed for AI.

fix(server): recover workflow runs whose queue job was lost - monitoring only (#22995)

Thomas Trompette98c71d7b3df3
diff --git a/packages/twenty-server/src/engine/core-modules/message-queue/drivers/bullmq.driver.ts b/packages/twenty-server/src/engine/core-modules/message-queue/drivers/bullmq.driver.ts
index dad792fc2e2..177e33e042c 100644
--- a/packages/twenty-server/src/engine/core-modules/message-queue/drivers/bullmq.driver.ts
+++ b/packages/twenty-server/src/engine/core-modules/message-queue/drivers/bullmq.driver.ts
@@ -19,8 +19,14 @@ import {
type QueueCronJobOptions,
type QueueJobOptions,
} from 'src/engine/core-modules/message-queue/drivers/interfaces/job-options.interface';
-import { type MessageQueueDriver } from 'src/engine/core-modules/message-queue/drivers/interfaces/message-queue-driver.interface';
-import { type MessageQueueJob } from 'src/engine/core-modules/message-queue/interfaces/message-queue-job.interface';
+import {
+ type InFlightQueueJob,
+ type MessageQueueDriver,
+} from 'src/engine/core-modules/message-queue/drivers/interfaces/message-queue-driver.interface';
+import {
+ type MessageQueueJob,
+ type MessageQueueJobData,
+} from 'src/engine/core-modules/message-queue/interfaces/message-queue-job.interface';
import { type MessageQueueWorkerOptions } from 'src/engine/core-modules/message-queue/interfaces/message-queue-worker-options.interface';
import { QUEUE_RETENTION } from 'src/engine/core-modules/message-queue/constants/queue-retention.constants';
@@ -250,6 +256,18 @@ export class BullMQDriver
shouldStoreInCache: false,
});
});
+
+ this.workerMap[queueName].on('stalled', (jobId) => {
+ this.logger.warn(
+ `Job ${jobId} stalled on queue ${queueName}: its worker stopped processing it without completing or failing it`,
+ );
+
+ void this.metricsService.incrementCounterForEvent({
+ key: MetricsKeys.JobStalled,
+ attributes: { queue: queueName },
+ shouldStoreInCache: false,
+ });
+ });
}
async addCron<T>({
@@ -322,7 +340,7 @@ export class BullMQDriver
}
// This ensures only one waiting job can be queued for a specific option.id
- if (options?.id) {
+ if (options?.id && !options?.allowDuplicatedPrefixes) {
const waitingJobs = await this.queueMap[queueName].getJobs(['waiting']);
const isJobAlreadyWaiting = waitingJobs.some(
@@ -351,4 +369,27 @@ export class BullMQDriver
await this.queueMap[queueName].add(jobName, data, queueOptions);
}
+
+ async getInFlightJobs<T extends MessageQueueJobData>(
+ queueName: MessageQueue,
+ ): Promise<InFlightQueueJob<T>[]> {
+ if (!this.queueMap[queueName]) {
+ throw new Error(
+ `Queue ${queueName} is not registered, make sure you have added it as a queue provider`,
+ );
+ }
+
+ const jobs = await this.queueMap[queueName].getJobs([
+ 'active',
+ 'waiting',
+ 'waiting-children',
+ 'paused',
+ 'prioritized',
+ 'delayed',
+ ]);
+
+ return jobs
+ .filter(isDefined)
+ .map((job) => ({ id: job.id, data: job.data }));
+ }
}
diff --git a/packages/twenty-server/src/engine/core-modules/message-queue/drivers/interfaces/job-options.interface.ts b/packages/twenty-server/src/engine/core-modules/message-queue/drivers/interfaces/job-options.interface.ts
index 9df438081a1..ee2f74919f8 100644
--- a/packages/twenty-server/src/engine/core-modules/message-queue/drivers/interfaces/job-options.interface.ts
+++ b/packages/twenty-server/src/engine/core-modules/message-queue/drivers/interfaces/job-options.interface.ts
@@ -1,5 +1,6 @@
export interface QueueJobOptions {
id?: string;
+ allowDuplicatedPrefixes?: boolean;
priority?: number;
retryLimit?: number;
delay?: number;
diff --git a/packages/twenty-server/src/engine/core-modules/message-queue/drivers/interfaces/message-queue-driver.interface.ts b/packages/twenty-server/src/engine/core-modules/message-queue/drivers/interfaces/message-queue-driver.interface.ts
index 117c4d7adec..8107f31c68e 100644
--- a/packages/twenty-server/src/engine/core-modules/message-queue/drivers/interfaces/message-queue-driver.interface.ts
+++ b/packages/twenty-server/src/engine/core-modules/message-queue/drivers/interfaces/message-queue-driver.interface.ts
@@ -42,4 +42,12 @@ export interface MessageQueueDriver {
jobId?: string;
}): Promise<void>;
register?(queueName: MessageQueue): void;
+ getInFlightJobs?<T extends MessageQueueJobData>(
+ queueName: MessageQueue,
+ ): Promise<InFlightQueueJob<T>[]>;
+}
+
+export interface InFlightQueueJob<T extends MessageQueueJobData> {
+ id?: string;
+ data: T;
}
diff --git a/packages/twenty-server/src/engine/core-modules/message-queue/services/message-queue.service.ts b/packages/twenty-server/src/engine/core-modules/message-queue/services/message-queue.service.ts
index 9f13086e337..71bc29801d6 100644
--- a/packages/twenty-server/src/engine/core-modules/message-queue/services/message-queue.service.ts
+++ b/packages/twenty-server/src/engine/core-modules/message-queue/services/message-queue.service.ts
@@ -4,7 +4,10 @@ import {
type QueueCronJobOptions,
type QueueJobOptions,
} from 'src/engine/core-modules/message-queue/drivers/interfaces/job-options.interface';
-import { MessageQueueDriver } from 'src/engine/core-modules/message-queue/drivers/interfaces/message-queue-driver.interface';
+import {
+ type InFlightQueueJob,
+ MessageQueueDriver,
+} from 'src/engine/core-modules/message-queue/drivers/interfaces/message-queue-driver.interface';
import {
type MessageQueueJobData,
type MessageQueueJob,
@@ -35,6 +38,16 @@ export class MessageQueueService {
return this.driver.add(this.queueName, jobName, data, options);
}
+ getInFlightJobs<T extends MessageQueueJobData>(): Promise<
+ InFlightQueueJob<T>[]
+ > {
+ if (typeof this.driver.getInFlightJobs !== 'function') {
+ return Promise.resolve([]);
+ }
+
+ return this.driver.getInFlightJobs(this.queueName);
+ }
+
addCron<T extends MessageQueueJobData | undefined>({
jobName,
data,
diff --git a/packages/twenty-server/src/engine/core-modules/metrics/types/metrics-keys.type.ts b/packages/twenty-server/src/engine/core-modules/metrics/types/metrics-keys.type.ts
index d1a12227dc6..8d189bc227f 100644
--- a/packages/twenty-server/src/engine/core-modules/metrics/types/metrics-keys.type.ts
+++ b/packages/twenty-server/src/engine/core-modules/metrics/types/metrics-keys.type.ts
@@ -23,6 +23,8 @@ export enum MetricsKeys {
WorkflowRunThrottled = 'workflow-run/throttled',
WorkflowRunFailedToEnqueue = 'workflow-run/failed/to-enqueue',
WorkflowRunSystemError = 'workflow-run/system-error',
+ WorkflowRunStuckRunningDetected = 'workflow-run/stuck-running/detected',
+ WorkflowRunStuckRunningFalsePositive = 'workflow-run/stuck-running/false-positive',
AiChatToolExecutionSucceeded = 'ai-chat/tool-execution-succeeded',
AiChatToolExecutionFailed = 'ai-chat/tool-execution-failed',
WorkflowAgentToolExecutionSucceeded = 'workflow-agent/tool-execution-succeeded',
@@ -52,6 +54,7 @@ export enum MetricsKeys {
CommonApiQueryRateLimited = 'common-api-query/rate-limited',
JobCompleted = 'job/completed',
JobFailed = 'job/failed',
+ JobStalled = 'job/stalled',
JobWaiting = 'job/waiting',
JobLatencyMs = 'job/latency-ms',
AiChatTurnLatencyMs = 'ai-chat/turn-latency-ms',
diff --git a/packages/twenty-server/src/modules/workflow/workflow-executor/workflow-actions/delay/delay.workflow-action.ts b/packages/twenty-server/src/modules/workflow/workflow-executor/workflow-actions/delay/delay.workflow-action.ts
index df95a027694..85cb7de3ba9 100644
--- a/packages/twenty-server/src/modules/workflow/workflow-executor/workflow-actions/delay/delay.workflow-action.ts
+++ b/packages/twenty-server/src/modules/workflow/workflow-executor/workflow-actions/delay/delay.workflow-action.ts
@@ -18,6 +18,7 @@ import { RESUME_DELAYED_WORKFLOW_JOB_NAME } from 'src/modules/workflow/workflow-
import { isWorkflowDelayAction } from 'src/modules/workflow/workflow-executor/workflow-actions/delay/guards/is-workflow-delay-action.guard';
import { ResumeDelayedWorkflowJobData } from 'src/modules/workflow/workflow-executor/workflow-actions/delay/types/resume-delayed-workflow-job-data.type';
import { WorkflowDelayActionInput } from 'src/modules/workflow/workflow-executor/workflow-actions/delay/types/workflow-delay-action-input.type';
+import { buildRunWorkflowJobOptions } from 'src/modules/workflow/workflow-runner/utils/build-run-workflow-job-options.util';
@Injectable()
export class DelayWorkflowAction implements WorkflowAction {
@@ -105,6 +106,7 @@ export class DelayWorkflowAction implements WorkflowAction {
stepId: currentStepId,
},
{
+ ...buildRunWorkflowJobOptions(runInfo.workflowRunId),
delay: delayInMs,
},
);
diff --git a/packages/twenty-server/src/modules/workflow/workflow-executor/workflow-actions/delay/jobs/resume-delayed-workflow.job.ts b/packages/twenty-server/src/modules/workflow/workflow-executor/workflow-actions/delay/jobs/resume-delayed-workflow.job.ts
index 0a01db6e43d..79e3d08bd72 100644
--- a/packages/twenty-server/src/modules/workflow/workflow-executor/workflow-actions/delay/jobs/resume-delayed-workflow.job.ts
+++ b/packages/twenty-server/src/modules/workflow/workflow-executor/workflow-actions/delay/jobs/resume-delayed-workflow.job.ts
@@ -19,6 +19,7 @@ import {
} from 'src/modules/workflow/workflow-runner/exceptions/workflow-run.exception';
import { RunWorkflowJob } from 'src/modules/workflow/workflow-runner/jobs/run-workflow.job';
import { type RunWorkflowJobData } from 'src/modules/workflow/workflow-runner/types/run-workflow-job-data.type';
+import { buildRunWorkflowJobOptions } from 'src/modules/workflow/workflow-runner/utils/build-run-workflow-job-options.util';
import { WorkflowRunWorkspaceService } from 'src/modules/workflow/workflow-runner/workflow-run/workflow-run.workspace-service';
@Processor({
@@ -92,6 +93,7 @@ export class ResumeDelayedWorkflowJob {
workflowRunId,
lastExecutedStepId: stepId,
},
+ buildRunWorkflowJobOptions(workflowRunId),
);
} catch (error) {
await this.workflowRunWorkspaceService.endWorkflowRun({
diff --git a/packages/twenty-server/src/modules/workflow/workflow-executor/workspace-services/__tests__/workflow-executor.workspace-service.spec.ts b/packages/twenty-server/src/modules/workflow/workflow-executor/workspace-services/__tests__/workflow-executor.workspace-service.spec.ts
index 2413e0e00b1..628097aa04e 100644
--- a/packages/twenty-server/src/modules/workflow/workflow-executor/workspace-services/__tests__/workflow-executor.workspace-service.spec.ts
+++ b/packages/twenty-server/src/modules/workflow/workflow-executor/workspace-services/__tests__/workflow-executor.workspace-service.spec.ts
@@ -429,6 +429,7 @@ describe('WorkflowExecutorWorkspaceService', () => {
workflowRunId: mockWorkflowRunId,
lastExecutedStepId: 'step-1',
},
+ { id: mockWorkflowRunId, allowDuplicatedPrefixes: true },
);
// Should not execute the next step (step-2) in the same job
diff --git a/packages/twenty-server/src/modules/workflow/workflow-executor/workspace-services/workflow-executor.workspace-service.ts b/packages/twenty-server/src/modules/workflow/workflow-executor/workspace-services/workflow-executor.workspace-service.ts
index bf09898df91..1bce3310f1a 100644
--- a/packages/twenty-server/src/modules/workflow/workflow-executor/workspace-services/workflow-executor.workspace-service.ts
+++ b/packages/twenty-server/src/modules/workflow/workflow-executor/workspace-services/workflow-executor.workspace-service.ts
@@ -49,6 +49,7 @@ import { getNextStepIdsForIterator } from 'src/modules/workflow/workflow-executo
import { WorkflowAction } from 'src/modules/workflow/workflow-executor/workflow-actions/types/workflow-action.type';
import { RUN_WORKFLOW_JOB_NAME } from 'src/modules/workflow/workflow-runner/constants/run-workflow-job-name';
import { type RunWorkflowJobData } from 'src/modules/workflow/workflow-runner/types/run-workflow-job-data.type';
+import { buildRunWorkflowJobOptions } from 'src/modules/workflow/workflow-runner/utils/build-run-workflow-job-options.util';
import { WorkflowRunWorkspaceService } from 'src/modules/workflow/workflow-runner/workflow-run/workflow-run.workspace-service';
const MAX_EXECUTED_STEPS_COUNT = 20;
@@ -612,6 +613,7 @@ export class WorkflowExecutorWorkspaceService {
workflowRunId,
lastExecutedStepId,
},
+ buildRunWorkflowJobOptions(workflowRunId),
);
}
}
diff --git a/packages/twenty-server/src/modules/workflow/workflow-runner/utils/build-run-workflow-job-options.util.ts b/packages/twenty-server/src/modules/workflow/workflow-runner/utils/build-run-workflow-job-options.util.ts
new file mode 100644
index 00000000000..52d20c0a12f
--- /dev/null
+++ b/packages/twenty-server/src/modules/workflow/workflow-runner/utils/build-run-workflow-job-options.util.ts
@@ -0,0 +1,8 @@
+import { type QueueJobOptions } from 'src/engine/core-modules/message-queue/drivers/interfaces/job-options.interface';
+
+export const buildRunWorkflowJobOptions = (
+ workflowRunId: string,
+): QueueJobOptions => ({
+ id: workflowRunId,
+ allowDuplicatedPrefixes: true,
+});
diff --git a/packages/twenty-server/src/modules/workflow/workflow-runner/workflow-run-queue/command/workflow-handle-staled-runs.command.ts b/packages/twenty-server/src/modules/workflow/workflow-runner/workflow-run-queue/command/workflow-handle-staled-runs.command.ts
index 36e1b0f2c15..5bfff6560cb 100644
--- a/packages/twenty-server/src/modules/workflow/workflow-runner/workflow-run-queue/command/workflow-handle-staled-runs.command.ts
+++ b/packages/twenty-server/src/modules/workflow/workflow-runner/workflow-run-queue/command/workflow-handle-staled-runs.command.ts
@@ -53,6 +53,10 @@ export class WorkflowHandleStaledRunsCommand extends CommandRunner {
await this.workflowHandleStaledRunsWorkspaceService.handleStuckStoppingRunsForWorkspace(
workspaceId,
);
+
+ await this.workflowHandleStaledRunsWorkspaceService.handleStuckRunningRunsForWorkspace(
+ workspaceId,
+ );
} catch (error) {
this.logger.error(
`Failed to handle staled runs for workspace ${workspaceId}`,
diff --git a/packages/twenty-server/src/modules/workflow/workflow-runner/workflow-run-queue/constants/stuck-running-runs-threshold.ts b/packages/twenty-server/src/modules/workflow/workflow-runner/workflow-run-queue/constants/stuck-running-runs-threshold.ts
new file mode 100644
index 00000000000..4db27d06c3e
--- /dev/null
+++ b/packages/twenty-server/src/modules/workflow/workflow-runner/workflow-run-queue/constants/stuck-running-runs-threshold.ts
@@ -0,0 +1 @@
+export const STUCK_RUNNING_RUNS_THRESHOLD_MS = 60 * 60 * 1000; // 1 hour
diff --git a/packages/twenty-server/src/modules/workflow/workflow-runner/workflow-run-queue/cron/jobs/workflow-handle-staled-runs.cron.job.ts b/packages/twenty-server/src/modules/workflow/workflow-runner/workflow-run-queue/cron/jobs/workflow-handle-staled-runs.cron.job.ts
index 366839d5d8a..e8aa33329a6 100644
--- a/packages/twenty-server/src/modules/workflow/workflow-runner/workflow-run-queue/cron/jobs/workflow-handle-staled-runs.cron.job.ts
+++ b/packages/twenty-server/src/modules/workflow/workflow-runner/workflow-run-queue/cron/jobs/workflow-handle-staled-runs.cron.job.ts
@@ -1,6 +1,7 @@
import { Logger } from '@nestjs/common';
import { InjectDataSource, InjectRepository } from '@nestjs/typeorm';
+import { isDefined } from 'twenty-shared/utils';
import { WorkspaceActivationStatus } from 'twenty-shared/workspace';
import { DataSource, Repository } from 'typeorm';
@@ -18,7 +19,9 @@ import { WorkspaceEntity } from 'src/engine/core-modules/workspace/workspace.ent
import { getWorkspaceSchemaName } from 'src/engine/workspace-datasource/utils/get-workspace-schema-name.util';
import { WorkflowRunStatus } from 'src/modules/workflow/common/standard-objects/workflow-run.workspace-entity';
import { STALED_RUNS_THRESHOLD_MS } from 'src/modules/workflow/workflow-runner/workflow-run-queue/constants/staled-runs-threshold';
+import { STUCK_RUNNING_RUNS_THRESHOLD_MS } from 'src/modules/workflow/workflow-runner/workflow-run-queue/constants/stuck-running-runs-threshold';
import { STUCK_STOPPING_RUNS_THRESHOLD_MS } from 'src/modules/workflow/workflow-runner/workflow-run-queue/constants/stuck-stopping-runs-threshold';
+import { getStuckRunningRunsMonitorCacheKey } from 'src/modules/workflow/workflow-runner/workflow-run-queue/utils/get-stuck-running-runs-monitor-cache-key.util';
import {
WorkflowHandleStaledRunsJob,
WorkflowHandleStaledRunsJobData,
@@ -101,12 +104,24 @@ export class WorkflowHandleStaledRunsCronJob {
}
private async checkAndEnqueue(workspaceId: string): Promise<boolean> {
- const [hasStaledRuns, hasStuckStoppingRuns] = await Promise.all([
+ const [
+ hasStaledRuns,
+ hasStuckStoppingRuns,
+ hasStuckRunningRuns,
+ hasFlaggedStuckRunningRuns,
+ ] = await Promise.all([
this.hasStaledRuns(workspaceId),
this.hasStuckStoppingRuns(workspaceId),
+ this.hasStuckRunningRuns(workspaceId),
+ this.hasFlaggedStuckRunningRuns(workspaceId),
]);
- if (hasStaledRuns || hasStuckStoppingRuns) {
+ if (
+ hasStaledRuns ||
+ hasStuckStoppingRuns ||
+ hasStuckRunningRuns ||
+ hasFlaggedStuckRunningRuns
+ ) {
await this.messageQueueService.add<WorkflowHandleStaledRunsJobData>(
WorkflowHandleStaledRunsJob.name,
{ workspaceId },
@@ -158,4 +173,28 @@ export class WorkflowHandleStaledRunsCronJob {
return result.length > 0;
}
+
+ private async hasStuckRunningRuns(workspaceId: string): Promise<boolean> {
+ const schemaName = getWorkspaceSchemaName(workspaceId);
+ const thresholdDate = new Date(
+ Date.now() - STUCK_RUNNING_RUNS_THRESHOLD_MS,
+ );
+
+ const result = await this.coreDataSource.query(
+ `SELECT 1 FROM ${schemaName}."workflowRun" WHERE "status" = $1 AND "updatedAt" < $2 LIMIT 1`,
+ [WorkflowRunStatus.RUNNING, thresholdDate],
+ );
+
+ return result.length > 0;
+ }
+
+ private async hasFlaggedStuckRunningRuns(
+ workspaceId: string,
+ ): Promise<boolean> {
+ const flaggedRuns = await this.cacheStorageService.get<
+ Record<string, string>
+ >(getStuckRunningRunsMonitorCacheKey(workspaceId));
+
+ return isDefined(flaggedRuns) && Object.keys(flaggedRuns).length > 0;
+ }
}
diff --git a/packages/twenty-server/src/modules/workflow/workflow-runner/workflow-run-queue/jobs/workflow-handle-staled-runs.job.ts b/packages/twenty-server/src/modules/workflow/workflow-runner/workflow-run-queue/jobs/workflow-handle-staled-runs.job.ts
index 02aea3dcc3d..1ca26fc5c05 100644
--- a/packages/twenty-server/src/modules/workflow/workflow-runner/workflow-run-queue/jobs/workflow-handle-staled-runs.job.ts
+++ b/packages/twenty-server/src/modules/workflow/workflow-runner/workflow-run-queue/jobs/workflow-handle-staled-runs.job.ts
@@ -26,6 +26,9 @@ export class WorkflowHandleStaledRunsJob {
this.workflowHandleStaledRunsWorkspaceService.handleStuckStoppingRunsForWorkspace(
workspaceId,
),
+ this.workflowHandleStaledRunsWorkspaceService.handleStuckRunningRunsForWorkspace(
+ workspaceId,
+ ),
]);
}
}
diff --git a/packages/twenty-server/src/modules/workflow/workflow-runner/workflow-run-queue/utils/__tests__/get-stuck-running-runs-find-options.util.spec.ts b/packages/twenty-server/src/modules/workflow/workflow-runner/workflow-run-queue/utils/__tests__/get-stuck-running-runs-find-options.util.spec.ts
new file mode 100644
index 00000000000..bb41bee7a6c
--- /dev/null
+++ b/packages/twenty-server/src/modules/workflow/workflow-runner/workflow-run-queue/utils/__tests__/get-stuck-running-runs-find-options.util.spec.ts
@@ -0,0 +1,14 @@
+import { FindOperator } from 'typeorm';
+
+import { WorkflowRunStatus } from 'src/modules/workflow/common/standard-objects/workflow-run.workspace-entity';
+import { getStuckRunningRunsFindOptions } from 'src/modules/workflow/workflow-runner/workflow-run-queue/utils/get-stuck-running-runs-find-options.util';
+
+describe('getStuckRunningRunsFindOptions', () => {
+ it('should match RUNNING runs older than the threshold', () => {
+ const where = getStuckRunningRunsFindOptions();
+
+ expect(where.status).toBe(WorkflowRunStatus.RUNNING);
+ expect(where.updatedAt).toBeInstanceOf(FindOperator);
+ expect((where.updatedAt as FindOperator<string>).type).toBe('lessThan');
+ });
+});
diff --git a/packages/twenty-server/src/modules/workflow/workflow-runner/workflow-run-queue/utils/get-stuck-running-runs-find-options.util.ts b/packages/twenty-server/src/modules/workflow/workflow-runner/workflow-run-queue/utils/get-stuck-running-runs-find-options.util.ts
new file mode 100644
index 00000000000..59db9a0c876
--- /dev/null
+++ b/packages/twenty-server/src/modules/workflow/workflow-runner/workflow-run-queue/utils/get-stuck-running-runs-find-options.util.ts
@@ -0,0 +1,19 @@
+import { type FindOptionsWhere, LessThan } from 'typeorm';
+
+import {
+ WorkflowRunStatus,
+ type WorkflowRunWorkspaceEntity,
+} from 'src/modules/workflow/common/standard-objects/workflow-run.workspace-entity';
+import { STUCK_RUNNING_RUNS_THRESHOLD_MS } from 'src/modules/workflow/workflow-runner/workflow-run-queue/constants/stuck-running-runs-threshold';
+
+export const getStuckRunningRunsFindOptions =
+ (): FindOptionsWhere<WorkflowRunWorkspaceEntity> => {
+ const thresholdDate = new Date(
+ Date.now() - STUCK_RUNNING_RUNS_THRESHOLD_MS,
+ );
+
+ return {
+ status: WorkflowRunStatus.RUNNING,
+ updatedAt: LessThan(thresholdDate.toISOString()),
+ };
+ };
diff --git a/packages/twenty-server/src/modules/workflow/workflow-runner/workflow-run-queue/utils/get-stuck-running-runs-monitor-cache-key.util.ts b/packages/twenty-server/src/modules/workflow/workflow-runner/workflow-run-queue/utils/get-stuck-running-runs-monitor-cache-key.util.ts
new file mode 100644
index 00000000000..551923734a0
--- /dev/null
+++ b/packages/twenty-server/src/modules/workflow/workflow-runner/workflow-run-queue/utils/get-stuck-running-runs-monitor-cache-key.util.ts
@@ -0,0 +1,2 @@
+export const getStuckRunningRunsMonitorCacheKey = (workspaceId: string) =>
+ `workflow-stuck-running-runs-monitor:${workspaceId}`;
diff --git a/packages/twenty-server/src/modules/workflow/workflow-runner/workflow-run-queue/workspace-services/__tests__/workflow-handle-staled-runs.workspace-service.spec.ts b/packages/twenty-server/src/modules/workflow/workflow-runner/workflow-run-queue/workspace-services/__tests__/workflow-handle-staled-runs.workspace-service.spec.ts
index 29492e1dd92..5888cae017f 100644
--- a/packages/twenty-server/src/modules/workflow/workflow-runner/workflow-run-queue/workspace-services/__tests__/workflow-handle-staled-runs.workspace-service.spec.ts
+++ b/packages/twenty-server/src/modules/workflow/workflow-runner/workflow-run-queue/workspace-services/__tests__/workflow-handle-staled-runs.workspace-service.spec.ts
@@ -1,5 +1,11 @@
import { Test, type TestingModule } from '@nestjs/testing';
+import { StepStatus } from 'twenty-shared/workflow';
+
+import { CacheStorageNamespace } from 'src/engine/core-modules/cache-storage/types/cache-storage-namespace.enum';
+import { MessageQueue } from 'src/engine/core-modules/message-queue/message-queue.constants';
+import { MetricsService } from 'src/engine/core-modules/metrics/metrics.service';
+import { MetricsKeys } from 'src/engine/core-modules/metrics/types/metrics-keys.type';
import { GlobalWorkspaceOrmManager } from 'src/engine/twenty-orm/global-workspace-datasource/global-workspace-orm.manager';
import { WorkflowRunStatus } from 'src/modules/workflow/common/standard-objects/workflow-run.workspace-entity';
import { WorkflowHandleStaledRunsWorkspaceService } from 'src/modules/workflow/workflow-runner/workflow-run-queue/workspace-services/workflow-handle-staled-runs.workspace-service';
@@ -26,6 +32,20 @@ const mockWorkflowThrottlingWorkspaceService = {
const mockWorkflowRunWorkspaceService = {
endWorkflowRun: jest.fn().mockResolvedValue(undefined),
+ getWorkflowRunOrFail: jest.fn(),
+};
+
+const mockMessageQueueService = {
+ getInFlightJobs: jest.fn().mockResolvedValue([]),
+};
+
+const mockMetricsService = {
+ incrementCounterForEvent: jest.fn().mockResolvedValue(undefined),
+};
+
+const mockCacheStorageService = {
+ get: jest.fn().mockResolvedValue(undefined),
+ set: jest.fn().mockResolvedValue(undefined),
};
// Mirrors QUERY_MAX_RECORDS, the per-batch cap the service applies. Kept as a
@@ -61,9 +81,29 @@ describe('WorkflowHandleStaledRunsWorkspaceService', () => {
provide: WorkflowRunWorkspaceService,
useValue: mockWorkflowRunWorkspaceService,
},
+ {
+ provide: `MESSAGE_QUEUE_${MessageQueue.workflowQueue}`,
+ useValue: mockMessageQueueService,
+ },
+ {
+ provide: MetricsService,
+ useValue: mockMetricsService,
+ },
+ {
+ provide: CacheStorageNamespace.ModuleWorkflow,
+ useValue: mockCacheStorageService,
+ },
],
}).compile();
+ mockWorkflowRunWorkspaceService.endWorkflowRun
+ .mockReset()
+ .mockResolvedValue(undefined);
+ mockWorkflowRunWorkspaceService.getWorkflowRunOrFail.mockReset();
+ mockMessageQueueService.getInFlightJobs.mockReset().mockResolvedValue([]);
+ mockCacheStorageService.get.mockReset().mockResolvedValue(undefined);
+ mockCacheStorageService.set.mockReset().mockResolvedValue(undefined);
+
service = module.get<WorkflowHandleStaledRunsWorkspaceService>(
WorkflowHandleStaledRunsWorkspaceService,
);
@@ -237,4 +277,293 @@ describe('WorkflowHandleStaledRunsWorkspaceService', () => {
});
});
});
+
+ describe('handleStuckRunningRunsForWorkspace', () => {
+ const buildRunningWorkflowRun = ({
+ id = 'run-0',
+ steps,
+ stepInfos,
+ }: {
+ id?: string;
+ // oxlint-disable-next-line typescript/no-explicit-any
+ steps: any[];
+ // oxlint-disable-next-line typescript/no-explicit-any
+ stepInfos: Record<string, any>;
+ }) => ({
+ id,
+ status: WorkflowRunStatus.RUNNING,
+ updatedAt: '2026-07-15T00:00:00.000Z',
+ state: { flow: { steps }, stepInfos },
+ });
+
+ const expectNoWorkflowRunEnded = () => {
+ expect(
+ mockWorkflowRunWorkspaceService.endWorkflowRun,
+ ).not.toHaveBeenCalled();
+ };
+
+ it('should do nothing when there are no stuck running nor flagged runs', async () => {
+ mockRepository.find.mockResolvedValueOnce([]);
+
+ await service.handleStuckRunningRunsForWorkspace(workspaceId);
+
+ expect(mockMessageQueueService.getInFlightJobs).not.toHaveBeenCalled();
+ expect(mockCacheStorageService.set).not.toHaveBeenCalled();
+ expectNoWorkflowRunEnded();
+ });
+
+ it('should not flag runs that still have an in-flight job matched by id prefix', async () => {
+ mockRepository.find.mockResolvedValueOnce([{ id: 'run-0' }]);
+ mockMessageQueueService.getInFlightJobs.mockResolvedValueOnce([
+ { id: 'run-0-8a521c10-92b1-4013-a4a7-71f20b1a4a4a', data: {} },
+ ]);
+
+ await service.handleStuckRunningRunsForWorkspace(workspaceId);
+
+ expect(
+ mockWorkflowRunWorkspaceService.getWorkflowRunOrFail,
+ ).not.toHaveBeenCalled();
+ expect(mockCacheStorageService.set).toHaveBeenCalledWith(
+ expect.any(String),
+ {},
+ );
+ expectNoWorkflowRunEnded();
+ });
+
+ it('should not flag runs whose pre-deploy job is matched by job data', async () => {
+ mockRepository.find.mockResolvedValueOnce([{ id: 'run-0' }]);
+ mockMessageQueueService.getInFlightJobs.mockResolvedValueOnce([
+ { id: '12345', data: { workspaceId, workflowRunId: 'run-0' } },
+ ]);
+
+ await service.handleStuckRunningRunsForWorkspace(workspaceId);
+
+ expect(
+ mockWorkflowRunWorkspaceService.getWorkflowRunOrFail,
+ ).not.toHaveBeenCalled();
+ expect(mockCacheStorageService.set).toHaveBeenCalledWith(
+ expect.any(String),
+ {},
+ );
+ expectNoWorkflowRunEnded();
+ });
+
+ it('should flag a run with an orphaned running step without ending it', async () => {
+ mockRepository.find.mockResolvedValueOnce([{ id: 'run-0' }]);
+ mockWorkflowRunWorkspaceService.getWorkflowRunOrFail.mockResolvedValueOnce(
+ buildRunningWorkflowRun({
+ steps: [{ id: 'step-1', type: 'CODE', nextStepIds: [] }],
+ stepInfos: { 'step-1': { status: StepStatus.RUNNING } },
+ }),
+ );
+
+ await service.handleStuckRunningRunsForWorkspace(workspaceId);
+
+ expect(mockMetricsService.incrementCounterForEvent).toHaveBeenCalledWith(
+ expect.objectContaining({
+ key: MetricsKeys.WorkflowRunStuckRunningDetected,
+ }),
+ );
+ expect(mockCacheStorageService.set).toHaveBeenCalledWith(
+ expect.any(String),
+ { 'run-0': expect.any(String) },
+ );
+ expectNoWorkflowRunEnded();
+ });
+
+ it('should not flag runs waiting on a pending step', async () => {
+ mockRepository.find.mockResolvedValueOnce([{ id: 'run-0' }]);
+ mockWorkflowRunWorkspaceService.getWorkflowRunOrFail.mockResolvedValueOnce(
+ buildRunningWorkflowRun({
+ steps: [{ id: 'step-1', type: 'DELAY', nextStepIds: [] }],
+ stepInfos: { 'step-1': { status: StepStatus.PENDING } },
+ }),
+ );
+
+ await service.handleStuckRunningRunsForWorkspace(workspaceId);
+
+ expect(
+ mockMetricsService.incrementCounterForEvent,
+ ).not.toHaveBeenCalled();
+ expect(mockCacheStorageService.set).toHaveBeenCalledWith(
+ expect.any(String),
+ {},
+ );
+ expectNoWorkflowRunEnded();
+ });
+
+ it('should flag a run with a failed branch even when another branch is pending', async () => {
+ mockRepository.find.mockResolvedValueOnce([{ id: 'run-0' }]);
+ mockWorkflowRunWorkspaceService.getWorkflowRunOrFail.mockResolvedValueOnce(
+ buildRunningWorkflowRun({
+ steps: [
+ { id: 'step-1', type: 'CODE', nextStepIds: [] },
+ { id: 'step-2', type: 'DELAY', nextStepIds: [] },
+ ],
+ stepInfos: {
+ 'step-1': { status: StepStatus.FAILED },
+ 'step-2': { status: StepStatus.PENDING },
+ },
+ }),
+ );
+
+ await service.handleStuckRunningRunsForWorkspace(workspaceId);
+
+ expect(mockMetricsService.incrementCounterForEvent).toHaveBeenCalledWith(
+ expect.objectContaining({
+ key: MetricsKeys.WorkflowRunStuckRunningDetected,
+ }),
+ );
+ expectNoWorkflowRunEnded();
+ });
+
+ it('should flag a run whose job was lost between two steps', async () => {
+ mockRepository.find.mockResolvedValueOnce([{ id: 'run-0' }]);
+ mockWorkflowRunWorkspaceService.getWorkflowRunOrFail.mockResolvedValueOnce(
+ buildRunningWorkflowRun({
+ steps: [
+ { id: 'step-1', type: 'CODE', nextStepIds: ['step-2'] },
+ { id: 'step-2', type: 'CODE', nextStepIds: [] },
+ ],
+ stepInfos: {
+ 'step-1': { status: StepStatus.SUCCESS },
+ 'step-2': { status: StepStatus.NOT_STARTED },
+ },
+ }),
+ );
+
+ await service.handleStuckRunningRunsForWorkspace(workspaceId);
+
+ expect(mockMetricsService.incrementCounterForEvent).toHaveBeenCalledWith(
+ expect.objectContaining({
+ key: MetricsKeys.WorkflowRunStuckRunningDetected,
+ }),
+ );
+ expectNoWorkflowRunEnded();
+ });
+
+ it('should not flag runs that progressed since the query', async () => {
+ mockRepository.find.mockResolvedValueOnce([{ id: 'run-0' }]);
+ mockWorkflowRunWorkspaceService.getWorkflowRunOrFail.mockResolvedValueOnce(
+ {
+ id: 'run-0',
+ status: WorkflowRunStatus.COMPLETED,
+ },
+ );
+
+ await service.handleStuckRunningRunsForWorkspace(workspaceId);
+
+ expect(
+ mockMetricsService.incrementCounterForEvent,
+ ).not.toHaveBeenCalled();
+ expect(mockCacheStorageService.set).toHaveBeenCalledWith(
+ expect.any(String),
+ {},
+ );
+ expectNoWorkflowRunEnded();
+ });
+
+ it('should keep a still-stuck flagged run without counting it again', async () => {
+ mockCacheStorageService.get.mockResolvedValueOnce({
+ 'run-0': '2026-07-16T00:00:00.000Z',
+ });
+ mockRepository.find.mockResolvedValueOnce([{ id: 'run-0' }]);
+ mockWorkflowRunWorkspaceService.getWorkflowRunOrFail.mockResolvedValueOnce(
+ buildRunningWorkflowRun({
+ steps: [{ id: 'step-1', type: 'CODE', nextStepIds: [] }],
+ stepInfos: { 'step-1': { status: StepStatus.RUNNING } },
+ }),
+ );
+
+ await service.handleStuckRunningRunsForWorkspace(workspaceId);
+
+ expect(
+ mockMetricsService.incrementCounterForEvent,
+ ).not.toHaveBeenCalled();
+ expect(mockCacheStorageService.set).toHaveBeenCalledWith(
+ expect.any(String),
+ { 'run-0': '2026-07-16T00:00:00.000Z' },
+ );
+ expectNoWorkflowRunEnded();
+ });
+
+ it('should record a false positive when a flagged run ended on its own', async () => {
+ mockCacheStorageService.get.mockResolvedValueOnce({
+ 'run-0': '2026-07-16T00:00:00.000Z',
+ });
+ mockRepository.find.mockResolvedValueOnce([]);
+ mockWorkflowRunWorkspaceService.getWorkflowRunOrFail.mockResolvedValueOnce(
+ {
+ id: 'run-0',
+ status: WorkflowRunStatus.COMPLETED,
+ },
+ );
+
+ await service.handleStuckRunningRunsForWorkspace(workspaceId);
+
+ expect(mockMetricsService.incrementCounterForEvent).toHaveBeenCalledWith(
+ expect.objectContaining({
+ key: MetricsKeys.WorkflowRunStuckRunningFalsePositive,
+ }),
+ );
+ expect(mockCacheStorageService.set).toHaveBeenCalledWith(
+ expect.any(String),
+ {},
+ );
+ expectNoWorkflowRunEnded();
+ });
+
+ it('should record a false positive when a flagged run got a new queue job', async () => {
+ mockCacheStorageService.get.mockResolvedValueOnce({
+ 'run-0': '2026-07-16T00:00:00.000Z',
+ });
+ mockRepository.find.mockResolvedValueOnce([]);
+ mockMessageQueueService.getInFlightJobs.mockResolvedValueOnce([
+ { id: 'run-0-8a521c10-92b1-4013-a4a7-71f20b1a4a4a', data: {} },
+ ]);
+ mockWorkflowRunWorkspaceService.getWorkflowRunOrFail.mockResolvedValueOnce(
+ buildRunningWorkflowRun({
+ steps: [{ id: 'step-1', type: 'CODE', nextStepIds: [] }],
+ stepInfos: { 'step-1': { status: StepStatus.RUNNING } },
+ }),
+ );
+
+ await service.handleStuckRunningRunsForWorkspace(workspaceId);
+
+ expect(mockMetricsService.incrementCounterForEvent).toHaveBeenCalledWith(
+ expect.objectContaining({
+ key: MetricsKeys.WorkflowRunStuckRunningFalsePositive,
+ }),
+ );
+ expect(mockCacheStorageService.set).toHaveBeenCalledWith(
+ expect.any(String),
+ {},
+ );
+ expectNoWorkflowRunEnded();
+ });
+
+ it('should keep checking remaining runs when one check fails', async () => {
+ mockRepository.find.mockResolvedValueOnce([
+ { id: 'run-0' },
+ { id: 'run-1' },
+ ]);
+ mockWorkflowRunWorkspaceService.getWorkflowRunOrFail
+ .mockRejectedValueOnce(new Error('boom'))
+ .mockResolvedValueOnce(
+ buildRunningWorkflowRun({
+ id: 'run-1',
+ steps: [{ id: 'step-1', type: 'CODE', nextStepIds: [] }],
+ stepInfos: { 'step-1': { status: StepStatus.RUNNING } },
+ }),
+ );
+
+ await service.handleStuckRunningRunsForWorkspace(workspaceId);
+
+ expect(mockCacheStorageService.set).toHaveBeenCalledWith(
+ expect.any(String),
+ { 'run-1': expect.any(String) },
+ );
+ expectNoWorkflowRunEnded();
+ });
+ });
});
diff --git a/packages/twenty-server/src/modules/workflow/workflow-runner/workflow-run-queue/workspace-services/workflow-handle-staled-runs.workspace-service.ts b/packages/twenty-server/src/modules/workflow/workflow-runner/workflow-run-queue/workspace-services/workflow-handle-staled-runs.workspace-service.ts
index d43c3be2562..b3d15ffec9b 100644
--- a/packages/twenty-server/src/modules/workflow/workflow-runner/workflow-run-queue/workspace-services/workflow-handle-staled-runs.workspace-service.ts
+++ b/packages/twenty-server/src/modules/workflow/workflow-runner/workflow-run-queue/workspace-services/workflow-handle-staled-runs.workspace-service.ts
@@ -1,14 +1,31 @@
import { Injectable, Logger } from '@nestjs/common';
import { QUERY_MAX_RECORDS } from 'twenty-shared/constants';
+import { isDefined } from 'twenty-shared/utils';
+import { StepStatus } from 'twenty-shared/workflow';
+import { type FindOptionsWhere } from 'typeorm';
+import { InjectCacheStorage } from 'src/engine/core-modules/cache-storage/decorators/cache-storage.decorator';
+import { CacheStorageService } from 'src/engine/core-modules/cache-storage/services/cache-storage.service';
+import { CacheStorageNamespace } from 'src/engine/core-modules/cache-storage/types/cache-storage-namespace.enum';
+import { InjectMessageQueue } from 'src/engine/core-modules/message-queue/decorators/message-queue.decorator';
+import { MessageQueue } from 'src/engine/core-modules/message-queue/message-queue.constants';
+import { MessageQueueService } from 'src/engine/core-modules/message-queue/services/message-queue.service';
+import { MetricsService } from 'src/engine/core-modules/metrics/metrics.service';
+import { MetricsKeys } from 'src/engine/core-modules/metrics/types/metrics-keys.type';
import { GlobalWorkspaceOrmManager } from 'src/engine/twenty-orm/global-workspace-datasource/global-workspace-orm.manager';
import { buildSystemAuthContext } from 'src/engine/twenty-orm/utils/build-system-auth-context.util';
import {
WorkflowRunStatus,
WorkflowRunWorkspaceEntity,
} from 'src/modules/workflow/common/standard-objects/workflow-run.workspace-entity';
+import { workflowHasRunningSteps } from 'src/modules/workflow/common/utils/workflow-has-running-steps.util';
+import { workflowShouldFail } from 'src/modules/workflow/workflow-executor/utils/workflow-should-fail.util';
+import { workflowShouldKeepRunning } from 'src/modules/workflow/workflow-executor/utils/workflow-should-keep-running.util';
+import { type RunWorkflowJobData } from 'src/modules/workflow/workflow-runner/types/run-workflow-job-data.type';
import { getStaledRunsFindOptions } from 'src/modules/workflow/workflow-runner/workflow-run-queue/utils/get-staled-runs-find-options.util';
+import { getStuckRunningRunsFindOptions } from 'src/modules/workflow/workflow-runner/workflow-run-queue/utils/get-stuck-running-runs-find-options.util';
+import { getStuckRunningRunsMonitorCacheKey } from 'src/modules/workflow/workflow-runner/workflow-run-queue/utils/get-stuck-running-runs-monitor-cache-key.util';
import { getStuckStoppingRunsFindOptions } from 'src/modules/workflow/workflow-runner/workflow-run-queue/utils/get-stuck-stopping-runs-find-options.util';
import { WorkflowThrottlingWorkspaceService } from 'src/modules/workflow/workflow-runner/workflow-run-queue/workspace-services/workflow-throttling.workspace-service';
import { WorkflowRunWorkspaceService } from 'src/modules/workflow/workflow-runner/workflow-run/workflow-run.workspace-service';
@@ -22,6 +39,11 @@ export class WorkflowHandleStaledRunsWorkspaceService {
private readonly globalWorkspaceOrmManager: GlobalWorkspaceOrmManager,
private readonly workflowThrottlingWorkspaceService: WorkflowThrottlingWorkspaceService,
private readonly workflowRunWorkspaceService: WorkflowRunWorkspaceService,
+ @InjectMessageQueue(MessageQueue.workflowQueue)
+ private readonly messageQueueService: MessageQueueService,
+ private readonly metricsService: MetricsService,
+ @InjectCacheStorage(CacheStorageNamespace.ModuleWorkflow)
+ private readonly cacheStorageService: CacheStorageService,
) {}
async handleStaledRunsForWorkspace(workspaceId: string) {
@@ -77,39 +99,10 @@ export class WorkflowHandleStaledRunsWorkspaceService {
}
async handleStuckStoppingRunsForWorkspace(workspaceId: string) {
- const authContext = buildSystemAuthContext(workspaceId);
-
- const stuckStoppingRunIds =
- await this.globalWorkspaceOrmManager.executeInWorkspaceContext(
- async () => {
- const workflowRunRepository =
- await this.globalWorkspaceOrmManager.getRepository(
- workspaceId,
- WorkflowRunWorkspaceEntity,
- { shouldBypassPermissionChecks: true },
- );
-
- const findOptions = getStuckStoppingRunsFindOptions();
-
- const runIds: string[] = [];
- let page: WorkflowRunWorkspaceEntity[];
-
- do {
- page = await workflowRunRepository.find({
- where: findOptions,
- select: { id: true },
- order: { createdAt: 'ASC', id: 'ASC' },
- take: QUERY_MAX_RECORDS,
- skip: runIds.length,
- });
-
- runIds.push(...page.map((workflowRun) => workflowRun.id));
- } while (page.length === QUERY_MAX_RECORDS);
-
- return runIds;
- },
- authContext,
- );
+ const stuckStoppingRunIds = await this.collectRunIds({
+ workspaceId,
+ findOptions: getStuckStoppingRunsFindOptions(),
+ });
for (const workflowRunId of stuckStoppingRunIds) {
try {
@@ -126,4 +119,194 @@ export class WorkflowHandleStaledRunsWorkspaceService {
}
}
}
+
+ // Monitoring mode: stuck RUNNING runs are only flagged, never finalized.
+ // Flagged runs are re-checked on every sweep; one that ends or gets a new
+ // job on its own is a false positive, disproving that it was stuck forever.
+ async handleStuckRunningRunsForWorkspace(workspaceId: string) {
+ const cacheKey = getStuckRunningRunsMonitorCacheKey(workspaceId);
+ const flaggedRuns =
+ (await this.cacheStorageService.get<Record<string, string>>(cacheKey)) ??
+ {};
+
+ const stuckRunningRunIds = await this.collectRunIds({
+ workspaceId,
+ findOptions: getStuckRunningRunsFindOptions(),
+ });
+
+ if (
+ stuckRunningRunIds.length === 0 &&
+ Object.keys(flaggedRuns).length === 0
+ ) {
+ return;
+ }
+
+ const inFlightJobs =
+ await this.messageQueueService.getInFlightJobs<RunWorkflowJobData>();
+
+ const hasInFlightJob = (workflowRunId: string) =>
+ inFlightJobs.some(
+ (job) =>
+ job.id?.startsWith(`${workflowRunId}-`) ||
+ job.data?.workflowRunId === workflowRunId,
+ );
+
+ const stillFlaggedRuns: Record<string, string> = {};
+
+ for (const [workflowRunId, detectedAt] of Object.entries(flaggedRuns)) {
+ try {
+ const workflowRun =
+ await this.workflowRunWorkspaceService.getWorkflowRunOrFail({
+ workflowRunId,
+ workspaceId,
+ });
+
+ if (
+ workflowRun.status !== WorkflowRunStatus.RUNNING ||
+ hasInFlightJob(workflowRunId)
+ ) {
+ this.logger.warn(
+ `Stuck running workflow run ${workflowRunId} in workspace ${workspaceId} was a false positive: flagged at ${detectedAt}, now in status ${workflowRun.status}`,
+ );
+
+ await this.metricsService.incrementCounterForEvent({
+ key: MetricsKeys.WorkflowRunStuckRunningFalsePositive,
+ eventId: workflowRunId,
+ });
+ } else {
+ stillFlaggedRuns[workflowRunId] = detectedAt;
+ }
+ } catch (error) {
+ stillFlaggedRuns[workflowRunId] = detectedAt;
+ this.logger.error(
+ `Failed to re-check flagged stuck running workflow run ${workflowRunId} for workspace ${workspaceId}`,
+ error,
+ );
+ }
+ }
+
+ for (const workflowRunId of stuckRunningRunIds) {
+ if (
+ isDefined(flaggedRuns[workflowRunId]) ||
+ hasInFlightJob(workflowRunId)
+ ) {
+ continue;
+ }
+
+ try {
+ const expectedOutcome = await this.computeStuckRunningRunOutcome({
+ workflowRunId,
+ workspaceId,
+ });
+
+ if (!isDefined(expectedOutcome)) {
+ continue;
+ }
+
+ stillFlaggedRuns[workflowRunId] = new Date().toISOString();
+
+ this.logger.warn(
+ `Workflow run ${workflowRunId} in workspace ${workspaceId} is stuck in RUNNING without a queue job and would have been finalized as ${expectedOutcome}`,
+ );
+
+ await this.metricsService.incrementCounterForEvent({
+ key: MetricsKeys.WorkflowRunStuckRunningDetected,
+ eventId: workflowRunId,
+ });
+ } catch (error) {
+ this.logger.error(
+ `Failed to check stuck running workflow run ${workflowRunId} for workspace ${workspaceId}`,
+ error,
+ );
+ }
+ }
+
+ await this.cacheStorageService.set(cacheKey, stillFlaggedRuns);
+ }
+
+ private async computeStuckRunningRunOutcome({
+ workflowRunId,
+ workspaceId,
+ }: {
+ workflowRunId: string;
+ workspaceId: string;
+ }): Promise<WorkflowRunStatus | undefined> {
+ const workflowRun =
+ await this.workflowRunWorkspaceService.getWorkflowRunOrFail({
+ workflowRunId,
+ workspaceId,
+ });
+
+ if (workflowRun.status !== WorkflowRunStatus.RUNNING) {
+ return undefined;
+ }
+
+ const stepInfos = workflowRun.state?.stepInfos;
+ const steps = workflowRun.state?.flow?.steps;
+
+ if (
+ !isDefined(stepInfos) ||
+ !isDefined(steps) ||
+ workflowHasRunningSteps({ stepInfos, steps })
+ ) {
+ return WorkflowRunStatus.FAILED;
+ }
+
+ if (workflowShouldFail({ stepInfos, steps })) {
+ return WorkflowRunStatus.FAILED;
+ }
+
+ const hasPendingSteps = steps.some(
+ (step) => stepInfos[step.id]?.status === StepStatus.PENDING,
+ );
+
+ if (hasPendingSteps) {
+ return undefined;
+ }
+
+ if (workflowShouldKeepRunning({ stepInfos, steps })) {
+ return WorkflowRunStatus.FAILED;
+ }
+
+ return WorkflowRunStatus.COMPLETED;
+ }
+
+ private async collectRunIds({
+ workspaceId,
+ findOptions,
+ }: {
+ workspaceId: string;
+ findOptions: FindOptionsWhere<WorkflowRunWorkspaceEntity>;
+ }): Promise<string[]> {
+ const authContext = buildSystemAuthContext(workspaceId);
+
+ return this.globalWorkspaceOrmManager.executeInWorkspaceContext(
+ async () => {
+ const workflowRunRepository =
+ await this.globalWorkspaceOrmManager.getRepository(
+ workspaceId,
+ WorkflowRunWorkspaceEntity,
+ { shouldBypassPermissionChecks: true },
+ );
+
+ const runIds: string[] = [];
+ let page: WorkflowRunWorkspaceEntity[];
+
+ do {
+ page = await workflowRunRepository.find({
+ where: findOptions,
+ select: { id: true },
+ order: { createdAt: 'ASC', id: 'ASC' },
+ take: QUERY_MAX_RECORDS,
+ skip: runIds.length,
+ });
+
+ runIds.push(...page.map((workflowRun) => workflowRun.id));
+ } while (page.length === QUERY_MAX_RECORDS);
+
+ return runIds;
+ },
+ authContext,
+ );
+ }
}
diff --git a/packages/twenty-server/src/modules/workflow/workflow-runner/workflow-run-queue/workspace-services/workflow-run-enqueue.workspace-service.ts b/packages/twenty-server/src/modules/workflow/workflow-runner/workflow-run-queue/workspace-services/workflow-run-enqueue.workspace-service.ts
index 84a2edd51cf..c1077b4e420 100644
--- a/packages/twenty-server/src/modules/workflow/workflow-runner/workflow-run-queue/workspace-services/workflow-run-enqueue.workspace-service.ts
+++ b/packages/twenty-server/src/modules/workflow/workflow-runner/workflow-run-queue/workspace-services/workflow-run-enqueue.workspace-service.ts
@@ -15,6 +15,7 @@ import {
} from 'src/modules/workflow/common/standard-objects/workflow-run.workspace-entity';
import { RunWorkflowJob } from 'src/modules/workflow/workflow-runner/jobs/run-workflow.job';
import { type RunWorkflowJobData } from 'src/modules/workflow/workflow-runner/types/run-workflow-job-data.type';
+import { buildRunWorkflowJobOptions } from 'src/modules/workflow/workflow-runner/utils/build-run-workflow-job-options.util';
import { NOT_STARTED_RUNS_FIND_OPTIONS } from 'src/modules/workflow/workflow-runner/workflow-run-queue/constants/not-started-runs-find-options';
import { WorkflowThrottlingWorkspaceService } from 'src/modules/workflow/workflow-runner/workflow-run-queue/workspace-services/workflow-throttling.workspace-service';
@@ -117,6 +118,7 @@ export class WorkflowRunEnqueueWorkspaceService {
workflowRunId,
workspaceId,
},
+ buildRunWorkflowJobOptions(workflowRunId),
);
}
diff --git a/packages/twenty-server/src/modules/workflow/workflow-runner/workspace-services/workflow-runner.workspace-service.ts b/packages/twenty-server/src/modules/workflow/workflow-runner/workspace-services/workflow-runner.workspace-service.ts
index 962322c2ed5..5681e78f2b5 100644
--- a/packages/twenty-server/src/modules/workflow/workflow-runner/workspace-services/workflow-runner.workspace-service.ts
+++ b/packages/twenty-server/src/modules/workflow/workflow-runner/workspace-services/workflow-runner.workspace-service.ts
@@ -28,6 +28,7 @@ import {
import { RunWorkflowJob } from 'src/modules/workflow/workflow-runner/jobs/run-workflow.job';
import { type RunWorkflowJobData } from 'src/modules/workflow/workflow-runner/types/run-workflow-job-data.type';
import { buildRetryStepInfos } from 'src/modules/workflow/workflow-runner/utils/build-retry-step-infos.util';
+import { buildRunWorkflowJobOptions } from 'src/modules/workflow/workflow-runner/utils/build-run-workflow-job-options.util';
import { getRunnableStepIds } from 'src/modules/workflow/workflow-runner/utils/get-runnable-step-ids.util';
import {
WorkflowRunEnqueueJob,
@@ -130,6 +131,7 @@ export class WorkflowRunnerWorkspaceService {
workflowRunId,
lastExecutedStepId,
},
+ buildRunWorkflowJobOptions(workflowRunId),
);
}
@@ -350,6 +352,7 @@ export class WorkflowRunnerWorkspaceService {
workflowRunId,
stepIdsToRetry: stepIdsToRun,
},
+ buildRunWorkflowJobOptions(workflowRunId),
);
} catch (error) {
// The job couldn't be enqueued: revert to the previous failed state so
@@ -446,6 +449,7 @@ export class WorkflowRunnerWorkspaceService {
workspaceId,
workflowRunId,
},
+ buildRunWorkflowJobOptions(workflowRunId),
);
return { workflowRunId };