Skip to content

AI Agents & MCPs & AI Workflow Automation • (~400 MCP servers for AI agents) • AI Automation / AI Agent with MCPs • AI Workflows & AI Agents • MCPs for AI Agents

fix(flow-runs): refuse retry from failed step for runs that did not fail (ENG-949) (#16707)

lina-alomari635a2cbe008f
diff --git a/packages/core/utils/src/lib/activepieces-error.ts b/packages/core/utils/src/lib/activepieces-error.ts
index 6b16d161a93..da32ea74c76 100644
--- a/packages/core/utils/src/lib/activepieces-error.ts
+++ b/packages/core/utils/src/lib/activepieces-error.ts
@@ -25,6 +25,7 @@ export type ApErrorParams =
| ExistingUserErrorParams
| FlowOperationErrorParams
| FlowOperationInProgressErrorParams
+ | FlowRunNotRetryableErrorParams
| FlowRunRetryOutsideRetentionErrorParams
| InvalidApiKeyParams
| InvalidAppConnectionParams
@@ -196,6 +197,14 @@ ErrorCode.SYSTEM_PROP_INVALID,
}
>
+export type FlowRunNotRetryableErrorParams = BaseErrorParams<
+ErrorCode.FLOW_RUN_NOT_RETRYABLE,
+{
+ flowRunId: FlowRunId
+ status: string
+}
+>
+
export type FlowRunRetryOutsideRetentionErrorParams = BaseErrorParams<
ErrorCode.FLOW_RUN_RETRY_OUTSIDE_RETENTION,
{
@@ -601,6 +610,7 @@ export enum ErrorCode {
FLOW_EXTERNAL_ID_ALREADY_EXISTS = 'FLOW_EXTERNAL_ID_ALREADY_EXISTS',
FLOW_OPERATION_INVALID = 'FLOW_OPERATION_INVALID',
FLOW_OPERATION_IN_PROGRESS = 'FLOW_OPERATION_IN_PROGRESS',
+ FLOW_RUN_NOT_RETRYABLE = 'FLOW_RUN_NOT_RETRYABLE',
FLOW_RUN_RETRY_OUTSIDE_RETENTION = 'FLOW_RUN_RETRY_OUTSIDE_RETENTION',
INVALID_API_KEY = 'INVALID_API_KEY',
INVALID_APP_CONNECTION = 'INVALID_APP_CONNECTION',
diff --git a/packages/server/api/src/app/flows/flow-run/flow-run-service.ts b/packages/server/api/src/app/flows/flow-run/flow-run-service.ts
index 5d54475e915..15cca2381fd 100644
--- a/packages/server/api/src/app/flows/flow-run/flow-run-service.ts
+++ b/packages/server/api/src/app/flows/flow-run/flow-run-service.ts
@@ -1,6 +1,6 @@
-import { ActivepiecesError, apId, ApId, Cursor, ErrorCode, FlowId, FlowRunId, FlowVersionId, isNil, PlatformId, ProjectId, SeekPage } from '@activepieces/core-utils'
+import { ActivepiecesError, apId, ApId, Cursor, ErrorCode, FlowId, FlowRunId, FlowVersionId, isEmpty, isNil, PlatformId, ProjectId, SeekPage } from '@activepieces/core-utils'
import { apDayjs, apDayjsDuration, wideEvent } from '@activepieces/server-utils'
-import { ExecuteFlowJobData, ExecutionType, ExecutioOutputFile, FileCompression, FileType, FlowRetryStrategy, FlowRun, FlowRunCountByStatus, FlowRunStatus, FlowRunWithRetryError, flowStructureUtil, FlowVersion, GenericStepOutput, isFlowRunStateTerminal, JobPayload, LATEST_JOB_DATA_SCHEMA_VERSION, logSerializer, LogSliceRef, ResumeReason, RunEnvironment, RunInternalError, SampleDataFileType, StepOutput, StepOutputStatus, StepOutputType, StreamStepProgress, WorkerJobType } from '@activepieces/shared'
+import { ExecuteFlowJobData, ExecutionType, ExecutioOutputFile, FAILED_STATES, FileCompression, FileType, FlowRetryStrategy, FlowRun, FlowRunCountByStatus, FlowRunStatus, FlowRunWithRetryError, flowStructureUtil, FlowVersion, GenericStepOutput, isFlowRunStateTerminal, JobPayload, LATEST_JOB_DATA_SCHEMA_VERSION, logSerializer, LogSliceRef, ResumeReason, RunEnvironment, RunInternalError, SampleDataFileType, StepOutput, StepOutputStatus, StepOutputType, StreamStepProgress, WorkerJobType } from '@activepieces/shared'
import { FastifyBaseLogger } from 'fastify'
import pLimit from 'p-limit'
import { ArrayContains, In, IsNull, Not, Repository, SelectQueryBuilder } from 'typeorm'
@@ -26,6 +26,7 @@ import { flowRunSideEffects } from './flow-run-side-effects'
import { runsMetadataQueue } from './flow-runs-queue'
const CANCELLABLE_STATUSES: FlowRunStatus[] = [FlowRunStatus.PAUSED, FlowRunStatus.QUEUED]
+const RETRYABLE_FROM_FAILED_STEP_STATUSES: FlowRunStatus[] = [...FAILED_STATES, FlowRunStatus.QUEUED, FlowRunStatus.RUNNING]
const PARENT_BOUND_RETRY_BACKOFF_MS = apDayjsDuration(30, 'second').asMilliseconds()
@@ -169,14 +170,29 @@ export const flowRunService = (log: FastifyBaseLogger) => ({
? await resolveStepOutput({ step: triggerStep, flowRun: oldFlowRun, log })
: undefined
- await flowRunRepo().update({
- id: oldFlowRun.id,
- projectId: oldFlowRun.projectId,
- }, {
- status: FlowRunStatus.QUEUED,
- startTime: apDayjs().toISOString(),
- finishTime: null,
- })
+ const { raw: requeuedRuns } = await flowRunRepo().createQueryBuilder()
+ .update()
+ .set({
+ status: FlowRunStatus.QUEUED,
+ startTime: apDayjs().toISOString(),
+ finishTime: null,
+ })
+ .where({
+ id: oldFlowRun.id,
+ projectId: oldFlowRun.projectId,
+ status: In(RETRYABLE_FROM_FAILED_STEP_STATUSES),
+ })
+ .returning('id')
+ .execute()
+ if (requeuedRuns.length === 0) {
+ throw new ActivepiecesError({
+ code: ErrorCode.FLOW_RUN_NOT_RETRYABLE,
+ params: {
+ flowRunId: oldFlowRun.id,
+ status: oldFlowRun.status,
+ },
+ })
+ }
const updatedFlowRun = await findFlowRunOrThrow(oldFlowRun.id)
const platformId = await projectService(log).getPlatformId(updatedFlowRun.projectId)
await flowRunSideEffects(log).onRetry({ flowRun: updatedFlowRun, platformId })
@@ -266,7 +282,12 @@ export const flowRunService = (log: FastifyBaseLogger) => ({
})
},
async bulkRetry(params: BulkRetryParams): Promise<FlowRunWithRetryError[]> {
- const filteredFlowRuns = await filterFlowRunsAndApplyFilters(params)
+ const isFromFailedStepByFilter = params.strategy === FlowRetryStrategy.FROM_FAILED_STEP && isEmpty(params.flowRunIds)
+ const statusFilter = isFromFailedStepByFilter ? restrictToFailedStatuses(params.status) : params.status
+ if (isFromFailedStepByFilter && isEmpty(statusFilter)) {
+ return []
+ }
+ const filteredFlowRuns = await filterFlowRunsAndApplyFilters({ ...params, status: statusFilter })
const limit = pLimit(10)
const results = await Promise.allSettled(
filteredFlowRuns.map(flowRun =>
@@ -573,6 +594,13 @@ async function getAllChildRuns(parentRunIds: string[]): Promise<FlowRun[]> {
}
+function restrictToFailedStatuses(statusFilter: FlowRunStatus[] | undefined): FlowRunStatus[] {
+ if (isNil(statusFilter) || statusFilter.length === 0) {
+ return FAILED_STATES
+ }
+ return statusFilter.filter((status) => FAILED_STATES.includes(status))
+}
+
async function filterFlowRunsAndApplyFilters(
params: FilterFlowRunsAndApplyFiltersParams,
): Promise<FlowRun[]> {
diff --git a/packages/server/api/src/app/helper/error-handler.ts b/packages/server/api/src/app/helper/error-handler.ts
index af40a2b1776..bd68c9ecc1d 100644
--- a/packages/server/api/src/app/helper/error-handler.ts
+++ b/packages/server/api/src/app/helper/error-handler.ts
@@ -126,6 +126,7 @@ const statusCodeMap: Partial<Record<ErrorCode, StatusCodes>> = {
[ErrorCode.MCP_PIECE_REQUIRES_CONNECTION]: StatusCodes.BAD_REQUEST,
[ErrorCode.MCP_PIECE_CONNECTION_MISMATCH]: StatusCodes.BAD_REQUEST,
[ErrorCode.DOES_NOT_MEET_BUSINESS_REQUIREMENTS]: StatusCodes.UNPROCESSABLE_ENTITY,
+ [ErrorCode.FLOW_RUN_NOT_RETRYABLE]: StatusCodes.CONFLICT,
[ErrorCode.FLOW_RUN_RETRY_OUTSIDE_RETENTION]: StatusCodes.GONE,
[ErrorCode.SANDBOX_CAPACITY_EXCEEDED]: StatusCodes.TOO_MANY_REQUESTS,
[ErrorCode.CHAT_CONTEXT_LIMIT_EXCEEDED]: StatusCodes.BAD_REQUEST,
diff --git a/packages/server/api/test/integration/ce/flows/flow-run/retry-from-failed-step-status.test.ts b/packages/server/api/test/integration/ce/flows/flow-run/retry-from-failed-step-status.test.ts
new file mode 100644
index 00000000000..db49d23a3b6
--- /dev/null
+++ b/packages/server/api/test/integration/ce/flows/flow-run/retry-from-failed-step-status.test.ts
@@ -0,0 +1,153 @@
+import { ErrorCode } from '@activepieces/core-utils'
+import { FAILED_STATES, FlowRetryStrategy, FlowRun, FlowRunStatus, FlowRunWithRetryError, FlowVersionState, RunEnvironment } from '@activepieces/shared'
+import { FastifyInstance } from 'fastify'
+import { db } from '../../../../helpers/db'
+import { createMockFlow, createMockFlowRun, createMockFlowVersion } from '../../../../helpers/mocks'
+import { createTestContext, TestContext } from '../../../../helpers/test-context'
+import { setupTestEnvironment, teardownTestEnvironment } from '../../../../helpers/test-setup'
+
+let app: FastifyInstance
+let ctx: TestContext
+
+beforeAll(async () => {
+ app = await setupTestEnvironment()
+})
+
+afterAll(async () => {
+ await teardownTestEnvironment()
+})
+
+beforeEach(async () => {
+ ctx = await createTestContext(app)
+})
+
+describe('Retry from failed step (POST /v1/flow-runs/:id/retry)', () => {
+ it.each([
+ FlowRunStatus.SUCCEEDED,
+ FlowRunStatus.PAUSED,
+ FlowRunStatus.CANCELED,
+ FlowRunStatus.LOG_SIZE_EXCEEDED,
+ ])('refuses a %s run and leaves it unchanged', async (status) => {
+ const flowRun = await createRunWithStatus(status)
+ const before = await readRun(flowRun.id)
+
+ const response = await ctx.post(`/v1/flow-runs/${flowRun.id}/retry`, {
+ strategy: FlowRetryStrategy.FROM_FAILED_STEP,
+ projectId: ctx.project.id,
+ })
+
+ expect(response.statusCode).toBe(409)
+ expect(response.json().code).toBe(ErrorCode.FLOW_RUN_NOT_RETRYABLE)
+ const after = await readRun(flowRun.id)
+ expect(after.status).toBe(status)
+ expect(after.startTime).toEqual(before.startTime)
+ })
+
+ it.each([
+ ...FAILED_STATES,
+ FlowRunStatus.QUEUED,
+ FlowRunStatus.RUNNING,
+ ])('accepts a %s run and queues it', async (status) => {
+ const flowRun = await createRunWithStatus(status)
+
+ const response = await ctx.post(`/v1/flow-runs/${flowRun.id}/retry`, {
+ strategy: FlowRetryStrategy.FROM_FAILED_STEP,
+ projectId: ctx.project.id,
+ })
+
+ expect(response.statusCode).toBe(200)
+ const after = await readRun(flowRun.id)
+ expect(after.status).toBe(FlowRunStatus.QUEUED)
+ })
+
+ it('retry from failed step by filter retries only failed runs', async () => {
+ const failedRun = await createRunWithStatus(FlowRunStatus.FAILED)
+ const otherRuns = await Promise.all([
+ FlowRunStatus.QUEUED,
+ FlowRunStatus.RUNNING,
+ FlowRunStatus.PAUSED,
+ FlowRunStatus.SUCCEEDED,
+ ].map((status) => createRunWithStatus(status)))
+ const otherRunsBefore = await Promise.all(otherRuns.map((run) => readRun(run.id)))
+
+ const response = await ctx.post('/v1/flow-runs/retry', {
+ projectId: ctx.project.id,
+ strategy: FlowRetryStrategy.FROM_FAILED_STEP,
+ })
+
+ expect(response.statusCode).toBe(200)
+ const results: FlowRunWithRetryError[] = response.json()
+ expect(results.map((run) => run.id)).toEqual([failedRun.id])
+ expect((await readRun(failedRun.id)).status).toBe(FlowRunStatus.QUEUED)
+ const otherRunsAfter = await Promise.all(otherRuns.map((run) => readRun(run.id)))
+ expect(otherRunsAfter).toEqual(otherRunsBefore)
+ })
+
+ it('retry from failed step by filter keeps only the failed statuses of the status filter', async () => {
+ const failedRun = await createRunWithStatus(FlowRunStatus.FAILED)
+ const runningRun = await createRunWithStatus(FlowRunStatus.RUNNING)
+ const runningBefore = await readRun(runningRun.id)
+
+ const response = await ctx.post('/v1/flow-runs/retry', {
+ projectId: ctx.project.id,
+ strategy: FlowRetryStrategy.FROM_FAILED_STEP,
+ status: [FlowRunStatus.FAILED, FlowRunStatus.RUNNING],
+ })
+
+ expect(response.statusCode).toBe(200)
+ const results: FlowRunWithRetryError[] = response.json()
+ expect(results.map((run) => run.id)).toEqual([failedRun.id])
+ expect(await readRun(runningRun.id)).toEqual(runningBefore)
+ })
+
+ it('retry from failed step by filter retries nothing when the status filter has no failed status', async () => {
+ const runningRun = await createRunWithStatus(FlowRunStatus.RUNNING)
+ const runningBefore = await readRun(runningRun.id)
+
+ const response = await ctx.post('/v1/flow-runs/retry', {
+ projectId: ctx.project.id,
+ strategy: FlowRetryStrategy.FROM_FAILED_STEP,
+ status: [FlowRunStatus.RUNNING],
+ })
+
+ expect(response.statusCode).toBe(200)
+ expect(response.json()).toEqual([])
+ expect(await readRun(runningRun.id)).toEqual(runningBefore)
+ })
+
+ it('retry from failed step by run ids still accepts a RUNNING run', async () => {
+ const runningRun = await createRunWithStatus(FlowRunStatus.RUNNING)
+
+ const response = await ctx.post('/v1/flow-runs/retry', {
+ projectId: ctx.project.id,
+ strategy: FlowRetryStrategy.FROM_FAILED_STEP,
+ flowRunIds: [runningRun.id],
+ })
+
+ expect(response.statusCode).toBe(200)
+ const results: FlowRunWithRetryError[] = response.json()
+ expect(results.map((run) => ({ id: run.id, error: run.error }))).toEqual([{ id: runningRun.id, error: undefined }])
+ expect((await readRun(runningRun.id)).status).toBe(FlowRunStatus.QUEUED)
+ })
+})
+
+async function createRunWithStatus(status: FlowRunStatus): Promise<FlowRun> {
+ const flow = createMockFlow({ projectId: ctx.project.id })
+ await db.save('flow', flow)
+ const flowVersion = createMockFlowVersion({ flowId: flow.id, state: FlowVersionState.LOCKED })
+ await db.save('flow_version', flowVersion)
+ const flowRun = createMockFlowRun({
+ projectId: ctx.project.id,
+ flowId: flow.id,
+ flowVersionId: flowVersion.id,
+ status,
+ environment: RunEnvironment.PRODUCTION,
+ created: new Date().toISOString(),
+ })
+ await db.save('flow_run', flowRun)
+ return flowRun
+}
+
+async function readRun(id: string): Promise<Pick<FlowRun, 'status' | 'startTime'>> {
+ return db.findOneByOrFail<Pick<FlowRun, 'status' | 'startTime'>>('flow_run', { id })
+}
diff --git a/packages/web/src/features/flow-runs/components/runs-table/failed-retry-runs-dialog.tsx b/packages/web/src/features/flow-runs/components/runs-table/failed-retry-runs-dialog.tsx
index ac09725cd19..27eff426600 100644
--- a/packages/web/src/features/flow-runs/components/runs-table/failed-retry-runs-dialog.tsx
+++ b/packages/web/src/features/flow-runs/components/runs-table/failed-retry-runs-dialog.tsx
@@ -66,15 +66,7 @@ export const FailedRetryRunsDialog = ({
</span>
</div>
<p className="text-xs text-gray-11">
- {run.error?.errorCode ===
- ErrorCode.FLOW_RUN_RETRY_OUTSIDE_RETENTION
- ? t(
- 'Retry is only available for {failedJobRetentionDays} after a run fails.',
- {
- failedJobRetentionDays: retentionDays,
- },
- )
- : run.error.errorMessage ?? t('Internal server error')}
+ {retryErrorMessage({ run, retentionDays })}
</p>
</div>
<Button
@@ -101,3 +93,27 @@ export const FailedRetryRunsDialog = ({
</Dialog>
);
};
+
+function retryErrorMessage({
+ run,
+ retentionDays,
+}: RetryErrorMessageParams): string {
+ switch (run.error.errorCode) {
+ case ErrorCode.FLOW_RUN_RETRY_OUTSIDE_RETENTION:
+ return t(
+ 'Retry is only available for {failedJobRetentionDays} after a run fails.',
+ {
+ failedJobRetentionDays: retentionDays,
+ },
+ );
+ case ErrorCode.FLOW_RUN_NOT_RETRYABLE:
+ return t('Only failed runs can be retried from failed step');
+ default:
+ return run.error.errorMessage ?? t('Internal server error');
+ }
+}
+
+type RetryErrorMessageParams = {
+ run: Required<FlowRunWithRetryError>;
+ retentionDays: number | null | undefined;
+};
diff --git a/packages/web/src/features/flow-runs/hooks/flow-run-hooks.ts b/packages/web/src/features/flow-runs/hooks/flow-run-hooks.ts
index d5815ce5bab..bab38319f9c 100644
--- a/packages/web/src/features/flow-runs/hooks/flow-run-hooks.ts
+++ b/packages/web/src/features/flow-runs/hooks/flow-run-hooks.ts
@@ -159,6 +159,15 @@ export const flowRunMutations = {
closeButton: true,
dismissible: true,
});
+ } else if (apError.code === ErrorCode.FLOW_RUN_NOT_RETRYABLE) {
+ toast.error(t('Retry failed'), {
+ description: t(
+ 'Only failed runs can be retried from failed step',
+ ),
+ duration: 5000,
+ closeButton: true,
+ dismissible: true,
+ });
} else if (apError.code === ErrorCode.QUOTA_EXCEEDED) {
billingDialogHooks.useManagePlanDialogStore.getState().openDialog();
}