AI Agents & MCPs & AI Workflow Automation • (~400 MCP servers for AI agents) • AI Automation / AI Agent with MCPs • AI Workflows & AI Agents • MCPs for AI Agents
TypeScript ai-agentai-agent-toolsai-agentsai-agents-frameworkmcpmcp-servermcp-toolsmcpsn8n-alternativeno-code-automationworkflowworkflow-automationworkflows
fix(flow-runs): refuse retry from failed step for runs that did not fail (ENG-949) (#16707)
lina-alomari635a2cbe008f
diff --git a/packages/core/utils/src/lib/activepieces-error.ts b/packages/core/utils/src/lib/activepieces-error.tsindex 6b16d161a93..da32ea74c76 100644--- a/packages/core/utils/src/lib/activepieces-error.ts+++ b/packages/core/utils/src/lib/activepieces-error.ts@@ -25,6 +25,7 @@ export type ApErrorParams =| ExistingUserErrorParams| FlowOperationErrorParams| FlowOperationInProgressErrorParams+ | FlowRunNotRetryableErrorParams| FlowRunRetryOutsideRetentionErrorParams| InvalidApiKeyParams| InvalidAppConnectionParams@@ -196,6 +197,14 @@ ErrorCode.SYSTEM_PROP_INVALID,}>+export type FlowRunNotRetryableErrorParams = BaseErrorParams<+ErrorCode.FLOW_RUN_NOT_RETRYABLE,+{+ flowRunId: FlowRunId+ status: string+}+>+export type FlowRunRetryOutsideRetentionErrorParams = BaseErrorParams<ErrorCode.FLOW_RUN_RETRY_OUTSIDE_RETENTION,{@@ -601,6 +610,7 @@ export enum ErrorCode {FLOW_EXTERNAL_ID_ALREADY_EXISTS = 'FLOW_EXTERNAL_ID_ALREADY_EXISTS',FLOW_OPERATION_INVALID = 'FLOW_OPERATION_INVALID',FLOW_OPERATION_IN_PROGRESS = 'FLOW_OPERATION_IN_PROGRESS',+ FLOW_RUN_NOT_RETRYABLE = 'FLOW_RUN_NOT_RETRYABLE',FLOW_RUN_RETRY_OUTSIDE_RETENTION = 'FLOW_RUN_RETRY_OUTSIDE_RETENTION',INVALID_API_KEY = 'INVALID_API_KEY',INVALID_APP_CONNECTION = 'INVALID_APP_CONNECTION',diff --git a/packages/server/api/src/app/flows/flow-run/flow-run-service.ts b/packages/server/api/src/app/flows/flow-run/flow-run-service.tsindex 5d54475e915..15cca2381fd 100644--- a/packages/server/api/src/app/flows/flow-run/flow-run-service.ts+++ b/packages/server/api/src/app/flows/flow-run/flow-run-service.ts@@ -1,6 +1,6 @@-import { ActivepiecesError, apId, ApId, Cursor, ErrorCode, FlowId, FlowRunId, FlowVersionId, isNil, PlatformId, ProjectId, SeekPage } from '@activepieces/core-utils'+import { ActivepiecesError, apId, ApId, Cursor, ErrorCode, FlowId, FlowRunId, FlowVersionId, isEmpty, isNil, PlatformId, ProjectId, SeekPage } from '@activepieces/core-utils'import { apDayjs, apDayjsDuration, wideEvent } from '@activepieces/server-utils'-import { ExecuteFlowJobData, ExecutionType, ExecutioOutputFile, FileCompression, FileType, FlowRetryStrategy, FlowRun, FlowRunCountByStatus, FlowRunStatus, FlowRunWithRetryError, flowStructureUtil, FlowVersion, GenericStepOutput, isFlowRunStateTerminal, JobPayload, LATEST_JOB_DATA_SCHEMA_VERSION, logSerializer, LogSliceRef, ResumeReason, RunEnvironment, RunInternalError, SampleDataFileType, StepOutput, StepOutputStatus, StepOutputType, StreamStepProgress, WorkerJobType } from '@activepieces/shared'+import { ExecuteFlowJobData, ExecutionType, ExecutioOutputFile, FAILED_STATES, FileCompression, FileType, FlowRetryStrategy, FlowRun, FlowRunCountByStatus, FlowRunStatus, FlowRunWithRetryError, flowStructureUtil, FlowVersion, GenericStepOutput, isFlowRunStateTerminal, JobPayload, LATEST_JOB_DATA_SCHEMA_VERSION, logSerializer, LogSliceRef, ResumeReason, RunEnvironment, RunInternalError, SampleDataFileType, StepOutput, StepOutputStatus, StepOutputType, StreamStepProgress, WorkerJobType } from '@activepieces/shared'import { FastifyBaseLogger } from 'fastify'import pLimit from 'p-limit'import { ArrayContains, In, IsNull, Not, Repository, SelectQueryBuilder } from 'typeorm'@@ -26,6 +26,7 @@ import { flowRunSideEffects } from './flow-run-side-effects'import { runsMetadataQueue } from './flow-runs-queue'const CANCELLABLE_STATUSES: FlowRunStatus[] = [FlowRunStatus.PAUSED, FlowRunStatus.QUEUED]+const RETRYABLE_FROM_FAILED_STEP_STATUSES: FlowRunStatus[] = [...FAILED_STATES, FlowRunStatus.QUEUED, FlowRunStatus.RUNNING]const PARENT_BOUND_RETRY_BACKOFF_MS = apDayjsDuration(30, 'second').asMilliseconds()@@ -169,14 +170,29 @@ export const flowRunService = (log: FastifyBaseLogger) => ({? await resolveStepOutput({ step: triggerStep, flowRun: oldFlowRun, log }): undefined- await flowRunRepo().update({- id: oldFlowRun.id,- projectId: oldFlowRun.projectId,- }, {- status: FlowRunStatus.QUEUED,- startTime: apDayjs().toISOString(),- finishTime: null,- })+ const { raw: requeuedRuns } = await flowRunRepo().createQueryBuilder()+ .update()+ .set({+ status: FlowRunStatus.QUEUED,+ startTime: apDayjs().toISOString(),+ finishTime: null,+ })+ .where({+ id: oldFlowRun.id,+ projectId: oldFlowRun.projectId,+ status: In(RETRYABLE_FROM_FAILED_STEP_STATUSES),+ })+ .returning('id')+ .execute()+ if (requeuedRuns.length === 0) {+ throw new ActivepiecesError({+ code: ErrorCode.FLOW_RUN_NOT_RETRYABLE,+ params: {+ flowRunId: oldFlowRun.id,+ status: oldFlowRun.status,+ },+ })+ }const updatedFlowRun = await findFlowRunOrThrow(oldFlowRun.id)const platformId = await projectService(log).getPlatformId(updatedFlowRun.projectId)await flowRunSideEffects(log).onRetry({ flowRun: updatedFlowRun, platformId })@@ -266,7 +282,12 @@ export const flowRunService = (log: FastifyBaseLogger) => ({})},async bulkRetry(params: BulkRetryParams): Promise<FlowRunWithRetryError[]> {- const filteredFlowRuns = await filterFlowRunsAndApplyFilters(params)+ const isFromFailedStepByFilter = params.strategy === FlowRetryStrategy.FROM_FAILED_STEP && isEmpty(params.flowRunIds)+ const statusFilter = isFromFailedStepByFilter ? restrictToFailedStatuses(params.status) : params.status+ if (isFromFailedStepByFilter && isEmpty(statusFilter)) {+ return []+ }+ const filteredFlowRuns = await filterFlowRunsAndApplyFilters({ ...params, status: statusFilter })const limit = pLimit(10)const results = await Promise.allSettled(filteredFlowRuns.map(flowRun =>@@ -573,6 +594,13 @@ async function getAllChildRuns(parentRunIds: string[]): Promise<FlowRun[]> {}+function restrictToFailedStatuses(statusFilter: FlowRunStatus[] | undefined): FlowRunStatus[] {+ if (isNil(statusFilter) || statusFilter.length === 0) {+ return FAILED_STATES+ }+ return statusFilter.filter((status) => FAILED_STATES.includes(status))+}+async function filterFlowRunsAndApplyFilters(params: FilterFlowRunsAndApplyFiltersParams,): Promise<FlowRun[]> {diff --git a/packages/server/api/src/app/helper/error-handler.ts b/packages/server/api/src/app/helper/error-handler.tsindex af40a2b1776..bd68c9ecc1d 100644--- a/packages/server/api/src/app/helper/error-handler.ts+++ b/packages/server/api/src/app/helper/error-handler.ts@@ -126,6 +126,7 @@ const statusCodeMap: Partial<Record<ErrorCode, StatusCodes>> = {[ErrorCode.MCP_PIECE_REQUIRES_CONNECTION]: StatusCodes.BAD_REQUEST,[ErrorCode.MCP_PIECE_CONNECTION_MISMATCH]: StatusCodes.BAD_REQUEST,[ErrorCode.DOES_NOT_MEET_BUSINESS_REQUIREMENTS]: StatusCodes.UNPROCESSABLE_ENTITY,+ [ErrorCode.FLOW_RUN_NOT_RETRYABLE]: StatusCodes.CONFLICT,[ErrorCode.FLOW_RUN_RETRY_OUTSIDE_RETENTION]: StatusCodes.GONE,[ErrorCode.SANDBOX_CAPACITY_EXCEEDED]: StatusCodes.TOO_MANY_REQUESTS,[ErrorCode.CHAT_CONTEXT_LIMIT_EXCEEDED]: StatusCodes.BAD_REQUEST,diff --git a/packages/server/api/test/integration/ce/flows/flow-run/retry-from-failed-step-status.test.ts b/packages/server/api/test/integration/ce/flows/flow-run/retry-from-failed-step-status.test.tsnew file mode 100644index 00000000000..db49d23a3b6--- /dev/null+++ b/packages/server/api/test/integration/ce/flows/flow-run/retry-from-failed-step-status.test.ts@@ -0,0 +1,153 @@+import { ErrorCode } from '@activepieces/core-utils'+import { FAILED_STATES, FlowRetryStrategy, FlowRun, FlowRunStatus, FlowRunWithRetryError, FlowVersionState, RunEnvironment } from '@activepieces/shared'+import { FastifyInstance } from 'fastify'+import { db } from '../../../../helpers/db'+import { createMockFlow, createMockFlowRun, createMockFlowVersion } from '../../../../helpers/mocks'+import { createTestContext, TestContext } from '../../../../helpers/test-context'+import { setupTestEnvironment, teardownTestEnvironment } from '../../../../helpers/test-setup'++let app: FastifyInstance+let ctx: TestContext++beforeAll(async () => {+ app = await setupTestEnvironment()+})++afterAll(async () => {+ await teardownTestEnvironment()+})++beforeEach(async () => {+ ctx = await createTestContext(app)+})++describe('Retry from failed step (POST /v1/flow-runs/:id/retry)', () => {+ it.each([+ FlowRunStatus.SUCCEEDED,+ FlowRunStatus.PAUSED,+ FlowRunStatus.CANCELED,+ FlowRunStatus.LOG_SIZE_EXCEEDED,+ ])('refuses a %s run and leaves it unchanged', async (status) => {+ const flowRun = await createRunWithStatus(status)+ const before = await readRun(flowRun.id)++ const response = await ctx.post(`/v1/flow-runs/${flowRun.id}/retry`, {+ strategy: FlowRetryStrategy.FROM_FAILED_STEP,+ projectId: ctx.project.id,+ })++ expect(response.statusCode).toBe(409)+ expect(response.json().code).toBe(ErrorCode.FLOW_RUN_NOT_RETRYABLE)+ const after = await readRun(flowRun.id)+ expect(after.status).toBe(status)+ expect(after.startTime).toEqual(before.startTime)+ })++ it.each([+ ...FAILED_STATES,+ FlowRunStatus.QUEUED,+ FlowRunStatus.RUNNING,+ ])('accepts a %s run and queues it', async (status) => {+ const flowRun = await createRunWithStatus(status)++ const response = await ctx.post(`/v1/flow-runs/${flowRun.id}/retry`, {+ strategy: FlowRetryStrategy.FROM_FAILED_STEP,+ projectId: ctx.project.id,+ })++ expect(response.statusCode).toBe(200)+ const after = await readRun(flowRun.id)+ expect(after.status).toBe(FlowRunStatus.QUEUED)+ })++ it('retry from failed step by filter retries only failed runs', async () => {+ const failedRun = await createRunWithStatus(FlowRunStatus.FAILED)+ const otherRuns = await Promise.all([+ FlowRunStatus.QUEUED,+ FlowRunStatus.RUNNING,+ FlowRunStatus.PAUSED,+ FlowRunStatus.SUCCEEDED,+ ].map((status) => createRunWithStatus(status)))+ const otherRunsBefore = await Promise.all(otherRuns.map((run) => readRun(run.id)))++ const response = await ctx.post('/v1/flow-runs/retry', {+ projectId: ctx.project.id,+ strategy: FlowRetryStrategy.FROM_FAILED_STEP,+ })++ expect(response.statusCode).toBe(200)+ const results: FlowRunWithRetryError[] = response.json()+ expect(results.map((run) => run.id)).toEqual([failedRun.id])+ expect((await readRun(failedRun.id)).status).toBe(FlowRunStatus.QUEUED)+ const otherRunsAfter = await Promise.all(otherRuns.map((run) => readRun(run.id)))+ expect(otherRunsAfter).toEqual(otherRunsBefore)+ })++ it('retry from failed step by filter keeps only the failed statuses of the status filter', async () => {+ const failedRun = await createRunWithStatus(FlowRunStatus.FAILED)+ const runningRun = await createRunWithStatus(FlowRunStatus.RUNNING)+ const runningBefore = await readRun(runningRun.id)++ const response = await ctx.post('/v1/flow-runs/retry', {+ projectId: ctx.project.id,+ strategy: FlowRetryStrategy.FROM_FAILED_STEP,+ status: [FlowRunStatus.FAILED, FlowRunStatus.RUNNING],+ })++ expect(response.statusCode).toBe(200)+ const results: FlowRunWithRetryError[] = response.json()+ expect(results.map((run) => run.id)).toEqual([failedRun.id])+ expect(await readRun(runningRun.id)).toEqual(runningBefore)+ })++ it('retry from failed step by filter retries nothing when the status filter has no failed status', async () => {+ const runningRun = await createRunWithStatus(FlowRunStatus.RUNNING)+ const runningBefore = await readRun(runningRun.id)++ const response = await ctx.post('/v1/flow-runs/retry', {+ projectId: ctx.project.id,+ strategy: FlowRetryStrategy.FROM_FAILED_STEP,+ status: [FlowRunStatus.RUNNING],+ })++ expect(response.statusCode).toBe(200)+ expect(response.json()).toEqual([])+ expect(await readRun(runningRun.id)).toEqual(runningBefore)+ })++ it('retry from failed step by run ids still accepts a RUNNING run', async () => {+ const runningRun = await createRunWithStatus(FlowRunStatus.RUNNING)++ const response = await ctx.post('/v1/flow-runs/retry', {+ projectId: ctx.project.id,+ strategy: FlowRetryStrategy.FROM_FAILED_STEP,+ flowRunIds: [runningRun.id],+ })++ expect(response.statusCode).toBe(200)+ const results: FlowRunWithRetryError[] = response.json()+ expect(results.map((run) => ({ id: run.id, error: run.error }))).toEqual([{ id: runningRun.id, error: undefined }])+ expect((await readRun(runningRun.id)).status).toBe(FlowRunStatus.QUEUED)+ })+})++async function createRunWithStatus(status: FlowRunStatus): Promise<FlowRun> {+ const flow = createMockFlow({ projectId: ctx.project.id })+ await db.save('flow', flow)+ const flowVersion = createMockFlowVersion({ flowId: flow.id, state: FlowVersionState.LOCKED })+ await db.save('flow_version', flowVersion)+ const flowRun = createMockFlowRun({+ projectId: ctx.project.id,+ flowId: flow.id,+ flowVersionId: flowVersion.id,+ status,+ environment: RunEnvironment.PRODUCTION,+ created: new Date().toISOString(),+ })+ await db.save('flow_run', flowRun)+ return flowRun+}++async function readRun(id: string): Promise<Pick<FlowRun, 'status' | 'startTime'>> {+ return db.findOneByOrFail<Pick<FlowRun, 'status' | 'startTime'>>('flow_run', { id })+}diff --git a/packages/web/src/features/flow-runs/components/runs-table/failed-retry-runs-dialog.tsx b/packages/web/src/features/flow-runs/components/runs-table/failed-retry-runs-dialog.tsxindex ac09725cd19..27eff426600 100644--- a/packages/web/src/features/flow-runs/components/runs-table/failed-retry-runs-dialog.tsx+++ b/packages/web/src/features/flow-runs/components/runs-table/failed-retry-runs-dialog.tsx@@ -66,15 +66,7 @@ export const FailedRetryRunsDialog = ({</span></div><p className="text-xs text-gray-11">- {run.error?.errorCode ===- ErrorCode.FLOW_RUN_RETRY_OUTSIDE_RETENTION- ? t(- 'Retry is only available for {failedJobRetentionDays} after a run fails.',- {- failedJobRetentionDays: retentionDays,- },- )- : run.error.errorMessage ?? t('Internal server error')}+ {retryErrorMessage({ run, retentionDays })}</p></div><Button@@ -101,3 +93,27 @@ export const FailedRetryRunsDialog = ({</Dialog>);};++function retryErrorMessage({+ run,+ retentionDays,+}: RetryErrorMessageParams): string {+ switch (run.error.errorCode) {+ case ErrorCode.FLOW_RUN_RETRY_OUTSIDE_RETENTION:+ return t(+ 'Retry is only available for {failedJobRetentionDays} after a run fails.',+ {+ failedJobRetentionDays: retentionDays,+ },+ );+ case ErrorCode.FLOW_RUN_NOT_RETRYABLE:+ return t('Only failed runs can be retried from failed step');+ default:+ return run.error.errorMessage ?? t('Internal server error');+ }+}++type RetryErrorMessageParams = {+ run: Required<FlowRunWithRetryError>;+ retentionDays: number | null | undefined;+};diff --git a/packages/web/src/features/flow-runs/hooks/flow-run-hooks.ts b/packages/web/src/features/flow-runs/hooks/flow-run-hooks.tsindex d5815ce5bab..bab38319f9c 100644--- a/packages/web/src/features/flow-runs/hooks/flow-run-hooks.ts+++ b/packages/web/src/features/flow-runs/hooks/flow-run-hooks.ts@@ -159,6 +159,15 @@ export const flowRunMutations = {closeButton: true,dismissible: true,});+ } else if (apError.code === ErrorCode.FLOW_RUN_NOT_RETRYABLE) {+ toast.error(t('Retry failed'), {+ description: t(+ 'Only failed runs can be retried from failed step',+ ),+ duration: 5000,+ closeButton: true,+ dismissible: true,+ });} else if (apError.code === ErrorCode.QUOTA_EXCEEDED) {billingDialogHooks.useManagePlanDialogStore.getState().openDialog();}