Skip to content

AI Agents & MCPs & AI Workflow Automation • (~400 MCP servers for AI agents) • AI Automation / AI Agent with MCPs • AI Workflows & AI Agents • MCPs for AI Agents

37.7 KB
Raw Blame
import { ActivepiecesError, apId, ApId, Cursor, ErrorCode, FlowId, FlowRunId, FlowVersionId, isEmpty, isNil, PlatformId, ProjectId, SeekPage } from '@activepieces/core-utils'
import { apDayjs, apDayjsDuration, wideEvent } from '@activepieces/server-utils'
import { ExecuteFlowJobData, ExecutionType, ExecutioOutputFile, FAILED_STATES, FileCompression, FileType, FlowRetryStrategy, FlowRun, FlowRunCountByStatus, FlowRunStatus, FlowRunWithRetryError, flowStructureUtil, FlowVersion, GenericStepOutput, isFlowRunStateTerminal, JobPayload, LATEST_JOB_DATA_SCHEMA_VERSION, logSerializer, LogSliceRef, ResumeReason, RunEnvironment, RunInternalError, SampleDataFileType, StepOutput, StepOutputStatus, StepOutputType, StreamStepProgress, WorkerJobType } from '@activepieces/shared'
import { FastifyBaseLogger } from 'fastify'
import pLimit from 'p-limit'
import { ArrayContains, In, IsNull, Not, Repository, SelectQueryBuilder } from 'typeorm'
import { repoFactory } from '../../core/db/repo-factory'
import { distributedLock } from '../../database/redis-connections'
import { fileCompressor } from '../../file/file-compressor'
import { fileService, getEffectiveExecutionDataRetentionDays } from '../../file/file.service'
import { buildPaginator } from '../../helper/pagination/build-paginator'
import { paginationHelper } from '../../helper/pagination/pagination-utils'
import { Order } from '../../helper/pagination/paginator'
import { system } from '../../helper/system/system'
import { AppSystemProp } from '../../helper/system/system-props'
import { assertRunCreditsNotExceeded, CreditCheckSpeed, shouldBlockRunOnCredits } from '../../platform/billing-provider'
import { projectService } from '../../project/project-service'
import { waitpointService } from '../../waitpoints/waitpoint-service'
import { jobQueue, JobType } from '../../workers/job-queue/job-queue'
import { payloadOffloader } from '../../workers/payload-offloader'
import { flowService } from '../flow/flow.service'
import { flowVersionService } from '../flow-version/flow-version.service'
import { sampleDataService } from '../step-run/sample-data.service'
import { FlowRunEntity } from './flow-run-entity'
import { flowRunSideEffects } from './flow-run-side-effects'
import { runsMetadataQueue } from './flow-runs-queue'

const CANCELLABLE_STATUSES: FlowRunStatus[] = [FlowRunStatus.PAUSED, FlowRunStatus.QUEUED]
const RETRYABLE_FROM_FAILED_STEP_STATUSES: FlowRunStatus[] = [...FAILED_STATES, FlowRunStatus.QUEUED, FlowRunStatus.RUNNING]
const PARENT_BOUND_RETRY_BACKOFF_MS = apDayjsDuration(30, 'second').asMilliseconds()


export const WEBHOOK_TIMEOUT_MS = system.getNumberOrThrow(AppSystemProp.WEBHOOK_TIMEOUT_SECONDS) * 1000
export const flowRunRepo = repoFactory<FlowRun>(FlowRunEntity)

export const flowRunService = (log: FastifyBaseLogger) => ({
    async upsert({ id, projectId }: { id: FlowRunId, projectId: ProjectId }): Promise<FlowRun> {
        const existingFlowRun = await flowRunRepo().findOneBy({ id, projectId })
        if (isNil(existingFlowRun)) {
            return flowRunRepo().save({ id, projectId })
        }
        return existingFlowRun
    },
    async list(params: ListParams): Promise<SeekPage<FlowRun>> {
        const decodedCursor = paginationHelper.decodeCursor(params.cursor)
        const paginator = buildPaginator<FlowRun>({
            entity: FlowRunEntity,
            query: {
                limit: params.limit,
                orderBy: isNil(params.parentWaitpointId)
                    ? [
                        { field: 'created', order: Order.DESC },
                        { field: 'id', order: Order.DESC },
                    ]
                    : [
                        { field: 'dispatchIndex', order: Order.ASC },
                        { field: 'id', order: Order.ASC },
                    ],
                afterCursor: decodedCursor.nextCursor,
                beforeCursor: decodedCursor.previousCursor,
            },
        })


        const whereClause: Record<string, unknown> = {
            projectId: params.projectId,
        }
        if (!isNil(params.environment)) {
            whereClause.environment = params.environment
        }
        let query = queryBuilderForFlowRun(flowRunRepo())
            .where(whereClause)
            .andWhere({ parentWaitpointId: params.parentWaitpointId ?? IsNull() })

        if (!isNil(params.parentWaitpointId)) {
            query = query.andWhere({ dispatchIndex: Not(IsNull()) })
        }

        if (!params.includeArchived) {
            query = query.andWhere({
                archivedAt: IsNull(),
            })
        }

        if (params.flowId) {
            query = query.andWhere({
                flowId: In(params.flowId),
            })
        }
        if (params.status) {
            query = query.andWhere({
                status: In(params.status),
            })
        }
        if (params.createdAfter) {
            query = query.andWhere('flow_run.created >= :createdAfter', {
                createdAfter: params.createdAfter,
            })
        }
        if (params.createdBefore) {
            query = query.andWhere('flow_run.created <= :createdBefore', {
                createdBefore: params.createdBefore,
            })
        }
        if (params.tags) {
            query = query.andWhere({ tags: ArrayContains(params.tags) })
        }

        if (!isNil(params.failedStepName)) {
            query = query.andWhere('flow_run."failedStep"->>\'name\' = :failedStepName', {
                failedStepName: params.failedStepName,
            })
        }
        if (!isNil(params.failedStepMessage)) {
            query = query.andWhere('flow_run."failedStep"->>\'message\' ILIKE :failedStepMessage', {
                failedStepMessage: `%${params.failedStepMessage}%`,
            })
        }
        if (params.flowRunIds) {
            query = query.andWhere({
                id: In(params.flowRunIds),
            })
        }

        const { data, cursor: newCursor } = await paginator.paginate(query)
        return paginationHelper.createPage<FlowRun>(data, newCursor)
    },
    async retry({ flowRunId, strategy, projectId }: RetryParams): Promise<FlowRun> {
        const oldFlowRun = await flowRunService(log).getOnePopulatedOrThrow({
            id: flowRunId,
            projectId,
        })
        log.info({ flowRun: { id: flowRunId }, flow: { id: oldFlowRun.flowId }, strategy }, 'Flow run retry initiated')

        if (!isNil(oldFlowRun.dispatchIndex) || !isNil(oldFlowRun.parentWaitpointId)) {
            const message = `The run ${flowRunId} started from a step inside its flow rather than from the trigger, so it cannot be retried. Retry the run that dispatched it instead.`
            throw new ActivepiecesError({
                code: ErrorCode.VALIDATION,
                params: { message },
            }, message)
        }

        const project = await projectService(log).getOneOrThrow(oldFlowRun.projectId)
        const retentionDays = getEffectiveExecutionDataRetentionDays(project.executionDataRetentionDays)
        if (
            isFlowRunStateTerminal({ status: oldFlowRun.status, ignoreInternalError: false }) &&
            isOutsideRetentionWindow(oldFlowRun.created, retentionDays)
        ) {
            throw new ActivepiecesError({
                code: ErrorCode.FLOW_RUN_RETRY_OUTSIDE_RETENTION,
                params: {
                    flowRunId: oldFlowRun.id,
                    failedJobRetentionDays: retentionDays,
                },
            })
        }

        await assertRunCreditsNotExceeded({
            platformId: project.platformId,
            environment: oldFlowRun.environment,
            creditCheckSpeed: CreditCheckSpeed.NORMAL,
            log,
        })

        switch (strategy) {
            case FlowRetryStrategy.FROM_FAILED_STEP: {
                const flowVersion = await flowVersionService(log).getOneOrThrow(oldFlowRun.flowVersionId)
                const triggerStep = oldFlowRun.steps?.[flowVersion.trigger.name]
                const triggerFailed = triggerStep?.status === StepOutputStatus.FAILED
                const triggerPayload = triggerFailed
                    ? await resolveStepOutput({ step: triggerStep, flowRun: oldFlowRun, log })
                    : undefined

                const { raw: requeuedRuns } = await flowRunRepo().createQueryBuilder()
                    .update()
                    .set({
                        status: FlowRunStatus.QUEUED,
                        startTime: apDayjs().toISOString(),
                        finishTime: null,
                    })
                    .where({
                        id: oldFlowRun.id,
                        projectId: oldFlowRun.projectId,
                        status: In(RETRYABLE_FROM_FAILED_STEP_STATUSES),
                    })
                    .returning('id')
                    .execute()
                if (requeuedRuns.length === 0) {
                    throw new ActivepiecesError({
                        code: ErrorCode.FLOW_RUN_NOT_RETRYABLE,
                        params: {
                            flowRunId: oldFlowRun.id,
                            status: oldFlowRun.status,
                        },
                    })
                }
                const updatedFlowRun = await findFlowRunOrThrow(oldFlowRun.id)
                const platformId = await projectService(log).getPlatformId(updatedFlowRun.projectId)
                await flowRunSideEffects(log).onRetry({ flowRun: updatedFlowRun, platformId })
                if (triggerFailed) {
                    return addToQueue({
                        flowRun: updatedFlowRun,
                        platformId,
                        payload: triggerPayload,
                        streamStepProgress: StreamStepProgress.NONE,
                        executeTrigger: true,
                        executionType: ExecutionType.BEGIN,
                        workerHandlerId: undefined,
                        httpRequestId: undefined,
                    }, log)
                }
                return addToQueue({
                    flowRun: updatedFlowRun,
                    platformId,
                    streamStepProgress: StreamStepProgress.NONE,
                    executionType: ExecutionType.RESUME,
                    resumeReason: ResumeReason.RETRY,
                    workerHandlerId: undefined,
                    httpRequestId: undefined,
                }, log)
            }
            case FlowRetryStrategy.ON_LATEST_VERSION: {
                const latestFlowVersion = await flowVersionService(log).getLatestLockedVersionOrThrow(
                    oldFlowRun.flowId,
                )
                const triggerStep = oldFlowRun.steps?.[latestFlowVersion.trigger.name]
                const triggerFailed = triggerStep?.status === StepOutputStatus.FAILED
                const payload = await resolveStepOutput({ step: triggerStep, flowRun: oldFlowRun, log })
                return this.start({
                    flowId: oldFlowRun.flowId,
                    payload,
                    platformId: await projectService(log).getPlatformId(oldFlowRun.projectId),
                    executionType: ExecutionType.BEGIN,
                    streamStepProgress: StreamStepProgress.NONE,
                    workerHandlerId: undefined,
                    httpRequestId: undefined,
                    executeTrigger: triggerFailed,
                    environment: oldFlowRun.environment,
                    flowVersionId: latestFlowVersion.id,
                    projectId: oldFlowRun.projectId,
                    failParentOnFailure: oldFlowRun.failParentOnFailure,
                    parentRunId: oldFlowRun.parentRunId,
                })
            }
        }
    },
    async cancel({ projectId, platformId, flowRunIds, excludeFlowRunIds, status, flowId, createdAfter, createdBefore }: CancelParams): Promise<void> {
        const filteredStatus = status ?? CANCELLABLE_STATUSES
        const flowRuns = await filterFlowRunsAndApplyFilters({
            projectId,
            flowRunIds,
            status: filteredStatus,
            flowId,
            createdAfter,
            createdBefore,
            excludeFlowRunIds,
        })
        const cancelParentFlowRuns = await Promise.allSettled(flowRuns.map(flowRun => cancelSingleRun(log, flowRun, platformId)))
        const childFlows = await getAllChildRuns(flowRuns.map(flowRun => flowRun.id))
        log.info({
            flowRunsCount: flowRuns.length,
            childFlowCount: childFlows.length,
        }, 'Found cancellable descendant flows')

        const canceChildlPromises = await Promise.allSettled(childFlows.map(flowRun => cancelSingleRun(log, flowRun, platformId)))
        if (cancelParentFlowRuns.some(r => r.status === 'rejected')) {
            throw cancelParentFlowRuns.find(r => r.status === 'rejected')!.reason
        }
        if (canceChildlPromises.some(r => r.status === 'rejected')) {
            throw canceChildlPromises.find(r => r.status === 'rejected')!.reason
        }
    },
    async existsBy(runId: FlowRunId): Promise<boolean> {
        return flowRunRepo().existsBy({ id: runId })
    },
    async bulkArchive(params: BulkArchiveActionParams): Promise<void> {
        const filteredFlowRuns = await filterFlowRunsAndApplyFilters(params)
        await flowRunRepo().update({
            id: In(filteredFlowRuns.map(flowRun => flowRun.id)),
            projectId: params.projectId,
        }, {
            archivedAt: new Date().toISOString(),
        })
    },
    async bulkRetry(params: BulkRetryParams): Promise<FlowRunWithRetryError[]> {
        const isFromFailedStepByFilter = params.strategy === FlowRetryStrategy.FROM_FAILED_STEP && isEmpty(params.flowRunIds)
        const statusFilter = isFromFailedStepByFilter ? restrictToFailedStatuses(params.status) : params.status
        if (isFromFailedStepByFilter && isEmpty(statusFilter)) {
            return []
        }
        const filteredFlowRuns = await filterFlowRunsAndApplyFilters({ ...params, status: statusFilter })
        const limit = pLimit(10)
        const results = await Promise.allSettled(
            filteredFlowRuns.map(flowRun =>
                limit(() => this.retry({ flowRunId: flowRun.id, strategy: params.strategy, projectId: params.projectId })),
            ),
        )
        return results.map((result, i) => {
            if (result.status === 'fulfilled') {
                return result.value
            }
            const error = result.reason instanceof ActivepiecesError ? result.reason : undefined
            return {
                ...filteredFlowRuns[i],
                error: {
                    errorCode: error?.error.code ?? ErrorCode.INTERNAL_SERVER_ERROR,
                    errorMessage: error?.message ?? 'Internal server error',
                },
            }
        })
    },
    async start({
        flowId,
        payload,
        executeTrigger,
        executionType,
        workerHandlerId,
        streamStepProgress,
        httpRequestId,
        projectId,
        flowVersionId,
        parentRunId,
        failParentOnFailure,
        platformId,
        stepNameToTest,
        environment,
    }: StartParams): Promise<FlowRun> {
        const newFlowRun = await queueOrCreateInstantly({
            projectId,
            flowVersionId,
            parentRunId,
            flowId,
            failParentOnFailure,
            stepNameToTest,
            environment,
        }, log)

        wideEvent.set({
            flowRun: {
                id: newFlowRun.id,
                environment,
                executionType,
            },
            flow: { id: flowId },
        })

        await addToQueue({
            flowRun: newFlowRun,
            platformId,
            payload,
            executeTrigger,
            executionType,
            workerHandlerId,
            httpRequestId,
            streamStepProgress,
        }, log)

        await flowRunSideEffects(log).onStart({ flowRun: newFlowRun, platformId })
        log.info({ flowRun: { id: newFlowRun.id }, flow: { id: flowId }, project: { id: projectId }, executionType }, 'Flow run started')
        return newFlowRun
    },

    async createQuotaExceededRun({ flowVersion, payload, projectId, environment, parentRunId, failParentOnFailure, triggeredBy, shouldExecuteTriggerOnRetry }: CreateQuotaExceededRunParams): Promise<FlowRun> {
        const now = new Date().toISOString()
        const logsFileId = apId()
        await persistQuotaExceededTriggerLog({ log, flowVersion, projectId, payload, logsFileId, shouldExecuteTriggerOnRetry })
        const flowRun: FlowRun = {
            id: apId(),
            projectId,
            flowId: flowVersion.flowId,
            flowVersionId: flowVersion.id,
            environment,
            parentRunId,
            failParentOnFailure: failParentOnFailure ?? true,
            status: FlowRunStatus.QUOTA_EXCEEDED,
            created: now,
            updated: now,
            startTime: now,
            finishTime: now,
            logsFileId,
            tags: [],
            steps: {},
            triggeredBy,
        }
        await runsMetadataQueue(log).add(flowRun)
        log.info({ flowRun: { id: flowRun.id }, flow: { id: flowVersion.flowId }, project: { id: projectId } }, 'Flow run admitted as QUOTA_EXCEEDED')
        return flowRun
    },

    async test({ projectId, flowVersionId, parentRunId, stepNameToTest, triggeredBy }: TestParams): Promise<FlowRun> {
        const flowVersion = await flowVersionService(log).getOneOrThrow(flowVersionId)
        await flowService(log).getOneOrThrow({ id: flowVersion.flowId, projectId })

        const triggerPayload = await sampleDataService(log).getOrReturnEmpty({
            projectId,
            flowVersion,
            stepName: flowVersion.trigger.name,
            type: SampleDataFileType.OUTPUT,
        })
        const flowRun = await queueOrCreateInstantly({
            projectId,
            flowId: flowVersion.flowId,
            flowVersionId: flowVersion.id,
            environment: RunEnvironment.TESTING,
            parentRunId,
            failParentOnFailure: undefined,
            stepNameToTest,
            triggeredBy,
        }, log)
        return addToQueue({
            flowRun,
            payload: triggerPayload,
            executionType: ExecutionType.BEGIN,
            workerHandlerId: undefined,
            httpRequestId: undefined,
            platformId: await projectService(log).getPlatformId(projectId),
            executeTrigger: false,
            streamStepProgress: StreamStepProgress.WEBSOCKET,
            sampleData: !isNil(stepNameToTest) ? await sampleDataService(log).getSampleDataForFlow({
                projectId,
                flowVersion,
                type: SampleDataFileType.OUTPUT,
                referencedBy: settingsOfStepUnderTest({ flowVersion, stepNameToTest }),
            }) : undefined,
        }, log)
    },
    async startManualTrigger({ projectId, flowVersionId, triggeredBy }: StartManualTriggerParams): Promise<FlowRun> {
        const flowVersion = await flowVersionService(log).getOneOrThrow(flowVersionId)
        await flowService(log).getOneOrThrow({ id: flowVersion.flowId, projectId })
        const triggerPayload = {}
        const platformId = await projectService(log).getPlatformId(projectId)

        const creditsExhausted = await shouldBlockRunOnCredits({
            platformId,
            environment: RunEnvironment.PRODUCTION,
            creditCheckSpeed: CreditCheckSpeed.NORMAL,
            log,
        })
        if (creditsExhausted) {
            return this.createQuotaExceededRun({
                flowVersion,
                payload: triggerPayload,
                projectId,
                environment: RunEnvironment.PRODUCTION,
                parentRunId: undefined,
                failParentOnFailure: undefined,
                triggeredBy,
                shouldExecuteTriggerOnRetry: false,
            })
        }

        const flowRun = await queueOrCreateInstantly({
            projectId,
            flowId: flowVersion.flowId,
            flowVersionId: flowVersion.id,
            environment: RunEnvironment.PRODUCTION,
            parentRunId: undefined,
            failParentOnFailure: undefined,
            stepNameToTest: undefined,
            triggeredBy,
        }, log)
        return addToQueue({
            flowRun,
            payload: triggerPayload,
            executionType: ExecutionType.BEGIN,
            workerHandlerId: undefined,
            httpRequestId: undefined,
            platformId,
            executeTrigger: false,
            streamStepProgress: StreamStepProgress.WEBSOCKET,
            sampleData: undefined,
        }, log)
    },
    async getOne(params: GetOneParams): Promise<FlowRun | null> {
        const flowRun = await queryBuilderForFlowRun(flowRunRepo()).where({
            id: params.id,
            ...(params.projectId ? { projectId: params.projectId } : {}),
        }).getOne()

        return flowRun
    },
    async getOneOrThrow(params: GetOneParams): Promise<FlowRun> {
        const flowRun = await this.getOne(params)

        if (isNil(flowRun)) {
            throw new ActivepiecesError({
                code: ErrorCode.ENTITY_NOT_FOUND,
                params: {
                    entityType: 'flow_run',
                    entityId: params.id,
                    message: 'Flow run not found',
                },
            })
        }

        return flowRun
    },
    async getStepsOrNull({ flowRun }: { flowRun: FlowRun }): Promise<Record<string, StepOutput> | null> {
        if (isNil(flowRun.logsFileId)) {
            return null
        }
        const stateFile = await readLogsFile(log, flowRun.logsFileId, flowRun.projectId)
        return stateFile?.executionState.steps ?? null
    },
    async countByStatus(params: CountByStatusParams): Promise<FlowRunCountByStatus[]> {
        let query = flowRunRepo().createQueryBuilder('flow_run')
            .select('flow_run.status', 'status')
            .addSelect('COUNT(*)', 'count')
            .where({
                projectId: params.projectId,
                environment: RunEnvironment.PRODUCTION,
                archivedAt: IsNull(),
                parentWaitpointId: IsNull(),
            })
            .groupBy('flow_run.status')

        if (params.createdAfter) {
            query = query.andWhere('flow_run.created >= :createdAfter', { createdAfter: params.createdAfter })
        }
        if (params.createdBefore) {
            query = query.andWhere('flow_run.created <= :createdBefore', { createdBefore: params.createdBefore })
        }

        const results = await query.getRawMany()
        return results.map((r: { status: FlowRunStatus, count: string }) => ({ status: r.status, count: parseInt(r.count, 10) }))
    },
    async getOnePopulatedOrThrow(params: GetOneParams): Promise<FlowRun> {
        const flowRun = await this.getOneOrThrow(params)
        let steps = {}
        let internalError: RunInternalError | undefined = undefined
        if (!isNil(flowRun.logsFileId)) {
            const stateFile = await readLogsFile(log, flowRun.logsFileId, flowRun.projectId)
            if (!isNil(stateFile)) {
                steps = stateFile.executionState.steps
                internalError = stateFile.internalError
            }
        }
        return {
            ...flowRun,
            steps,
            internalError,
        }
    },
})


export async function cancelSingleRun(log: FastifyBaseLogger, flowRun: FlowRun, platformId: string): Promise<void> {
    await distributedLock(log).runExclusive({
        key: `runs_metadata_${flowRun.id}`,
        timeoutInSeconds: 30,
        fn: async () => {
            await jobQueue(log).removeAllFlowRunJobs({ flowRunId: flowRun.id, platformId, projectId: flowRun.projectId, flowId: flowRun.flowId })
            await waitpointService(log).deleteByFlowRunId({ flowRunId: flowRun.id, projectId: flowRun.projectId })
            await runsMetadataQueue(log).add({
                id: flowRun.id,
                projectId: flowRun.projectId,
                status: FlowRunStatus.CANCELED,
            })
        },
    })
    log.info({
        flowRun: { id: flowRun.id },
        flow: { id: flowRun.flowId },
    }, 'Flow run cancelled')
}

async function getAllChildRuns(parentRunIds: string[]): Promise<FlowRun[]> {
    if (parentRunIds.length === 0) {
        return []
    }

    const query = `
        WITH RECURSIVE descendants AS (
            SELECT *
            FROM flow_run
            WHERE "parentRunId" = ANY($1)
              AND status = ANY($2)

            UNION ALL

            SELECT f.*
            FROM flow_run f
            INNER JOIN descendants d ON f."parentRunId" = d.id
            WHERE f.status = ANY($2)
        )
        SELECT * FROM descendants;
    `

    const params = [
        parentRunIds,
        CANCELLABLE_STATUSES,
    ]

    const results = await flowRunRepo().query(query, params)
    return results as FlowRun[]
}


function restrictToFailedStatuses(statusFilter: FlowRunStatus[] | undefined): FlowRunStatus[] {
    if (isNil(statusFilter) || statusFilter.length === 0) {
        return FAILED_STATES
    }
    return statusFilter.filter((status) => FAILED_STATES.includes(status))
}

async function filterFlowRunsAndApplyFilters(
    params: FilterFlowRunsAndApplyFiltersParams,
): Promise<FlowRun[]> {
    const explicitFlowRunIds = params.flowRunIds ?? []
    let query = flowRunRepo().createQueryBuilder('flow_run').where({
        projectId: params.projectId,
        environment: RunEnvironment.PRODUCTION,
    })

    if (explicitFlowRunIds.length > 0) {
        query = query.andWhere({
            id: In(explicitFlowRunIds),
        })
    }
    else {
        query = query.andWhere({
            parentWaitpointId: IsNull(),
        })
    }

    if (!isNil(params.archived)) {
        query = query.andWhere({
            archivedAt: params.archived ? Not(IsNull()) : IsNull(),
        })
    }

    if (params.flowId && params.flowId.length > 0) {
        query = query.andWhere({
            flowId: In(params.flowId),
        })
    }
    if (params.status && params.status.length > 0) {
        query = query.andWhere({
            status: In(params.status),
        })
    }
    if (params.createdAfter) {
        query = query.andWhere('flow_run.created >= :createdAfter', {
            createdAfter: params.createdAfter,
        })
    }
    if (params.createdBefore) {
        query = query.andWhere('flow_run.created <= :createdBefore', {
            createdBefore: params.createdBefore,
        })
    }
    if (params.excludeFlowRunIds && params.excludeFlowRunIds.length > 0) {
        query = query.andWhere({
            id: Not(In(params.excludeFlowRunIds)),
        })
    }

    if (params.failedStepName) {
        query = query.andWhere('flow_run."failedStep"->>\'name\' = :failedStepName', {
            failedStepName: params.failedStepName,
        })
    }
    if (params.failedStepMessage) {
        query = query.andWhere('flow_run."failedStep"->>\'message\' ILIKE :failedStepMessage', {
            failedStepMessage: `%${params.failedStepMessage}%`,
        })
    }

    const flowRuns = await query.getMany()
    return flowRuns
}


export async function addToQueue(params: AddToQueueParams, log: FastifyBaseLogger): Promise<FlowRun> {
    const logsFileId = params.flowRun.logsFileId ?? apId()

    let jobPayload: JobPayload = { type: 'inline', value: null }
    if (!isNil(params.payload) && isNil(params.workerHandlerId) && !(params.preferInlinePayload ?? false)) {
        jobPayload = await payloadOffloader.offloadPayload(log, params.payload, params.flowRun.projectId, params.platformId)
    }
    else if (!isNil(params.payload)) {
        jobPayload = await payloadOffloader.maybeOffloadPayload(log, params.payload, params.flowRun.projectId, params.platformId)
    }

    const commonJobData = {
        schemaVersion: LATEST_JOB_DATA_SCHEMA_VERSION,
        workerHandlerId: params.workerHandlerId ?? null,
        projectId: params.flowRun.projectId,
        platformId: params.platformId,
        environment: params.flowRun.environment,
        flowId: params.flowRun.flowId,
        runId: params.flowRun.id,
        jobType: WorkerJobType.EXECUTE_FLOW as const,
        flowVersionId: params.flowRun.flowVersionId,
        payload: jobPayload,
        httpRequestId: params.httpRequestId,
        streamStepProgress: params.streamStepProgress,
        stepNameToTest: params.flowRun.stepNameToTest ?? undefined,
        sampleData: params.sampleData,
        logsFileId,
        parentWaitpointId: params.flowRun.parentWaitpointId ?? undefined,
    }
    const data: ExecuteFlowJobData = params.executionType === ExecutionType.RESUME
        ? {
            ...commonJobData,
            executionType: ExecutionType.RESUME,
            resumeReason: params.resumeReason,
        }
        : {
            ...commonJobData,
            executionType: ExecutionType.BEGIN,
            executeTrigger: params.executeTrigger,
            entryStepName: params.entryStepName,
        }
    const aParentIsBlockedOnThisRun = params.flowRun.failParentOnFailure && !isNil(params.flowRun.parentRunId)
    await jobQueue(log).add({
        id: params.jobId ?? params.flowRun.id,
        type: JobType.ONE_TIME,
        data,
        retryBackoffMs: aParentIsBlockedOnThisRun ? PARENT_BOUND_RETRY_BACKOFF_MS : undefined,
    })
    return params.flowRun
}

export async function findFlowRunOrThrow(flowRunId: FlowRunId): Promise<FlowRun> {
    const flowRun = await queryBuilderForFlowRun(flowRunRepo()).where({ id: flowRunId }).getOne()
    if (isNil(flowRun)) {
        throw new ActivepiecesError({
            code: ErrorCode.ENTITY_NOT_FOUND,
            params: {
                entityType: 'flow_run',
                entityId: flowRunId,
                message: 'Flow run not found',
            },
        })
    }
    return flowRun
}

function queryBuilderForFlowRun(repo: Repository<FlowRun>): SelectQueryBuilder<FlowRun> {
    return repo.createQueryBuilder('flow_run')
        .leftJoin('flow_run.flowVersion', 'flowVersion')
        .addSelect(['flowVersion.id', 'flowVersion.displayName'])
}

async function resolveStepOutput({ step, flowRun, log }: ResolveStepOutputParams): Promise<unknown> {
    if (isNil(step)) {
        return undefined
    }
    if (step.outputType !== StepOutputType.SLICE) {
        return step.output
    }
    const ref = step.output as LogSliceRef
    const file = await fileService(log).getDataOrUndefined({
        projectId: flowRun.projectId,
        fileId: ref.fileId,
        type: FileType.FLOW_RUN_LOG_SLICE,
    })
    if (isNil(file)) {
        throw new ActivepiecesError({
            code: ErrorCode.ENTITY_NOT_FOUND,
            params: {
                entityType: 'file',
                entityId: ref.fileId,
                message: `Trigger output was offloaded to storage but its slice file is missing; flow run ${flowRun.id} cannot be retried without the trigger payload`,
            },
        })
    }
    return JSON.parse(file.data.toString('utf-8'))
}

async function readLogsFile(log: FastifyBaseLogger, logsFileId: string, projectId: string): Promise<ExecutioOutputFile | null> {
    const result = await fileService(log).getDataOrUndefined({
        projectId,
        fileId: logsFileId,
        type: FileType.FLOW_RUN_LOG,
    })
    if (isNil(result)) {
        return null
    }
    return JSON.parse(result.data.toString('utf-8'))
}

async function persistQuotaExceededTriggerLog({ log, flowVersion, projectId, payload, logsFileId, shouldExecuteTriggerOnRetry }: PersistQuotaExceededTriggerLogParams): Promise<void> {
    const triggerStep = GenericStepOutput.create({
        input: {},
        type: flowVersion.trigger.type,
        status: shouldExecuteTriggerOnRetry ? StepOutputStatus.FAILED : StepOutputStatus.SUCCEEDED,
        output: payload,
    })
    const outputFile: ExecutioOutputFile = {
        executionState: {
            steps: { [flowVersion.trigger.name]: triggerStep },
            tags: [],
        },
    }
    const data = await fileCompressor.compress({
        data: await logSerializer.serialize(outputFile),
        compression: FileCompression.ZSTD,
    })
    const platformId = await projectService(log).getPlatformId(projectId)
    await fileService(log).save({
        fileId: logsFileId,
        projectId,
        platformId,
        type: FileType.FLOW_RUN_LOG,
        data,
        size: data.length,
        compression: FileCompression.ZSTD,
    })
}

async function queueOrCreateInstantly(params: CreateParams, log: FastifyBaseLogger): Promise<FlowRun> {
    const now = new Date().toISOString()
    const flowRun: FlowRun = {
        id: apId(),
        projectId: params.projectId,
        flowId: params.flowId,
        flowVersionId: params.flowVersionId,
        environment: params.environment,
        parentRunId: params.parentRunId,
        failParentOnFailure: params.failParentOnFailure ?? true,
        status: FlowRunStatus.QUEUED,
        stepNameToTest: params.stepNameToTest,
        created: now,
        updated: now,
        tags: [],
        steps: {},
        triggeredBy: params.triggeredBy,
    }
    switch (params.environment) {
        case RunEnvironment.TESTING:
            return flowRunRepo().save(flowRun)
        case RunEnvironment.PRODUCTION:
            await runsMetadataQueue(log).add(flowRun)
            return flowRun
    }
}

export function isOutsideRetentionWindow(createdTime: string, retentionDays: number): boolean {
    if (!createdTime) return false
    return apDayjs(createdTime).add(retentionDays, 'day').isBefore(apDayjs())
}

function settingsOfStepUnderTest({ flowVersion, stepNameToTest }: SettingsOfStepUnderTestParams): unknown {
    const stepUnderTest = flowStructureUtil.getStep(stepNameToTest, flowVersion.trigger)
    if (isNil(stepUnderTest)) {
        return undefined
    }
    return flowStructureUtil.getAllChildSteps(stepUnderTest).map((step) => step.settings)
}

type SettingsOfStepUnderTestParams = {
    flowVersion: FlowVersion
    stepNameToTest: string
}

type CreateParams = {
    projectId: ProjectId
    flowVersionId: FlowVersionId
    triggeredBy?: string
    parentRunId?: FlowRunId
    failParentOnFailure: boolean | undefined
    stepNameToTest?: string
    flowId: FlowId
    environment: RunEnvironment
}

type ListParams = {
    projectId: ProjectId
    flowId: FlowId[] | undefined
    status: FlowRunStatus[] | undefined
    cursor: Cursor | null
    tags?: string[]
    limit: number
    createdAfter?: string
    createdBefore?: string
    failedStepName?: string
    failedStepMessage?: string
    flowRunIds?: FlowRunId[]
    includeArchived?: boolean
    environment?: RunEnvironment
    parentWaitpointId?: ApId
}

type GetOneParams = {
    id: FlowRunId
    projectId: ProjectId | undefined
}

type ResolveStepOutputParams = {
    step: StepOutput | undefined
    flowRun: FlowRun
    log: FastifyBaseLogger
}

type AddToQueueParamsCommon = {
    flowRun: FlowRun
    platformId: PlatformId
    payload?: unknown
    workerHandlerId: string | undefined
    httpRequestId: string | undefined
    streamStepProgress: StreamStepProgress
    sampleData?: Record<string, unknown>
    jobId?: string
    preferInlinePayload?: boolean
}

export type AddToQueueParams = AddToQueueParamsCommon & (
    | { executionType: ExecutionType.BEGIN, executeTrigger: boolean, entryStepName?: string }
    | { executionType: ExecutionType.RESUME, resumeReason: ResumeReason }
)


type CreateQuotaExceededRunParams = {
    flowVersion: FlowVersion
    payload: unknown
    projectId: ProjectId
    environment: RunEnvironment
    parentRunId?: FlowRunId
    failParentOnFailure: boolean | undefined
    triggeredBy?: string
    shouldExecuteTriggerOnRetry: boolean
}

type PersistQuotaExceededTriggerLogParams = {
    log: FastifyBaseLogger
    flowVersion: FlowVersion
    projectId: ProjectId
    payload: unknown
    logsFileId: string
    shouldExecuteTriggerOnRetry: boolean
}

type StartParams = {
    flowId: FlowId
    payload: unknown
    platformId: PlatformId
    environment: RunEnvironment
    flowVersionId: FlowVersionId
    projectId: ProjectId
    parentRunId?: FlowRunId
    failParentOnFailure: boolean | undefined
    stepNameToTest?: string
    executeTrigger: boolean
    executionType: ExecutionType.BEGIN
    workerHandlerId: string | undefined
    httpRequestId: string | undefined
    streamStepProgress: StreamStepProgress
    sampleData?: Record<string, unknown>
}



type TestParams = {
    projectId: ProjectId
    flowVersionId: FlowVersionId
    triggeredBy?: string
    parentRunId?: FlowRunId
    stepNameToTest?: string
}

type StartManualTriggerParams = {
    projectId: ProjectId
    flowVersionId: FlowVersionId
    triggeredBy: string
}
type RetryParams = {
    flowRunId: FlowRunId
    strategy: FlowRetryStrategy
    projectId: ProjectId
}

type CancelParams = {
    projectId: ProjectId
    platformId: PlatformId
    flowRunIds?: FlowRunId[]
    excludeFlowRunIds?: FlowRunId[]
    status?: FlowRunStatus[]
    flowId?: FlowId[]
    createdAfter?: string
    createdBefore?: string
}

type BulkRetryParams = {
    projectId: ProjectId
    flowRunIds?: FlowRunId[]
    strategy: FlowRetryStrategy
    status?: FlowRunStatus[]
    flowId?: FlowId[]
    createdAfter?: string
    archived?: boolean
    createdBefore?: string
    excludeFlowRunIds?: FlowRunId[]
    failedStepName?: string
    failedStepMessage?: string
}

type BulkArchiveActionParams = {
    projectId: ProjectId
    flowRunIds?: FlowRunId[]
    status?: FlowRunStatus[]
    flowId?: FlowId[]
    createdAfter?: string
    archived?: boolean
    createdBefore?: string
    excludeFlowRunIds?: FlowRunId[]
    failedStepName?: string
    failedStepMessage?: string
}

type CountByStatusParams = {
    projectId: ProjectId
    createdAfter?: string
    createdBefore?: string
}

type FilterFlowRunsAndApplyFiltersParams = {
    projectId: ProjectId
    flowRunIds?: FlowRunId[]
    status?: FlowRunStatus[]
    archived?: boolean
    flowId?: FlowId[]
    createdAfter?: string
    createdBefore?: string
    excludeFlowRunIds?: FlowRunId[]
    failedStepName?: string
    failedStepMessage?: string
}