Skip to content

AI Agents & MCPs & AI Workflow Automation • (~400 MCP servers for AI agents) • AI Automation / AI Agent with MCPs • AI Workflows & AI Agents • MCPs for AI Agents

47.2 KB
Raw Blame
import { ActivepiecesError, apId, Cursor, ErrorCode, FlowId, FlowVersionId, isNil, Metadata, PlatformId, ProjectId, SeekPage, tryCatch, UserId } from '@activepieces/core-utils'
import { apDayjs, apDayjsDuration } from '@activepieces/server-utils'
import { CreateFlowRequest, Flow, FlowCreator, FlowOperationRequest, FlowOperationStatus, FlowOperationType, flowPieceUtil, FlowStatus, flowStructureUtil, FlowTrigger, FlowTriggerType, FlowVersion, FlowVersionState, PopulatedFlow, requiredActionsUtil, SharedTemplate, TelemetryEventName, TemplateStatus, TemplateType, TriggerSource, UncategorizedFolderId, UserWithMetaInformation } from '@activepieces/shared'
import dayjs from 'dayjs'
import { FastifyBaseLogger } from 'fastify'
import { EntityManager, In, IsNull, Not } from 'typeorm'
import { transaction } from '../../core/db/transaction'
import { distributedLock } from '../../database/redis-connections'
import { buildPaginator } from '../../helper/pagination/build-paginator'
import { paginationHelper } from '../../helper/pagination/pagination-utils'
import Paginator, { CURSOR_SELECT_PREFIX, Order } from '../../helper/pagination/paginator'
import { rejectedPromiseHandler } from '../../helper/promise-handler'
import { system } from '../../helper/system/system'
import { AppSystemProp } from '../../helper/system/system-props'
import { SystemJobName } from '../../helper/system-jobs/common'
import { systemJobsSchedule } from '../../helper/system-jobs/system-job'
import { telemetry } from '../../helper/telemetry.utils'
import { projectService } from '../../project/project-service'
import { triggerSourceService } from '../../trigger/trigger-source/trigger-source-service'
import { triggerUtils } from '../../trigger/trigger-source/trigger-utils'
import { flowVersionMigrationService } from '../flow-version/flow-version-migration.service'
import { flowVersionRepo, flowVersionService } from '../flow-version/flow-version.service'
import { flowFolderService } from '../folder/folder.service'
import { flowExecutionCache } from './flow-execution-cache'
import { flowPublishHooks, publishHooksFactory } from './flow-publish-hooks'
import { flowPublishUtils } from './flow-publish-utils'
import { flowSideEffects } from './flow-service-side-effects'
import { FlowEntity } from './flow.entity'
import { flowRepo } from './flow.repo'



const notDeleting = { operationStatus: Not(FlowOperationStatus.DELETING) }

const nonDraftOperationTypes: FlowOperationType[] = [
    FlowOperationType.CHANGE_STATUS,
    FlowOperationType.CHANGE_FOLDER,
    FlowOperationType.UPDATE_METADATA,
    FlowOperationType.UPDATE_MINUTES_SAVED,
    FlowOperationType.UPDATE_OWNER,
]

export const flowService = (log: FastifyBaseLogger) => ({
    async create({ projectId, request, externalId, ownerId, templateId, createdBy, ip, emitEvents = true }: CreateParams): Promise<PopulatedFlow> {
        await assertExternalIdIsUnique({ projectId, externalId })
        const folderId = await getFolderIdFromRequest({ projectId, folderId: request.folderId, folderName: request.folderName, log })
        const newFlow: NewFlow = {
            id: apId(),
            projectId,
            folderId,
            status: FlowStatus.DISABLED,
            ownerId,
            publishedVersionId: null,
            externalId: externalId ?? apId(),
            metadata: request.metadata,
            operationStatus: FlowOperationStatus.NONE,
            templateId,
            createdBy,
        }
        const { savedFlow, savedFlowVersion } = await transaction(async (entityManager) => {
            const flow = await flowRepo(entityManager).save(newFlow)
            const flowVersion = await flowVersionService(log).createEmptyVersion({
                flowId: flow.id,
                displayName: request.displayName,
                notes: [],
                schemaVersion: null,
                entityManager,
            })
            return { savedFlow: flow, savedFlowVersion: flowVersion }
        })

        rejectedPromiseHandler(
            telemetry(log).trackProject({
                projectId: savedFlow.projectId,
                event: {
                    name: TelemetryEventName.CREATED_FLOW,
                    payload: {
                        flowId: savedFlow.id,
                    },
                },
            }),
            log,
        )

        log.info({ flow: { id: savedFlow.id }, project: { id: projectId }, displayName: request.displayName }, 'Flow created')
        const createdFlow = {
            ...savedFlow,
            version: savedFlowVersion,
        }
        if (emitEvents) {
            flowSideEffects(log).onCreated({
                platformId: await projectService(log).getPlatformId(projectId),
                projectId,
                userId: ownerId,
                ip,
                flow: createdFlow,
            })
        }
        return createdFlow
    },

    async list({
        projectIds,
        platformId,
        cursorRequest,
        limit = Paginator.NO_LIMIT,
        folderId,
        folderIds,
        status,
        name,
        connectionExternalIds,
        agentExternalIds,
        externalIds,
        flowIds,
        versionState = FlowVersionState.DRAFT,
        includeTriggerSource = true,
        sortBy,
        order,
    }: ListParams): Promise<SeekPage<PopulatedFlow>> {
        assertSortIsNotCombinedWithCursor({ sortBy, cursor: cursorRequest })
        const decodedCursor = paginationHelper.decodeCursor(cursorRequest)
        const paginator = buildPaginator({
            entity: FlowEntity,
            alias: 'ff',
            query: {
                limit,
                orderBy: [
                    { field: 'status', order: Order.DESC },
                    { field: 'updated', order: Order.DESC },
                ],
                afterCursor: decodedCursor.nextCursor,
                beforeCursor: decodedCursor.previousCursor,
            },
        })

        const queryBuilder = flowRepo().createQueryBuilder('ff').where(notDeleting)

        if (projectIds) {
            queryBuilder.andWhere({ projectId: In(projectIds) })
        }
        else {
            queryBuilder
                .innerJoin('project', 'project', 'project.id = ff."projectId"')
                .andWhere('project."platformId" = :platformId', { platformId })
        }

        if (folderId !== undefined) {
            queryBuilder.andWhere({ folderId: folderId === UncategorizedFolderId ? IsNull() : folderId })
        }

        if (folderIds !== undefined) {
            queryBuilder.andWhere({ folderId: In(folderIds) })
        }

        if (status !== undefined) {
            queryBuilder.andWhere({ status: In(status) })
        }

        const latestVersionSubquery = flowVersionRepo()
            .createQueryBuilder('fv_sub')
            .select('fv_sub.id')
            .where('fv_sub."flowId" = ff.id')
            .orderBy('fv_sub.created', 'DESC')
            .limit(1)

        if (versionState === FlowVersionState.DRAFT) {
            queryBuilder.leftJoinAndMapOne(
                'ff.version',
                'flow_version',
                'latest_version',
                `latest_version."flowId" = ff.id AND latest_version.id = (${latestVersionSubquery.getQuery()})`,
            )
        }
        else {
            queryBuilder.leftJoin(
                'flow_version',
                'latest_version',
                `latest_version."flowId" = ff.id AND latest_version.id = (${latestVersionSubquery.getQuery()})`,
            )
            queryBuilder.innerJoinAndMapOne(
                'ff.version',
                'flow_version',
                'published_version',
                'published_version.id = ff.publishedVersionId',
            )
        }

        if (includeTriggerSource) {
            queryBuilder.leftJoinAndMapOne(
                'ff.triggerSource',
                'trigger_source',
                'ts',
                'ts."flowId" = ff.id AND ts.deleted IS NULL AND ts.simulate = false',
            )
        }

        if (name !== undefined) {
            queryBuilder.andWhere('LOWER(latest_version."displayName") LIKE LOWER(:name)', { name: `%${name}%` })
        }

        if (externalIds !== undefined) {
            queryBuilder.andWhere('ff."externalId" IN (:...externalIds)', { externalIds })
        }

        if (flowIds !== undefined) {
            queryBuilder.andWhere('ff.id IN (:...flowIds)', { flowIds })
        }

        if (connectionExternalIds !== undefined) {
            const filteredVersion = versionState === FlowVersionState.DRAFT ? 'latest_version' : 'published_version'
            queryBuilder.andWhere(`${filteredVersion}."connectionIds" && :connectionExternalIds`, { connectionExternalIds })
        }

        if (agentExternalIds !== undefined) {
            queryBuilder.andWhere('latest_version."agentIds" && :agentExternalIds', { agentExternalIds })
        }

        if (!isNil(sortBy)) {
            const nameSortAlias = `${CURSOR_SELECT_PREFIX}name`
            const nameSortColumn = versionState === FlowVersionState.DRAFT ? 'latest_version."displayName"' : 'published_version."displayName"'
            queryBuilder.addSelect(`LOWER(COALESCE(${nameSortColumn}, ''))`, nameSortAlias)
            queryBuilder.addOrderBy(nameSortAlias, order ?? 'ASC')
        }

        const paginationResult = await paginator.paginate<Flow & { version: FlowVersion | null, triggerSource?: TriggerSource }>(queryBuilder)

        const populatedFlows = await Promise.all(paginationResult.data.map(async (flow) => {
            if (isNil(flow.version)) {
                throw new ActivepiecesError({
                    code: ErrorCode.ENTITY_NOT_FOUND,
                    params: {
                        entityType: 'FlowVersion',
                        message: `flowId=${flow.id}`,
                    },
                })
            }
            const migratedVersion = await flowVersionMigrationService(log).migrate(flow.version, flow.projectId, platformId)
            return {
                ...flow,
                version: migratedVersion,
                triggerSource: includeTriggerSource && flow.triggerSource
                    ? {
                        schedule: flow.triggerSource.schedule,
                        pieceName: flow.triggerSource.pieceName,
                    }
                    : undefined,
            }
        }))
        return paginationHelper.createPage(populatedFlows, isNil(sortBy) ? paginationResult.cursor : null)
    },
    async exists({ id, projectId }: FlowExistsParams): Promise<boolean> {
        return flowRepo().existsBy({
            id,
            projectId,
            ...notDeleting,
        })
    },
    async getOneById(id: string): Promise<Flow | null> {
        const flow = await flowRepo().findOneBy({
            id,
            ...notDeleting,
        })
        if (isNil(flow)) {
            return null
        }
        const projectExists = await projectService(log).exists({
            projectId: flow.projectId,
        })
        if (!projectExists) {
            return null
        }
        return flow
    },
    async getOne({ id, projectId, entityManager, includeDeleting = false }: GetOneParams): Promise<Flow | null> {
        const projectExists = await projectService(log).exists({
            projectId,
        })
        if (!projectExists) {
            return null
        }
        return flowRepo(entityManager).findOneBy({
            id,
            projectId,
            ...(includeDeleting ? {} : notDeleting),
        })
    },

    async getOneOrThrow(params: GetOneParams): Promise<Flow> {
        const flow = await this.getOne(params)
        assertFlowIsNotNull(flow)
        return flow
    },

    async getOnePopulated({
        id,
        projectId,
        versionId,
        removeConnectionsName = false,
        removeSampleData = false,
        entityManager,
        includeDeleting = false,
    }: GetOnePopulatedParams): Promise<PopulatedFlow | null> {
        const flow = await this.getOne({ id, projectId, entityManager, includeDeleting })
        if (isNil(flow)) {
            return null
        }

        const flowVersion = await flowVersionService(log).getFlowVersionOrThrow({
            flowId: id,
            versionId,
            removeConnectionsName,
            removeSampleData,
            entityManager,
            projectId,
        })

        const triggerSource = await triggerSourceService(log).getByFlowId({
            flowId: id,
            projectId,
            simulate: false,
        })

        return {
            ...flow,
            version: flowVersion,
            triggerSource: triggerSource ? {
                schedule: triggerSource.schedule,
                pieceName: triggerSource.pieceName,
            } : undefined,
        }
    },

    async getOnePopulatedOrThrow(params: GetOnePopulatedParams): Promise<PopulatedFlow> {
        const flow = await this.getOnePopulated(params)

        assertFlowIsNotNull(flow)
        return flow
    },

    async update({
        id,
        userId = null,
        projectId,
        platformId,
        operation,
        previousFlow,
        ip,
        emitEvents = true,
        skipRequiredActionsCheck = false,
        editorSessionId = null,
        auditExplanation,
    }: UpdateParams): Promise<PopulatedFlow> {
        const flowBeforeOperation = emitEvents
            ? previousFlow ?? await this.getOnePopulatedOrThrow({ id, projectId })
            : undefined

        switch (operation.type) {
            case FlowOperationType.LOCK_AND_PUBLISH: {
                await withStatusLock({
                    id,
                    log,
                    fn: () => publish({
                        id,
                        projectId,
                        platformId,
                        userId,
                        requestedStatus: operation.request.status ?? FlowStatus.ENABLED,
                        skipRequiredActionsCheck,
                        log,
                    }),
                })
                break
            }

            case FlowOperationType.CHANGE_STATUS: {
                await applyStatusChange({ id, projectId, newStatus: operation.request.status }, log)
                break
            }

            case FlowOperationType.CHANGE_FOLDER: {
                const folderId = operation.request.folderId === UncategorizedFolderId ? null : operation.request.folderId
                if (!isNil(folderId)) {
                    await flowFolderService(log).getOneOrThrow({ projectId, folderId })
                }
                await flowRepo().update({ id, projectId }, {
                    folderId,
                })
                log.info({ flow: { id }, folderId }, 'Flow moved to folder')
                break
            }

            case FlowOperationType.UPDATE_METADATA: {
                await this.updateMetadata({
                    id,
                    projectId,
                    metadata: operation.request.metadata,
                })
                break
            }

            case FlowOperationType.UPDATE_MINUTES_SAVED: {
                await flowRepo().update(id, {
                    timeSavedPerRun: operation.request.timeSavedPerRun,
                })
                break
            }

            case FlowOperationType.UPDATE_OWNER: {
                await flowRepo().update(id, {
                    ownerId: operation.request.ownerId,
                })
                break
            }
            case FlowOperationType.ADD_NOTE:
            case FlowOperationType.UPDATE_NOTE:
            case FlowOperationType.DELETE_NOTE: {
                await withDraftLock({ flowId: id, log, fn: async () => {
                    const lastVersion = await flowVersionService(
                        log,
                    ).getFlowVersionOrThrow({
                        flowId: id,
                        versionId: undefined,
                    })
                    await flowVersionService(log).applyOperation({
                        userId,
                        projectId,
                        platformId,
                        flowVersion: lastVersion,
                        userOperation: operation,
                    })
                } })
                break
            }
            default: {
                await withDraftLock({ flowId: id, log, fn: () => writeDraft({ id, projectId, platformId, userId, operation, log }) })
            }
        }

        if (isDraftWrite(operation.type)) {
            await flowSideEffects(log).onDraftWritten({ flowId: id, projectId, editorSessionId })
        }

        const updatedFlow = await this.getOnePopulatedOrThrow({
            id,
            projectId,
        })
        if (!isNil(flowBeforeOperation)) {
            flowSideEffects(log).onOperationApplied({
                platformId,
                projectId,
                userId,
                ip,
                flow: updatedFlow,
                previousVersion: flowBeforeOperation.version,
                previousStatus: flowBeforeOperation.status,
                operation,
                auditExplanation,
            })
        }
        return updatedFlow
    },

    async replaceConnection({ id, projectId, platformId, userId, sourceExternalId, targetExternalId, applyToPublishedVersion, auditExplanation }: ReplaceConnectionParams): Promise<void> {
        const flowBeforeReplace = await this.getOnePopulatedOrThrow({ id, projectId })
        const { appliedOperations, error } = await replaceConnectionForDraftAndPublishedVersions({ id, projectId, platformId, userId, sourceExternalId, targetExternalId, applyToPublishedVersion, log })
        if (appliedOperations.length > 0) {
            await flowSideEffects(log).onDraftWritten({ flowId: id, projectId, editorSessionId: null })
            appliedOperations.forEach(({ operation, flowAfterOperation }, index) => {
                const flowBeforeOperation = index === 0 ? flowBeforeReplace : appliedOperations[index - 1].flowAfterOperation
                flowSideEffects(log).onOperationApplied({
                    platformId,
                    projectId,
                    userId,
                    flow: flowAfterOperation,
                    previousVersion: flowBeforeOperation.version,
                    previousStatus: flowBeforeOperation.status,
                    operation,
                    auditExplanation,
                })
            })
        }
        if (!isNil(error)) {
            throw error
        }
    },

    async updatedPublishedVersionId({
        id,
        userId,
        projectId,
        platformId,
        flowVersionToPublish,
        requestedStatus,
    }: UpdatePublishedVersionIdParams): Promise<UpdatePublishedVersionIdResult> {
        const flowToUpdate = await this.getOneOrThrow({ id, projectId })
        const liveVersion = flowToUpdate.status === FlowStatus.ENABLED && !isNil(flowToUpdate.publishedVersionId)
            ? await flowVersionService(log).getFlowVersionOrThrow({ flowId: id, versionId: flowToUpdate.publishedVersionId })
            : undefined
        const republishParams = { previouslyPublished: liveVersion, toPublish: flowVersionToPublish, requestedStatus }
        const isRepublish = flowPublishUtils.isRepublish(republishParams)
        const isTriggerUnchanged = flowPublishUtils.isTriggerUnchanged(republishParams)

        if (flowToUpdate.status === FlowStatus.ENABLED && !isNil(flowToUpdate.publishedVersionId)) {
            await triggerSourceService(log).disable({
                flowId: flowToUpdate.id,
                projectId: flowToUpdate.projectId,
                simulate: false,
                ignoreError: true,
                isRepublish,
                isTriggerUnchanged,
            })
        }

        const publishedFlow = await transaction(async (entityManager) => {
            await flowPublishHooks.get(log).assertReferencesResolve({
                projectId,
                agentExternalIds: flowVersionToPublish.agentIds ?? [],
                entityManager,
            })
            const lockedFlowVersion = await lockFlowVersionIfNotLocked({
                flowVersion: flowVersionToPublish,
                userId,
                projectId,
                platformId,
                entityManager,
                log,
            })
            await this.setPublishedVersion({
                flow: flowToUpdate,
                lockedVersion: lockedFlowVersion,
                entityManager,
            })
            await flowExecutionCache(log).invalidate(flowToUpdate.id)
            return {
                ...flowToUpdate,
                publishedVersionId: lockedFlowVersion.id,
                status: FlowStatus.DISABLED,
                version: lockedFlowVersion,
            }
        })
        const { websocketService } = await import('../../core/websockets.service')
        websocketService.notifyWorkers().flowPublished({ flowId: publishedFlow.id, flowVersionId: publishedFlow.version.id, projectId: publishedFlow.projectId })
        rejectedPromiseHandler(telemetry(log).trackProject({
            projectId,
            event: {
                name: TelemetryEventName.FLOW_PUBLISHED,
                payload: { flowId: id },
            },
        }), log)
        return { publishedFlow, isRepublish, isTriggerUnchanged }
    },

    async setPublishedVersion({ flow, lockedVersion, entityManager }: SetPublishedVersionParams): Promise<void> {
        await flowPublishHooks.get(log).assertReferencesResolve({
            projectId: flow.projectId,
            agentExternalIds: lockedVersion.agentIds ?? [],
            entityManager,
        })
        await flowRepo(entityManager).update({ id: flow.id }, {
            publishedVersionId: lockedVersion.id,
            status: FlowStatus.DISABLED,
        })
    },

    async withStatusLock<T>({ id, fn }: { id: FlowId, fn: () => Promise<T> }): Promise<T> {
        return withStatusLock({ id, log, fn })
    },

    async changeStatusHoldingLock(params: ChangeStatusParams): Promise<void> {
        await changeStatus(params, log)
    },

    async delete({ id, projectId, previousFlow, userId, ip, emitEvents = true }: DeleteParams): Promise<void> {
        const deletedFlow = emitEvents
            ? previousFlow ?? await this.getOnePopulatedOrThrow({ id, projectId, includeDeleting: true })
            : undefined
        const flow = await this.getOneOrThrow({
            id,
            projectId,
            includeDeleting: true,
        })
        await flowRepo().update({ id, projectId }, {
            status: FlowStatus.DISABLED,
            operationStatus: FlowOperationStatus.DELETING,
        })
        await flowExecutionCache(log).invalidate(id)
        await this.addDeleteFlowJob(flow)
        log.info({ flow: { id }, project: { id: projectId } }, 'Flow deletion requested')
        if (!isNil(deletedFlow)) {
            flowSideEffects(log).onDeleted({
                platformId: await projectService(log).getPlatformId(projectId),
                projectId,
                userId,
                ip,
                flow: deletedFlow,
            })
        }
    },

    async deleteAllByPlatformId(platformId: PlatformId): Promise<void> {
        const projectIds = await projectService(log).getProjectIdsByPlatform(platformId)
        const flows = await flowRepo().findBy({
            projectId: In(projectIds),
        })
        await Promise.all(flows.map((flow) => this.delete({ id: flow.id, projectId: flow.projectId, emitEvents: false })))
    },

    async getTemplate({
        flowId,
        userMetadata,
        versionId,
        projectId,
    }: GetTemplateParams): Promise<SharedTemplate> {
        const flow = await this.getOnePopulatedOrThrow({
            id: flowId,
            projectId,
            versionId,
            removeConnectionsName: true,
            removeSampleData: true,
        })

        const template: SharedTemplate = {
            name: flow.version.displayName,
            summary: '',
            description: '',
            pieces: Array.from(new Set(flowPieceUtil.getUsedPieces(flow.version.trigger))),
            flows: [flow.version],
            tags: [],
            blogUrl: '',
            metadata: {
                externalId: flow.externalId,
            },
            author: userMetadata ? `${userMetadata.firstName} ${userMetadata.lastName}` : '',
            categories: [],
            type: TemplateType.SHARED,
            status: TemplateStatus.PUBLISHED,
        }
        return template
    },

    async count({ projectId, folderId, status }: CountParams): Promise<number> {
        if (folderId === undefined) {
            return flowRepo().countBy({ projectId, status })
        }

        return flowRepo().countBy({
            folderId: folderId !== UncategorizedFolderId ? folderId : IsNull(),
            projectId,
            status,
        })
    },

    async existsByProjectAndStatus(params: ExistsByProjectAndStatusParams): Promise<boolean> {
        const { projectId, status, entityManager } = params

        return flowRepo(entityManager).existsBy({
            projectId,
            status,
        })
    },

    async updateMetadata({
        id,
        projectId,
        metadata,
    }: UpdateMetadataParams): Promise<PopulatedFlow> {
        const flowToUpdate = await this.getOneOrThrow({
            id,
            projectId,
        })

        flowToUpdate.metadata = metadata

        await flowRepo().save(flowToUpdate)

        return this.getOnePopulatedOrThrow({
            id,
            projectId,
        })
    },

    async updateLastModified({ flowId, projectId, entityManager }: UpdateLastModifiedParams): Promise<void> {
        await flowRepo(entityManager).update({
            id: flowId,
            projectId,
        }, {
            updated: dayjs().toISOString(),
        })
    },

    addDeleteFlowJob: async (flow: Flow): Promise<void> => {
        await systemJobsSchedule(log).upsertJob({
            job: {
                name: SystemJobName.DELETE_FLOW,
                data: {
                    flow,
                    preDeleteDone: false,
                },
                jobId: `delete-flow-${flow.id}`,
            },
            schedule: {
                type: 'one-time',
                date: apDayjs(),
            },
            customConfig: {
                backoff: {
                    type: 'exponential',
                    delay: apDayjsDuration(5, 'second').asMilliseconds(),
                },
            },
        })
    },

    async countFlowsByProjects(projectIds: ProjectId[]): Promise<Map<ProjectId, number>> {
        if (projectIds.length === 0) return new Map()
        
        const result = await flowRepo()
            .createQueryBuilder('flow')
            .select('flow.projectId', 'projectId')
            .addSelect('COUNT(*)', 'count')
            .where('flow.projectId IN (:...projectIds)', { projectIds })
            .groupBy('flow.projectId')
            .getRawMany()
        
        return new Map(result.map(r => [r.projectId, parseInt(r.count)]))
    },

    async countActiveFlowsByProjects(projectIds: ProjectId[]): Promise<Map<ProjectId, number>> {
        if (projectIds.length === 0) return new Map()
        
        const result = await flowRepo()
            .createQueryBuilder('flow')
            .select('flow.projectId', 'projectId')
            .addSelect('COUNT(*)', 'count')
            .where('flow.projectId IN (:...projectIds)', { projectIds })
            .andWhere('flow.status = :status', { status: FlowStatus.ENABLED })
            .andWhere('flow.operationStatus != :deleting', { deleting: FlowOperationStatus.DELETING })
            .groupBy('flow.projectId')
            .getRawMany()
        
        return new Map(result.map(r => [r.projectId, parseInt(r.count)]))
    },

    async getLastFlowUpdatedByProjects(projectIds: ProjectId[]): Promise<Map<ProjectId, string>> {
        if (projectIds.length === 0) return new Map()

        const result = await flowRepo()
            .createQueryBuilder('flow')
            .select('flow.projectId', 'projectId')
            .addSelect('MAX(flow.updated)', 'lastUpdated')
            .where('flow.projectId IN (:...projectIds)', { projectIds })
            .andWhere('flow.operationStatus != :deleting', { deleting: FlowOperationStatus.DELETING })
            .groupBy('flow.projectId')
            .getRawMany()

        return new Map(result.map(r => [r.projectId, new Date(r.lastUpdated).toISOString()]))
    },
})


function isDraftWrite(type: FlowOperationType): boolean {
    return !nonDraftOperationTypes.includes(type)
}

async function assertRequiredActionsPresent({ projectId, platformId, flowVersion, log }: AssertRequiredActionsParams): Promise<void> {
    const result = await flowPublishHooks.get(log).findMissingRequiredActions({ projectId, platformId, flowVersion })
    if (isNil(result)) {
        return
    }
    throw new ActivepiecesError({
        code: ErrorCode.REQUIRED_ACTIONS_MISSING,
        params: {
            message: requiredActionsUtil.buildRequiredActionsMissingErrorMessage(result),
            mode: result.mode,
            requiredActions: result.requiredActions,
            missingActions: result.missingActions,
            skippedActions: result.skippedActions,
        },
    })
}

function assertTriggerIsSelected({ trigger }: { trigger: FlowTrigger }): void {
    if (trigger.type === FlowTriggerType.PIECE && !isNil(trigger.settings.triggerName)) {
        return
    }
    throw new ActivepiecesError({
        code: ErrorCode.FLOW_OPERATION_INVALID,
        params: { message: 'Cannot enable a flow without a trigger, select a trigger first' },
    })
}

const lockFlowVersionIfNotLocked = async ({
    flowVersion,
    userId,
    projectId,
    platformId,
    entityManager,
    log,
}: LockFlowVersionIfNotLockedParams): Promise<FlowVersion> => {
    if (flowVersion.state === FlowVersionState.LOCKED) {
        return flowVersion
    }

    return flowVersionService(log).applyOperation({
        userId,
        projectId,
        platformId,
        flowVersion,
        userOperation: {
            type: FlowOperationType.LOCK_FLOW,
            request: {},
        },
        entityManager,
    })
}


async function withDraftLock<T>({ flowId, log, fn }: {
    flowId: FlowId
    log: FastifyBaseLogger
    fn: () => Promise<T>
}): Promise<T> {
    return distributedLock(log).runExclusive({
        key: `flow-draft-update-${flowId}`,
        timeoutInSeconds: DRAFT_UPDATE_LOCK_TIMEOUT_SECONDS,
        fn,
    })
}

async function writeDraft({ id, projectId, platformId, userId, operation, log }: WriteDraftParams): Promise<void> {
    const { version: lastVersion, createdNewDraft } = await createNewDraftIfVersionIsPublished({
        flowId: id,
        projectId,
        platformId,
        userId,
        log,
    })
    const { error } = await tryCatch(() => flowVersionService(log).applyOperation({
        userId,
        projectId,
        platformId,
        flowVersion: lastVersion,
        userOperation: operation,
    }))
    if (!isNil(error)) {
        if (createdNewDraft) {
            await tryCatch(() => flowVersionRepo().delete({ id: lastVersion.id, flowId: id }))
        }
        throw error
    }
}

async function publish({ id, projectId, platformId, userId, requestedStatus, skipRequiredActionsCheck, log }: PublishParams): Promise<void> {
    const flow = await flowService(log).getOneOrThrow({ id, projectId })
    const flowVersionToPublish = await flowVersionService(log).getFlowVersionOrThrow({ flowId: id, versionId: undefined })
    if (!skipRequiredActionsCheck) {
        await assertRequiredActionsPresent({
            projectId,
            platformId,
            flowVersion: flowVersionToPublish,
            log,
        })
    }
    if (requestedStatus === FlowStatus.ENABLED) {
        assertTriggerIsSelected({ trigger: flowVersionToPublish.trigger })
        await triggerUtils(log).getPieceTriggerOrThrow({ flowVersion: flowVersionToPublish, projectId })
    }
    const route = await publishHooksFactory.get(log).routePublish({ flow, projectId, platformId, userId })
    if (route === 'NEEDS_APPROVAL') {
        await publishHooksFactory.get(log).submitForApproval({
            flow,
            flowVersionToPublish,
            userId,
            projectId,
            platformId,
            requestedStatus,
        })
        return
    }
    const { isRepublish, isTriggerUnchanged } = await flowService(log).updatedPublishedVersionId({
        id,
        userId,
        projectId,
        platformId,
        flowVersionToPublish,
        requestedStatus,
    })
    await changeStatus({ id, projectId, newStatus: requestedStatus, isRepublish, isTriggerUnchanged }, log)
}

async function replaceConnectionForDraftAndPublishedVersions({ id, projectId, platformId, userId, sourceExternalId, targetExternalId, applyToPublishedVersion, log }: ReplaceConnectionForDraftAndPublishedVersionsParams): Promise<ReplaceConnectionResult> {
    return withDraftLock({
        flowId: id,
        log,
        fn: async () => {
            if (!applyToPublishedVersion) {
                return replaceConnectionInDraft({ id, projectId, platformId, userId, sourceExternalId, targetExternalId, log })
            }
            return withStatusLock({
                id,
                log,
                fn: async () => {
                    const flow = await flowService(log).getOneOrThrow({ id, projectId })
                    const publishedVersionId = flow.publishedVersionId
                    if (isNil(publishedVersionId)) {
                        return replaceConnectionInDraft({ id, projectId, platformId, userId, sourceExternalId, targetExternalId, log })
                    }
                    const latestVersion = await flowVersionService(log).getFlowVersionOrThrow({ flowId: id, versionId: undefined })
                    const publishedVersion = await flowVersionService(log).getFlowVersionOrThrow({ flowId: id, versionId: publishedVersionId })
                    const replaceInDraftOperation = createImportOperationToReplaceConnection({ flowVersion: latestVersion, sourceExternalId, targetExternalId })
                    const replaceInPublishedVersionOperation = createImportOperationToReplaceConnection({ flowVersion: publishedVersion, sourceExternalId, targetExternalId })
                    const appliedOperations: AppliedOperation[] = []
                    const { error } = await tryCatch(async () => {
                        await writeDraft({ id, projectId, platformId, userId, operation: replaceInPublishedVersionOperation, log })
                        appliedOperations.push(await recordAppliedOperation({ id, projectId, operation: replaceInPublishedVersionOperation, log }))
                        await publish({ id, projectId, platformId, userId, requestedStatus: flow.status, skipRequiredActionsCheck: true, log })
                        appliedOperations.push(await recordAppliedOperation({ id, projectId, operation: { type: FlowOperationType.LOCK_AND_PUBLISH, request: { status: flow.status } }, log }))
                    })
                    const hadUnpublishedChanges = latestVersion.id !== publishedVersionId
                    if (!hadUnpublishedChanges && isNil(error)) {
                        return { appliedOperations, error }
                    }
                    const { error: restoreDraftError } = await tryCatch(async () => {
                        await writeDraft({ id, projectId, platformId, userId, operation: replaceInDraftOperation, log })
                        appliedOperations.push(await recordAppliedOperation({ id, projectId, operation: replaceInDraftOperation, log }))
                    })
                    return { appliedOperations, error: error ?? restoreDraftError }
                },
            })
        },
    })
}

async function replaceConnectionInDraft({ id, projectId, platformId, userId, sourceExternalId, targetExternalId, log }: ReplaceConnectionInDraftParams): Promise<ReplaceConnectionResult> {
    const latestVersion = await flowVersionService(log).getFlowVersionOrThrow({ flowId: id, versionId: undefined })
    if (!latestVersion.connectionIds.includes(sourceExternalId)) {
        return { appliedOperations: [], error: null }
    }
    const replaceInDraftOperation = createImportOperationToReplaceConnection({ flowVersion: latestVersion, sourceExternalId, targetExternalId })
    await writeDraft({ id, projectId, platformId, userId, operation: replaceInDraftOperation, log })
    return { appliedOperations: [await recordAppliedOperation({ id, projectId, operation: replaceInDraftOperation, log })], error: null }
}

async function recordAppliedOperation({ id, projectId, operation, log }: RecordAppliedOperationParams): Promise<AppliedOperation> {
    const flowAfterOperation = await flowService(log).getOnePopulatedOrThrow({ id, projectId })
    return { operation, flowAfterOperation }
}

function createImportOperationToReplaceConnection({ flowVersion, sourceExternalId, targetExternalId }: CreateImportOperationToReplaceConnectionParams): FlowOperationRequest {
    return {
        type: FlowOperationType.IMPORT_FLOW,
        request: flowStructureUtil.replaceConnection({ flowVersion, sourceExternalId, targetExternalId }),
    }
}

async function applyStatusChange(params: ChangeStatusParams, log: FastifyBaseLogger): Promise<void> {
    await withStatusLock({ id: params.id, log, fn: () => changeStatus(params, log) })
}

async function withStatusLock<T>({ id, log, fn }: WithStatusLockParams<T>): Promise<T> {
    const triggerTimeout = system.getNumberOrThrow(AppSystemProp.TRIGGER_TIMEOUT_SECONDS)
    return distributedLock(log).runExclusive({
        key: `flow-status-change-${id}`,
        timeoutInSeconds: triggerTimeout * 2 + 30,
        fn,
    })
}

async function changeStatus(params: ChangeStatusParams, log: FastifyBaseLogger): Promise<void> {
    const flowToUpdate = await flowService(log).getOneOrThrow({
        id: params.id,
        projectId: params.projectId,
    })
    if (flowToUpdate.status === params.newStatus) {
        return
    }

    const publishedFlowVersionId = flowToUpdate.publishedVersionId
    if (isNil(publishedFlowVersionId)) {
        throw new ActivepiecesError({
            code: ErrorCode.FLOW_OPERATION_INVALID,
            params: { message: 'Publish the flow before turning it on' },
        })
    }
    const publishedFlowVersion = await flowVersionService(log).getFlowVersionOrThrow({
        flowId: flowToUpdate.id,
        versionId: publishedFlowVersionId,
    })
    if (params.newStatus === FlowStatus.ENABLED) {
        assertTriggerIsSelected({ trigger: publishedFlowVersion.trigger })
    }

    await flowSideEffects(log).preUpdateStatus({
        flowToUpdate,
        publishedFlowVersion,
        newStatus: params.newStatus,
        templateId: flowToUpdate.templateId ?? undefined,
        isRepublish: params.isRepublish,
        isTriggerUnchanged: params.isTriggerUnchanged,
    })

    await flowRepo().save({
        ...flowToUpdate,
        status: params.newStatus,
        publishedVersionId: publishedFlowVersion.id,
    })
    await flowExecutionCache(log).invalidate(params.id)
}

export const getFolderIdFromRequest = async ({ projectId, folderId, folderName, log }: { projectId: string, folderId: string | undefined, folderName: string | undefined, log: FastifyBaseLogger }) => {
    if (folderId) {
        await flowFolderService(log).getOneOrThrow({ projectId, folderId })
        return folderId
    }
    if (folderName) {
        return (await flowFolderService(log).upsert({
            projectId,
            request: {
                projectId,
                displayName: folderName,
            },
        })).id
    }
    return null
}

const assertFlowIsNotNull: <T extends Flow>(
    flow: T | null
) => asserts flow is T = <T>(flow: T | null) => {
    if (isNil(flow)) {
        throw new ActivepiecesError({
            code: ErrorCode.ENTITY_NOT_FOUND,
            params: {},
        })
    }
}

function assertSortIsNotCombinedWithCursor({ sortBy, cursor }: { sortBy: 'NAME' | undefined, cursor: Cursor | undefined | null }): void {
    if (isNil(sortBy) || isNil(cursor)) {
        return
    }
    throw new ActivepiecesError({
        code: ErrorCode.GENERIC_ERROR,
        params: { message: 'sortBy cannot be combined with cursor' },
    })
}

async function assertExternalIdIsUnique({ projectId, externalId }: { projectId: ProjectId, externalId: string | undefined }): Promise<void> {
    if (isNil(externalId)) {
        return
    }
    const exists = await flowRepo().existsBy({ projectId, externalId, ...notDeleting })
    if (exists) {
        throw new ActivepiecesError({
            code: ErrorCode.FLOW_EXTERNAL_ID_ALREADY_EXISTS,
            params: { externalId },
        })
    }
}

type CreateParams = EventEmissionParams & {
    projectId: ProjectId
    request: CreateFlowRequest
    ownerId?: UserId
    externalId?: string
    templateId?: string
    createdBy?: FlowCreator
}

type ListParamsBase = {
    cursorRequest?: Cursor
    limit?: number
    folderId?: string
    folderIds?: string[]
    status?: FlowStatus[]
    name?: string
    versionState?: FlowVersionState
    externalIds?: string[]
    flowIds?: FlowId[]
    connectionExternalIds?: string[]
    agentExternalIds?: string[]
    includeTriggerSource?: boolean
    sortBy?: 'NAME'
    order?: 'ASC' | 'DESC'
}

type ListParams = ListParamsBase & (
    | { projectIds: ProjectId[], platformId?: never }
    | { projectIds?: never, platformId: PlatformId }
)

type GetOneParams = {
    id: FlowId
    projectId: ProjectId
    entityManager?: EntityManager
    includeDeleting?: boolean
}

type GetOnePopulatedParams = GetOneParams & {
    versionId?: FlowVersionId
    removeConnectionsName?: boolean
    removeSampleData?: boolean
}

type GetTemplateParams = {
    flowId: FlowId
    userMetadata: UserWithMetaInformation | null
    projectId: ProjectId
    versionId: FlowVersionId | undefined
}

type CountParams = {
    projectId: ProjectId
    folderId?: string
    status?: FlowStatus
}

type UpdateParams = EventEmissionParams & {
    id: FlowId
    userId?: UserId | null
    projectId: ProjectId
    operation: FlowOperationRequest
    platformId: PlatformId
    previousFlow?: PopulatedFlow
    skipRequiredActionsCheck?: boolean
    editorSessionId?: string | null
    auditExplanation?: string
}

type ReplaceConnectionParams = {
    id: FlowId
    projectId: ProjectId
    platformId: PlatformId
    userId: UserId
    sourceExternalId: string
    targetExternalId: string
    applyToPublishedVersion: boolean
    auditExplanation: string
}

type ReplaceConnectionForDraftAndPublishedVersionsParams = Omit<ReplaceConnectionParams, 'auditExplanation'> & {
    log: FastifyBaseLogger
}

type ReplaceConnectionInDraftParams = Omit<ReplaceConnectionForDraftAndPublishedVersionsParams, 'applyToPublishedVersion'>

type ReplaceConnectionResult = {
    appliedOperations: AppliedOperation[]
    error: Error | null
}

type AppliedOperation = {
    operation: FlowOperationRequest
    flowAfterOperation: PopulatedFlow
}

type RecordAppliedOperationParams = {
    id: FlowId
    projectId: ProjectId
    operation: FlowOperationRequest
    log: FastifyBaseLogger
}

type CreateImportOperationToReplaceConnectionParams = {
    flowVersion: FlowVersion
    sourceExternalId: string
    targetExternalId: string
}

type WriteDraftParams = {
    id: FlowId
    projectId: ProjectId
    platformId: PlatformId
    userId: UserId | null
    operation: FlowOperationRequest
    log: FastifyBaseLogger
}

type PublishParams = {
    id: FlowId
    projectId: ProjectId
    platformId: PlatformId
    userId: UserId | null
    requestedStatus: FlowStatus
    skipRequiredActionsCheck: boolean
    log: FastifyBaseLogger
}

type UpdatePublishedVersionIdParams = {
    id: FlowId
    userId: UserId | null
    platformId: PlatformId
    projectId: ProjectId
    flowVersionToPublish: FlowVersion
    requestedStatus: FlowStatus
}

type ChangeStatusParams = {
    id: FlowId
    projectId: ProjectId
    newStatus: FlowStatus
    isRepublish?: boolean
    isTriggerUnchanged?: boolean
}

type WithStatusLockParams<T> = {
    id: FlowId
    log: FastifyBaseLogger
    fn: () => Promise<T>
}

type UpdatePublishedVersionIdResult = {
    publishedFlow: PopulatedFlow
    isRepublish: boolean
    isTriggerUnchanged: boolean
}

type SetPublishedVersionParams = {
    flow: Flow
    lockedVersion: FlowVersion
    entityManager: EntityManager
}

type DeleteParams = EventEmissionParams & {
    id: FlowId
    projectId: ProjectId
    userId?: UserId
    previousFlow?: PopulatedFlow
}

type EventEmissionParams = {
    ip?: string
    emitEvents?: boolean
}


type NewFlow = Omit<Flow, 'created' | 'updated'>

type LockFlowVersionIfNotLockedParams = {
    flowVersion: FlowVersion
    userId: UserId | null
    projectId: ProjectId
    platformId: PlatformId
    entityManager: EntityManager
    log: FastifyBaseLogger
}

type FlowExistsParams = {
    id: FlowId
    projectId: ProjectId
}

type ExistsByProjectAndStatusParams = {
    projectId: ProjectId
    status: FlowStatus
    entityManager: EntityManager
}

type AssertRequiredActionsParams = {
    projectId: ProjectId
    platformId: PlatformId
    flowVersion: FlowVersion
    log: FastifyBaseLogger
}

type UpdateMetadataParams = {
    id: FlowId
    projectId: ProjectId
    metadata: Metadata | null | undefined
}

type UpdateLastModifiedParams = {
    flowId: FlowId
    projectId: ProjectId
    entityManager?: EntityManager
}

/** When the latest version is locked (published snapshot), creates a new draft and imports it. */
async function createNewDraftIfVersionIsPublished({
    flowId,
    projectId,
    platformId,
    userId,
    log,
}: {
    flowId: FlowId
    projectId: ProjectId
    platformId: PlatformId
    userId: UserId | null
    log: FastifyBaseLogger
}): Promise<{ version: FlowVersion, createdNewDraft: boolean }> {
    let lastVersion = await flowVersionService(log).getFlowVersionOrThrow({
        flowId,
        versionId: undefined,
    })
    const createdNewDraft = lastVersion.state === FlowVersionState.LOCKED
    if (lastVersion.state === FlowVersionState.LOCKED) {
        const lockedVersion = lastVersion
        const operations: FlowOperationRequest[] = [{
            type: FlowOperationType.IMPORT_FLOW,
            request: lockedVersion,
        }]
        if (
            lockedVersion.trigger.type === FlowTriggerType.PIECE &&
            !isNil(lockedVersion.trigger.settings.sampleData)
        ) {
            operations.push({
                type: FlowOperationType.UPDATE_SAMPLE_DATA_INFO,
                request: {
                    stepName: lockedVersion.trigger.name,
                    sampleDataSettings: lockedVersion.trigger.settings.sampleData,
                },
            })
        }
        lastVersion = await transaction(async (entityManager) => {
            let draftVersion = await flowVersionService(log).createEmptyVersion({
                flowId,
                displayName: lockedVersion.displayName,
                notes: lockedVersion.notes,
                schemaVersion: lockedVersion.schemaVersion,
                entityManager,
            })
            for (const operation of operations) {
                draftVersion = await flowVersionService(log).applyOperation({
                    userId,
                    projectId,
                    platformId,
                    flowVersion: draftVersion,
                    userOperation: operation,
                    entityManager,
                })
            }
            return draftVersion
        })
    }
    return { version: lastVersion, createdNewDraft }
}

const DRAFT_UPDATE_LOCK_TIMEOUT_SECONDS = 60