Skip to content

AI Agents & MCPs & AI Workflow Automation • (~400 MCP servers for AI agents) • AI Automation / AI Agent with MCPs • AI Workflows & AI Agents • MCPs for AI Agents

15.1 KB
Raw Blame
import { ActivepiecesError, apId, ErrorCode, isNil } from '@activepieces/core-utils'
import { FlowRun, FlowRunStatus, isFailedState, PauseType } from '@activepieces/shared'
import dayjs from 'dayjs'
import { FastifyBaseLogger } from 'fastify'
import { EntityManager, FindOptionsWhere, In, Not } from 'typeorm'
import { z } from 'zod'
import { repoFactory } from '../core/db/repo-factory'
import { transaction } from '../core/db/transaction'
import { distributedStore } from '../database/redis-connections'
import { flowRunRepo } from '../flows/flow-run/flow-run-service'
import { system } from '../helper/system/system'
import { AppSystemProp } from '../helper/system/system-props'
import { barrierSourceKey } from './barrier-queue-factory'
import { WaitpointEntity } from './waitpoint-entity'
import { waitpointTimeoutJob } from './waitpoint-timeout-job'
import { CompleteParams, CompleteResult, CreateForPauseParams, CreateForPauseResult, FindPendingByVersionParams, HandleResumeSignalParams, Waitpoint, WaitpointStatus } from './waitpoint-types'

const waitpointRepo = repoFactory(WaitpointEntity)

export const waitpointService = (log: FastifyBaseLogger) => ({
    async createForPause(params: CreateForPauseParams): Promise<CreateForPauseResult> {
        const preCompleted = await waitpointRepo().findOneBy({
            flowRunId: params.flowRunId,
            projectId: params.projectId,
            stepName: params.stepName,
            status: WaitpointStatus.COMPLETED,
        })
        if (!isNil(preCompleted)) {
            log.info({ flowRun: { id: params.flowRunId }, step: { name: params.stepName }, existingStatus: preCompleted.status }, '[waitpointService#createForPause] Waitpoint already pre-completed for this step')
            return { inserted: false, waitpoint: preCompleted }
        }

        const flowRun = await flowRunRepo().findOneByOrFail({ id: params.flowRunId, projectId: params.projectId })
        const resumeDateTime = clampWaitpointResumeDeadline({ requested: params.resumeDateTime, type: params.type, flowRunCreated: flowRun.created, flowRunId: flowRun.id })

        const id = apId()
        await waitpointRepo()
            .createQueryBuilder()
            .insert()
            .into('waitpoint')
            .values({
                id,
                flowRunId: params.flowRunId,
                projectId: params.projectId,
                stepName: params.stepName,
                type: params.type,
                version: params.version,
                status: WaitpointStatus.PENDING,
                resumeDateTime: resumeDateTime ?? null,
                responseToSend: params.responseToSend ?? null,
                workerHandlerId: params.workerHandlerId ?? null,
                httpRequestId: params.httpRequestId ?? null,
                resumePayload: null,
                sealed: false,
                policy: null,
            })
            .orIgnore()
            .execute()

        const waitpoint = await waitpointRepo().findOneByOrFail({ flowRunId: params.flowRunId, projectId: params.projectId, stepName: params.stepName })
        const inserted = waitpoint.id === id
        if (inserted) {
            log.info({ flowRun: { id: params.flowRunId }, waitpoint: { id } }, '[waitpointService#createForPause] Waitpoint created')
        }
        else {
            log.info({ flowRun: { id: params.flowRunId }, existingStatus: waitpoint.status }, '[waitpointService#createForPause] Waitpoint already exists')
        }
        if (isNil(resumeDateTime)) {
            return { inserted, waitpoint }
        }
        const { status } = await waitpointTimeoutJob.schedule({
            flowRunId: params.flowRunId,
            projectId: params.projectId,
            waitpointId: waitpoint.id,
            resumeDateTime,
            log,
        })
        const revivesDeadLetteredDeadline = status === 'retried' || !isNil(waitpoint.deadLetteredAt)
        if (!revivesDeadLetteredDeadline) {
            return { inserted, waitpoint }
        }
        await waitpointRepo().update({ id: waitpoint.id, projectId: params.projectId }, { deadLetteredAt: null })
        if (!isNil(waitpoint.deadLetteredAt)) {
            log.info({ waitpoint: { id: waitpoint.id }, flowRun: { id: params.flowRunId } }, '[waitpointService#createForPause] Re-armed a dead-lettered deadline, so the sweep covers it again')
        }
        return { inserted, waitpoint: { ...waitpoint, deadLetteredAt: null } }
    },

    async complete(params: CompleteParams): Promise<CompleteResult> {
        return transaction(async (entityManager) => {
            const repo = waitpointRepo(entityManager)

            const pending = await repo
                .createQueryBuilder('waitpoint')
                .setLock('pessimistic_write')
                .where({ id: params.waitpointId, flowRunId: params.flowRunId, projectId: params.projectId, status: WaitpointStatus.PENDING })
                .getOne()

            if (isNil(pending)) {
                log.info({ flowRun: { id: params.flowRunId }, waitpoint: { id: params.waitpointId } }, '[waitpointService#complete] No pending waitpoint matches; dropping stale resume signal')
                return { completedExisting: false, waitpoint: null }
            }

            const updated: Waitpoint = {
                ...pending,
                status: WaitpointStatus.COMPLETED,
                resumePayload: params.resumePayload,
                workerHandlerId: params.workerHandlerId ?? pending.workerHandlerId,
            }
            await repo.save(updated)
            log.info({ flowRun: { id: params.flowRunId } }, '[waitpointService#complete] Completed existing PENDING waitpoint')
            return { completedExisting: true, waitpoint: updated }
        })
    },

    async handleResumeSignal(params: HandleResumeSignalParams): Promise<boolean> {
        const { flowRunId, waitpointId, flowRunStatus, projectId, resumePayload, workerHandlerId, onReady } = params

        if (flowRunStatus === FlowRunStatus.PAUSED) {
            const delivery = await transaction<PausedDelivery>(async (entityManager) => {
                const repo = waitpointRepo(entityManager)
                const found = await repo
                    .createQueryBuilder('waitpoint')
                    .setLock('pessimistic_write')
                    .where({ id: waitpointId, flowRunId, projectId })
                    .getOne()
                if (isNil(found)) {
                    return 'unknown-waitpoint'
                }
                if (found.status === WaitpointStatus.CONSUMED) {
                    return 'already-delivered'
                }
                await onReady(found)
                await this.consume({ waitpoint: found, entityManager })
                return 'dispatched'
            })
            if (delivery === 'unknown-waitpoint') {
                log.info({ flowRun: { id: flowRunId }, waitpoint: { id: waitpointId } }, '[waitpointService#handleResumeSignal] Stale waitpointId, ignoring')
                return false
            }
            if (delivery === 'already-delivered') {
                log.info({ flowRun: { id: flowRunId }, waitpoint: { id: waitpointId } }, '[waitpointService#handleResumeSignal] Waitpoint was already delivered, ignoring')
                return false
            }
            await waitpointTimeoutJob.remove({ waitpointId, flowRunId, log })
            log.info({ flowRun: { id: flowRunId }, waitpoint: { id: waitpointId } }, '[waitpointService#handleResumeSignal] Resume triggered')
            return true
        }

        if (flowRunStatus === FlowRunStatus.RUNNING || flowRunStatus === FlowRunStatus.QUEUED) {
            const { completedExisting } = await this.complete({ flowRunId, projectId, waitpointId, resumePayload, workerHandlerId })
            if (!completedExisting) {
                log.info({ flowRun: { id: flowRunId }, waitpoint: { id: waitpointId } }, '[waitpointService#handleResumeSignal] Stale resume signal during RUNNING/QUEUED, ignoring')
                return false
            }
            log.info({ flowRun: { id: flowRunId } }, '[waitpointService#handleResumeSignal] Marked PENDING waitpoint COMPLETED while flow still RUNNING/QUEUED; runsMetadataQueue will trigger resume on PAUSED upload')
            return true
        }

        log.info({ flowRun: { id: flowRunId }, flowRunStatus }, '[waitpointService#handleResumeSignal] Flow run not in resumable state, ignoring')
        return false
    },

    async findPendingByVersion({ flowRunId, projectId, version }: FindPendingByVersionParams): Promise<Waitpoint | null> {
        return waitpointRepo().findOne({
            where: { flowRunId, projectId, status: WaitpointStatus.PENDING, version },
        })
    },

    async findByIdAndFlowRunId({ waitpointId, flowRunId, projectId }: FindByIdAndFlowRunIdParams): Promise<Waitpoint | null> {
        return waitpointRepo().findOneBy({ id: waitpointId, flowRunId, projectId })
    },

    async findUndeliveredCompletedWaitpoint({ flowRunId, projectId }: FindUndeliveredCompletedWaitpointParams): Promise<Waitpoint | null> {
        const barrierPending = await this.hasPendingBarrier({ flowRunId, projectId })
        if (barrierPending) {
            return null
        }
        return waitpointRepo().findOne({
            where: { flowRunId, projectId, status: WaitpointStatus.COMPLETED },
            order: { created: 'DESC' },
        })
    },

    async hasPendingBarrier({ flowRunId, projectId }: HasPendingBarrierParams): Promise<boolean> {
        return waitpointRepo().existsBy({ flowRunId, projectId, type: PauseType.BARRIER, status: WaitpointStatus.PENDING })
    },

    async hasBarrier({ flowRunId, projectId }: HasBarrierParams): Promise<boolean> {
        return waitpointRepo().existsBy({ flowRunId, projectId, type: PauseType.BARRIER })
    },

    async consume({ waitpoint, entityManager }: ConsumeParams): Promise<void> {
        const repo = waitpointRepo(entityManager)
        if (waitpoint.type === PauseType.BARRIER) {
            await repo.save({ ...waitpoint, status: WaitpointStatus.CONSUMED })
            log.info({ waitpoint: { id: waitpoint.id }, flowRun: { id: waitpoint.flowRunId } }, '[waitpointService#consume] Barrier kept as CONSUMED so the run stays barrier-owned until it ends')
            return
        }
        await repo.delete({ id: waitpoint.id, projectId: waitpoint.projectId })
        log.info({ waitpoint: { id: waitpoint.id }, flowRun: { id: waitpoint.flowRunId } }, '[waitpointService#consume] Waitpoint consumed and deleted')
    },

    async findSubflowWaitpoint({ flowRunId, projectId }: FindSubflowWaitpointParams): Promise<Waitpoint | null> {
        const pending = await waitpointRepo().findOne({
            where: { flowRunId, projectId, status: WaitpointStatus.PENDING, type: Not(PauseType.BARRIER) },
            order: { created: 'DESC' },
        })
        if (!isNil(pending)) {
            return pending
        }
        return waitpointRepo().findOne({
            where: { flowRunId, projectId, type: Not(PauseType.BARRIER) },
            order: { created: 'DESC' },
        })
    },

    async deleteByFlowRunId({ flowRunId, projectId }: DeleteByFlowRunIdParams): Promise<void> {
        await deleteWaitpointsWithTimers({ where: { flowRunId, projectId }, flowRunId, log })
        log.info({ flowRun: { id: flowRunId } }, '[waitpointService#deleteByFlowRunId] Waitpoint deleted')
    },

    async deleteForFinishedRun({ flowRun }: DeleteForFinishedRunParams): Promise<void> {
        const failedStepName = flowRun.failedStep?.name
        if (!isFailedState(flowRun.status) || isNil(failedStepName)) {
            await this.deleteByFlowRunId({ flowRunId: flowRun.id, projectId: flowRun.projectId })
            return
        }
        const waitpoints = await waitpointRepo().find({
            select: { id: true, stepName: true, type: true },
            where: { flowRunId: flowRun.id, projectId: flowRun.projectId },
        })
        const discardedIds = waitpoints
            .filter((waitpoint) => waitpoint.type === PauseType.BARRIER || stepNameOfWaitpointKey({ waitpointKey: waitpoint.stepName }) === failedStepName)
            .map((waitpoint) => waitpoint.id)
        if (discardedIds.length > 0) {
            await deleteWaitpointsWithTimers({ where: { id: In(discardedIds), flowRunId: flowRun.id, projectId: flowRun.projectId }, flowRunId: flowRun.id, log })
        }
        log.info({ flowRun: { id: flowRun.id } }, '[waitpointService#deleteForFinishedRun] Deleted the failed step\'s waitpoints and any barrier, kept the waitpoints of the steps that finished')
    },
})

async function deleteWaitpointsWithTimers({ where, flowRunId, log }: DeleteWaitpointsWithTimersParams): Promise<void> {
    const result = await waitpointRepo()
        .createQueryBuilder()
        .delete()
        .where(where)
        .returning(['id', 'resumeDateTime', 'type'])
        .execute()
    const deleted = DeletedWaitpointRows.parse(result.raw ?? [])
    const timed = deleted.filter((waitpoint) => !isNil(waitpoint.resumeDateTime))
    await Promise.all(timed.map((waitpoint) => waitpointTimeoutJob.remove({ waitpointId: waitpoint.id, flowRunId, log })))
    const barrierSourceKeys = deleted.filter((waitpoint) => waitpoint.type === PauseType.BARRIER).map((waitpoint) => barrierSourceKey(waitpoint.id))
    await distributedStore.delete(barrierSourceKeys)
}

function stepNameOfWaitpointKey({ waitpointKey }: { waitpointKey: string }): string {
    return waitpointKey.split('/').at(-1) ?? waitpointKey
}

function clampWaitpointResumeDeadline({ requested, type, flowRunCreated, flowRunId }: ClampWaitpointResumeDeadlineParams): string | undefined {
    const pauseTimeoutDays = system.getNumberOrThrow(AppSystemProp.PAUSED_FLOW_TIMEOUT_DAYS)
    const runDeadline = dayjs(flowRunCreated).add(pauseTimeoutDays, 'day')
    if (isNil(requested)) {
        return type === PauseType.WEBHOOK ? runDeadline.toISOString() : undefined
    }
    if (dayjs(requested).isAfter(runDeadline)) {
        throw new ActivepiecesError({
            code: ErrorCode.PAUSED_FLOW_TIMEOUT_EXCEEDED,
            params: { pauseTimeoutDays, flowRunId },
        })
    }
    return requested
}

const DeletedWaitpointRows = z.array(z.object({
    id: z.string(),
    resumeDateTime: z.coerce.date().nullable(),
    type: z.enum(PauseType),
}))

type PausedDelivery = 'dispatched' | 'already-delivered' | 'unknown-waitpoint'

type ClampWaitpointResumeDeadlineParams = {
    requested: string | undefined
    type: `${PauseType}`
    flowRunCreated: string
    flowRunId: string
}

type DeleteByFlowRunIdParams = {
    flowRunId: string
    projectId: string
}

type DeleteForFinishedRunParams = {
    flowRun: FlowRun
}

type DeleteWaitpointsWithTimersParams = {
    where: FindOptionsWhere<Waitpoint>
    flowRunId: string
    log: FastifyBaseLogger
}

type HasPendingBarrierParams = {
    flowRunId: string
    projectId: string
}

type HasBarrierParams = {
    flowRunId: string
    projectId: string
}

type ConsumeParams = {
    waitpoint: Waitpoint
    entityManager?: EntityManager
}

type FindByIdAndFlowRunIdParams = {
    waitpointId: string
    flowRunId: string
    projectId: string
}

type FindUndeliveredCompletedWaitpointParams = {
    flowRunId: string
    projectId: string
}

type FindSubflowWaitpointParams = {
    flowRunId: string
    projectId: string
}