Skip to content

AI Agents & MCPs & AI Workflow Automation • (~400 MCP servers for AI agents) • AI Automation / AI Agent with MCPs • AI Workflows & AI Agents • MCPs for AI Agents

12.4 KB
Raw Blame
import { inspect } from 'node:util'
import { ActivepiecesError, ErrorCode, isNil, spreadIfDefined, tryCatch } from '@activepieces/core-utils'
import { onCallService, wideEvent } from '@activepieces/server-utils'
import { BeginExecuteFlowOperation, EngineOperationType, EngineResponseStatus, ExecuteFlowJobData, ExecutionType, FailedStep, FlowActionType, FlowRunStatus, flowStructureUtil, FlowVersion, ResumeExecuteFlowOperation, RunInternalError, RunInternalErrorSource, WorkerJobType } from '@activepieces/shared'
import { system, WorkerSystemProp } from '../../config/configs'
import { workerSettings } from '../../config/worker-settings'
import { FireAndForgetJobResult, JobContext, JobHandler, JobResultKind } from '../types'
import { isSandboxTimeout } from '../utils/sandbox-helpers'

export const executeFlowJob: JobHandler<ExecuteFlowJobData, FireAndForgetJobResult> = {
    jobType: WorkerJobType.EXECUTE_FLOW,
    async execute(ctx: JobContext, data: ExecuteFlowJobData): Promise<FireAndForgetJobResult> {
        const timeoutInSeconds = workerSettings.getSettings().FLOW_TIMEOUT_SECONDS

        const { data: resolved, error: provisionError } = await tryCatch(() =>
            ctx.resolver.resolve({ platformId: data.platformId, internalApiUrl: ctx.internalApiUrl, engineToken: ctx.engineToken, flow: { id: data.flowId, versionId: data.flowVersionId, projectId: data.projectId } }),
        )
        if (provisionError) {
            await reportFlowStatus({ ctx, data, status: FlowRunStatus.INTERNAL_ERROR, internalError: toInternalError(RunInternalErrorSource.WORKER, provisionError) })
            throw provisionError
        }

        // A deleted/disabled flow can't run — the run is correctly marked FAILED, but the job itself must
        // COMPLETE, not return INTERNAL_ERROR. INTERNAL_ERROR fails+retries the job and pages oncall for a
        // user condition (the flow was disabled/removed while jobs were still queued).
        if (resolved.kind === 'flow-not-found') {
            ctx.log.info({ flowVersion: { id: data.flowVersionId } }, 'Flow version not found, skipping')
            await reportFlowStatus({ ctx, data, status: FlowRunStatus.FAILED })
            return { kind: JobResultKind.FIRE_AND_FORGET, status: EngineResponseStatus.OK }
        }

        if (resolved.kind === 'disabled') {
            await reportFlowStatus({ ctx, data, status: FlowRunStatus.FAILED, failedStep: resolved.failedStep })
            return { kind: JobResultKind.FIRE_AND_FORGET, status: EngineResponseStatus.OK }
        }

        // resolved.kind === 'ready' — flowVersion is guaranteed present when flow: is passed to resolve
        if (isNil(resolved.flowVersion)) {
            const error = new ActivepiecesError({ code: ErrorCode.VALIDATION, params: { message: 'flowVersion missing after resolve' } })
            await reportFlowStatus({ ctx, data, status: FlowRunStatus.INTERNAL_ERROR, internalError: toInternalError(RunInternalErrorSource.WORKER, error) })
            throw error
        }
        const flowVersion: FlowVersion = resolved.flowVersion

        if (data.executionType === ExecutionType.RESUME && isNil(data.logsFileId)) {
            const error = new ActivepiecesError({
                code: ErrorCode.RESUME_LOGS_FILE_MISSING,
                params: { runId: data.runId },
            }, 'logsFileId is missing for RESUME operation')
            await reportFlowStatus({ ctx, data, status: FlowRunStatus.INTERNAL_ERROR, internalError: toInternalError(RunInternalErrorSource.WORKER, error) })
            throw error
        }

        try {
            const operation = buildFlowOperation(ctx, data, flowVersion, timeoutInSeconds)
            const result = await ctx.runtime.execute({
                workerIndex: ctx.workerIndex,
                log: ctx.log,
                operationType: EngineOperationType.EXECUTE_FLOW,
                operation,
                timeoutInSeconds,
                provision: resolved.provision,
            })

            // Best-effort latency breakdown for the runs page; reuses the run-log metadata upload
            // with no status so it only merges the timings the engine's own report can't measure.
            await tryCatch(() => ctx.apiClient.uploadRunLog({
                runId: data.runId,
                projectId: data.projectId,
                provisionMs: result.timings.provisionMs,
                bootMs: result.timings.bootMs,
                runMs: result.timings.runMs,
            }))

            if (result.status === EngineResponseStatus.LOG_SIZE_EXCEEDED) {
                await reportFlowStatus({ ctx, data, status: FlowRunStatus.LOG_SIZE_EXCEEDED })
                return { kind: JobResultKind.FIRE_AND_FORGET, status: EngineResponseStatus.LOG_SIZE_EXCEEDED, logs: result.logs }
            }

            if (result.status === EngineResponseStatus.INTERNAL_ERROR) {
                await reportFlowStatus({ ctx, data, status: FlowRunStatus.INTERNAL_ERROR, internalError: {
                    source: RunInternalErrorSource.ENGINE,
                    message: result.error ?? 'Engine reported an internal error without details',
                    occurredAt: new Date().toISOString(),
                } })
                return { kind: JobResultKind.FIRE_AND_FORGET, status: EngineResponseStatus.INTERNAL_ERROR, logs: result.logs }
            }

            return { kind: JobResultKind.FIRE_AND_FORGET, status: EngineResponseStatus.OK, logs: result.logs }
        }
        catch (e) {
            if (isSandboxTimeout(e)) {
                await reportFlowStatus({ ctx, data, status: FlowRunStatus.TIMEOUT })
                return { kind: JobResultKind.FIRE_AND_FORGET, status: EngineResponseStatus.TIMEOUT }
            }
            if (e instanceof ActivepiecesError) {
                if (e.error.code === ErrorCode.SANDBOX_MEMORY_ISSUE) {
                    await reportFlowStatus({ ctx, data, status: FlowRunStatus.MEMORY_LIMIT_EXCEEDED })
                    return { kind: JobResultKind.FIRE_AND_FORGET, status: EngineResponseStatus.MEMORY_ISSUE }
                }
                if (e.error.code === ErrorCode.SANDBOX_LOG_SIZE_EXCEEDED) {
                    await reportFlowStatus({ ctx, data, status: FlowRunStatus.LOG_SIZE_EXCEEDED })
                    return { kind: JobResultKind.FIRE_AND_FORGET, status: EngineResponseStatus.LOG_SIZE_EXCEEDED }
                }
                if (e.error.code === ErrorCode.PIECE_BUNDLE_NOT_AVAILABLE) {
                    const { pieceName, pieceVersion } = e.error.params
                    const owner = findStepOwningPiece({ flowVersion, pieceName, pieceVersion })
                    await reportFlowStatus({
                        ctx,
                        data,
                        status: FlowRunStatus.FAILED,
                        failedStep: {
                            name: owner.name,
                            displayName: owner.displayName,
                            message: `Piece "${pieceName}@${pieceVersion}" is unavailable in this environment (bundle not accessible). Confirm the piece and version are installed and visible in this project, or update the step to a supported version.`,
                        },
                    })
                    return { kind: JobResultKind.FIRE_AND_FORGET, status: EngineResponseStatus.OK }
                }
            }
            await reportFlowStatus({ ctx, data, status: FlowRunStatus.INTERNAL_ERROR, internalError: toInternalError(RunInternalErrorSource.WORKER, e) })
            throw e
        }
    },
}

function buildFlowOperation(
    ctx: JobContext,
    data: ExecuteFlowJobData,
    flowVersion: FlowVersion,
    timeoutInSeconds: number,
): BeginExecuteFlowOperation | ResumeExecuteFlowOperation {
    const base = {
        flowVersion,
        flowRunId: data.runId,
        projectId: data.projectId,
        workerHandlerId: data.workerHandlerId ?? null,
        runEnvironment: data.environment,
        httpRequestId: data.httpRequestId ?? null,
        streamStepProgress: data.streamStepProgress,
        stepNameToTest: data.stepNameToTest ?? null,
        logsFileId: data.logsFileId,
        timeoutInSeconds,
        platformId: data.platformId,
        engineToken: ctx.engineToken,
        internalApiUrl: ctx.internalApiUrl,
        publicApiUrl: ctx.publicApiUrl,
    }

    if (data.executionType === ExecutionType.RESUME) {
        return {
            ...base,
            executionType: ExecutionType.RESUME,
            resumePayload: data.payload,
            resumeReason: data.resumeReason,
        }
    }

    return {
        ...base,
        executionType: ExecutionType.BEGIN,
        triggerPayload: data.payload,
        executeTrigger: data.executeTrigger ?? false,
        entryStepName: data.entryStepName,
        sampleData: data.sampleData,
    }
}

function findStepOwningPiece({ flowVersion, pieceName, pieceVersion }: FindStepOwningPieceParams): { name: string, displayName: string } {
    const match = flowStructureUtil.getAllSteps(flowVersion.trigger).find(step => {
        if (step.type !== FlowActionType.PIECE) return false
        return step.settings.pieceName === pieceName && step.settings.pieceVersion === pieceVersion
    })
    if (match) return { name: match.name, displayName: match.displayName }
    return { name: flowVersion.trigger.name, displayName: flowVersion.trigger.displayName }
}

function toInternalError(source: RunInternalErrorSource, error: unknown): RunInternalError {
    const isApError = error instanceof ActivepiecesError
    const base = error instanceof Error
        ? [error.name, error.message, error.stack].filter(Boolean).join('\n')
        : inspect(error, { depth: 1 })
    return {
        source,
        message: base,
        code: isApError ? error.error.code : undefined,
        occurredAt: new Date().toISOString(),
    }
}

async function reportFlowStatus({ ctx, data, status, internalError, failedStep }: ReportFlowStatusParams): Promise<void> {
    const willRetry = status === FlowRunStatus.INTERNAL_ERROR && !ctx.lastAttempt
    recordRunOutcomeOnWideEvent({ status, internalError, willRetry, environment: data.environment, failedStepName: failedStep?.name })
    // A status report has no log file of its own; carry logsFileId only for an internalError the server may
    // persist into one (see uploadRunLog). Sending it on a plain status report would dangle flow_run.logsFileId.
    await ctx.apiClient.uploadRunLog({
        runId: data.runId,
        status,
        projectId: data.projectId,
        streamStepProgress: data.streamStepProgress,
        finishTime: new Date().toISOString(),
        ...(isNil(internalError) ? {} : { logsFileId: data.logsFileId }),
        internalError,
        failedStep,
        willRetry,
        ...spreadIfDefined('workerHandlerId', data.workerHandlerId ?? undefined),
        ...spreadIfDefined('httpRequestId', data.httpRequestId),
    })

    if (status === FlowRunStatus.INTERNAL_ERROR && isDedicatedWorker()) {
        onCallService(ctx.log, workerSettings.getSettings().PAGE_ONCALL_WEBHOOK).page({
            code: ErrorCode.ENGINE_OPERATION_FAILURE,
            message: `Flow run ${data.runId} ended with INTERNAL_ERROR`,
            params: { runId: data.runId, flowId: data.flowId, projectId: data.projectId, willRetry },
        }).catch((e) => ctx.log.error({ flowRun: { id: data.runId }, error: inspect(e) }, 'Failed to send on-call page for INTERNAL_ERROR'))
    }
}

function recordRunOutcomeOnWideEvent({ status, internalError, willRetry, environment, failedStepName }: RecordRunOutcomeParams): void {
    wideEvent.set({ flowRun: { status, willRetry, environment } })
    if (!isNil(failedStepName)) {
        wideEvent.set({ step: { name: failedStepName } })
    }
    if (isNil(internalError)) {
        return
    }
    wideEvent.set({ flowRun: { internalErrorSource: internalError.source, ...spreadIfDefined('internalErrorCode', internalError.code) } })
    const error = new Error(internalError.message)
    error.stack = internalError.message
    wideEvent.error(error)
}

function isDedicatedWorker(): boolean {
    return !isNil(system.get(WorkerSystemProp.WORKER_GROUP_ID))
}

type ReportFlowStatusParams = {
    ctx: JobContext
    data: ExecuteFlowJobData
    status: FlowRunStatus
    internalError?: RunInternalError
    failedStep?: FailedStep
}

type RecordRunOutcomeParams = {
    status: FlowRunStatus
    internalError?: RunInternalError
    willRetry: boolean
    environment: ExecuteFlowJobData['environment']
    failedStepName?: string
}

type FindStepOwningPieceParams = {
    flowVersion: FlowVersion
    pieceName: string
    pieceVersion: string
}