Skip to content

AI Agents & MCPs & AI Workflow Automation • (~400 MCP servers for AI agents) • AI Automation / AI Agent with MCPs • AI Workflows & AI Agents • MCPs for AI Agents

10.7 KB
Raw Blame
import { ApId, isNil, tryCatch } from '@activepieces/core-utils'
import { apDayjsDuration, memoryLock } from '@activepieces/server-utils'
import { EventDestinationJobData, ExecuteAgentRunJobData, ExecuteAiJobData, ExecuteFlowJobData, ExecutePersonalizationResearchJobData, getDefaultJobPriority, JOB_PRIORITY, JobData, PollingJobData, RenewWebhookJobData, ScheduleOptions, TriggerSourceScheduleType, UserInteractionJobData, WebhookJobData, WorkerJobType } from '@activepieces/shared'
import { Job, Queue } from 'bullmq'
import { FastifyBaseLogger } from 'fastify'
import { redisConnections } from '../../database/redis-connections'
import { system } from '../../helper/system/system'
import { AppSystemProp } from '../../helper/system/system-props'
import { QueueName } from '../job'
import { workerGroupRouting } from '../worker-groups/worker-group-routing'

const EIGHT_MINUTES_IN_MILLISECONDS = apDayjsDuration(8, 'minute').asMilliseconds()
const REDIS_FAILED_JOB_RETENTION_DAYS = apDayjsDuration(system.getNumberOrThrow(AppSystemProp.REDIS_FAILED_JOB_RETENTION_DAYS), 'day').asSeconds()
const REDIS_FAILED_JOB_RETRY_COUNT = system.getNumberOrThrow(AppSystemProp.REDIS_FAILED_JOB_RETENTION_MAX_COUNT)

const dedicatedWorkersQueues = new Map<string, Queue>()

export const jobQueue = (log: FastifyBaseLogger) => ({
    async init(): Promise<void> {
        await ensureQueueExists({ log, queueName: QueueName.WORKER_JOBS })
        log.info('[jobQueue#init] Dynamic queue system initialized')
    },
    async add(params: AddJobParams<JobType>): Promise<Job | null> {
        const { type, data } = params

        const platformId = data.platformId
        const projectId = 'projectId' in data ? data.projectId : null
        const flowId = 'flowId' in data ? data.flowId : null
        const queueName = await workerGroupRouting.resolveQueueForJob({ platformId, projectId, flowId, jobType: data.jobType, log })
        const queue = await ensureQueueExists({ log, queueName })

        switch (type) {
            case JobType.REPEATING: {
                const { scheduleOptions } = params
                await queue.upsertJobScheduler(data.flowVersionId, scheduleOptions.type === TriggerSourceScheduleType.INTERVAL ? {
                    every: scheduleOptions.intervalMs,
                    startDate: Date.now() + scheduleOptions.intervalMs,
                } : {
                    pattern: scheduleOptions.cronExpression,
                    tz: scheduleOptions.timezone,
                }, {
                    name: data.flowVersionId,
                    data,
                    opts: {
                        priority: JOB_PRIORITY[getDefaultJobPriority(data)],
                    },
                })
                return null
            }
            case JobType.ONE_TIME: {
                return queue.add(params.id, data, {
                    priority: JOB_PRIORITY[getDefaultJobPriority(data)],
                    delay: params.delay,
                    jobId: params.id,
                    ...(isNil(params.retryBackoffMs) ? {} : { backoff: { type: 'exponential', delay: params.retryBackoffMs } }),
                    ...(data.jobType === WorkerJobType.EVENT_DESTINATION ? { removeOnFail: true } : {}),
                    ...([WorkerJobType.EXECUTE_AGENT_RUN, WorkerJobType.EXECUTE_AI].includes(data.jobType) ? { attempts: 1 } : {}),
                    ...isUserInteractionJob(data.jobType) ? {
                        attempts: 1,
                        removeOnComplete: { age: 300 },
                        removeOnFail: true,
                    } : {},
                })
            }
        }
    },

    async removeRepeatingJob({ flowVersionId }: { flowVersionId: ApId }): Promise<void> {
        const allQueues = [...dedicatedWorkersQueues.values()].filter(queue => !isNil(queue))

        await Promise.allSettled(
            allQueues.map(queue => queue.removeJobScheduler(flowVersionId)),
        )

        log.info({
            flowVersion: { id: flowVersionId },
        }, '[jobQueue#removeRepeatingJob] removed jobs from all queues')
    },

    async removeOneTimeJob({ jobId, platformId, projectId, flowId, jobType }: RemoveOneTimeJobParams): Promise<void> {
        const queueName = await workerGroupRouting.resolveQueueForJob({ platformId, projectId, flowId, jobType, log })
        const queue = await ensureQueueExists({ log, queueName })
        const job = await queue.getJob(jobId)
        if (!isNil(job)) {
            await job.remove()
            log.info({
                job: { id: jobId },
                queueName,
            }, '[jobQueue#removeOneTimeJob] removed job from queue')
            return
        }
        log.info({
            job: { id: jobId },
            queueName,
        }, '[jobQueue#removeOneTimeJob] job not found in queue')
    },

    async cancelAndReportNeverStarted({ jobId, platformId, projectId, jobType }: RemoveOneTimeJobParams): Promise<boolean> {
        const queueName = await workerGroupRouting.resolveQueueForJob({ platformId, projectId, jobType, log })
        const queue = await ensureQueueExists({ log, queueName })
        const job = await queue.getJob(jobId)
        if (isNil(job)) {
            log.info({ job: { id: jobId }, queueName }, '[jobQueue#cancelAndReportNeverStarted] job not found')
            return false
        }
        const everDequeued = !isNil(job.processedOn)
        const { error } = await tryCatch(() => job.remove())
        if (error) {
            log.info({ job: { id: jobId, everDequeued }, queueName, error: String(error) }, '[jobQueue#cancelAndReportNeverStarted] a worker holds the job')
            return false
        }
        log.info({ job: { id: jobId, everDequeued }, queueName }, '[jobQueue#cancelAndReportNeverStarted] removed the abandoned job')
        return !everDequeued
    },

    async getOrCreateQueue({ queueName }: { queueName: string }): Promise<Queue> {
        return ensureQueueExists({ log, queueName })
    },

    getAllQueues(): Queue[] {
        const queues = [...dedicatedWorkersQueues.values()].filter(queue => !isNil(queue))
        return queues
    },

    getSharedQueue(): Queue {
        const queue = dedicatedWorkersQueues.get(QueueName.WORKER_JOBS)
        if (isNil(queue)) {
            throw Error('Shared queue not initialized')
        }
        return queue
    },
    async removeAllFlowRunJobs({ flowRunId, platformId, projectId, flowId }: RemoveAllFlowRunJobsParams): Promise<void> {
        const queueName = await workerGroupRouting.resolveQueueForJob({ platformId, projectId, flowId, jobType: WorkerJobType.EXECUTE_FLOW, log })
        const queue = await ensureQueueExists({ log, queueName })
        const allJobs = await queue.getJobs(['waiting', 'delayed'])
        const matching = allJobs.filter((j) => j.id?.startsWith(flowRunId))
        await Promise.allSettled(matching.map((j) => j.remove()))
        log.info({ flowRun: { id: flowRunId }, queueName, removedIds: matching.map((j) => j.id) }, '[jobQueue#removeAllFlowRunJobs] done')
    },

    async close(): Promise<void> {
        log.info('[jobQueue#close] Closing job queue')
        const allQueues = [...dedicatedWorkersQueues.values()].filter(queue => !isNil(queue))
        await Promise.allSettled(
            allQueues.map(queue => queue.close()),
        )
    },
})

async function ensureQueueExists({ log, queueName }: { log: FastifyBaseLogger, queueName: string }): Promise<Queue> {
    const existingQueue = dedicatedWorkersQueues.get(queueName)
    if (!isNil(existingQueue)) {
        return existingQueue
    }
    return memoryLock.runExclusive({
        key: `ensure_queue_exists_${queueName}`,
        fn: async () => {
            const existingQueue = dedicatedWorkersQueues.get(queueName)
            if (!isNil(existingQueue)) {
                return existingQueue
            }

            const queue = new Queue(queueName, {
                connection: await redisConnections.create(),
                defaultJobOptions: {
                    attempts: 2,
                    backoff: {
                        type: 'exponential',
                        delay: EIGHT_MINUTES_IN_MILLISECONDS,
                    },
                    removeOnComplete: true,
                    removeOnFail: {
                        age: REDIS_FAILED_JOB_RETENTION_DAYS,
                        count: REDIS_FAILED_JOB_RETRY_COUNT,
                    },
                },
            })

            await queue.removeGlobalConcurrency()
            await queue.waitUntilReady()

            dedicatedWorkersQueues.set(queueName, queue)

            log.info({
                queueName,
            }, '[jobQueue#ensureQueueExists] Queue created')
            return queue
        },
    })
}

const USER_INTERACTION_JOB_TYPES = new Set([
    WorkerJobType.EXECUTE_PROPERTY,
    WorkerJobType.EXECUTE_VALIDATION,
    WorkerJobType.EXECUTE_RESOLVE_CONNECTION_IDENTIFIER,
    WorkerJobType.EXECUTE_TRIGGER_HOOK,
    WorkerJobType.EXECUTE_EXTRACT_PIECE_INFORMATION,
    WorkerJobType.EXECUTE_TOKEN_REFRESH,
    WorkerJobType.EXECUTE_ACTION,
])

export function isUserInteractionJob(jobType: WorkerJobType): boolean {
    return USER_INTERACTION_JOB_TYPES.has(jobType)
}

export function isUserInteractionJobData(jobData: JobData): jobData is UserInteractionJobData {
    return USER_INTERACTION_JOB_TYPES.has(jobData.jobType)
}

export function callerWaitingForResponse(jobData: JobData): { webserverId: string, requestId: string } | null {
    if (isUserInteractionJobData(jobData)) {
        return { webserverId: jobData.webserverId, requestId: jobData.requestId }
    }
    if (jobData.jobType === WorkerJobType.EXECUTE_AI && !isNil(jobData.webserverId)) {
        return { webserverId: jobData.webserverId, requestId: jobData.requestId }
    }
    return null
}

export enum JobType {
    REPEATING = 'repeating',
    ONE_TIME = 'one_time',
}

type RemoveOneTimeJobParams = {
    jobId: ApId
    platformId: string | null
    projectId?: string | null
    flowId?: string | null
    jobType?: WorkerJobType
}

type RemoveAllFlowRunJobsParams = {
    flowRunId: string
    platformId: string | null
    projectId?: string | null
    flowId?: string | null
}

type BaseAddParams<JD extends Omit<JobData, 'engineToken'>, JT extends JobType> = {
    id: ApId
    data: JD
    type: JT
    delay?: number
}
type RepeatingJobAddParams = BaseAddParams<PollingJobData | RenewWebhookJobData, JobType.REPEATING> & {
    scheduleOptions: ScheduleOptions
}
type OneTimeJobAddParams = BaseAddParams<ExecuteFlowJobData | WebhookJobData | UserInteractionJobData | EventDestinationJobData | ExecuteAgentRunJobData | ExecuteAiJobData | ExecutePersonalizationResearchJobData, JobType.ONE_TIME> & {
    retryBackoffMs?: number
}

export type AddJobParams<type extends JobType> = type extends JobType.REPEATING ? RepeatingJobAddParams : OneTimeJobAddParams