AI Agents & MCPs & AI Workflow Automation • (~400 MCP servers for AI agents) • AI Automation / AI Agent with MCPs • AI Workflows & AI Agents • MCPs for AI Agents
TypeScript ai-agentai-agent-toolsai-agentsai-agents-frameworkmcpmcp-servermcp-toolsmcpsn8n-alternativeno-code-automationworkflowworkflow-automationworkflows
import { ApId, isNil, tryCatch } from '@activepieces/core-utils'
import { apDayjsDuration, memoryLock } from '@activepieces/server-utils'
import { EventDestinationJobData, ExecuteAgentRunJobData, ExecuteAiJobData, ExecuteFlowJobData, ExecutePersonalizationResearchJobData, getDefaultJobPriority, JOB_PRIORITY, JobData, PollingJobData, RenewWebhookJobData, ScheduleOptions, TriggerSourceScheduleType, UserInteractionJobData, WebhookJobData, WorkerJobType } from '@activepieces/shared'
import { Job, Queue } from 'bullmq'
import { FastifyBaseLogger } from 'fastify'
import { redisConnections } from '../../database/redis-connections'
import { system } from '../../helper/system/system'
import { AppSystemProp } from '../../helper/system/system-props'
import { QueueName } from '../job'
import { workerGroupRouting } from '../worker-groups/worker-group-routing'
const EIGHT_MINUTES_IN_MILLISECONDS = apDayjsDuration(8, 'minute').asMilliseconds()
const REDIS_FAILED_JOB_RETENTION_DAYS = apDayjsDuration(system.getNumberOrThrow(AppSystemProp.REDIS_FAILED_JOB_RETENTION_DAYS), 'day').asSeconds()
const REDIS_FAILED_JOB_RETRY_COUNT = system.getNumberOrThrow(AppSystemProp.REDIS_FAILED_JOB_RETENTION_MAX_COUNT)
const dedicatedWorkersQueues = new Map<string, Queue>()
export const jobQueue = (log: FastifyBaseLogger) => ({
async init(): Promise<void> {
await ensureQueueExists({ log, queueName: QueueName.WORKER_JOBS })
log.info('[jobQueue#init] Dynamic queue system initialized')
},
async add(params: AddJobParams<JobType>): Promise<Job | null> {
const { type, data } = params
const platformId = data.platformId
const projectId = 'projectId' in data ? data.projectId : null
const flowId = 'flowId' in data ? data.flowId : null
const queueName = await workerGroupRouting.resolveQueueForJob({ platformId, projectId, flowId, jobType: data.jobType, log })
const queue = await ensureQueueExists({ log, queueName })
switch (type) {
case JobType.REPEATING: {
const { scheduleOptions } = params
await queue.upsertJobScheduler(data.flowVersionId, scheduleOptions.type === TriggerSourceScheduleType.INTERVAL ? {
every: scheduleOptions.intervalMs,
startDate: Date.now() + scheduleOptions.intervalMs,
} : {
pattern: scheduleOptions.cronExpression,
tz: scheduleOptions.timezone,
}, {
name: data.flowVersionId,
data,
opts: {
priority: JOB_PRIORITY[getDefaultJobPriority(data)],
},
})
return null
}
case JobType.ONE_TIME: {
return queue.add(params.id, data, {
priority: JOB_PRIORITY[getDefaultJobPriority(data)],
delay: params.delay,
jobId: params.id,
...(isNil(params.retryBackoffMs) ? {} : { backoff: { type: 'exponential', delay: params.retryBackoffMs } }),
...(data.jobType === WorkerJobType.EVENT_DESTINATION ? { removeOnFail: true } : {}),
...([WorkerJobType.EXECUTE_AGENT_RUN, WorkerJobType.EXECUTE_AI].includes(data.jobType) ? { attempts: 1 } : {}),
...isUserInteractionJob(data.jobType) ? {
attempts: 1,
removeOnComplete: { age: 300 },
removeOnFail: true,
} : {},
})
}
}
},
async removeRepeatingJob({ flowVersionId }: { flowVersionId: ApId }): Promise<void> {
const allQueues = [...dedicatedWorkersQueues.values()].filter(queue => !isNil(queue))
await Promise.allSettled(
allQueues.map(queue => queue.removeJobScheduler(flowVersionId)),
)
log.info({
flowVersion: { id: flowVersionId },
}, '[jobQueue#removeRepeatingJob] removed jobs from all queues')
},
async removeOneTimeJob({ jobId, platformId, projectId, flowId, jobType }: RemoveOneTimeJobParams): Promise<void> {
const queueName = await workerGroupRouting.resolveQueueForJob({ platformId, projectId, flowId, jobType, log })
const queue = await ensureQueueExists({ log, queueName })
const job = await queue.getJob(jobId)
if (!isNil(job)) {
await job.remove()
log.info({
job: { id: jobId },
queueName,
}, '[jobQueue#removeOneTimeJob] removed job from queue')
return
}
log.info({
job: { id: jobId },
queueName,
}, '[jobQueue#removeOneTimeJob] job not found in queue')
},
async cancelAndReportNeverStarted({ jobId, platformId, projectId, jobType }: RemoveOneTimeJobParams): Promise<boolean> {
const queueName = await workerGroupRouting.resolveQueueForJob({ platformId, projectId, jobType, log })
const queue = await ensureQueueExists({ log, queueName })
const job = await queue.getJob(jobId)
if (isNil(job)) {
log.info({ job: { id: jobId }, queueName }, '[jobQueue#cancelAndReportNeverStarted] job not found')
return false
}
const everDequeued = !isNil(job.processedOn)
const { error } = await tryCatch(() => job.remove())
if (error) {
log.info({ job: { id: jobId, everDequeued }, queueName, error: String(error) }, '[jobQueue#cancelAndReportNeverStarted] a worker holds the job')
return false
}
log.info({ job: { id: jobId, everDequeued }, queueName }, '[jobQueue#cancelAndReportNeverStarted] removed the abandoned job')
return !everDequeued
},
async getOrCreateQueue({ queueName }: { queueName: string }): Promise<Queue> {
return ensureQueueExists({ log, queueName })
},
getAllQueues(): Queue[] {
const queues = [...dedicatedWorkersQueues.values()].filter(queue => !isNil(queue))
return queues
},
getSharedQueue(): Queue {
const queue = dedicatedWorkersQueues.get(QueueName.WORKER_JOBS)
if (isNil(queue)) {
throw Error('Shared queue not initialized')
}
return queue
},
async removeAllFlowRunJobs({ flowRunId, platformId, projectId, flowId }: RemoveAllFlowRunJobsParams): Promise<void> {
const queueName = await workerGroupRouting.resolveQueueForJob({ platformId, projectId, flowId, jobType: WorkerJobType.EXECUTE_FLOW, log })
const queue = await ensureQueueExists({ log, queueName })
const allJobs = await queue.getJobs(['waiting', 'delayed'])
const matching = allJobs.filter((j) => j.id?.startsWith(flowRunId))
await Promise.allSettled(matching.map((j) => j.remove()))
log.info({ flowRun: { id: flowRunId }, queueName, removedIds: matching.map((j) => j.id) }, '[jobQueue#removeAllFlowRunJobs] done')
},
async close(): Promise<void> {
log.info('[jobQueue#close] Closing job queue')
const allQueues = [...dedicatedWorkersQueues.values()].filter(queue => !isNil(queue))
await Promise.allSettled(
allQueues.map(queue => queue.close()),
)
},
})
async function ensureQueueExists({ log, queueName }: { log: FastifyBaseLogger, queueName: string }): Promise<Queue> {
const existingQueue = dedicatedWorkersQueues.get(queueName)
if (!isNil(existingQueue)) {
return existingQueue
}
return memoryLock.runExclusive({
key: `ensure_queue_exists_${queueName}`,
fn: async () => {
const existingQueue = dedicatedWorkersQueues.get(queueName)
if (!isNil(existingQueue)) {
return existingQueue
}
const queue = new Queue(queueName, {
connection: await redisConnections.create(),
defaultJobOptions: {
attempts: 2,
backoff: {
type: 'exponential',
delay: EIGHT_MINUTES_IN_MILLISECONDS,
},
removeOnComplete: true,
removeOnFail: {
age: REDIS_FAILED_JOB_RETENTION_DAYS,
count: REDIS_FAILED_JOB_RETRY_COUNT,
},
},
})
await queue.removeGlobalConcurrency()
await queue.waitUntilReady()
dedicatedWorkersQueues.set(queueName, queue)
log.info({
queueName,
}, '[jobQueue#ensureQueueExists] Queue created')
return queue
},
})
}
const USER_INTERACTION_JOB_TYPES = new Set([
WorkerJobType.EXECUTE_PROPERTY,
WorkerJobType.EXECUTE_VALIDATION,
WorkerJobType.EXECUTE_RESOLVE_CONNECTION_IDENTIFIER,
WorkerJobType.EXECUTE_TRIGGER_HOOK,
WorkerJobType.EXECUTE_EXTRACT_PIECE_INFORMATION,
WorkerJobType.EXECUTE_TOKEN_REFRESH,
WorkerJobType.EXECUTE_ACTION,
])
export function isUserInteractionJob(jobType: WorkerJobType): boolean {
return USER_INTERACTION_JOB_TYPES.has(jobType)
}
export function isUserInteractionJobData(jobData: JobData): jobData is UserInteractionJobData {
return USER_INTERACTION_JOB_TYPES.has(jobData.jobType)
}
export function callerWaitingForResponse(jobData: JobData): { webserverId: string, requestId: string } | null {
if (isUserInteractionJobData(jobData)) {
return { webserverId: jobData.webserverId, requestId: jobData.requestId }
}
if (jobData.jobType === WorkerJobType.EXECUTE_AI && !isNil(jobData.webserverId)) {
return { webserverId: jobData.webserverId, requestId: jobData.requestId }
}
return null
}
export enum JobType {
REPEATING = 'repeating',
ONE_TIME = 'one_time',
}
type RemoveOneTimeJobParams = {
jobId: ApId
platformId: string | null
projectId?: string | null
flowId?: string | null
jobType?: WorkerJobType
}
type RemoveAllFlowRunJobsParams = {
flowRunId: string
platformId: string | null
projectId?: string | null
flowId?: string | null
}
type BaseAddParams<JD extends Omit<JobData, 'engineToken'>, JT extends JobType> = {
id: ApId
data: JD
type: JT
delay?: number
}
type RepeatingJobAddParams = BaseAddParams<PollingJobData | RenewWebhookJobData, JobType.REPEATING> & {
scheduleOptions: ScheduleOptions
}
type OneTimeJobAddParams = BaseAddParams<ExecuteFlowJobData | WebhookJobData | UserInteractionJobData | EventDestinationJobData | ExecuteAgentRunJobData | ExecuteAiJobData | ExecutePersonalizationResearchJobData, JobType.ONE_TIME> & {
retryBackoffMs?: number
}
export type AddJobParams<type extends JobType> = type extends JobType.REPEATING ? RepeatingJobAddParams : OneTimeJobAddParams