Skip to content

AI Agents & MCPs & AI Workflow Automation • (~400 MCP servers for AI agents) • AI Automation / AI Agent with MCPs • AI Workflows & AI Agents • MCPs for AI Agents

15.7 KB
Raw Blame
import { isNil, tryCatch } from '@activepieces/core-utils'
import { ConsumeJobRequest, ConsumeJobResponse, EngineResponseStatus, JobData } from '@activepieces/shared'
import { Worker as BullMQWorker, Job, UnrecoverableError } from 'bullmq'
import { FastifyBaseLogger } from 'fastify'
import { Socket } from 'socket.io'
import { accessTokenManager } from '../../authentication/lib/access-token-manager'
import { redisConnections } from '../../database/redis-connections'
import { engineResponseWatcher } from '../engine-response-watcher'
import { QueueName } from '../job'
import { jobMigrations } from '../migrations/job-data-migrations'
import { pollingExclusivityInterceptor } from './interceptors/polling-exclusivity-interceptor'
import { rateLimiterInterceptor } from './interceptors/rate-limiter-interceptor'
import { zombiePollingInterceptor } from './interceptors/zombie-polling-interceptor'
import { jobAssignmentTracker } from './job-assignment-tracker'
import { jobFailureLogger } from './job-failure-logger'
import { InterceptorVerdict, JobInterceptor } from './job-interceptor'
import { callerWaitingForResponse } from './job-queue'
import { createQueueDispatcher, QueueDispatcher } from './queue-dispatcher'

const DRAIN_DELAY_SECONDS = 15
const LOCK_DURATION_MS = 120_000

const interceptors: JobInterceptor[] = [rateLimiterInterceptor, zombiePollingInterceptor, pollingExclusivityInterceptor]
const workerPromises = new Map<string, Promise<BullMQWorker>>()
const dispatchers = new Map<string, QueueDispatcher>()

function ensureBullMQWorker(queueName: string, log: FastifyBaseLogger): Promise<BullMQWorker> {
    const existing = workerPromises.get(queueName)
    if (existing) return existing

    const promise = createBullMQWorker(queueName, log)
    workerPromises.set(queueName, promise)
    return promise
}

async function createBullMQWorker(queueName: string, log: FastifyBaseLogger): Promise<BullMQWorker> {
    const worker = new BullMQWorker(
        queueName,
        undefined,
        {
            connection: await redisConnections.create(),
            concurrency: 500,
            autorun: false,
            lockDuration: LOCK_DURATION_MS,
            stalledInterval: 30_000,
            maxStalledCount: 3,
            drainDelay: DRAIN_DELAY_SECONDS,
        },
    )
    await worker.waitUntilReady()
    await worker.startStalledCheckTimer()

    worker.on('stalled', (jobId: string) => {
        log.warn({ queueName, job: { id: jobId } }, '[jobBroker] Job stalled — BullMQ will retry automatically')
    })

    log.info({ queueName }, '[jobBroker] BullMQ worker initialized')
    return worker
}

async function fetchJobFromRedis(queueName: string, jobId: string, log: FastifyBaseLogger): Promise<Job | null> {
    const worker = await ensureBullMQWorker(queueName, log)
    const job = await Job.fromId(worker, jobId)
    if (isNil(job)) {
        log.warn({ job: { id: jobId }, queueName }, '[jobBroker] Job not found in Redis')
        return null
    }
    return job
}

function ensureDispatcher(queueName: string, worker: BullMQWorker, log: FastifyBaseLogger): QueueDispatcher {
    const existing = dispatchers.get(queueName)
    if (existing) return existing

    const dispatcher = createQueueDispatcher({
        queueName,
        worker,
        dequeue: tryDequeue,
        onOrphanedJob: returnJobToQueue,
        log,
    })
    dispatchers.set(queueName, dispatcher)
    return dispatcher
}

async function tryDequeue(worker: BullMQWorker, queueName: string, log: FastifyBaseLogger): Promise<ConsumeJobRequest | null> {
    const token = `token-${Date.now()}-${Math.random().toString(36).slice(2)}`
    const job = await worker.getNextJob(token)
    if (isNil(job)) {
        return null  // waiting list empty — drainDelay provided backpressure
    }

    if (job.deferredFailure) {
        log.warn(
            { queueName, job: { id: job.id }, jobName: job.name, deferredFailure: job.deferredFailure },
            '[jobBroker#tryDequeue] Failing job with deferred failure (BullMQ stalled limit exceeded)',
        )
        const deferredError = new UnrecoverableError(job.deferredFailure)
        const { error: failError } = await tryCatch(() => job.moveToFailed(deferredError, token, false))
        if (failError) {
            log.error(
                { queueName, job: { id: job.id }, error: String(failError) },
                '[jobBroker#tryDequeue] Failed to fail deferred-failure job',
            )
        }
        else {
            jobFailureLogger.logJobFailed({ queueName, jobId: job.id, jobType: job.name, error: deferredError, log })
        }
        return tryDequeue(worker, queueName, log)
    }

    log.info({ queueName, job: { id: job.id, pickupWaitMs: (job.processedOn ?? Date.now()) - job.timestamp - (job.opts.delay ?? 0) }, jobName: job.name }, '[jobBroker#tryDequeue] Dequeued job')

    const originalSchemaVersion = (job.data as Record<string, unknown>).schemaVersion
    const migratedData = await jobMigrations(log).apply(job.data)
    const jobId = job.id ?? job.name

    if (migratedData.schemaVersion !== originalSchemaVersion) {
        await job.updateData(migratedData)
    }

    const parseResult = JobData.safeParse(migratedData)
    if (!parseResult.success) {
        const issues = parseResult.error.issues.map(issue => `${issue.path.join('.') || '<root>'}: ${issue.message}`).join('; ')
        const reason = `Job data failed schema validation after migration: ${issues}`
        log.error(
            { queueName, job: { id: jobId, type: migratedData.jobType }, schemaVersion: migratedData.schemaVersion, issues: parseResult.error.issues },
            '[jobBroker#tryDequeue] Failing job with invalid schema as unrecoverable',
        )
        const schemaError = new UnrecoverableError(reason)
        const { error: failError } = await tryCatch(() => job.moveToFailed(schemaError, token, false))
        if (failError) {
            log.error({ queueName, job: { id: jobId }, error: String(failError) }, '[jobBroker#tryDequeue] Failed to fail invalid-schema job')
        }
        else {
            jobFailureLogger.logJobFailed({ queueName, jobId, jobType: migratedData.jobType, error: schemaError, log })
        }
        return tryDequeue(worker, queueName, log)
    }

    const interceptorResult = await runInterceptors({ jobId, jobData: migratedData, job, log })
    if (interceptorResult === 'DISCARD') {
        await job.moveToCompleted(null, token, false)
        return tryDequeue(worker, queueName, log)
    }
    if (interceptorResult !== null) {
        await job.moveToDelayed(Date.now() + interceptorResult.delayInMs, token)
        if (interceptorResult.priority !== undefined) {
            await job.changePriority({ priority: interceptorResult.priority })
        }
        return tryDequeue(worker, queueName, log)  // retry getNextJob instead of returning null
    }

    const engineToken = await accessTokenManager(log).generateEngineToken({
        jobId,
        projectId: migratedData.projectId as string,
        platformId: migratedData.platformId,
    })

    return {
        jobId,
        jobData: migratedData,
        attempsStarted: job.attemptsMade,
        lastAttempt: job.attemptsMade + 1 >= (job.opts.attempts ?? 1),
        engineToken,
        token,
        queueName,
    }
}

async function returnJobToQueue(jobId: string, token: string, queueName: string, log: FastifyBaseLogger): Promise<void> {
    const job = await fetchJobFromRedis(queueName, jobId, log)
    if (isNil(job)) {
        return
    }
    const jobData = JobData.parse(job.data)
    await job.moveToDelayed(Date.now() + 100, token)
    for (const interceptor of interceptors) {
        const { error } = await tryCatch(() => interceptor.onJobFinished({ jobId, jobData, failed: false, log }))
        if (error) {
            log.error({ job: { id: jobId }, error: String(error) }, '[jobBroker#returnJobToQueue] interceptor cleanup failed')
        }
    }
    log.info({ job: { id: jobId } }, '[jobBroker#returnJobToQueue] orphaned job returned to queue')
}

async function runInterceptors({ jobId, jobData, job, log }: { jobId: string, jobData: JobData, job: Job, log: FastifyBaseLogger }): Promise<{ delayInMs: number, priority?: number } | 'DISCARD' | null> {
    const passed: JobInterceptor[] = []
    for (const interceptor of interceptors) {
        const result = await interceptor.preDispatch({ jobId, jobData, job, log })
        if (result.verdict === InterceptorVerdict.DISCARD) {
            for (const passedInterceptor of passed) {
                const { error } = await tryCatch(() => passedInterceptor.onJobFinished({ jobId, jobData, failed: false, log }))
                if (error) {
                    log.error({ job: { id: jobId }, error: String(error) }, '[jobBroker] Failed to clean up interceptor on discard')
                }
            }
            return 'DISCARD'
        }
        if (result.verdict === InterceptorVerdict.REJECT) {
            for (const passedInterceptor of passed) {
                const { error } = await tryCatch(() => passedInterceptor.onJobFinished({ jobId, jobData, failed: false, log }))
                if (error) {
                    log.error({ job: { id: jobId }, error: String(error) }, '[jobBroker] Failed to clean up interceptor on reject')
                }
            }
            return { delayInMs: result.delayInMs, priority: result.priority }
        }
        passed.push(interceptor)
    }
    return null
}

function buildFailedReason(errorMessage: string, logs?: string): string {
    if (!logs) return errorMessage
    return `${errorMessage}\n${logs}`
}

export { tryDequeue }

export const jobBroker = (log: FastifyBaseLogger) => ({
    async init(): Promise<void> {
        await ensureBullMQWorker(QueueName.WORKER_JOBS, log)
        log.info('[jobBroker] Job broker initialized')
    },

    async poll(queueName: string = QueueName.WORKER_JOBS, socket?: Pick<Socket, 'id' | 'connected'>): Promise<ConsumeJobRequest | null> {
        const worker = await ensureBullMQWorker(queueName, log)
        const dispatcher = ensureDispatcher(queueName, worker, log)
        const job = await dispatcher.poll(socket)
        if (!isNil(job) && !isNil(socket)) {
            jobAssignmentTracker.record({ connectionId: socket.id, jobId: job.jobId, token: job.token, queueName: job.queueName })
        }
        return job
    },

    async releaseConnectionJobs(connectionId: string): Promise<void> {
        const jobs = jobAssignmentTracker.takeByConnection(connectionId)
        if (jobs.length === 0) {
            return
        }
        log.info({ connection: { id: connectionId }, jobCount: jobs.length }, '[jobBroker] Worker connection closed with in-flight jobs — returning them to the queue')
        for (const { jobId, token, queueName } of jobs) {
            const { error } = await tryCatch(() => returnJobToQueue(jobId, token, queueName, log))
            if (error) {
                log.error({ connection: { id: connectionId }, job: { id: jobId }, error: String(error) }, '[jobBroker] Failed to return in-flight job on disconnect — leaving for stalled-scan recovery')
            }
        }
    },

    async completeJob(input: ConsumeJobResponse & { jobId: string, token: string, queueName: string }): Promise<void> {
        const job = await fetchJobFromRedis(input.queueName, input.jobId, log)
        if (isNil(job)) {
            return
        }

        const jobData = JobData.parse(job.data)
        const waitingCaller = callerWaitingForResponse(jobData)

        const { error } = await tryCatch(async () => {
            if (input.status === EngineResponseStatus.INTERNAL_ERROR) {
                if (waitingCaller) {
                    // User-interaction jobs (piece-metadata extraction, validation, property/auth, trigger
                    // hooks) are synchronous request/response — the caller awaits the result with a timeout.
                    // Return the error to that caller and COMPLETE the job instead of moving it to failed: the
                    // exponential-backoff retry only fires long after the caller has timed out, so it serves no
                    // one and just piles up dead jobs in the failed queue.
                    await engineResponseWatcher(log).publish(waitingCaller.webserverId, waitingCaller.requestId, {
                        status: EngineResponseStatus.INTERNAL_ERROR,
                        response: undefined,
                        error: input.errorMessage ?? 'Internal error',
                        logs: input.logs,
                    })
                    await job.moveToCompleted({ response: undefined }, input.token, false)
                    return
                }
                const engineErrorMessage = input.errorMessage ?? 'Internal error'
                const engineError = new Error(buildFailedReason(engineErrorMessage, input.logs))
                await job.moveToFailed(engineError, input.token)
                if ((job.attemptsMade ?? 0) >= (job.opts?.attempts ?? 1)) {
                    jobFailureLogger.logJobFailed({
                        queueName: input.queueName,
                        jobId: input.jobId,
                        jobType: jobData.jobType,
                        error: engineError,
                        log,
                        signatureOverride: jobFailureLogger.signatureFromMessage(engineErrorMessage, 'EngineError'),
                    })
                }
                return
            }

            await job.moveToCompleted({ response: input.response ?? undefined }, input.token, false)
            if (waitingCaller) {
                await engineResponseWatcher(log).publish(waitingCaller.webserverId, waitingCaller.requestId, {
                    status: input.status,
                    response: input.response,
                    error: input.errorMessage,
                    logs: input.logs,
                })
            }
        })
        if (error) {
            log.error({ job: { id: input.jobId }, error: String(error), originalError: input.errorMessage }, '[jobBroker] Failed to move job to final state — leaving for stalled-scan recovery')
            if (waitingCaller) {
                await engineResponseWatcher(log).publish(waitingCaller.webserverId, waitingCaller.requestId, {
                    status: EngineResponseStatus.INTERNAL_ERROR,
                    response: undefined,
                    error: String(error),
                })
            }
        }

        jobAssignmentTracker.clear({ jobId: input.jobId, queueName: input.queueName })

        const failed = input.status === EngineResponseStatus.INTERNAL_ERROR || !isNil(error)
        for (const interceptor of interceptors) {
            const { error: interceptorError } = await tryCatch(() => interceptor.onJobFinished({ jobId: input.jobId, jobData, failed, log }))
            if (interceptorError) {
                log.error({ job: { id: input.jobId }, error: String(interceptorError) }, '[jobBroker] Interceptor onJobFinished failed')
            }
        }
    },

    async extendLock(input: { jobId: string, token: string, queueName: string }): Promise<void> {
        const job = await fetchJobFromRedis(input.queueName, input.jobId, log)
        if (isNil(job)) {
            return
        }
        await job.extendLock(input.token, LOCK_DURATION_MS)
        log.debug({ job: { id: input.jobId } }, '[jobBroker] Lock extended')
    },

    async close(): Promise<void> {
        for (const dispatcher of dispatchers.values()) {
            dispatcher.close()
        }
        dispatchers.clear()

        const workers = await Promise.allSettled([...workerPromises.values()])
        await Promise.allSettled(
            workers
                .filter((r): r is PromiseFulfilledResult<BullMQWorker> => r.status === 'fulfilled')
                .map(r => r.value.close()),
        )
        workerPromises.clear()
    },
})