AI Agents & MCPs & AI Workflow Automation • (~400 MCP servers for AI agents) • AI Automation / AI Agent with MCPs • AI Workflows & AI Agents • MCPs for AI Agents
TypeScript ai-agentai-agent-toolsai-agentsai-agents-frameworkmcpmcp-servermcp-toolsmcpsn8n-alternativeno-code-automationworkflowworkflow-automationworkflows
import { apId } from '@activepieces/core-utils'
import { FlowTriggerType, LATEST_JOB_DATA_SCHEMA_VERSION, WorkerGroupScope, WorkerJobType } from '@activepieces/shared'
import { FastifyInstance } from 'fastify'
import { workerGroupQueues } from '../../../../src/app/workers/worker-groups/worker-group-queues'
import { jobQueue } from '../../../../src/app/workers/job-queue/job-queue'
import { platformQueueMigrationService } from '../../../../src/app/workers/worker-groups/queue-migration.service'
import { setupTestEnvironment, teardownTestEnvironment } from '../../../helpers/test-setup'
let app: FastifyInstance
beforeAll(async () => {
app = await setupTestEnvironment()
})
afterAll(async () => {
await teardownTestEnvironment()
})
describe('platformQueueMigrationService', () => {
let fromQueueName: string
let toQueueName: string
beforeEach(() => {
fromQueueName = workerGroupQueues.getQueueName({ scope: WorkerGroupScope.PLATFORM, workerGroupId: apId() })
toQueueName = workerGroupQueues.getQueueName({ scope: WorkerGroupScope.PLATFORM, workerGroupId: apId() })
})
afterEach(async () => {
const fromQueue = await jobQueue(app.log).getOrCreateQueue({ queueName: fromQueueName })
const toQueue = await jobQueue(app.log).getOrCreateQueue({ queueName: toQueueName })
await fromQueue.obliterate({ force: true })
await toQueue.obliterate({ force: true })
})
it('returns early without touching queues when fromQueueName equals toQueueName', async () => {
const platformId = apId()
const flowVersionId = apId()
const sameQueue = workerGroupQueues.getQueueName({ scope: WorkerGroupScope.PLATFORM, workerGroupId: apId() })
const queue = await jobQueue(app.log).getOrCreateQueue({ queueName: sameQueue })
await queue.upsertJobScheduler(
flowVersionId,
{ pattern: '*/5 * * * *', tz: 'UTC' },
{ name: flowVersionId, data: buildPollingJobData({ platformId, flowVersionId }) },
)
await platformQueueMigrationService(app.log).migrateJobs({
fromQueueName: sameQueue,
toQueueName: sameQueue,
platformId,
})
expect(await queue.getJobSchedulersCount()).toBe(1)
await queue.obliterate({ force: true })
})
it('moves a POLLING scheduler from source queue to target queue', async () => {
const platformId = apId()
const flowVersionId = apId()
const fromQueue = await jobQueue(app.log).getOrCreateQueue({ queueName: fromQueueName })
const toQueue = await jobQueue(app.log).getOrCreateQueue({ queueName: toQueueName })
await fromQueue.upsertJobScheduler(
flowVersionId,
{ pattern: '*/5 * * * *', tz: 'UTC' },
{ name: flowVersionId, data: buildPollingJobData({ platformId, flowVersionId }) },
)
await platformQueueMigrationService(app.log).migrateJobs({ fromQueueName, toQueueName, platformId })
expect(await fromQueue.getJobSchedulersCount()).toBe(0)
expect(await toQueue.getJobSchedulersCount()).toBe(1)
const [movedScheduler] = await toQueue.getJobSchedulers(0, 0)
expect(movedScheduler.id ?? movedScheduler.key).toBe(flowVersionId)
expect(movedScheduler.template?.data?.platformId).toBe(platformId)
expect(movedScheduler.pattern).toBe('*/5 * * * *')
})
it('does not migrate schedulers belonging to a different platform', async () => {
const platformA = apId()
const platformB = apId()
const flowVersionA = apId()
const flowVersionB = apId()
const fromQueue = await jobQueue(app.log).getOrCreateQueue({ queueName: fromQueueName })
const toQueue = await jobQueue(app.log).getOrCreateQueue({ queueName: toQueueName })
await fromQueue.upsertJobScheduler(
flowVersionA,
{ pattern: '*/5 * * * *', tz: 'UTC' },
{ name: flowVersionA, data: buildPollingJobData({ platformId: platformA, flowVersionId: flowVersionA }) },
)
await fromQueue.upsertJobScheduler(
flowVersionB,
{ pattern: '*/10 * * * *', tz: 'UTC' },
{ name: flowVersionB, data: buildPollingJobData({ platformId: platformB, flowVersionId: flowVersionB }) },
)
await platformQueueMigrationService(app.log).migrateJobs({ fromQueueName, toQueueName, platformId: platformA })
expect(await fromQueue.getJobSchedulersCount()).toBe(1)
expect(await toQueue.getJobSchedulersCount()).toBe(1)
const [remaining] = await fromQueue.getJobSchedulers(0, 0)
expect(remaining.template?.data?.platformId).toBe(platformB)
const [moved] = await toQueue.getJobSchedulers(0, 0)
expect(moved.template?.data?.platformId).toBe(platformA)
})
it('removes the orphaned next-run delayed job from the source queue after scheduler migration', async () => {
const platformId = apId()
const flowVersionId = apId()
const fromQueue = await jobQueue(app.log).getOrCreateQueue({ queueName: fromQueueName })
const toQueue = await jobQueue(app.log).getOrCreateQueue({ queueName: toQueueName })
await fromQueue.upsertJobScheduler(
flowVersionId,
{ pattern: '*/5 * * * *', tz: 'UTC' },
{ name: flowVersionId, data: buildPollingJobData({ platformId, flowVersionId }) },
)
// upsertJobScheduler always queues the next-run delayed instance immediately
expect(await fromQueue.getDelayedCount()).toBe(1)
await platformQueueMigrationService(app.log).migrateJobs({ fromQueueName, toQueueName, platformId })
expect(await fromQueue.getDelayedCount()).toBe(0)
// Target queue has the scheduler and its own next-run delayed job
expect(await toQueue.getDelayedCount()).toBe(1)
})
it('does not remove delayed jobs belonging to other-platform schedulers on the source queue', async () => {
const platformA = apId()
const platformB = apId()
const flowVersionA = apId()
const flowVersionB = apId()
const fromQueue = await jobQueue(app.log).getOrCreateQueue({ queueName: fromQueueName })
await fromQueue.upsertJobScheduler(
flowVersionA,
{ pattern: '*/5 * * * *', tz: 'UTC' },
{ name: flowVersionA, data: buildPollingJobData({ platformId: platformA, flowVersionId: flowVersionA }) },
)
await fromQueue.upsertJobScheduler(
flowVersionB,
{ pattern: '*/10 * * * *', tz: 'UTC' },
{ name: flowVersionB, data: buildPollingJobData({ platformId: platformB, flowVersionId: flowVersionB }) },
)
expect(await fromQueue.getDelayedCount()).toBe(2)
await platformQueueMigrationService(app.log).migrateJobs({ fromQueueName, toQueueName, platformId: platformA })
// Only platformA's orphaned delayed job should be removed
expect(await fromQueue.getDelayedCount()).toBe(1)
})
it('moves regular (one-time) waiting jobs for the platform', async () => {
const platformId = apId()
const jobId = apId()
const fromQueue = await jobQueue(app.log).getOrCreateQueue({ queueName: fromQueueName })
const toQueue = await jobQueue(app.log).getOrCreateQueue({ queueName: toQueueName })
await fromQueue.add(jobId, buildPollingJobData({ platformId, flowVersionId: apId() }), { jobId })
await platformQueueMigrationService(app.log).migrateJobs({ fromQueueName, toQueueName, platformId })
expect(await fromQueue.getWaitingCount()).toBe(0)
expect(await toQueue.getWaitingCount()).toBe(1)
const [movedJob] = await toQueue.getJobs(['waiting'], 0, 0)
expect(movedJob.data.platformId).toBe(platformId)
})
it('does not move regular jobs belonging to a different platform', async () => {
const platformA = apId()
const platformB = apId()
const fromQueue = await jobQueue(app.log).getOrCreateQueue({ queueName: fromQueueName })
const toQueue = await jobQueue(app.log).getOrCreateQueue({ queueName: toQueueName })
const jobIdA = apId()
const jobIdB = apId()
await fromQueue.add(jobIdA, buildPollingJobData({ platformId: platformA, flowVersionId: apId() }), { jobId: jobIdA })
await fromQueue.add(jobIdB, buildPollingJobData({ platformId: platformB, flowVersionId: apId() }), { jobId: jobIdB })
await platformQueueMigrationService(app.log).migrateJobs({ fromQueueName, toQueueName, platformId: platformA })
expect(await fromQueue.getWaitingCount()).toBe(1)
expect(await toQueue.getWaitingCount()).toBe(1)
})
describe('migrateFlowJobs', () => {
it('moves only the target flow executable jobs', async () => {
const flowA = apId()
const flowB = apId()
const fromQueue = await jobQueue(app.log).getOrCreateQueue({ queueName: fromQueueName })
const toQueue = await jobQueue(app.log).getOrCreateQueue({ queueName: toQueueName })
const jobIdA = apId()
const jobIdB = apId()
await fromQueue.add(jobIdA, buildWebhookJobData({ flowId: flowA }), { jobId: jobIdA })
await fromQueue.add(jobIdB, buildWebhookJobData({ flowId: flowB }), { jobId: jobIdB })
await platformQueueMigrationService(app.log).migrateFlowJobs({ fromQueueName, toQueueName, flowId: flowA })
expect(await fromQueue.getWaitingCount()).toBe(1)
expect(await toQueue.getWaitingCount()).toBe(1)
const [movedJob] = await toQueue.getJobs(['waiting'], 0, 0)
expect(movedJob.data.flowId).toBe(flowA)
})
it('does not move non-routable job types that carry the same flowId', async () => {
const flowId = apId()
const fromQueue = await jobQueue(app.log).getOrCreateQueue({ queueName: fromQueueName })
const toQueue = await jobQueue(app.log).getOrCreateQueue({ queueName: toQueueName })
const jobId = apId()
const pollingData = { ...buildPollingJobData({ platformId: apId(), flowVersionId: apId() }), flowId }
await fromQueue.add(jobId, pollingData, { jobId })
await platformQueueMigrationService(app.log).migrateFlowJobs({ fromQueueName, toQueueName, flowId })
expect(await fromQueue.getWaitingCount()).toBe(1)
expect(await toQueue.getWaitingCount()).toBe(0)
})
})
describe('batch logic', () => {
it('migrates all schedulers when count exceeds batchSize', async () => {
const platformId = apId()
const fromQueue = await jobQueue(app.log).getOrCreateQueue({ queueName: fromQueueName })
const toQueue = await jobQueue(app.log).getOrCreateQueue({ queueName: toQueueName })
const total = 5
for (let i = 0; i < total; i++) {
const flowVersionId = apId()
await fromQueue.upsertJobScheduler(
flowVersionId,
{ pattern: '*/5 * * * *', tz: 'UTC' },
{ name: flowVersionId, data: buildPollingJobData({ platformId, flowVersionId }) },
)
}
await platformQueueMigrationService(app.log).migrateJobs({ fromQueueName, toQueueName, platformId, batchSize: 2 })
expect(await fromQueue.getJobSchedulersCount()).toBe(0)
expect(await toQueue.getJobSchedulersCount()).toBe(total)
})
it('leaves other-platform schedulers on source when count exceeds batchSize', async () => {
const platformA = apId()
const platformB = apId()
const fromQueue = await jobQueue(app.log).getOrCreateQueue({ queueName: fromQueueName })
const toQueue = await jobQueue(app.log).getOrCreateQueue({ queueName: toQueueName })
// Interleave platformA and platformB schedulers to exercise multi-batch offset tracking
for (let i = 0; i < 3; i++) {
const idA = apId()
await fromQueue.upsertJobScheduler(idA, { pattern: '*/5 * * * *', tz: 'UTC' }, { name: idA, data: buildPollingJobData({ platformId: platformA, flowVersionId: idA }) })
const idB = apId()
await fromQueue.upsertJobScheduler(idB, { pattern: '*/10 * * * *', tz: 'UTC' }, { name: idB, data: buildPollingJobData({ platformId: platformB, flowVersionId: idB }) })
}
await platformQueueMigrationService(app.log).migrateJobs({ fromQueueName, toQueueName, platformId: platformA, batchSize: 2 })
expect(await fromQueue.getJobSchedulersCount()).toBe(3)
expect(await toQueue.getJobSchedulersCount()).toBe(3)
const remaining = await fromQueue.getJobSchedulers(0, -1)
expect(remaining.every(s => s.template?.data?.platformId === platformB)).toBe(true)
})
it('migrates all regular jobs when count exceeds batchSize', async () => {
const platformId = apId()
const fromQueue = await jobQueue(app.log).getOrCreateQueue({ queueName: fromQueueName })
const toQueue = await jobQueue(app.log).getOrCreateQueue({ queueName: toQueueName })
const total = 5
for (let i = 0; i < total; i++) {
const jobId = apId()
await fromQueue.add(jobId, buildPollingJobData({ platformId, flowVersionId: apId() }), { jobId })
}
await platformQueueMigrationService(app.log).migrateJobs({ fromQueueName, toQueueName, platformId, batchSize: 2 })
expect(await fromQueue.getWaitingCount()).toBe(0)
expect(await toQueue.getWaitingCount()).toBe(total)
})
})
})
function buildPollingJobData({ platformId, flowVersionId }: { platformId: string, flowVersionId: string }) {
return {
projectId: apId(),
platformId,
schemaVersion: LATEST_JOB_DATA_SCHEMA_VERSION,
flowVersionId,
flowId: apId(),
triggerType: FlowTriggerType.PIECE,
jobType: WorkerJobType.EXECUTE_POLLING,
}
}
function buildWebhookJobData({ flowId }: { flowId: string }) {
return {
projectId: apId(),
platformId: apId(),
schemaVersion: LATEST_JOB_DATA_SCHEMA_VERSION,
flowId,
jobType: WorkerJobType.EXECUTE_WEBHOOK,
}
}