AI Agents & MCPs & AI Workflow Automation • (~400 MCP servers for AI agents) • AI Automation / AI Agent with MCPs • AI Workflows & AI Agents • MCPs for AI Agents
TypeScript ai-agentai-agent-toolsai-agentsai-agents-frameworkmcpmcp-servermcp-toolsmcpsn8n-alternativeno-code-automationworkflowworkflow-automationworkflows
feat(server): flow-level worker groups (#16483)
Chaker Atallahdfeed9d5a58e
diff --git a/packages/core/execution/src/lib/flows/flow.ts b/packages/core/execution/src/lib/flows/flow.tsindex 3dc4444bce7..d1a17522e55 100755--- a/packages/core/execution/src/lib/flows/flow.ts+++ b/packages/core/execution/src/lib/flows/flow.ts@@ -57,6 +57,7 @@ export const Flow = z.object({timeSavedPerRun: Nullable(z.number()),templateId: Nullable(z.string()),createdBy: Nullable(FlowCreator),+ workerGroupId: Nullable(z.string()),})export type Flow = z.infer<typeof Flow>diff --git a/packages/core/execution/src/lib/workers/index.ts b/packages/core/execution/src/lib/workers/index.tsindex 923a096dd3d..044eed495a5 100644--- a/packages/core/execution/src/lib/workers/index.ts+++ b/packages/core/execution/src/lib/workers/index.ts@@ -16,6 +16,7 @@ export enum WorkerMachineType {export enum WorkerGroupScope {PLATFORM = 'platform',PROJECT = 'project',+ FLOW = 'flow',}export enum NetworkMode {diff --git a/packages/core/shared/package.json b/packages/core/shared/package.jsonindex d2e61e16353..1a4e9c38c75 100644--- a/packages/core/shared/package.json+++ b/packages/core/shared/package.json@@ -1,6 +1,6 @@{"name": "@activepieces/shared",- "version": "0.203.0",+ "version": "0.204.0","type": "commonjs","sideEffects": false,"main": "./dist/src/index.js",diff --git a/packages/core/shared/src/lib/management/project/project-requests.ts b/packages/core/shared/src/lib/management/project/project-requests.tsindex 9566fed9e09..7e32bf5d528 100644--- a/packages/core/shared/src/lib/management/project/project-requests.ts+++ b/packages/core/shared/src/lib/management/project/project-requests.ts@@ -16,6 +16,7 @@ export const UpdateProjectPlatformRequest = z.object({}).optional(),globalConnectionExternalIds: z.array(z.string()).optional(),maxConcurrentJobs: z.optional(Nullable(z.number().int().positive())),+ /** @deprecated use POST /v1/worker-groups/projects/:id instead */workerGroupId: z.optional(Nullable(z.string())),executionDataRetentionDays: z.optional(Nullable(z.number().int().positive())),sensitive: z.boolean().optional(),diff --git a/packages/server/api/src/app/database/migration/postgres/1864000000000-AddWorkerGroupIdToFlow.ts b/packages/server/api/src/app/database/migration/postgres/1864000000000-AddWorkerGroupIdToFlow.tsnew file mode 100644index 00000000000..1f77075ad16--- /dev/null+++ b/packages/server/api/src/app/database/migration/postgres/1864000000000-AddWorkerGroupIdToFlow.ts@@ -0,0 +1,36 @@+import { QueryRunner } from 'typeorm'+import { system } from '../../../helper/system/system'+import { AppSystemProp } from '../../../helper/system/system-props'+import { DatabaseType } from '../../database-type'+import { Migration } from '../../migration'++export class AddWorkerGroupIdToFlow1864000000000 implements Migration {+ name = 'AddWorkerGroupIdToFlow1864000000000'+ breaking = false+ release = '0.92.0'+ transaction = false++ public async up(queryRunner: QueryRunner): Promise<void> {+ await queryRunner.query(`+ ALTER TABLE "flow"+ ADD COLUMN IF NOT EXISTS "workerGroupId" character varying+ `)+ if (isPGlite()) {+ await queryRunner.query(`+ CREATE INDEX IF NOT EXISTS "idx_flow_worker_group" ON "flow" ("workerGroupId")+ `)+ }+ else {+ await queryRunner.query(`+ CREATE INDEX CONCURRENTLY IF NOT EXISTS "idx_flow_worker_group" ON "flow" ("workerGroupId")+ `)+ }+ }++ public async down(queryRunner: QueryRunner): Promise<void> {+ await queryRunner.query('DROP INDEX IF EXISTS "idx_flow_worker_group"')+ await queryRunner.query('ALTER TABLE "flow" DROP COLUMN IF EXISTS "workerGroupId"')+ }+}++const isPGlite = (): boolean => system.get(AppSystemProp.DB_TYPE) === DatabaseType.PGLITEdiff --git a/packages/server/api/src/app/database/postgres-connection.ts b/packages/server/api/src/app/database/postgres-connection.tsindex ba9feb847e7..5e23fc25848 100644--- a/packages/server/api/src/app/database/postgres-connection.ts+++ b/packages/server/api/src/app/database/postgres-connection.ts@@ -452,6 +452,7 @@ import { AddPlatformModelTier1859000000000 } from './migration/postgres/18590000import { AddPieceSetRequiredActions1860000000000 } from './migration/postgres/1860000000000-AddPieceSetRequiredActions'import { AddEventDestinationEnabledAndFormat1862000000000 } from './migration/postgres/1862000000000-AddEventDestinationEnabledAndFormat'import { AddEventDestinationHeaders1863000000000 } from './migration/postgres/1863000000000-AddEventDestinationHeaders'+import { AddWorkerGroupIdToFlow1864000000000 } from './migration/postgres/1864000000000-AddWorkerGroupIdToFlow'import { AddPlatformTierFastAndConversationTier1864000000000 } from './migration/postgres/1864000000000-AddPlatformTierFastAndConversationTier'import { AddAuditLogRetentionDaysToPlatform1865000000000 } from './migration/postgres/1865000000000-AddAuditLogRetentionDaysToPlatform'import { AddAiSpecificModelsVisibleToPlatformConfiguration1866000000000 } from './migration/postgres/1866000000000-AddAiSpecificModelsVisibleToPlatformConfiguration'@@ -938,6 +939,7 @@ export const getMigrations = (): (new () => Migration)[] => {AddPieceSetRequiredActions1860000000000,AddEventDestinationEnabledAndFormat1862000000000,AddEventDestinationHeaders1863000000000,+ AddWorkerGroupIdToFlow1864000000000,AddPlatformTierFastAndConversationTier1864000000000,AddAuditLogRetentionDaysToPlatform1865000000000,AddAiSpecificModelsVisibleToPlatformConfiguration1866000000000,diff --git a/packages/server/api/src/app/ee/projects/platform-project-controller.ts b/packages/server/api/src/app/ee/projects/platform-project-controller.tsindex a8a6d1f2db1..8b6a2af78ad 100644--- a/packages/server/api/src/app/ee/projects/platform-project-controller.ts+++ b/packages/server/api/src/app/ee/projects/platform-project-controller.ts@@ -9,7 +9,6 @@ import { securityAccess } from '../../core/security/authorization/fastify-securiimport { platformService } from '../../platform/platform.service'import { projectService } from '../../project/project-service'import { userService } from '../../user/user-service'-import { machineService } from '../../workers/machine/machine-service'import { platformProjectService } from './platform-project-service'const DEFAULT_LIMIT_SIZE = 50@@ -38,19 +37,6 @@ export const platformProjectController: FastifyPluginAsyncZod = async (app) => {await reply.status(StatusCodes.CREATED).send(projectWithUsage)})- app.get('/worker-groups', ListWorkerGroupsRequest, async (request) => {- const platform = await platformService(request.log).getOneWithPlanOrThrow(request.principal.platform.id)- if (!platform.plan.workerGroupsEnabled) {- throw new ActivepiecesError({- code: ErrorCode.FEATURE_DISABLED,- params: {- message: 'Isolated workers are not enabled for this platform',- },- })- }- return machineService(request.log).listProjectWorkerGroups()- })-app.get('/', ListProjectRequestForPlatform, async (request, _reply) => {const userId = await getUserId(request.principal, request.log)const user = await userService(request.log).getOneOrFail({ id: userId })@@ -202,25 +188,6 @@ const UpdateProjectRequest = {},}-const ListWorkerGroupsRequest = {- config: {- security: securityAccess.platformAdminOnly([PrincipalType.USER, PrincipalType.SERVICE]),- },- schema: {- tags: ['projects'],- security: [SERVICE_KEY_SECURITY_OPENAPI],- response: {- [StatusCodes.OK]: z.object({- groups: z.array(z.object({- label: z.string(),- slots: z.number(),- })),- sharedSlots: z.number(),- }),- },- },-}-const CreateProjectRequest = {config: {security: securityAccess.publicPlatform([PrincipalType.USER, PrincipalType.SERVICE]),diff --git a/packages/server/api/src/app/ee/projects/platform-project-service.ts b/packages/server/api/src/app/ee/projects/platform-project-service.tsindex 67cfbad47ae..6d16f7c8d0f 100644--- a/packages/server/api/src/app/ee/projects/platform-project-service.ts+++ b/packages/server/api/src/app/ee/projects/platform-project-service.ts@@ -138,7 +138,6 @@ export const platformProjectService = (log: FastifyBaseLogger) => ({...rest,...(resolvedPoolId !== undefined ? { poolId: resolvedPoolId } : {}),...(maxConcurrentJobs !== undefined ? { maxConcurrentJobs } : {}),- ...(platformPlan.workerGroupsEnabled && workerGroupId !== undefined ? { workerGroupId } : {}),}, entityManager)if (platformPlan.globalConnectionsEnabled && globalConnectionExternalIds) {const projectGlobalConnections = await appConnectionsRepo(entityManager).find({@@ -195,13 +194,11 @@ export const platformProjectService = (log: FastifyBaseLogger) => ({if (resolvedPoolId !== undefined) {await concurrencyPoolService(log).assignProject({ projectId, poolId: resolvedPoolId })}- const previousGroupId = project.workerGroupId ?? null- if (platformPlan.workerGroupsEnabled && workerGroupId !== undefined && previousGroupId !== workerGroupId) {- await projectWorkerGroupService(log).migrateProjectToGroup({+ if (platformPlan.workerGroupsEnabled && workerGroupId !== undefined) {+ await projectWorkerGroupService(log).assignProjectToGroup({projectId,platformId: project.platformId,- fromGroupId: previousGroupId,- toGroupId: workerGroupId,+ workerGroupId,})}return this.getWithPlanAndUsageOrThrow(projectId)diff --git a/packages/server/api/src/app/flows/flow-run/flow-run-service.ts b/packages/server/api/src/app/flows/flow-run/flow-run-service.tsindex fc51add6d64..86ca41c6a0f 100644--- a/packages/server/api/src/app/flows/flow-run/flow-run-service.ts+++ b/packages/server/api/src/app/flows/flow-run/flow-run-service.ts@@ -526,7 +526,7 @@ async function cancelSingleRun(log: FastifyBaseLogger, flowRun: FlowRun, platforkey: `runs_metadata_${flowRun.id}`,timeoutInSeconds: 30,fn: async () => {- await jobQueue(log).removeAllFlowRunJobs({ flowRunId: flowRun.id, platformId, projectId: flowRun.projectId })+ await jobQueue(log).removeAllFlowRunJobs({ flowRunId: flowRun.id, platformId, projectId: flowRun.projectId, flowId: flowRun.flowId })await waitpointService(log).deleteByFlowRunId({ flowRunId: flowRun.id, projectId: flowRun.projectId })await runsMetadataQueue(log).add({id: flowRun.id,diff --git a/packages/server/api/src/app/flows/flow/flow.controller.ts b/packages/server/api/src/app/flows/flow/flow.controller.tsindex f4812adf393..c2882c0e865 100644--- a/packages/server/api/src/app/flows/flow/flow.controller.ts+++ b/packages/server/api/src/app/flows/flow/flow.controller.ts@@ -299,7 +299,7 @@ const GetFlowRequestOptions = {const DeleteFlowRequestOptions = {config: {security: securityAccess.project(- [PrincipalType.USER, PrincipalType.SERVICE],+ [PrincipalType.USER, PrincipalType.SERVICE],Permission.WRITE_FLOW, {type: ProjectResourceType.TABLE,tableName: FlowEntity,@@ -317,3 +317,4 @@ const DeleteFlowRequestOptions = {},},}+diff --git a/packages/server/api/src/app/flows/flow/flow.entity.ts b/packages/server/api/src/app/flows/flow/flow.entity.tsindex 17cbd372f99..7a868547133 100644--- a/packages/server/api/src/app/flows/flow/flow.entity.ts+++ b/packages/server/api/src/app/flows/flow/flow.entity.ts@@ -79,6 +79,10 @@ export const FlowEntity = new EntitySchema<FlowSchema>({type: 'jsonb',nullable: true,},+ workerGroupId: {+ type: String,+ nullable: true,+ },},indices: [{@@ -112,6 +116,11 @@ export const FlowEntity = new EntitySchema<FlowSchema>({columns: ['updated'],where: `"operationStatus" = '${FlowOperationStatus.DELETING}'`,},+ {+ name: 'idx_flow_worker_group',+ columns: ['workerGroupId'],+ unique: false,+ },],relations: {runs: {diff --git a/packages/server/api/src/app/project/project-service.ts b/packages/server/api/src/app/project/project-service.tsindex 05fe9f4bc59..8543a809be6 100644--- a/packages/server/api/src/app/project/project-service.ts+++ b/packages/server/api/src/app/project/project-service.ts@@ -79,7 +79,6 @@ export const projectService = (log: FastifyBaseLogger) => ({...spreadIfDefined('sensitive', request.sensitive),...(request.poolId !== undefined ? { poolId: request.poolId } : {}),...(request.maxConcurrentJobs !== undefined ? { maxConcurrentJobs: request.maxConcurrentJobs } : {}),- ...(request.workerGroupId !== undefined ? { workerGroupId: request.workerGroupId } : {}),...spreadIfNotUndefined('executionDataRetentionDays', request.executionDataRetentionDays),}@@ -89,9 +88,6 @@ export const projectService = (log: FastifyBaseLogger) => ({} : {}await projectRepo(entityManager).update({ id: projectId }, { ...baseUpdate, ...teamUpdate })- if (request.workerGroupId !== undefined) {- await projectWorkerGroupService(log).invalidate({ projectId })- }return this.getOneOrThrow(projectId)},@@ -293,7 +289,6 @@ type UpdateTeamProjectParams = {metadata?: MetadatapoolId?: string | nullmaxConcurrentJobs?: number | null- workerGroupId?: string | nullexecutionDataRetentionDays?: number | nullicon?: ProjectIconsensitive?: boolean@@ -307,7 +302,6 @@ type UpdatePersonalProjectParams = {metadata?: MetadatapoolId?: string | nullmaxConcurrentJobs?: number | null- workerGroupId?: string | nullexecutionDataRetentionDays?: number | nullsensitive?: boolean}diff --git a/packages/server/api/src/app/workers/job-queue/job-queue.ts b/packages/server/api/src/app/workers/job-queue/job-queue.tsindex 23f81ba551f..d78dbd2052d 100644--- a/packages/server/api/src/app/workers/job-queue/job-queue.ts+++ b/packages/server/api/src/app/workers/job-queue/job-queue.ts@@ -25,7 +25,8 @@ export const jobQueue = (log: FastifyBaseLogger) => ({const platformId = data.platformIdconst projectId = 'projectId' in data ? data.projectId : null- const queueName = await workerGroupRouting.resolveQueueForJob({ platformId, projectId, jobType: data.jobType, log })+ const flowId = 'flowId' in data ? data.flowId : null+ const queueName = await workerGroupRouting.resolveQueueForJob({ platformId, projectId, flowId, jobType: data.jobType, log })const queue = await ensureQueueExists({ log, queueName })switch (type) {@@ -76,8 +77,8 @@ export const jobQueue = (log: FastifyBaseLogger) => ({}, '[jobQueue#removeRepeatingJob] removed jobs from all queues')},- async removeOneTimeJob({ jobId, platformId, projectId, jobType }: RemoveOneTimeJobParams): Promise<void> {- const queueName = await workerGroupRouting.resolveQueueForJob({ platformId, projectId, jobType, log })+ async removeOneTimeJob({ jobId, platformId, projectId, flowId, jobType }: RemoveOneTimeJobParams): Promise<void> {+ const queueName = await workerGroupRouting.resolveQueueForJob({ platformId, projectId, flowId, jobType, log })const queue = await ensureQueueExists({ log, queueName })const job = await queue.getJob(jobId)if (!isNil(job)) {@@ -128,8 +129,8 @@ export const jobQueue = (log: FastifyBaseLogger) => ({}return queue},- async removeAllFlowRunJobs({ flowRunId, platformId, projectId }: RemoveAllFlowRunJobsParams): Promise<void> {- const queueName = await workerGroupRouting.resolveQueueForJob({ platformId, projectId, jobType: WorkerJobType.EXECUTE_FLOW, log })+ async removeAllFlowRunJobs({ flowRunId, platformId, projectId, flowId }: RemoveAllFlowRunJobsParams): Promise<void> {+ const queueName = await workerGroupRouting.resolveQueueForJob({ platformId, projectId, flowId, jobType: WorkerJobType.EXECUTE_FLOW, log })const queue = await ensureQueueExists({ log, queueName })const allJobs = await queue.getJobs(['waiting', 'delayed'])const matching = allJobs.filter((j) => j.id?.startsWith(flowRunId))@@ -225,6 +226,7 @@ type RemoveOneTimeJobParams = {jobId: ApIdplatformId: string | nullprojectId?: string | null+ flowId?: string | nulljobType?: WorkerJobType}@@ -232,6 +234,7 @@ type RemoveAllFlowRunJobsParams = {flowRunId: stringplatformId: string | nullprojectId?: string | null+ flowId?: string | null}type BaseAddParams<JD extends Omit<JobData, 'engineToken'>, JT extends JobType> = {diff --git a/packages/server/api/src/app/workers/machine/machine-controller.ts b/packages/server/api/src/app/workers/machine/machine-controller.tsindex 358296f63b0..a8a723eaa9f 100644--- a/packages/server/api/src/app/workers/machine/machine-controller.ts+++ b/packages/server/api/src/app/workers/machine/machine-controller.ts@@ -18,8 +18,17 @@ export const workerMachineController: FastifyPluginAsyncZod = async (app) => {websocketService.addListener(PrincipalType.WORKER, WebsocketServerEvent.FETCH_WORKER_SETTINGS, (socket) => {return async (request: WorkerMachineHealthcheckRequest, _principal, _projectId, callback?: (data: unknown) => void) => {const rawWorkerGroupValue = socket.handshake.auth?.workerGroupId+ // AP_PROJECT_WORKER is deprecated in favor of AP_WORKER_GROUP_SCOPE but kept for backward compatibility.const projectWorker = socket.handshake.auth?.projectWorker === true- const assignment = workerGroupQueues.parseWorkerGroupValue({ value: typeof rawWorkerGroupValue === 'string' ? rawWorkerGroupValue : undefined, projectWorker })+ const rawWorkerGroupScope = socket.handshake.auth?.workerGroupScope+ const scope = workerGroupQueues.resolveScope({+ projectWorker,+ scope: typeof rawWorkerGroupScope === 'string' ? rawWorkerGroupScope : undefined,+ })+ const assignment = workerGroupQueues.parseWorkerGroupValue({+ value: typeof rawWorkerGroupValue === 'string' ? rawWorkerGroupValue : undefined,+ scope,+ })const response = await machineService(app.log).onConnection(request, assignment)callback?.(response)createRpcServer<WorkerToApiContract>(socket, createHandlers(app.log, assignment, socket.id), app.log)diff --git a/packages/server/api/src/app/workers/machine/machine-service.ts b/packages/server/api/src/app/workers/machine/machine-service.tsindex 7380a446cbc..464e47b7254 100644--- a/packages/server/api/src/app/workers/machine/machine-service.ts+++ b/packages/server/api/src/app/workers/machine/machine-service.ts@@ -101,7 +101,7 @@ export const machineService = (log: FastifyBaseLogger) => {if (worker.workerGroupScope === WorkerGroupScope.PLATFORM) {return !isNil(platformWorkerGroupId) && worker.workerGroupId === platformWorkerGroupId}- if (worker.workerGroupScope === WorkerGroupScope.PROJECT) {+ if (worker.workerGroupScope === WorkerGroupScope.PROJECT || worker.workerGroupScope === WorkerGroupScope.FLOW) {return true}return isNil(platformWorkerGroupId)diff --git a/packages/server/api/src/app/workers/worker-groups/flow-worker-group.service.ts b/packages/server/api/src/app/workers/worker-groups/flow-worker-group.service.tsnew file mode 100644index 00000000000..d48c96f93d0--- /dev/null+++ b/packages/server/api/src/app/workers/worker-groups/flow-worker-group.service.ts@@ -0,0 +1,73 @@+import { ActivepiecesError, ErrorCode, isNil, tryCatch } from '@activepieces/core-utils'+import { apDayjsDuration } from '@activepieces/server-utils'+import { WorkerGroupScope, WorkerJobType } from '@activepieces/shared'+import { FastifyBaseLogger } from 'fastify'+import { distributedStore } from '../../database/redis-connections'+import { flowRepo } from '../../flows/flow/flow.repo'+import { platformQueueMigrationService } from './queue-migration.service'+import { NO_WORKER_GROUP_SENTINEL, workerGroupQueues } from './worker-group-queues'+import { workerGroupRouting } from './worker-group-routing'++export const flowWorkerGroupService = (log: FastifyBaseLogger) => ({+ async getFlowWorkerGroup({ flowId }: { flowId: string }): Promise<string | null> {+ const cached = await distributedStore.get<string>(getFlowWorkerGroupCacheKey(flowId))+ if (!isNil(cached)) {+ return cached === NO_WORKER_GROUP_SENTINEL ? null : cached+ }++ const flow = await flowRepo().findOne({+ select: ['id', 'workerGroupId'],+ where: { id: flowId },+ })++ const stored = flow?.workerGroupId ?? null+ const workerGroupId = !isNil(stored) && workerGroupQueues.isRoutableWorkerGroupId(stored) ? stored : null+ await distributedStore.put(getFlowWorkerGroupCacheKey(flowId), workerGroupId ?? NO_WORKER_GROUP_SENTINEL, CACHE_TTL_SECONDS)+ return workerGroupId+ },++ async invalidate({ flowId }: { flowId: string }): Promise<void> {+ await distributedStore.delete(getFlowWorkerGroupCacheKey(flowId))+ },++ async assignFlowToGroup({ flowId, projectId, platformId, workerGroupId }: AssignFlowToGroupParams): Promise<void> {+ if (!isNil(workerGroupId) && !workerGroupQueues.isRoutableWorkerGroupId(workerGroupId)) {+ throw new ActivepiecesError({+ code: ErrorCode.VALIDATION,+ params: {+ message: `"${workerGroupId}" is not a valid worker group id`,+ },+ })+ }+ const flow = await flowRepo().findOneOrFail({+ select: ['id', 'workerGroupId'],+ where: { id: flowId, projectId },+ })+ const previousGroupId = flow.workerGroupId ?? null+ await flowRepo().update({ id: flowId, projectId }, { workerGroupId })+ await this.invalidate({ flowId })+ if (previousGroupId === workerGroupId) {+ return+ }+ const ungroupedQueueName = await workerGroupRouting.resolveQueueForJob({ platformId, projectId, jobType: WorkerJobType.EXECUTE_FLOW, log })+ const { error } = await tryCatch(() => platformQueueMigrationService(log).migrateFlowJobs({+ fromQueueName: isNil(previousGroupId) ? ungroupedQueueName : workerGroupQueues.getQueueName({ scope: WorkerGroupScope.FLOW, workerGroupId: previousGroupId }),+ toQueueName: isNil(workerGroupId) ? ungroupedQueueName : workerGroupQueues.getQueueName({ scope: WorkerGroupScope.FLOW, workerGroupId }),+ flowId,+ }))+ if (error) {+ log.error({ flow: { id: flowId }, project: { id: projectId }, error }, '[flowWorkerGroupService#assignFlowToGroup] Failed to migrate waiting jobs to the new worker group queue')+ }+ },+})++const getFlowWorkerGroupCacheKey = (flowId: string): string => `flow:${flowId}:worker_group`++const CACHE_TTL_SECONDS = apDayjsDuration(5, 'minute').asSeconds()++type AssignFlowToGroupParams = {+ flowId: string+ projectId: string+ platformId: string+ workerGroupId: string | null+}diff --git a/packages/server/api/src/app/workers/worker-groups/platform-worker-group.service.ts b/packages/server/api/src/app/workers/worker-groups/platform-worker-group.service.tsindex 45374087aac..7979685be32 100644--- a/packages/server/api/src/app/workers/worker-groups/platform-worker-group.service.ts+++ b/packages/server/api/src/app/workers/worker-groups/platform-worker-group.service.ts@@ -1,5 +1,6 @@import { isNil } from '@activepieces/core-utils'import { apDayjsDuration } from '@activepieces/server-utils'+import { WorkerGroupScope } from '@activepieces/shared'import { FastifyBaseLogger } from 'fastify'import { distributedStore } from '../../database/redis-connections'import { platformPlanRepo } from '../../ee/platform/platform-plan/platform-plan.service'@@ -65,8 +66,8 @@ export const platformWorkerGroupService = (log: FastifyBaseLogger) => ({async moveJobsToTargetQueue({ platformId, workerGroupId }: { platformId: string, workerGroupId: string | null }): Promise<void> {const currentGroupId = await platformWorkerGroupService(log).getWorkerGroupId({ platformId })- const targetQueue = isNil(workerGroupId) ? QueueName.WORKER_JOBS : workerGroupQueues.getPlatformGroupQueueName(workerGroupId)- const fromQueueName = isNil(currentGroupId) ? QueueName.WORKER_JOBS : workerGroupQueues.getPlatformGroupQueueName(currentGroupId)+ const targetQueue = isNil(workerGroupId) ? QueueName.WORKER_JOBS : workerGroupQueues.getQueueName({ scope: WorkerGroupScope.PLATFORM, workerGroupId })+ const fromQueueName = isNil(currentGroupId) ? QueueName.WORKER_JOBS : workerGroupQueues.getQueueName({ scope: WorkerGroupScope.PLATFORM, workerGroupId: currentGroupId })await platformQueueMigrationService(log).migrateJobs({ fromQueueName, toQueueName: targetQueue, platformId })},})diff --git a/packages/server/api/src/app/workers/worker-groups/project-worker-group.service.ts b/packages/server/api/src/app/workers/worker-groups/project-worker-group.service.tsindex a36ba595eed..65d01670b05 100644--- a/packages/server/api/src/app/workers/worker-groups/project-worker-group.service.ts+++ b/packages/server/api/src/app/workers/worker-groups/project-worker-group.service.ts@@ -1,12 +1,13 @@-import { isNil, spreadIfDefined, tryCatch } from '@activepieces/core-utils'+import { ActivepiecesError, ErrorCode, isNil, spreadIfDefined, tryCatch } from '@activepieces/core-utils'import { apDayjsDuration } from '@activepieces/server-utils'+import { WorkerGroupScope } from '@activepieces/shared'import { FastifyBaseLogger } from 'fastify'import { distributedStore } from '../../database/redis-connections'import { projectRepo } from '../../project/project-repo'import { QueueName } from '../job'import { platformWorkerGroupService } from './platform-worker-group.service'import { platformQueueMigrationService } from './queue-migration.service'-import { workerGroupQueues } from './worker-group-queues'+import { NO_WORKER_GROUP_SENTINEL, workerGroupQueues } from './worker-group-queues'export const projectWorkerGroupService = (log: FastifyBaseLogger) => ({async getProjectWorkerGroup({ projectId, platformId }: { projectId: string, platformId?: string | null }): Promise<string | null> {@@ -16,11 +17,12 @@ export const projectWorkerGroupService = (log: FastifyBaseLogger) => ({}const project = await projectRepo().findOne({- select: ['workerGroupId'],+ select: ['id', 'workerGroupId'],where: { id: projectId, ...spreadIfDefined('platformId', platformId ?? undefined) },})- const workerGroupId = project?.workerGroupId ?? null+ const stored = project?.workerGroupId ?? null+ const workerGroupId = !isNil(stored) && workerGroupQueues.isRoutableWorkerGroupId(stored) ? stored : nullawait distributedStore.put(getProjectWorkerGroupCacheKey(projectId), workerGroupId ?? NO_WORKER_GROUP_SENTINEL, CACHE_TTL_SECONDS)return workerGroupId},@@ -36,32 +38,57 @@ export const projectWorkerGroupService = (log: FastifyBaseLogger) => ({await distributedStore.delete(getProjectWorkerGroupCacheKey(projectId))},- async migrateProjectToGroup({ projectId, platformId, fromGroupId, toGroupId }: MigrateProjectToGroupParams): Promise<void> {+ async assignProjectToGroup({ projectId, platformId, workerGroupId }: AssignProjectToGroupParams): Promise<void> {+ if (!isNil(workerGroupId) && !workerGroupQueues.isRoutableWorkerGroupId(workerGroupId)) {+ throw new ActivepiecesError({+ code: ErrorCode.VALIDATION,+ params: {+ message: `"${workerGroupId}" is not a valid worker group id`,+ },+ })+ }+ const project = await projectRepo().findOne({+ select: ['id', 'workerGroupId'],+ where: { id: projectId, platformId },+ })+ if (isNil(project)) {+ throw new ActivepiecesError({+ code: ErrorCode.ENTITY_NOT_FOUND,+ params: {+ entityId: projectId,+ entityType: 'project',+ },+ })+ }+ const previousGroupId = project.workerGroupId ?? null+ await projectRepo().update({ id: projectId, platformId }, { workerGroupId })+ await this.invalidate({ projectId })+ if (previousGroupId === workerGroupId) {+ return+ }const ungroupedQueueName = await resolveUngroupedQueueName({ platformId, log })const { error } = await tryCatch(() => platformQueueMigrationService(log).migrateProjectJobs({- fromQueueName: isNil(fromGroupId) ? ungroupedQueueName : workerGroupQueues.getProjectGroupQueueName(fromGroupId),- toQueueName: isNil(toGroupId) ? ungroupedQueueName : workerGroupQueues.getProjectGroupQueueName(toGroupId),+ fromQueueName: isNil(previousGroupId) ? ungroupedQueueName : workerGroupQueues.getQueueName({ scope: WorkerGroupScope.PROJECT, workerGroupId: previousGroupId }),+ toQueueName: isNil(workerGroupId) ? ungroupedQueueName : workerGroupQueues.getQueueName({ scope: WorkerGroupScope.PROJECT, workerGroupId }),projectId,}))if (error) {- log.error({ project: { id: projectId }, platform: { id: platformId }, error }, '[projectWorkerGroupService#migrateProjectToGroup] Failed to migrate waiting jobs to the new worker group queue')+ log.error({ project: { id: projectId }, platform: { id: platformId }, error }, '[projectWorkerGroupService#assignProjectToGroup] Failed to migrate waiting jobs to the new worker group queue')}},})async function resolveUngroupedQueueName({ platformId, log }: { platformId: string, log: FastifyBaseLogger }): Promise<string> {const platformGroupId = await platformWorkerGroupService(log).getWorkerGroupId({ platformId })- return isNil(platformGroupId) ? QueueName.WORKER_JOBS : workerGroupQueues.getPlatformGroupQueueName(platformGroupId)+ return isNil(platformGroupId) ? QueueName.WORKER_JOBS : workerGroupQueues.getQueueName({ scope: WorkerGroupScope.PLATFORM, workerGroupId: platformGroupId })}const getProjectWorkerGroupCacheKey = (projectId: string): string => `project:${projectId}:worker_group`-const NO_WORKER_GROUP_SENTINEL = '__none__'const CACHE_TTL_SECONDS = apDayjsDuration(5, 'minute').asSeconds()-type MigrateProjectToGroupParams = {+type AssignProjectToGroupParams = {projectId: stringplatformId: string- fromGroupId: string | null- toGroupId: string | null+ workerGroupId: string | null}diff --git a/packages/server/api/src/app/workers/worker-groups/queue-migration.service.ts b/packages/server/api/src/app/workers/worker-groups/queue-migration.service.tsindex 010ca77f7e7..dc875ba95c1 100644--- a/packages/server/api/src/app/workers/worker-groups/queue-migration.service.ts+++ b/packages/server/api/src/app/workers/worker-groups/queue-migration.service.ts@@ -24,6 +24,15 @@ export const platformQueueMigrationService = (log: FastifyBaseLogger) => ({await migrateRegularJobs({ sourceQueue, targetQueue, batchSize, belongsTo: job => job.data?.projectId === projectId && PROJECT_GROUP_ROUTABLE_JOB_TYPES.has(job.data?.jobType) })},+ async migrateFlowJobs({ fromQueueName, toQueueName, flowId, batchSize = 200 }: MigrateFlowJobsParams): Promise<void> {+ if (fromQueueName === toQueueName) {+ return+ }+ const sourceQueue = await jobQueue(log).getOrCreateQueue({ queueName: fromQueueName })+ const targetQueue = await jobQueue(log).getOrCreateQueue({ queueName: toQueueName })++ await migrateRegularJobs({ sourceQueue, targetQueue, batchSize, belongsTo: job => job.data?.flowId === flowId && PROJECT_GROUP_ROUTABLE_JOB_TYPES.has(job.data?.jobType) })+ },})async function migrateRegularJobs({ sourceQueue, targetQueue, belongsTo, batchSize }: MigrateRegularJobsParams): Promise<void> {@@ -178,6 +187,13 @@ type MigrateProjectJobsParams = {batchSize?: number}+type MigrateFlowJobsParams = {+ fromQueueName: string+ toQueueName: string+ flowId: string+ batchSize?: number+}+type MigrateRegularJobsParams = {sourceQueue: QueuetargetQueue: Queuediff --git a/packages/server/api/src/app/workers/worker-groups/worker-group-queues.ts b/packages/server/api/src/app/workers/worker-groups/worker-group-queues.tsindex 3248d23f444..f0fd19d24c9 100644--- a/packages/server/api/src/app/workers/worker-groups/worker-group-queues.ts+++ b/packages/server/api/src/app/workers/worker-groups/worker-group-queues.ts@@ -1,31 +1,49 @@import { isNil } from '@activepieces/core-utils'import { WorkerGroupScope, WorkerJobType } from '@activepieces/shared'-const parseWorkerGroupValue = ({ value, projectWorker }: { value: string | undefined, projectWorker: boolean }): WorkerGroupAssignment | null => {+const parseWorkerGroupValue = ({ value, scope }: ParseWorkerGroupValueParams): WorkerGroupAssignment | null => {if (isNil(value) || value.length === 0) {return null}return {- scope: projectWorker ? WorkerGroupScope.PROJECT : WorkerGroupScope.PLATFORM,+ scope,id: value,}}-const getPlatformGroupQueueName = (workerGroupId: string): string => {- // TODO Rename this to workerGroups-workerGroupId-jobs in the future and migrate existings jobs there.- return `platform-${workerGroupId}-jobs`+const resolveScope = ({ projectWorker, scope }: ResolveScopeParams): WorkerGroupScope => {+ const explicitScope = Object.values(WorkerGroupScope).find((knownScope) => knownScope === scope)+ if (!isNil(explicitScope)) {+ return explicitScope+ }+ return projectWorker ? WorkerGroupScope.PROJECT : WorkerGroupScope.PLATFORM+}++const getQueueName = ({ scope, workerGroupId }: GetQueueNameParams): string => {+ switch (scope) {+ case WorkerGroupScope.FLOW:+ return `flow-${workerGroupId}-jobs`+ case WorkerGroupScope.PROJECT:+ return `project-${workerGroupId}-jobs`+ case WorkerGroupScope.PLATFORM:+ // TODO Rename this to workerGroups-workerGroupId-jobs in the future and migrate existings jobs there.+ return `platform-${workerGroupId}-jobs`+ }}-const getProjectGroupQueueName = (workerGroupId: string): string => {- return `project-${workerGroupId}-jobs`+const isRoutableWorkerGroupId = (value: string): boolean => {+ return value.length > 0 && value !== NO_WORKER_GROUP_SENTINEL}export const workerGroupQueues = {parseWorkerGroupValue,- getPlatformGroupQueueName,- getProjectGroupQueueName,+ resolveScope,+ getQueueName,+ isRoutableWorkerGroupId,}+export const NO_WORKER_GROUP_SENTINEL = '__none__'+export const PROJECT_GROUP_ROUTABLE_JOB_TYPES = new Set<WorkerJobType>([WorkerJobType.EXECUTE_FLOW,WorkerJobType.EXECUTE_WEBHOOK,@@ -35,3 +53,18 @@ export type WorkerGroupAssignment = {scope: WorkerGroupScopeid: string}++type ParseWorkerGroupValueParams = {+ value: string | undefined+ scope: WorkerGroupScope+}++type ResolveScopeParams = {+ projectWorker: boolean+ scope: string | undefined+}++type GetQueueNameParams = {+ scope: WorkerGroupScope+ workerGroupId: string+}diff --git a/packages/server/api/src/app/workers/worker-groups/worker-group-routing.ts b/packages/server/api/src/app/workers/worker-groups/worker-group-routing.tsindex 9e64928d669..724a7ee9633 100644--- a/packages/server/api/src/app/workers/worker-groups/worker-group-routing.ts+++ b/packages/server/api/src/app/workers/worker-groups/worker-group-routing.ts@@ -2,17 +2,24 @@ import { isNil } from '@activepieces/core-utils'import { WorkerGroupScope, WorkerJobType } from '@activepieces/shared'import { FastifyBaseLogger } from 'fastify'import { QueueName } from '../job'+import { flowWorkerGroupService } from './flow-worker-group.service'import { platformWorkerGroupService } from './platform-worker-group.service'import { projectWorkerGroupService } from './project-worker-group.service'import { PROJECT_GROUP_ROUTABLE_JOB_TYPES, WorkerGroupAssignment, workerGroupQueues } from './worker-group-queues'-const resolveQueueForJob = async ({ platformId, projectId, jobType, log }: ResolveQueueForJobParams): Promise<string> => {+const resolveQueueForJob = async ({ platformId, projectId, flowId, jobType, log }: ResolveQueueForJobParams): Promise<string> => {if (!isNil(platformId) && !isNil(projectId) && !isNil(jobType) && PROJECT_GROUP_ROUTABLE_JOB_TYPES.has(jobType)) {const workerGroupsEnabled = await platformWorkerGroupService(log).isWorkerGroupsEnabled({ platformId })if (workerGroupsEnabled) {+ if (!isNil(flowId)) {+ const flowGroupId = await flowWorkerGroupService(log).getFlowWorkerGroup({ flowId })+ if (!isNil(flowGroupId)) {+ return workerGroupQueues.getQueueName({ scope: WorkerGroupScope.FLOW, workerGroupId: flowGroupId })+ }+ }const projectGroupId = await projectWorkerGroupService(log).getProjectWorkerGroup({ projectId, platformId })if (!isNil(projectGroupId)) {- return workerGroupQueues.getProjectGroupQueueName(projectGroupId)+ return workerGroupQueues.getQueueName({ scope: WorkerGroupScope.PROJECT, workerGroupId: projectGroupId })}}}@@ -23,16 +30,14 @@ const resolveQueueForJob = async ({ platformId, projectId, jobType, log }: Resolif (isNil(groupId)) {return QueueName.WORKER_JOBS}- return workerGroupQueues.getPlatformGroupQueueName(groupId)+ return workerGroupQueues.getQueueName({ scope: WorkerGroupScope.PLATFORM, workerGroupId: groupId })}const resolveQueueForWorker = (assignment: WorkerGroupAssignment | null): string => {if (isNil(assignment)) {return QueueName.WORKER_JOBS}- return assignment.scope === WorkerGroupScope.PROJECT- ? workerGroupQueues.getProjectGroupQueueName(assignment.id)- : workerGroupQueues.getPlatformGroupQueueName(assignment.id)+ return workerGroupQueues.getQueueName({ scope: assignment.scope, workerGroupId: assignment.id })}export const workerGroupRouting = {@@ -43,6 +48,7 @@ export const workerGroupRouting = {type ResolveQueueForJobParams = {platformId: string | nullprojectId?: string | null+ flowId?: string | nulljobType?: WorkerJobTypelog: FastifyBaseLogger}diff --git a/packages/server/api/src/app/workers/worker-groups/worker-group.controller.ts b/packages/server/api/src/app/workers/worker-groups/worker-group.controller.tsnew file mode 100644index 00000000000..7346f1433a8--- /dev/null+++ b/packages/server/api/src/app/workers/worker-groups/worker-group.controller.ts@@ -0,0 +1,119 @@+import { ActivepiecesError, ApId, ErrorCode, isNil } from '@activepieces/core-utils'+import { PrincipalType, SERVICE_KEY_SECURITY_OPENAPI } from '@activepieces/shared'+import { FastifyBaseLogger } from 'fastify'+import { FastifyPluginAsyncZod } from 'fastify-type-provider-zod'+import { StatusCodes } from 'http-status-codes'+import { z } from 'zod'+import { securityAccess } from '../../core/security/authorization/fastify-security'+import { flowRepo } from '../../flows/flow/flow.repo'+import { projectService } from '../../project/project-service'+import { machineService } from '../machine/machine-service'+import { flowWorkerGroupService } from './flow-worker-group.service'+import { platformWorkerGroupService } from './platform-worker-group.service'+import { projectWorkerGroupService } from './project-worker-group.service'++export const workerGroupController: FastifyPluginAsyncZod = async (app) => {+ app.get('/', ListWorkerGroupsRequestOptions, async (request) => {+ await assertWorkerGroupsEnabled(request.principal.platform.id, request.log)+ return machineService(request.log).listProjectWorkerGroups()+ })++ app.post('/projects/:id', UpdateProjectWorkerGroupRequestOptions, async (request, reply) => {+ const platformId = request.principal.platform.id+ await assertWorkerGroupsEnabled(platformId, request.log)+ await projectWorkerGroupService(request.log).assignProjectToGroup({+ projectId: request.params.id,+ platformId,+ workerGroupId: request.body.workerGroupId,+ })+ return reply.status(StatusCodes.OK).send()+ })++ app.post('/flows/:id', UpdateFlowWorkerGroupRequestOptions, async (request, reply) => {+ const flow = await flowRepo().findOneBy({ id: request.params.id })+ const platformId = isNil(flow) ? null : await projectService(request.log).getPlatformId(flow.projectId)+ if (isNil(flow) || platformId !== request.principal.platform.id) {+ throw new ActivepiecesError({+ code: ErrorCode.ENTITY_NOT_FOUND,+ params: {+ entityId: request.params.id,+ entityType: 'flow',+ },+ })+ }+ await assertWorkerGroupsEnabled(platformId, request.log)+ await flowWorkerGroupService(request.log).assignFlowToGroup({+ flowId: flow.id,+ projectId: flow.projectId,+ platformId,+ workerGroupId: request.body.workerGroupId,+ })+ return reply.status(StatusCodes.OK).send()+ })+}++async function assertWorkerGroupsEnabled(platformId: string, log: FastifyBaseLogger): Promise<void> {+ const workerGroupsEnabled = await platformWorkerGroupService(log).isWorkerGroupsEnabled({ platformId })+ if (!workerGroupsEnabled) {+ throw new ActivepiecesError({+ code: ErrorCode.FEATURE_DISABLED,+ params: {+ message: 'Worker groups are not enabled for this platform',+ },+ })+ }+}++const WorkerGroupAssignmentBody = z.object({+ workerGroupId: z.string().min(1).nullable(),+})++const ListWorkerGroupsRequestOptions = {+ config: {+ security: securityAccess.platformAdminOnly([PrincipalType.USER, PrincipalType.SERVICE]),+ },+ schema: {+ tags: ['worker-groups'],+ security: [SERVICE_KEY_SECURITY_OPENAPI],+ description: 'List worker groups and their capacity',+ response: {+ [StatusCodes.OK]: z.object({+ groups: z.array(z.object({+ label: z.string(),+ slots: z.number(),+ })),+ sharedSlots: z.number(),+ }),+ },+ },+}++const UpdateProjectWorkerGroupRequestOptions = {+ config: {+ security: securityAccess.platformAdminOnly([PrincipalType.USER, PrincipalType.SERVICE]),+ },+ schema: {+ tags: ['worker-groups'],+ security: [SERVICE_KEY_SECURITY_OPENAPI],+ description: 'Assign a project to a worker group',+ params: z.object({+ id: ApId,+ }),+ body: WorkerGroupAssignmentBody,+ },+}++const UpdateFlowWorkerGroupRequestOptions = {+ config: {+ security: securityAccess.platformAdminOnly([PrincipalType.USER, PrincipalType.SERVICE]),+ },+ schema: {+ tags: ['worker-groups'],+ security: [SERVICE_KEY_SECURITY_OPENAPI],+ description: 'Assign a flow to a worker group',+ params: z.object({+ id: ApId,+ }),+ body: WorkerGroupAssignmentBody,+ },+}diff --git a/packages/server/api/src/app/workers/worker-module.ts b/packages/server/api/src/app/workers/worker-module.tsindex b222b338a12..8bb724e0560 100644--- a/packages/server/api/src/app/workers/worker-module.ts+++ b/packages/server/api/src/app/workers/worker-module.ts@@ -11,6 +11,7 @@ import { jobBroker } from './job-queue/job-broker'import { jobQueue } from './job-queue/job-queue'import { workerMachineController } from './machine/machine-controller'import { queueMigration } from './migrations/queue-migration-runner'+import { workerGroupController } from './worker-groups/worker-group.controller'export const workerModule: FastifyPluginAsyncZod = async (app) => {await app.register(flowEngineWorker, {prefix: '/v1/engine',@@ -18,6 +19,9 @@ export const workerModule: FastifyPluginAsyncZod = async (app) => {await app.register(workerMachineController, {prefix: '/v1/worker-machines',})+ await app.register(workerGroupController, {+ prefix: '/v1/worker-groups',+ })await jobQueue(app.log).init()await runsMetadataQueue(app.log).init()diff --git a/packages/server/api/test/integration/ce/workers/platform-queue-migration.test.ts b/packages/server/api/test/integration/ce/workers/platform-queue-migration.test.tsindex 44455828cea..5ae1c7ce8dc 100644--- a/packages/server/api/test/integration/ce/workers/platform-queue-migration.test.ts+++ b/packages/server/api/test/integration/ce/workers/platform-queue-migration.test.ts@@ -1,5 +1,5 @@import { apId } from '@activepieces/core-utils'-import { FlowTriggerType, LATEST_JOB_DATA_SCHEMA_VERSION, WorkerJobType } from '@activepieces/shared'+import { FlowTriggerType, LATEST_JOB_DATA_SCHEMA_VERSION, WorkerGroupScope, WorkerJobType } from '@activepieces/shared'import { FastifyInstance } from 'fastify'import { workerGroupQueues } from '../../../../src/app/workers/worker-groups/worker-group-queues'import { jobQueue } from '../../../../src/app/workers/job-queue/job-queue'@@ -21,8 +21,8 @@ describe('platformQueueMigrationService', () => {let toQueueName: stringbeforeEach(() => {- fromQueueName = workerGroupQueues.getPlatformGroupQueueName(apId())- toQueueName = workerGroupQueues.getPlatformGroupQueueName(apId())+ fromQueueName = workerGroupQueues.getQueueName({ scope: WorkerGroupScope.PLATFORM, workerGroupId: apId() })+ toQueueName = workerGroupQueues.getQueueName({ scope: WorkerGroupScope.PLATFORM, workerGroupId: apId() })})afterEach(async () => {@@ -35,7 +35,7 @@ describe('platformQueueMigrationService', () => {it('returns early without touching queues when fromQueueName equals toQueueName', async () => {const platformId = apId()const flowVersionId = apId()- const sameQueue = workerGroupQueues.getPlatformGroupQueueName(apId())+ const sameQueue = workerGroupQueues.getQueueName({ scope: WorkerGroupScope.PLATFORM, workerGroupId: apId() })const queue = await jobQueue(app.log).getOrCreateQueue({ queueName: sameQueue })await queue.upsertJobScheduler(@@ -188,6 +188,43 @@ describe('platformQueueMigrationService', () => {expect(await toQueue.getWaitingCount()).toBe(1)})+ describe('migrateFlowJobs', () => {+ it('moves only the target flow executable jobs', async () => {+ const flowA = apId()+ const flowB = apId()+ const fromQueue = await jobQueue(app.log).getOrCreateQueue({ queueName: fromQueueName })+ const toQueue = await jobQueue(app.log).getOrCreateQueue({ queueName: toQueueName })++ const jobIdA = apId()+ const jobIdB = apId()+ await fromQueue.add(jobIdA, buildWebhookJobData({ flowId: flowA }), { jobId: jobIdA })+ await fromQueue.add(jobIdB, buildWebhookJobData({ flowId: flowB }), { jobId: jobIdB })++ await platformQueueMigrationService(app.log).migrateFlowJobs({ fromQueueName, toQueueName, flowId: flowA })++ expect(await fromQueue.getWaitingCount()).toBe(1)+ expect(await toQueue.getWaitingCount()).toBe(1)++ const [movedJob] = await toQueue.getJobs(['waiting'], 0, 0)+ expect(movedJob.data.flowId).toBe(flowA)+ })++ it('does not move non-routable job types that carry the same flowId', async () => {+ const flowId = apId()+ const fromQueue = await jobQueue(app.log).getOrCreateQueue({ queueName: fromQueueName })+ const toQueue = await jobQueue(app.log).getOrCreateQueue({ queueName: toQueueName })++ const jobId = apId()+ const pollingData = { ...buildPollingJobData({ platformId: apId(), flowVersionId: apId() }), flowId }+ await fromQueue.add(jobId, pollingData, { jobId })++ await platformQueueMigrationService(app.log).migrateFlowJobs({ fromQueueName, toQueueName, flowId })++ expect(await fromQueue.getWaitingCount()).toBe(1)+ expect(await toQueue.getWaitingCount()).toBe(0)+ })+ })+describe('batch logic', () => {it('migrates all schedulers when count exceeds batchSize', async () => {const platformId = apId()@@ -263,3 +300,13 @@ function buildPollingJobData({ platformId, flowVersionId }: { platformId: stringjobType: WorkerJobType.EXECUTE_POLLING,}}++function buildWebhookJobData({ flowId }: { flowId: string }) {+ return {+ projectId: apId(),+ platformId: apId(),+ schemaVersion: LATEST_JOB_DATA_SCHEMA_VERSION,+ flowId,+ jobType: WorkerJobType.EXECUTE_WEBHOOK,+ }+}diff --git a/packages/server/api/test/unit/app/workers/worker-groups/worker-group-queues.test.ts b/packages/server/api/test/unit/app/workers/worker-groups/worker-group-queues.test.tsnew file mode 100644index 00000000000..62952147c93--- /dev/null+++ b/packages/server/api/test/unit/app/workers/worker-groups/worker-group-queues.test.ts@@ -0,0 +1,41 @@+import { WorkerGroupScope } from '@activepieces/shared'+import { describe, expect, it } from 'vitest'+import { workerGroupQueues } from '../../../../../src/app/workers/worker-groups/worker-group-queues'++describe('workerGroupQueues.parseWorkerGroupValue', () => {+ it('returns null when no group id is set', () => {+ expect(workerGroupQueues.parseWorkerGroupValue({ value: undefined, scope: WorkerGroupScope.PROJECT })).toBeNull()+ expect(workerGroupQueues.parseWorkerGroupValue({ value: '', scope: WorkerGroupScope.FLOW })).toBeNull()+ })++ it('returns an assignment carrying the given scope', () => {+ expect(workerGroupQueues.parseWorkerGroupValue({ value: 'g1', scope: WorkerGroupScope.FLOW })).toEqual({ scope: WorkerGroupScope.FLOW, id: 'g1' })+ expect(workerGroupQueues.parseWorkerGroupValue({ value: 'g1', scope: WorkerGroupScope.PLATFORM })).toEqual({ scope: WorkerGroupScope.PLATFORM, id: 'g1' })+ })+})++describe('workerGroupQueues.resolveScope', () => {+ it('uses the explicit scope when valid', () => {+ expect(workerGroupQueues.resolveScope({ projectWorker: true, scope: 'flow' })).toBe(WorkerGroupScope.FLOW)+ expect(workerGroupQueues.resolveScope({ projectWorker: true, scope: 'platform' })).toBe(WorkerGroupScope.PLATFORM)+ expect(workerGroupQueues.resolveScope({ projectWorker: false, scope: 'project' })).toBe(WorkerGroupScope.PROJECT)+ })++ it('falls back to the projectWorker flag when the scope is missing or invalid', () => {+ expect(workerGroupQueues.resolveScope({ projectWorker: true, scope: undefined })).toBe(WorkerGroupScope.PROJECT)+ expect(workerGroupQueues.resolveScope({ projectWorker: false, scope: undefined })).toBe(WorkerGroupScope.PLATFORM)+ expect(workerGroupQueues.resolveScope({ projectWorker: true, scope: 'bogus' })).toBe(WorkerGroupScope.PROJECT)+ })+})++describe('workerGroupQueues.isRoutableWorkerGroupId', () => {+ it('rejects values no worker can ever claim', () => {+ expect(workerGroupQueues.isRoutableWorkerGroupId('')).toBe(false)+ expect(workerGroupQueues.isRoutableWorkerGroupId('__none__')).toBe(false)+ })++ it('accepts ordinary labels', () => {+ expect(workerGroupQueues.isRoutableWorkerGroupId('g1')).toBe(true)+ expect(workerGroupQueues.isRoutableWorkerGroupId('canary')).toBe(true)+ })+})diff --git a/packages/server/api/test/unit/app/workers/worker-groups/worker-group-routing.test.ts b/packages/server/api/test/unit/app/workers/worker-groups/worker-group-routing.test.tsnew file mode 100644index 00000000000..92cbd0abab7--- /dev/null+++ b/packages/server/api/test/unit/app/workers/worker-groups/worker-group-routing.test.ts@@ -0,0 +1,126 @@+import { WorkerGroupScope, WorkerJobType } from '@activepieces/shared'+import { FastifyBaseLogger } from 'fastify'+import { beforeEach, describe, expect, it, vi } from 'vitest'++const mockIsWorkerGroupsEnabled = vi.fn()+const mockGetPlatformWorkerGroupId = vi.fn()+const mockGetProjectWorkerGroup = vi.fn()+const mockGetFlowWorkerGroup = vi.fn()++vi.mock('../../../../../src/app/workers/worker-groups/platform-worker-group.service', () => ({+ platformWorkerGroupService: () => ({+ isWorkerGroupsEnabled: mockIsWorkerGroupsEnabled,+ getWorkerGroupId: mockGetPlatformWorkerGroupId,+ }),+}))++vi.mock('../../../../../src/app/workers/worker-groups/project-worker-group.service', () => ({+ projectWorkerGroupService: () => ({+ getProjectWorkerGroup: mockGetProjectWorkerGroup,+ }),+}))++vi.mock('../../../../../src/app/workers/worker-groups/flow-worker-group.service', () => ({+ flowWorkerGroupService: () => ({+ getFlowWorkerGroup: mockGetFlowWorkerGroup,+ }),+}))++const mockLog = { info: vi.fn(), error: vi.fn(), warn: vi.fn(), debug: vi.fn() } as unknown as FastifyBaseLogger++type WorkerGroupRouting = typeof import('../../../../../src/app/workers/worker-groups/worker-group-routing').workerGroupRouting++async function loadRouting(): Promise<WorkerGroupRouting> {+ const mod = await import('../../../../../src/app/workers/worker-groups/worker-group-routing')+ return mod.workerGroupRouting+}++describe('workerGroupRouting', () => {+ let routing: WorkerGroupRouting++ beforeEach(async () => {+ vi.clearAllMocks()+ vi.resetModules()+ mockIsWorkerGroupsEnabled.mockResolvedValue(true)+ mockGetPlatformWorkerGroupId.mockResolvedValue(null)+ mockGetProjectWorkerGroup.mockResolvedValue(null)+ mockGetFlowWorkerGroup.mockResolvedValue(null)+ routing = await loadRouting()+ })++ describe('resolveQueueForJob', () => {+ it('routes to the flow group queue when the flow has a group', async () => {+ mockGetFlowWorkerGroup.mockResolvedValue('g1')+ mockGetProjectWorkerGroup.mockResolvedValue('pg')++ const queue = await routing.resolveQueueForJob({ platformId: 'pl1', projectId: 'pr1', flowId: 'f1', jobType: WorkerJobType.EXECUTE_FLOW, log: mockLog })++ expect(queue).toBe('flow-g1-jobs')+ })++ it('falls back to the project group queue when the flow has no group', async () => {+ mockGetProjectWorkerGroup.mockResolvedValue('pg')++ const queue = await routing.resolveQueueForJob({ platformId: 'pl1', projectId: 'pr1', flowId: 'f1', jobType: WorkerJobType.EXECUTE_WEBHOOK, log: mockLog })++ expect(queue).toBe('project-pg-jobs')+ })++ it('falls back to the platform group queue when neither flow nor project has a group', async () => {+ mockGetPlatformWorkerGroupId.mockResolvedValue('plg')++ const queue = await routing.resolveQueueForJob({ platformId: 'pl1', projectId: 'pr1', flowId: 'f1', jobType: WorkerJobType.EXECUTE_FLOW, log: mockLog })++ expect(queue).toBe('platform-plg-jobs')+ })++ it('falls back to the shared queue when no group applies', async () => {+ const queue = await routing.resolveQueueForJob({ platformId: 'pl1', projectId: 'pr1', flowId: 'f1', jobType: WorkerJobType.EXECUTE_FLOW, log: mockLog })++ expect(queue).toBe('workerJobs')+ })++ it('skips flow and project lookups when worker groups are disabled', async () => {+ mockIsWorkerGroupsEnabled.mockResolvedValue(false)+ mockGetFlowWorkerGroup.mockResolvedValue('g1')+ mockGetProjectWorkerGroup.mockResolvedValue('pg')++ const queue = await routing.resolveQueueForJob({ platformId: 'pl1', projectId: 'pr1', flowId: 'f1', jobType: WorkerJobType.EXECUTE_FLOW, log: mockLog })++ expect(queue).toBe('workerJobs')+ expect(mockGetFlowWorkerGroup).not.toHaveBeenCalled()+ expect(mockGetProjectWorkerGroup).not.toHaveBeenCalled()+ })++ it('skips flow and project lookups for non-routable job types', async () => {+ mockGetFlowWorkerGroup.mockResolvedValue('g1')++ const queue = await routing.resolveQueueForJob({ platformId: 'pl1', projectId: 'pr1', flowId: 'f1', jobType: WorkerJobType.EXECUTE_PROPERTY, log: mockLog })++ expect(queue).toBe('workerJobs')+ expect(mockGetFlowWorkerGroup).not.toHaveBeenCalled()+ expect(mockGetProjectWorkerGroup).not.toHaveBeenCalled()+ })++ it('skips the flow lookup when no flowId is provided', async () => {+ mockGetProjectWorkerGroup.mockResolvedValue('pg')++ const queue = await routing.resolveQueueForJob({ platformId: 'pl1', projectId: 'pr1', jobType: WorkerJobType.EXECUTE_FLOW, log: mockLog })++ expect(queue).toBe('project-pg-jobs')+ expect(mockGetFlowWorkerGroup).not.toHaveBeenCalled()+ })+ })++ describe('resolveQueueForWorker', () => {+ it('routes a shared worker to the shared queue', async () => {+ expect(routing.resolveQueueForWorker(null)).toBe('workerJobs')+ })++ it('routes each scope to its queue family', async () => {+ expect(routing.resolveQueueForWorker({ scope: WorkerGroupScope.FLOW, id: 'g1' })).toBe('flow-g1-jobs')+ expect(routing.resolveQueueForWorker({ scope: WorkerGroupScope.PROJECT, id: 'g1' })).toBe('project-g1-jobs')+ expect(routing.resolveQueueForWorker({ scope: WorkerGroupScope.PLATFORM, id: 'g1' })).toBe('platform-g1-jobs')+ })+ })+})diff --git a/packages/server/worker/src/lib/config/configs.ts b/packages/server/worker/src/lib/config/configs.tsindex f12b68e9597..f215a906bb7 100644--- a/packages/server/worker/src/lib/config/configs.ts+++ b/packages/server/worker/src/lib/config/configs.ts@@ -46,6 +46,7 @@ export enum WorkerSystemProp {BETTERSTACK_HOST = 'AP_BETTERSTACK_HOST',LOAD_TRANSLATIONS_FOR_DEV_PIECES = 'AP_LOAD_TRANSLATIONS_FOR_DEV_PIECES',WORKER_GROUP_ID = 'AP_WORKER_GROUP_ID',+ WORKER_GROUP_SCOPE = 'AP_WORKER_GROUP_SCOPE',IS_CANARY_WORKER = 'AP_IS_CANARY_WORKER',PROJECT_WORKER = 'AP_PROJECT_WORKER',WORKER_CONCURRENCY = 'AP_WORKER_CONCURRENCY',diff --git a/packages/server/worker/src/lib/worker.ts b/packages/server/worker/src/lib/worker.tsindex 1c8762548d3..968dd1a88e8 100644--- a/packages/server/worker/src/lib/worker.ts+++ b/packages/server/worker/src/lib/worker.ts@@ -65,8 +65,9 @@ export const worker = {await modelTierCatalog.warmUp()const workerGroupId = system.get(WorkerSystemProp.WORKER_GROUP_ID)const projectWorker = system.getBoolean(WorkerSystemProp.PROJECT_WORKER) ?? true+ const workerGroupScope = system.get(WorkerSystemProp.WORKER_GROUP_SCOPE)socket = io(socketUrl.url, {- auth: { token: workerToken, workerId, workerGroupId, projectWorker },+ auth: { token: workerToken, workerId, workerGroupId, projectWorker, workerGroupScope },path: socketUrl.path,transports: ['websocket'],reconnection: true,diff --git a/packages/web/src/app/routes/platform/infra/workers/assign-projects-dialog.tsx b/packages/web/src/app/routes/platform/infra/workers/assign-projects-dialog.tsxindex b0c92f82934..7a5691a953e 100644--- a/packages/web/src/app/routes/platform/infra/workers/assign-projects-dialog.tsx+++ b/packages/web/src/app/routes/platform/infra/workers/assign-projects-dialog.tsx@@ -17,6 +17,7 @@ import {} from '@/components/ui/dialog';import { Input } from '@/components/ui/input';import { ScrollArea } from '@/components/ui/scroll-area';+import { workersApi } from '@/features/platform-admin/api/workers-api';import { projectCollectionUtils } from '@/features/projects/stores/project-collection';import { AdminControl, adminControl } from '@/lib/admin-control';import { cn } from '@/lib/utils';@@ -75,19 +76,21 @@ function AssignProjectsContent({};const handleSave = async () => {+ let changed = false;for (const project of allProjects) {const wasInGroup = project.workerGroupId === groupLabel;const isNowChecked = checkedIds.has(project.id);if (isNowChecked && !wasInGroup) {- await projectCollectionUtils.update(project.id, {- workerGroupId: groupLabel,- });+ await workersApi.assignProject(project.id, groupLabel);+ changed = true;} else if (!isNowChecked && wasInGroup) {- await projectCollectionUtils.update(project.id, {- workerGroupId: null,- });+ await workersApi.assignProject(project.id, null);+ changed = true;}}+ if (changed) {+ await projectCollectionUtils.refetchProjects();+ }onOpenChange(false);};diff --git a/packages/web/src/features/platform-admin/api/workers-api.ts b/packages/web/src/features/platform-admin/api/workers-api.tsindex b874a40c6d6..2e343ce8995 100644--- a/packages/web/src/features/platform-admin/api/workers-api.ts+++ b/packages/web/src/features/platform-admin/api/workers-api.ts@@ -7,7 +7,17 @@ export const workersApi = {return api.get<WorkerMachineWithStatus[]>('/v1/worker-machines');},listWorkerGroups() {- return api.get<WorkerPoolCapacity>('/v1/projects/worker-groups');+ return api.get<WorkerPoolCapacity>('/v1/worker-groups');+ },+ assignProject(projectId: string, workerGroupId: string | null) {+ return api.post<void>(`/v1/worker-groups/projects/${projectId}`, {+ workerGroupId,+ });+ },+ assignFlow(flowId: string, workerGroupId: string | null) {+ return api.post<void>(`/v1/worker-groups/flows/${flowId}`, {+ workerGroupId,+ });},};