Skip to content

AI Agents & MCPs & AI Workflow Automation • (~400 MCP servers for AI agents) • AI Automation / AI Agent with MCPs • AI Workflows & AI Agents • MCPs for AI Agents

fix(api): prevent overlapping polling-trigger executions per flow version (#17150)

Chaker Atallahba22f79c9368
diff --git a/packages/server/api/src/app/database/redis/keys.ts b/packages/server/api/src/app/database/redis/keys.ts
index bd972823cae..533fa5b3124 100644
--- a/packages/server/api/src/app/database/redis/keys.ts
+++ b/packages/server/api/src/app/database/redis/keys.ts
@@ -1,4 +1,4 @@
-import { PlatformId, ProjectId } from '@activepieces/core-utils'
+import { ApId, PlatformId, ProjectId } from '@activepieces/core-utils'
export const getAiProviderConfirmKey = (providerId: string): string => `ai_provider:confirm-downgrade:${providerId}`
export const getManagedAiProviderKeyLockKey = (platformId: PlatformId): string => `ai_provider_managed_key_${platformId}`
@@ -21,6 +21,7 @@ export const getConcurrencyPoolLimitKey = (poolId: string): string => `concurren
export const getConcurrencyPoolSetKey = (poolId: string): string => `active_jobs_set:pool:${poolId}`
export const getConcurrencyPoolParkedKey = (poolId: string): string => `parked_jobs_set:pool:${poolId}`
export const getMcpOAuthRevokedGrantKey = (grantId: string): string => `mcp_oauth:revoked_grant:${grantId}`
+export const getPollingExclusivityKey = (flowVersionId: ApId): string => `polling_exclusivity:${flowVersionId}`
export const BILLING_ENFORCED_TTL_SECONDS = 24 * 60 * 60
export const PLATFORM_PLAN_NAME_TTL_SECONDS = 24 * 60 * 60
diff --git a/packages/server/api/src/app/workers/job-queue/interceptors/polling-exclusivity-interceptor.ts b/packages/server/api/src/app/workers/job-queue/interceptors/polling-exclusivity-interceptor.ts
new file mode 100644
index 00000000000..2f2630148c2
--- /dev/null
+++ b/packages/server/api/src/app/workers/job-queue/interceptors/polling-exclusivity-interceptor.ts
@@ -0,0 +1,58 @@
+import { JobData, PollingJobData, WorkerJobType } from '@activepieces/shared'
+import { FastifyBaseLogger } from 'fastify'
+import { getPollingExclusivityKey } from '../../../database/redis/keys'
+import { redisConnections } from '../../../database/redis-connections'
+import { system } from '../../../helper/system/system'
+import { AppSystemProp } from '../../../helper/system/system-props'
+import { InterceptorResult, InterceptorVerdict, JobInterceptor } from '../job-interceptor'
+
+const RETRY_DELAY_MS = 10_000
+const LOCK_SAFETY_MARGIN_SECONDS = 60
+
+function isPollingJob(jobData: JobData): jobData is PollingJobData {
+ return jobData.jobType === WorkerJobType.EXECUTE_POLLING
+}
+
+function getLockTtlSeconds(): number {
+ return system.getNumberOrThrow(AppSystemProp.TRIGGER_TIMEOUT_SECONDS) + LOCK_SAFETY_MARGIN_SECONDS
+}
+
+async function tryAcquireLock({ flowVersionId, jobId }: { flowVersionId: string, jobId: string }): Promise<boolean> {
+ const redis = await redisConnections.useExisting()
+ const result = await redis.set(getPollingExclusivityKey(flowVersionId), jobId, 'EX', getLockTtlSeconds(), 'NX')
+ return result !== null
+}
+
+async function releaseLock({ flowVersionId, jobId }: { flowVersionId: string, jobId: string }): Promise<void> {
+ const redis = await redisConnections.useExisting()
+ const key = getPollingExclusivityKey(flowVersionId)
+ const owner = await redis.get(key)
+ if (owner === jobId) {
+ await redis.del(key)
+ }
+}
+
+export const pollingExclusivityInterceptor: JobInterceptor = {
+ async preDispatch({ jobId, jobData, log }: { jobId: string, jobData: JobData, log: FastifyBaseLogger }): Promise<InterceptorResult> {
+ if (!isPollingJob(jobData)) {
+ return { verdict: InterceptorVerdict.ALLOW }
+ }
+ const { flowVersionId } = jobData
+ const acquired = await tryAcquireLock({ flowVersionId, jobId })
+ if (acquired) {
+ log.debug({ job: { id: jobId }, flowVersion: { id: flowVersionId } }, '[pollingExclusivityInterceptor] Lock acquired')
+ return { verdict: InterceptorVerdict.ALLOW }
+ }
+ log.info({ job: { id: jobId }, flowVersion: { id: flowVersionId } }, '[pollingExclusivityInterceptor] A poll is already in flight for this flow version, deferring')
+ return { verdict: InterceptorVerdict.REJECT, delayInMs: RETRY_DELAY_MS }
+ },
+
+ async onJobFinished({ jobId, jobData, log }: { jobId: string, jobData: JobData, failed: boolean, log: FastifyBaseLogger }): Promise<void> {
+ if (!isPollingJob(jobData)) {
+ return
+ }
+ const { flowVersionId } = jobData
+ await releaseLock({ flowVersionId, jobId })
+ log.debug({ job: { id: jobId }, flowVersion: { id: flowVersionId } }, '[pollingExclusivityInterceptor] Lock released')
+ },
+}
diff --git a/packages/server/api/src/app/workers/job-queue/job-broker.ts b/packages/server/api/src/app/workers/job-queue/job-broker.ts
index c0d42faa2a0..789e30e16ba 100644
--- a/packages/server/api/src/app/workers/job-queue/job-broker.ts
+++ b/packages/server/api/src/app/workers/job-queue/job-broker.ts
@@ -8,6 +8,7 @@ import { redisConnections } from '../../database/redis-connections'
import { engineResponseWatcher } from '../engine-response-watcher'
import { QueueName } from '../job'
import { jobMigrations } from '../migrations/job-data-migrations'
+import { pollingExclusivityInterceptor } from './interceptors/polling-exclusivity-interceptor'
import { rateLimiterInterceptor } from './interceptors/rate-limiter-interceptor'
import { zombiePollingInterceptor } from './interceptors/zombie-polling-interceptor'
import { jobAssignmentTracker } from './job-assignment-tracker'
@@ -19,7 +20,7 @@ import { createQueueDispatcher, QueueDispatcher } from './queue-dispatcher'
const DRAIN_DELAY_SECONDS = 15
const LOCK_DURATION_MS = 120_000
-const interceptors: JobInterceptor[] = [rateLimiterInterceptor, zombiePollingInterceptor]
+const interceptors: JobInterceptor[] = [rateLimiterInterceptor, zombiePollingInterceptor, pollingExclusivityInterceptor]
const workerPromises = new Map<string, Promise<BullMQWorker>>()
const dispatchers = new Map<string, QueueDispatcher>()