AI Agents & MCPs & AI Workflow Automation • (~400 MCP servers for AI agents) • AI Automation / AI Agent with MCPs • AI Workflows & AI Agents • MCPs for AI Agents
TypeScript ai-agentai-agent-toolsai-agentsai-agents-frameworkmcpmcp-servermcp-toolsmcpsn8n-alternativeno-code-automationworkflowworkflow-automationworkflows
import { chunk, isNil, tryCatch } from '@activepieces/core-utils'
import { wideEvent } from '@activepieces/server-utils'
import { FlowRunStatus, PauseType } from '@activepieces/shared'
import dayjs from 'dayjs'
import { FastifyBaseLogger } from 'fastify'
import { In, LessThanOrEqual } from 'typeorm'
import { repoFactory } from '../core/db/repo-factory'
import { distributedStore } from '../database/redis-connections'
import { systemJobIds } from '../helper/system-jobs/common'
import { systemJobsSchedule } from '../helper/system-jobs/system-job'
import { barrierQueue } from './barrier-queue'
import { WaitpointEntity } from './waitpoint-entity'
import { waitpointTimeoutJob } from './waitpoint-timeout-job'
import { Waitpoint, WaitpointStatus } from './waitpoint-types'
const waitpointRepo = repoFactory(WaitpointEntity)
export async function sweepOverdueDeadlines({ log, pageSize, maxPages, maxRedelivered }: SweepOverdueDeadlinesParams): Promise<string[]> {
const sweep = await runSweepPages({
log,
pageSize: pageSize ?? SCAN_PAGE_SIZE,
maxPages: maxPages ?? MAX_SCAN_PAGES_PER_TICK,
})
if (sweep.armed.length > 0) {
log.info({ armedCount: sweep.armed.length, scannedCount: sweep.scannedCount }, '[sweepOverdueDeadlines] Re-armed overdue waitpoint deadlines')
}
if (sweep.deadLettered.length > 0) {
log.warn({
deadLetteredCount: sweep.deadLettered.length,
scannedCount: sweep.scannedCount,
sample: sweep.deadLettered.slice(0, DEAD_LETTER_SAMPLE_SIZE).map((waitpoint) => waitpoint.id),
}, '[sweepOverdueDeadlines] Dead-lettered deadlines whose jobs exhausted their attempts; their runs stay paused')
}
if (!isNil(sweep.resumeFrom)) {
log.warn({
stopReason: sweep.stopReason,
scannedCount: sweep.scannedCount,
armedCount: sweep.armed.length,
resumeFrom: sweep.resumeFrom,
}, '[sweepOverdueDeadlines] Tick budget spent before the end of the backlog; the next tick resumes from the cursor')
}
const redelivery = await redeliverUndeliveredBarriers({ log, maxRedelivered: maxRedelivered ?? MAX_REDELIVERED_PER_TICK })
wideEvent.set({
waitpointSweep: {
scannedCount: sweep.scannedCount,
armedCount: sweep.armed.length,
deadLetteredCount: sweep.deadLettered.length,
stopReason: sweep.stopReason,
deadlineBacklogCarried: !isNil(sweep.resumeFrom),
barriersEnqueuedCount: redelivery.enqueued,
barriersEnqueueFailedCount: redelivery.enqueueFailed,
barrierBacklogCarried: redelivery.backlogCarried,
},
})
return sweep.armed
}
async function redeliverUndeliveredBarriers({ log, maxRedelivered }: RedeliverUndeliveredBarriersParams): Promise<RedeliveryOutcome> {
const cursor = await readBarrierCursor()
const undelivered = await findUndeliveredBarriers({
staleBefore: dayjs().subtract(UNDELIVERED_BARRIER_GRACE_MINUTES, 'minute').toISOString(),
cursor,
limit: maxRedelivered,
})
if (undelivered.length === 0) {
await rememberBarrierCursor(undefined)
return { enqueued: 0, enqueueFailed: 0, backlogCarried: false }
}
log.warn({
undeliveredCount: undelivered.length,
resumedFrom: cursor,
sample: undelivered.slice(0, DEAD_LETTER_SAMPLE_SIZE).map((barrier) => barrier.id),
}, '[redeliverUndeliveredBarriers] Found closed barriers that were never delivered; handing them to the barrier queue')
const { enqueued, enqueueFailed } = await enqueueBarrierEvaluations({ undelivered, log })
const backlogCarried = undelivered.length >= maxRedelivered
const resumeFrom = backlogCarried ? toBarrierCursor(undelivered[undelivered.length - 1]) : undefined
await rememberBarrierCursor(resumeFrom)
if (!isNil(resumeFrom)) {
log.warn({
enqueuedCount: enqueued,
enqueueFailedCount: enqueueFailed,
resumeFrom,
}, '[redeliverUndeliveredBarriers] Batch full before the end of the backlog; the next tick resumes from the cursor')
}
return { enqueued, enqueueFailed, backlogCarried }
}
async function enqueueBarrierEvaluations({ undelivered, log }: EnqueueBarrierEvaluationsParams): Promise<EnqueueOutcome> {
const outcomes = await Promise.all(undelivered.map(async (barrier) => {
const { error } = await tryCatch(() => barrierQueue(log).enqueueEvaluation({ barrierId: barrier.id, projectId: barrier.projectId }))
if (!isNil(error)) {
log.error({ error, waitpoint: { id: barrier.id }, flowRun: { id: barrier.flowRunId } }, '[redeliverUndeliveredBarriers] Could not enqueue an undelivered barrier; the next tick retries it')
return false
}
return true
}))
const enqueued = outcomes.filter(Boolean).length
return { enqueued, enqueueFailed: outcomes.length - enqueued }
}
async function findUndeliveredBarriers({ staleBefore, cursor, limit }: FindUndeliveredBarriersParams): Promise<Waitpoint[]> {
const query = waitpointRepo()
.createQueryBuilder('waitpoint')
.innerJoin('flow_run', 'flowRun', '"flowRun"."id" = "waitpoint"."flowRunId"')
.where('"waitpoint"."type" = :type', { type: PauseType.BARRIER })
.andWhere('"waitpoint"."status" = :status', { status: WaitpointStatus.COMPLETED })
.andWhere('"waitpoint"."updated" < :staleBefore', { staleBefore })
.andWhere('"flowRun"."status" = :runStatus', { runStatus: FlowRunStatus.PAUSED })
if (!isNil(cursor)) {
query.andWhere('("waitpoint"."updated", "waitpoint"."id") > (:cursorUpdated, :cursorId)', {
cursorUpdated: cursor.updated,
cursorId: cursor.id,
})
}
return query
.orderBy('"waitpoint"."updated"', 'ASC')
.addOrderBy('"waitpoint"."id"', 'ASC')
.limit(limit)
.getMany()
}
async function readBarrierCursor(): Promise<BarrierRecoveryCursor | undefined> {
const stored = await distributedStore.get<BarrierRecoveryCursor>(BARRIER_RECOVERY_CURSOR_KEY)
return stored ?? undefined
}
async function rememberBarrierCursor(cursor: BarrierRecoveryCursor | undefined): Promise<void> {
if (isNil(cursor)) {
await distributedStore.delete(BARRIER_RECOVERY_CURSOR_KEY)
return
}
await distributedStore.put(BARRIER_RECOVERY_CURSOR_KEY, cursor, DEADLINE_SWEEP_CURSOR_TTL_SECONDS)
}
function toBarrierCursor(barrier: Waitpoint): BarrierRecoveryCursor {
return { updated: barrier.updated, id: barrier.id }
}
async function runSweepPages({ log, pageSize, maxPages }: RunSweepPagesParams): Promise<SweepOutcome> {
const scanStartedAt = await readDatabaseTime()
const armed: string[] = []
const deadLettered: Waitpoint[] = []
let scannedCount = 0
let cursor = await readStoredCursor()
for (let page = 0; page < maxPages; page++) {
const overdue = await findOverdueWaitpoints({ now: scanStartedAt, cursor, pageSize })
if (overdue.length === 0) {
return finishSweep({ armed, deadLettered, scannedCount, resumeFrom: undefined, stopReason: 'drained' })
}
scannedCount += overdue.length
const probes = await probeDeadlineJobs({ overdue, log })
const pageDeadLettered = probes.filter((probe) => probe.state === 'dead-lettered').map((probe) => probe.waitpoint)
await stampDeadLettered({ deadLettered: pageDeadLettered, scanStartedAt })
deadLettered.push(...pageDeadLettered)
const unarmed = probes.filter((probe) => probe.state === 'unarmed').map((probe) => probe.waitpoint)
const arming = await armDeadlines({ unarmed, remainingQuota: MAX_ARMED_PER_TICK - armed.length, log })
armed.push(...arming.armed)
if (arming.quotaExhausted) {
const resumeFrom = isNil(arming.lastHandled) ? cursor : toCursor(arming.lastHandled)
return finishSweep({ armed, deadLettered, scannedCount, resumeFrom, stopReason: 'arm-quota' })
}
cursor = toCursor(overdue[overdue.length - 1])
if (overdue.length < pageSize) {
return finishSweep({ armed, deadLettered, scannedCount, resumeFrom: undefined, stopReason: 'drained' })
}
}
return finishSweep({ armed, deadLettered, scannedCount, resumeFrom: cursor, stopReason: 'page-budget' })
}
async function finishSweep({ armed, deadLettered, scannedCount, resumeFrom, stopReason }: SweepOutcome): Promise<SweepOutcome> {
await rememberCursor(resumeFrom)
return { armed, deadLettered, scannedCount, resumeFrom, stopReason }
}
async function readStoredCursor(): Promise<DeadlineCursor | undefined> {
const stored = await distributedStore.get<DeadlineCursor>(DEADLINE_SWEEP_CURSOR_KEY)
return stored ?? undefined
}
async function rememberCursor(cursor: DeadlineCursor | undefined): Promise<void> {
if (isNil(cursor)) {
await distributedStore.delete(DEADLINE_SWEEP_CURSOR_KEY)
return
}
await distributedStore.put(DEADLINE_SWEEP_CURSOR_KEY, cursor, DEADLINE_SWEEP_CURSOR_TTL_SECONDS)
}
async function readDatabaseTime(): Promise<string> {
const rows = await waitpointRepo().query<{ now: Date }[]>('SELECT now() AS now')
return dayjs(rows[0].now).toISOString()
}
function toCursor(waitpoint: Waitpoint): DeadlineCursor | undefined {
return isNil(waitpoint.resumeDateTime) ? undefined : { resumeDateTime: waitpoint.resumeDateTime, id: waitpoint.id }
}
async function findOverdueWaitpoints({ now, cursor, pageSize }: FindOverdueWaitpointsParams): Promise<Waitpoint[]> {
const query = waitpointRepo()
.createQueryBuilder('waitpoint')
.innerJoin('flow_run', 'flowRun', '"flowRun"."id" = "waitpoint"."flowRunId"')
.where('"waitpoint"."status" = :status', { status: WaitpointStatus.PENDING })
.andWhere('"waitpoint"."resumeDateTime" < :now', { now })
.andWhere('"waitpoint"."deadLetteredAt" IS NULL')
.andWhere('"flowRun"."status" = :runStatus', { runStatus: FlowRunStatus.PAUSED })
if (!isNil(cursor)) {
query.andWhere('("waitpoint"."resumeDateTime", "waitpoint"."id") > (:cursorResumeDateTime, :cursorId)', {
cursorResumeDateTime: cursor.resumeDateTime,
cursorId: cursor.id,
})
}
return query
.orderBy('"waitpoint"."resumeDateTime"', 'ASC')
.addOrderBy('"waitpoint"."id"', 'ASC')
.limit(pageSize)
.getMany()
}
async function probeDeadlineJobs({ overdue, log }: ProbeDeadlineJobsParams): Promise<DeadlineProbe[]> {
const resolvedChunks: DeadlineProbe[][] = []
for (const batch of chunk(overdue, DEADLINE_PROBE_CHUNK_SIZE)) {
resolvedChunks.push(await Promise.all(batch.map((waitpoint) => probeDeadlineJob({ waitpoint, log }))))
}
return resolvedChunks.flat()
}
async function probeDeadlineJob({ waitpoint, log }: ProbeDeadlineJobParams): Promise<DeadlineProbe> {
const jobState = await systemJobsSchedule(log).getJobState(systemJobIds.resumeDelay({ waitpointId: waitpoint.id }))
if (jobState === 'unknown') {
return { waitpoint, state: 'unarmed' }
}
return { waitpoint, state: jobState === 'failed' ? 'dead-lettered' : 'armed' }
}
async function stampDeadLettered({ deadLettered, scanStartedAt }: StampDeadLetteredParams): Promise<void> {
const stampedAt = dayjs().toISOString()
const idsByProject = deadLettered.reduce<Record<string, string[]>>((grouped, waitpoint) => ({
...grouped,
[waitpoint.projectId]: [...grouped[waitpoint.projectId] ?? [], waitpoint.id],
}), {})
await Promise.all(Object.entries(idsByProject).map(([projectId, ids]) =>
waitpointRepo().update({ id: In(ids), projectId, updated: LessThanOrEqual(scanStartedAt) }, { deadLetteredAt: stampedAt }),
))
}
async function armDeadlines({ unarmed, remainingQuota, log }: ArmDeadlinesParams): Promise<ArmOutcome> {
const armed: string[] = []
let lastHandled: Waitpoint | undefined = undefined
for (const waitpoint of unarmed) {
if (armed.length >= remainingQuota) {
return { armed, quotaExhausted: true, lastHandled }
}
lastHandled = waitpoint
if (isNil(waitpoint.resumeDateTime)) {
continue
}
const { status } = await waitpointTimeoutJob.schedule({
flowRunId: waitpoint.flowRunId,
projectId: waitpoint.projectId,
waitpointId: waitpoint.id,
resumeDateTime: waitpoint.resumeDateTime,
log,
})
if (status === 'added') {
armed.push(waitpoint.id)
}
}
return { armed, quotaExhausted: false, lastHandled }
}
const SCAN_PAGE_SIZE = 500
const MAX_SCAN_PAGES_PER_TICK = 8
const MAX_ARMED_PER_TICK = 500
const DEADLINE_PROBE_CHUNK_SIZE = 50
const DEAD_LETTER_SAMPLE_SIZE = 10
const DEADLINE_SWEEP_CURSOR_TTL_SECONDS = 600
const UNDELIVERED_BARRIER_GRACE_MINUTES = 2
const MAX_REDELIVERED_PER_TICK = 100
export const DEADLINE_SWEEP_CURSOR_KEY = 'waitpoint:deadline-sweep:cursor'
export const BARRIER_RECOVERY_CURSOR_KEY = 'waitpoint:barrier-recovery:cursor'
type SweepOverdueDeadlinesParams = {
log: FastifyBaseLogger
pageSize?: number
maxPages?: number
maxRedelivered?: number
}
type RedeliverUndeliveredBarriersParams = {
log: FastifyBaseLogger
maxRedelivered: number
}
type RedeliveryOutcome = {
enqueued: number
enqueueFailed: number
backlogCarried: boolean
}
type EnqueueOutcome = {
enqueued: number
enqueueFailed: number
}
type EnqueueBarrierEvaluationsParams = {
undelivered: Waitpoint[]
log: FastifyBaseLogger
}
type FindUndeliveredBarriersParams = {
staleBefore: string
cursor: BarrierRecoveryCursor | undefined
limit: number
}
type BarrierRecoveryCursor = {
updated: string
id: string
}
type RunSweepPagesParams = {
log: FastifyBaseLogger
pageSize: number
maxPages: number
}
type SweepOutcome = {
armed: string[]
deadLettered: Waitpoint[]
scannedCount: number
resumeFrom: DeadlineCursor | undefined
stopReason: 'drained' | 'arm-quota' | 'page-budget'
}
type DeadlineCursor = {
resumeDateTime: string
id: string
}
type DeadlineProbe = {
waitpoint: Waitpoint
state: 'unarmed' | 'armed' | 'dead-lettered'
}
type FindOverdueWaitpointsParams = {
now: string
cursor: DeadlineCursor | undefined
pageSize: number
}
type ProbeDeadlineJobsParams = {
overdue: Waitpoint[]
log: FastifyBaseLogger
}
type ProbeDeadlineJobParams = {
waitpoint: Waitpoint
log: FastifyBaseLogger
}
type StampDeadLetteredParams = {
deadLettered: Waitpoint[]
scanStartedAt: string
}
type ArmDeadlinesParams = {
unarmed: Waitpoint[]
remainingQuota: number
log: FastifyBaseLogger
}
type ArmOutcome = {
armed: string[]
quotaExhausted: boolean
lastHandled: Waitpoint | undefined
}