AI Agents & MCPs & AI Workflow Automation • (~400 MCP servers for AI agents) • AI Automation / AI Agent with MCPs • AI Workflows & AI Agents • MCPs for AI Agents
TypeScript ai-agentai-agent-toolsai-agentsai-agents-frameworkmcpmcp-servermcp-toolsmcpsn8n-alternativeno-code-automationworkflowworkflow-automationworkflows
import { apId, assertNotNullOrUndefined, isNil, sanitizeObjectForPostgresql, spreadIfDefined } from '@activepieces/core-utils'
import { BarrierSignalStatus, FlowRun, FlowRunStatus, isFlowRunStateTerminal, RunTimeline } from '@activepieces/shared'
import { Queue, Worker } from 'bullmq'
import { FastifyBaseLogger } from 'fastify'
import { distributedLock, distributedStore, redisConnections } from '../../database/redis-connections'
import { domainHelper } from '../../helper/domain-helper'
import { exceptionHandler } from '../../helper/exception-handler'
import { system } from '../../helper/system/system'
import { AppSystemProp } from '../../helper/system/system-props'
import { projectService } from '../../project/project-service'
import { barrierService } from '../../waitpoints/barrier-service'
import { resumeService } from '../../waitpoints/resume-service'
import { waitpointService } from '../../waitpoints/waitpoint-service'
import { legacyRedisMetadataKey, QueueName, redisMetadataKey, runsMetadataDeduplicationId, RunsMetadataJobData, RunsMetadataQueueConfig, runsMetadataQueueFactory, RunsMetadataUpsertData } from '../../workers/job'
import { jobFailureLogger } from '../../workers/job-queue/job-failure-logger'
import { flowService } from '../flow/flow.service'
import { flowRunRepo } from './flow-run-service'
import { flowRunSideEffects } from './flow-run-side-effects'
import { buildRunTimeline } from './run-timeline'
let runsMetadataWorker: Worker<RunsMetadataJobData> | undefined = undefined
const queue = runsMetadataQueueFactory({ createRedisConnection: redisConnections.create, distributedStore })
export const runsMetadataQueue = (log: FastifyBaseLogger) => ({
async init(): Promise<void> {
const queueName = QueueName.RUNS_METADATA
const config: RunsMetadataQueueConfig = {
redisFailedJobRetentionDays: system.getNumberOrThrow(AppSystemProp.REDIS_FAILED_JOB_RETENTION_DAYS),
redisFailedJobRetentionMaxCount: system.getNumberOrThrow(AppSystemProp.REDIS_FAILED_JOB_RETENTION_MAX_COUNT),
}
await queue.init(config)
runsMetadataWorker = new Worker<RunsMetadataJobData>(
queueName,
async (job) => {
log.info({
job: { id: job.id },
flowRun: { id: job.data.runId },
}, '[runsMetadataQueue#worker] Saving runs metadata')
assertNotNullOrUndefined(job.data.projectId, 'projectId')
const key = redisMetadataKey({ projectId: job.data.projectId, runId: job.data.runId })
await distributedLock(log).runExclusive({
key: `runs_metadata_${job.data.runId}`,
timeoutInSeconds: 30,
fn: async () => {
try {
await runsMetadataQueue(log).get().removeDeduplicationKey(runsMetadataDeduplicationId({ projectId: job.data.projectId, runId: job.data.runId }))
const rawRunMetadata = await distributedStore.hgetJson<RunsMetadataUpsertData>(key)
?? await claimMetadataWrittenBeforeTheKeyWasScoped(job.data)
if (isNil(rawRunMetadata) || Object.keys(rawRunMetadata).length === 0) {
log.info({
job: { id: job.id },
flowRun: { id: job.data.runId },
}, '[runsMetadataQueue#worker] Runs metadata not found, skipping job')
return
}
const runMetadata = sanitizeObjectForPostgresql(rawRunMetadata)
const flowRunScope = { id: job.data.runId, projectId: job.data.projectId }
const existingFlowRun = await flowRunRepo().findOneBy(flowRunScope)
let savedFlowRun: FlowRun
if (!isNil(existingFlowRun)) {
const timeline = buildTimeline({ existingFlowRun, runMetadata })
await flowRunRepo().update(flowRunScope, {
...spreadIfDefined('timeline', timeline),
...spreadIfDefined('flowId', runMetadata.flowId),
...spreadIfDefined('flowVersionId', runMetadata.flowVersionId),
...spreadIfDefined('environment', runMetadata.environment),
...spreadIfDefined('startTime', runMetadata.startTime),
...spreadIfDefined('finishTime', runMetadata.finishTime),
...spreadIfDefined('status', runMetadata.status),
...spreadIfDefined('tags', runMetadata.tags),
...spreadIfDefined('failedStep', runMetadata.failedStep),
...spreadIfDefined('stepNameToTest', runMetadata.stepNameToTest),
...spreadIfDefined('parentRunId', runMetadata.parentRunId),
...spreadIfDefined('parentWaitpointId', runMetadata.parentWaitpointId),
...spreadIfDefined('failParentOnFailure', runMetadata.failParentOnFailure),
...spreadIfDefined('logsFileId', runMetadata.logsFileId),
...spreadIfDefined('updated', runMetadata.updated),
...spreadIfDefined('stepsCount', runMetadata.stepsCount),
})
const updatedFlowRun = await flowRunRepo().findOneBy(flowRunScope)
if (isNil(updatedFlowRun)) {
log.info({
job: { id: job.id },
flowRun: { id: job.data.runId },
}, '[runsMetadataQueue#worker] Flow run was deleted during update, skipping job')
return
}
savedFlowRun = updatedFlowRun
}
else {
const idBelongsToAnotherProject = await flowRunRepo().countBy({ id: job.data.runId }) > 0
if (idBelongsToAnotherProject) {
log.warn({
job: { id: job.id },
flowRun: { id: job.data.runId },
project: { id: job.data.projectId },
}, '[runsMetadataQueue#worker] Run metadata reported for a run owned by another project, skipping job')
await discardMetadata({ key, requestId: runMetadata.requestId })
return
}
const flowId = runMetadata.flowId
const flowExists = !isNil(flowId) && await flowService(log).exists({ id: flowId, projectId: job.data.projectId })
if (!flowExists) {
log.info({
job: { id: job.id },
flowRun: { id: job.data.runId },
}, '[runsMetadataQueue#worker] Flow does not exist (deleted), skipping job')
await discardMetadata({ key, requestId: runMetadata.requestId })
return
}
savedFlowRun = await flowRunRepo().save({ ...runMetadata, ...flowRunScope })
}
const parentRunId = savedFlowRun.parentRunId
const shouldMarkParentAsFailed = !isNil(parentRunId) && childRunFailsParent({
status: savedFlowRun.status,
failParentOnFailure: savedFlowRun.failParentOnFailure,
willRetry: runMetadata.willRetry,
})
if (!isNil(savedFlowRun.parentWaitpointId) && isFlowRunStateTerminal({ status: savedFlowRun.status, ignoreInternalError: runMetadata.willRetry === true })) {
await barrierService(log).receiveSignal({
refId: savedFlowRun.id,
projectId: savedFlowRun.projectId,
status: toSignalStatus(savedFlowRun.status),
result: { runId: savedFlowRun.id, status: savedFlowRun.status },
})
}
else if (shouldMarkParentAsFailed) {
await markParentRunAsFailed({
parentRunId,
childRunId: savedFlowRun.id,
childStatus: savedFlowRun.status,
childFailureMessage: savedFlowRun.failedStep?.message,
projectId: savedFlowRun.projectId,
log,
})
}
if (!isNil(runMetadata.requestId)) {
await distributedStore.deleteKeyIfFieldValueMatches(key, 'requestId', runMetadata.requestId)
}
if (!isNil(runMetadata.finishTime)) {
const platformId = await projectService(log).getPlatformId(savedFlowRun.projectId)
await flowRunSideEffects(log).onFinish({ flowRun: savedFlowRun, platformId })
}
if (savedFlowRun.status === FlowRunStatus.PAUSED) {
const undeliveredWaitpoint = await waitpointService(log).findUndeliveredCompletedWaitpoint({ flowRunId: savedFlowRun.id, projectId: savedFlowRun.projectId })
if (!isNil(undeliveredWaitpoint)) {
await resumeService(log).resumeTrustedWithoutLock({
flowRunId: savedFlowRun.id,
waitpointId: undeliveredWaitpoint.id,
resumePayload: undeliveredWaitpoint.resumePayload,
})
}
}
}
catch (error) {
log.error({
error,
data: job.data,
}, '[runsMetadataQueue#worker] Error saving runs metadata')
exceptionHandler.handle(error, log)
throw error
}
},
})
},
{
connection: await redisConnections.create(),
concurrency: system.getNumberOrThrow(AppSystemProp.RUNS_METADATA_UPDATE_CONCURRENCY),
autorun: true,
},
)
runsMetadataWorker.on('failed', (job, err) => {
const attemptsUsed = job?.attemptsMade ?? 0
const maxAttempts = job?.opts?.attempts ?? 1
if (attemptsUsed >= maxAttempts) {
jobFailureLogger.logJobFailed({
queueName: QueueName.RUNS_METADATA,
jobId: job?.id,
jobType: job?.name ?? 'update-run-metadata',
error: err,
log,
})
}
})
await runsMetadataWorker.waitUntilReady()
},
async add(params: RunsMetadataUpsertData): Promise<void> {
log.info({
flowRun: { id: params.id },
project: { id: params.projectId },
}, '[runsMetadataQueue#add] Adding runs metadata to queue')
await queue.add(params)
},
get(): Queue<RunsMetadataJobData> {
return queue.get()
},
async close(): Promise<void> {
if (queue.get()) {
await queue.get().close()
}
if (runsMetadataWorker) {
await runsMetadataWorker.close()
}
},
})
async function discardMetadata({ key, requestId }: DiscardMetadataParams): Promise<void> {
if (isNil(requestId)) {
return
}
await distributedStore.deleteKeyIfFieldValueMatches(key, 'requestId', requestId)
}
async function claimMetadataWrittenBeforeTheKeyWasScoped({ runId, projectId }: RunsMetadataJobData): Promise<RunsMetadataUpsertData | null> {
const legacyKey = legacyRedisMetadataKey(runId)
const legacyMetadata = await distributedStore.hgetJson<RunsMetadataUpsertData>(legacyKey)
if (isNil(legacyMetadata) || legacyMetadata.projectId !== projectId) {
return null
}
await distributedStore.delete(legacyKey)
return legacyMetadata
}
function buildTimeline({ existingFlowRun, runMetadata }: BuildTimelineParams): RunTimeline | undefined {
return buildRunTimeline({
existingTimeline: existingFlowRun.timeline,
created: existingFlowRun.created,
startTime: runMetadata.startTime ?? existingFlowRun.startTime,
finishTime: runMetadata.finishTime ?? existingFlowRun.finishTime,
provisionMs: runMetadata.provisionMs,
bootMs: runMetadata.bootMs,
runMs: runMetadata.runMs,
})
}
export function childRunFailsParent({ status, failParentOnFailure, willRetry }: ChildRunFailsParentParams): boolean {
return failParentOnFailure
&& status !== FlowRunStatus.SUCCEEDED
&& isFlowRunStateTerminal({ status, ignoreInternalError: willRetry === true })
}
export async function markParentRunAsFailed({
parentRunId,
childRunId,
childStatus,
childFailureMessage,
projectId,
log,
}: MarkParentRunAsFailedParams): Promise<void> {
const flowRun = await flowRunRepo().findOneBy({
id: parentRunId,
projectId,
})
if (isNil(flowRun) || isFlowRunStateTerminal({ status: flowRun.status, ignoreInternalError: false })) {
return
}
const childRunUrl = await domainHelper.getPublicUrl({ path: `/projects/${projectId}/runs/${childRunId}` })
const errorPayload = {
body: {
status: 'error',
data: {
message: childFailureMessage ?? 'Subflow execution failed',
status: childStatus,
link: childRunUrl,
},
},
headers: {},
queryParams: {},
}
const existingWaitpoint = await waitpointService(log).findSubflowWaitpoint({ flowRunId: parentRunId, projectId: flowRun.projectId })
const result = await waitpointService(log).complete({
flowRunId: parentRunId,
projectId: flowRun.projectId,
waitpointId: existingWaitpoint?.id ?? apId(),
resumePayload: errorPayload,
})
if (result.completedExisting && !isNil(result.waitpoint)) {
await resumeService(log).resumeFromWaitpoint({
flowRunId: parentRunId,
waitpointId: result.waitpoint.id,
resumePayload: result.waitpoint.resumePayload,
})
}
}
function toSignalStatus(status: FlowRunStatus): BarrierSignalStatus {
if (status === FlowRunStatus.SUCCEEDED) {
return BarrierSignalStatus.SUCCEEDED
}
if (status === FlowRunStatus.CANCELED) {
return BarrierSignalStatus.CANCELED
}
return BarrierSignalStatus.FAILED
}
type ChildRunFailsParentParams = {
status: FlowRunStatus
failParentOnFailure: boolean
willRetry?: boolean
}
type DiscardMetadataParams = {
key: string
requestId?: string
}
type BuildTimelineParams = {
existingFlowRun: FlowRun
runMetadata: RunsMetadataUpsertData
}
type MarkParentRunAsFailedParams = {
parentRunId: string
childRunId: string
childStatus: FlowRunStatus
childFailureMessage?: string
projectId: string
log: FastifyBaseLogger
}