Skip to content

AI Agents & MCPs & AI Workflow Automation • (~400 MCP servers for AI agents) • AI Automation / AI Agent with MCPs • AI Workflows & AI Agents • MCPs for AI Agents

13.1 KB
Raw Blame
import { apId, isNil } from '@activepieces/core-utils'
import { AgentConversationStatus, AgentPromptOverride, LATEST_JOB_DATA_SCHEMA_VERSION, MAX_AGENT_TEXT_LENGTH, PersistedAgentRole, SimulateAgentRequest, WorkerJobType } from '@activepieces/shared'
import { FastifyBaseLogger, FastifyReply, FastifyRequest } from 'fastify'
import { FastifyPluginAsyncZod } from 'fastify-type-provider-zod'
import { StatusCodes } from 'http-status-codes'
import { z } from 'zod'
import { securityAccess } from '../../core/security/authorization/fastify-security'
import { system } from '../../helper/system/system'
import { AppSystemProp } from '../../helper/system/system-props'
import { platformService } from '../../platform/platform.service'
import { jobQueue, JobType } from '../../workers/job-queue/job-queue'
import { agentConversationService } from './agent-conversation-service'
import { agentHelpers, EVAL_CONVERSATION_ID_PREFIX, isEvalConversationId } from './agent-helpers'
import { agentPrompt } from './prompt/agent-prompt'

const API_KEY_HEADER = 'api-key'
const API_KEY = system.get(AppSystemProp.API_KEY)
const SIMULATION_POLL_INTERVAL_MS = 1_500
const SIMULATION_MAX_ATTEMPTS = 120
const SIMULATION_TIMEOUT_STATUS = 'TIMEOUT'

type SimulationStatus = AgentConversationStatus | typeof SIMULATION_TIMEOUT_STATUS

// Same guard as admin-platform.controller: a single global AP_API_KEY, not a per-platform key.
async function checkApiKeyPreHandler(req: FastifyRequest, res: FastifyReply): Promise<void> {
    const key = req.headers[API_KEY_HEADER] as string | undefined
    if (isNil(API_KEY) || key !== API_KEY) {
        await res.status(StatusCodes.FORBIDDEN).send({ message: 'Forbidden' })
        throw new Error('Forbidden')
    }
}

export const agentEvalModule: FastifyPluginAsyncZod = async (app) => {
    app.addHook('preHandler', checkApiKeyPreHandler)
    await app.register(agentEvalController, { prefix: '/v1/agents' })
}

const agentEvalController: FastifyPluginAsyncZod = async (app) => {
    app.get('/eval/prompt-sources', PromptSourcesRoute, async (_request, reply) => {
        return reply.status(StatusCodes.OK).send(agentPrompt.sources)
    })

    // Lets a caller replaying a foreign conversation (e.g. the console in dev, where the
    // conversation's real platform lives in cloud) resolve a local platform to sandbox the run in.
    app.get('/eval/sandbox-platform', SandboxPlatformRoute, async (request, reply) => {
        const platform = await platformService(request.log).getOldestPlatform()
        if (isNil(platform)) {
            return reply.status(StatusCodes.NOT_FOUND).send({ message: 'No platform exists on this instance' })
        }
        return reply.status(StatusCodes.OK).send({ platformId: platform.id })
    })

    app.post('/eval/simulate', SimulateRoute, async (request, reply) => {
        const log = request.log
        const { platformId, userMessage, userMessages, promptOverride } = request.body
        const turns = userMessages ?? [userMessage as string]

        // Eval is an internal api-key dry-run, so it doesn't require the platform's chatEnabled entitlement.
        const platform = await platformService(log).getOneOrThrow(platformId)
        const evalUserId = platform.ownerId

        const conversation = await agentConversationService(log).createConversation({
            platformId,
            userId: evalUserId,
            request: {},
        })

        // Replay the turns sequentially in one conversation: each turn's history accumulates,
        // so the candidate prompt is exercised across the whole conversation.
        let lastRunId = ''
        let settled: { status: SimulationStatus, uiMessages: unknown[] | null } = { status: SIMULATION_TIMEOUT_STATUS, uiMessages: null }
        let priorAssistantTurns = 0
        for (const turn of turns) {
            lastRunId = apId()
            await jobQueue(log).add({
                id: apId(),
                type: JobType.ONE_TIME,
                data: {
                    schemaVersion: LATEST_JOB_DATA_SCHEMA_VERSION,
                    jobType: WorkerJobType.EXECUTE_AGENT_RUN,
                    conversationId: conversation.id,
                    runId: lastRunId,
                    projectId: null,
                    platformId,
                    userId: evalUserId,
                    userMessage: turn,
                    modelName: null,
                    promptOverride,
                    dryRun: true,
                },
            })

            settled = await waitForSimulationResult({ conversationId: conversation.id, platformId, userId: evalUserId, priorAssistantTurns, log })
            if (settled.status === AgentConversationStatus.ERROR || settled.status === SIMULATION_TIMEOUT_STATUS) {
                break
            }
            priorAssistantTurns = countAssistantTurns(settled.uiMessages)
        }

        return reply.status(StatusCodes.OK).send({
            conversationId: conversation.id,
            runId: lastRunId,
            status: settled.status,
            uiMessages: settled.uiMessages ?? [],
        })
    })

    // Enqueue one turn and return; the caller polls /state for progress. Dry-run by default (tools
    // not executed); pass executeTools:true to run tools live against the owner's real connections.
    app.post('/eval/turn/start', EvalTurnStartRoute, async (request, reply) => {
        const log = request.log
        const { conversationId, platformId, userMessage, promptOverride, executeTools, discoveryOnly } = request.body

        let convId: string
        let evalPlatformId: string
        let evalUserId: string
        let priorAssistantTurns: number
        if (!isNil(conversationId)) {
            // Only continue conversations this eval flow created — never an arbitrary (e.g. a real
            // user's) conversation, even with a valid API key.
            if (!isEvalConversationId(conversationId)) {
                return reply.status(StatusCodes.NOT_FOUND).send({ message: 'Conversation not found' })
            }
            const existing = await agentHelpers.conversationRepo().findOneBy({ id: conversationId })
            if (isNil(existing)) {
                return reply.status(StatusCodes.NOT_FOUND).send({ message: 'Conversation not found' })
            }
            // Reject overlapping turns: a turn is already in flight for this conversation. The caller
            // serializes turns, so this only fires on a double-submit — don't race two workers on one row.
            if (existing.status === AgentConversationStatus.STREAMING) {
                return reply.status(StatusCodes.CONFLICT).send({ message: 'A turn is already running for this conversation' })
            }
            convId = existing.id
            evalPlatformId = existing.platformId
            evalUserId = existing.userId
            priorAssistantTurns = countAssistantTurns(existing.uiMessages)
        }
        else {
            if (isNil(platformId)) {
                return reply.status(StatusCodes.BAD_REQUEST).send({ message: 'platformId is required to start a new conversation' })
            }
            const platform = await platformService(log).getOneOrThrow(platformId)
            evalPlatformId = platformId
            evalUserId = platform.ownerId
            const conversation = await agentConversationService(log).createConversation({ platformId, userId: evalUserId, request: {}, id: (EVAL_CONVERSATION_ID_PREFIX + apId()).slice(0, 21) })
            convId = conversation.id
            priorAssistantTurns = 0
        }

        const runId = apId()
        await jobQueue(log).add({
            id: apId(),
            type: JobType.ONE_TIME,
            data: {
                schemaVersion: LATEST_JOB_DATA_SCHEMA_VERSION,
                jobType: WorkerJobType.EXECUTE_AGENT_RUN,
                conversationId: convId,
                runId,
                projectId: null,
                platformId: evalPlatformId,
                userId: evalUserId,
                userMessage,
                modelName: null,
                promptOverride,
                // discovery-only still needs the real tool set (dry-run strips MCP piece tools
                // entirely), so it runs non-dry but with execution neutralized in the worker.
                dryRun: executeTools !== true && discoveryOnly !== true,
                discoveryOnly: discoveryOnly === true,
            },
        })

        return reply.status(StatusCodes.OK).send({ conversationId: convId, runId, priorAssistantTurns })
    })

    // Pure observer of an eval conversation's live state — read the row directly (not
    // getConversationOrThrow, which resets a stale STREAMING row to IDLE).
    app.get('/eval/conversations/:conversationId/state', EvalStateRoute, async (request, reply) => {
        if (!isEvalConversationId(request.params.conversationId)) {
            return reply.status(StatusCodes.NOT_FOUND).send({ message: 'Conversation not found' })
        }
        const conversation = await agentHelpers.conversationRepo().findOneBy({ id: request.params.conversationId })
        if (isNil(conversation)) {
            return reply.status(StatusCodes.NOT_FOUND).send({ message: 'Conversation not found' })
        }
        return reply.status(StatusCodes.OK).send({
            status: conversation.status,
            uiMessages: conversation.uiMessages ?? [],
        })
    })
}

async function waitForSimulationResult({ conversationId, platformId, userId, priorAssistantTurns, log }: { conversationId: string, platformId: string, userId: string, priorAssistantTurns: number, log: FastifyBaseLogger }): Promise<{ status: SimulationStatus, uiMessages: unknown[] | null }> {
    // Read the row directly, not via getConversationOrThrow (which resets a stale STREAMING
    // conversation to IDLE), so the poller stays a pure observer.
    for (let attempt = 0; attempt < SIMULATION_MAX_ATTEMPTS; attempt++) {
        await new Promise((resolve) => setTimeout(resolve, SIMULATION_POLL_INTERVAL_MS))
        const settled = readSettledState(await agentHelpers.conversationRepo().findOneBy({ id: conversationId, platformId, userId }), priorAssistantTurns)
        if (settled) {
            return settled
        }
    }
    log.warn({ conversation: { id: conversationId } }, 'Chat simulation timed out before completing')
    return { status: SIMULATION_TIMEOUT_STATUS, uiMessages: null }
}

function readSettledState(conversation: { status: AgentConversationStatus, uiMessages: unknown[] | null } | null, priorAssistantTurns: number): { status: AgentConversationStatus, uiMessages: unknown[] | null } | null {
    if (!conversation) {
        return null
    }
    const uiMessages = conversation.uiMessages ?? null
    if (conversation.status === AgentConversationStatus.ERROR) {
        return { status: conversation.status, uiMessages }
    }
    // Settle only once THIS turn's assistant reply has landed: the user message is persisted at
    // the start, and earlier turns already left assistant messages, so we wait for the assistant
    // count to grow past the count we had before this turn.
    if (conversation.status === AgentConversationStatus.IDLE && countAssistantTurns(uiMessages) > priorAssistantTurns) {
        return { status: conversation.status, uiMessages }
    }
    return null
}

function countAssistantTurns(uiMessages: unknown[] | null): number {
    return (uiMessages ?? []).filter((message) =>
        typeof message === 'object' && message !== null && 'role' in message && message.role === PersistedAgentRole.ASSISTANT,
    ).length
}

const PromptSourcesRoute = {
    config: {
        security: securityAccess.public(),
    },
    schema: {
        tags: ['agent'],
    },
}

const SandboxPlatformRoute = {
    config: {
        security: securityAccess.public(),
    },
    schema: {
        tags: ['agent'],
    },
}

const SimulateRoute = {
    config: {
        security: securityAccess.public(),
    },
    schema: {
        tags: ['agent'],
        body: SimulateAgentRequest,
    },
}

const EvalTurnStartRequest = z.object({
    conversationId: z.string().optional(),
    platformId: z.string().optional(),
    userMessage: z.string().min(1).max(MAX_AGENT_TEXT_LENGTH),
    promptOverride: AgentPromptOverride.optional(),
    // Opt-in (default off): run the turn with tools actually executing against the platform
    // owner's real connections, instead of the dry-run playground stub. The failure-mode eval
    // harness needs real discovery/execution results to measure how the agent uses pieces;
    // dry-run returns "not executed" for every cross-project tool, which hides piece-use behavior.
    executeTools: z.boolean().optional(),
    // Opt-in (default off): real discovery, but ap_execute_action neutralized and approval gates
    // auto-resolved — measures how the agent navigates to a runnable call with zero side effects.
    discoveryOnly: z.boolean().optional(),
})

const EvalStateParams = z.object({
    conversationId: z.string(),
})

const EvalTurnStartRoute = {
    config: {
        security: securityAccess.public(),
    },
    schema: {
        tags: ['agent'],
        body: EvalTurnStartRequest,
    },
}

const EvalStateRoute = {
    config: {
        security: securityAccess.public(),
    },
    schema: {
        tags: ['agent'],
        params: EvalStateParams,
    },
}