Skip to content

AI Agents & MCPs & AI Workflow Automation • (~400 MCP servers for AI agents) • AI Automation / AI Agent with MCPs • AI Workflows & AI Agents • MCPs for AI Agents

11.4 KB
Raw Blame
import { AIProviderName, chunk, isNil, tryCatch } from '@activepieces/core-utils'
import { AgentConversation, AgentRunSource, ApEdition, PersistedAgentMessage, PersistedAgentPartType, PersistedToolCallStatus } from '@activepieces/shared'
import { FastifyBaseLogger } from 'fastify'
import { FirstCandidate } from '../../ai/ai-model-candidates'
import { isNotOneOfTheseEditions } from '../../database/database-common'
import { rejectedPromiseHandler } from '../../helper/promise-handler'
import { system } from '../../helper/system/system'
import { AppSystemProp } from '../../helper/system/system-props'
import { platformService } from '../../platform/platform.service'
import { userService } from '../../user/user-service'
import { platformPlanRepo } from '../platform/platform-plan/platform-plan.service'
import { agentHelpers } from './agent-helpers'
import { agentModelTier, agentTierCandidates } from './agent-model-tier'
import { agentHistory } from './history/agent-history'

const CONSOLE_TELEMETRY_URL = 'https://console.activepieces.com/api/chat-analytics/external/sync'
const BATCH_SIZE = 50
const REQUEST_TIMEOUT_MS = 30000

export const chatAnalyticsTelemetry = (log: FastifyBaseLogger) => ({
    sendConversationUpdate({ conversation }: {
        conversation: AgentConversation
    }): void {
        rejectedPromiseHandler(syncConversations({ conversations: [conversation], log }), log)
    },
})

export const chatAnalyticsBulkSync = (log: FastifyBaseLogger) => ({
    async syncAll({ conversations }: {
        conversations: AgentConversation[]
    }): Promise<{ synced: number, failed: number }> {
        const result = await syncConversations({ conversations, log })
        return { synced: result.pushed, failed: result.skipped + result.failed }
    },
})

async function syncConversations({ conversations, log }: {
    conversations: AgentConversation[]
    log: FastifyBaseLogger
}): Promise<{ pushed: number, skipped: number, failed: number }> {
    if (isNotOneOfTheseEditions([ApEdition.CLOUD])) {
        return { pushed: 0, skipped: conversations.length, failed: 0 }
    }

    const skippedBySource = conversations.length
    conversations = conversations.filter((conversation) => conversation.source === AgentRunSource.CHAT)

    if (conversations.length === 0) {
        return { pushed: 0, skipped: skippedBySource, failed: 0 }
    }

    const platformIds = [...new Set(conversations.map((c) => c.platformId))]
    const licenseKeyByPlatform = await resolveLicenseKeysByPlatform({ platformIds })
    const { userCache, platformCache, providerCache, tierCache } = await resolveLookups({ conversations, log })

    let pushed = 0
    let skipped = 0
    let failed = 0
    for (const batch of chunk(conversations, BATCH_SIZE)) {
        const result = await pushBatch({ conversations: batch, licenseKeyByPlatform, log, userCache, platformCache, providerCache, tierCache })
        pushed += result.pushed
        skipped += result.skipped
        if (!result.success) {
            failed += batch.length - result.skipped
        }
    }

    return { pushed, skipped, failed }
}

async function resolveLicenseKeysByPlatform({ platformIds }: {
    platformIds: string[]
}): Promise<Map<string, string>> {
    if (platformIds.length === 0) {
        return new Map()
    }

    const rows = await platformPlanRepo()
        .createQueryBuilder('platform_plan')
        .select('platform_plan.platformId', 'platformId')
        .addSelect('platform_plan.licenseKey', 'licenseKey')
        .where('platform_plan.platformId IN (:...platformIds)', { platformIds })
        .andWhere('platform_plan.licenseKey IS NOT NULL')
        .getRawMany<{ platformId: string, licenseKey: string }>()

    const map = new Map<string, string>()
    for (const row of rows) {
        map.set(row.platformId, row.licenseKey)
    }
    return map
}

function chatProviderCacheKey(conversation: AgentConversation): string {
    return `${conversation.platformId}:${conversation.projectId ?? ''}:${conversation.providerConfigId ?? ''}`
}

async function resolveLookups({ conversations, log }: {
    conversations: AgentConversation[]
    log: FastifyBaseLogger
}): Promise<ConversationLookups> {
    const uniqueUserIds = [...new Set(conversations.map((c) => c.userId))]
    const uniquePlatformIds = [...new Set(conversations.map((c) => c.platformId))]
    const uniqueScopes = [...new Map(conversations.map((c) => [chatProviderCacheKey(c), c])).values()]

    const [userEntries, platformNameEntries, providerEntries, tierCache] = await Promise.all([
        Promise.all(uniqueUserIds.map(async (userId): Promise<[string, string | null]> => [userId, await resolveUserEmail({ userId, log })])),
        Promise.all(uniquePlatformIds.map(async (platformId): Promise<[string, string | null]> => [platformId, await resolvePlatformName({ platformId, log })])),
        Promise.all(uniqueScopes.map(async (conversation): Promise<[string, AIProviderName | null]> => [
            chatProviderCacheKey(conversation),
            await resolveProviderName({ conversation, log }),
        ])),
        agentModelTier(log).mainModelsOf({ conversations }),
    ])

    return {
        userCache: new Map(userEntries),
        platformCache: new Map(platformNameEntries),
        providerCache: new Map(providerEntries),
        tierCache,
    }
}

async function pushBatch({ conversations, licenseKeyByPlatform, log, userCache, platformCache, providerCache, tierCache }: {
    conversations: AgentConversation[]
    licenseKeyByPlatform: Map<string, string>
    log: FastifyBaseLogger
    userCache?: Map<string, string | null>
    platformCache?: Map<string, string | null>
    providerCache?: Map<string, AIProviderName | null>
    tierCache?: Map<string, FirstCandidate | null>
}): Promise<{ pushed: number, skipped: number, success: boolean }> {
    const secret = system.get(AppSystemProp.CONSOLE_API_SECRET_KEY)
    if (isNil(secret)) {
        return { pushed: 0, skipped: conversations.length, success: true }
    }

    const payloads: Record<string, unknown>[] = []
    for (const conversation of conversations) {
        const licenseKey = licenseKeyByPlatform.get(conversation.platformId) ?? null
        const payloadResult = await tryCatch(() => toSyncPayload({ conversation, licenseKey, log, userCache, platformCache, providerCache, tierCache }))
        if (payloadResult.error) {
            log.error({ conversation: { id: conversation.id }, error: String(payloadResult.error) }, 'Failed to build sync payload for conversation, skipping')
            continue
        }
        payloads.push(payloadResult.data)
    }

    const skipped = conversations.length - payloads.length

    if (payloads.length === 0) {
        return { pushed: 0, skipped, success: true }
    }

    const result = await tryCatch(() => fetch(CONSOLE_TELEMETRY_URL, {
        method: 'POST',
        headers: {
            'Content-Type': 'application/json',
            'Authorization': `Bearer ${secret}`,
        },
        body: JSON.stringify({ conversations: payloads }),
        signal: AbortSignal.timeout(REQUEST_TIMEOUT_MS),
    }))

    if (result.error) {
        log.error({ error: result.error }, 'Failed to push chat analytics telemetry')
        return { pushed: 0, skipped, success: false }
    }
    if (!result.data.ok) {
        const body = await tryCatch(() => result.data.text())
        log.error({ status: result.data.status, body: body.error ? 'unreadable' : body.data }, 'Failed to push chat analytics telemetry: non-2xx response')
        return { pushed: 0, skipped, success: false }
    }
    return { pushed: payloads.length, skipped, success: true }
}

async function toSyncPayload({ conversation, licenseKey, log, userCache, platformCache, providerCache, tierCache }: {
    conversation: AgentConversation
    licenseKey: string | null
    log: FastifyBaseLogger
    userCache?: Map<string, string | null>
    platformCache?: Map<string, string | null>
    providerCache?: Map<string, AIProviderName | null>
    tierCache?: Map<string, FirstCandidate | null>
}): Promise<Record<string, unknown>> {
    const userEmail = userCache?.get(conversation.userId) ?? await resolveUserEmail({ userId: conversation.userId, log })
    const platformName = platformCache?.get(conversation.platformId) ?? await resolvePlatformName({ platformId: conversation.platformId, log })
    const tierKey = agentTierCandidates.tierCacheKey({ platformId: conversation.platformId, tierId: conversation.modelTierId ?? '' })
    const tierModel = tierCache?.has(tierKey) ? tierCache.get(tierKey) ?? null : await agentModelTier(log).mainModelOf({ conversation })
    const provider = tierModel?.provider ?? providerCache?.get(chatProviderCacheKey(conversation)) ?? await resolveProviderName({ conversation, log })

    const messages = agentHistory.resolveMessages({ conversation, log })

    return {
        id: conversation.id,
        licenseKey,
        platformId: conversation.platformId,
        platformName,
        userId: conversation.userId,
        userEmail,
        title: conversation.title,
        modelName: tierModel?.modelId ?? agentHelpers.resolveModelIdForAnalytics({ selectedModel: conversation.modelName ?? null, provider, surface: 'chat', pinned: !isNil(conversation.providerConfigId) }),
        provider,
        messages,
        messageCount: messages.length,
        toolCallsSummary: extractToolCallsSummary(messages),
        isActive: false,
        createdAt: conversation.created,
        updatedAt: conversation.updated,
    }
}


function extractToolCallsSummary(messages: PersistedAgentMessage[]): Array<{ name: string, successCount: number, failureCount: number }> | null {
    const usage: Record<string, { successCount: number, failureCount: number }> = {}

    for (const msg of messages) {
        for (const part of msg.parts) {
            if (part.type !== PersistedAgentPartType.TOOL_CALL) continue

            const name = part.toolName
            if (!usage[name]) {
                usage[name] = { successCount: 0, failureCount: 0 }
            }
            if (part.status === PersistedToolCallStatus.COMPLETED) {
                usage[name].successCount++
            }
            else {
                usage[name].failureCount++
            }
        }
    }

    const entries = Object.entries(usage).map(([name, counts]) => ({ name, ...counts }))
    return entries.length > 0 ? entries : null
}

async function resolveUserEmail({ userId, log }: { userId: string, log: FastifyBaseLogger }): Promise<string | null> {
    const result = await tryCatch(() => userService(log).getMetaInformation({ id: userId }))
    return result.error ? null : result.data.email
}

async function resolvePlatformName({ platformId, log }: { platformId: string, log: FastifyBaseLogger }): Promise<string | null> {
    const result = await tryCatch(() => platformService(log).getOneOrThrow(platformId))
    return result.error ? null : result.data.name
}

async function resolveProviderName({ conversation, log }: { conversation: AgentConversation, log: FastifyBaseLogger }): Promise<AIProviderName | null> {
    const result = await tryCatch(() => agentHelpers.resolveConversationProviderName({ platformId: conversation.platformId, projectId: conversation.projectId ?? null, providerConfigId: conversation.providerConfigId ?? null, log }))
    return result.error ? null : result.data
}

type ConversationLookups = {
    userCache: Map<string, string | null>
    platformCache: Map<string, string | null>
    providerCache: Map<string, AIProviderName | null>
    tierCache: Map<string, FirstCandidate | null>
}