Skip to content

AI Agents & MCPs & AI Workflow Automation • (~400 MCP servers for AI agents) • AI Automation / AI Agent with MCPs • AI Workflows & AI Agents • MCPs for AI Agents

22.7 KB
Raw Blame
import { ActivepiecesAiBilling, ActivepiecesAiConsumerSource, ActivepiecesError, AiUsageEnvironment, AiUsageOrigin, apId, ErrorCode, isNil, spreadIfDefined, tryCatch } from '@activepieces/core-utils'
import { aiUtils } from '@activepieces/server-utils'
import { AI_PROVIDER_CAPABILITIES, FileCompression, FileType, GetProviderConfigResponse, KnowledgeBaseEmbeddingSource, KnowledgeBaseFile } from '@activepieces/shared'
import { SharedV3ProviderOptions } from '@ai-sdk/provider'
import { EmbeddingModel, embedMany } from 'ai'
import { parse as parseCsv } from 'csv-parse/sync'
import { FastifyBaseLogger } from 'fastify'
import { In, IsNull, Not } from 'typeorm'
import { aiProviderService } from '../ai/ai-provider-service'
import { aiToolConfigService } from '../ai/ai-tool-config-service'
import { repoFactory } from '../core/db/repo-factory'
import { transaction } from '../core/db/transaction'
import { databaseConnection } from '../database/database-connection'
import { distributedLock } from '../database/redis-connections'
import { fileService } from '../file/file.service'
import { KnowledgeBaseChunkEntity } from './knowledge-base-chunk.entity'
import { KnowledgeBaseFileEntity } from './knowledge-base-file.entity'

const kbFileRepo = repoFactory(KnowledgeBaseFileEntity)
const kbChunkRepo = repoFactory(KnowledgeBaseChunkEntity)

const INSERT_BATCH_SIZE = 100
const CHUNK_SIZE_CHARS = 2000
const CHUNK_OVERLAP_CHARS = 200
const EMBED_BATCH_SIZE = 50
const KNOWLEDGE_BASE_BILLING_CONVERSATION = 'knowledge-base'

function chunkText(text: string): string[] {
    const chunks: string[] = []
    let start = 0
    while (start < text.length) {
        const end = Math.min(start + CHUNK_SIZE_CHARS, text.length)
        chunks.push(text.slice(start, end))
        if (end >= text.length) break
        start = end - CHUNK_OVERLAP_CHARS
    }
    return chunks
}

function chunkCsvText(csvText: string): string[] {
    const records: string[][] = parseCsv(csvText, { relax_column_count: true })
    if (records.length === 0) return []

    const headerLine = records[0].join(',')
    const chunks: string[] = []
    let currentRows: string[] = []
    let currentLength = headerLine.length + 1

    for (let i = 1; i < records.length; i++) {
        const rowLine = records[i].join(',')
        if (currentLength + rowLine.length + 1 > CHUNK_SIZE_CHARS && currentRows.length > 0) {
            chunks.push(headerLine + '\n' + currentRows.join('\n'))
            currentRows = []
            currentLength = headerLine.length + 1
        }
        currentRows.push(rowLine)
        currentLength += rowLine.length + 1
    }
    if (currentRows.length > 0) {
        chunks.push(headerLine + '\n' + currentRows.join('\n'))
    }
    return chunks
}

async function chunksOf({ data, fileName }: { data: Buffer, fileName: string }): Promise<string[]> {
    if (fileName.toLowerCase().endsWith('.csv')) {
        return chunkCsvText(data.toString('utf-8'))
    }
    return chunkText(await extractTextFromFile(data, fileName))
}

async function embedAll({ texts, embedFn }: { texts: string[], embedFn: EmbedFn }): Promise<number[][]> {
    const embeddings: number[][] = []
    for (let start = 0; start < texts.length; start += EMBED_BATCH_SIZE) {
        const batch = texts.slice(start, start + EMBED_BATCH_SIZE)
        const batchEmbeddings = await embedFn(batch)
        if (batchEmbeddings.length !== batch.length) {
            throw new Error(`Embedding count mismatch: expected ${batch.length}, got ${batchEmbeddings.length}`)
        }
        embeddings.push(...batchEmbeddings)
    }
    return embeddings
}

function toVector(embedding: number[]): string {
    return `[${embedding.join(',')}]`
}

async function extractTextFromFile(fileBuffer: Buffer, fileName: string): Promise<string> {
    const lowerName = (fileName ?? '').toLowerCase()
    if (lowerName.endsWith('.pdf')) {
        const { extractText, getDocumentProxy } = await import('unpdf')
        const pdf = await getDocumentProxy(new Uint8Array(fileBuffer))
        const { text } = await extractText(pdf, { mergePages: true })
        return text
    }

    if (lowerName.endsWith('.docx')) {
        const mammoth = await import('mammoth')
        const result = await mammoth.extractRawText({ buffer: fileBuffer })
        return result.value
    }

    return fileBuffer.toString('utf-8')
}

export const knowledgeBaseService = (log: FastifyBaseLogger) => ({
    async uploadFile(params: UploadFileParams): Promise<KnowledgeBaseFile> {
        const { projectId, data, fileName, displayName, embedFn } = params
        const texts = (await chunksOf({ data, fileName })).filter((text) => text.trim().length > 0)
        if (texts.length === 0) {
            throw new ActivepiecesError({
                code: ErrorCode.VALIDATION,
                params: { message: KNOWLEDGE_BASE_FILE_HAS_NO_TEXT },
            })
        }
        const embeddings = await embedAll({ texts, embedFn })
        const savedFile = await fileService(log).save({
            projectId,
            data,
            size: data.length,
            type: FileType.KNOWLEDGE_BASE,
            compression: FileCompression.NONE,
            fileName,
        })
        const { data: kbFile, error } = await tryCatch(() => transaction(async (entityManager) => {
            const file = await entityManager.getRepository(KnowledgeBaseFileEntity).save({ id: apId(), projectId, fileId: savedFile.id, displayName })
            const rows = texts.map((content, chunkIndex) => ({
                id: apId(),
                projectId,
                knowledgeBaseFileId: file.id,
                chunkIndex,
                content,
                embedding: toVector(embeddings[chunkIndex]),
                metadata: { chunkIndex, totalChunks: texts.length },
            }))
            for (let start = 0; start < rows.length; start += INSERT_BATCH_SIZE) {
                await entityManager.getRepository(KnowledgeBaseChunkEntity).insert(rows.slice(start, start + INSERT_BATCH_SIZE))
            }
            return file
        }))
        if (error) {
            await fileService(log).delete({ projectId, fileId: savedFile.id })
            throw error
        }
        return kbFile
    },

    async embedderFor(params: { projectId: string, platformId: string }): Promise<EmbedFn> {
        const { projectId, platformId } = params
        const { model, providerOptions } = await this.embeddingModelFor({
            platformId,
            projectId,
            billing: { source: ActivepiecesAiConsumerSource.CHAT, platformId, projectId, conversationId: KNOWLEDGE_BASE_BILLING_CONVERSATION, context: { origin: AiUsageOrigin.KNOWLEDGE_BASE, environment: AiUsageEnvironment.PRODUCTION } },
        })
        return this.embedFnOf({ model, providerOptions })
    },

    async embeddingModelFor({ platformId, projectId, billing }: { platformId: string, projectId: string, billing: ActivepiecesAiBilling }): Promise<{ model: EmbeddingModel, providerOptions: SharedV3ProviderOptions }> {
        const source = await this.embeddingSourceFor({ platformId, projectId })
        if (isNil(source)) {
            throw new ActivepiecesError({
                code: ErrorCode.VALIDATION,
                params: { message: KNOWLEDGE_BASE_NEEDS_AI_PROVIDER },
            })
        }
        const { credentials, modelId } = source
        return aiUtils.createEmbeddingModel({ credentials, modelId, platformId, providerConfigId: credentials.configId, billing })
    },

    async embeddingSourceFor({ platformId, projectId }: { platformId: string, projectId: string }): Promise<{ credentials: GetProviderConfigResponse, modelId: string } | null> {
        const scope = { type: 'project' as const, projectId }
        const choice = (await aiToolConfigService(log).getProviderChoices({ platformId })).embeddings
        const chosen = isNil(choice) ? null : await aiProviderService(log).findKeyConfig({ platformId, configId: choice.aiProviderId, scope })
        if (!isNil(chosen)) {
            const pickedModel = choice?.modelId ?? AI_PROVIDER_CAPABILITIES[chosen.provider].defaultEmbeddingModel
            return isNil(pickedModel) ? null : { credentials: chosen, modelId: pickedModel }
        }
        const defaultKey = await aiProviderService(log).getChatProvider({ platformId, scope })
        const defaultKeyModel = isNil(defaultKey) ? undefined : AI_PROVIDER_CAPABILITIES[defaultKey.provider].defaultEmbeddingModel
        if (!isNil(defaultKey) && !isNil(defaultKeyModel)) {
            return { credentials: defaultKey, modelId: defaultKeyModel }
        }
        const credits = await aiProviderService(log).findCreditsConfig({ platformId, scope })
        const creditsModel = isNil(credits) ? undefined : AI_PROVIDER_CAPABILITIES[credits.provider].defaultEmbeddingModel
        return isNil(credits) || isNil(creditsModel) ? null : { credentials: credits, modelId: creditsModel }
    },

    async getEmbeddingSource({ platformId, projectId }: { platformId: string, projectId: string }): Promise<KnowledgeBaseEmbeddingSource> {
        const source = await this.embeddingSourceFor({ platformId, projectId })
        return { provider: source?.credentials.provider ?? null, modelId: source?.modelId ?? null }
    },

    embedFnOf(params: { model: EmbeddingModel, providerOptions: SharedV3ProviderOptions }): EmbedFn {
        const { model, providerOptions } = params
        return async (texts) => {
            const { embeddings } = await embedMany({ model, values: texts, providerOptions })
            return embeddings.map((embedding) => aiUtils.toStorageEmbedding(embedding))
        }
    },

    async embedMissingChunks(params: EmbedMissingChunksParams): Promise<number> {
        const { projectId, knowledgeBaseFileId, resolveEmbedFn } = params
        const hasMissing = await kbChunkRepo().existsBy({ projectId, knowledgeBaseFileId, embedding: IsNull() })
        if (!hasMissing) {
            return 0
        }
        return distributedLock(log).runExclusive({
            key: `knowledge_base_embed_${knowledgeBaseFileId}`,
            timeoutInSeconds: 60,
            fn: async () => {
                const missing = await kbChunkRepo().find({
                    where: { projectId, knowledgeBaseFileId, embedding: IsNull() },
                    select: ['id', 'content'],
                    order: { chunkIndex: 'ASC' },
                })
                if (missing.length === 0) {
                    return 0
                }
                const embeddings = await embedAll({ texts: missing.map((chunk) => chunk.content), embedFn: await resolveEmbedFn() })
                await transaction(async (entityManager) => {
                    for (let start = 0; start < missing.length; start += INSERT_BATCH_SIZE) {
                        const batch = missing.slice(start, start + INSERT_BATCH_SIZE)
                        await entityManager.query(
                            `UPDATE knowledge_base_chunk AS kbc
                             SET embedding = missing.embedding::vector
                             FROM unnest($1::varchar[], $2::text[], $3::text[]) AS missing(id, content, embedding)
                             WHERE kbc.id = missing.id AND kbc.content = missing.content AND kbc.embedding IS NULL
                               AND kbc."projectId" = $4 AND kbc."knowledgeBaseFileId" = $5`,
                            [batch.map((chunk) => chunk.id), batch.map((chunk) => chunk.content), embeddings.slice(start, start + INSERT_BATCH_SIZE).map(toVector), projectId, knowledgeBaseFileId],
                        )
                    }
                })
                return missing.length
            },
        })
    },

    async search(params: SearchParams): Promise<SearchResult[]> {
        const { projectId, knowledgeBaseFileIds, queryEmbedding, limit, similarityThreshold } = params
        const embeddingStr = toVector(queryEmbedding)

        const results = await databaseConnection().query(
            `SELECT kbc.id, kbc.content, kbc.metadata, kbc."chunkIndex",
                    kbc.embedding <=> $1::vector AS distance
             FROM knowledge_base_chunk kbc
             WHERE kbc."projectId" = $2
               AND kbc."knowledgeBaseFileId" = ANY($3)
               AND kbc.embedding IS NOT NULL
             ORDER BY distance
             LIMIT $4`,
            [embeddingStr, projectId, knowledgeBaseFileIds, limit],
        )

        return results
            .map((row: SearchRow) => ({
                id: row.id,
                content: row.content,
                metadata: row.metadata,
                chunkIndex: row.chunkIndex,
                score: Math.max(0, 1 - row.distance),
            }))
            .filter((row: SearchResult) => similarityThreshold === undefined || row.score >= similarityThreshold)
    },

    async listFiles(params: { projectId: string }): Promise<KnowledgeBaseFile[]> {
        return kbFileRepo().find({
            where: { projectId: params.projectId },
            order: { created: 'DESC' },
        })
    },

    async createFile(params: CreateFileParams): Promise<KnowledgeBaseFile> {
        const kbFile = {
            id: apId(),
            projectId: params.projectId,
            fileId: params.fileId,
            displayName: params.displayName,
        }
        return kbFileRepo().save(kbFile)
    },

    async deleteFile(params: { projectId: string, id: string }): Promise<void> {
        const kbFile = await kbFileRepo().findOneBy({
            id: params.id,
            projectId: params.projectId,
        })
        if (!kbFile) {
            return
        }
        await kbFileRepo().delete({
            id: params.id,
            projectId: params.projectId,
        })
        await fileService(log).delete({
            projectId: params.projectId,
            fileId: kbFile.fileId,
        })
    },

    async getFileOrThrow(params: { projectId: string, id: string }): Promise<KnowledgeBaseFile> {
        const file = await kbFileRepo().findOneBy({
            id: params.id,
            projectId: params.projectId,
        })
        if (!file) {
            throw new ActivepiecesError({
                code: ErrorCode.ENTITY_NOT_FOUND,
                params: {
                    entityType: 'KnowledgeBaseFile',
                    entityId: params.id,
                },
            })
        }
        return file
    },

    async getChunkCount(params: { projectId: string, knowledgeBaseFileId: string }): Promise<number> {
        return kbChunkRepo().count({ where: { projectId: params.projectId, knowledgeBaseFileId: params.knowledgeBaseFileId } })
    },

    async isSearchable(params: { projectId: string, knowledgeBaseFileId: string }): Promise<boolean> {
        const { projectId, knowledgeBaseFileId } = params
        const [hasChunks, hasUnindexedChunks] = await Promise.all([
            kbChunkRepo().existsBy({ projectId, knowledgeBaseFileId }),
            kbChunkRepo().existsBy({ projectId, knowledgeBaseFileId, embedding: IsNull() }),
        ])
        return hasChunks && !hasUnindexedChunks
    },

    async extractChunks(params: { projectId: string, knowledgeBaseFileId: string }): Promise<string[]> {
        const kbFile = await kbFileRepo().findOneBy({
            id: params.knowledgeBaseFileId,
            projectId: params.projectId,
        })
        if (!kbFile) {
            throw new ActivepiecesError({
                code: ErrorCode.ENTITY_NOT_FOUND,
                params: {
                    entityType: 'KnowledgeBaseFile',
                    entityId: params.knowledgeBaseFileId,
                },
            })
        }

        const fileData = await fileService(log).getDataOrThrow({
            projectId: params.projectId,
            fileId: kbFile.fileId,
        })

        return chunksOf({ data: fileData.data, fileName: fileData.fileName || kbFile.displayName })
    },

    async storeChunks(params: StoreChunksParams): Promise<void> {
        const { projectId, knowledgeBaseFileId, chunks } = params
        const isFullRestore = chunks.every((chunk) => isNil(chunk.id))

        await transaction(async (entityManager) => {
            const owner = await entityManager.getRepository(KnowledgeBaseFileEntity)
                .createQueryBuilder('file')
                .setLock('pessimistic_write')
                .where('file.id = :knowledgeBaseFileId AND file."projectId" = :projectId', { knowledgeBaseFileId, projectId })
                .getOne()
            if (isNil(owner)) {
                throw new ActivepiecesError({
                    code: ErrorCode.ENTITY_NOT_FOUND,
                    params: { entityType: 'KnowledgeBaseFile', entityId: knowledgeBaseFileId },
                })
            }
            const repo = entityManager.getRepository(KnowledgeBaseChunkEntity)
            const stored = await repo.find({ where: { projectId, knowledgeBaseFileId }, select: ['id', 'chunkIndex'] })
            const storedById = new Map(stored.map((row) => [row.id, row]))
            const idByIndex = new Map(stored.map((row) => [row.chunkIndex, row.id]))

            const writeByIndex = new Map<number, { id?: string, values: Record<string, unknown> }>()
            const submittedIds = new Set<string>()
            for (const [position, chunk] of chunks.entries()) {
                const values = {
                    ...spreadIfDefined('content', chunk.content),
                    ...spreadIfDefined('embedding', chunk.embedding ? toVector(chunk.embedding) : undefined),
                    ...spreadIfDefined('metadata', chunk.metadata),
                }
                if (isNil(chunk.id)) {
                    writeByIndex.set(chunk.chunkIndex ?? position, { values: { content: '', metadata: {}, ...values } })
                    continue
                }
                if (submittedIds.has(chunk.id)) {
                    throw new ActivepiecesError({
                        code: ErrorCode.VALIDATION,
                        params: { message: `Chunk ${chunk.id} is listed more than once in the same request` },
                    })
                }
                submittedIds.add(chunk.id)
                const edited = storedById.get(chunk.id)
                if (isNil(edited)) {
                    throw new ActivepiecesError({
                        code: ErrorCode.ENTITY_NOT_FOUND,
                        params: { entityType: 'KnowledgeBaseChunk', entityId: chunk.id },
                    })
                }
                writeByIndex.set(chunk.chunkIndex ?? edited.chunkIndex, { id: chunk.id, values })
            }

            const claimedIds = new Set([...writeByIndex.values()].map((write) => write.id).filter((id) => !isNil(id)))
            const adoptableIdByIndex = new Map([...idByIndex].filter(([, id]) => !claimedIds.has(id)))
            const writes = [...writeByIndex].map(([chunkIndex, write]) => ({
                chunkIndex,
                id: write.id ?? adoptableIdByIndex.get(chunkIndex),
                values: write.values,
            }))
            const keptIds = new Set(writes.map((write) => write.id).filter((id) => !isNil(id)))
            const displacedIds = stored
                .filter((row) => !keptIds.has(row.id) && writeByIndex.has(row.chunkIndex))
                .map((row) => row.id)
            for (let start = 0; start < displacedIds.length; start += INSERT_BATCH_SIZE) {
                await repo.delete({ id: In(displacedIds.slice(start, start + INSERT_BATCH_SIZE)), projectId, knowledgeBaseFileId })
            }

            const inserts = writes
                .filter((write) => isNil(write.id))
                .map((write) => ({ id: apId(), projectId, knowledgeBaseFileId, chunkIndex: write.chunkIndex, ...write.values }))
            for (const write of writes) {
                if (isNil(write.id)) {
                    continue
                }
                await repo.update({ id: write.id, projectId, knowledgeBaseFileId }, { ...write.values, chunkIndex: write.chunkIndex })
            }
            for (let start = 0; start < inserts.length; start += INSERT_BATCH_SIZE) {
                await repo.insert(inserts.slice(start, start + INSERT_BATCH_SIZE))
            }

            if (!isFullRestore) {
                return
            }
            const submittedIndexes = [...writeByIndex.keys()]
            const cleanup = repo.createQueryBuilder()
                .delete()
                .where('"projectId" = :projectId AND "knowledgeBaseFileId" = :knowledgeBaseFileId', { projectId, knowledgeBaseFileId })
            if (submittedIndexes.length > 0) {
                cleanup.andWhere('"chunkIndex" NOT IN (:...submittedIndexes)', { submittedIndexes })
            }
            await cleanup.execute()
        })
    },

    async listChunks(params: ListChunksParams): Promise<ChunkListItem[]> {
        return kbChunkRepo().find({
            where: {
                projectId: params.projectId,
                knowledgeBaseFileId: params.knowledgeBaseFileId,
                ...params.embedded === false ? { embedding: IsNull() } : {},
                ...params.embedded === true ? { embedding: Not(IsNull()) } : {},
            },
            select: ['id', 'content', 'chunkIndex'],
            order: { chunkIndex: 'ASC' },
        })
    },

    async getFilesByIds(params: { projectId: string, ids: string[] }): Promise<KnowledgeBaseFile[]> {
        if (params.ids.length === 0) return []
        return kbFileRepo().find({
            where: params.ids.map(id => ({
                id,
                projectId: params.projectId,
            })),
        })
    },
})

export const KNOWLEDGE_BASE_NEEDS_AI_PROVIDER = 'KNOWLEDGE_BASE_NEEDS_AI_PROVIDER'
export const KNOWLEDGE_BASE_FILE_HAS_NO_TEXT = 'KNOWLEDGE_BASE_FILE_HAS_NO_TEXT'

type UploadFileParams = {
    projectId: string
    data: Buffer
    fileName: string
    displayName: string
    embedFn: EmbedFn
}

type EmbedFn = (texts: string[]) => Promise<number[][]>

type EmbedMissingChunksParams = {
    projectId: string
    knowledgeBaseFileId: string
    resolveEmbedFn: () => Promise<EmbedFn>
}

type SearchParams = {
    projectId: string
    knowledgeBaseFileIds: string[]
    queryEmbedding: number[]
    limit: number
    similarityThreshold?: number
}

type SearchRow = {
    id: string
    content: string
    metadata: Record<string, unknown>
    chunkIndex: number
    distance: number
}

type SearchResult = {
    id: string
    content: string
    metadata: Record<string, unknown>
    chunkIndex: number
    score: number
}

type CreateFileParams = {
    projectId: string
    fileId: string
    displayName: string
}

type StoreChunksParams = {
    projectId: string
    knowledgeBaseFileId: string
    chunks: {
        id?: string
        content?: string
        embedding?: number[]
        chunkIndex?: number
        metadata?: Record<string, unknown>
    }[]
}

type ListChunksParams = {
    projectId: string
    knowledgeBaseFileId: string
    embedded?: boolean
}

type ChunkListItem = {
    id: string
    content: string
    chunkIndex: number
}