Skip to content

AI Agents & MCPs & AI Workflow Automation • (~400 MCP servers for AI agents) • AI Automation / AI Agent with MCPs • AI Workflows & AI Agents • MCPs for AI Agents

16.9 KB
Raw Blame
import { Readable } from 'node:stream'
import { buffer as streamToBuffer } from 'node:stream/consumers'
import { setTimeout as delay } from 'node:timers/promises'
import { ActivepiecesError, apId, assertNotNullOrUndefined, ErrorCode, isMultipartFile, isNil, ProjectId } from '@activepieces/core-utils'
import { File, FileCompression, FileId, FileLocation, FileType, Project } from '@activepieces/shared'
import dayjs from 'dayjs'
import { FastifyBaseLogger } from 'fastify'
import { In, LessThan, LessThanOrEqual } from 'typeorm'
import { repoFactory } from '../core/db/repo-factory'
import { exceptionHandler } from '../helper/exception-handler'
import { jwtUtils } from '../helper/jwt-utils'
import { system } from '../helper/system/system'
import { AppSystemProp } from '../helper/system/system-props'
import { projectRepo } from '../project/project-repo'
import { fileCompressor } from './file-compressor'
import { FileEntity } from './file.entity'
import { s3Helper } from './s3-helper'

const ALLOWED_SIGNED_FILE_TYPES: FileType[] = [FileType.FLOW_STEP_FILE, FileType.FLOW_RUN_LOG_SLICE]

const IMAGE_MIME_TYPES = ['image/jpeg', 'image/png', 'image/gif', 'image/webp', 'image/svg+xml', 'image/tiff', 'image/bmp', 'image/ico', 'image/avif', 'image/apng']

export const fileRepo = repoFactory<File>(FileEntity)
const EXECUTION_DATA_RETENTION_DAYS = system.getNumberOrThrow(AppSystemProp.EXECUTION_DATA_RETENTION_DAYS)
const FILE_CLEANUP_STATEMENT_TIMEOUT_MS = 5 * 60 * 1000
const FILE_CLEANUP_PAUSE_BETWEEN_ITERATIONS_MS = 1000

type BaseFile = Pick<File, 'id' | 'projectId' | 'platformId' | 'type' | 'fileName' | 'compression' | 'size' | 'metadata' | 'created' | 'updated'>

const saveFileToDb = async (baseFile: BaseFile, data: Buffer | null) => {
    assertNotNullOrUndefined(data, 'data is required')
    return fileRepo().save({
        ...baseFile,
        location: FileLocation.DB,
        data,
    })
}
export const fileService = (log: FastifyBaseLogger) => ({
    async save(params: SaveParams): Promise<File> {
        const baseFile: BaseFile = {
            id: params.fileId ?? apId(),
            projectId: params.projectId,
            platformId: params.platformId,
            type: params.type,
            fileName: params.fileName,
            compression: params.compression,
            size: params.size,
            metadata: params.metadata,
            created: dayjs().toISOString(),
            updated: dayjs().toISOString(),
        }
        const location = getLocationForFile(params.type)
        switch (location) {
            case FileLocation.DB: {
                if (params.data instanceof Readable) {
                    const data = await streamToBuffer(params.data)
                    return saveFileToDb({ ...baseFile, size: data.length }, data)
                }
                return saveFileToDb(baseFile, params.data)
            }
            case FileLocation.S3: {
                const s3Key = await s3Helper(log).constructS3Key(params.platformId, params.projectId, params.type, baseFile.id)
                // A stream can be consumed once, so it has no S3-error DB fallback.
                if (params.data instanceof Readable) {
                    const size = await s3Helper(log).uploadStream(s3Key, params.data)
                    return fileRepo().save({ ...baseFile, size, location: FileLocation.S3, s3Key })
                }
                try {
                    if (!isNil(params.data)) {
                        await s3Helper(log).uploadFile(s3Key, params.data)
                    }
                    return await fileRepo().save({ ...baseFile, location: FileLocation.S3, s3Key })
                }
                catch (error) {
                    if (isNil(params.data)) {
                        throw error
                    }
                    exceptionHandler.handle(error, log)
                    return saveFileToDb(baseFile, params.data)
                }
            }
        }
    },
    async exists(params: GetOneParams): Promise<boolean> {
        const file = await fileRepo().findOneBy({
            projectId: params.projectId,
            id: params.fileId,
            type: normalizeTypeFilter(params.type),
        })
        return !isNil(file)
    },
    async getFile({ projectId, fileId, type }: GetOneParams): Promise<File | null> {
        const file = await fileRepo().findOneBy({
            projectId,
            id: fileId,
            type: normalizeTypeFilter(type),
        })
        return file
    },
    async getFileOrThrow(params: GetOneParams): Promise<File> {
        const file = !isNil(params.fileId) ? await this.getFile(params) : undefined
        if (isNil(file)) {
            throw new ActivepiecesError({
                code: ErrorCode.ENTITY_NOT_FOUND,
                params: {
                    entityType: 'file',
                    entityId: params.fileId,
                    message: 'File not found',
                },
            })
        }
        return file
    },
    async getDataOrUndefined({ projectId, fileId, type }: GetOneParams): Promise<GetDataResponse | undefined> {
        try {
            return await this.getDataOrThrow({ projectId, fileId, type })
        }
        catch (error) {
            log.error({
                error,
            }, '[FileService#getData] error')
            return undefined
        }

    },
    async getDataOrThrow({ projectId, platformId, fileId, type }: GetOneParams): Promise<GetDataResponse> {
        const file = await fileRepo().findOneBy({
            projectId,
            platformId,
            id: fileId,
            type: normalizeTypeFilter(type),
        })
        if (isNil(file)) {
            throw new ActivepiecesError({
                code: ErrorCode.ENTITY_NOT_FOUND,
                params: {
                    entityType: 'file',
                    entityId: fileId,
                    message: 'File not found',
                },
            })
        }
        const data = await fileCompressor.decompress({
            data: file.location === FileLocation.DB ? file.data : await s3Helper(log).getFile(file.s3Key!),
            compression: file.compression,
        })
        return {
            metadata: file.metadata ?? undefined,
            data,
            fileName: file.fileName ?? undefined,
        }
    },
    async delete(params: { projectId: ProjectId, fileId: FileId }): Promise<void> {
        const file = await fileRepo().findOneBy({
            id: params.fileId,
            projectId: params.projectId,
        })
        if (isNil(file)) {
            return
        }
        if (!isNil(file.s3Key)) {
            await s3Helper(log).deleteFiles([file.s3Key])
        }
        await fileRepo().delete({ id: file.id })
    },
    async deleteStaleBulk(types: FileType[]) {
        const maximumFilesToDeletePerIteration = 4000
        const maximumFilesToDeletePerRun = 1_000_000
        const customRetentionProjects = await projectRepo().find({
            select: ['id', 'executionDataRetentionDays'],
            where: {
                executionDataRetentionDays: LessThan(EXECUTION_DATA_RETENTION_DAYS),
            },
        })
        const cleanupPasses: CleanupPass[] = [
            ...Array.from(groupProjectIdsByRetentionDays(customRetentionProjects), ([retentionDays, projectIds]) => ({
                retentionDateBoundary: dayjs().subtract(retentionDays, 'days').toISOString(),
                projectIds,
            })),
            {
                retentionDateBoundary: dayjs().subtract(EXECUTION_DATA_RETENTION_DAYS, 'days').toISOString(),
                projectIds: undefined,
            },
        ]
        let totalAffected = 0
        // Iterate one type at a time with an equality predicate AND an explicit ORDER BY created ASC
        // so the select hits the (type, created) index (idx_file_type_created_desc) as an index scan.
        // Either a `type IN (...)` predicate OR a missing ORDER BY makes the planner fall back to a
        // sequential scan of the file table (150M+ rows) for any type that is >10% of it — its
        // LIMIT-cost heuristic estimates the seq scan will find 4000 hits in ~2000 pages, but the
        // cleanup deletes wade through dead tuples and hit statement_timeout, so the cleanup never
        // drains and the backlog grows. The delete is by primary key only.
        // Cap the work per run so a large backlog drains across the hourly schedule instead of one
        // multi-hour run (which could outlive its worker lock); the next run resumes from the oldest.
        for (const pass of cleanupPasses) {
            for (const type of types) {
                let affected: undefined | number = undefined
                while ((isNil(affected) || affected === maximumFilesToDeletePerIteration) && totalAffected < maximumFilesToDeletePerRun) {
                    const staleFiles = await fileRepo().manager.transaction(async (em) => {
                        await em.query(`SET LOCAL statement_timeout = ${FILE_CLEANUP_STATEMENT_TIMEOUT_MS}`)
                        return em.getRepository(FileEntity).find({
                            select: ['id', 's3Key'],
                            where: {
                                type,
                                created: LessThanOrEqual(pass.retentionDateBoundary),
                                ...(pass.projectIds ? { projectId: In(pass.projectIds) } : {}),
                            },
                            order: { created: 'ASC' },
                            take: maximumFilesToDeletePerIteration,
                        })
                    })

                    if (staleFiles.length === 0) {
                        affected = 0
                        break
                    }

                    const s3Keys = staleFiles.filter(f => !isNil(f.s3Key)).map(f => f.s3Key!)
                    await s3Helper(log).deleteFiles(s3Keys)

                    const result = await fileRepo().delete({
                        id: In(staleFiles.map(file => file.id)),
                    })
                    affected = result.affected || 0
                    totalAffected += affected
                    log.info({
                        counts: affected,
                        type,
                    }, '[FileService#deleteStaleBulk] iteration completed')
                    await delay(FILE_CLEANUP_PAUSE_BETWEEN_ITERATIONS_MS)
                }
            }
        }
        log.info({
            totalAffected,
            types,
        }, '[FileService#deleteStaleBulk] completed')
    },
    async getFileByToken(token: string): Promise<Omit<File, 'data'>> {
        try {
            const decodedToken = await jwtUtils.decodeAndVerify<FileToken>({
                jwt: token,
                key: await jwtUtils.getJwtSecret(),
            })
            const fileType = decodedToken.fileType ?? FileType.FLOW_STEP_FILE
            if (!ALLOWED_SIGNED_FILE_TYPES.includes(fileType)) {
                throw new Error(`File type ${fileType} not allowed for signed download`)
            }
            return await this.getFileOrThrow({
                fileId: decodedToken.fileId,
                type: fileType,
            })
        }
        catch (e) {
            throw new ActivepiecesError({
                code: ErrorCode.INVALID_BEARER_TOKEN,
                params: {
                    message: 'invalid token or expired for the step file',
                },
            })
        }
    },
    extractBufferOrUndefined(value: unknown): Buffer | undefined {
        if (value === undefined || value === null) {
            return undefined
        }
        if (Buffer.isBuffer(value)) {
            return value
        }
        if (typeof value === 'string') {
            return Buffer.from(value, 'utf-8')
        }
        if (value instanceof Uint8Array) {
            return Buffer.from(value)
        }
        throw new ActivepiecesError({
            code: ErrorCode.VALIDATION,
            params: { message: 'File data must be a Buffer' },
        })
    },
    async uploadPublicAsset(params: UploadPublicAssetParams): Promise<string | undefined> {
        const { file, type, platformId, allowedMimeTypes = IMAGE_MIME_TYPES, maxFileSizeInBytes, metadata } = params

        if (isNil(file)) {
            return undefined
        }

        if (!isMultipartFile(file)) {
            throw new ActivepiecesError({
                code: ErrorCode.VALIDATION,
                params: {
                    message: 'File must be a multipart file',
                },
            })
        }

        if (!allowedMimeTypes.includes(file.mimetype ?? '')) {
            throw new ActivepiecesError({
                code: ErrorCode.VALIDATION,
                params: {
                    message: `Invalid file type. Allowed types: ${allowedMimeTypes.join(', ')}`,
                },
            })
        }

        if (!isNil(maxFileSizeInBytes) && file.data.length > maxFileSizeInBytes) {
            throw new ActivepiecesError({
                code: ErrorCode.VALIDATION,
                params: {
                    message: `File size exceeds ${Math.round(maxFileSizeInBytes / (1024 * 1024))}MB limit`,
                },
            })
        }

        const savedFile = await this.save({
            data: file.data,
            size: file.data.length,
            type,
            compression: FileCompression.NONE,
            platformId,
            fileName: file.filename,
            metadata: {
                ...metadata,
                mimetype: file.mimetype ?? '',
            },
        })

        return `${system.get(AppSystemProp.FRONTEND_URL)}/api/v1/platforms/assets/${savedFile.id}`
    },
})

export type GetDataResponse = {
    metadata?: Record<string, string>
    data: Buffer
    fileName?: string
}

function normalizeTypeFilter(type: FileType | FileType[] | undefined) {
    return Array.isArray(type) ? In(type) : type
}

function groupProjectIdsByRetentionDays(projects: Pick<Project, 'id' | 'executionDataRetentionDays'>[]): Map<number, ProjectId[]> {
    const retentionDaysToProjectIds = new Map<number, ProjectId[]>()
    for (const project of projects) {
        const effectiveRetentionDays = getEffectiveExecutionDataRetentionDays(project.executionDataRetentionDays)
        if (effectiveRetentionDays >= EXECUTION_DATA_RETENTION_DAYS) {
            continue
        }
        const projectIds = retentionDaysToProjectIds.get(effectiveRetentionDays) ?? []
        projectIds.push(project.id)
        retentionDaysToProjectIds.set(effectiveRetentionDays, projectIds)
    }
    return retentionDaysToProjectIds
}

export function getLocationForFile(type: FileType) {
    const FILE_LOCATION = system.getOrThrow<FileLocation>(AppSystemProp.FILE_STORAGE_LOCATION)
    if (type === FileType.FLOW_BUNDLE || type === FileType.PREWARM_SCOPE || isExecutionDataFileThatExpires(type)) {
        return FILE_LOCATION
    }
    return FileLocation.DB
}

export function getDownloadName(file: Pick<File, 'id' | 'fileName' | 'type'>): string {
    return file.fileName ?? `${file.id}.${file.type === FileType.FLOW_RUN_LOG_SLICE ? 'json' : 'bin'}`
}

export function getEffectiveExecutionDataRetentionDays(executionDataRetentionDays: number | null | undefined): number {
    if (isNil(executionDataRetentionDays)) {
        return EXECUTION_DATA_RETENTION_DAYS
    }
    const pausedFlowTimeoutDays = system.getNumberOrThrow(AppSystemProp.PAUSED_FLOW_TIMEOUT_DAYS)
    return Math.min(EXECUTION_DATA_RETENTION_DAYS, Math.max(executionDataRetentionDays, pausedFlowTimeoutDays))
}

function isExecutionDataFileThatExpires(type: FileType) {
    switch (type) {
        case FileType.FLOW_RUN_LOG:
        case FileType.FLOW_RUN_LOG_SLICE:
        case FileType.FLOW_STEP_FILE:
        case FileType.TRIGGER_PAYLOAD:
        case FileType.TRIGGER_EVENT_FILE:
        case FileType.WEBHOOK_PAYLOAD:
        case FileType.MCP_CALL_PAYLOAD:
            return true
        case FileType.PLATFORM_ASSET:
        case FileType.USER_PROFILE_PICTURE:
        case FileType.SAMPLE_DATA:
        case FileType.SAMPLE_DATA_INPUT:
        case FileType.PACKAGE_ARCHIVE:
        case FileType.PROJECT_RELEASE:
        case FileType.FLOW_VERSION_BACKUP:
        case FileType.KNOWLEDGE_BASE:
            return false
        default:
            throw new Error(`File type ${type} is not supported`)
    }
}

type SaveParams = {
    fileId?: FileId | undefined
    projectId?: ProjectId
    data: Buffer | Readable | null
    size?: number
    type: FileType
    platformId?: string
    fileName?: string
    compression: FileCompression
    metadata?: Record<string, string>
}

type GetOneParams = {
    fileId?: FileId
    projectId?: ProjectId
    platformId?: string
    type?: FileType | FileType[]
}

type FileToken = {
    fileId: string
    fileType?: FileType
}

type CleanupPass = {
    retentionDateBoundary: string
    projectIds: ProjectId[] | undefined
}

type UploadPublicAssetParams = {
    file: unknown
    type: FileType
    platformId: string
    allowedMimeTypes?: string[]
    maxFileSizeInBytes?: number
    metadata?: Record<string, string>
}