Skip to content

AI Agents & MCPs & AI Workflow Automation • (~400 MCP servers for AI agents) • AI Automation / AI Agent with MCPs • AI Workflows & AI Agents • MCPs for AI Agents

7.6 KB
Raw Blame

import { execFileSync } from 'node:child_process'
import { existsSync } from 'node:fs'
import { readdir, stat } from 'node:fs/promises'
import { resolve, join, relative, dirname } from 'node:path'
import { cwd } from 'node:process'
import * as semver from 'semver'
import { tryCatch } from '@activepieces/core-utils'
import { readPackageJson } from './files'
import { StatusCodes } from 'http-status-codes'
import { pieceTranslation, PieceMetadata } from '@activepieces/pieces-framework'

const LOAD_PIECE_METADATA_CHILD = resolve(
    __dirname,
    '..',
    'pieces',
    'load-piece-metadata-child.mjs',
)

type LoadedPieceChildPayload = {
    metadata: Omit<PieceMetadata, 'name' | 'version'>;
    minimumSupportedRelease: string | null;
    maximumSupportedRelease: string | null;
    authors: string[];
};

export const AP_CLOUD_API_BASE = 'https://cloud.activepieces.com/api/v1';
export const PIECES_FOLDER = 'packages/pieces'
export const COMMUNITY_PIECE_FOLDER = 'packages/pieces/community'
export const NON_PIECES_PACKAGES = ['@activepieces/pieces-framework', '@activepieces/pieces-common']

const validateSupportedRelease = (minRelease: string | undefined, maxRelease: string | undefined) => {
    if (minRelease !== undefined && !semver.valid(minRelease)) {
        throw Error(`[validateSupportedRelease] "minimumSupportedRelease" should be a valid semver version`)
    }

    if (maxRelease !== undefined && !semver.valid(maxRelease)) {
        throw Error(`[validateSupportedRelease] "maximumSupportedRelease" should be a valid semver version`)
    }

    if (minRelease !== undefined && maxRelease !== undefined && semver.gt(minRelease, maxRelease)) {
        throw Error(`[validateSupportedRelease] "minimumSupportedRelease" should be less than "maximumSupportedRelease"`)
    }
}

const validateMetadata = (pieceMetadata: PieceMetadata): void => {
    console.info(`[validateMetadata] pieceName=${pieceMetadata.name}`)
    validateSupportedRelease(
        pieceMetadata.minimumSupportedRelease,
        pieceMetadata.maximumSupportedRelease,
    )
}


export function getCommunityPieceFolder(pieceName: string): string {
    return join(COMMUNITY_PIECE_FOLDER, pieceName)
}


export async function findAllPiecesDirectoryInSource(): Promise<string[]> {
    const piecesPath = resolve(cwd(), 'packages', 'pieces')
    const paths = await traverseFolder(piecesPath)
    return paths.map(p => relative(cwd(), p))
}

export const pieceMetadataExists = async (
    pieceName: string,
    pieceVersion: string
): Promise<boolean> => {
    const cloudResponse = await fetch(
        `${AP_CLOUD_API_BASE}/pieces/${pieceName}?version=${pieceVersion}`
    );

    const pieceExist: Record<number, boolean> = {
        [StatusCodes.OK]: true,
        [StatusCodes.NOT_FOUND]: false
    };

    if (
        pieceExist[cloudResponse.status] === null ||
        pieceExist[cloudResponse.status] === undefined
    ) {
        throw new Error(await cloudResponse.text());
    }

    return pieceExist[cloudResponse.status];
};

export async function findNewPieces(): Promise<FindNewPiecesResult> {
    const changedDistPaths = getChangedPiecesDistPaths()
    const paths = changedDistPaths ?? await findAllDistPaths()

    console.info(`[findNewPieces] scanning ${paths.length} dist paths${changedDistPaths ? ' (scoped to changed)' : ' (all)'}`)

    const pieces: PieceMetadata[] = []
    const failures: PieceLoadFailure[] = []

    // Adding batches because of memory limit when we have a lot of pieces
    const batchSize = 75
    for (let i = 0; i < paths.length; i += batchSize) {
        const batch = paths.slice(i, i + batchSize)
        const batchResults = await Promise.all(batch.map(async (folderPath) => {
            const { data, error } = await tryCatch(async () => {
                const packageJson = await readPackageJson(folderPath);
                if (NON_PIECES_PACKAGES.includes(packageJson.name)) {
                    return null;
                }
                const exists = await pieceMetadataExists(packageJson.name, packageJson.version)
                if (exists) {
                    return null;
                }
                return loadPieceFromFolder(folderPath)
            })
            if (error !== null) {
                return { path: folderPath, error: error.message } satisfies PieceLoadFailure
            }
            return data
        }))

        for (const result of batchResults) {
            if (result === null) {
                continue
            }
            if ('error' in result) {
                failures.push(result)
            }
            else {
                pieces.push(result)
            }
        }
    }

    return { pieces, failures };
}

function getChangedPiecesDistPaths(): string[] | null {
    const changedPieces = process.env['CHANGED_PIECES']
    if (!changedPieces || changedPieces.trim() === '') {
        return null
    }
    return changedPieces.split('\n').filter(Boolean).map(p => {
        return resolve(cwd(), p, 'dist')
    }).filter(p => {
        const exists = existsSync(join(p, 'package.json'))
        if (!exists) {
            console.info(`[getChangedPiecesDistPaths] skipping, no build output at ${p}`)
        }
        return exists
    })
}

async function findAllDistPaths(): Promise<string[]> {
    const sourcePiecesPath = resolve(cwd(), 'packages', 'pieces')
    const sourceFolders = await traverseFolder(sourcePiecesPath)
    const distPaths: string[] = []
    for (const folder of sourceFolders) {
        const distPath = join(folder, 'dist')
        const distPackageJson = join(distPath, 'package.json')
        if (existsSync(distPackageJson)) {
            distPaths.push(distPath)
        }
    }
    return distPaths
}

async function traverseFolder(folderPath: string): Promise<string[]> {
    const paths: string[] = []
    const directoryExists = await stat(folderPath).catch(() => null)

    if (directoryExists && directoryExists.isDirectory()) {
        const files = await readdir(folderPath)

        for (const file of files) {
            const filePath = join(folderPath, file)
            const fileStats = await stat(filePath)
            if (fileStats.isDirectory() && file !== 'node_modules' && file !== 'dist') {
                paths.push(...await traverseFolder(filePath))
            }
            else if (file === 'package.json') {
                paths.push(folderPath)
            }
        }
    }
    return paths
}

async function loadPieceFromFolder(folderPath: string): Promise<PieceMetadata> {
    const packageJson = await readPackageJson(folderPath);
    const payload = loadPieceViaChildProcess(folderPath);
    const pieceSourcePath = dirname(folderPath)
    const i18n = await pieceTranslation.initializeI18n(pieceSourcePath)
    const metadata: PieceMetadata = {
        ...payload.metadata,
        name: packageJson.name,
        version: packageJson.version,
        i18n,
        authors: payload.authors,
        directoryPath: folderPath,
        minimumSupportedRelease: payload.minimumSupportedRelease ?? '0.0.0',
        maximumSupportedRelease: payload.maximumSupportedRelease ?? '99999.99999.9999',
    };

    validateMetadata(metadata);
    return metadata;
}

function loadPieceViaChildProcess(folderPath: string): LoadedPieceChildPayload {
    const stdout = execFileSync('node', [LOAD_PIECE_METADATA_CHILD, folderPath], {
        encoding: 'utf8',
        stdio: ['ignore', 'pipe', 'inherit'],
        maxBuffer: 64 * 1024 * 1024,
    })
    return JSON.parse(stdout) as LoadedPieceChildPayload
}

type PieceLoadFailure = {
    path: string
    error: string
}

type FindNewPiecesResult = {
    pieces: PieceMetadata[]
    failures: PieceLoadFailure[]
}