AI Agents & MCPs & AI Workflow Automation • (~400 MCP servers for AI agents) • AI Automation / AI Agent with MCPs • AI Workflows & AI Agents • MCPs for AI Agents
TypeScript ai-agentai-agent-toolsai-agentsai-agents-frameworkmcpmcp-servermcp-toolsmcpsn8n-alternativeno-code-automationworkflowworkflow-automationworkflows
fix(engine): force reinstall piece when require fails (#16106)
Chaker Atallahe1d1479a90be
diff --git a/packages/core/execution/src/lib/engine/engine-contract.ts b/packages/core/execution/src/lib/engine/engine-contract.tsindex 0a137004cfe..055f468c8e9 100644--- a/packages/core/execution/src/lib/engine/engine-contract.ts+++ b/packages/core/execution/src/lib/engine/engine-contract.ts@@ -8,3 +8,12 @@ export type WorkerNotifyContract = {stdout(input: EngineStdout): voidstderr(input: EngineStderr): void}++export type WorkerRpcContract = {+ forceReinstallPiece(input: ForceReinstallPieceRequest): Promise<void>+}++export type ForceReinstallPieceRequest = {+ pieceName: string+ pieceVersion: string+}diff --git a/packages/core/execution/src/lib/engine/execution-errors.ts b/packages/core/execution/src/lib/engine/execution-errors.tsindex b9dd400a457..50ca456f80f 100644--- a/packages/core/execution/src/lib/engine/execution-errors.ts+++ b/packages/core/execution/src/lib/engine/execution-errors.ts@@ -142,6 +142,13 @@ export class EngineGenericError extends ExecutionError {}}+export class RequireError extends ExecutionError {+ constructor(piecePath: string, cause?: unknown) {+ const causeMessage = cause instanceof Error ? cause.message : String(cause)+ super('RequireError', formatMessage(`Failed to require piece module at ${piecePath}: ${causeMessage}`), ExecutionErrorType.ENGINE, cause)+ }+}+export class SSRFBlockedError extends ExecutionError {constructor({ host, ip, cause }: { host: string, ip: string, cause?: unknown }) {super(diff --git a/packages/server/engine/src/lib/helper/piece-loader.ts b/packages/server/engine/src/lib/helper/piece-loader.tsindex b5e01946a59..d6da35eca3c 100644--- a/packages/server/engine/src/lib/helper/piece-loader.ts+++ b/packages/server/engine/src/lib/helper/piece-loader.ts@@ -1,10 +1,11 @@import fs from 'fs/promises'import { createRequire } from 'node:module'import path from 'path'-import { ActivepiecesError, ErrorCode, isNil } from '@activepieces/core-utils'+import { ActivepiecesError, ErrorCode, isNil, tryCatchSync } from '@activepieces/core-utils'import { Action, Piece, PiecePropertyMap, Trigger } from '@activepieces/pieces-framework'-import { EngineGenericError, extractPieceFromModule, getPackageAliasForPiece, getPieceNameFromAlias, trimVersionFromAlias } from '@activepieces/shared'+import { EngineGenericError, extractPieceFromModule, getPackageAliasForPiece, getPieceNameFromAlias, RequireError, trimVersionFromAlias } from '@activepieces/shared'import { utils } from '../utils'+import { workerSocket } from '../worker-socket'export const pieceLoader = {loadPieceOrThrow: async (@@ -17,7 +18,7 @@ export const pieceLoader = {devPieces,})const piecePath = await pieceLoader.getPiecePath({ packageName, devPieces })- const module = createRequire(__filename)(piecePath)+ const module = await requireWithReinstallRetry({ piecePath, pieceName, pieceVersion })const piece = extractPieceFromModule<Piece>({module,@@ -131,6 +132,31 @@ export const pieceLoader = {},}+const MODULE_RESOLUTION_ERROR_CODES = ['MODULE_NOT_FOUND', 'ERR_MODULE_NOT_FOUND']++function isModuleResolutionError(error: unknown): boolean {+ return error instanceof Error && 'code' in error && typeof error.code === 'string' && MODULE_RESOLUTION_ERROR_CODES.includes(error.code)+}++async function requireWithReinstallRetry({ piecePath, pieceName, pieceVersion }: RequireWithReinstallRetryParams): Promise<Record<string, unknown>> {+ const { data: module, error: requireError } = tryCatchSync<Record<string, unknown>>(() => createRequire(__filename)(piecePath))+ if (!requireError) {+ return module+ }+ if (!isModuleResolutionError(requireError)) {+ throw requireError+ }+ const reinstalled = await workerSocket.requestPieceReinstall({ pieceName, pieceVersion })+ if (!reinstalled) {+ throw new RequireError(piecePath, requireError)+ }+ const { data: retriedModule, error: retryError } = tryCatchSync<Record<string, unknown>>(() => createRequire(__filename)(piecePath))+ if (retryError) {+ throw new RequireError(piecePath, retryError)+ }+ return retriedModule+}+async function findInDistFolder(packageName: string): Promise<string | null> {const sourcePiecesPath = path.resolve('packages/pieces')if (!await utils.folderExists(sourcePiecesPath)) {@@ -247,6 +273,12 @@ type GetPiecePathParams = {devPieces: string[]}+type RequireWithReinstallRetryParams = {+ piecePath: string+ pieceName: string+ pieceVersion: string+}+type LoadPieceParams = {pieceName: stringpieceVersion: stringdiff --git a/packages/server/engine/src/lib/worker-socket.ts b/packages/server/engine/src/lib/worker-socket.tsindex 88dac2cf725..8d1a2110067 100644--- a/packages/server/engine/src/lib/worker-socket.ts+++ b/packages/server/engine/src/lib/worker-socket.ts@@ -1,20 +1,26 @@import { inspect } from 'node:util'+import { isNil, tryCatch } from '@activepieces/core-utils'import {createNotifyClient,+ createRpcClient,createRpcServer,EngineContract,EngineResponse,ERROR_MESSAGES_TO_REDACT,+ ForceReinstallPieceRequest,WorkerNotifyContract,+ WorkerRpcContract,} from '@activepieces/shared'import { io, type ManagerOptions, type Socket, type SocketOptions } from 'socket.io-client'import { flowRunProgressReporter } from './helper/flow-run-progress-reporter'import { execute } from './operations'const INITIAL_CONNECT_TIMEOUT_MS = 60_000+const PIECE_REINSTALL_RPC_TIMEOUT_MS = 120_000let socket: Socket | undefinedlet notifyClient: WorkerNotifyContract | undefined+let rpcClient: WorkerRpcContract | undefinedlet initialConnectWatchdog: NodeJS.Timeout | undefinedfunction clearInitialConnectWatchdog(): void {@@ -41,6 +47,7 @@ export const workerSocket = {}, INITIAL_CONNECT_TIMEOUT_MS)notifyClient = createNotifyClient<WorkerNotifyContract>(socket)+ rpcClient = createRpcClient<WorkerRpcContract>(socket, PIECE_REINSTALL_RPC_TIMEOUT_MS)socket.on('connect', () => {clearInitialConnectWatchdog()@@ -110,6 +117,19 @@ export const workerSocket = {notifyClient?.stderr({ message: inspect(error) })},+ requestPieceReinstall: async (input: ForceReinstallPieceRequest): Promise<boolean> => {+ if (isNil(socket) || !socket.connected || isNil(rpcClient)) {+ return false+ }+ const client = rpcClient+ const { error } = await tryCatch(() => client.forceReinstallPiece(input))+ if (error) {+ notifyClient?.stderr({ message: `Piece reinstall request failed: ${inspect(error)}\n` })+ return false+ }+ return true+ },+disconnect: (): void => {clearInitialConnectWatchdog()socket?.disconnect()diff --git a/packages/server/sandbox/src/lib/cache/pieces/piece-installer.ts b/packages/server/sandbox/src/lib/cache/pieces/piece-installer.tsindex 45cda2c3adc..a07bd1d5c6f 100644--- a/packages/server/sandbox/src/lib/cache/pieces/piece-installer.ts+++ b/packages/server/sandbox/src/lib/cache/pieces/piece-installer.ts@@ -15,10 +15,10 @@ const relativePiecePath = (piece: PiecePackage) => join('./', 'pieces', `${piececonst piecePath = (rootWorkspace: string, piece: PiecePackage) => join(rootWorkspace, 'pieces', `${piece.pieceName}-${piece.pieceVersion}`)export const pieceInstaller = (log: ApLogger, basePath: string, getSettings: () => SandboxSettings) => ({- async install({ pieces, includeFilters, publicApiUrl, engineToken, bestEffort }: InstallParams): Promise<void> {+ async install({ pieces, includeFilters, publicApiUrl, engineToken, bestEffort, force }: InstallParams): Promise<void> {const groupedPieces = groupPiecesByPackagePath(pieces, basePath, getSettings)const installPromises = Object.entries(groupedPieces).map(async ([packagePath, piecesInGroup]) => {- await installPieces(packagePath, piecesInGroup, includeFilters, log, { publicApiUrl, engineToken }, getSettings, bestEffort ?? false)+ await installPieces(packagePath, piecesInGroup, includeFilters, log, { publicApiUrl, engineToken }, getSettings, bestEffort ?? false, force ?? false)})await Promise.all(installPromises)},@@ -26,6 +26,10 @@ export const pieceInstaller = (log: ApLogger, basePath: string, getSettings: ()getCustomPiecesPath(platformId: string): string {return getCustomPiecesPath(basePath, platformId, getSettings)},++ resolveWorkspace(piece: PiecePackage): string {+ return resolveWorkspaceForPiece({ piece, basePath, getSettings })+ },})function getCustomPiecesPath(basePath: string, platformId: string, getSettings: () => SandboxSettings): string {@@ -42,7 +46,7 @@ function getCustomPiecesPath(basePath: string, platformId: string, getSettings:}}-async function installPieces(rootWorkspace: string, pieces: PiecePackage[], includeFilters: boolean, log: ApLogger, bundleSource: BundleSource, getSettings: () => SandboxSettings, bestEffort: boolean): Promise<void> {+async function installPieces(rootWorkspace: string, pieces: PiecePackage[], includeFilters: boolean, log: ApLogger, bundleSource: BundleSource, getSettings: () => SandboxSettings, bestEffort: boolean, force: boolean): Promise<void> {const devPieces = getSettings().DEV_PIECESconst nonDevPieces = pieces.filter(piece => !devPieces.includes(getPieceNameFromAlias(piece.pieceName)))const { validPieces, invalidPieces } = partitionValidPieceNames(nonDevPieces)@@ -52,7 +56,7 @@ async function installPieces(rootWorkspace: string, pieces: PiecePackage[], inclinvalidPieces: invalidPieces.map(piece => `${piece.pieceName}@${piece.pieceVersion}`),}, '[pieceInstaller] Skipping pieces with invalid package names to protect the shared lockfile')}- const { piecesToInstall } = await partitionPiecesToInstall(rootWorkspace, validPieces)+ const { piecesToInstall } = await partitionPiecesToInstall(rootWorkspace, validPieces, force)if (isEmpty(piecesToInstall)) {log.debug({ rootWorkspace }, '[pieceInstaller] No new pieces to install (already installed)')@@ -61,12 +65,13 @@ async function installPieces(rootWorkspace: string, pieces: PiecePackage[], incllog.info({rootWorkspace,piecesToInstall: piecesToInstall.map(piece => `${piece.pieceName}-${piece.pieceVersion}`),+ force,}, '[pieceInstaller] Installing pieces in workspace')await memoryLock.runExclusive({key: `install-pieces-${rootWorkspace}`,fn: async () => {- const { piecesToInstall } = await partitionPiecesToInstall(rootWorkspace, validPieces)+ const { piecesToInstall } = await partitionPiecesToInstall(rootWorkspace, validPieces, force)if (isEmpty(piecesToInstall)) {log.info({ rootWorkspace }, '[pieceInstaller] No new pieces to install in lock (already installed)')return@@ -101,6 +106,7 @@ async function installPieces(rootWorkspace: string, pieces: PiecePackage[], inclconst { error: batchError } = await tryCatch(async () => bunRunner(log).install({path: rootWorkspace,filtersPath: includeFilters ? downloaded.map(relativePiecePath) : [],+ force,}))if (isNil(batchError)) {@@ -124,7 +130,7 @@ async function installPieces(rootWorkspace: string, pieces: PiecePackage[], inclerror: batchError,}, '[pieceInstaller] Batch install failed, retrying pieces individually')- const failedPieces = await tryInstallPiecesIndividually(rootWorkspace, downloaded, log)+ const failedPieces = await tryInstallPiecesIndividually(rootWorkspace, downloaded, log, force)if (failedPieces.length > 0) {const names = failedPieces.map(p => `${p.pieceName}@${p.pieceVersion}`).join(', ')@@ -178,6 +184,7 @@ async function tryInstallPiecesIndividually(rootWorkspace: string,pieces: PiecePackage[],log: ApLogger,+ force: boolean,): Promise<PiecePackage[]> {const failures: PiecePackage[] = []for (const piece of pieces) {@@ -185,6 +192,7 @@ async function tryInstallPiecesIndividually(bunRunner(log).install({path: rootWorkspace,filtersPath: [relativePiecePath(piece)],+ force,}),)if (error) {@@ -203,21 +211,22 @@ async function tryInstallPiecesIndividually(}function groupPiecesByPackagePath(pieces: PiecePackage[], basePath: string, getSettings: () => SandboxSettings): Record<string, PiecePackage[]> {- const paths = cacheUtils(basePath)- return groupBy(pieces, (piece) => {- switch (piece.packageType) {- case PackageType.ARCHIVE:+ return groupBy(pieces, (piece) => resolveWorkspaceForPiece({ piece, basePath, getSettings }))+}++function resolveWorkspaceForPiece({ piece, basePath, getSettings }: ResolveWorkspaceParams): string {+ switch (piece.packageType) {+ case PackageType.ARCHIVE:+ return getCustomPiecesPath(basePath, piece.platformId, getSettings)+ case PackageType.REGISTRY: {+ if (piece.pieceType === PieceType.CUSTOM && !isNil(piece.platformId)) {return getCustomPiecesPath(basePath, piece.platformId, getSettings)- case PackageType.REGISTRY: {- if (piece.pieceType === PieceType.CUSTOM && !isNil(piece.platformId)) {- return getCustomPiecesPath(basePath, piece.platformId, getSettings)- }- return paths.getGlobalCacheCommonPath()}- default:- throw new Error('Invalid package type')+ return cacheUtils(basePath).getGlobalCacheCommonPath()}- })+ default:+ throw new Error('Invalid package type')+ }}const WORKSPACE_BUNFIG = '[install]\nlinker = "isolated"\nminimumReleaseAge = 259200\n'@@ -324,7 +333,10 @@ function pieceBundleEndpointUrl(publicApiUrl: string, piece: PiecePackage): strireturn `${base}?name=${encodeURIComponent(piece.pieceName)}&version=${encodeURIComponent(piece.pieceVersion)}`}-async function partitionPiecesToInstall(rootWorkspace: string, pieces: PiecePackage[]): Promise<PieceInstallationResult> {+async function partitionPiecesToInstall(rootWorkspace: string, pieces: PiecePackage[], force: boolean): Promise<PieceInstallationResult> {+ if (force) {+ return { piecesToInstall: pieces }+ }const piecesWithCheck = await Promise.all(pieces.map(async (piece) => {const installed = await pieceCheckIfAlreadyInstalled(rootWorkspace, piece)@@ -407,6 +419,7 @@ type InstallParams = {publicApiUrl: stringengineToken: stringbestEffort?: boolean+ force?: boolean}type BundleSource = {@@ -433,3 +446,9 @@ type DownloadBundleParams = {piece: PiecePackagebundleSource: BundleSource}++type ResolveWorkspaceParams = {+ piece: PiecePackage+ basePath: string+ getSettings: () => SandboxSettings+}diff --git a/packages/server/sandbox/src/lib/engine-rpc-handlers.ts b/packages/server/sandbox/src/lib/engine-rpc-handlers.tsnew file mode 100644index 00000000000..96627c73c83--- /dev/null+++ b/packages/server/sandbox/src/lib/engine-rpc-handlers.ts@@ -0,0 +1,70 @@+import { stat, writeFile } from 'node:fs/promises'+import { dirname, join } from 'path'+import { isEmpty, tryCatch } from '@activepieces/core-utils'+import { type ApLogger, fileSystemUtils } from '@activepieces/server-utils'+import { ForceReinstallPieceRequest, WorkerRpcContract } from '@activepieces/shared'+import { pieceInstaller } from './cache/pieces/piece-installer'+import { ProvisionInput, SandboxSettings } from './types'+import { diskLock } from './utils/distributed-lock'++const FORCE_REINSTALL_COOLDOWN_MS = 2 * 60 * 1000+const FORCE_REINSTALL_LOCK_TIMEOUT_MS = 60 * 1000++export const engineRpcHandlers = ({ log, basePath, getSettings, provision }: EngineRpcHandlersParams): WorkerRpcContract => ({+ async forceReinstallPiece({ pieceName, pieceVersion }: ForceReinstallPieceRequest): Promise<void> {+ const piecesToReinstall = provision.pieces.filter((piece) => piece.pieceName === pieceName && piece.pieceVersion === pieceVersion)+ if (isEmpty(piecesToReinstall)) {+ return+ }+ const pieceKey = `${pieceName}@${pieceVersion}`+ const installer = pieceInstaller(log, basePath, getSettings)+ await diskLock(join(basePath, 'force-install-lock')).runExclusive({+ key: installer.resolveWorkspace(piecesToReinstall[0]),+ timeoutMs: FORCE_REINSTALL_LOCK_TIMEOUT_MS,+ fn: async () => {+ const stampPath = reinstallStampPath(basePath, pieceKey)+ if (await isWithinCooldown(stampPath)) {+ log.warn({+ piece: { name: pieceName, version: pieceVersion },+ }, '[engineRpcHandlers] Skipping forced piece reinstall, last attempt is within cooldown')+ return+ }+ await markReinstallAttempt(stampPath)+ log.warn({+ piece: { name: pieceName, version: pieceVersion },+ }, '[engineRpcHandlers] Engine failed to require a piece module, forcing piece reinstall')+ await installer.install({+ pieces: piecesToReinstall,+ includeFilters: true,+ publicApiUrl: provision.publicApiUrl,+ engineToken: provision.engineToken,+ force: true,+ })+ },+ })+ },+})++function reinstallStampPath(basePath: string, pieceKey: string): string {+ return join(basePath, 'force-reinstall', pieceKey.replace(/[^a-zA-Z0-9._-]/g, '-'))+}++async function isWithinCooldown(stampPath: string): Promise<boolean> {+ const { data: stampStat, error } = await tryCatch(() => stat(stampPath))+ if (error) {+ return false+ }+ return Date.now() - stampStat.mtimeMs < FORCE_REINSTALL_COOLDOWN_MS+}++async function markReinstallAttempt(stampPath: string): Promise<void> {+ await fileSystemUtils.threadSafeMkdir(dirname(stampPath))+ await writeFile(stampPath, '', 'utf8')+}++type EngineRpcHandlersParams = {+ log: ApLogger+ basePath: string+ getSettings: () => SandboxSettings+ provision: ProvisionInput+}diff --git a/packages/server/sandbox/src/lib/sandbox.ts b/packages/server/sandbox/src/lib/sandbox.tsindex 237c75b6005..7ed25c9c777 100644--- a/packages/server/sandbox/src/lib/sandbox.ts+++ b/packages/server/sandbox/src/lib/sandbox.ts@@ -5,6 +5,7 @@ import { type ApLogger, apVersionUtil, wideEvent } from '@activepieces/server-utimport { PrewarmScopeFileContent, WorkerToApiContract } from '@activepieces/shared'import { cacheUtils } from './cache/cache-paths'import { localExecutionCache } from './cache/local-execution-cache'+import { engineRpcHandlers } from './engine-rpc-handlers'import { createResolver } from './resolver'import { createSandboxManager, SandboxManager } from './sandbox-manager'import {@@ -90,7 +91,10 @@ export function createSandboxRuntime({ concurrency = 1, basePath, getSettings }:const runStartedAt = Date.now()const runResult = await wideEvent.timed({name: 'sandboxRun',- fn: () => sandbox.execute(operationType, operation, { timeoutInSeconds: runTimeoutInSeconds }),+ fn: () => sandbox.execute(operationType, operation, {+ timeoutInSeconds: runTimeoutInSeconds,+ engineRpc: engineRpcHandlers({ log, basePath, getSettings, provision }),+ }),})runMs = Date.now() - runStartedAtreturn runResultdiff --git a/packages/server/sandbox/src/lib/sandbox/sandbox.ts b/packages/server/sandbox/src/lib/sandbox/sandbox.tsindex dc15c0bfaf0..bbc7494d74d 100644--- a/packages/server/sandbox/src/lib/sandbox/sandbox.ts+++ b/packages/server/sandbox/src/lib/sandbox/sandbox.ts@@ -3,7 +3,7 @@ import { randomBytes, timingSafeEqual } from 'crypto'import { createServer, Server as HttpServer } from 'http'import path from 'path'import { ActivepiecesError, assertNotNullOrUndefined, ErrorCode, isNil, tryCatch } from '@activepieces/core-utils'-import { createNotifyServer, createRpcClient, EngineContract, EngineOperation, EngineOperationType, EngineResponse, EngineStderr, EngineStdout, RpcTimeoutError, WorkerNotifyContract } from '@activepieces/shared'+import { createNotifyServer, createRpcClient, createRpcServer, EngineContract, EngineOperation, EngineOperationType, EngineResponse, EngineStderr, EngineStdout, RpcTimeoutError, WorkerNotifyContract, WorkerRpcContract } from '@activepieces/shared'import { Socket, Server as SocketIOServer } from 'socket.io'import treeKill from 'tree-kill'import { cacheUtils } from '../cache/cache-paths'@@ -240,6 +240,8 @@ export function createSandbox(},}, log)+ createRpcServer<WorkerRpcContract>(executeSocket, executeOptions.engineRpc ?? unsupportedEngineRpc, log)+timeout = setTimeout(async () => {killedByTimeout = truelog.debug({ sandbox: { id: sandboxId } }, 'Killing sandbox by timeout')@@ -296,6 +298,7 @@ export function createSandbox(clearTimeout(timeout)}executeSocket?.removeAllListeners('rpc-notify')+ executeSocket?.removeAllListeners('rpc')executeProcess?.removeAllListeners('close')executeProcess?.removeAllListeners('error')}@@ -449,6 +452,12 @@ function authenticateHandshake({ getExpectedToken, log, sandboxId }: {}}+const unsupportedEngineRpc: WorkerRpcContract = {+ async forceReinstallPiece(): Promise<void> {+ throw new Error('Engine RPC is not supported for this execution')+ },+}+type ProcessExitParams = {sandboxId: stringoperationType: EngineOperationTypediff --git a/packages/server/sandbox/src/lib/sandbox/types.ts b/packages/server/sandbox/src/lib/sandbox/types.tsindex 5e8dca2ca6d..3ed75483fa5 100644--- a/packages/server/sandbox/src/lib/sandbox/types.ts+++ b/packages/server/sandbox/src/lib/sandbox/types.ts@@ -1,5 +1,5 @@import { ChildProcess } from 'child_process'-import { EngineOperation, EngineOperationType, EngineResponse } from '@activepieces/shared'+import { EngineOperation, EngineOperationType, EngineResponse, WorkerRpcContract } from '@activepieces/shared'export type SandboxMount = {hostPath: string@@ -59,6 +59,7 @@ export type SandboxInitOptions = {export type SandboxOptions = {timeoutInSeconds: number+ engineRpc?: WorkerRpcContract}export type SandboxLogger = {diff --git a/packages/server/sandbox/src/lib/utils/bun-runner.ts b/packages/server/sandbox/src/lib/utils/bun-runner.tsindex 98f273a0a90..75a4301f36c 100644--- a/packages/server/sandbox/src/lib/utils/bun-runner.ts+++ b/packages/server/sandbox/src/lib/utils/bun-runner.ts@@ -7,13 +7,14 @@ import { stepFolderResolvePlugin } from './esbuild-build-options'import { CommandOutput, spawnWithKill } from './exec'export const bunRunner = (log: ApLogger) => ({- async install({ path, filtersPath }: InstallParams): Promise<CommandOutput> {+ async install({ path, filtersPath, force }: InstallParams): Promise<CommandOutput> {const filterArgs: string[] = filtersPath.map(sanitizeFilterPath).flatMap((p) => ['--filter', `./${p}`])const args = ['install','--ignore-scripts',+ ...(force ? ['--force'] : []),...filterArgs,]await fileSystemUtils.threadSafeMkdir(path)@@ -84,6 +85,7 @@ function formatBuildError({ error, entryFile }: FormatBuildErrorParams): stringtype InstallParams = {path: stringfiltersPath: string[]+ force?: boolean}type BuildParams = {diff --git a/packages/server/sandbox/src/lib/utils/distributed-lock.ts b/packages/server/sandbox/src/lib/utils/distributed-lock.tsnew file mode 100644index 00000000000..e348b4d0886--- /dev/null+++ b/packages/server/sandbox/src/lib/utils/distributed-lock.ts@@ -0,0 +1,97 @@+import { randomUUID } from 'node:crypto'+import { mkdir, open, readFile, rename, rm, stat } from 'node:fs/promises'+import { setTimeout as sleep } from 'node:timers/promises'+import { join } from 'path'+import { isNil, tryCatch } from '@activepieces/core-utils'+import { ApLock } from '@activepieces/server-utils'++const POLL_INTERVAL_MS = 500+const STALE_LOCK_MS = 5 * 60 * 1000++export const diskLock = (locksPath: string) => ({+ acquire: async (key: string, timeoutMs?: number): Promise<ApLock> => {+ const lockPath = join(locksPath, `${sanitizeKey(key)}.lock`)+ const token = randomUUID()+ const deadline = isNil(timeoutMs) ? null : Date.now() + timeoutMs+ await mkdir(locksPath, { recursive: true })+ while (!await tryAcquireOnce({ lockPath, token })) {+ await takeoverIfStale(lockPath)+ if (!isNil(deadline) && Date.now() >= deadline) {+ throw new Error(`Timed out acquiring disk lock for key ${key}`)+ }+ await sleep(POLL_INTERVAL_MS)+ }+ return {+ release: async () => releaseIfOwned({ lockPath, token }),+ }+ },+ runExclusive: async <T>({ key, fn, timeoutMs }: RunExclusiveParams<T>): Promise<T> => {+ const lock = await diskLock(locksPath).acquire(key, timeoutMs)+ try {+ return await fn()+ }+ finally {+ await lock.release()+ }+ },+})++function sanitizeKey(key: string): string {+ return key.replace(/[^a-zA-Z0-9._-]/g, '-')+}++async function tryAcquireOnce({ lockPath, token }: LockParams): Promise<boolean> {+ const { data: handle, error } = await tryCatch(() => open(lockPath, 'wx'))+ if (error) {+ if (getErrorCode(error) === 'EEXIST') {+ return false+ }+ throw error+ }+ try {+ await handle.writeFile(token, 'utf8')+ }+ finally {+ await handle.close()+ }+ return true+}++async function takeoverIfStale(lockPath: string): Promise<void> {+ const { data: lockStat, error } = await tryCatch(() => stat(lockPath))+ if (error || Date.now() - lockStat.mtimeMs < STALE_LOCK_MS) {+ return+ }+ const stalePath = `${lockPath}.stale-${randomUUID()}`+ const { error: renameError } = await tryCatch(() => rename(lockPath, stalePath))+ if (renameError) {+ return+ }+ await rm(stalePath, { force: true })+}++async function releaseIfOwned({ lockPath, token }: LockParams): Promise<void> {+ const { data: owner, error } = await tryCatch(() => readFile(lockPath, 'utf8'))+ if (error || owner !== token) {+ return+ }+ await rm(lockPath, { force: true })+}++function getErrorCode(error: unknown): string | undefined {+ if (error instanceof Error && 'code' in error && typeof error.code === 'string') {+ return error.code+ }+ return undefined+}++type LockParams = {+ lockPath: string+ token: string+}++type RunExclusiveParams<T> = {+ key: string+ fn: () => Promise<T>+ timeoutMs?: number+}diff --git a/packages/server/sandbox/test/lib/utils/distributed-lock.test.ts b/packages/server/sandbox/test/lib/utils/distributed-lock.test.tsnew file mode 100644index 00000000000..525d490db71--- /dev/null+++ b/packages/server/sandbox/test/lib/utils/distributed-lock.test.ts@@ -0,0 +1,140 @@+import { randomUUID } from 'node:crypto'+import { mkdir, readdir, readFile, rm, utimes, writeFile } from 'node:fs/promises'+import { tmpdir } from 'node:os'+import { join } from 'node:path'+import { setTimeout as sleep } from 'node:timers/promises'+import { afterEach, describe, expect, it } from 'vitest'+import { diskLock } from '../../../src/lib/utils/distributed-lock'++const folders: string[] = []++function uniqueLocksPath(): string {+ const folder = join(tmpdir(), `disk-lock-test-${randomUUID()}`)+ folders.push(folder)+ return folder+}++afterEach(async () => {+ for (const f of folders) {+ await rm(f, { recursive: true, force: true })+ }+ folders.length = 0+})++describe('diskLock', () => {+ it('runs fn, returns its value and removes the lock file', async () => {+ const locksPath = uniqueLocksPath()+ const result = await diskLock(locksPath).runExclusive({+ key: 'my-key',+ fn: async () => 'value',+ })+ expect(result).toBe('value')+ expect(await readdir(locksPath)).toEqual([])+ })++ it('never lets two holders of the same key overlap', async () => {+ const locksPath = uniqueLocksPath()+ let active = 0+ let maxActive = 0+ let runs = 0+ const contend = () => diskLock(locksPath).runExclusive({+ key: 'same-key',+ fn: async () => {+ active += 1+ maxActive = Math.max(maxActive, active)+ await sleep(150)+ active -= 1+ runs += 1+ },+ })+ await Promise.all([contend(), contend(), contend()])+ expect(runs).toBe(3)+ expect(maxActive).toBe(1)+ })++ it('does not block holders of different keys', async () => {+ const locksPath = uniqueLocksPath()+ let slowStillRunning = false+ const slow = diskLock(locksPath).runExclusive({+ key: 'key-a',+ fn: async () => {+ slowStillRunning = true+ await sleep(400)+ slowStillRunning = false+ },+ })+ await sleep(50)+ await diskLock(locksPath).runExclusive({+ key: 'key-b',+ fn: async () => {+ expect(slowStillRunning).toBe(true)+ },+ })+ await slow+ })++ it('releases the lock when fn throws', async () => {+ const locksPath = uniqueLocksPath()+ await expect(diskLock(locksPath).runExclusive({+ key: 'throwing-key',+ fn: async () => {+ throw new Error('boom')+ },+ })).rejects.toThrow('boom')+ expect(await readdir(locksPath)).toEqual([])+ const result = await diskLock(locksPath).runExclusive({+ key: 'throwing-key',+ fn: async () => 'recovered',+ })+ expect(result).toBe('recovered')+ })++ it('times out while another holder keeps the lock', async () => {+ const locksPath = uniqueLocksPath()+ const holder = diskLock(locksPath).runExclusive({+ key: 'held-key',+ fn: async () => sleep(2000),+ })+ await sleep(50)+ await expect(diskLock(locksPath).runExclusive({+ key: 'held-key',+ timeoutMs: 100,+ fn: async () => 'should not run',+ })).rejects.toThrow('Timed out acquiring disk lock')+ await holder+ })++ it('takes over a stale lock from a dead holder', async () => {+ const locksPath = uniqueLocksPath()+ await mkdir(locksPath, { recursive: true })+ const staleLockPath = join(locksPath, 'dead-key.lock')+ await writeFile(staleLockPath, 'dead-holder-token', 'utf8')+ const sixMinutesAgo = (Date.now() - 6 * 60 * 1000) / 1000+ await utimes(staleLockPath, sixMinutesAgo, sixMinutesAgo)+ const result = await diskLock(locksPath).runExclusive({+ key: 'dead-key',+ fn: async () => 'took over',+ })+ expect(result).toBe('took over')+ expect(await readdir(locksPath)).toEqual([])+ })++ it('does not delete a lock it no longer owns on release', async () => {+ const locksPath = uniqueLocksPath()+ const lock = await diskLock(locksPath).acquire('contested-key')+ const [lockFile] = await readdir(locksPath)+ await writeFile(join(locksPath, lockFile), 'successor-token', 'utf8')+ await lock.release()+ expect(await readdir(locksPath)).toEqual([lockFile])+ expect(await readFile(join(locksPath, lockFile), 'utf8')).toBe('successor-token')+ })++ it('keeps the lock file inside locksPath for hostile keys', async () => {+ const locksPath = uniqueLocksPath()+ const lock = await diskLock(locksPath).acquire('../../etc/@scope/passwd')+ const entries = await readdir(locksPath)+ expect(entries).toHaveLength(1)+ expect(entries[0]).toMatch(/^[A-Za-z0-9._-]+\.lock$/)+ await lock.release()+ })+})