Skip to content

AI Agents & MCPs & AI Workflow Automation • (~400 MCP servers for AI agents) • AI Automation / AI Agent with MCPs • AI Workflows & AI Agents • MCPs for AI Agents

fix(engine): force reinstall piece when require fails (#16106)

Chaker Atallahe1d1479a90be
diff --git a/packages/core/execution/src/lib/engine/engine-contract.ts b/packages/core/execution/src/lib/engine/engine-contract.ts
index 0a137004cfe..055f468c8e9 100644
--- a/packages/core/execution/src/lib/engine/engine-contract.ts
+++ b/packages/core/execution/src/lib/engine/engine-contract.ts
@@ -8,3 +8,12 @@ export type WorkerNotifyContract = {
stdout(input: EngineStdout): void
stderr(input: EngineStderr): void
}
+
+export type WorkerRpcContract = {
+ forceReinstallPiece(input: ForceReinstallPieceRequest): Promise<void>
+}
+
+export type ForceReinstallPieceRequest = {
+ pieceName: string
+ pieceVersion: string
+}
diff --git a/packages/core/execution/src/lib/engine/execution-errors.ts b/packages/core/execution/src/lib/engine/execution-errors.ts
index b9dd400a457..50ca456f80f 100644
--- a/packages/core/execution/src/lib/engine/execution-errors.ts
+++ b/packages/core/execution/src/lib/engine/execution-errors.ts
@@ -142,6 +142,13 @@ export class EngineGenericError extends ExecutionError {
}
}
+export class RequireError extends ExecutionError {
+ constructor(piecePath: string, cause?: unknown) {
+ const causeMessage = cause instanceof Error ? cause.message : String(cause)
+ super('RequireError', formatMessage(`Failed to require piece module at ${piecePath}: ${causeMessage}`), ExecutionErrorType.ENGINE, cause)
+ }
+}
+
export class SSRFBlockedError extends ExecutionError {
constructor({ host, ip, cause }: { host: string, ip: string, cause?: unknown }) {
super(
diff --git a/packages/server/engine/src/lib/helper/piece-loader.ts b/packages/server/engine/src/lib/helper/piece-loader.ts
index b5e01946a59..d6da35eca3c 100644
--- a/packages/server/engine/src/lib/helper/piece-loader.ts
+++ b/packages/server/engine/src/lib/helper/piece-loader.ts
@@ -1,10 +1,11 @@
import fs from 'fs/promises'
import { createRequire } from 'node:module'
import path from 'path'
-import { ActivepiecesError, ErrorCode, isNil } from '@activepieces/core-utils'
+import { ActivepiecesError, ErrorCode, isNil, tryCatchSync } from '@activepieces/core-utils'
import { Action, Piece, PiecePropertyMap, Trigger } from '@activepieces/pieces-framework'
-import { EngineGenericError, extractPieceFromModule, getPackageAliasForPiece, getPieceNameFromAlias, trimVersionFromAlias } from '@activepieces/shared'
+import { EngineGenericError, extractPieceFromModule, getPackageAliasForPiece, getPieceNameFromAlias, RequireError, trimVersionFromAlias } from '@activepieces/shared'
import { utils } from '../utils'
+import { workerSocket } from '../worker-socket'
export const pieceLoader = {
loadPieceOrThrow: async (
@@ -17,7 +18,7 @@ export const pieceLoader = {
devPieces,
})
const piecePath = await pieceLoader.getPiecePath({ packageName, devPieces })
- const module = createRequire(__filename)(piecePath)
+ const module = await requireWithReinstallRetry({ piecePath, pieceName, pieceVersion })
const piece = extractPieceFromModule<Piece>({
module,
@@ -131,6 +132,31 @@ export const pieceLoader = {
},
}
+const MODULE_RESOLUTION_ERROR_CODES = ['MODULE_NOT_FOUND', 'ERR_MODULE_NOT_FOUND']
+
+function isModuleResolutionError(error: unknown): boolean {
+ return error instanceof Error && 'code' in error && typeof error.code === 'string' && MODULE_RESOLUTION_ERROR_CODES.includes(error.code)
+}
+
+async function requireWithReinstallRetry({ piecePath, pieceName, pieceVersion }: RequireWithReinstallRetryParams): Promise<Record<string, unknown>> {
+ const { data: module, error: requireError } = tryCatchSync<Record<string, unknown>>(() => createRequire(__filename)(piecePath))
+ if (!requireError) {
+ return module
+ }
+ if (!isModuleResolutionError(requireError)) {
+ throw requireError
+ }
+ const reinstalled = await workerSocket.requestPieceReinstall({ pieceName, pieceVersion })
+ if (!reinstalled) {
+ throw new RequireError(piecePath, requireError)
+ }
+ const { data: retriedModule, error: retryError } = tryCatchSync<Record<string, unknown>>(() => createRequire(__filename)(piecePath))
+ if (retryError) {
+ throw new RequireError(piecePath, retryError)
+ }
+ return retriedModule
+}
+
async function findInDistFolder(packageName: string): Promise<string | null> {
const sourcePiecesPath = path.resolve('packages/pieces')
if (!await utils.folderExists(sourcePiecesPath)) {
@@ -247,6 +273,12 @@ type GetPiecePathParams = {
devPieces: string[]
}
+type RequireWithReinstallRetryParams = {
+ piecePath: string
+ pieceName: string
+ pieceVersion: string
+}
+
type LoadPieceParams = {
pieceName: string
pieceVersion: string
diff --git a/packages/server/engine/src/lib/worker-socket.ts b/packages/server/engine/src/lib/worker-socket.ts
index 88dac2cf725..8d1a2110067 100644
--- a/packages/server/engine/src/lib/worker-socket.ts
+++ b/packages/server/engine/src/lib/worker-socket.ts
@@ -1,20 +1,26 @@
import { inspect } from 'node:util'
+import { isNil, tryCatch } from '@activepieces/core-utils'
import {
createNotifyClient,
+ createRpcClient,
createRpcServer,
EngineContract,
EngineResponse,
ERROR_MESSAGES_TO_REDACT,
+ ForceReinstallPieceRequest,
WorkerNotifyContract,
+ WorkerRpcContract,
} from '@activepieces/shared'
import { io, type ManagerOptions, type Socket, type SocketOptions } from 'socket.io-client'
import { flowRunProgressReporter } from './helper/flow-run-progress-reporter'
import { execute } from './operations'
const INITIAL_CONNECT_TIMEOUT_MS = 60_000
+const PIECE_REINSTALL_RPC_TIMEOUT_MS = 120_000
let socket: Socket | undefined
let notifyClient: WorkerNotifyContract | undefined
+let rpcClient: WorkerRpcContract | undefined
let initialConnectWatchdog: NodeJS.Timeout | undefined
function clearInitialConnectWatchdog(): void {
@@ -41,6 +47,7 @@ export const workerSocket = {
}, INITIAL_CONNECT_TIMEOUT_MS)
notifyClient = createNotifyClient<WorkerNotifyContract>(socket)
+ rpcClient = createRpcClient<WorkerRpcContract>(socket, PIECE_REINSTALL_RPC_TIMEOUT_MS)
socket.on('connect', () => {
clearInitialConnectWatchdog()
@@ -110,6 +117,19 @@ export const workerSocket = {
notifyClient?.stderr({ message: inspect(error) })
},
+ requestPieceReinstall: async (input: ForceReinstallPieceRequest): Promise<boolean> => {
+ if (isNil(socket) || !socket.connected || isNil(rpcClient)) {
+ return false
+ }
+ const client = rpcClient
+ const { error } = await tryCatch(() => client.forceReinstallPiece(input))
+ if (error) {
+ notifyClient?.stderr({ message: `Piece reinstall request failed: ${inspect(error)}\n` })
+ return false
+ }
+ return true
+ },
+
disconnect: (): void => {
clearInitialConnectWatchdog()
socket?.disconnect()
diff --git a/packages/server/sandbox/src/lib/cache/pieces/piece-installer.ts b/packages/server/sandbox/src/lib/cache/pieces/piece-installer.ts
index 45cda2c3adc..a07bd1d5c6f 100644
--- a/packages/server/sandbox/src/lib/cache/pieces/piece-installer.ts
+++ b/packages/server/sandbox/src/lib/cache/pieces/piece-installer.ts
@@ -15,10 +15,10 @@ const relativePiecePath = (piece: PiecePackage) => join('./', 'pieces', `${piece
const piecePath = (rootWorkspace: string, piece: PiecePackage) => join(rootWorkspace, 'pieces', `${piece.pieceName}-${piece.pieceVersion}`)
export const pieceInstaller = (log: ApLogger, basePath: string, getSettings: () => SandboxSettings) => ({
- async install({ pieces, includeFilters, publicApiUrl, engineToken, bestEffort }: InstallParams): Promise<void> {
+ async install({ pieces, includeFilters, publicApiUrl, engineToken, bestEffort, force }: InstallParams): Promise<void> {
const groupedPieces = groupPiecesByPackagePath(pieces, basePath, getSettings)
const installPromises = Object.entries(groupedPieces).map(async ([packagePath, piecesInGroup]) => {
- await installPieces(packagePath, piecesInGroup, includeFilters, log, { publicApiUrl, engineToken }, getSettings, bestEffort ?? false)
+ await installPieces(packagePath, piecesInGroup, includeFilters, log, { publicApiUrl, engineToken }, getSettings, bestEffort ?? false, force ?? false)
})
await Promise.all(installPromises)
},
@@ -26,6 +26,10 @@ export const pieceInstaller = (log: ApLogger, basePath: string, getSettings: ()
getCustomPiecesPath(platformId: string): string {
return getCustomPiecesPath(basePath, platformId, getSettings)
},
+
+ resolveWorkspace(piece: PiecePackage): string {
+ return resolveWorkspaceForPiece({ piece, basePath, getSettings })
+ },
})
function getCustomPiecesPath(basePath: string, platformId: string, getSettings: () => SandboxSettings): string {
@@ -42,7 +46,7 @@ function getCustomPiecesPath(basePath: string, platformId: string, getSettings:
}
}
-async function installPieces(rootWorkspace: string, pieces: PiecePackage[], includeFilters: boolean, log: ApLogger, bundleSource: BundleSource, getSettings: () => SandboxSettings, bestEffort: boolean): Promise<void> {
+async function installPieces(rootWorkspace: string, pieces: PiecePackage[], includeFilters: boolean, log: ApLogger, bundleSource: BundleSource, getSettings: () => SandboxSettings, bestEffort: boolean, force: boolean): Promise<void> {
const devPieces = getSettings().DEV_PIECES
const nonDevPieces = pieces.filter(piece => !devPieces.includes(getPieceNameFromAlias(piece.pieceName)))
const { validPieces, invalidPieces } = partitionValidPieceNames(nonDevPieces)
@@ -52,7 +56,7 @@ async function installPieces(rootWorkspace: string, pieces: PiecePackage[], incl
invalidPieces: invalidPieces.map(piece => `${piece.pieceName}@${piece.pieceVersion}`),
}, '[pieceInstaller] Skipping pieces with invalid package names to protect the shared lockfile')
}
- const { piecesToInstall } = await partitionPiecesToInstall(rootWorkspace, validPieces)
+ const { piecesToInstall } = await partitionPiecesToInstall(rootWorkspace, validPieces, force)
if (isEmpty(piecesToInstall)) {
log.debug({ rootWorkspace }, '[pieceInstaller] No new pieces to install (already installed)')
@@ -61,12 +65,13 @@ async function installPieces(rootWorkspace: string, pieces: PiecePackage[], incl
log.info({
rootWorkspace,
piecesToInstall: piecesToInstall.map(piece => `${piece.pieceName}-${piece.pieceVersion}`),
+ force,
}, '[pieceInstaller] Installing pieces in workspace')
await memoryLock.runExclusive({
key: `install-pieces-${rootWorkspace}`,
fn: async () => {
- const { piecesToInstall } = await partitionPiecesToInstall(rootWorkspace, validPieces)
+ const { piecesToInstall } = await partitionPiecesToInstall(rootWorkspace, validPieces, force)
if (isEmpty(piecesToInstall)) {
log.info({ rootWorkspace }, '[pieceInstaller] No new pieces to install in lock (already installed)')
return
@@ -101,6 +106,7 @@ async function installPieces(rootWorkspace: string, pieces: PiecePackage[], incl
const { error: batchError } = await tryCatch(async () => bunRunner(log).install({
path: rootWorkspace,
filtersPath: includeFilters ? downloaded.map(relativePiecePath) : [],
+ force,
}))
if (isNil(batchError)) {
@@ -124,7 +130,7 @@ async function installPieces(rootWorkspace: string, pieces: PiecePackage[], incl
error: batchError,
}, '[pieceInstaller] Batch install failed, retrying pieces individually')
- const failedPieces = await tryInstallPiecesIndividually(rootWorkspace, downloaded, log)
+ const failedPieces = await tryInstallPiecesIndividually(rootWorkspace, downloaded, log, force)
if (failedPieces.length > 0) {
const names = failedPieces.map(p => `${p.pieceName}@${p.pieceVersion}`).join(', ')
@@ -178,6 +184,7 @@ async function tryInstallPiecesIndividually(
rootWorkspace: string,
pieces: PiecePackage[],
log: ApLogger,
+ force: boolean,
): Promise<PiecePackage[]> {
const failures: PiecePackage[] = []
for (const piece of pieces) {
@@ -185,6 +192,7 @@ async function tryInstallPiecesIndividually(
bunRunner(log).install({
path: rootWorkspace,
filtersPath: [relativePiecePath(piece)],
+ force,
}),
)
if (error) {
@@ -203,21 +211,22 @@ async function tryInstallPiecesIndividually(
}
function groupPiecesByPackagePath(pieces: PiecePackage[], basePath: string, getSettings: () => SandboxSettings): Record<string, PiecePackage[]> {
- const paths = cacheUtils(basePath)
- return groupBy(pieces, (piece) => {
- switch (piece.packageType) {
- case PackageType.ARCHIVE:
+ return groupBy(pieces, (piece) => resolveWorkspaceForPiece({ piece, basePath, getSettings }))
+}
+
+function resolveWorkspaceForPiece({ piece, basePath, getSettings }: ResolveWorkspaceParams): string {
+ switch (piece.packageType) {
+ case PackageType.ARCHIVE:
+ return getCustomPiecesPath(basePath, piece.platformId, getSettings)
+ case PackageType.REGISTRY: {
+ if (piece.pieceType === PieceType.CUSTOM && !isNil(piece.platformId)) {
return getCustomPiecesPath(basePath, piece.platformId, getSettings)
- case PackageType.REGISTRY: {
- if (piece.pieceType === PieceType.CUSTOM && !isNil(piece.platformId)) {
- return getCustomPiecesPath(basePath, piece.platformId, getSettings)
- }
- return paths.getGlobalCacheCommonPath()
}
- default:
- throw new Error('Invalid package type')
+ return cacheUtils(basePath).getGlobalCacheCommonPath()
}
- })
+ default:
+ throw new Error('Invalid package type')
+ }
}
const WORKSPACE_BUNFIG = '[install]\nlinker = "isolated"\nminimumReleaseAge = 259200\n'
@@ -324,7 +333,10 @@ function pieceBundleEndpointUrl(publicApiUrl: string, piece: PiecePackage): stri
return `${base}?name=${encodeURIComponent(piece.pieceName)}&version=${encodeURIComponent(piece.pieceVersion)}`
}
-async function partitionPiecesToInstall(rootWorkspace: string, pieces: PiecePackage[]): Promise<PieceInstallationResult> {
+async function partitionPiecesToInstall(rootWorkspace: string, pieces: PiecePackage[], force: boolean): Promise<PieceInstallationResult> {
+ if (force) {
+ return { piecesToInstall: pieces }
+ }
const piecesWithCheck = await Promise.all(
pieces.map(async (piece) => {
const installed = await pieceCheckIfAlreadyInstalled(rootWorkspace, piece)
@@ -407,6 +419,7 @@ type InstallParams = {
publicApiUrl: string
engineToken: string
bestEffort?: boolean
+ force?: boolean
}
type BundleSource = {
@@ -433,3 +446,9 @@ type DownloadBundleParams = {
piece: PiecePackage
bundleSource: BundleSource
}
+
+type ResolveWorkspaceParams = {
+ piece: PiecePackage
+ basePath: string
+ getSettings: () => SandboxSettings
+}
diff --git a/packages/server/sandbox/src/lib/engine-rpc-handlers.ts b/packages/server/sandbox/src/lib/engine-rpc-handlers.ts
new file mode 100644
index 00000000000..96627c73c83
--- /dev/null
+++ b/packages/server/sandbox/src/lib/engine-rpc-handlers.ts
@@ -0,0 +1,70 @@
+import { stat, writeFile } from 'node:fs/promises'
+import { dirname, join } from 'path'
+import { isEmpty, tryCatch } from '@activepieces/core-utils'
+import { type ApLogger, fileSystemUtils } from '@activepieces/server-utils'
+import { ForceReinstallPieceRequest, WorkerRpcContract } from '@activepieces/shared'
+import { pieceInstaller } from './cache/pieces/piece-installer'
+import { ProvisionInput, SandboxSettings } from './types'
+import { diskLock } from './utils/distributed-lock'
+
+const FORCE_REINSTALL_COOLDOWN_MS = 2 * 60 * 1000
+const FORCE_REINSTALL_LOCK_TIMEOUT_MS = 60 * 1000
+
+export const engineRpcHandlers = ({ log, basePath, getSettings, provision }: EngineRpcHandlersParams): WorkerRpcContract => ({
+ async forceReinstallPiece({ pieceName, pieceVersion }: ForceReinstallPieceRequest): Promise<void> {
+ const piecesToReinstall = provision.pieces.filter((piece) => piece.pieceName === pieceName && piece.pieceVersion === pieceVersion)
+ if (isEmpty(piecesToReinstall)) {
+ return
+ }
+ const pieceKey = `${pieceName}@${pieceVersion}`
+ const installer = pieceInstaller(log, basePath, getSettings)
+ await diskLock(join(basePath, 'force-install-lock')).runExclusive({
+ key: installer.resolveWorkspace(piecesToReinstall[0]),
+ timeoutMs: FORCE_REINSTALL_LOCK_TIMEOUT_MS,
+ fn: async () => {
+ const stampPath = reinstallStampPath(basePath, pieceKey)
+ if (await isWithinCooldown(stampPath)) {
+ log.warn({
+ piece: { name: pieceName, version: pieceVersion },
+ }, '[engineRpcHandlers] Skipping forced piece reinstall, last attempt is within cooldown')
+ return
+ }
+ await markReinstallAttempt(stampPath)
+ log.warn({
+ piece: { name: pieceName, version: pieceVersion },
+ }, '[engineRpcHandlers] Engine failed to require a piece module, forcing piece reinstall')
+ await installer.install({
+ pieces: piecesToReinstall,
+ includeFilters: true,
+ publicApiUrl: provision.publicApiUrl,
+ engineToken: provision.engineToken,
+ force: true,
+ })
+ },
+ })
+ },
+})
+
+function reinstallStampPath(basePath: string, pieceKey: string): string {
+ return join(basePath, 'force-reinstall', pieceKey.replace(/[^a-zA-Z0-9._-]/g, '-'))
+}
+
+async function isWithinCooldown(stampPath: string): Promise<boolean> {
+ const { data: stampStat, error } = await tryCatch(() => stat(stampPath))
+ if (error) {
+ return false
+ }
+ return Date.now() - stampStat.mtimeMs < FORCE_REINSTALL_COOLDOWN_MS
+}
+
+async function markReinstallAttempt(stampPath: string): Promise<void> {
+ await fileSystemUtils.threadSafeMkdir(dirname(stampPath))
+ await writeFile(stampPath, '', 'utf8')
+}
+
+type EngineRpcHandlersParams = {
+ log: ApLogger
+ basePath: string
+ getSettings: () => SandboxSettings
+ provision: ProvisionInput
+}
diff --git a/packages/server/sandbox/src/lib/sandbox.ts b/packages/server/sandbox/src/lib/sandbox.ts
index 237c75b6005..7ed25c9c777 100644
--- a/packages/server/sandbox/src/lib/sandbox.ts
+++ b/packages/server/sandbox/src/lib/sandbox.ts
@@ -5,6 +5,7 @@ import { type ApLogger, apVersionUtil, wideEvent } from '@activepieces/server-ut
import { PrewarmScopeFileContent, WorkerToApiContract } from '@activepieces/shared'
import { cacheUtils } from './cache/cache-paths'
import { localExecutionCache } from './cache/local-execution-cache'
+import { engineRpcHandlers } from './engine-rpc-handlers'
import { createResolver } from './resolver'
import { createSandboxManager, SandboxManager } from './sandbox-manager'
import {
@@ -90,7 +91,10 @@ export function createSandboxRuntime({ concurrency = 1, basePath, getSettings }:
const runStartedAt = Date.now()
const runResult = await wideEvent.timed({
name: 'sandboxRun',
- fn: () => sandbox.execute(operationType, operation, { timeoutInSeconds: runTimeoutInSeconds }),
+ fn: () => sandbox.execute(operationType, operation, {
+ timeoutInSeconds: runTimeoutInSeconds,
+ engineRpc: engineRpcHandlers({ log, basePath, getSettings, provision }),
+ }),
})
runMs = Date.now() - runStartedAt
return runResult
diff --git a/packages/server/sandbox/src/lib/sandbox/sandbox.ts b/packages/server/sandbox/src/lib/sandbox/sandbox.ts
index dc15c0bfaf0..bbc7494d74d 100644
--- a/packages/server/sandbox/src/lib/sandbox/sandbox.ts
+++ b/packages/server/sandbox/src/lib/sandbox/sandbox.ts
@@ -3,7 +3,7 @@ import { randomBytes, timingSafeEqual } from 'crypto'
import { createServer, Server as HttpServer } from 'http'
import path from 'path'
import { ActivepiecesError, assertNotNullOrUndefined, ErrorCode, isNil, tryCatch } from '@activepieces/core-utils'
-import { createNotifyServer, createRpcClient, EngineContract, EngineOperation, EngineOperationType, EngineResponse, EngineStderr, EngineStdout, RpcTimeoutError, WorkerNotifyContract } from '@activepieces/shared'
+import { createNotifyServer, createRpcClient, createRpcServer, EngineContract, EngineOperation, EngineOperationType, EngineResponse, EngineStderr, EngineStdout, RpcTimeoutError, WorkerNotifyContract, WorkerRpcContract } from '@activepieces/shared'
import { Socket, Server as SocketIOServer } from 'socket.io'
import treeKill from 'tree-kill'
import { cacheUtils } from '../cache/cache-paths'
@@ -240,6 +240,8 @@ export function createSandbox(
},
}, log)
+ createRpcServer<WorkerRpcContract>(executeSocket, executeOptions.engineRpc ?? unsupportedEngineRpc, log)
+
timeout = setTimeout(async () => {
killedByTimeout = true
log.debug({ sandbox: { id: sandboxId } }, 'Killing sandbox by timeout')
@@ -296,6 +298,7 @@ export function createSandbox(
clearTimeout(timeout)
}
executeSocket?.removeAllListeners('rpc-notify')
+ executeSocket?.removeAllListeners('rpc')
executeProcess?.removeAllListeners('close')
executeProcess?.removeAllListeners('error')
}
@@ -449,6 +452,12 @@ function authenticateHandshake({ getExpectedToken, log, sandboxId }: {
}
}
+const unsupportedEngineRpc: WorkerRpcContract = {
+ async forceReinstallPiece(): Promise<void> {
+ throw new Error('Engine RPC is not supported for this execution')
+ },
+}
+
type ProcessExitParams = {
sandboxId: string
operationType: EngineOperationType
diff --git a/packages/server/sandbox/src/lib/sandbox/types.ts b/packages/server/sandbox/src/lib/sandbox/types.ts
index 5e8dca2ca6d..3ed75483fa5 100644
--- a/packages/server/sandbox/src/lib/sandbox/types.ts
+++ b/packages/server/sandbox/src/lib/sandbox/types.ts
@@ -1,5 +1,5 @@
import { ChildProcess } from 'child_process'
-import { EngineOperation, EngineOperationType, EngineResponse } from '@activepieces/shared'
+import { EngineOperation, EngineOperationType, EngineResponse, WorkerRpcContract } from '@activepieces/shared'
export type SandboxMount = {
hostPath: string
@@ -59,6 +59,7 @@ export type SandboxInitOptions = {
export type SandboxOptions = {
timeoutInSeconds: number
+ engineRpc?: WorkerRpcContract
}
export type SandboxLogger = {
diff --git a/packages/server/sandbox/src/lib/utils/bun-runner.ts b/packages/server/sandbox/src/lib/utils/bun-runner.ts
index 98f273a0a90..75a4301f36c 100644
--- a/packages/server/sandbox/src/lib/utils/bun-runner.ts
+++ b/packages/server/sandbox/src/lib/utils/bun-runner.ts
@@ -7,13 +7,14 @@ import { stepFolderResolvePlugin } from './esbuild-build-options'
import { CommandOutput, spawnWithKill } from './exec'
export const bunRunner = (log: ApLogger) => ({
- async install({ path, filtersPath }: InstallParams): Promise<CommandOutput> {
+ async install({ path, filtersPath, force }: InstallParams): Promise<CommandOutput> {
const filterArgs: string[] = filtersPath
.map(sanitizeFilterPath)
.flatMap((p) => ['--filter', `./${p}`])
const args = [
'install',
'--ignore-scripts',
+ ...(force ? ['--force'] : []),
...filterArgs,
]
await fileSystemUtils.threadSafeMkdir(path)
@@ -84,6 +85,7 @@ function formatBuildError({ error, entryFile }: FormatBuildErrorParams): string
type InstallParams = {
path: string
filtersPath: string[]
+ force?: boolean
}
type BuildParams = {
diff --git a/packages/server/sandbox/src/lib/utils/distributed-lock.ts b/packages/server/sandbox/src/lib/utils/distributed-lock.ts
new file mode 100644
index 00000000000..e348b4d0886
--- /dev/null
+++ b/packages/server/sandbox/src/lib/utils/distributed-lock.ts
@@ -0,0 +1,97 @@
+import { randomUUID } from 'node:crypto'
+import { mkdir, open, readFile, rename, rm, stat } from 'node:fs/promises'
+import { setTimeout as sleep } from 'node:timers/promises'
+import { join } from 'path'
+import { isNil, tryCatch } from '@activepieces/core-utils'
+import { ApLock } from '@activepieces/server-utils'
+
+const POLL_INTERVAL_MS = 500
+const STALE_LOCK_MS = 5 * 60 * 1000
+
+export const diskLock = (locksPath: string) => ({
+ acquire: async (key: string, timeoutMs?: number): Promise<ApLock> => {
+ const lockPath = join(locksPath, `${sanitizeKey(key)}.lock`)
+ const token = randomUUID()
+ const deadline = isNil(timeoutMs) ? null : Date.now() + timeoutMs
+ await mkdir(locksPath, { recursive: true })
+ while (!await tryAcquireOnce({ lockPath, token })) {
+ await takeoverIfStale(lockPath)
+ if (!isNil(deadline) && Date.now() >= deadline) {
+ throw new Error(`Timed out acquiring disk lock for key ${key}`)
+ }
+ await sleep(POLL_INTERVAL_MS)
+ }
+ return {
+ release: async () => releaseIfOwned({ lockPath, token }),
+ }
+ },
+ runExclusive: async <T>({ key, fn, timeoutMs }: RunExclusiveParams<T>): Promise<T> => {
+ const lock = await diskLock(locksPath).acquire(key, timeoutMs)
+ try {
+ return await fn()
+ }
+ finally {
+ await lock.release()
+ }
+ },
+})
+
+function sanitizeKey(key: string): string {
+ return key.replace(/[^a-zA-Z0-9._-]/g, '-')
+}
+
+async function tryAcquireOnce({ lockPath, token }: LockParams): Promise<boolean> {
+ const { data: handle, error } = await tryCatch(() => open(lockPath, 'wx'))
+ if (error) {
+ if (getErrorCode(error) === 'EEXIST') {
+ return false
+ }
+ throw error
+ }
+ try {
+ await handle.writeFile(token, 'utf8')
+ }
+ finally {
+ await handle.close()
+ }
+ return true
+}
+
+async function takeoverIfStale(lockPath: string): Promise<void> {
+ const { data: lockStat, error } = await tryCatch(() => stat(lockPath))
+ if (error || Date.now() - lockStat.mtimeMs < STALE_LOCK_MS) {
+ return
+ }
+ const stalePath = `${lockPath}.stale-${randomUUID()}`
+ const { error: renameError } = await tryCatch(() => rename(lockPath, stalePath))
+ if (renameError) {
+ return
+ }
+ await rm(stalePath, { force: true })
+}
+
+async function releaseIfOwned({ lockPath, token }: LockParams): Promise<void> {
+ const { data: owner, error } = await tryCatch(() => readFile(lockPath, 'utf8'))
+ if (error || owner !== token) {
+ return
+ }
+ await rm(lockPath, { force: true })
+}
+
+function getErrorCode(error: unknown): string | undefined {
+ if (error instanceof Error && 'code' in error && typeof error.code === 'string') {
+ return error.code
+ }
+ return undefined
+}
+
+type LockParams = {
+ lockPath: string
+ token: string
+}
+
+type RunExclusiveParams<T> = {
+ key: string
+ fn: () => Promise<T>
+ timeoutMs?: number
+}
diff --git a/packages/server/sandbox/test/lib/utils/distributed-lock.test.ts b/packages/server/sandbox/test/lib/utils/distributed-lock.test.ts
new file mode 100644
index 00000000000..525d490db71
--- /dev/null
+++ b/packages/server/sandbox/test/lib/utils/distributed-lock.test.ts
@@ -0,0 +1,140 @@
+import { randomUUID } from 'node:crypto'
+import { mkdir, readdir, readFile, rm, utimes, writeFile } from 'node:fs/promises'
+import { tmpdir } from 'node:os'
+import { join } from 'node:path'
+import { setTimeout as sleep } from 'node:timers/promises'
+import { afterEach, describe, expect, it } from 'vitest'
+import { diskLock } from '../../../src/lib/utils/distributed-lock'
+
+const folders: string[] = []
+
+function uniqueLocksPath(): string {
+ const folder = join(tmpdir(), `disk-lock-test-${randomUUID()}`)
+ folders.push(folder)
+ return folder
+}
+
+afterEach(async () => {
+ for (const f of folders) {
+ await rm(f, { recursive: true, force: true })
+ }
+ folders.length = 0
+})
+
+describe('diskLock', () => {
+ it('runs fn, returns its value and removes the lock file', async () => {
+ const locksPath = uniqueLocksPath()
+ const result = await diskLock(locksPath).runExclusive({
+ key: 'my-key',
+ fn: async () => 'value',
+ })
+ expect(result).toBe('value')
+ expect(await readdir(locksPath)).toEqual([])
+ })
+
+ it('never lets two holders of the same key overlap', async () => {
+ const locksPath = uniqueLocksPath()
+ let active = 0
+ let maxActive = 0
+ let runs = 0
+ const contend = () => diskLock(locksPath).runExclusive({
+ key: 'same-key',
+ fn: async () => {
+ active += 1
+ maxActive = Math.max(maxActive, active)
+ await sleep(150)
+ active -= 1
+ runs += 1
+ },
+ })
+ await Promise.all([contend(), contend(), contend()])
+ expect(runs).toBe(3)
+ expect(maxActive).toBe(1)
+ })
+
+ it('does not block holders of different keys', async () => {
+ const locksPath = uniqueLocksPath()
+ let slowStillRunning = false
+ const slow = diskLock(locksPath).runExclusive({
+ key: 'key-a',
+ fn: async () => {
+ slowStillRunning = true
+ await sleep(400)
+ slowStillRunning = false
+ },
+ })
+ await sleep(50)
+ await diskLock(locksPath).runExclusive({
+ key: 'key-b',
+ fn: async () => {
+ expect(slowStillRunning).toBe(true)
+ },
+ })
+ await slow
+ })
+
+ it('releases the lock when fn throws', async () => {
+ const locksPath = uniqueLocksPath()
+ await expect(diskLock(locksPath).runExclusive({
+ key: 'throwing-key',
+ fn: async () => {
+ throw new Error('boom')
+ },
+ })).rejects.toThrow('boom')
+ expect(await readdir(locksPath)).toEqual([])
+ const result = await diskLock(locksPath).runExclusive({
+ key: 'throwing-key',
+ fn: async () => 'recovered',
+ })
+ expect(result).toBe('recovered')
+ })
+
+ it('times out while another holder keeps the lock', async () => {
+ const locksPath = uniqueLocksPath()
+ const holder = diskLock(locksPath).runExclusive({
+ key: 'held-key',
+ fn: async () => sleep(2000),
+ })
+ await sleep(50)
+ await expect(diskLock(locksPath).runExclusive({
+ key: 'held-key',
+ timeoutMs: 100,
+ fn: async () => 'should not run',
+ })).rejects.toThrow('Timed out acquiring disk lock')
+ await holder
+ })
+
+ it('takes over a stale lock from a dead holder', async () => {
+ const locksPath = uniqueLocksPath()
+ await mkdir(locksPath, { recursive: true })
+ const staleLockPath = join(locksPath, 'dead-key.lock')
+ await writeFile(staleLockPath, 'dead-holder-token', 'utf8')
+ const sixMinutesAgo = (Date.now() - 6 * 60 * 1000) / 1000
+ await utimes(staleLockPath, sixMinutesAgo, sixMinutesAgo)
+ const result = await diskLock(locksPath).runExclusive({
+ key: 'dead-key',
+ fn: async () => 'took over',
+ })
+ expect(result).toBe('took over')
+ expect(await readdir(locksPath)).toEqual([])
+ })
+
+ it('does not delete a lock it no longer owns on release', async () => {
+ const locksPath = uniqueLocksPath()
+ const lock = await diskLock(locksPath).acquire('contested-key')
+ const [lockFile] = await readdir(locksPath)
+ await writeFile(join(locksPath, lockFile), 'successor-token', 'utf8')
+ await lock.release()
+ expect(await readdir(locksPath)).toEqual([lockFile])
+ expect(await readFile(join(locksPath, lockFile), 'utf8')).toBe('successor-token')
+ })
+
+ it('keeps the lock file inside locksPath for hostile keys', async () => {
+ const locksPath = uniqueLocksPath()
+ const lock = await diskLock(locksPath).acquire('../../etc/@scope/passwd')
+ const entries = await readdir(locksPath)
+ expect(entries).toHaveLength(1)
+ expect(entries[0]).toMatch(/^[A-Za-z0-9._-]+\.lock$/)
+ await lock.release()
+ })
+})