AI Agents & MCPs & AI Workflow Automation • (~400 MCP servers for AI agents) • AI Automation / AI Agent with MCPs • AI Workflows & AI Agents • MCPs for AI Agents
TypeScript ai-agentai-agent-toolsai-agentsai-agents-frameworkmcpmcp-servermcp-toolsmcpsn8n-alternativeno-code-automationworkflowworkflow-automationworkflows
fix(sandbox): rebuild code steps in place with a disk-locked, forced dependency install (#16435)
Chaker Atallah80b632aa90f8
diff --git a/packages/server/sandbox/src/lib/cache/cache-state.ts b/packages/server/sandbox/src/lib/cache/cache-state.tsindex 41c295330c3..a3c461a1657 100644--- a/packages/server/sandbox/src/lib/cache/cache-state.ts+++ b/packages/server/sandbox/src/lib/cache/cache-state.ts@@ -1,4 +1,4 @@-import { readFile } from 'node:fs/promises'+import { readFile, rm } from 'node:fs/promises'import { join } from 'path'import { isNil } from '@activepieces/core-utils'import { fileSystemUtils, memoryLock } from '@activepieces/server-utils'@@ -49,6 +49,9 @@ export const cacheState = (folderPath: string) => {},})},+ clear: async (): Promise<void> => {+ await rm(cachePath(folderPath), { force: true })+ },saveCache: async (key: string, value: string): Promise<CacheMap> => {await fileSystemUtils.threadSafeMkdir(folderPath)const cacheFilePath = cachePath(folderPath)diff --git a/packages/server/sandbox/src/lib/cache/flow/code/code-builder.ts b/packages/server/sandbox/src/lib/cache/flow/code/code-builder.tsindex b873d1d4ca5..aabb7d57bc0 100644--- a/packages/server/sandbox/src/lib/cache/flow/code/code-builder.ts+++ b/packages/server/sandbox/src/lib/cache/flow/code/code-builder.ts@@ -6,6 +6,7 @@ import { type ApLogger, cryptoUtils, fileSystemUtils, wideEvent } from '@activepimport { ExecutionMode } from '@activepieces/shared'import { CodeArtifact, SandboxSettings } from '../../../types'import { bunRunner } from '../../../utils/bun-runner'+import { diskLock } from '../../../utils/distributed-lock'import { cacheState } from '../../cache-state'import { codeCache } from './code-cache'import { denoStepTranspiler } from './deno-step-transpiler'@@ -37,6 +38,10 @@ const TS_CONFIG_CONTENT = `const DENO_TRANSPILER_VERSION = 'sucrase-v1'+const INSTALL_LOCKS_DIR = '.install-locks'++const INSTALL_LOCK_TIMEOUT_MS = 15 * 60 * 1000+const INVALID_ARTIFACT_ERROR_PLACEHOLDER = '__AP_ERROR_MESSAGE__'const INVALID_DENO_ARTIFACT_TEMPLATE = `@@ -83,12 +88,9 @@ export const codeBuilder = (log: ApLogger, getSettings: () => SandboxSettings) =|| (useDeno && hasDependencies && !existsSync(path.join(codePath, 'node_modules')))},installFn: async () => {- const codeNeedCleanUp = await fileSystemUtils.fileExists(codePath)- if (codeNeedCleanUp) {- await rm(codePath, { recursive: true })- }-await fileSystemUtils.threadSafeMkdir(codePath)+ await cache.clear()+ await removeStaleBuildOutputs({ codes, flowVersionId, stepName: name })const installError = await wideEvent.timed({name: 'codeDeps',@@ -96,6 +98,7 @@ export const codeBuilder = (log: ApLogger, getSettings: () => SandboxSettings) =const { error } = await tryCatch(() => installDependencies({path: codePath,packageJson,+ locksPath: path.join(codesFolderPath, INSTALL_LOCKS_DIR),}, log))if (error) {log.info({ codePath, error }, 'Dependency installation error')@@ -201,12 +204,27 @@ function getPackageJson(packageJson: string, getSettings: () => SandboxSettings)})}-async function installDependencies({ path, packageJson }: InstallDependenciesParams, log: ApLogger): Promise<void> {- await fs.writeFile(`${path}/package.json`, packageJson, 'utf8')- const deps = Object.entries(JSON.parse(packageJson).dependencies ?? {})- if (deps.length > 0) {- await bunRunner(log).install({ path, filtersPath: [] })- }+async function removeStaleBuildOutputs({ codes, flowVersionId, stepName }: RemoveStaleBuildOutputsParams): Promise<void> {+ const ref = { flowVersionId, stepName }+ await Promise.all([+ rm(codes.stepEntryPath(ref), { force: true }),+ rm(codes.compiledStepPath(ref), { force: true }),+ rm(codes.transpiledStepPath(ref), { force: true }),+ ])+}++async function installDependencies({ path, packageJson, locksPath }: InstallDependenciesParams, log: ApLogger): Promise<void> {+ await diskLock(locksPath).runExclusive({+ key: path,+ timeoutMs: INSTALL_LOCK_TIMEOUT_MS,+ fn: async () => {+ await fs.writeFile(`${path}/package.json`, packageJson, 'utf8')+ const deps = Object.entries(JSON.parse(packageJson).dependencies ?? {})+ if (deps.length > 0) {+ await bunRunner(log).install({ path, filtersPath: [], force: true })+ }+ },+ })}async function compileCode({ path, code }: CompileCodeParams, log: ApLogger): Promise<void> {@@ -237,9 +255,16 @@ type ProcessCodeStepParams = {codesFolderPath: string}+type RemoveStaleBuildOutputsParams = {+ codes: ReturnType<typeof codeCache>+ flowVersionId: string+ stepName: string+}+type InstallDependenciesParams = {path: stringpackageJson: string+ locksPath: string}type CompileCodeParams = {diff --git a/packages/server/sandbox/test/lib/cache/flow/code/code-builder.test.ts b/packages/server/sandbox/test/lib/cache/flow/code/code-builder.test.tsindex 85261c95470..3e43f55c6e0 100644--- a/packages/server/sandbox/test/lib/cache/flow/code/code-builder.test.ts+++ b/packages/server/sandbox/test/lib/cache/flow/code/code-builder.test.ts@@ -1,6 +1,6 @@import { randomUUID } from 'node:crypto'import { existsSync } from 'node:fs'-import { mkdir, readFile, rm, writeFile } from 'node:fs/promises'+import { mkdir, readdir, readFile, rm, writeFile } from 'node:fs/promises'import { tmpdir } from 'node:os'import { join } from 'node:path'import { ExecutionMode, FlowVersionState, NetworkMode } from '@activepieces/shared'@@ -375,3 +375,61 @@ describe('codeBuilder.processCodeStep', () => {await expect(readFile(entryPath, 'utf8')).resolves.toBe(TRANSPILED_SOURCE)})})++describe('codeBuilder in-place rebuilds', () => {+ it('keeps node_modules across a rebuild and only replaces the build outputs', async () => {+ const codesFolderPath = uniqueFolder()+ const artifact = buildArtifact('{"dependencies":{"pkg":"1.0.0"}}')+ const ref = { flowVersionId: artifact.flowVersionId, stepName: artifact.name }+ const stepDir = codeCache(codesFolderPath).stepDir(ref)+ mockInstallSuccess()++ await expect(+ codeBuilder(noopLog, getSettings).processCodeStep({ artifact, codesFolderPath }),+ ).resolves.toBe('success')++ const markerPath = join(stepDir, 'node_modules', 'marker.txt')+ await writeFile(markerPath, 'installed-earlier', 'utf8')++ const changedSource = 'export const code = async () => 43'+ const changedArtifact = { ...artifact, sourceCode: { ...artifact.sourceCode, code: changedSource } }+ await expect(+ codeBuilder(noopLog, getSettings).processCodeStep({ artifact: changedArtifact, codesFolderPath }),+ ).resolves.toBe('success')++ await expect(readFile(markerPath, 'utf8')).resolves.toBe('installed-earlier')+ await expect(readFile(codeCache(codesFolderPath).stepEntryPath(ref), 'utf8')).resolves.toBe(changedSource)+ })++ it('installs dependencies with force so an interrupted earlier install is fully re-linked', async () => {+ const codesFolderPath = uniqueFolder()+ const artifact = buildArtifact('{"dependencies":{"pkg":"1.0.0"}}')+ mockInstallSuccess()++ await expect(+ codeBuilder(noopLog, getSettings).processCodeStep({ artifact, codesFolderPath }),+ ).resolves.toBe('success')++ expect(installMock).toHaveBeenCalledWith(expect.objectContaining({ force: true }))+ })++ it('runs the dependency install while holding the shared disk lock', async () => {+ const codesFolderPath = uniqueFolder()+ const artifact = buildArtifact('{"dependencies":{"pkg":"1.0.0"}}')+ let lockFilesDuringInstall: string[] | null = null+ installMock.mockImplementation(async ({ path }: { path: string }) => {+ lockFilesDuringInstall = await readdir(join(codesFolderPath, '.install-locks'))+ await mkdir(join(path, 'node_modules'), { recursive: true })+ return { stdout: '', stderr: '' }+ })++ await expect(+ codeBuilder(noopLog, getSettings).processCodeStep({ artifact, codesFolderPath }),+ ).resolves.toBe('success')++ expect(lockFilesDuringInstall).not.toBeNull()+ expect(lockFilesDuringInstall!.filter((name) => name.endsWith('.lock'))).toHaveLength(1)+ const lockFilesAfter = await readdir(join(codesFolderPath, '.install-locks'))+ expect(lockFilesAfter.filter((name) => name.endsWith('.lock'))).toHaveLength(0)+ })+})