Open-source, secure environment with real-world tools for enterprise-grade agents.
Python agentaiai-agentai-agentscode-interpretercopilotdevelopmentdevtoolsgptgpt-4javascriptllmnextjsopenaipythonreactsoftwaretypescript
Use chunked transfer encoding for tar uploads (#1118)
Mish Ushakov8efc53c8149d
diff --git a/.changeset/chunked-upload.md b/.changeset/chunked-upload.mdnew file mode 100644index 000000000..15f54a230--- /dev/null+++ b/.changeset/chunked-upload.md@@ -0,0 +1,5 @@+---+"e2b": patch+---++Use chunked transfer encoding for tar uploads to reduce memory usagediff --git a/packages/js-sdk/src/template/buildApi.ts b/packages/js-sdk/src/template/buildApi.tsindex ffffe1e50..d56d953c6 100644--- a/packages/js-sdk/src/template/buildApi.ts+++ b/packages/js-sdk/src/template/buildApi.ts@@ -111,7 +111,7 @@ export async function uploadFile(const { fileName, url, fileContextPath, ignorePatterns, resolveSymlinks } =optionstry {- const { contentLength, uploadStream } = await tarFileStreamUpload(+ const uploadStream = await tarFileStreamUpload(fileName,fileContextPath,ignorePatterns,@@ -123,9 +123,6 @@ export async function uploadFile(method: 'PUT',// @ts-expect-errorbody: uploadStream,- headers: {- 'Content-Length': contentLength.toString(),- },duplex: 'half',})diff --git a/packages/js-sdk/src/template/utils.ts b/packages/js-sdk/src/template/utils.tsindex ad907ea83..4a03fc5ad 100644--- a/packages/js-sdk/src/template/utils.ts+++ b/packages/js-sdk/src/template/utils.ts@@ -373,7 +373,7 @@ export async function tarFileStream(// gzip.portable ensures deterministic gzip header without affecting file modesreturn create({- gzip: { portable: true },+ gzip: true,cwd: fileContextPath,follow: resolveSymlinks,noDirRecurse: true,@@ -383,12 +383,12 @@ export async function tarFileStream(}/**- * Create a tar stream and calculate its compressed size for upload.+ * Create a tar stream for upload using chunked transfer encoding.** @param fileName Glob pattern for files to include* @param fileContextPath Base directory for resolving file paths* @param resolveSymlinks Whether to follow symbolic links- * @returns Object containing the content length and upload stream+ * @returns A readable stream of the gzipped tar archive*/export async function tarFileStreamUpload(fileName: string,@@ -396,27 +396,12 @@ export async function tarFileStreamUpload(ignorePatterns: string[],resolveSymlinks: boolean) {- // First pass: calculate the compressed size- const sizeCalculationStream = await tarFileStream(+ return tarFileStream(fileName,fileContextPath,ignorePatterns,resolveSymlinks)- let contentLength = 0- for await (const chunk of sizeCalculationStream as unknown as AsyncIterable<Buffer>) {- contentLength += chunk.length- }-- return {- contentLength,- uploadStream: await tarFileStream(- fileName,- fileContextPath,- ignorePatterns,- resolveSymlinks- ),- }}/**