Skip to content

armgit / E2B

Public Imported from GitHub
0

Open-source, secure environment with real-world tools for enterprise-grade agents.

Use chunked transfer encoding for tar uploads (#1118)

Mish Ushakov8efc53c8149d
diff --git a/.changeset/chunked-upload.md b/.changeset/chunked-upload.md
new file mode 100644
index 000000000..15f54a230
--- /dev/null
+++ b/.changeset/chunked-upload.md
@@ -0,0 +1,5 @@
+---
+"e2b": patch
+---
+
+Use chunked transfer encoding for tar uploads to reduce memory usage
diff --git a/packages/js-sdk/src/template/buildApi.ts b/packages/js-sdk/src/template/buildApi.ts
index ffffe1e50..d56d953c6 100644
--- a/packages/js-sdk/src/template/buildApi.ts
+++ b/packages/js-sdk/src/template/buildApi.ts
@@ -111,7 +111,7 @@ export async function uploadFile(
const { fileName, url, fileContextPath, ignorePatterns, resolveSymlinks } =
options
try {
- const { contentLength, uploadStream } = await tarFileStreamUpload(
+ const uploadStream = await tarFileStreamUpload(
fileName,
fileContextPath,
ignorePatterns,
@@ -123,9 +123,6 @@ export async function uploadFile(
method: 'PUT',
// @ts-expect-error
body: uploadStream,
- headers: {
- 'Content-Length': contentLength.toString(),
- },
duplex: 'half',
})
diff --git a/packages/js-sdk/src/template/utils.ts b/packages/js-sdk/src/template/utils.ts
index ad907ea83..4a03fc5ad 100644
--- a/packages/js-sdk/src/template/utils.ts
+++ b/packages/js-sdk/src/template/utils.ts
@@ -373,7 +373,7 @@ export async function tarFileStream(
// gzip.portable ensures deterministic gzip header without affecting file modes
return create(
{
- gzip: { portable: true },
+ gzip: true,
cwd: fileContextPath,
follow: resolveSymlinks,
noDirRecurse: true,
@@ -383,12 +383,12 @@ export async function tarFileStream(
}
/**
- * Create a tar stream and calculate its compressed size for upload.
+ * Create a tar stream for upload using chunked transfer encoding.
*
* @param fileName Glob pattern for files to include
* @param fileContextPath Base directory for resolving file paths
* @param resolveSymlinks Whether to follow symbolic links
- * @returns Object containing the content length and upload stream
+ * @returns A readable stream of the gzipped tar archive
*/
export async function tarFileStreamUpload(
fileName: string,
@@ -396,27 +396,12 @@ export async function tarFileStreamUpload(
ignorePatterns: string[],
resolveSymlinks: boolean
) {
- // First pass: calculate the compressed size
- const sizeCalculationStream = await tarFileStream(
+ return tarFileStream(
fileName,
fileContextPath,
ignorePatterns,
resolveSymlinks
)
- let contentLength = 0
- for await (const chunk of sizeCalculationStream as unknown as AsyncIterable<Buffer>) {
- contentLength += chunk.length
- }
-
- return {
- contentLength,
- uploadStream: await tarFileStream(
- fileName,
- fileContextPath,
- ignorePatterns,
- resolveSymlinks
- ),
- }
}
/**