Skip to content

armgit / E2B

Public Imported from GitHub
0

Open-source, secure environment with real-world tools for enterprise-grade agents.

Stream template build-context upload from disk instead of buffering in memory (#1435)

Mish Ushakovbb1696871b49
diff --git a/.changeset/stream-template-build-upload.md b/.changeset/stream-template-build-upload.md
new file mode 100644
index 000000000..c71e96ba5
--- /dev/null
+++ b/.changeset/stream-template-build-upload.md
@@ -0,0 +1,6 @@
+---
+"e2b": patch
+"@e2b/python-sdk": patch
+---
+
+Template builds: the build-context tar archive is now spooled to a temporary file and streamed from disk during upload instead of being held in memory (JS and Python, sync and async), while keeping the explicit `Content-Length` required by S3 presigned URLs. Temp-file cleanup is now best-effort, so a cleanup failure after the upload no longer masks a successful upload as an error (nor overwrites the real upload error on failure). The Python SDK now uploads the archive with a 1-hour default timeout (overridable via `request_timeout`) instead of the 60s general API timeout, matching the JS SDK and preventing large uploads from timing out.
diff --git a/packages/js-sdk/src/template/buildApi.ts b/packages/js-sdk/src/template/buildApi.ts
index 819981ad3..581d26eba 100644
--- a/packages/js-sdk/src/template/buildApi.ts
+++ b/packages/js-sdk/src/template/buildApi.ts
@@ -1,10 +1,11 @@
+import type { Readable } from 'node:stream'
import { ApiClient, handleApiError, components } from '../api'
import { buildRequestSignal } from '../connectionConfig'
import { dynamicImport } from '../utils'
import { BuildError, FileUploadError, TemplateError } from '../errors'
import { FILE_UPLOAD_TIMEOUT_MS } from './consts'
import { LogEntry } from './logger'
-import { getBuildStepIndex, tarFileStreamUpload } from './utils'
+import { getBuildStepIndex, tarFileStream } from './utils'
import {
BuildStatusReason,
TemplateBuildStatus,
@@ -121,38 +122,40 @@ export async function uploadFile(
) {
const { fileName, url, fileContextPath, ignorePatterns, resolveSymlinks } =
options
+ // Spool the archive to a temporary file and stream it from disk instead of
+ // buffering in memory. S3 presigned PUT URLs reject Transfer-Encoding:
+ // chunked with 501 NotImplemented (see e2b-dev/e2b#1243), so the upload
+ // sends an explicit Content-Length, which fetch honors for stream bodies.
+ // The spooled file deletes itself once the stream is consumed, so there is
+ // no cleanup to manage here. The Python SDK takes the same approach
+ // (build_api.py:upload_file).
+ let uploadStream: Readable | undefined
try {
- const uploadStream = await tarFileStreamUpload(
+ // Dynamically import so the browser bundle doesn't pull in node:stream.
+ const { Readable } =
+ await dynamicImport<typeof import('node:stream')>('node:stream')
+
+ const tar = await tarFileStream(
fileName,
fileContextPath,
ignorePatterns,
resolveSymlinks
)
-
- // Buffer the archive before uploading so fetch sets Content-Length.
- // S3 presigned PUT URLs reject Transfer-Encoding: chunked with 501
- // NotImplemented, which is what Node's fetch falls back to when the
- // body is a Readable without a known length. See e2b-dev/e2b#1243.
- // The Python SDK takes the same approach (build_api.py:upload_file).
- // Dynamically import so the browser bundle doesn't pull in node:stream.
- // tar's Pack extends Minipass and is iterable as AsyncIterable<Buffer> at
- // runtime, but the cli's tsconfig (preserveSymlinks) doesn't surface that
- // through the type chain — cast via unknown.
- const { buffer } = await dynamicImport<
- typeof import('node:stream/consumers')
- >('node:stream/consumers')
- const uploadBody = await buffer(
- uploadStream as unknown as AsyncIterable<Buffer>
- )
+ uploadStream = tar.stream
const res = await fetch(url, {
method: 'PUT',
- body: uploadBody,
+ body: Readable.toWeb(uploadStream) as ReadableStream<Uint8Array>,
+ headers: {
+ 'Content-Length': tar.size.toString(),
+ },
+ // Streaming request bodies require half-duplex mode.
+ duplex: 'half',
signal: buildRequestSignal(
abortOpts?.requestTimeoutMs ?? FILE_UPLOAD_TIMEOUT_MS,
abortOpts?.signal
),
- })
+ } as RequestInit)
if (!res.ok) {
throw new FileUploadError(
@@ -161,6 +164,10 @@ export async function uploadFile(
)
}
} catch (error) {
+ // Ensure the spooled archive is removed even if fetch never consumed the
+ // stream (e.g. it threw before reading the body). Destroying the stream
+ // fires its `close` handler, which deletes the temp file.
+ uploadStream?.destroy()
if (error instanceof FileUploadError) {
throw error
}
diff --git a/packages/js-sdk/src/template/utils.ts b/packages/js-sdk/src/template/utils.ts
index e4e83a37c..d5476ebc1 100644
--- a/packages/js-sdk/src/template/utils.ts
+++ b/packages/js-sdk/src/template/utils.ts
@@ -1,9 +1,11 @@
import crypto from 'node:crypto'
import fs from 'node:fs'
+import os from 'node:os'
import path from 'node:path'
import { dynamicImport, dynamicRequire } from '../utils'
import { TemplateError } from '../errors'
import { BASE_STEP_NAME, FINALIZE_STEP_NAME } from './consts'
+import type { Readable } from 'node:stream'
import type { Path } from 'glob'
import type { BuildOptions } from './types'
@@ -352,21 +354,33 @@ export function padOctal(mode: number): string {
}
/**
- * Create a compressed tar stream of files matching a pattern.
+ * Create a gzipped tar archive of files matching a pattern and return a
+ * readable stream over it.
+ *
+ * The archive is spooled to a temporary file on disk so it can be uploaded as
+ * a stream with a known `Content-Length` instead of being buffered in memory.
+ * The temporary file is removed automatically once the returned stream is
+ * closed — whether it was fully read, errored, or destroyed. This mirrors the
+ * Python SDK's `tar_file_stream`, where closing the temp-file handle deletes
+ * it (Node has no portable delete-on-close, so cleanup is tied to the stream's
+ * `close` event instead).
*
* @param fileName Glob pattern for files to include
* @param fileContextPath Base directory for resolving file paths
* @param ignorePatterns Ignore patterns to exclude from the archive
* @param resolveSymlinks Whether to follow symbolic links
- * @returns A readable stream of the gzipped tar archive
+ * @returns The readable stream and the archive size in bytes
*/
export async function tarFileStream(
fileName: string,
fileContextPath: string,
ignorePatterns: string[],
resolveSymlinks: boolean
-) {
+): Promise<{ stream: Readable; size: number }> {
const { create } = await dynamicImport<typeof import('tar')>('tar')
+ // Dynamically import so the browser bundle doesn't pull in node:fs.
+ const { createReadStream } =
+ await dynamicImport<typeof import('node:fs')>('node:fs')
const allFiles = await getAllFilesInPath(
fileName,
@@ -377,38 +391,38 @@ export async function tarFileStream(
const filePaths = allFiles.map((file) => file.relativePosix())
- // gzip.portable ensures deterministic gzip header without affecting file modes
- return create(
- {
- gzip: true,
- cwd: fileContextPath,
- follow: resolveSymlinks,
- noDirRecurse: true,
- },
- filePaths
+ const tmpDir = await fs.promises.mkdtemp(
+ path.join(os.tmpdir(), 'e2b-template-')
)
-}
+ const tarPath = path.join(tmpDir, 'context.tar.gz')
+ // Best-effort removal so it can never mask the archive-creation error or the
+ // upload result. A leaked temp dir is non-fatal — the OS reclaims it.
+ const removeTmpDir = () =>
+ fs.promises.rm(tmpDir, { recursive: true, force: true }).catch(() => {})
-/**
- * Create a tar stream for upload using chunked transfer encoding.
- *
- * @param fileName Glob pattern for files to include
- * @param fileContextPath Base directory for resolving file paths
- * @param resolveSymlinks Whether to follow symbolic links
- * @returns A readable stream of the gzipped tar archive
- */
-export async function tarFileStreamUpload(
- fileName: string,
- fileContextPath: string,
- ignorePatterns: string[],
- resolveSymlinks: boolean
-) {
- return tarFileStream(
- fileName,
- fileContextPath,
- ignorePatterns,
- resolveSymlinks
- )
+ try {
+ await create(
+ {
+ gzip: true,
+ cwd: fileContextPath,
+ follow: resolveSymlinks,
+ noDirRecurse: true,
+ file: tarPath,
+ },
+ filePaths
+ )
+
+ const { size } = await fs.promises.stat(tarPath)
+
+ const stream = createReadStream(tarPath)
+ // Remove the spooled archive once the stream is done — `close` fires after
+ // the fd is released on every path (end, error, or destroy).
+ stream.once('close', removeTmpDir)
+ return { stream, size }
+ } catch (err) {
+ await removeTmpDir()
+ throw err
+ }
}
/**
diff --git a/packages/js-sdk/tests/template/uploadFile.test.ts b/packages/js-sdk/tests/template/uploadFile.test.ts
index 3d0cd25b0..119301e22 100644
--- a/packages/js-sdk/tests/template/uploadFile.test.ts
+++ b/packages/js-sdk/tests/template/uploadFile.test.ts
@@ -9,7 +9,8 @@ import { uploadFile } from '../../src/template/buildApi'
// Regression test for e2b-dev/e2b#1243 — uploadFile used to pass a Node
// Readable directly to fetch, which made undici fall back to
// Transfer-Encoding: chunked. S3 presigned PUT URLs reject that with 501
-// NotImplemented. The fix buffers the archive first so Content-Length is set.
+// NotImplemented. The fix spools the archive to a temporary file and streams
+// it from disk with an explicit Content-Length.
describe('uploadFile transfer encoding', () => {
let testDir: string
let server: Server
diff --git a/packages/js-sdk/tests/template/utils/tarFileStream.test.ts b/packages/js-sdk/tests/template/utils/tarFileStream.test.ts
index d08278cd2..3b86dbbcc 100644
--- a/packages/js-sdk/tests/template/utils/tarFileStream.test.ts
+++ b/packages/js-sdk/tests/template/utils/tarFileStream.test.ts
@@ -1,9 +1,18 @@
-import { expect, test, describe, beforeAll, afterAll, beforeEach } from 'vitest'
-import { writeFile, mkdir, rm, symlink, readFile } from 'fs/promises'
-import { createWriteStream } from 'fs'
+import {
+ expect,
+ test,
+ describe,
+ beforeAll,
+ afterAll,
+ beforeEach,
+ vi,
+} from 'vitest'
+import { writeFile, mkdir, rm, symlink, readFile, access } from 'fs/promises'
+import fs from 'node:fs'
+import { Readable } from 'node:stream'
+import { pipeline } from 'node:stream/promises'
import { join } from 'path'
import { tmpdir } from 'os'
-import { pipeline } from 'stream/promises'
import { tarFileStream } from '../../../src/template/utils'
import * as tar from 'tar'
import { ReadEntry } from 'tar'
@@ -26,39 +35,64 @@ describe('tarFileStream', () => {
})
/**
- * Extract tar contents into a dictionary mapping paths to file contents.
+ * Wait until a path no longer exists. Cleanup runs asynchronously from the
+ * stream's `close` handler, so it may not complete in the same tick.
*/
- async function extractTarContents(
- stream: any
- ): Promise<Map<string, Buffer | null>> {
- // Write stream to a temporary file (like Python's BytesIO)
- const tarFile = join(tmpdir(), `tar-test-${Date.now()}.tar.gz`)
- const writeStream = createWriteStream(tarFile)
- await pipeline(stream, writeStream)
-
- const contents = new Map<string, Buffer | null>()
+ async function waitUntilGone(target: string): Promise<void> {
+ for (let i = 0; i < 100; i++) {
+ try {
+ await access(target)
+ } catch {
+ return
+ }
+ await new Promise((resolve) => setTimeout(resolve, 10))
+ }
+ throw new Error(`path was not removed: ${target}`)
+ }
- // Extract files to a temp directory and collect members
- const extractDir = join(tmpdir(), `tar-extract-${Date.now()}`)
+ /**
+ * Pipe an archive stream into tar's extractor and collect its entries.
+ */
+ async function extractTarStream(
+ stream: Readable
+ ): Promise<{ extractDir: string; members: ReadEntry[] }> {
+ const extractDir = join(
+ tmpdir(),
+ `tar-extract-${Date.now()}-${Math.random()}`
+ )
await mkdir(extractDir, { recursive: true })
const members: ReadEntry[] = []
- await tar.extract({
- file: tarFile,
- cwd: extractDir,
- gzip: true,
- onentry: (entry: ReadEntry) => {
- members.push(entry)
- },
- })
-
- // Read file contents
+ await pipeline(
+ stream,
+ tar.extract({
+ cwd: extractDir,
+ gzip: true,
+ onentry: (entry: ReadEntry) => {
+ members.push(entry)
+ },
+ })
+ )
+
+ return { extractDir, members }
+ }
+
+ /**
+ * Extract archive contents into a map of path -> file contents.
+ */
+ async function extractTarContents(
+ stream: Readable
+ ): Promise<Map<string, Buffer | null>> {
+ const { extractDir, members } = await extractTarStream(stream)
+ const contents = new Map<string, Buffer | null>()
+
for (const member of members) {
if (member.type === 'File') {
- const filePath = join(extractDir, member.path)
try {
- const content = await readFile(filePath)
- contents.set(member.path, content)
+ contents.set(
+ member.path,
+ await readFile(join(extractDir, member.path))
+ )
} catch {
// File might not exist
}
@@ -69,76 +103,55 @@ describe('tarFileStream', () => {
}
}
- // Cleanup
- await rm(tarFile, { force: true })
await rm(extractDir, { recursive: true, force: true })
-
return contents
}
/**
- * Get tar members
+ * Extract archive members into a map of path -> entry.
*/
- async function getTarMembers(stream: any): Promise<Map<string, ReadEntry>> {
- // Write stream to a temporary file
- const tarFile = join(tmpdir(), `tar-test-${Date.now()}.tar.gz`)
- const writeStream = createWriteStream(tarFile)
- await pipeline(stream, writeStream)
-
- const members = new Map<string, ReadEntry>()
-
- const extractDir = join(tmpdir(), `tar-extract-${Date.now()}`)
- await mkdir(extractDir, { recursive: true })
-
- await tar.extract({
- file: tarFile,
- cwd: extractDir,
- gzip: true,
- onentry: (entry: ReadEntry) => {
- members.set(entry.path, entry)
- },
- })
-
- // Cleanup
- await rm(tarFile, { force: true })
+ async function getTarMembers(
+ stream: Readable
+ ): Promise<Map<string, ReadEntry>> {
+ const { extractDir, members } = await extractTarStream(stream)
+ const map = new Map<string, ReadEntry>()
+ for (const member of members) {
+ map.set(member.path, member)
+ }
await rm(extractDir, { recursive: true, force: true })
-
- return members
+ return map
}
test('should create tar with simple files', async () => {
- // Create test files
await writeFile(join(testDir, 'file1.txt'), 'content1')
await writeFile(join(testDir, 'file2.txt'), 'content2')
- const stream = await tarFileStream('*.txt', testDir, [], false)
+ const { stream, size } = await tarFileStream('*.txt', testDir, [], false)
+ expect(size).toBeGreaterThan(0)
+
const contents = await extractTarContents(stream)
expect(contents.size).toBe(2)
- expect(contents.has('file1.txt')).toBe(true)
- expect(contents.has('file2.txt')).toBe(true)
expect(contents.get('file1.txt')?.toString()).toBe('content1')
expect(contents.get('file2.txt')?.toString()).toBe('content2')
})
test('should respect ignore patterns', async () => {
- // Create test files
await writeFile(join(testDir, 'file1.txt'), 'content1')
await writeFile(join(testDir, 'file2.txt'), 'content2')
await writeFile(join(testDir, 'temp.txt'), 'temp content')
await writeFile(join(testDir, 'backup.txt'), 'backup content')
- const stream = await tarFileStream(
+ const { stream } = await tarFileStream(
'*.txt',
testDir,
['temp*', 'backup*'],
false
)
+
const contents = await extractTarContents(stream)
expect(contents.size).toBe(2)
- expect(contents.has('file1.txt')).toBe(true)
- expect(contents.has('file2.txt')).toBe(true)
expect(contents.get('file1.txt')?.toString()).toBe('content1')
expect(contents.get('file2.txt')?.toString()).toBe('content2')
expect(contents.has('temp.txt')).toBe(false)
@@ -146,17 +159,15 @@ describe('tarFileStream', () => {
})
test('should handle nested files', async () => {
- // Create nested directory structure
const nestedDir = join(testDir, 'src', 'components')
await mkdir(nestedDir, { recursive: true })
await writeFile(join(testDir, 'src', 'index.ts'), 'index content')
await writeFile(join(nestedDir, 'Button.tsx'), 'button content')
- const stream = await tarFileStream('src', testDir, [], false)
- const contents = await extractTarContents(stream)
+ const { stream } = await tarFileStream('src', testDir, [], false)
- // Should include the directory and files
+ const contents = await extractTarContents(stream)
const paths = Array.from(contents.keys())
expect(paths.some((p) => p.includes('src'))).toBe(true)
expect(paths.some((p) => p.includes('index.ts'))).toBe(true)
@@ -164,14 +175,10 @@ describe('tarFileStream', () => {
})
test('should resolve symlinks when enabled', async () => {
- // Create original file
- const originalPath = join(testDir, 'original.txt')
- await writeFile(originalPath, 'original content')
+ await writeFile(join(testDir, 'original.txt'), 'original content')
- // Create symlink
- const symlinkPath = join(testDir, 'link.txt')
try {
- await symlink('original.txt', symlinkPath)
+ await symlink('original.txt', join(testDir, 'link.txt'))
} catch (error: any) {
// Skip test if symlinks are not supported on this platform
if (error.code === 'ENOSYS' || error.code === 'EPERM') {
@@ -180,27 +187,19 @@ describe('tarFileStream', () => {
throw error
}
- // Test with resolveSymlinks=true
- const stream = await tarFileStream('*.txt', testDir, [], true)
- const contents = await extractTarContents(stream)
+ const { stream } = await tarFileStream('*.txt', testDir, [], true)
- // Both files should be in tar
- expect(contents.has('original.txt')).toBe(true)
- expect(contents.has('link.txt')).toBe(true)
- // Symlink should be resolved (contain actual content, not link)
+ const contents = await extractTarContents(stream)
expect(contents.get('original.txt')?.toString()).toBe('original content')
+ // Symlink should be resolved (contain actual content, not link)
expect(contents.get('link.txt')?.toString()).toBe('original content')
})
test('should preserve symlinks when disabled', async () => {
- // Create original file
- const originalPath = join(testDir, 'original.txt')
- await writeFile(originalPath, 'original content')
+ await writeFile(join(testDir, 'original.txt'), 'original content')
- // Create symlink
- const symlinkPath = join(testDir, 'link.txt')
try {
- await symlink('original.txt', symlinkPath)
+ await symlink('original.txt', join(testDir, 'link.txt'))
} catch (error: any) {
// Skip test if symlinks are not supported on this platform
if (error.code === 'ENOSYS' || error.code === 'EPERM') {
@@ -209,21 +208,65 @@ describe('tarFileStream', () => {
throw error
}
- // Test with resolveSymlinks=false
- const stream = await tarFileStream('*.txt', testDir, [], false)
- const members = await getTarMembers(stream)
-
- // Both files should be in tar
- expect(members.has('original.txt')).toBe(true)
- expect(members.has('link.txt')).toBe(true)
+ const { stream } = await tarFileStream('*.txt', testDir, [], false)
- // Original should be a regular file
- const original = members.get('original.txt')!
- expect(original.type).toBe('File')
-
- // Link should be a symlink
+ const members = await getTarMembers(stream)
+ expect(members.get('original.txt')?.type).toBe('File')
const link = members.get('link.txt')!
expect(link.type).toBe('SymbolicLink')
expect(link.linkpath).toBe('original.txt')
})
+
+ test('removes the spooled archive once the stream is consumed', async () => {
+ await writeFile(join(testDir, 'file1.txt'), 'content1')
+
+ // Capture the temp dir the helper creates so we can assert it is gone.
+ let tmpDir: string | undefined
+ const mkdtemp = fs.promises.mkdtemp.bind(fs.promises)
+ const spy = vi
+ .spyOn(fs.promises, 'mkdtemp')
+ .mockImplementation(async (prefix: any, ...rest: any[]) => {
+ const dir = await mkdtemp(prefix, ...rest)
+ tmpDir = dir
+ return dir
+ })
+
+ try {
+ const { stream } = await tarFileStream('*.txt', testDir, [], false)
+ expect(tmpDir).toBeDefined()
+ await access(tmpDir!)
+
+ // Drain the stream to completion; the `close` handler then removes the
+ // spooled archive.
+ stream.resume()
+ await waitUntilGone(tmpDir!)
+ } finally {
+ spy.mockRestore()
+ }
+ })
+
+ test('cleans up the spooled archive if it is never consumed', async () => {
+ await writeFile(join(testDir, 'file1.txt'), 'content1')
+
+ let tmpDir: string | undefined
+ const mkdtemp = fs.promises.mkdtemp.bind(fs.promises)
+ const spy = vi
+ .spyOn(fs.promises, 'mkdtemp')
+ .mockImplementation(async (prefix: any, ...rest: any[]) => {
+ const dir = await mkdtemp(prefix, ...rest)
+ tmpDir = dir
+ return dir
+ })
+
+ try {
+ const { stream } = await tarFileStream('*.txt', testDir, [], false)
+ await access(tmpDir!)
+
+ // Destroying without reading still fires `close` and removes the file.
+ stream.destroy()
+ await waitUntilGone(tmpDir!)
+ } finally {
+ spy.mockRestore()
+ }
+ })
})
diff --git a/packages/python-sdk/e2b/template/consts.py b/packages/python-sdk/e2b/template/consts.py
index 41d6cb879..eb0070671 100644
--- a/packages/python-sdk/e2b/template/consts.py
+++ b/packages/python-sdk/e2b/template/consts.py
@@ -28,3 +28,12 @@ Default setting for whether to resolve symbolic links when copying files.
When False, symlinks are copied as symlinks rather than following them.
"""
RESOLVE_SYMLINKS = False
+
+"""
+Default timeout (in seconds) for uploading the build-context archive to the
+S3 presigned URL. Uploads of large archives can take far longer than the 60s
+general API request timeout, so the upload uses a 1-hour default unless the
+caller passes an explicit ``request_timeout``. This matches the JS SDK's
+``FILE_UPLOAD_TIMEOUT_MS``.
+"""
+FILE_UPLOAD_TIMEOUT_SECONDS = 3600
diff --git a/packages/python-sdk/e2b/template/utils.py b/packages/python-sdk/e2b/template/utils.py
index 2902dbca8..cdfc62547 100644
--- a/packages/python-sdk/e2b/template/utils.py
+++ b/packages/python-sdk/e2b/template/utils.py
@@ -1,14 +1,14 @@
import hashlib
import os
-import io
import tarfile
+import tempfile
import json
import stat
from wcmatch import glob
import re
import inspect
from types import TracebackType, FrameType
-from typing import List, Optional, Union
+from typing import IO, List, Optional, Union
from e2b.exceptions import TemplateException
from e2b.template.consts import BASE_STEP_NAME, FINALIZE_STEP_NAME
@@ -259,32 +259,47 @@ def tar_file_stream(
file_context_path: str,
ignore_patterns: List[str],
resolve_symlinks: bool,
-) -> io.BytesIO:
+) -> IO[bytes]:
"""
- Create a tar stream of files matching a pattern.
+ Create a tar archive of files matching a pattern in a temporary file.
+
+ The archive is spooled to disk so it can be uploaded as a stream instead
+ of being buffered in memory. The temporary file is deleted when closed.
:param file_name: Glob pattern for files to include
:param file_context_path: Base directory for resolving file paths
:param ignore_patterns: Ignore patterns
:param resolve_symlinks: Whether to resolve symbolic links
- :return: Tar stream
+ :return: Binary file object positioned at the start of the archive
"""
- tar_buffer = io.BytesIO()
- with tarfile.open(
- fileobj=tar_buffer,
- mode="w:gz",
- dereference=resolve_symlinks,
- ) as tar:
- files = get_all_files_in_path(
- file_name, file_context_path, ignore_patterns, True
- )
- for file in files:
- tar.add(
- file, arcname=os.path.relpath(file, file_context_path), recursive=False
+ tar_file = tempfile.TemporaryFile()
+ try:
+ with tarfile.open(
+ fileobj=tar_file,
+ mode="w:gz",
+ dereference=resolve_symlinks,
+ ) as tar:
+ files = get_all_files_in_path(
+ file_name, file_context_path, ignore_patterns, True
)
-
- return tar_buffer
+ for file in files:
+ tar.add(
+ file,
+ arcname=os.path.relpath(file, file_context_path),
+ recursive=False,
+ )
+
+ tar_file.seek(0)
+ return tar_file
+ except Exception:
+ # Best-effort cleanup: a close failure must not replace the real
+ # archive-creation error.
+ try:
+ tar_file.close()
+ except Exception:
+ pass
+ raise
def strip_ansi_escape_codes(text: str) -> str:
diff --git a/packages/python-sdk/e2b/template_async/build_api.py b/packages/python-sdk/e2b/template_async/build_api.py
index 44eada8cc..d306cea17 100644
--- a/packages/python-sdk/e2b/template_async/build_api.py
+++ b/packages/python-sdk/e2b/template_async/build_api.py
@@ -1,10 +1,12 @@
import asyncio
+import os
from types import TracebackType
from typing import Callable, Optional, List, Union
import httpx
from e2b.api import handle_api_exception
+from e2b.io_utils import aiter_io_chunks
from e2b.api.client.api.templates import (
post_v3_templates,
get_templates_template_id_files_hash,
@@ -37,6 +39,7 @@ from e2b.template.types import (
TemplateTag,
TemplateTagInfo,
)
+from e2b.template.consts import FILE_UPLOAD_TIMEOUT_SECONDS
from e2b.template.utils import get_build_step_index, tar_file_stream
@@ -105,21 +108,46 @@ async def upload_file(
ignore_patterns: List[str],
resolve_symlinks: bool,
stack_trace: Optional[TracebackType],
+ request_timeout: Optional[float] = None,
):
+ # Uploading a large build-context archive can take far longer than the 60s
+ # general API timeout, so default to a 1-hour upload timeout unless the
+ # caller set an explicit request_timeout. Matches the JS SDK
+ # (FILE_UPLOAD_TIMEOUT_MS).
+ upload_timeout = (
+ request_timeout if request_timeout is not None else FILE_UPLOAD_TIMEOUT_SECONDS
+ )
try:
- tar_buffer = tar_file_stream(
+ tar_file = tar_file_stream(
file_name, context_path, ignore_patterns, resolve_symlinks
)
-
- async with httpx.AsyncClient(
- timeout=api_client._timeout,
- verify=api_client._verify_ssl,
- follow_redirects=api_client._follow_redirects,
- proxy=getattr(api_client, "_proxy", None),
- http2=False,
- ) as client:
- response = await client.put(url, content=tar_buffer.getvalue())
- response.raise_for_status()
+ try:
+ size = os.fstat(tar_file.fileno()).st_size
+
+ async with httpx.AsyncClient(
+ timeout=httpx.Timeout(upload_timeout),
+ verify=api_client._verify_ssl,
+ follow_redirects=api_client._follow_redirects,
+ proxy=getattr(api_client, "_proxy", None),
+ http2=False,
+ ) as client:
+ # Stream the archive from disk via an async iterator. The
+ # explicit Content-Length suppresses chunked transfer
+ # encoding, which S3 presigned URLs reject.
+ response = await client.put(
+ url,
+ content=aiter_io_chunks(tar_file),
+ headers={"Content-Length": str(size)},
+ )
+ response.raise_for_status()
+ finally:
+ # Closing the spooled temp file is best-effort: a failure here
+ # must not mask a successful upload as a FileUploadException,
+ # nor overwrite a real upload error.
+ try:
+ tar_file.close()
+ except Exception:
+ pass
except httpx.HTTPStatusError as e:
raise FileUploadException(f"Failed to upload file: {e}").with_traceback(
stack_trace
diff --git a/packages/python-sdk/e2b/template_async/main.py b/packages/python-sdk/e2b/template_async/main.py
index 8324c5ca0..78d80c94c 100644
--- a/packages/python-sdk/e2b/template_async/main.py
+++ b/packages/python-sdk/e2b/template_async/main.py
@@ -41,6 +41,7 @@ class AsyncTemplate(TemplateBase):
memory_mb: int = 1024,
skip_cache: bool = False,
on_build_logs: Optional[Callable[[LogEntry], None]] = None,
+ request_timeout: Optional[float] = None,
) -> BuildInfo:
"""
Internal implementation of the template build process
@@ -127,6 +128,7 @@ class AsyncTemplate(TemplateBase):
],
resolve_symlinks,
stack_trace,
+ request_timeout=request_timeout,
)
if on_build_logs:
on_build_logs(
@@ -248,6 +250,9 @@ class AsyncTemplate(TemplateBase):
memory_mb=memory_mb,
skip_cache=skip_cache,
on_build_logs=on_build_logs,
+ # Only honor an explicitly set request_timeout for uploads;
+ # otherwise upload_file applies its 1-hour default.
+ request_timeout=opts.get("request_timeout"),
)
if on_build_logs:
@@ -337,6 +342,9 @@ class AsyncTemplate(TemplateBase):
memory_mb=memory_mb,
skip_cache=skip_cache,
on_build_logs=on_build_logs,
+ # Only honor an explicitly set request_timeout for uploads;
+ # otherwise upload_file applies its 1-hour default.
+ request_timeout=opts.get("request_timeout"),
)
@staticmethod
diff --git a/packages/python-sdk/e2b/template_sync/build_api.py b/packages/python-sdk/e2b/template_sync/build_api.py
index 257492928..243fc81dc 100644
--- a/packages/python-sdk/e2b/template_sync/build_api.py
+++ b/packages/python-sdk/e2b/template_sync/build_api.py
@@ -37,6 +37,7 @@ from e2b.template.types import (
TemplateTag,
TemplateTagInfo,
)
+from e2b.template.consts import FILE_UPLOAD_TIMEOUT_SECONDS
from e2b.template.utils import get_build_step_index, tar_file_stream
@@ -105,20 +106,40 @@ def upload_file(
ignore_patterns: List[str],
resolve_symlinks: bool,
stack_trace: Optional[TracebackType],
+ request_timeout: Optional[float] = None,
):
+ # Uploading a large build-context archive can take far longer than the 60s
+ # general API timeout, so default to a 1-hour upload timeout unless the
+ # caller set an explicit request_timeout. Matches the JS SDK
+ # (FILE_UPLOAD_TIMEOUT_MS).
+ upload_timeout = (
+ request_timeout if request_timeout is not None else FILE_UPLOAD_TIMEOUT_SECONDS
+ )
try:
- tar_buffer = tar_file_stream(
+ tar_file = tar_file_stream(
file_name, context_path, ignore_patterns, resolve_symlinks
)
- with httpx.Client(
- timeout=api_client._timeout,
- verify=api_client._verify_ssl,
- follow_redirects=api_client._follow_redirects,
- proxy=getattr(api_client, "_proxy", None),
- http2=False,
- ) as client:
- response = client.put(url, content=tar_buffer.getvalue())
- response.raise_for_status()
+ try:
+ with httpx.Client(
+ timeout=httpx.Timeout(upload_timeout),
+ verify=api_client._verify_ssl,
+ follow_redirects=api_client._follow_redirects,
+ proxy=getattr(api_client, "_proxy", None),
+ http2=False,
+ ) as client:
+ # httpx streams the archive from disk in chunks and sets
+ # Content-Length from the file size—S3 presigned URLs reject
+ # chunked transfer encoding.
+ response = client.put(url, content=tar_file)
+ response.raise_for_status()
+ finally:
+ # Closing the spooled temp file is best-effort: a failure here
+ # must not mask a successful upload as a FileUploadException,
+ # nor overwrite a real upload error.
+ try:
+ tar_file.close()
+ except Exception:
+ pass
except httpx.HTTPStatusError as e:
raise FileUploadException(f"Failed to upload file: {e}").with_traceback(
stack_trace
diff --git a/packages/python-sdk/e2b/template_sync/main.py b/packages/python-sdk/e2b/template_sync/main.py
index d1d4f35f5..94414de2f 100644
--- a/packages/python-sdk/e2b/template_sync/main.py
+++ b/packages/python-sdk/e2b/template_sync/main.py
@@ -41,6 +41,7 @@ class Template(TemplateBase):
memory_mb: int = 1024,
skip_cache: bool = False,
on_build_logs: Optional[Callable[[LogEntry], None]] = None,
+ request_timeout: Optional[float] = None,
) -> BuildInfo:
"""
Internal implementation of the template build process
@@ -127,6 +128,7 @@ class Template(TemplateBase):
],
resolve_symlinks,
stack_trace,
+ request_timeout=request_timeout,
)
if on_build_logs:
on_build_logs(
@@ -248,6 +250,9 @@ class Template(TemplateBase):
memory_mb=memory_mb,
skip_cache=skip_cache,
on_build_logs=on_build_logs,
+ # Only honor an explicitly set request_timeout for uploads;
+ # otherwise upload_file applies its 1-hour default.
+ request_timeout=opts.get("request_timeout"),
)
if on_build_logs:
@@ -337,6 +342,9 @@ class Template(TemplateBase):
memory_mb=memory_mb,
skip_cache=skip_cache,
on_build_logs=on_build_logs,
+ # Only honor an explicitly set request_timeout for uploads;
+ # otherwise upload_file applies its 1-hour default.
+ request_timeout=opts.get("request_timeout"),
)
@staticmethod
diff --git a/packages/python-sdk/tests/async/template_async/test_upload_file.py b/packages/python-sdk/tests/async/template_async/test_upload_file.py
index 472a4be54..d4f4b0414 100644
--- a/packages/python-sdk/tests/async/template_async/test_upload_file.py
+++ b/packages/python-sdk/tests/async/template_async/test_upload_file.py
@@ -1,15 +1,20 @@
import threading
from http.server import BaseHTTPRequestHandler, HTTPServer
+from unittest import mock
+
+import httpx
from e2b.api.client.client import AuthenticatedClient
+from e2b.template import utils as template_utils
+from e2b.template.consts import FILE_UPLOAD_TIMEOUT_SECONDS
from e2b.template_async.build_api import upload_file
# Regression test for e2b-dev/e2b#1243 — upload_file must set Content-Length
# and must not fall back to Transfer-Encoding: chunked. S3 presigned PUT URLs
-# reject chunked encoding with 501 NotImplemented. httpx sets Content-Length
-# automatically when we pass bytes (tar_buffer.getvalue()); this test guards
-# against someone swapping the bytes for a generator/stream later.
+# reject chunked encoding with 501 NotImplemented. The archive is streamed
+# from a temporary file on disk with a known Content-Length instead of being
+# buffered in memory; this test guards that contract.
#
# The mock server runs in a daemon thread and doesn't need to be async — the
# httpx.AsyncClient connects to it via asyncio sockets without blocking the
@@ -75,3 +80,115 @@ async def test_upload_file_sets_content_length_and_no_chunked_encoding(tmp_path)
if transfer_encoding is not None:
assert "chunked" not in transfer_encoding.lower()
assert "Authorization" not in state["headers"]
+
+
+async def _capture_upload_timeout(tmp_path, request_timeout=None):
+ """Run upload_file against a local server, capturing the httpx timeout."""
+ (tmp_path / "hello.txt").write_text("hello world")
+
+ server, thread, state = _make_server()
+ host, port = server.server_address
+ url = f"http://{host}:{port}/upload"
+
+ captured = {}
+ real_client = httpx.AsyncClient
+
+ def spy_client(*args, **kwargs):
+ captured["timeout"] = kwargs.get("timeout")
+ return real_client(*args, **kwargs)
+
+ try:
+ client = AuthenticatedClient(base_url="http://test", token="test")
+ kwargs = {}
+ if request_timeout is not None:
+ kwargs["request_timeout"] = request_timeout
+ with mock.patch(
+ "e2b.template_async.build_api.httpx.AsyncClient", side_effect=spy_client
+ ):
+ await upload_file(
+ api_client=client,
+ file_name="*.txt",
+ context_path=str(tmp_path),
+ url=url,
+ ignore_patterns=[],
+ resolve_symlinks=False,
+ stack_trace=None,
+ **kwargs,
+ )
+ finally:
+ server.shutdown()
+ server.server_close()
+ thread.join(timeout=5)
+
+ return captured["timeout"]
+
+
+async def test_upload_file_defaults_to_one_hour_timeout(tmp_path):
+ # Uploads of large archives need far longer than the 60s general API
+ # timeout, so the default upload timeout is 1 hour (matches the JS SDK).
+ timeout = await _capture_upload_timeout(tmp_path)
+ assert timeout == httpx.Timeout(FILE_UPLOAD_TIMEOUT_SECONDS)
+
+
+async def test_upload_file_honors_explicit_request_timeout(tmp_path):
+ # An explicitly set request_timeout overrides the 1-hour upload default.
+ timeout = await _capture_upload_timeout(tmp_path, request_timeout=5.0)
+ assert timeout == httpx.Timeout(5.0)
+
+
+async def test_upload_file_ignores_post_upload_close_failure(tmp_path):
+ # Regression test: once S3 has accepted the archive, closing the spooled
+ # temp file in the `finally` block can raise. That failure must not be
+ # wrapped as a FileUploadException — the upload already succeeded.
+ (tmp_path / "hello.txt").write_text("hello world")
+
+ server, thread, state = _make_server()
+ host, port = server.server_address
+ url = f"http://{host}:{port}/upload"
+
+ real_tar_file_stream = template_utils.tar_file_stream
+
+ class _FailingCloseFile:
+ # Proxies a real spooled temp file but raises on close(). The
+ # underlying file object is a C type whose `close` attribute can't
+ # be reassigned, so we wrap it instead.
+ def __init__(self, inner):
+ self._inner = inner
+
+ def __getattr__(self, name):
+ return getattr(self._inner, name)
+
+ def __iter__(self):
+ return iter(self._inner)
+
+ def close(self):
+ # Run the real close so we don't leak the temp file, then
+ # simulate a close failure surfacing from the `finally`.
+ self._inner.close()
+ raise OSError("close failed")
+
+ def failing_close_stream(*args, **kwargs):
+ return _FailingCloseFile(real_tar_file_stream(*args, **kwargs))
+
+ try:
+ client = AuthenticatedClient(base_url="http://test", token="test")
+ with mock.patch(
+ "e2b.template_async.build_api.tar_file_stream",
+ side_effect=failing_close_stream,
+ ):
+ # Must not raise despite close() failing after a 200 response.
+ await upload_file(
+ api_client=client,
+ file_name="*.txt",
+ context_path=str(tmp_path),
+ url=url,
+ ignore_patterns=[],
+ resolve_symlinks=False,
+ stack_trace=None,
+ )
+ finally:
+ server.shutdown()
+ server.server_close()
+ thread.join(timeout=5)
+
+ assert state["headers"] is not None
diff --git a/packages/python-sdk/tests/shared/template/utils/test_tar_file_stream.py b/packages/python-sdk/tests/shared/template/utils/test_tar_file_stream.py
index 672b937e9..b274e4955 100644
--- a/packages/python-sdk/tests/shared/template/utils/test_tar_file_stream.py
+++ b/packages/python-sdk/tests/shared/template/utils/test_tar_file_stream.py
@@ -1,8 +1,8 @@
import os
import tempfile
import tarfile
-import io
import pytest
+from typing import IO
from e2b.template.utils import tar_file_stream
@@ -14,7 +14,7 @@ class TestTarFileStream:
yield tmpdir.name
tmpdir.cleanup()
- def _extract_tar_contents(self, tar_buffer: io.BytesIO) -> dict:
+ def _extract_tar_contents(self, tar_buffer: IO[bytes]) -> dict:
"""Extract tar contents into a dictionary mapping paths to file contents."""
tar_buffer.seek(0)
contents = {}
diff --git a/packages/python-sdk/tests/sync/template_sync/test_upload_file.py b/packages/python-sdk/tests/sync/template_sync/test_upload_file.py
index 609e3178d..e2dd9a83a 100644
--- a/packages/python-sdk/tests/sync/template_sync/test_upload_file.py
+++ b/packages/python-sdk/tests/sync/template_sync/test_upload_file.py
@@ -1,15 +1,20 @@
import threading
from http.server import BaseHTTPRequestHandler, HTTPServer
+from unittest import mock
+
+import httpx
from e2b.api.client.client import AuthenticatedClient
+from e2b.template import utils as template_utils
+from e2b.template.consts import FILE_UPLOAD_TIMEOUT_SECONDS
from e2b.template_sync.build_api import upload_file
# Regression test for e2b-dev/e2b#1243 — upload_file must set Content-Length
# and must not fall back to Transfer-Encoding: chunked. S3 presigned PUT URLs
-# reject chunked encoding with 501 NotImplemented. httpx sets Content-Length
-# automatically when we pass bytes (tar_buffer.getvalue()); this test guards
-# against someone swapping the bytes for a generator/stream later.
+# reject chunked encoding with 501 NotImplemented. The archive is streamed
+# from a temporary file on disk with a known Content-Length instead of being
+# buffered in memory; this test guards that contract.
def _make_server():
@@ -71,3 +76,115 @@ def test_upload_file_sets_content_length_and_no_chunked_encoding(tmp_path):
if transfer_encoding is not None:
assert "chunked" not in transfer_encoding.lower()
assert "Authorization" not in state["headers"]
+
+
+def _capture_upload_timeout(tmp_path, request_timeout=None):
+ """Run upload_file against a local server, capturing the httpx timeout."""
+ (tmp_path / "hello.txt").write_text("hello world")
+
+ server, thread, state = _make_server()
+ host, port = server.server_address
+ url = f"http://{host}:{port}/upload"
+
+ captured = {}
+ real_client = httpx.Client
+
+ def spy_client(*args, **kwargs):
+ captured["timeout"] = kwargs.get("timeout")
+ return real_client(*args, **kwargs)
+
+ try:
+ client = AuthenticatedClient(base_url="http://test", token="test")
+ kwargs = {}
+ if request_timeout is not None:
+ kwargs["request_timeout"] = request_timeout
+ with mock.patch(
+ "e2b.template_sync.build_api.httpx.Client", side_effect=spy_client
+ ):
+ upload_file(
+ api_client=client,
+ file_name="*.txt",
+ context_path=str(tmp_path),
+ url=url,
+ ignore_patterns=[],
+ resolve_symlinks=False,
+ stack_trace=None,
+ **kwargs,
+ )
+ finally:
+ server.shutdown()
+ server.server_close()
+ thread.join(timeout=5)
+
+ return captured["timeout"]
+
+
+def test_upload_file_defaults_to_one_hour_timeout(tmp_path):
+ # Uploads of large archives need far longer than the 60s general API
+ # timeout, so the default upload timeout is 1 hour (matches the JS SDK).
+ timeout = _capture_upload_timeout(tmp_path)
+ assert timeout == httpx.Timeout(FILE_UPLOAD_TIMEOUT_SECONDS)
+
+
+def test_upload_file_honors_explicit_request_timeout(tmp_path):
+ # An explicitly set request_timeout overrides the 1-hour upload default.
+ timeout = _capture_upload_timeout(tmp_path, request_timeout=5.0)
+ assert timeout == httpx.Timeout(5.0)
+
+
+def test_upload_file_ignores_post_upload_close_failure(tmp_path):
+ # Regression test: once S3 has accepted the archive, closing the spooled
+ # temp file in the `finally` block can raise. That failure must not be
+ # wrapped as a FileUploadException — the upload already succeeded.
+ (tmp_path / "hello.txt").write_text("hello world")
+
+ server, thread, state = _make_server()
+ host, port = server.server_address
+ url = f"http://{host}:{port}/upload"
+
+ real_tar_file_stream = template_utils.tar_file_stream
+
+ class _FailingCloseFile:
+ # Proxies a real spooled temp file but raises on close(). The
+ # underlying file object is a C type whose `close` attribute can't
+ # be reassigned, so we wrap it instead.
+ def __init__(self, inner):
+ self._inner = inner
+
+ def __getattr__(self, name):
+ return getattr(self._inner, name)
+
+ def __iter__(self):
+ return iter(self._inner)
+
+ def close(self):
+ # Run the real close so we don't leak the temp file, then
+ # simulate a close failure surfacing from the `finally`.
+ self._inner.close()
+ raise OSError("close failed")
+
+ def failing_close_stream(*args, **kwargs):
+ return _FailingCloseFile(real_tar_file_stream(*args, **kwargs))
+
+ try:
+ client = AuthenticatedClient(base_url="http://test", token="test")
+ with mock.patch(
+ "e2b.template_sync.build_api.tar_file_stream",
+ side_effect=failing_close_stream,
+ ):
+ # Must not raise despite close() failing after a 200 response.
+ upload_file(
+ api_client=client,
+ file_name="*.txt",
+ context_path=str(tmp_path),
+ url=url,
+ ignore_patterns=[],
+ resolve_symlinks=False,
+ stack_trace=None,
+ )
+ finally:
+ server.shutdown()
+ server.server_close()
+ thread.join(timeout=5)
+
+ assert state["headers"] is not None