Open-source, secure environment with real-world tools for enterprise-grade agents.
Python agentaiai-agentai-agentscode-interpretercopilotdevelopmentdevtoolsgptgpt-4javascriptllmnextjsopenaipythonreactsoftwaretypescript
Stream template build-context upload from disk instead of buffering in memory (#1435)
Mish Ushakovbb1696871b49
diff --git a/.changeset/stream-template-build-upload.md b/.changeset/stream-template-build-upload.mdnew file mode 100644index 000000000..c71e96ba5--- /dev/null+++ b/.changeset/stream-template-build-upload.md@@ -0,0 +1,6 @@+---+"e2b": patch+"@e2b/python-sdk": patch+---++Template builds: the build-context tar archive is now spooled to a temporary file and streamed from disk during upload instead of being held in memory (JS and Python, sync and async), while keeping the explicit `Content-Length` required by S3 presigned URLs. Temp-file cleanup is now best-effort, so a cleanup failure after the upload no longer masks a successful upload as an error (nor overwrites the real upload error on failure). The Python SDK now uploads the archive with a 1-hour default timeout (overridable via `request_timeout`) instead of the 60s general API timeout, matching the JS SDK and preventing large uploads from timing out.diff --git a/packages/js-sdk/src/template/buildApi.ts b/packages/js-sdk/src/template/buildApi.tsindex 819981ad3..581d26eba 100644--- a/packages/js-sdk/src/template/buildApi.ts+++ b/packages/js-sdk/src/template/buildApi.ts@@ -1,10 +1,11 @@+import type { Readable } from 'node:stream'import { ApiClient, handleApiError, components } from '../api'import { buildRequestSignal } from '../connectionConfig'import { dynamicImport } from '../utils'import { BuildError, FileUploadError, TemplateError } from '../errors'import { FILE_UPLOAD_TIMEOUT_MS } from './consts'import { LogEntry } from './logger'-import { getBuildStepIndex, tarFileStreamUpload } from './utils'+import { getBuildStepIndex, tarFileStream } from './utils'import {BuildStatusReason,TemplateBuildStatus,@@ -121,38 +122,40 @@ export async function uploadFile() {const { fileName, url, fileContextPath, ignorePatterns, resolveSymlinks } =options+ // Spool the archive to a temporary file and stream it from disk instead of+ // buffering in memory. S3 presigned PUT URLs reject Transfer-Encoding:+ // chunked with 501 NotImplemented (see e2b-dev/e2b#1243), so the upload+ // sends an explicit Content-Length, which fetch honors for stream bodies.+ // The spooled file deletes itself once the stream is consumed, so there is+ // no cleanup to manage here. The Python SDK takes the same approach+ // (build_api.py:upload_file).+ let uploadStream: Readable | undefinedtry {- const uploadStream = await tarFileStreamUpload(+ // Dynamically import so the browser bundle doesn't pull in node:stream.+ const { Readable } =+ await dynamicImport<typeof import('node:stream')>('node:stream')++ const tar = await tarFileStream(fileName,fileContextPath,ignorePatterns,resolveSymlinks)-- // Buffer the archive before uploading so fetch sets Content-Length.- // S3 presigned PUT URLs reject Transfer-Encoding: chunked with 501- // NotImplemented, which is what Node's fetch falls back to when the- // body is a Readable without a known length. See e2b-dev/e2b#1243.- // The Python SDK takes the same approach (build_api.py:upload_file).- // Dynamically import so the browser bundle doesn't pull in node:stream.- // tar's Pack extends Minipass and is iterable as AsyncIterable<Buffer> at- // runtime, but the cli's tsconfig (preserveSymlinks) doesn't surface that- // through the type chain — cast via unknown.- const { buffer } = await dynamicImport<- typeof import('node:stream/consumers')- >('node:stream/consumers')- const uploadBody = await buffer(- uploadStream as unknown as AsyncIterable<Buffer>- )+ uploadStream = tar.streamconst res = await fetch(url, {method: 'PUT',- body: uploadBody,+ body: Readable.toWeb(uploadStream) as ReadableStream<Uint8Array>,+ headers: {+ 'Content-Length': tar.size.toString(),+ },+ // Streaming request bodies require half-duplex mode.+ duplex: 'half',signal: buildRequestSignal(abortOpts?.requestTimeoutMs ?? FILE_UPLOAD_TIMEOUT_MS,abortOpts?.signal),- })+ } as RequestInit)if (!res.ok) {throw new FileUploadError(@@ -161,6 +164,10 @@ export async function uploadFile()}} catch (error) {+ // Ensure the spooled archive is removed even if fetch never consumed the+ // stream (e.g. it threw before reading the body). Destroying the stream+ // fires its `close` handler, which deletes the temp file.+ uploadStream?.destroy()if (error instanceof FileUploadError) {throw error}diff --git a/packages/js-sdk/src/template/utils.ts b/packages/js-sdk/src/template/utils.tsindex e4e83a37c..d5476ebc1 100644--- a/packages/js-sdk/src/template/utils.ts+++ b/packages/js-sdk/src/template/utils.ts@@ -1,9 +1,11 @@import crypto from 'node:crypto'import fs from 'node:fs'+import os from 'node:os'import path from 'node:path'import { dynamicImport, dynamicRequire } from '../utils'import { TemplateError } from '../errors'import { BASE_STEP_NAME, FINALIZE_STEP_NAME } from './consts'+import type { Readable } from 'node:stream'import type { Path } from 'glob'import type { BuildOptions } from './types'@@ -352,21 +354,33 @@ export function padOctal(mode: number): string {}/**- * Create a compressed tar stream of files matching a pattern.+ * Create a gzipped tar archive of files matching a pattern and return a+ * readable stream over it.+ *+ * The archive is spooled to a temporary file on disk so it can be uploaded as+ * a stream with a known `Content-Length` instead of being buffered in memory.+ * The temporary file is removed automatically once the returned stream is+ * closed — whether it was fully read, errored, or destroyed. This mirrors the+ * Python SDK's `tar_file_stream`, where closing the temp-file handle deletes+ * it (Node has no portable delete-on-close, so cleanup is tied to the stream's+ * `close` event instead).** @param fileName Glob pattern for files to include* @param fileContextPath Base directory for resolving file paths* @param ignorePatterns Ignore patterns to exclude from the archive* @param resolveSymlinks Whether to follow symbolic links- * @returns A readable stream of the gzipped tar archive+ * @returns The readable stream and the archive size in bytes*/export async function tarFileStream(fileName: string,fileContextPath: string,ignorePatterns: string[],resolveSymlinks: boolean-) {+): Promise<{ stream: Readable; size: number }> {const { create } = await dynamicImport<typeof import('tar')>('tar')+ // Dynamically import so the browser bundle doesn't pull in node:fs.+ const { createReadStream } =+ await dynamicImport<typeof import('node:fs')>('node:fs')const allFiles = await getAllFilesInPath(fileName,@@ -377,38 +391,38 @@ export async function tarFileStream(const filePaths = allFiles.map((file) => file.relativePosix())- // gzip.portable ensures deterministic gzip header without affecting file modes- return create(- {- gzip: true,- cwd: fileContextPath,- follow: resolveSymlinks,- noDirRecurse: true,- },- filePaths+ const tmpDir = await fs.promises.mkdtemp(+ path.join(os.tmpdir(), 'e2b-template-'))-}+ const tarPath = path.join(tmpDir, 'context.tar.gz')+ // Best-effort removal so it can never mask the archive-creation error or the+ // upload result. A leaked temp dir is non-fatal — the OS reclaims it.+ const removeTmpDir = () =>+ fs.promises.rm(tmpDir, { recursive: true, force: true }).catch(() => {})-/**- * Create a tar stream for upload using chunked transfer encoding.- *- * @param fileName Glob pattern for files to include- * @param fileContextPath Base directory for resolving file paths- * @param resolveSymlinks Whether to follow symbolic links- * @returns A readable stream of the gzipped tar archive- */-export async function tarFileStreamUpload(- fileName: string,- fileContextPath: string,- ignorePatterns: string[],- resolveSymlinks: boolean-) {- return tarFileStream(- fileName,- fileContextPath,- ignorePatterns,- resolveSymlinks- )+ try {+ await create(+ {+ gzip: true,+ cwd: fileContextPath,+ follow: resolveSymlinks,+ noDirRecurse: true,+ file: tarPath,+ },+ filePaths+ )++ const { size } = await fs.promises.stat(tarPath)++ const stream = createReadStream(tarPath)+ // Remove the spooled archive once the stream is done — `close` fires after+ // the fd is released on every path (end, error, or destroy).+ stream.once('close', removeTmpDir)+ return { stream, size }+ } catch (err) {+ await removeTmpDir()+ throw err+ }}/**diff --git a/packages/js-sdk/tests/template/uploadFile.test.ts b/packages/js-sdk/tests/template/uploadFile.test.tsindex 3d0cd25b0..119301e22 100644--- a/packages/js-sdk/tests/template/uploadFile.test.ts+++ b/packages/js-sdk/tests/template/uploadFile.test.ts@@ -9,7 +9,8 @@ import { uploadFile } from '../../src/template/buildApi'// Regression test for e2b-dev/e2b#1243 — uploadFile used to pass a Node// Readable directly to fetch, which made undici fall back to// Transfer-Encoding: chunked. S3 presigned PUT URLs reject that with 501-// NotImplemented. The fix buffers the archive first so Content-Length is set.+// NotImplemented. The fix spools the archive to a temporary file and streams+// it from disk with an explicit Content-Length.describe('uploadFile transfer encoding', () => {let testDir: stringlet server: Serverdiff --git a/packages/js-sdk/tests/template/utils/tarFileStream.test.ts b/packages/js-sdk/tests/template/utils/tarFileStream.test.tsindex d08278cd2..3b86dbbcc 100644--- a/packages/js-sdk/tests/template/utils/tarFileStream.test.ts+++ b/packages/js-sdk/tests/template/utils/tarFileStream.test.ts@@ -1,9 +1,18 @@-import { expect, test, describe, beforeAll, afterAll, beforeEach } from 'vitest'-import { writeFile, mkdir, rm, symlink, readFile } from 'fs/promises'-import { createWriteStream } from 'fs'+import {+ expect,+ test,+ describe,+ beforeAll,+ afterAll,+ beforeEach,+ vi,+} from 'vitest'+import { writeFile, mkdir, rm, symlink, readFile, access } from 'fs/promises'+import fs from 'node:fs'+import { Readable } from 'node:stream'+import { pipeline } from 'node:stream/promises'import { join } from 'path'import { tmpdir } from 'os'-import { pipeline } from 'stream/promises'import { tarFileStream } from '../../../src/template/utils'import * as tar from 'tar'import { ReadEntry } from 'tar'@@ -26,39 +35,64 @@ describe('tarFileStream', () => {})/**- * Extract tar contents into a dictionary mapping paths to file contents.+ * Wait until a path no longer exists. Cleanup runs asynchronously from the+ * stream's `close` handler, so it may not complete in the same tick.*/- async function extractTarContents(- stream: any- ): Promise<Map<string, Buffer | null>> {- // Write stream to a temporary file (like Python's BytesIO)- const tarFile = join(tmpdir(), `tar-test-${Date.now()}.tar.gz`)- const writeStream = createWriteStream(tarFile)- await pipeline(stream, writeStream)-- const contents = new Map<string, Buffer | null>()+ async function waitUntilGone(target: string): Promise<void> {+ for (let i = 0; i < 100; i++) {+ try {+ await access(target)+ } catch {+ return+ }+ await new Promise((resolve) => setTimeout(resolve, 10))+ }+ throw new Error(`path was not removed: ${target}`)+ }- // Extract files to a temp directory and collect members- const extractDir = join(tmpdir(), `tar-extract-${Date.now()}`)+ /**+ * Pipe an archive stream into tar's extractor and collect its entries.+ */+ async function extractTarStream(+ stream: Readable+ ): Promise<{ extractDir: string; members: ReadEntry[] }> {+ const extractDir = join(+ tmpdir(),+ `tar-extract-${Date.now()}-${Math.random()}`+ )await mkdir(extractDir, { recursive: true })const members: ReadEntry[] = []- await tar.extract({- file: tarFile,- cwd: extractDir,- gzip: true,- onentry: (entry: ReadEntry) => {- members.push(entry)- },- })-- // Read file contents+ await pipeline(+ stream,+ tar.extract({+ cwd: extractDir,+ gzip: true,+ onentry: (entry: ReadEntry) => {+ members.push(entry)+ },+ })+ )++ return { extractDir, members }+ }++ /**+ * Extract archive contents into a map of path -> file contents.+ */+ async function extractTarContents(+ stream: Readable+ ): Promise<Map<string, Buffer | null>> {+ const { extractDir, members } = await extractTarStream(stream)+ const contents = new Map<string, Buffer | null>()+for (const member of members) {if (member.type === 'File') {- const filePath = join(extractDir, member.path)try {- const content = await readFile(filePath)- contents.set(member.path, content)+ contents.set(+ member.path,+ await readFile(join(extractDir, member.path))+ )} catch {// File might not exist}@@ -69,76 +103,55 @@ describe('tarFileStream', () => {}}- // Cleanup- await rm(tarFile, { force: true })await rm(extractDir, { recursive: true, force: true })-return contents}/**- * Get tar members+ * Extract archive members into a map of path -> entry.*/- async function getTarMembers(stream: any): Promise<Map<string, ReadEntry>> {- // Write stream to a temporary file- const tarFile = join(tmpdir(), `tar-test-${Date.now()}.tar.gz`)- const writeStream = createWriteStream(tarFile)- await pipeline(stream, writeStream)-- const members = new Map<string, ReadEntry>()-- const extractDir = join(tmpdir(), `tar-extract-${Date.now()}`)- await mkdir(extractDir, { recursive: true })-- await tar.extract({- file: tarFile,- cwd: extractDir,- gzip: true,- onentry: (entry: ReadEntry) => {- members.set(entry.path, entry)- },- })-- // Cleanup- await rm(tarFile, { force: true })+ async function getTarMembers(+ stream: Readable+ ): Promise<Map<string, ReadEntry>> {+ const { extractDir, members } = await extractTarStream(stream)+ const map = new Map<string, ReadEntry>()+ for (const member of members) {+ map.set(member.path, member)+ }await rm(extractDir, { recursive: true, force: true })-- return members+ return map}test('should create tar with simple files', async () => {- // Create test filesawait writeFile(join(testDir, 'file1.txt'), 'content1')await writeFile(join(testDir, 'file2.txt'), 'content2')- const stream = await tarFileStream('*.txt', testDir, [], false)+ const { stream, size } = await tarFileStream('*.txt', testDir, [], false)+ expect(size).toBeGreaterThan(0)+const contents = await extractTarContents(stream)expect(contents.size).toBe(2)- expect(contents.has('file1.txt')).toBe(true)- expect(contents.has('file2.txt')).toBe(true)expect(contents.get('file1.txt')?.toString()).toBe('content1')expect(contents.get('file2.txt')?.toString()).toBe('content2')})test('should respect ignore patterns', async () => {- // Create test filesawait writeFile(join(testDir, 'file1.txt'), 'content1')await writeFile(join(testDir, 'file2.txt'), 'content2')await writeFile(join(testDir, 'temp.txt'), 'temp content')await writeFile(join(testDir, 'backup.txt'), 'backup content')- const stream = await tarFileStream(+ const { stream } = await tarFileStream('*.txt',testDir,['temp*', 'backup*'],false)+const contents = await extractTarContents(stream)expect(contents.size).toBe(2)- expect(contents.has('file1.txt')).toBe(true)- expect(contents.has('file2.txt')).toBe(true)expect(contents.get('file1.txt')?.toString()).toBe('content1')expect(contents.get('file2.txt')?.toString()).toBe('content2')expect(contents.has('temp.txt')).toBe(false)@@ -146,17 +159,15 @@ describe('tarFileStream', () => {})test('should handle nested files', async () => {- // Create nested directory structureconst nestedDir = join(testDir, 'src', 'components')await mkdir(nestedDir, { recursive: true })await writeFile(join(testDir, 'src', 'index.ts'), 'index content')await writeFile(join(nestedDir, 'Button.tsx'), 'button content')- const stream = await tarFileStream('src', testDir, [], false)- const contents = await extractTarContents(stream)+ const { stream } = await tarFileStream('src', testDir, [], false)- // Should include the directory and files+ const contents = await extractTarContents(stream)const paths = Array.from(contents.keys())expect(paths.some((p) => p.includes('src'))).toBe(true)expect(paths.some((p) => p.includes('index.ts'))).toBe(true)@@ -164,14 +175,10 @@ describe('tarFileStream', () => {})test('should resolve symlinks when enabled', async () => {- // Create original file- const originalPath = join(testDir, 'original.txt')- await writeFile(originalPath, 'original content')+ await writeFile(join(testDir, 'original.txt'), 'original content')- // Create symlink- const symlinkPath = join(testDir, 'link.txt')try {- await symlink('original.txt', symlinkPath)+ await symlink('original.txt', join(testDir, 'link.txt'))} catch (error: any) {// Skip test if symlinks are not supported on this platformif (error.code === 'ENOSYS' || error.code === 'EPERM') {@@ -180,27 +187,19 @@ describe('tarFileStream', () => {throw error}- // Test with resolveSymlinks=true- const stream = await tarFileStream('*.txt', testDir, [], true)- const contents = await extractTarContents(stream)+ const { stream } = await tarFileStream('*.txt', testDir, [], true)- // Both files should be in tar- expect(contents.has('original.txt')).toBe(true)- expect(contents.has('link.txt')).toBe(true)- // Symlink should be resolved (contain actual content, not link)+ const contents = await extractTarContents(stream)expect(contents.get('original.txt')?.toString()).toBe('original content')+ // Symlink should be resolved (contain actual content, not link)expect(contents.get('link.txt')?.toString()).toBe('original content')})test('should preserve symlinks when disabled', async () => {- // Create original file- const originalPath = join(testDir, 'original.txt')- await writeFile(originalPath, 'original content')+ await writeFile(join(testDir, 'original.txt'), 'original content')- // Create symlink- const symlinkPath = join(testDir, 'link.txt')try {- await symlink('original.txt', symlinkPath)+ await symlink('original.txt', join(testDir, 'link.txt'))} catch (error: any) {// Skip test if symlinks are not supported on this platformif (error.code === 'ENOSYS' || error.code === 'EPERM') {@@ -209,21 +208,65 @@ describe('tarFileStream', () => {throw error}- // Test with resolveSymlinks=false- const stream = await tarFileStream('*.txt', testDir, [], false)- const members = await getTarMembers(stream)-- // Both files should be in tar- expect(members.has('original.txt')).toBe(true)- expect(members.has('link.txt')).toBe(true)+ const { stream } = await tarFileStream('*.txt', testDir, [], false)- // Original should be a regular file- const original = members.get('original.txt')!- expect(original.type).toBe('File')-- // Link should be a symlink+ const members = await getTarMembers(stream)+ expect(members.get('original.txt')?.type).toBe('File')const link = members.get('link.txt')!expect(link.type).toBe('SymbolicLink')expect(link.linkpath).toBe('original.txt')})++ test('removes the spooled archive once the stream is consumed', async () => {+ await writeFile(join(testDir, 'file1.txt'), 'content1')++ // Capture the temp dir the helper creates so we can assert it is gone.+ let tmpDir: string | undefined+ const mkdtemp = fs.promises.mkdtemp.bind(fs.promises)+ const spy = vi+ .spyOn(fs.promises, 'mkdtemp')+ .mockImplementation(async (prefix: any, ...rest: any[]) => {+ const dir = await mkdtemp(prefix, ...rest)+ tmpDir = dir+ return dir+ })++ try {+ const { stream } = await tarFileStream('*.txt', testDir, [], false)+ expect(tmpDir).toBeDefined()+ await access(tmpDir!)++ // Drain the stream to completion; the `close` handler then removes the+ // spooled archive.+ stream.resume()+ await waitUntilGone(tmpDir!)+ } finally {+ spy.mockRestore()+ }+ })++ test('cleans up the spooled archive if it is never consumed', async () => {+ await writeFile(join(testDir, 'file1.txt'), 'content1')++ let tmpDir: string | undefined+ const mkdtemp = fs.promises.mkdtemp.bind(fs.promises)+ const spy = vi+ .spyOn(fs.promises, 'mkdtemp')+ .mockImplementation(async (prefix: any, ...rest: any[]) => {+ const dir = await mkdtemp(prefix, ...rest)+ tmpDir = dir+ return dir+ })++ try {+ const { stream } = await tarFileStream('*.txt', testDir, [], false)+ await access(tmpDir!)++ // Destroying without reading still fires `close` and removes the file.+ stream.destroy()+ await waitUntilGone(tmpDir!)+ } finally {+ spy.mockRestore()+ }+ })})diff --git a/packages/python-sdk/e2b/template/consts.py b/packages/python-sdk/e2b/template/consts.pyindex 41d6cb879..eb0070671 100644--- a/packages/python-sdk/e2b/template/consts.py+++ b/packages/python-sdk/e2b/template/consts.py@@ -28,3 +28,12 @@ Default setting for whether to resolve symbolic links when copying files.When False, symlinks are copied as symlinks rather than following them."""RESOLVE_SYMLINKS = False++"""+Default timeout (in seconds) for uploading the build-context archive to the+S3 presigned URL. Uploads of large archives can take far longer than the 60s+general API request timeout, so the upload uses a 1-hour default unless the+caller passes an explicit ``request_timeout``. This matches the JS SDK's+``FILE_UPLOAD_TIMEOUT_MS``.+"""+FILE_UPLOAD_TIMEOUT_SECONDS = 3600diff --git a/packages/python-sdk/e2b/template/utils.py b/packages/python-sdk/e2b/template/utils.pyindex 2902dbca8..cdfc62547 100644--- a/packages/python-sdk/e2b/template/utils.py+++ b/packages/python-sdk/e2b/template/utils.py@@ -1,14 +1,14 @@import hashlibimport os-import ioimport tarfile+import tempfileimport jsonimport statfrom wcmatch import globimport reimport inspectfrom types import TracebackType, FrameType-from typing import List, Optional, Union+from typing import IO, List, Optional, Unionfrom e2b.exceptions import TemplateExceptionfrom e2b.template.consts import BASE_STEP_NAME, FINALIZE_STEP_NAME@@ -259,32 +259,47 @@ def tar_file_stream(file_context_path: str,ignore_patterns: List[str],resolve_symlinks: bool,-) -> io.BytesIO:+) -> IO[bytes]:"""- Create a tar stream of files matching a pattern.+ Create a tar archive of files matching a pattern in a temporary file.++ The archive is spooled to disk so it can be uploaded as a stream instead+ of being buffered in memory. The temporary file is deleted when closed.:param file_name: Glob pattern for files to include:param file_context_path: Base directory for resolving file paths:param ignore_patterns: Ignore patterns:param resolve_symlinks: Whether to resolve symbolic links- :return: Tar stream+ :return: Binary file object positioned at the start of the archive"""- tar_buffer = io.BytesIO()- with tarfile.open(- fileobj=tar_buffer,- mode="w:gz",- dereference=resolve_symlinks,- ) as tar:- files = get_all_files_in_path(- file_name, file_context_path, ignore_patterns, True- )- for file in files:- tar.add(- file, arcname=os.path.relpath(file, file_context_path), recursive=False+ tar_file = tempfile.TemporaryFile()+ try:+ with tarfile.open(+ fileobj=tar_file,+ mode="w:gz",+ dereference=resolve_symlinks,+ ) as tar:+ files = get_all_files_in_path(+ file_name, file_context_path, ignore_patterns, True)-- return tar_buffer+ for file in files:+ tar.add(+ file,+ arcname=os.path.relpath(file, file_context_path),+ recursive=False,+ )++ tar_file.seek(0)+ return tar_file+ except Exception:+ # Best-effort cleanup: a close failure must not replace the real+ # archive-creation error.+ try:+ tar_file.close()+ except Exception:+ pass+ raisedef strip_ansi_escape_codes(text: str) -> str:diff --git a/packages/python-sdk/e2b/template_async/build_api.py b/packages/python-sdk/e2b/template_async/build_api.pyindex 44eada8cc..d306cea17 100644--- a/packages/python-sdk/e2b/template_async/build_api.py+++ b/packages/python-sdk/e2b/template_async/build_api.py@@ -1,10 +1,12 @@import asyncio+import osfrom types import TracebackTypefrom typing import Callable, Optional, List, Unionimport httpxfrom e2b.api import handle_api_exception+from e2b.io_utils import aiter_io_chunksfrom e2b.api.client.api.templates import (post_v3_templates,get_templates_template_id_files_hash,@@ -37,6 +39,7 @@ from e2b.template.types import (TemplateTag,TemplateTagInfo,)+from e2b.template.consts import FILE_UPLOAD_TIMEOUT_SECONDSfrom e2b.template.utils import get_build_step_index, tar_file_stream@@ -105,21 +108,46 @@ async def upload_file(ignore_patterns: List[str],resolve_symlinks: bool,stack_trace: Optional[TracebackType],+ request_timeout: Optional[float] = None,):+ # Uploading a large build-context archive can take far longer than the 60s+ # general API timeout, so default to a 1-hour upload timeout unless the+ # caller set an explicit request_timeout. Matches the JS SDK+ # (FILE_UPLOAD_TIMEOUT_MS).+ upload_timeout = (+ request_timeout if request_timeout is not None else FILE_UPLOAD_TIMEOUT_SECONDS+ )try:- tar_buffer = tar_file_stream(+ tar_file = tar_file_stream(file_name, context_path, ignore_patterns, resolve_symlinks)-- async with httpx.AsyncClient(- timeout=api_client._timeout,- verify=api_client._verify_ssl,- follow_redirects=api_client._follow_redirects,- proxy=getattr(api_client, "_proxy", None),- http2=False,- ) as client:- response = await client.put(url, content=tar_buffer.getvalue())- response.raise_for_status()+ try:+ size = os.fstat(tar_file.fileno()).st_size++ async with httpx.AsyncClient(+ timeout=httpx.Timeout(upload_timeout),+ verify=api_client._verify_ssl,+ follow_redirects=api_client._follow_redirects,+ proxy=getattr(api_client, "_proxy", None),+ http2=False,+ ) as client:+ # Stream the archive from disk via an async iterator. The+ # explicit Content-Length suppresses chunked transfer+ # encoding, which S3 presigned URLs reject.+ response = await client.put(+ url,+ content=aiter_io_chunks(tar_file),+ headers={"Content-Length": str(size)},+ )+ response.raise_for_status()+ finally:+ # Closing the spooled temp file is best-effort: a failure here+ # must not mask a successful upload as a FileUploadException,+ # nor overwrite a real upload error.+ try:+ tar_file.close()+ except Exception:+ passexcept httpx.HTTPStatusError as e:raise FileUploadException(f"Failed to upload file: {e}").with_traceback(stack_tracediff --git a/packages/python-sdk/e2b/template_async/main.py b/packages/python-sdk/e2b/template_async/main.pyindex 8324c5ca0..78d80c94c 100644--- a/packages/python-sdk/e2b/template_async/main.py+++ b/packages/python-sdk/e2b/template_async/main.py@@ -41,6 +41,7 @@ class AsyncTemplate(TemplateBase):memory_mb: int = 1024,skip_cache: bool = False,on_build_logs: Optional[Callable[[LogEntry], None]] = None,+ request_timeout: Optional[float] = None,) -> BuildInfo:"""Internal implementation of the template build process@@ -127,6 +128,7 @@ class AsyncTemplate(TemplateBase):],resolve_symlinks,stack_trace,+ request_timeout=request_timeout,)if on_build_logs:on_build_logs(@@ -248,6 +250,9 @@ class AsyncTemplate(TemplateBase):memory_mb=memory_mb,skip_cache=skip_cache,on_build_logs=on_build_logs,+ # Only honor an explicitly set request_timeout for uploads;+ # otherwise upload_file applies its 1-hour default.+ request_timeout=opts.get("request_timeout"),)if on_build_logs:@@ -337,6 +342,9 @@ class AsyncTemplate(TemplateBase):memory_mb=memory_mb,skip_cache=skip_cache,on_build_logs=on_build_logs,+ # Only honor an explicitly set request_timeout for uploads;+ # otherwise upload_file applies its 1-hour default.+ request_timeout=opts.get("request_timeout"),)@staticmethoddiff --git a/packages/python-sdk/e2b/template_sync/build_api.py b/packages/python-sdk/e2b/template_sync/build_api.pyindex 257492928..243fc81dc 100644--- a/packages/python-sdk/e2b/template_sync/build_api.py+++ b/packages/python-sdk/e2b/template_sync/build_api.py@@ -37,6 +37,7 @@ from e2b.template.types import (TemplateTag,TemplateTagInfo,)+from e2b.template.consts import FILE_UPLOAD_TIMEOUT_SECONDSfrom e2b.template.utils import get_build_step_index, tar_file_stream@@ -105,20 +106,40 @@ def upload_file(ignore_patterns: List[str],resolve_symlinks: bool,stack_trace: Optional[TracebackType],+ request_timeout: Optional[float] = None,):+ # Uploading a large build-context archive can take far longer than the 60s+ # general API timeout, so default to a 1-hour upload timeout unless the+ # caller set an explicit request_timeout. Matches the JS SDK+ # (FILE_UPLOAD_TIMEOUT_MS).+ upload_timeout = (+ request_timeout if request_timeout is not None else FILE_UPLOAD_TIMEOUT_SECONDS+ )try:- tar_buffer = tar_file_stream(+ tar_file = tar_file_stream(file_name, context_path, ignore_patterns, resolve_symlinks)- with httpx.Client(- timeout=api_client._timeout,- verify=api_client._verify_ssl,- follow_redirects=api_client._follow_redirects,- proxy=getattr(api_client, "_proxy", None),- http2=False,- ) as client:- response = client.put(url, content=tar_buffer.getvalue())- response.raise_for_status()+ try:+ with httpx.Client(+ timeout=httpx.Timeout(upload_timeout),+ verify=api_client._verify_ssl,+ follow_redirects=api_client._follow_redirects,+ proxy=getattr(api_client, "_proxy", None),+ http2=False,+ ) as client:+ # httpx streams the archive from disk in chunks and sets+ # Content-Length from the file size—S3 presigned URLs reject+ # chunked transfer encoding.+ response = client.put(url, content=tar_file)+ response.raise_for_status()+ finally:+ # Closing the spooled temp file is best-effort: a failure here+ # must not mask a successful upload as a FileUploadException,+ # nor overwrite a real upload error.+ try:+ tar_file.close()+ except Exception:+ passexcept httpx.HTTPStatusError as e:raise FileUploadException(f"Failed to upload file: {e}").with_traceback(stack_tracediff --git a/packages/python-sdk/e2b/template_sync/main.py b/packages/python-sdk/e2b/template_sync/main.pyindex d1d4f35f5..94414de2f 100644--- a/packages/python-sdk/e2b/template_sync/main.py+++ b/packages/python-sdk/e2b/template_sync/main.py@@ -41,6 +41,7 @@ class Template(TemplateBase):memory_mb: int = 1024,skip_cache: bool = False,on_build_logs: Optional[Callable[[LogEntry], None]] = None,+ request_timeout: Optional[float] = None,) -> BuildInfo:"""Internal implementation of the template build process@@ -127,6 +128,7 @@ class Template(TemplateBase):],resolve_symlinks,stack_trace,+ request_timeout=request_timeout,)if on_build_logs:on_build_logs(@@ -248,6 +250,9 @@ class Template(TemplateBase):memory_mb=memory_mb,skip_cache=skip_cache,on_build_logs=on_build_logs,+ # Only honor an explicitly set request_timeout for uploads;+ # otherwise upload_file applies its 1-hour default.+ request_timeout=opts.get("request_timeout"),)if on_build_logs:@@ -337,6 +342,9 @@ class Template(TemplateBase):memory_mb=memory_mb,skip_cache=skip_cache,on_build_logs=on_build_logs,+ # Only honor an explicitly set request_timeout for uploads;+ # otherwise upload_file applies its 1-hour default.+ request_timeout=opts.get("request_timeout"),)@staticmethoddiff --git a/packages/python-sdk/tests/async/template_async/test_upload_file.py b/packages/python-sdk/tests/async/template_async/test_upload_file.pyindex 472a4be54..d4f4b0414 100644--- a/packages/python-sdk/tests/async/template_async/test_upload_file.py+++ b/packages/python-sdk/tests/async/template_async/test_upload_file.py@@ -1,15 +1,20 @@import threadingfrom http.server import BaseHTTPRequestHandler, HTTPServer+from unittest import mock++import httpxfrom e2b.api.client.client import AuthenticatedClient+from e2b.template import utils as template_utils+from e2b.template.consts import FILE_UPLOAD_TIMEOUT_SECONDSfrom e2b.template_async.build_api import upload_file# Regression test for e2b-dev/e2b#1243 — upload_file must set Content-Length# and must not fall back to Transfer-Encoding: chunked. S3 presigned PUT URLs-# reject chunked encoding with 501 NotImplemented. httpx sets Content-Length-# automatically when we pass bytes (tar_buffer.getvalue()); this test guards-# against someone swapping the bytes for a generator/stream later.+# reject chunked encoding with 501 NotImplemented. The archive is streamed+# from a temporary file on disk with a known Content-Length instead of being+# buffered in memory; this test guards that contract.## The mock server runs in a daemon thread and doesn't need to be async — the# httpx.AsyncClient connects to it via asyncio sockets without blocking the@@ -75,3 +80,115 @@ async def test_upload_file_sets_content_length_and_no_chunked_encoding(tmp_path)if transfer_encoding is not None:assert "chunked" not in transfer_encoding.lower()assert "Authorization" not in state["headers"]+++async def _capture_upload_timeout(tmp_path, request_timeout=None):+ """Run upload_file against a local server, capturing the httpx timeout."""+ (tmp_path / "hello.txt").write_text("hello world")++ server, thread, state = _make_server()+ host, port = server.server_address+ url = f"http://{host}:{port}/upload"++ captured = {}+ real_client = httpx.AsyncClient++ def spy_client(*args, **kwargs):+ captured["timeout"] = kwargs.get("timeout")+ return real_client(*args, **kwargs)++ try:+ client = AuthenticatedClient(base_url="http://test", token="test")+ kwargs = {}+ if request_timeout is not None:+ kwargs["request_timeout"] = request_timeout+ with mock.patch(+ "e2b.template_async.build_api.httpx.AsyncClient", side_effect=spy_client+ ):+ await upload_file(+ api_client=client,+ file_name="*.txt",+ context_path=str(tmp_path),+ url=url,+ ignore_patterns=[],+ resolve_symlinks=False,+ stack_trace=None,+ **kwargs,+ )+ finally:+ server.shutdown()+ server.server_close()+ thread.join(timeout=5)++ return captured["timeout"]+++async def test_upload_file_defaults_to_one_hour_timeout(tmp_path):+ # Uploads of large archives need far longer than the 60s general API+ # timeout, so the default upload timeout is 1 hour (matches the JS SDK).+ timeout = await _capture_upload_timeout(tmp_path)+ assert timeout == httpx.Timeout(FILE_UPLOAD_TIMEOUT_SECONDS)+++async def test_upload_file_honors_explicit_request_timeout(tmp_path):+ # An explicitly set request_timeout overrides the 1-hour upload default.+ timeout = await _capture_upload_timeout(tmp_path, request_timeout=5.0)+ assert timeout == httpx.Timeout(5.0)+++async def test_upload_file_ignores_post_upload_close_failure(tmp_path):+ # Regression test: once S3 has accepted the archive, closing the spooled+ # temp file in the `finally` block can raise. That failure must not be+ # wrapped as a FileUploadException — the upload already succeeded.+ (tmp_path / "hello.txt").write_text("hello world")++ server, thread, state = _make_server()+ host, port = server.server_address+ url = f"http://{host}:{port}/upload"++ real_tar_file_stream = template_utils.tar_file_stream++ class _FailingCloseFile:+ # Proxies a real spooled temp file but raises on close(). The+ # underlying file object is a C type whose `close` attribute can't+ # be reassigned, so we wrap it instead.+ def __init__(self, inner):+ self._inner = inner++ def __getattr__(self, name):+ return getattr(self._inner, name)++ def __iter__(self):+ return iter(self._inner)++ def close(self):+ # Run the real close so we don't leak the temp file, then+ # simulate a close failure surfacing from the `finally`.+ self._inner.close()+ raise OSError("close failed")++ def failing_close_stream(*args, **kwargs):+ return _FailingCloseFile(real_tar_file_stream(*args, **kwargs))++ try:+ client = AuthenticatedClient(base_url="http://test", token="test")+ with mock.patch(+ "e2b.template_async.build_api.tar_file_stream",+ side_effect=failing_close_stream,+ ):+ # Must not raise despite close() failing after a 200 response.+ await upload_file(+ api_client=client,+ file_name="*.txt",+ context_path=str(tmp_path),+ url=url,+ ignore_patterns=[],+ resolve_symlinks=False,+ stack_trace=None,+ )+ finally:+ server.shutdown()+ server.server_close()+ thread.join(timeout=5)++ assert state["headers"] is not Nonediff --git a/packages/python-sdk/tests/shared/template/utils/test_tar_file_stream.py b/packages/python-sdk/tests/shared/template/utils/test_tar_file_stream.pyindex 672b937e9..b274e4955 100644--- a/packages/python-sdk/tests/shared/template/utils/test_tar_file_stream.py+++ b/packages/python-sdk/tests/shared/template/utils/test_tar_file_stream.py@@ -1,8 +1,8 @@import osimport tempfileimport tarfile-import ioimport pytest+from typing import IOfrom e2b.template.utils import tar_file_stream@@ -14,7 +14,7 @@ class TestTarFileStream:yield tmpdir.nametmpdir.cleanup()- def _extract_tar_contents(self, tar_buffer: io.BytesIO) -> dict:+ def _extract_tar_contents(self, tar_buffer: IO[bytes]) -> dict:"""Extract tar contents into a dictionary mapping paths to file contents."""tar_buffer.seek(0)contents = {}diff --git a/packages/python-sdk/tests/sync/template_sync/test_upload_file.py b/packages/python-sdk/tests/sync/template_sync/test_upload_file.pyindex 609e3178d..e2dd9a83a 100644--- a/packages/python-sdk/tests/sync/template_sync/test_upload_file.py+++ b/packages/python-sdk/tests/sync/template_sync/test_upload_file.py@@ -1,15 +1,20 @@import threadingfrom http.server import BaseHTTPRequestHandler, HTTPServer+from unittest import mock++import httpxfrom e2b.api.client.client import AuthenticatedClient+from e2b.template import utils as template_utils+from e2b.template.consts import FILE_UPLOAD_TIMEOUT_SECONDSfrom e2b.template_sync.build_api import upload_file# Regression test for e2b-dev/e2b#1243 — upload_file must set Content-Length# and must not fall back to Transfer-Encoding: chunked. S3 presigned PUT URLs-# reject chunked encoding with 501 NotImplemented. httpx sets Content-Length-# automatically when we pass bytes (tar_buffer.getvalue()); this test guards-# against someone swapping the bytes for a generator/stream later.+# reject chunked encoding with 501 NotImplemented. The archive is streamed+# from a temporary file on disk with a known Content-Length instead of being+# buffered in memory; this test guards that contract.def _make_server():@@ -71,3 +76,115 @@ def test_upload_file_sets_content_length_and_no_chunked_encoding(tmp_path):if transfer_encoding is not None:assert "chunked" not in transfer_encoding.lower()assert "Authorization" not in state["headers"]+++def _capture_upload_timeout(tmp_path, request_timeout=None):+ """Run upload_file against a local server, capturing the httpx timeout."""+ (tmp_path / "hello.txt").write_text("hello world")++ server, thread, state = _make_server()+ host, port = server.server_address+ url = f"http://{host}:{port}/upload"++ captured = {}+ real_client = httpx.Client++ def spy_client(*args, **kwargs):+ captured["timeout"] = kwargs.get("timeout")+ return real_client(*args, **kwargs)++ try:+ client = AuthenticatedClient(base_url="http://test", token="test")+ kwargs = {}+ if request_timeout is not None:+ kwargs["request_timeout"] = request_timeout+ with mock.patch(+ "e2b.template_sync.build_api.httpx.Client", side_effect=spy_client+ ):+ upload_file(+ api_client=client,+ file_name="*.txt",+ context_path=str(tmp_path),+ url=url,+ ignore_patterns=[],+ resolve_symlinks=False,+ stack_trace=None,+ **kwargs,+ )+ finally:+ server.shutdown()+ server.server_close()+ thread.join(timeout=5)++ return captured["timeout"]+++def test_upload_file_defaults_to_one_hour_timeout(tmp_path):+ # Uploads of large archives need far longer than the 60s general API+ # timeout, so the default upload timeout is 1 hour (matches the JS SDK).+ timeout = _capture_upload_timeout(tmp_path)+ assert timeout == httpx.Timeout(FILE_UPLOAD_TIMEOUT_SECONDS)+++def test_upload_file_honors_explicit_request_timeout(tmp_path):+ # An explicitly set request_timeout overrides the 1-hour upload default.+ timeout = _capture_upload_timeout(tmp_path, request_timeout=5.0)+ assert timeout == httpx.Timeout(5.0)+++def test_upload_file_ignores_post_upload_close_failure(tmp_path):+ # Regression test: once S3 has accepted the archive, closing the spooled+ # temp file in the `finally` block can raise. That failure must not be+ # wrapped as a FileUploadException — the upload already succeeded.+ (tmp_path / "hello.txt").write_text("hello world")++ server, thread, state = _make_server()+ host, port = server.server_address+ url = f"http://{host}:{port}/upload"++ real_tar_file_stream = template_utils.tar_file_stream++ class _FailingCloseFile:+ # Proxies a real spooled temp file but raises on close(). The+ # underlying file object is a C type whose `close` attribute can't+ # be reassigned, so we wrap it instead.+ def __init__(self, inner):+ self._inner = inner++ def __getattr__(self, name):+ return getattr(self._inner, name)++ def __iter__(self):+ return iter(self._inner)++ def close(self):+ # Run the real close so we don't leak the temp file, then+ # simulate a close failure surfacing from the `finally`.+ self._inner.close()+ raise OSError("close failed")++ def failing_close_stream(*args, **kwargs):+ return _FailingCloseFile(real_tar_file_stream(*args, **kwargs))++ try:+ client = AuthenticatedClient(base_url="http://test", token="test")+ with mock.patch(+ "e2b.template_sync.build_api.tar_file_stream",+ side_effect=failing_close_stream,+ ):+ # Must not raise despite close() failing after a 200 response.+ upload_file(+ api_client=client,+ file_name="*.txt",+ context_path=str(tmp_path),+ url=url,+ ignore_patterns=[],+ resolve_symlinks=False,+ stack_trace=None,+ )+ finally:+ server.shutdown()+ server.server_close()+ thread.join(timeout=5)++ assert state["headers"] is not None