Skip to content

armgit / E2B

Public Imported from GitHub
0

Open-source, secure environment with real-world tools for enterprise-grade agents.

feat: use application/octet-stream for sandbox file uploads (#1242)

Mish Ushakovcf35f61b4464
diff --git a/.changeset/moody-baths-dig.md b/.changeset/moody-baths-dig.md
new file mode 100644
index 000000000..6f6f2555c
--- /dev/null
+++ b/.changeset/moody-baths-dig.md
@@ -0,0 +1,6 @@
+---
+'@e2b/python-sdk': minor
+'e2b': minor
+---
+
+switch to application/octet-stream for file uploads
diff --git a/packages/js-sdk/src/envd/versions.ts b/packages/js-sdk/src/envd/versions.ts
index 16fffb3a4..6348ff675 100644
--- a/packages/js-sdk/src/envd/versions.ts
+++ b/packages/js-sdk/src/envd/versions.ts
@@ -3,3 +3,4 @@ export const ENVD_DEBUG_FALLBACK = '99.99.99'
export const ENVD_COMMANDS_STDIN = '0.3.0'
export const ENVD_DEFAULT_USER = '0.4.0'
export const ENVD_ENVD_CLOSE = '0.5.2'
+export const ENVD_OCTET_STREAM_UPLOAD = '0.5.7'
diff --git a/packages/js-sdk/src/sandbox/filesystem/index.ts b/packages/js-sdk/src/sandbox/filesystem/index.ts
index 5a15602ec..cf58160c7 100644
--- a/packages/js-sdk/src/sandbox/filesystem/index.ts
+++ b/packages/js-sdk/src/sandbox/filesystem/index.ts
@@ -29,6 +29,7 @@ import type { Timestamp } from '@bufbuild/protobuf/wkt'
import { compareVersions } from 'compare-versions'
import {
ENVD_DEFAULT_USER,
+ ENVD_OCTET_STREAM_UPLOAD,
ENVD_VERSION_RECURSIVE_WATCH,
} from '../../envd/versions'
import {
@@ -380,12 +381,6 @@ export class Filesystem {
if (writeFiles.length === 0) return [] as WriteInfo[]
- const formData = new FormData()
- for (let i = 0; i < writeFiles.length; i++) {
- const file = writeFiles[i]
- formData.append('file', await toBlob(file.data), writeFiles[i].path)
- }
-
let user = writeOpts?.user
if (
user == undefined &&
@@ -394,29 +389,89 @@ export class Filesystem {
user = defaultUsername
}
- const res = await this.envdApi.api.POST('/files', {
- params: {
- query: {
- path,
- username: user,
+ const useOctetStream =
+ compareVersions(this.envdApi.version, ENVD_OCTET_STREAM_UPLOAD) >= 0
+
+ const results: WriteInfo[] = []
+
+ if (useOctetStream) {
+ const uploadResults = await Promise.all(
+ writeFiles.map(async (file) => {
+ const filePath = path ?? (file as WriteEntry).path
+ const blob = await toBlob(file.data)
+
+ const res = await this.envdApi.api.POST('/files', {
+ params: {
+ query: {
+ path: filePath,
+ username: user,
+ },
+ },
+ bodySerializer: () => blob,
+ headers: {
+ 'Content-Type': 'application/octet-stream',
+ },
+ signal: this.connectionConfig.getSignal(
+ writeOpts?.requestTimeoutMs
+ ),
+ body: {},
+ })
+
+ const err = await handleFilesystemEnvdApiError(res)
+ if (err) {
+ throw err
+ }
+
+ const files = res.data as WriteInfo[]
+ if (!files || files.length === 0) {
+ throw new Error(
+ 'Expected to receive information about written file'
+ )
+ }
+
+ return files
+ })
+ )
+
+ for (const files of uploadResults) {
+ results.push(...files)
+ }
+ } else {
+ const formData = new FormData()
+ for (const file of writeFiles) {
+ formData.append(
+ 'file',
+ await toBlob(file.data),
+ (file as WriteEntry).path ?? path!
+ )
+ }
+
+ const res = await this.envdApi.api.POST('/files', {
+ params: {
+ query: {
+ path,
+ username: user,
+ },
},
- },
- bodySerializer: () => formData,
- signal: this.connectionConfig.getSignal(opts?.requestTimeoutMs),
- body: {},
- })
+ bodySerializer: () => formData,
+ signal: this.connectionConfig.getSignal(writeOpts?.requestTimeoutMs),
+ body: {},
+ })
+
+ const err = await handleFilesystemEnvdApiError(res)
+ if (err) {
+ throw err
+ }
- const err = await handleFilesystemEnvdApiError(res)
- if (err) {
- throw err
- }
+ const files = res.data as WriteInfo[]
+ if (!files || files.length === 0) {
+ throw new Error('Expected to receive information about written file')
+ }
- const files = res.data as WriteInfo[]
- if (!files) {
- throw new Error('Expected to receive information about written file')
+ results.push(...files)
}
- return files.length === 1 && path ? files[0] : files
+ return results.length === 1 && path ? results[0] : results
}
/**
diff --git a/packages/python-sdk/e2b/envd/versions.py b/packages/python-sdk/e2b/envd/versions.py
index f0674b530..94e9e238c 100644
--- a/packages/python-sdk/e2b/envd/versions.py
+++ b/packages/python-sdk/e2b/envd/versions.py
@@ -5,3 +5,4 @@ ENVD_DEBUG_FALLBACK = Version("99.99.99")
ENVD_COMMANDS_STDIN = Version("0.3.0")
ENVD_DEFAULT_USER = Version("0.4.0")
ENVD_ENVD_CLOSE = Version("0.5.2")
+ENVD_OCTET_STREAM_UPLOAD = Version("0.5.7")
diff --git a/packages/python-sdk/e2b/sandbox_async/filesystem/filesystem.py b/packages/python-sdk/e2b/sandbox_async/filesystem/filesystem.py
index f0c4c1cbc..bfaf5492f 100644
--- a/packages/python-sdk/e2b/sandbox_async/filesystem/filesystem.py
+++ b/packages/python-sdk/e2b/sandbox_async/filesystem/filesystem.py
@@ -1,3 +1,4 @@
+import asyncio
from io import IOBase, TextIOBase
from typing import IO, AsyncIterator, List, Literal, Optional, Union, overload
@@ -16,7 +17,11 @@ from e2b.connection_config import (
from e2b.envd.api import ENVD_API_FILES_ROUTE, ahandle_envd_api_exception
from e2b.envd.filesystem import filesystem_connect, filesystem_pb2
from e2b.envd.rpc import authentication_header, handle_rpc_exception
-from e2b.envd.versions import ENVD_DEFAULT_USER, ENVD_VERSION_RECURSIVE_WATCH
+from e2b.envd.versions import (
+ ENVD_DEFAULT_USER,
+ ENVD_OCTET_STREAM_UPLOAD,
+ ENVD_VERSION_RECURSIVE_WATCH,
+)
from e2b.exceptions import (
FileNotFoundException,
InvalidArgumentException,
@@ -223,51 +228,108 @@ class Filesystem:
if username is None and self._envd_version < ENVD_DEFAULT_USER:
username = default_username
- params = {}
- if username:
- params["username"] = username
- if len(files) == 1:
- params["path"] = files[0]["path"]
-
- # Prepare the files for the multipart/form-data request
- httpx_files = []
- for file in files:
- file_path, file_data = file["path"], file["data"]
- if isinstance(file_data, (str, bytes)):
- # str and bytes can be passed directly
- httpx_files.append(("file", (file_path, file_data)))
- elif isinstance(file_data, TextIOBase):
- # Text streams must be read first
- httpx_files.append(("file", (file_path, file_data.read())))
- elif isinstance(file_data, IOBase):
- # Binary streams can be passed directly
- httpx_files.append(("file", (file_path, file_data)))
- else:
- raise InvalidArgumentException(
- f"Unsupported data type for file {file_path}"
+ if len(files) == 0:
+ return []
+
+ use_octet_stream = self._envd_version >= ENVD_OCTET_STREAM_UPLOAD
+
+ results: List[WriteInfo] = []
+
+ if use_octet_stream:
+
+ async def _upload_file(file):
+ file_path, file_data = file["path"], file["data"]
+
+ if isinstance(file_data, str):
+ content = file_data.encode("utf-8")
+ elif isinstance(file_data, bytes):
+ content = file_data
+ elif isinstance(file_data, TextIOBase):
+ content = file_data.read().encode("utf-8")
+ elif isinstance(file_data, IOBase):
+ content = file_data.read()
+ else:
+ raise InvalidArgumentException(
+ f"Unsupported data type for file {file_path}"
+ )
+
+ params = {"path": file_path}
+ if username:
+ params["username"] = username
+
+ r = await self._envd_api.post(
+ ENVD_API_FILES_ROUTE,
+ content=content,
+ headers={"Content-Type": "application/octet-stream"},
+ params=params,
+ timeout=self._connection_config.get_request_timeout(
+ request_timeout
+ ),
)
- # Allow passing empty list of files
- if len(httpx_files) == 0:
- return []
+ err = await _ahandle_filesystem_envd_api_exception(r)
+ if err:
+ raise err
- r = await self._envd_api.post(
- ENVD_API_FILES_ROUTE,
- files=httpx_files,
- params=params,
- timeout=self._connection_config.get_request_timeout(request_timeout),
- )
+ write_result = r.json()
- err = await _ahandle_filesystem_envd_api_exception(r)
- if err:
- raise err
+ if not isinstance(write_result, list) or len(write_result) == 0:
+ raise SandboxException(
+ "Expected to receive information about written file"
+ )
+
+ return [WriteInfo(**f) for f in write_result]
+
+ upload_results = await asyncio.gather(
+ *[_upload_file(file) for file in files]
+ )
+ for file_results in upload_results:
+ results.extend(file_results)
+ else:
+ params = {}
+ if username:
+ params["username"] = username
+ if len(files) == 1:
+ params["path"] = files[0]["path"]
+
+ httpx_files = []
+ for file in files:
+ file_path, file_data = file["path"], file["data"]
+ if isinstance(file_data, (str, bytes)):
+ httpx_files.append(("file", (file_path, file_data)))
+ elif isinstance(file_data, TextIOBase):
+ httpx_files.append(("file", (file_path, file_data.read())))
+ elif isinstance(file_data, IOBase):
+ httpx_files.append(("file", (file_path, file_data)))
+ else:
+ raise InvalidArgumentException(
+ f"Unsupported data type for file {file_path}"
+ )
- write_files = r.json()
+ if len(httpx_files) == 0:
+ return []
+
+ r = await self._envd_api.post(
+ ENVD_API_FILES_ROUTE,
+ files=httpx_files,
+ params=params,
+ timeout=self._connection_config.get_request_timeout(request_timeout),
+ )
+
+ err = await _ahandle_filesystem_envd_api_exception(r)
+ if err:
+ raise err
+
+ write_result = r.json()
+
+ if not isinstance(write_result, list) or len(write_result) == 0:
+ raise SandboxException(
+ "Expected to receive information about written file"
+ )
- if not isinstance(write_files, list) or len(write_files) == 0:
- raise SandboxException("Expected to receive information about written file")
+ results.extend([WriteInfo(**f) for f in write_result])
- return [WriteInfo(**file) for file in write_files]
+ return results
async def list(
self,
diff --git a/packages/python-sdk/e2b/sandbox_sync/filesystem/filesystem.py b/packages/python-sdk/e2b/sandbox_sync/filesystem/filesystem.py
index a1d271f27..7548a9e27 100644
--- a/packages/python-sdk/e2b/sandbox_sync/filesystem/filesystem.py
+++ b/packages/python-sdk/e2b/sandbox_sync/filesystem/filesystem.py
@@ -18,7 +18,11 @@ from e2b_connect.client import Code
from e2b.envd.api import ENVD_API_FILES_ROUTE, handle_envd_api_exception
from e2b.envd.filesystem import filesystem_connect, filesystem_pb2
from e2b.envd.rpc import authentication_header, handle_rpc_exception
-from e2b.envd.versions import ENVD_DEFAULT_USER, ENVD_VERSION_RECURSIVE_WATCH
+from e2b.envd.versions import (
+ ENVD_DEFAULT_USER,
+ ENVD_OCTET_STREAM_UPLOAD,
+ ENVD_VERSION_RECURSIVE_WATCH,
+)
from e2b.exceptions import (
FileNotFoundException,
InvalidArgumentException,
@@ -223,51 +227,101 @@ class Filesystem:
if username is None and self._envd_version < ENVD_DEFAULT_USER:
username = default_username
- params = {}
- if username:
- params["username"] = username
- if len(files) == 1:
- params["path"] = files[0]["path"]
-
- # Prepare the files for the multipart/form-data request
- httpx_files = []
- for file in files:
- file_path, file_data = file["path"], file["data"]
- if isinstance(file_data, (str, bytes)):
- # str and bytes can be passed directly
- httpx_files.append(("file", (file_path, file_data)))
- elif isinstance(file_data, TextIOBase):
- # Text streams must be read first
- httpx_files.append(("file", (file_path, file_data.read())))
- elif isinstance(file_data, IOBase):
- # Binary streams can be passed directly
- httpx_files.append(("file", (file_path, file_data)))
- else:
- raise InvalidArgumentException(
- f"Unsupported data type for file {file_path}"
+ if len(files) == 0:
+ return []
+
+ use_octet_stream = self._envd_version >= ENVD_OCTET_STREAM_UPLOAD
+
+ results: List[WriteInfo] = []
+
+ if use_octet_stream:
+ for file in files:
+ file_path, file_data = file["path"], file["data"]
+
+ if isinstance(file_data, str):
+ content = file_data.encode("utf-8")
+ elif isinstance(file_data, bytes):
+ content = file_data
+ elif isinstance(file_data, TextIOBase):
+ content = file_data.read().encode("utf-8")
+ elif isinstance(file_data, IOBase):
+ content = file_data.read()
+ else:
+ raise InvalidArgumentException(
+ f"Unsupported data type for file {file_path}"
+ )
+
+ params = {"path": file_path}
+ if username:
+ params["username"] = username
+
+ r = self._envd_api.post(
+ ENVD_API_FILES_ROUTE,
+ content=content,
+ headers={"Content-Type": "application/octet-stream"},
+ params=params,
+ timeout=self._connection_config.get_request_timeout(
+ request_timeout
+ ),
)
- # Allow passing empty list of files
- if len(httpx_files) == 0:
- return []
+ err = _handle_filesystem_envd_api_exception(r)
+ if err:
+ raise err
- r = self._envd_api.post(
- ENVD_API_FILES_ROUTE,
- files=httpx_files,
- params=params,
- timeout=self._connection_config.get_request_timeout(request_timeout),
- )
+ write_result = r.json()
- err = _handle_filesystem_envd_api_exception(r)
- if err:
- raise err
+ if not isinstance(write_result, list) or len(write_result) == 0:
+ raise SandboxException(
+ "Expected to receive information about written file"
+ )
+
+ results.extend([WriteInfo(**f) for f in write_result])
+ else:
+ params = {}
+ if username:
+ params["username"] = username
+ if len(files) == 1:
+ params["path"] = files[0]["path"]
+
+ httpx_files = []
+ for file in files:
+ file_path, file_data = file["path"], file["data"]
+ if isinstance(file_data, (str, bytes)):
+ httpx_files.append(("file", (file_path, file_data)))
+ elif isinstance(file_data, TextIOBase):
+ httpx_files.append(("file", (file_path, file_data.read())))
+ elif isinstance(file_data, IOBase):
+ httpx_files.append(("file", (file_path, file_data)))
+ else:
+ raise InvalidArgumentException(
+ f"Unsupported data type for file {file_path}"
+ )
- write_files = r.json()
+ if len(httpx_files) == 0:
+ return []
+
+ r = self._envd_api.post(
+ ENVD_API_FILES_ROUTE,
+ files=httpx_files,
+ params=params,
+ timeout=self._connection_config.get_request_timeout(request_timeout),
+ )
+
+ err = _handle_filesystem_envd_api_exception(r)
+ if err:
+ raise err
+
+ write_result = r.json()
+
+ if not isinstance(write_result, list) or len(write_result) == 0:
+ raise SandboxException(
+ "Expected to receive information about written file"
+ )
- if not isinstance(write_files, list) or len(write_files) == 0:
- raise SandboxException("Expected to receive information about written file")
+ results.extend([WriteInfo(**f) for f in write_result])
- return [WriteInfo(**file) for file in write_files]
+ return results
def list(
self,