Open-source, secure environment with real-world tools for enterprise-grade agents.
Python agentaiai-agentai-agentscode-interpretercopilotdevelopmentdevtoolsgptgpt-4javascriptllmnextjsopenaipythonreactsoftwaretypescript
feat: use application/octet-stream for sandbox file uploads (#1242)
Mish Ushakovcf35f61b4464
diff --git a/.changeset/moody-baths-dig.md b/.changeset/moody-baths-dig.mdnew file mode 100644index 000000000..6f6f2555c--- /dev/null+++ b/.changeset/moody-baths-dig.md@@ -0,0 +1,6 @@+---+'@e2b/python-sdk': minor+'e2b': minor+---++switch to application/octet-stream for file uploadsdiff --git a/packages/js-sdk/src/envd/versions.ts b/packages/js-sdk/src/envd/versions.tsindex 16fffb3a4..6348ff675 100644--- a/packages/js-sdk/src/envd/versions.ts+++ b/packages/js-sdk/src/envd/versions.ts@@ -3,3 +3,4 @@ export const ENVD_DEBUG_FALLBACK = '99.99.99'export const ENVD_COMMANDS_STDIN = '0.3.0'export const ENVD_DEFAULT_USER = '0.4.0'export const ENVD_ENVD_CLOSE = '0.5.2'+export const ENVD_OCTET_STREAM_UPLOAD = '0.5.7'diff --git a/packages/js-sdk/src/sandbox/filesystem/index.ts b/packages/js-sdk/src/sandbox/filesystem/index.tsindex 5a15602ec..cf58160c7 100644--- a/packages/js-sdk/src/sandbox/filesystem/index.ts+++ b/packages/js-sdk/src/sandbox/filesystem/index.ts@@ -29,6 +29,7 @@ import type { Timestamp } from '@bufbuild/protobuf/wkt'import { compareVersions } from 'compare-versions'import {ENVD_DEFAULT_USER,+ ENVD_OCTET_STREAM_UPLOAD,ENVD_VERSION_RECURSIVE_WATCH,} from '../../envd/versions'import {@@ -380,12 +381,6 @@ export class Filesystem {if (writeFiles.length === 0) return [] as WriteInfo[]- const formData = new FormData()- for (let i = 0; i < writeFiles.length; i++) {- const file = writeFiles[i]- formData.append('file', await toBlob(file.data), writeFiles[i].path)- }-let user = writeOpts?.userif (user == undefined &&@@ -394,29 +389,89 @@ export class Filesystem {user = defaultUsername}- const res = await this.envdApi.api.POST('/files', {- params: {- query: {- path,- username: user,+ const useOctetStream =+ compareVersions(this.envdApi.version, ENVD_OCTET_STREAM_UPLOAD) >= 0++ const results: WriteInfo[] = []++ if (useOctetStream) {+ const uploadResults = await Promise.all(+ writeFiles.map(async (file) => {+ const filePath = path ?? (file as WriteEntry).path+ const blob = await toBlob(file.data)++ const res = await this.envdApi.api.POST('/files', {+ params: {+ query: {+ path: filePath,+ username: user,+ },+ },+ bodySerializer: () => blob,+ headers: {+ 'Content-Type': 'application/octet-stream',+ },+ signal: this.connectionConfig.getSignal(+ writeOpts?.requestTimeoutMs+ ),+ body: {},+ })++ const err = await handleFilesystemEnvdApiError(res)+ if (err) {+ throw err+ }++ const files = res.data as WriteInfo[]+ if (!files || files.length === 0) {+ throw new Error(+ 'Expected to receive information about written file'+ )+ }++ return files+ })+ )++ for (const files of uploadResults) {+ results.push(...files)+ }+ } else {+ const formData = new FormData()+ for (const file of writeFiles) {+ formData.append(+ 'file',+ await toBlob(file.data),+ (file as WriteEntry).path ?? path!+ )+ }++ const res = await this.envdApi.api.POST('/files', {+ params: {+ query: {+ path,+ username: user,+ },},- },- bodySerializer: () => formData,- signal: this.connectionConfig.getSignal(opts?.requestTimeoutMs),- body: {},- })+ bodySerializer: () => formData,+ signal: this.connectionConfig.getSignal(writeOpts?.requestTimeoutMs),+ body: {},+ })++ const err = await handleFilesystemEnvdApiError(res)+ if (err) {+ throw err+ }- const err = await handleFilesystemEnvdApiError(res)- if (err) {- throw err- }+ const files = res.data as WriteInfo[]+ if (!files || files.length === 0) {+ throw new Error('Expected to receive information about written file')+ }- const files = res.data as WriteInfo[]- if (!files) {- throw new Error('Expected to receive information about written file')+ results.push(...files)}- return files.length === 1 && path ? files[0] : files+ return results.length === 1 && path ? results[0] : results}/**diff --git a/packages/python-sdk/e2b/envd/versions.py b/packages/python-sdk/e2b/envd/versions.pyindex f0674b530..94e9e238c 100644--- a/packages/python-sdk/e2b/envd/versions.py+++ b/packages/python-sdk/e2b/envd/versions.py@@ -5,3 +5,4 @@ ENVD_DEBUG_FALLBACK = Version("99.99.99")ENVD_COMMANDS_STDIN = Version("0.3.0")ENVD_DEFAULT_USER = Version("0.4.0")ENVD_ENVD_CLOSE = Version("0.5.2")+ENVD_OCTET_STREAM_UPLOAD = Version("0.5.7")diff --git a/packages/python-sdk/e2b/sandbox_async/filesystem/filesystem.py b/packages/python-sdk/e2b/sandbox_async/filesystem/filesystem.pyindex f0c4c1cbc..bfaf5492f 100644--- a/packages/python-sdk/e2b/sandbox_async/filesystem/filesystem.py+++ b/packages/python-sdk/e2b/sandbox_async/filesystem/filesystem.py@@ -1,3 +1,4 @@+import asynciofrom io import IOBase, TextIOBasefrom typing import IO, AsyncIterator, List, Literal, Optional, Union, overload@@ -16,7 +17,11 @@ from e2b.connection_config import (from e2b.envd.api import ENVD_API_FILES_ROUTE, ahandle_envd_api_exceptionfrom e2b.envd.filesystem import filesystem_connect, filesystem_pb2from e2b.envd.rpc import authentication_header, handle_rpc_exception-from e2b.envd.versions import ENVD_DEFAULT_USER, ENVD_VERSION_RECURSIVE_WATCH+from e2b.envd.versions import (+ ENVD_DEFAULT_USER,+ ENVD_OCTET_STREAM_UPLOAD,+ ENVD_VERSION_RECURSIVE_WATCH,+)from e2b.exceptions import (FileNotFoundException,InvalidArgumentException,@@ -223,51 +228,108 @@ class Filesystem:if username is None and self._envd_version < ENVD_DEFAULT_USER:username = default_username- params = {}- if username:- params["username"] = username- if len(files) == 1:- params["path"] = files[0]["path"]-- # Prepare the files for the multipart/form-data request- httpx_files = []- for file in files:- file_path, file_data = file["path"], file["data"]- if isinstance(file_data, (str, bytes)):- # str and bytes can be passed directly- httpx_files.append(("file", (file_path, file_data)))- elif isinstance(file_data, TextIOBase):- # Text streams must be read first- httpx_files.append(("file", (file_path, file_data.read())))- elif isinstance(file_data, IOBase):- # Binary streams can be passed directly- httpx_files.append(("file", (file_path, file_data)))- else:- raise InvalidArgumentException(- f"Unsupported data type for file {file_path}"+ if len(files) == 0:+ return []++ use_octet_stream = self._envd_version >= ENVD_OCTET_STREAM_UPLOAD++ results: List[WriteInfo] = []++ if use_octet_stream:++ async def _upload_file(file):+ file_path, file_data = file["path"], file["data"]++ if isinstance(file_data, str):+ content = file_data.encode("utf-8")+ elif isinstance(file_data, bytes):+ content = file_data+ elif isinstance(file_data, TextIOBase):+ content = file_data.read().encode("utf-8")+ elif isinstance(file_data, IOBase):+ content = file_data.read()+ else:+ raise InvalidArgumentException(+ f"Unsupported data type for file {file_path}"+ )++ params = {"path": file_path}+ if username:+ params["username"] = username++ r = await self._envd_api.post(+ ENVD_API_FILES_ROUTE,+ content=content,+ headers={"Content-Type": "application/octet-stream"},+ params=params,+ timeout=self._connection_config.get_request_timeout(+ request_timeout+ ),)- # Allow passing empty list of files- if len(httpx_files) == 0:- return []+ err = await _ahandle_filesystem_envd_api_exception(r)+ if err:+ raise err- r = await self._envd_api.post(- ENVD_API_FILES_ROUTE,- files=httpx_files,- params=params,- timeout=self._connection_config.get_request_timeout(request_timeout),- )+ write_result = r.json()- err = await _ahandle_filesystem_envd_api_exception(r)- if err:- raise err+ if not isinstance(write_result, list) or len(write_result) == 0:+ raise SandboxException(+ "Expected to receive information about written file"+ )++ return [WriteInfo(**f) for f in write_result]++ upload_results = await asyncio.gather(+ *[_upload_file(file) for file in files]+ )+ for file_results in upload_results:+ results.extend(file_results)+ else:+ params = {}+ if username:+ params["username"] = username+ if len(files) == 1:+ params["path"] = files[0]["path"]++ httpx_files = []+ for file in files:+ file_path, file_data = file["path"], file["data"]+ if isinstance(file_data, (str, bytes)):+ httpx_files.append(("file", (file_path, file_data)))+ elif isinstance(file_data, TextIOBase):+ httpx_files.append(("file", (file_path, file_data.read())))+ elif isinstance(file_data, IOBase):+ httpx_files.append(("file", (file_path, file_data)))+ else:+ raise InvalidArgumentException(+ f"Unsupported data type for file {file_path}"+ )- write_files = r.json()+ if len(httpx_files) == 0:+ return []++ r = await self._envd_api.post(+ ENVD_API_FILES_ROUTE,+ files=httpx_files,+ params=params,+ timeout=self._connection_config.get_request_timeout(request_timeout),+ )++ err = await _ahandle_filesystem_envd_api_exception(r)+ if err:+ raise err++ write_result = r.json()++ if not isinstance(write_result, list) or len(write_result) == 0:+ raise SandboxException(+ "Expected to receive information about written file"+ )- if not isinstance(write_files, list) or len(write_files) == 0:- raise SandboxException("Expected to receive information about written file")+ results.extend([WriteInfo(**f) for f in write_result])- return [WriteInfo(**file) for file in write_files]+ return resultsasync def list(self,diff --git a/packages/python-sdk/e2b/sandbox_sync/filesystem/filesystem.py b/packages/python-sdk/e2b/sandbox_sync/filesystem/filesystem.pyindex a1d271f27..7548a9e27 100644--- a/packages/python-sdk/e2b/sandbox_sync/filesystem/filesystem.py+++ b/packages/python-sdk/e2b/sandbox_sync/filesystem/filesystem.py@@ -18,7 +18,11 @@ from e2b_connect.client import Codefrom e2b.envd.api import ENVD_API_FILES_ROUTE, handle_envd_api_exceptionfrom e2b.envd.filesystem import filesystem_connect, filesystem_pb2from e2b.envd.rpc import authentication_header, handle_rpc_exception-from e2b.envd.versions import ENVD_DEFAULT_USER, ENVD_VERSION_RECURSIVE_WATCH+from e2b.envd.versions import (+ ENVD_DEFAULT_USER,+ ENVD_OCTET_STREAM_UPLOAD,+ ENVD_VERSION_RECURSIVE_WATCH,+)from e2b.exceptions import (FileNotFoundException,InvalidArgumentException,@@ -223,51 +227,101 @@ class Filesystem:if username is None and self._envd_version < ENVD_DEFAULT_USER:username = default_username- params = {}- if username:- params["username"] = username- if len(files) == 1:- params["path"] = files[0]["path"]-- # Prepare the files for the multipart/form-data request- httpx_files = []- for file in files:- file_path, file_data = file["path"], file["data"]- if isinstance(file_data, (str, bytes)):- # str and bytes can be passed directly- httpx_files.append(("file", (file_path, file_data)))- elif isinstance(file_data, TextIOBase):- # Text streams must be read first- httpx_files.append(("file", (file_path, file_data.read())))- elif isinstance(file_data, IOBase):- # Binary streams can be passed directly- httpx_files.append(("file", (file_path, file_data)))- else:- raise InvalidArgumentException(- f"Unsupported data type for file {file_path}"+ if len(files) == 0:+ return []++ use_octet_stream = self._envd_version >= ENVD_OCTET_STREAM_UPLOAD++ results: List[WriteInfo] = []++ if use_octet_stream:+ for file in files:+ file_path, file_data = file["path"], file["data"]++ if isinstance(file_data, str):+ content = file_data.encode("utf-8")+ elif isinstance(file_data, bytes):+ content = file_data+ elif isinstance(file_data, TextIOBase):+ content = file_data.read().encode("utf-8")+ elif isinstance(file_data, IOBase):+ content = file_data.read()+ else:+ raise InvalidArgumentException(+ f"Unsupported data type for file {file_path}"+ )++ params = {"path": file_path}+ if username:+ params["username"] = username++ r = self._envd_api.post(+ ENVD_API_FILES_ROUTE,+ content=content,+ headers={"Content-Type": "application/octet-stream"},+ params=params,+ timeout=self._connection_config.get_request_timeout(+ request_timeout+ ),)- # Allow passing empty list of files- if len(httpx_files) == 0:- return []+ err = _handle_filesystem_envd_api_exception(r)+ if err:+ raise err- r = self._envd_api.post(- ENVD_API_FILES_ROUTE,- files=httpx_files,- params=params,- timeout=self._connection_config.get_request_timeout(request_timeout),- )+ write_result = r.json()- err = _handle_filesystem_envd_api_exception(r)- if err:- raise err+ if not isinstance(write_result, list) or len(write_result) == 0:+ raise SandboxException(+ "Expected to receive information about written file"+ )++ results.extend([WriteInfo(**f) for f in write_result])+ else:+ params = {}+ if username:+ params["username"] = username+ if len(files) == 1:+ params["path"] = files[0]["path"]++ httpx_files = []+ for file in files:+ file_path, file_data = file["path"], file["data"]+ if isinstance(file_data, (str, bytes)):+ httpx_files.append(("file", (file_path, file_data)))+ elif isinstance(file_data, TextIOBase):+ httpx_files.append(("file", (file_path, file_data.read())))+ elif isinstance(file_data, IOBase):+ httpx_files.append(("file", (file_path, file_data)))+ else:+ raise InvalidArgumentException(+ f"Unsupported data type for file {file_path}"+ )- write_files = r.json()+ if len(httpx_files) == 0:+ return []++ r = self._envd_api.post(+ ENVD_API_FILES_ROUTE,+ files=httpx_files,+ params=params,+ timeout=self._connection_config.get_request_timeout(request_timeout),+ )++ err = _handle_filesystem_envd_api_exception(r)+ if err:+ raise err++ write_result = r.json()++ if not isinstance(write_result, list) or len(write_result) == 0:+ raise SandboxException(+ "Expected to receive information about written file"+ )- if not isinstance(write_files, list) or len(write_files) == 0:- raise SandboxException("Expected to receive information about written file")+ results.extend([WriteInfo(**f) for f in write_result])- return [WriteInfo(**file) for file in write_files]+ return resultsdef list(self,