AutoGPT is the vision of accessible AI for everyone, to use and to build on. Our mission is to provide the tools, so that you can focus on what matters.
feat(blocks): Store sandbox files to workspace (#12073)
Nicholas Tindlecb166dd6fb80
diff --git a/autogpt_platform/backend/backend/blocks/claude_code.py b/autogpt_platform/backend/backend/blocks/claude_code.pyindex 1919406c6f..2e870f02b6 100644--- a/autogpt_platform/backend/backend/blocks/claude_code.py+++ b/autogpt_platform/backend/backend/blocks/claude_code.py@@ -1,10 +1,10 @@import jsonimport shleximport uuid-from typing import Literal, Optional+from typing import TYPE_CHECKING, Literal, Optionalfrom e2b import AsyncSandbox as BaseAsyncSandbox-from pydantic import BaseModel, SecretStr+from pydantic import SecretStrfrom backend.blocks._base import (Block,@@ -20,6 +20,13 @@ from backend.data.model import (SchemaField,)from backend.integrations.providers import ProviderName+from backend.util.sandbox_files import (+ SandboxFileOutput,+ extract_and_store_sandbox_files,+)++if TYPE_CHECKING:+ from backend.executor.utils import ExecutionContextclass ClaudeCodeExecutionError(Exception):@@ -174,22 +181,15 @@ class ClaudeCodeBlock(Block):advanced=True,)- class FileOutput(BaseModel):- """A file extracted from the sandbox."""-- path: str- relative_path: str # Path relative to working directory (for GitHub, etc.)- name: str- content: str-class Output(BlockSchemaOutput):response: str = SchemaField(description="The output/response from Claude Code execution")- files: list["ClaudeCodeBlock.FileOutput"] = SchemaField(+ files: list[SandboxFileOutput] = SchemaField(description=("List of text files created/modified by Claude Code during this execution. "- "Each file has 'path', 'relative_path', 'name', and 'content' fields."+ "Each file has 'path', 'relative_path', 'name', 'content', and 'workspace_ref' fields. "+ "workspace_ref contains a workspace:// URI if the file was stored to workspace."))conversation_history: str = SchemaField(@@ -252,6 +252,7 @@ class ClaudeCodeBlock(Block):"relative_path": "index.html","name": "index.html","content": "<html>Hello World</html>",+ "workspace_ref": None,}],),@@ -267,11 +268,12 @@ class ClaudeCodeBlock(Block):"execute_claude_code": lambda *args, **kwargs: ("Created index.html with hello world content", # response[- ClaudeCodeBlock.FileOutput(+ SandboxFileOutput(path="/home/user/index.html",relative_path="index.html",name="index.html",content="<html>Hello World</html>",+ workspace_ref=None,)], # files"User: Create a hello world HTML file\n"@@ -294,7 +296,8 @@ class ClaudeCodeBlock(Block):existing_sandbox_id: str,conversation_history: str,dispose_sandbox: bool,- ) -> tuple[str, list["ClaudeCodeBlock.FileOutput"], str, str, str]:+ execution_context: "ExecutionContext",+ ) -> tuple[str, list[SandboxFileOutput], str, str, str]:"""Execute Claude Code in an E2B sandbox.@@ -449,14 +452,18 @@ class ClaudeCodeBlock(Block):else:new_conversation_history = turn_entry- # Extract files created/modified during this run- files = await self._extract_files(- sandbox, working_directory, start_timestamp+ # Extract files created/modified during this run and store to workspace+ sandbox_files = await extract_and_store_sandbox_files(+ sandbox=sandbox,+ working_directory=working_directory,+ execution_context=execution_context,+ since_timestamp=start_timestamp,+ text_only=True,)return (response,- files,+ sandbox_files, # Already SandboxFileOutput objectsnew_conversation_history,current_session_id,sandbox_id,@@ -471,140 +478,6 @@ class ClaudeCodeBlock(Block):if dispose_sandbox and sandbox:await sandbox.kill()- async def _extract_files(- self,- sandbox: BaseAsyncSandbox,- working_directory: str,- since_timestamp: str | None = None,- ) -> list["ClaudeCodeBlock.FileOutput"]:- """- Extract text files created/modified during this Claude Code execution.-- Args:- sandbox: The E2B sandbox instance- working_directory: Directory to search for files- since_timestamp: ISO timestamp - only return files modified after this time-- Returns:- List of FileOutput objects with path, relative_path, name, and content- """- files: list[ClaudeCodeBlock.FileOutput] = []-- # Text file extensions we can safely read as text- text_extensions = {- ".txt",- ".md",- ".html",- ".htm",- ".css",- ".js",- ".ts",- ".jsx",- ".tsx",- ".json",- ".xml",- ".yaml",- ".yml",- ".toml",- ".ini",- ".cfg",- ".conf",- ".py",- ".rb",- ".php",- ".java",- ".c",- ".cpp",- ".h",- ".hpp",- ".cs",- ".go",- ".rs",- ".swift",- ".kt",- ".scala",- ".sh",- ".bash",- ".zsh",- ".sql",- ".graphql",- ".env",- ".gitignore",- ".dockerfile",- "Dockerfile",- ".vue",- ".svelte",- ".astro",- ".mdx",- ".rst",- ".tex",- ".csv",- ".log",- }-- try:- # List files recursively using find command- # Exclude node_modules and .git directories, but allow hidden files- # like .env and .gitignore (they're filtered by text_extensions later)- # Filter by timestamp to only get files created/modified during this run- safe_working_dir = shlex.quote(working_directory)- timestamp_filter = ""- if since_timestamp:- timestamp_filter = f"-newermt {shlex.quote(since_timestamp)} "- find_result = await sandbox.commands.run(- f"find {safe_working_dir} -type f "- f"{timestamp_filter}"- f"-not -path '*/node_modules/*' "- f"-not -path '*/.git/*' "- f"2>/dev/null"- )-- if find_result.stdout:- for file_path in find_result.stdout.strip().split("\n"):- if not file_path:- continue-- # Check if it's a text file we can read- is_text = any(- file_path.endswith(ext) for ext in text_extensions- ) or file_path.endswith("Dockerfile")-- if is_text:- try:- content = await sandbox.files.read(file_path)- # Handle bytes or string- if isinstance(content, bytes):- content = content.decode("utf-8", errors="replace")-- # Extract filename from path- file_name = file_path.split("/")[-1]-- # Calculate relative path by stripping working directory- relative_path = file_path- if file_path.startswith(working_directory):- relative_path = file_path[len(working_directory) :]- # Remove leading slash if present- if relative_path.startswith("/"):- relative_path = relative_path[1:]-- files.append(- ClaudeCodeBlock.FileOutput(- path=file_path,- relative_path=relative_path,- name=file_name,- content=content,- )- )- except Exception:- # Skip files that can't be read- pass-- except Exception:- # If file extraction fails, return empty results- pass-- return files-def _escape_prompt(self, prompt: str) -> str:"""Escape the prompt for safe shell execution."""# Use single quotes and escape any single quotes in the prompt@@ -617,6 +490,7 @@ class ClaudeCodeBlock(Block):*,e2b_credentials: APIKeyCredentials,anthropic_credentials: APIKeyCredentials,+ execution_context: "ExecutionContext",**kwargs,) -> BlockOutput:try:@@ -637,6 +511,7 @@ class ClaudeCodeBlock(Block):existing_sandbox_id=input_data.sandbox_id,conversation_history=input_data.conversation_history,dispose_sandbox=input_data.dispose_sandbox,+ execution_context=execution_context,)yield "response", responsediff --git a/autogpt_platform/backend/backend/blocks/code_executor.py b/autogpt_platform/backend/backend/blocks/code_executor.pyindex 766f44b7bb..26bf9acd4f 100644--- a/autogpt_platform/backend/backend/blocks/code_executor.py+++ b/autogpt_platform/backend/backend/blocks/code_executor.py@@ -1,5 +1,5 @@from enum import Enum-from typing import Any, Literal, Optional+from typing import TYPE_CHECKING, Any, Literal, Optionalfrom e2b_code_interpreter import AsyncSandboxfrom e2b_code_interpreter import Result as E2BExecutionResult@@ -20,6 +20,13 @@ from backend.data.model import (SchemaField,)from backend.integrations.providers import ProviderName+from backend.util.sandbox_files import (+ SandboxFileOutput,+ extract_and_store_sandbox_files,+)++if TYPE_CHECKING:+ from backend.executor.utils import ExecutionContextTEST_CREDENTIALS = APIKeyCredentials(id="01234567-89ab-cdef-0123-456789abcdef",@@ -85,6 +92,9 @@ class CodeExecutionResult(MainCodeExecutionResult):class BaseE2BExecutorMixin:"""Shared implementation methods for E2B executor blocks."""+ # Default working directory in E2B sandboxes+ WORKING_DIR = "/home/user"+async def execute_code(self,api_key: str,@@ -95,14 +105,21 @@ class BaseE2BExecutorMixin:timeout: Optional[int] = None,sandbox_id: Optional[str] = None,dispose_sandbox: bool = False,+ execution_context: Optional["ExecutionContext"] = None,+ extract_files: bool = False,):"""Unified code execution method that handles all three use cases:1. Create new sandbox and execute (ExecuteCodeBlock)2. Create new sandbox, execute, and return sandbox_id (InstantiateCodeSandboxBlock)3. Connect to existing sandbox and execute (ExecuteCodeStepBlock)++ Args:+ extract_files: If True and execution_context provided, extract files+ created/modified during execution and store to workspace.""" # noqasandbox = None+ files: list[SandboxFileOutput] = []try:if sandbox_id:# Connect to existing sandbox (ExecuteCodeStepBlock case)@@ -118,6 +135,12 @@ class BaseE2BExecutorMixin:for cmd in setup_commands:await sandbox.commands.run(cmd)+ # Capture timestamp before execution to scope file extraction+ start_timestamp = None+ if extract_files:+ ts_result = await sandbox.commands.run("date -u +%Y-%m-%dT%H:%M:%S")+ start_timestamp = ts_result.stdout.strip() if ts_result.stdout else None+# Execute the codeexecution = await sandbox.run_code(code,@@ -133,7 +156,24 @@ class BaseE2BExecutorMixin:stdout_logs = "".join(execution.logs.stdout)stderr_logs = "".join(execution.logs.stderr)- return results, text_output, stdout_logs, stderr_logs, sandbox.sandbox_id+ # Extract files created/modified during this execution+ if extract_files and execution_context:+ files = await extract_and_store_sandbox_files(+ sandbox=sandbox,+ working_directory=self.WORKING_DIR,+ execution_context=execution_context,+ since_timestamp=start_timestamp,+ text_only=False, # Include binary files too+ )++ return (+ results,+ text_output,+ stdout_logs,+ stderr_logs,+ sandbox.sandbox_id,+ files,+ )finally:# Dispose of sandbox if requested to reduce usage costsif dispose_sandbox and sandbox:@@ -238,6 +278,12 @@ class ExecuteCodeBlock(Block, BaseE2BExecutorMixin):description="Standard output logs from execution")stderr_logs: str = SchemaField(description="Standard error logs from execution")+ files: list[SandboxFileOutput] = SchemaField(+ description=(+ "Files created or modified during execution. "+ "Each file has path, name, content, and workspace_ref (if stored)."+ ),+ )def __init__(self):super().__init__(@@ -259,23 +305,30 @@ class ExecuteCodeBlock(Block, BaseE2BExecutorMixin):("results", []),("response", "Hello World"),("stdout_logs", "Hello World\n"),+ ("files", []),],test_mock={- "execute_code": lambda api_key, code, language, template_id, setup_commands, timeout, dispose_sandbox: ( # noqa+ "execute_code": lambda api_key, code, language, template_id, setup_commands, timeout, dispose_sandbox, execution_context, extract_files: ( # noqa[], # results"Hello World", # text_output"Hello World\n", # stdout_logs"", # stderr_logs"sandbox_id", # sandbox_id+ [], # files),},)async def run(- self, input_data: Input, *, credentials: APIKeyCredentials, **kwargs+ self,+ input_data: Input,+ *,+ credentials: APIKeyCredentials,+ execution_context: "ExecutionContext",+ **kwargs,) -> BlockOutput:try:- results, text_output, stdout, stderr, _ = await self.execute_code(+ results, text_output, stdout, stderr, _, files = await self.execute_code(api_key=credentials.api_key.get_secret_value(),code=input_data.code,language=input_data.language,@@ -283,6 +336,8 @@ class ExecuteCodeBlock(Block, BaseE2BExecutorMixin):setup_commands=input_data.setup_commands,timeout=input_data.timeout,dispose_sandbox=input_data.dispose_sandbox,+ execution_context=execution_context,+ extract_files=True,)# Determine result object shape & filter out empty formats@@ -296,6 +351,8 @@ class ExecuteCodeBlock(Block, BaseE2BExecutorMixin):yield "stdout_logs", stdoutif stderr:yield "stderr_logs", stderr+ # Always yield files (empty list if none)+ yield "files", [f.model_dump() for f in files]except Exception as e:yield "error", str(e)@@ -393,6 +450,7 @@ class InstantiateCodeSandboxBlock(Block, BaseE2BExecutorMixin):"Hello World\n", # stdout_logs"", # stderr_logs"sandbox_id", # sandbox_id+ [], # files),},)@@ -401,7 +459,7 @@ class InstantiateCodeSandboxBlock(Block, BaseE2BExecutorMixin):self, input_data: Input, *, credentials: APIKeyCredentials, **kwargs) -> BlockOutput:try:- _, text_output, stdout, stderr, sandbox_id = await self.execute_code(+ _, text_output, stdout, stderr, sandbox_id, _ = await self.execute_code(api_key=credentials.api_key.get_secret_value(),code=input_data.setup_code,language=input_data.language,@@ -500,6 +558,7 @@ class ExecuteCodeStepBlock(Block, BaseE2BExecutorMixin):"Hello World\n", # stdout_logs"", # stderr_logssandbox_id, # sandbox_id+ [], # files),},)@@ -508,7 +567,7 @@ class ExecuteCodeStepBlock(Block, BaseE2BExecutorMixin):self, input_data: Input, *, credentials: APIKeyCredentials, **kwargs) -> BlockOutput:try:- results, text_output, stdout, stderr, _ = await self.execute_code(+ results, text_output, stdout, stderr, _, _ = await self.execute_code(api_key=credentials.api_key.get_secret_value(),code=input_data.step_code,language=input_data.language,diff --git a/autogpt_platform/backend/backend/util/sandbox_files.py b/autogpt_platform/backend/backend/util/sandbox_files.pynew file mode 100644index 0000000000..9db53ded14--- /dev/null+++ b/autogpt_platform/backend/backend/util/sandbox_files.py@@ -0,0 +1,288 @@+"""+Shared utilities for extracting and storing files from E2B sandboxes.++This module provides common file extraction and workspace storage functionality+for blocks that run code in E2B sandboxes (Claude Code, Code Executor, etc.).+"""++import base64+import logging+import mimetypes+import shlex+from dataclasses import dataclass+from typing import TYPE_CHECKING++from pydantic import BaseModel++from backend.util.file import store_media_file+from backend.util.type import MediaFileType++if TYPE_CHECKING:+ from e2b import AsyncSandbox as BaseAsyncSandbox++ from backend.executor.utils import ExecutionContext++logger = logging.getLogger(__name__)++# Text file extensions that can be safely read and stored as text+TEXT_EXTENSIONS = {+ ".txt",+ ".md",+ ".html",+ ".htm",+ ".css",+ ".js",+ ".ts",+ ".jsx",+ ".tsx",+ ".json",+ ".xml",+ ".yaml",+ ".yml",+ ".toml",+ ".ini",+ ".cfg",+ ".conf",+ ".py",+ ".rb",+ ".php",+ ".java",+ ".c",+ ".cpp",+ ".h",+ ".hpp",+ ".cs",+ ".go",+ ".rs",+ ".swift",+ ".kt",+ ".scala",+ ".sh",+ ".bash",+ ".zsh",+ ".sql",+ ".graphql",+ ".env",+ ".gitignore",+ ".dockerfile",+ "Dockerfile",+ ".vue",+ ".svelte",+ ".astro",+ ".mdx",+ ".rst",+ ".tex",+ ".csv",+ ".log",+}+++class SandboxFileOutput(BaseModel):+ """A file extracted from a sandbox and optionally stored in workspace."""++ path: str+ """Full path in the sandbox."""++ relative_path: str+ """Path relative to the working directory."""++ name: str+ """Filename only."""++ content: str+ """File content as text (for backward compatibility)."""++ workspace_ref: str | None = None+ """Workspace reference (workspace://{id}#mime) if stored, None otherwise."""+++@dataclass+class ExtractedFile:+ """Internal representation of an extracted file before storage."""++ path: str+ relative_path: str+ name: str+ content: bytes+ is_text: bool+++async def extract_sandbox_files(+ sandbox: "BaseAsyncSandbox",+ working_directory: str,+ since_timestamp: str | None = None,+ text_only: bool = True,+) -> list[ExtractedFile]:+ """+ Extract files from an E2B sandbox.++ Args:+ sandbox: The E2B sandbox instance+ working_directory: Directory to search for files+ since_timestamp: ISO timestamp - only return files modified after this time+ text_only: If True, only extract text files (default). If False, extract all files.++ Returns:+ List of ExtractedFile objects with path, content, and metadata+ """+ files: list[ExtractedFile] = []++ try:+ # Build find command+ safe_working_dir = shlex.quote(working_directory)+ timestamp_filter = ""+ if since_timestamp:+ timestamp_filter = f"-newermt {shlex.quote(since_timestamp)} "++ find_result = await sandbox.commands.run(+ f"find {safe_working_dir} -type f "+ f"{timestamp_filter}"+ f"-not -path '*/node_modules/*' "+ f"-not -path '*/.git/*' "+ f"2>/dev/null"+ )++ if not find_result.stdout:+ return files++ for file_path in find_result.stdout.strip().split("\n"):+ if not file_path:+ continue++ # Check if it's a text file+ is_text = any(file_path.endswith(ext) for ext in TEXT_EXTENSIONS)++ # Skip non-text files if text_only mode+ if text_only and not is_text:+ continue++ try:+ # Read file content as bytes+ content = await sandbox.files.read(file_path, format="bytes")+ if isinstance(content, str):+ content = content.encode("utf-8")+ elif isinstance(content, bytearray):+ content = bytes(content)++ # Extract filename from path+ file_name = file_path.split("/")[-1]++ # Calculate relative path+ relative_path = file_path+ if file_path.startswith(working_directory):+ relative_path = file_path[len(working_directory) :]+ if relative_path.startswith("/"):+ relative_path = relative_path[1:]++ files.append(+ ExtractedFile(+ path=file_path,+ relative_path=relative_path,+ name=file_name,+ content=content,+ is_text=is_text,+ )+ )+ except Exception as e:+ logger.debug(f"Failed to read file {file_path}: {e}")+ continue++ except Exception as e:+ logger.warning(f"File extraction failed: {e}")++ return files+++async def store_sandbox_files(+ extracted_files: list[ExtractedFile],+ execution_context: "ExecutionContext",+) -> list[SandboxFileOutput]:+ """+ Store extracted sandbox files to workspace and return output objects.++ Args:+ extracted_files: List of files extracted from sandbox+ execution_context: Execution context for workspace storage++ Returns:+ List of SandboxFileOutput objects with workspace refs+ """+ outputs: list[SandboxFileOutput] = []++ for file in extracted_files:+ # Decode content for text files (for backward compat content field)+ if file.is_text:+ try:+ content_str = file.content.decode("utf-8", errors="replace")+ except Exception:+ content_str = ""+ else:+ content_str = f"[Binary file: {len(file.content)} bytes]"++ # Build data URI (needed for storage and as binary fallback)+ mime_type = mimetypes.guess_type(file.name)[0] or "application/octet-stream"+ data_uri = f"data:{mime_type};base64,{base64.b64encode(file.content).decode()}"++ # Try to store in workspace+ workspace_ref: str | None = None+ try:+ result = await store_media_file(+ file=MediaFileType(data_uri),+ execution_context=execution_context,+ return_format="for_block_output",+ )+ if result.startswith("workspace://"):+ workspace_ref = result+ elif not file.is_text:+ # Non-workspace context (graph execution): store_media_file+ # returned a data URI — use it as content so binary data isn't lost.+ content_str = result+ except Exception as e:+ logger.warning(f"Failed to store file {file.name} to workspace: {e}")+ # For binary files, fall back to data URI to prevent data loss+ if not file.is_text:+ content_str = data_uri++ outputs.append(+ SandboxFileOutput(+ path=file.path,+ relative_path=file.relative_path,+ name=file.name,+ content=content_str,+ workspace_ref=workspace_ref,+ )+ )++ return outputs+++async def extract_and_store_sandbox_files(+ sandbox: "BaseAsyncSandbox",+ working_directory: str,+ execution_context: "ExecutionContext",+ since_timestamp: str | None = None,+ text_only: bool = True,+) -> list[SandboxFileOutput]:+ """+ Extract files from sandbox and store them in workspace.++ This is the main entry point combining extraction and storage.++ Args:+ sandbox: The E2B sandbox instance+ working_directory: Directory to search for files+ execution_context: Execution context for workspace storage+ since_timestamp: ISO timestamp - only return files modified after this time+ text_only: If True, only extract text files++ Returns:+ List of SandboxFileOutput objects with content and workspace refs+ """+ extracted = await extract_sandbox_files(+ sandbox=sandbox,+ working_directory=working_directory,+ since_timestamp=since_timestamp,+ text_only=text_only,+ )++ return await store_sandbox_files(extracted, execution_context)diff --git a/docs/integrations/block-integrations/llm.md b/docs/integrations/block-integrations/llm.mdindex 20a5147fcd..9c96ef56c0 100644--- a/docs/integrations/block-integrations/llm.md+++ b/docs/integrations/block-integrations/llm.md@@ -563,7 +563,7 @@ The block supports conversation continuation through three mechanisms:|--------|-------------|------|| error | Error message if execution failed | str || response | The output/response from Claude Code execution | str |-| files | List of text files created/modified by Claude Code during this execution. Each file has 'path', 'relative_path', 'name', and 'content' fields. | List[FileOutput] |+| files | List of text files created/modified by Claude Code during this execution. Each file has 'path', 'relative_path', 'name', 'content', and 'workspace_ref' fields. workspace_ref contains a workspace:// URI if the file was stored to workspace. | List[SandboxFileOutput] || conversation_history | Full conversation history including this turn. Pass this to conversation_history input to continue on a fresh sandbox if the previous sandbox timed out. | str || session_id | Session ID for this conversation. Pass this back along with sandbox_id to continue the conversation. | str || sandbox_id | ID of the sandbox instance. Pass this back along with session_id to continue the conversation. This is None if dispose_sandbox was True (sandbox was disposed). | str |diff --git a/docs/integrations/block-integrations/misc.md b/docs/integrations/block-integrations/misc.mdindex 4c199bebb4..ad6300ae88 100644--- a/docs/integrations/block-integrations/misc.md+++ b/docs/integrations/block-integrations/misc.md@@ -215,6 +215,7 @@ The sandbox includes pip and npm pre-installed. Set timeout to limit execution t| response | Text output (if any) of the main execution result | str || stdout_logs | Standard output logs from execution | str || stderr_logs | Standard error logs from execution | str |+| files | Files created or modified during execution. Each file has path, name, content, and workspace_ref (if stored). | List[SandboxFileOutput] |### Possible use case<!-- MANUAL: use_case -->