AutoGPT is the vision of accessible AI for everyone, to use and to build on. Our mission is to provide the tools, so that you can focus on what matters.
fix(backend): Extract binary files from ClaudeCodeBlock sandbox (#12125)
Bently63ba53f02b6e
diff --git a/autogpt_platform/backend/backend/blocks/claude_code.py b/autogpt_platform/backend/backend/blocks/claude_code.pyindex 03c8f70312..8e1720e19a 100644--- a/autogpt_platform/backend/backend/blocks/claude_code.py+++ b/autogpt_platform/backend/backend/blocks/claude_code.py@@ -188,9 +188,11 @@ class ClaudeCodeBlock(Block):)files: list[SandboxFileOutput] = SchemaField(description=(- "List of text files created/modified by Claude Code during this execution. "+ "List of files created/modified by Claude Code during this execution. "+ "Includes text files and binary files (images, PDFs, etc.). ""Each file has 'path', 'relative_path', 'name', 'content', and 'workspace_ref' fields. "- "workspace_ref contains a workspace:// URI if the file was stored to workspace."+ "workspace_ref contains a workspace:// URI for workspace storage. "+ "For binary files, content contains a placeholder; use workspace_ref to access the file."))conversation_history: str = SchemaField(@@ -454,13 +456,15 @@ class ClaudeCodeBlock(Block):else:new_conversation_history = turn_entry- # Extract files created/modified during this run and store to workspace+ # Extract files created/modified during this run and store to workspace.+ # Binary files (images, PDFs, etc.) are stored via store_media_file+ # which handles virus scanning and workspace storage.sandbox_files = await extract_and_store_sandbox_files(sandbox=sandbox,working_directory=working_directory,execution_context=execution_context,since_timestamp=start_timestamp,- text_only=True,+ text_only=False,)return (diff --git a/autogpt_platform/backend/backend/util/sandbox_files.py b/autogpt_platform/backend/backend/util/sandbox_files.pyindex 9db53ded14..795545b5a6 100644--- a/autogpt_platform/backend/backend/util/sandbox_files.py+++ b/autogpt_platform/backend/backend/util/sandbox_files.py@@ -15,6 +15,7 @@ from typing import TYPE_CHECKINGfrom pydantic import BaseModelfrom backend.util.file import store_media_file+from backend.util.settings import Configfrom backend.util.type import MediaFileTypeif TYPE_CHECKING:@@ -74,8 +75,51 @@ TEXT_EXTENSIONS = {".tex",".csv",".log",+ ".svg", # SVG is XML-based text}+# Binary file extensions we explicitly support extracting+BINARY_EXTENSIONS = {+ # Images+ ".png",+ ".jpg",+ ".jpeg",+ ".gif",+ ".webp",+ ".ico",+ ".bmp",+ ".tiff",+ ".tif",+ # Documents+ ".pdf",+ # Archives+ ".zip",+ ".tar",+ ".gz",+ ".7z",+ # Audio+ ".mp3",+ ".aac",+ ".wav",+ ".ogg",+ ".flac",+ # Video+ ".mp4",+ ".webm",+ ".mov",+ ".avi",+ # Fonts+ ".woff",+ ".woff2",+ ".ttf",+ ".otf",+ ".eot",+}++# Maximum file size for binary extraction (50MB)+# Prevents OOM from accidentally extracting huge files+MAX_BINARY_FILE_SIZE = 50 * 1024 * 1024+class SandboxFileOutput(BaseModel):"""A file extracted from a sandbox and optionally stored in workspace."""@@ -120,7 +164,8 @@ async def extract_sandbox_files(sandbox: The E2B sandbox instanceworking_directory: Directory to search for filessince_timestamp: ISO timestamp - only return files modified after this time- text_only: If True, only extract text files (default). If False, extract all files.+ text_only: If True, only extract text files. If False, also extract+ supported binary files (images, PDFs, etc.).Returns:List of ExtractedFile objects with path, content, and metadata@@ -149,15 +194,48 @@ async def extract_sandbox_files(if not file_path:continue- # Check if it's a text file- is_text = any(file_path.endswith(ext) for ext in TEXT_EXTENSIONS)+ # Check file type (case-insensitive for extensions)+ file_path_lower = file_path.lower()+ is_text = any(+ file_path_lower.endswith(ext.lower()) for ext in TEXT_EXTENSIONS+ )+ is_binary = any(+ file_path_lower.endswith(ext.lower()) for ext in BINARY_EXTENSIONS+ )++ # Skip files with unrecognized extensions+ if not is_text and not is_binary:+ continue- # Skip non-text files if text_only mode+ # In text_only mode, skip binary filesif text_only and not is_text:continuetry:- # Read file content as bytes+ # Check file size before reading to prevent OOM+ stat_result = await sandbox.commands.run(+ f"stat -c %s {shlex.quote(file_path)} 2>/dev/null"+ )+ if stat_result.exit_code != 0 or not stat_result.stdout:+ logger.debug(f"Skipping {file_path}: could not determine file size")+ continue++ try:+ file_size = int(stat_result.stdout.strip())+ except ValueError:+ logger.debug(+ f"Skipping {file_path}: unexpected stat output "+ f"{stat_result.stdout.strip()!r}"+ )+ continue++ if file_size > MAX_BINARY_FILE_SIZE:+ logger.info(+ f"Skipping {file_path}: size {file_size} bytes "+ f"exceeds limit {MAX_BINARY_FILE_SIZE}"+ )+ continue+content = await sandbox.files.read(file_path, format="bytes")if isinstance(content, str):content = content.encode("utf-8")@@ -239,8 +317,12 @@ async def store_sandbox_files(content_str = resultexcept Exception as e:logger.warning(f"Failed to store file {file.name} to workspace: {e}")- # For binary files, fall back to data URI to prevent data loss- if not file.is_text:+ # For binary files, fall back to the data URI to prevent data loss —+ # but respect the same configured size cap storage enforces, so a+ # storage failure never balloons the execution payload with content+ # that was too large (or unlucky enough) to store in the first place.+ max_inline = Config().max_file_size_mb * 1024 * 1024+ if not file.is_text and len(data_uri) <= max_inline:content_str = data_urioutputs.append(diff --git a/autogpt_platform/backend/backend/util/sandbox_files_test.py b/autogpt_platform/backend/backend/util/sandbox_files_test.pynew file mode 100644index 0000000000..54319f52c3--- /dev/null+++ b/autogpt_platform/backend/backend/util/sandbox_files_test.py@@ -0,0 +1,263 @@+"""Tests for sandbox file extraction and workspace storage."""++import base64+from unittest.mock import AsyncMock, MagicMock, patch++import pytest++from backend.util.sandbox_files import (+ MAX_BINARY_FILE_SIZE,+ ExtractedFile,+ extract_sandbox_files,+ store_sandbox_files,+)++_WORKDIR = "/home/user"+_PNG_BYTES = b"\x89PNG\r\n\x1a\n" + bytes(range(256))+++def _sandbox(+ file_listing: list[str],+ file_sizes: dict[str, str | None] | None = None,+ file_contents: dict[str, bytes] | None = None,+) -> MagicMock:+ """Fake E2B sandbox: one `find` call, then a `stat` call per file.++ ``file_sizes`` maps path -> stat stdout (None = stat fails with exit 1).+ """+ sizes = file_sizes or {}+ contents = file_contents or {}+ sandbox = MagicMock()++ async def run(command: str):+ result = MagicMock()+ if command.startswith("find "):+ result.exit_code = 0+ result.stdout = "\n".join(file_listing)+ return result+ for path, size in sizes.items():+ if path in command:+ result.exit_code = 1 if size is None else 0+ result.stdout = size or ""+ return result+ result.exit_code = 0+ result.stdout = "4"+ return result++ async def read(path: str, format: str = "bytes"):+ return contents.get(path, b"data")++ sandbox.commands.run = AsyncMock(side_effect=run)+ sandbox.files.read = AsyncMock(side_effect=read)+ return sandbox+++def _ctx() -> MagicMock:+ return MagicMock()+++# ── extract_sandbox_files ──────────────────────────────────────────+++async def test_extract_text_file():+ sandbox = _sandbox(+ [f"{_WORKDIR}/notes.md"], file_contents={f"{_WORKDIR}/notes.md": b"# hi"}+ )+ files = await extract_sandbox_files(sandbox, _WORKDIR)+ assert len(files) == 1+ assert files[0].is_text is True+ assert files[0].content == b"# hi"+ assert files[0].relative_path == "notes.md"+ assert files[0].name == "notes.md"+++async def test_extract_binary_file_when_allowed():+ sandbox = _sandbox(+ [f"{_WORKDIR}/chart.png"],+ file_contents={f"{_WORKDIR}/chart.png": _PNG_BYTES},+ )+ files = await extract_sandbox_files(sandbox, _WORKDIR, text_only=False)+ assert len(files) == 1+ assert files[0].is_text is False+ assert files[0].content == _PNG_BYTES+++async def test_binary_extension_matching_is_case_insensitive():+ sandbox = _sandbox(+ [f"{_WORKDIR}/PHOTO.JPG"],+ file_contents={f"{_WORKDIR}/PHOTO.JPG": b"\xff\xd8\xff"},+ )+ files = await extract_sandbox_files(sandbox, _WORKDIR, text_only=False)+ assert [f.name for f in files] == ["PHOTO.JPG"]+++async def test_text_only_mode_skips_binary_files():+ sandbox = _sandbox([f"{_WORKDIR}/chart.png", f"{_WORKDIR}/readme.txt"])+ files = await extract_sandbox_files(sandbox, _WORKDIR, text_only=True)+ assert [f.name for f in files] == ["readme.txt"]+++async def test_unrecognized_extension_is_skipped():+ sandbox = _sandbox([f"{_WORKDIR}/core.dump", f"{_WORKDIR}/binary.exe"])+ files = await extract_sandbox_files(sandbox, _WORKDIR, text_only=False)+ assert files == []+++async def test_oversized_file_is_skipped():+ path = f"{_WORKDIR}/huge.png"+ sandbox = _sandbox([path], file_sizes={path: str(MAX_BINARY_FILE_SIZE + 1)})+ files = await extract_sandbox_files(sandbox, _WORKDIR, text_only=False)+ assert files == []+ sandbox.files.read.assert_not_awaited()+++async def test_stat_failure_skips_file():+ path = f"{_WORKDIR}/gone.png"+ sandbox = _sandbox([path], file_sizes={path: None})+ files = await extract_sandbox_files(sandbox, _WORKDIR, text_only=False)+ assert files == []+ sandbox.files.read.assert_not_awaited()+++async def test_unparseable_stat_output_skips_file():+ path = f"{_WORKDIR}/odd.png"+ sandbox = _sandbox([path], file_sizes={path: "not-a-number"})+ files = await extract_sandbox_files(sandbox, _WORKDIR, text_only=False)+ assert files == []+++async def test_read_failure_skips_file_but_not_others():+ bad, good = f"{_WORKDIR}/bad.png", f"{_WORKDIR}/good.png"+ sandbox = _sandbox([bad, good], file_contents={good: b"ok"})+ sandbox.files.read = AsyncMock(+ side_effect=lambda path, format="bytes": (+ (_ for _ in ()).throw(OSError("io")) if path == bad else b"ok"+ )+ )+ files = await extract_sandbox_files(sandbox, _WORKDIR, text_only=False)+ assert [f.name for f in files] == ["good.png"]+++async def test_empty_find_output_returns_no_files():+ sandbox = _sandbox([])+ assert await extract_sandbox_files(sandbox, _WORKDIR) == []+++# ── store_sandbox_files ────────────────────────────────────────────+++def _extracted(name: str, content: bytes, is_text: bool) -> ExtractedFile:+ return ExtractedFile(+ path=f"{_WORKDIR}/{name}",+ relative_path=name,+ name=name,+ content=content,+ is_text=is_text,+ )+++async def test_store_text_file_sets_content_and_workspace_ref():+ store = AsyncMock(return_value="workspace://abc123")+ with patch("backend.util.sandbox_files.store_media_file", store):+ outputs = await store_sandbox_files(+ [_extracted("notes.md", b"# hi", is_text=True)], _ctx()+ )+ assert outputs[0].content == "# hi"+ assert outputs[0].workspace_ref == "workspace://abc123"+++async def test_store_binary_file_uses_placeholder_content():+ store = AsyncMock(return_value="workspace://img1")+ with patch("backend.util.sandbox_files.store_media_file", store):+ outputs = await store_sandbox_files(+ [_extracted("chart.png", _PNG_BYTES, is_text=False)], _ctx()+ )+ assert outputs[0].workspace_ref == "workspace://img1"+ assert outputs[0].content == f"[Binary file: {len(_PNG_BYTES)} bytes]"+ # The stored payload is the base64 data URI of the raw bytes.+ assert store.await_args is not None+ assert store.await_args.kwargs["file"] == (+ f"data:image/png;base64,{base64.b64encode(_PNG_BYTES).decode()}"+ )+++async def test_store_binary_in_graph_context_keeps_data_uri_content():+ # Non-workspace context: store_media_file returns a data URI instead of a+ # workspace ref — that URI must become the content so the bytes survive.+ data_uri = f"data:image/png;base64,{base64.b64encode(_PNG_BYTES).decode()}"+ store = AsyncMock(return_value=data_uri)+ with patch("backend.util.sandbox_files.store_media_file", store):+ outputs = await store_sandbox_files(+ [_extracted("chart.png", _PNG_BYTES, is_text=False)], _ctx()+ )+ assert outputs[0].workspace_ref is None+ assert outputs[0].content == data_uri+++async def test_store_failure_falls_back_to_data_uri_for_binary():+ store = AsyncMock(side_effect=RuntimeError("storage down"))+ with patch("backend.util.sandbox_files.store_media_file", store):+ outputs = await store_sandbox_files(+ [_extracted("chart.png", _PNG_BYTES, is_text=False)], _ctx()+ )+ assert outputs[0].workspace_ref is None+ assert outputs[0].content.startswith("data:image/png;base64,")+++async def test_store_failure_fallback_respects_size_cap():+ # A binary too large for the configured cap must NOT be inlined as a data+ # URI on storage failure — the placeholder stays, so a broken storage+ # backend can't balloon execution payloads.+ store = AsyncMock(side_effect=RuntimeError("storage down"))+ config = MagicMock(max_file_size_mb=1)+ big = bytes(2 * 1024 * 1024) # 2MB raw -> ~2.7MB data URI > 1MB cap+ with (+ patch("backend.util.sandbox_files.store_media_file", store),+ patch("backend.util.sandbox_files.Config", return_value=config),+ ):+ outputs = await store_sandbox_files(+ [_extracted("huge.png", big, is_text=False)], _ctx()+ )+ assert outputs[0].workspace_ref is None+ assert outputs[0].content == f"[Binary file: {len(big)} bytes]"+++async def test_store_failure_keeps_decoded_text_content():+ store = AsyncMock(side_effect=RuntimeError("storage down"))+ with patch("backend.util.sandbox_files.store_media_file", store):+ outputs = await store_sandbox_files(+ [_extracted("notes.md", b"# hi", is_text=True)], _ctx()+ )+ assert outputs[0].workspace_ref is None+ assert outputs[0].content == "# hi"+++async def test_store_unknown_extension_defaults_octet_stream_mime():+ store = AsyncMock(return_value="workspace://f1")+ with patch("backend.util.sandbox_files.store_media_file", store):+ # mimetypes tables differ per platform (.7z is known on Linux but not+ # Windows) — use an extension no table knows.+ await store_sandbox_files(+ [_extracted("blob.zz9unknown", b"\x00\x01", is_text=False)], _ctx()+ )+ assert store.await_args is not None+ assert store.await_args.kwargs["file"].startswith(+ "data:application/octet-stream;base64,"+ )diff --git a/docs/integrations/block-integrations/claude_code.md b/docs/integrations/block-integrations/claude_code.mdindex fea67cb494..78f9ccaf5d 100644--- a/docs/integrations/block-integrations/claude_code.md+++ b/docs/integrations/block-integrations/claude_code.md@@ -16,7 +16,7 @@ When activated, the block:- Install dependencies (npm, pip, etc.)- Run terminal commands- Build and test applications-5. Extracts all text files created/modified during execution+5. Extracts all files created/modified during execution (text files and binary files like images, PDFs, etc.)6. Returns the response and files, optionally keeping the sandbox alive for follow-up tasksThe block supports conversation continuation through three mechanisms:@@ -42,7 +42,7 @@ The block supports conversation continuation through three mechanisms:| Output | Description ||--------|-------------|| Response | The output/response from Claude Code execution |-| Files | List of text files created/modified during execution. Each file includes path, relative_path, name, and content fields |+| Files | List of files (text and binary) created/modified during execution. Includes images, PDFs, and other supported formats. Each file has path, relative_path, name, content, and workspace_ref fields. Binary files are stored in workspace and accessible via workspace_ref || Conversation History | Full conversation history including this turn. Use to restore context on a fresh sandbox || Session ID | Session ID for this conversation. Pass back with sandbox_id to continue the conversation || Sandbox ID | ID of the sandbox instance (null if disposed). Pass back with session_id to continue the conversation |diff --git a/docs/integrations/block-integrations/llm.md b/docs/integrations/block-integrations/llm.mdindex 804c342760..66c0b9f2f4 100644--- a/docs/integrations/block-integrations/llm.md+++ b/docs/integrations/block-integrations/llm.md@@ -535,7 +535,7 @@ When activated, the block:2. Installs the latest version of Claude Code in the sandbox3. Optionally runs setup commands to prepare the environment4. Executes your prompt using Claude Code, which can create/edit files, install dependencies, run terminal commands, and build applications-5. Extracts all text files created/modified during execution+5. Extracts all files created/modified during execution (text files and binary files like images, PDFs, etc.)6. Returns the response and files, optionally keeping the sandbox alive for follow-up tasksThe block supports conversation continuation through three mechanisms:@@ -563,7 +563,7 @@ The block supports conversation continuation through three mechanisms:|--------|-------------|------|| error | Error message if execution failed | str || response | The output/response from Claude Code execution | str |-| files | List of text files created/modified by Claude Code during this execution. Each file has 'path', 'relative_path', 'name', 'content', and 'workspace_ref' fields. workspace_ref contains a workspace:// URI if the file was stored to workspace. | List[SandboxFileOutput] |+| files | List of files created/modified by Claude Code during this execution. Includes text files and binary files (images, PDFs, etc.). Each file has 'path', 'relative_path', 'name', 'content', and 'workspace_ref' fields. workspace_ref contains a workspace:// URI for workspace storage. For binary files, content contains a placeholder; use workspace_ref to access the file. | List[SandboxFileOutput] || conversation_history | Full conversation history including this turn. Pass this to conversation_history input to continue on a fresh sandbox if the previous sandbox timed out. | str || session_id | Session ID for this conversation. Pass this back along with sandbox_id to continue the conversation. | str || sandbox_id | ID of the sandbox instance. Pass this back along with session_id to continue the conversation. This is None if dispose_sandbox was True (sandbox was disposed). | str |