AutoGPT is the vision of accessible AI for everyone, to use and to build on. Our mission is to provide the tools, so that you can focus on what matters.
feat(copilot): persist large tool outputs to workspace with retrieval instructions (#12279)
Zamil Majdy592830ce9b27
diff --git a/autogpt_platform/backend/backend/copilot/sdk/e2b_file_tools.py b/autogpt_platform/backend/backend/copilot/sdk/e2b_file_tools.pyindex 98216ff58e..5041bd0668 100644--- a/autogpt_platform/backend/backend/copilot/sdk/e2b_file_tools.py+++ b/autogpt_platform/backend/backend/copilot/sdk/e2b_file_tools.py@@ -280,7 +280,9 @@ E2B_FILE_TOOLS: list[tuple[str, str, dict[str, Any], Callable[..., Any]]] = [("write_file","Write or create a file in the cloud sandbox (/home/user). "- "Parent directories are created automatically.",+ "Parent directories are created automatically. "+ "To copy a workspace file into the sandbox, use "+ "read_workspace_file with save_to_path instead.",{"type": "object","properties": {diff --git a/autogpt_platform/backend/backend/copilot/sdk/service.py b/autogpt_platform/backend/backend/copilot/sdk/service.pyindex 79e150d1e1..870247ad42 100644--- a/autogpt_platform/backend/backend/copilot/sdk/service.py+++ b/autogpt_platform/backend/backend/copilot/sdk/service.py@@ -177,6 +177,13 @@ Long-running tools (create_agent, edit_agent, etc.) are handledasynchronously. You will receive an immediate response; the actual resultis delivered to the user via a background stream.+### Large tool outputs+When a tool output exceeds the display limit, it is automatically saved to+the persistent workspace. The truncated output includes a+`<tool-output-truncated>` tag with the workspace path. Use+`read_workspace_file(path="...", offset=N, length=50000)` to retrieve+additional sections.+### Sub-agent tasks- When using the Task tool, NEVER set `run_in_background` to true.All tasks must run in the foreground.diff --git a/autogpt_platform/backend/backend/copilot/tools/base.py b/autogpt_platform/backend/backend/copilot/tools/base.pyindex da2a644edf..1225d22e18 100644--- a/autogpt_platform/backend/backend/copilot/tools/base.py+++ b/autogpt_platform/backend/backend/copilot/tools/base.py@@ -1,5 +1,6 @@"""Base classes and shared utilities for chat tools."""+import jsonimport loggingfrom typing import Any@@ -7,11 +8,98 @@ from openai.types.chat import ChatCompletionToolParamfrom backend.copilot.model import ChatSessionfrom backend.copilot.response_model import StreamToolOutputAvailable+from backend.data.db_accessors import workspace_db+from backend.util.truncate import truncate+from backend.util.workspace import WorkspaceManagerfrom .models import ErrorResponse, NeedLoginResponse, ToolResponseBaselogger = logging.getLogger(__name__)+# Persist full tool output to workspace when it exceeds this threshold.+# Must be below _MAX_TOOL_OUTPUT_SIZE (100K) in response_model.py so we+# capture the data before model_post_init middle-out truncation discards it.+_LARGE_OUTPUT_THRESHOLD = 80_000++# Character budget for the middle-out preview. The total preview + wrapper+# must stay below BOTH:+# - _MAX_TOOL_OUTPUT_SIZE (100K) in response_model.py (our own truncation)+# - Claude SDK's ~100 KB tool-result spill-to-disk threshold+# to avoid double truncation/spilling. 95K + ~300 wrapper = ~95.3K, under both.+_PREVIEW_CHARS = 95_000+++# Fields whose values are binary/base64 data — truncating them produces+# garbage, so we replace them with a human-readable size summary instead.+_BINARY_FIELD_NAMES = {"content_base64"}+++def _summarize_binary_fields(raw_json: str) -> str:+ """Replace known binary fields with a size summary so truncate() doesn't+ produce garbled base64 in the middle-out preview."""+ try:+ data = json.loads(raw_json)+ except (json.JSONDecodeError, TypeError):+ return raw_json++ if not isinstance(data, dict):+ return raw_json++ changed = False+ for key in _BINARY_FIELD_NAMES:+ if key in data and isinstance(data[key], str) and len(data[key]) > 1_000:+ byte_size = len(data[key]) * 3 // 4 # approximate decoded size+ data[key] = f"<binary, ~{byte_size:,} bytes>"+ changed = True++ return json.dumps(data, ensure_ascii=False) if changed else raw_json+++async def _persist_and_summarize(+ raw_output: str,+ user_id: str,+ session_id: str,+ tool_call_id: str,+) -> str:+ """Persist full output to workspace and return a middle-out preview with retrieval instructions.++ On failure, returns the original ``raw_output`` unchanged so that the+ existing ``model_post_init`` middle-out truncation handles it as before.+ """+ file_path = f"tool-outputs/{tool_call_id}.json"+ try:+ workspace = await workspace_db().get_or_create_workspace(user_id)+ manager = WorkspaceManager(user_id, workspace.id, session_id)+ await manager.write_file(+ content=raw_output.encode("utf-8"),+ filename=f"{tool_call_id}.json",+ path=file_path,+ mime_type="application/json",+ overwrite=True,+ )+ except Exception:+ logger.warning(+ "Failed to persist large tool output for %s",+ tool_call_id,+ exc_info=True,+ )+ return raw_output # fall back to normal truncation++ total = len(raw_output)+ preview = truncate(_summarize_binary_fields(raw_output), _PREVIEW_CHARS)+ retrieval = (+ f"\nFull output ({total:,} chars) saved to workspace. "+ f"Use read_workspace_file("+ f'path="{file_path}", offset=<char_offset>, length=50000) '+ f"to read any section."+ )+ return (+ f'<tool-output-truncated total_chars={total} path="{file_path}">\n'+ f"{preview}\n"+ f"{retrieval}\n"+ f"</tool-output-truncated>"+ )+class BaseTool:"""Base class for all chat tools."""@@ -67,7 +155,7 @@ class BaseTool:"""Execute the tool with authentication check.Args:- user_id: User ID (may be anonymous like "anon_123")+ user_id: User ID (None for anonymous users)session_id: Chat session ID**kwargs: Tool-specific parameters@@ -91,10 +179,21 @@ class BaseTool:try:result = await self._execute(user_id, session, **kwargs)+ raw_output = result.model_dump_json()++ if (+ len(raw_output) > _LARGE_OUTPUT_THRESHOLD+ and user_id+ and session.session_id+ ):+ raw_output = await _persist_and_summarize(+ raw_output, user_id, session.session_id, tool_call_id+ )+return StreamToolOutputAvailable(toolCallId=tool_call_id,toolName=self.name,- output=result.model_dump_json(),+ output=raw_output,)except Exception as e:logger.error(f"Error in {self.name}: {e}", exc_info=True)diff --git a/autogpt_platform/backend/backend/copilot/tools/base_test.py b/autogpt_platform/backend/backend/copilot/tools/base_test.pynew file mode 100644index 0000000000..803b76baba--- /dev/null+++ b/autogpt_platform/backend/backend/copilot/tools/base_test.py@@ -0,0 +1,194 @@+"""Tests for BaseTool large-output persistence in execute()."""++from unittest.mock import AsyncMock, MagicMock, patch++import pytest++from backend.copilot.tools.base import (+ _LARGE_OUTPUT_THRESHOLD,+ BaseTool,+ _persist_and_summarize,+ _summarize_binary_fields,+)+from backend.copilot.tools.models import ResponseType, ToolResponseBase+++class _HugeOutputTool(BaseTool):+ """Fake tool that returns an arbitrarily large output."""++ def __init__(self, output_size: int) -> None:+ self._output_size = output_size++ @property+ def name(self) -> str:+ return "huge_output_tool"++ @property+ def description(self) -> str:+ return "Returns a huge output"++ @property+ def parameters(self) -> dict:+ return {"type": "object", "properties": {}}++ async def _execute(self, user_id, session, **kwargs) -> ToolResponseBase:+ return ToolResponseBase(+ type=ResponseType.ERROR,+ message="x" * self._output_size,+ )+++# ---------------------------------------------------------------------------+# _persist_and_summarize+# ---------------------------------------------------------------------------+++class TestPersistAndSummarize:+ @pytest.mark.asyncio+ async def test_returns_middle_out_preview_with_retrieval_instructions(self):+ raw = "A" * 200_000++ mock_workspace = MagicMock()+ mock_workspace.id = "ws-1"+ mock_db = AsyncMock()+ mock_db.get_or_create_workspace = AsyncMock(return_value=mock_workspace)++ mock_manager = AsyncMock()++ with (+ patch("backend.copilot.tools.base.workspace_db", return_value=mock_db),+ patch(+ "backend.copilot.tools.base.WorkspaceManager",+ return_value=mock_manager,+ ),+ ):+ result = await _persist_and_summarize(raw, "user-1", "session-1", "tc-123")++ assert "<tool-output-truncated" in result+ assert "</tool-output-truncated>" in result+ assert "total_chars=200000" in result+ assert 'path="tool-outputs/tc-123.json"' in result+ assert "read_workspace_file" in result+ # Middle-out sentinel from truncate()+ assert "omitted" in result+ # Total result is much shorter than the raw output+ assert len(result) < len(raw)++ # Verify write_file was called with full content+ mock_manager.write_file.assert_awaited_once()+ call_kwargs = mock_manager.write_file.call_args+ assert call_kwargs.kwargs["content"] == raw.encode("utf-8")+ assert call_kwargs.kwargs["path"] == "tool-outputs/tc-123.json"++ @pytest.mark.asyncio+ async def test_fallback_on_workspace_error(self):+ """If workspace write fails, return raw output for normal truncation."""+ raw = "B" * 200_000+ mock_db = AsyncMock()+ mock_db.get_or_create_workspace = AsyncMock(side_effect=RuntimeError("boom"))++ with patch("backend.copilot.tools.base.workspace_db", return_value=mock_db):+ result = await _persist_and_summarize(raw, "user-1", "session-1", "tc-fail")++ assert result == raw # unchanged — fallback to normal truncation+++# ---------------------------------------------------------------------------+# BaseTool.execute — integration with persistence+# ---------------------------------------------------------------------------+++class TestBaseToolExecuteLargeOutput:+ @pytest.mark.asyncio+ async def test_small_output_not_persisted(self):+ """Outputs under the threshold go through without persistence."""+ tool = _HugeOutputTool(output_size=100)+ session = MagicMock()+ session.session_id = "s-1"++ with patch(+ "backend.copilot.tools.base._persist_and_summarize",+ new_callable=AsyncMock,+ ) as persist_mock:+ result = await tool.execute("user-1", session, "tc-small")+ persist_mock.assert_not_awaited()+ assert "<tool-output-truncated" not in str(result.output)++ @pytest.mark.asyncio+ async def test_large_output_persisted(self):+ """Outputs over the threshold trigger persistence + preview."""+ tool = _HugeOutputTool(output_size=_LARGE_OUTPUT_THRESHOLD + 10_000)+ session = MagicMock()+ session.session_id = "s-1"++ mock_workspace = MagicMock()+ mock_workspace.id = "ws-1"+ mock_db = AsyncMock()+ mock_db.get_or_create_workspace = AsyncMock(return_value=mock_workspace)+ mock_manager = AsyncMock()++ with (+ patch("backend.copilot.tools.base.workspace_db", return_value=mock_db),+ patch(+ "backend.copilot.tools.base.WorkspaceManager",+ return_value=mock_manager,+ ),+ ):+ result = await tool.execute("user-1", session, "tc-big")++ assert "<tool-output-truncated" in str(result.output)+ assert "read_workspace_file" in str(result.output)+ mock_manager.write_file.assert_awaited_once()++ @pytest.mark.asyncio+ async def test_no_persistence_without_user_id(self):+ """Anonymous users skip persistence (no workspace)."""+ tool = _HugeOutputTool(output_size=_LARGE_OUTPUT_THRESHOLD + 10_000)+ session = MagicMock()+ session.session_id = "s-1"++ # user_id=None → should not attempt persistence+ with patch(+ "backend.copilot.tools.base._persist_and_summarize",+ new_callable=AsyncMock,+ ) as persist_mock:+ result = await tool.execute(None, session, "tc-anon")+ persist_mock.assert_not_awaited()+ # Output is set but not wrapped in <tool-output-truncated> tags+ # (it will be middle-out truncated by model_post_init instead)+ assert "<tool-output-truncated" not in str(result.output)+++# ---------------------------------------------------------------------------+# _summarize_binary_fields+# ---------------------------------------------------------------------------+++class TestSummarizeBinaryFields:+ def test_replaces_large_content_base64(self):+ import json++ data = {"content_base64": "A" * 10_000, "name": "file.png"}+ result = json.loads(_summarize_binary_fields(json.dumps(data)))+ assert result["name"] == "file.png"+ assert "<binary" in result["content_base64"]+ assert "bytes>" in result["content_base64"]++ def test_preserves_small_content_base64(self):+ import json++ data = {"content_base64": "AQID", "name": "tiny.bin"}+ result_str = _summarize_binary_fields(json.dumps(data))+ result = json.loads(result_str)+ assert result["content_base64"] == "AQID" # unchanged++ def test_non_json_passthrough(self):+ raw = "not json at all"+ assert _summarize_binary_fields(raw) == raw++ def test_no_binary_fields_unchanged(self):+ import json++ data = {"message": "hello", "type": "info"}+ raw = json.dumps(data)+ assert _summarize_binary_fields(raw) == rawdiff --git a/autogpt_platform/backend/backend/copilot/tools/workspace_files.py b/autogpt_platform/backend/backend/copilot/tools/workspace_files.pyindex 1bbc36227f..ee7e11dff8 100644--- a/autogpt_platform/backend/backend/copilot/tools/workspace_files.py+++ b/autogpt_platform/backend/backend/copilot/tools/workspace_files.py@@ -448,8 +448,10 @@ class ReadWorkspaceFileTool(BaseTool):"Specify either file_id or path to identify the file. ""For small text files, returns content directly. ""For large or binary files, returns metadata and a download URL. "- "Optionally use 'save_to_path' to copy the file to the ephemeral "- "working directory for processing with bash_exec or SDK tools. "+ "Use 'save_to_path' to copy the file to the working directory "+ "(sandbox or ephemeral) for processing with bash_exec or file tools. "+ "Use 'offset' and 'length' for paginated reads of large files "+ "(e.g., persisted tool outputs). ""Paths are scoped to the current session by default. ""Use /sessions/<session_id>/... for cross-session access.")@@ -473,9 +475,10 @@ class ReadWorkspaceFileTool(BaseTool):"save_to_path": {"type": "string","description": (- "If provided, save the file to this path in the ephemeral "- "working directory (e.g., '/tmp/copilot-.../data.csv') "- "so it can be processed with bash_exec or SDK tools. "+ "If provided, save the file to this path in the working "+ "directory (cloud sandbox when E2B is active, or "+ "ephemeral dir otherwise) so it can be processed with "+ "bash_exec or file tools. ""The file content is still returned in the response."),},@@ -486,6 +489,20 @@ class ReadWorkspaceFileTool(BaseTool):"Default is false (auto-selects based on file size/type)."),},+ "offset": {+ "type": "integer",+ "description": (+ "Character offset to start reading from (0-based). "+ "Use with 'length' for paginated reads of large files."+ ),+ },+ "length": {+ "type": "integer",+ "description": (+ "Maximum number of characters to return. "+ "Defaults to full file. Use with 'offset' for paginated reads."+ ),+ },},"required": [], # At least one must be provided}@@ -510,6 +527,8 @@ class ReadWorkspaceFileTool(BaseTool):path: Optional[str] = kwargs.get("path")save_to_path: Optional[str] = kwargs.get("save_to_path")force_download_url: bool = kwargs.get("force_download_url", False)+ char_offset: int = max(0, kwargs.get("offset", 0))+ char_length: Optional[int] = kwargs.get("length")if not file_id and not path:return ErrorResponse(@@ -532,6 +551,34 @@ class ReadWorkspaceFileTool(BaseTool):return resultsave_to_path = result+ # Ranged read: return a character slice directly.+ if char_offset > 0 or char_length is not None:+ raw = cached_content or await manager.read_file_by_id(target_file_id)+ text = raw.decode("utf-8", errors="replace")+ total_chars = len(text)+ end = (+ char_offset + char_length+ if char_length is not None+ else total_chars+ )+ slice_text = text[char_offset:end]+ return WorkspaceFileContentResponse(+ file_id=file_info.id,+ name=file_info.name,+ path=file_info.path,+ mime_type="text/plain",+ content_base64=base64.b64encode(slice_text.encode("utf-8")).decode(+ "utf-8"+ ),+ message=(+ f"Read chars {char_offset}–"+ f"{char_offset + len(slice_text)} "+ f"of {total_chars:,} total "+ f"from {file_info.name}"+ ),+ session_id=session_id,+ )+is_small = file_info.size_bytes <= self.MAX_INLINE_SIZE_BYTESis_text = _is_text_mime(file_info.mime_type)is_image = file_info.mime_type in _IMAGE_MIME_TYPESdiff --git a/autogpt_platform/backend/backend/copilot/tools/workspace_files_test.py b/autogpt_platform/backend/backend/copilot/tools/workspace_files_test.pyindex 0370ad8d98..9718a3ac18 100644--- a/autogpt_platform/backend/backend/copilot/tools/workspace_files_test.py+++ b/autogpt_platform/backend/backend/copilot/tools/workspace_files_test.py@@ -236,6 +236,65 @@ async def test_workspace_file_round_trip(setup_test_data):assert not any(f.file_id == file_id for f in list_resp2.files)+# ---------------------------------------------------------------------------+# Ranged reads (offset / length)+# ---------------------------------------------------------------------------++[email protected](loop_scope="session")+async def test_read_workspace_file_with_offset_and_length(setup_test_data):+ """Read a slice of a text file using offset and length."""+ user = setup_test_data["user"]+ session = make_session(user.id)++ # Write a known-content file+ content = "ABCDEFGHIJKLMNOPQRSTUVWXYZ" * 100 # 2600 chars+ write_tool = WriteWorkspaceFileTool()+ write_resp = await write_tool._execute(+ user_id=user.id,+ session=session,+ filename="ranged_test.txt",+ content=content,+ )+ assert isinstance(write_resp, WorkspaceWriteResponse), write_resp.message+ file_id = write_resp.file_id++ from backend.copilot.tools.workspace_files import WorkspaceFileContentResponse++ read_tool = ReadWorkspaceFileTool()++ # Read with offset=100, length=50+ resp = await read_tool._execute(+ user_id=user.id, session=session, file_id=file_id, offset=100, length=50+ )+ assert isinstance(resp, WorkspaceFileContentResponse), resp.message+ decoded = base64.b64decode(resp.content_base64).decode()+ assert decoded == content[100:150]+ assert "100" in resp.message+ assert "2,600" in resp.message # total chars (comma-formatted)++ # Read with offset only (no length) — returns from offset to end+ resp2 = await read_tool._execute(+ user_id=user.id, session=session, file_id=file_id, offset=2500+ )+ assert isinstance(resp2, WorkspaceFileContentResponse)+ decoded2 = base64.b64decode(resp2.content_base64).decode()+ assert decoded2 == content[2500:]+ assert len(decoded2) == 100++ # Read with offset beyond file length — returns empty string+ resp3 = await read_tool._execute(+ user_id=user.id, session=session, file_id=file_id, offset=9999, length=10+ )+ assert isinstance(resp3, WorkspaceFileContentResponse)+ decoded3 = base64.b64decode(resp3.content_base64).decode()+ assert decoded3 == ""++ # Cleanup+ delete_tool = DeleteWorkspaceFileTool()+ await delete_tool._execute(user_id=user.id, session=session, file_id=file_id)++@pytest.mark.asyncio(loop_scope="session")async def test_write_workspace_file_source_path(setup_test_data):"""E2E: write a file from ephemeral source_path to workspace."""