AutoGPT is the vision of accessible AI for everyone, to use and to build on. Our mission is to provide the tools, so that you can focus on what matters.
fix(copilot): prevent background agent stalls and context hallucination (#12167)
Zamil Majdy0818cd6683a3
diff --git a/autogpt_platform/backend/backend/copilot/sdk/security_hooks.py b/autogpt_platform/backend/backend/copilot/sdk/security_hooks.pyindex afb144b1e6..396304fd27 100644--- a/autogpt_platform/backend/backend/copilot/sdk/security_hooks.py+++ b/autogpt_platform/backend/backend/copilot/sdk/security_hooks.py@@ -188,8 +188,19 @@ def create_security_hooks(# Rate-limit Task (sub-agent) spawns per sessionif tool_name == "Task":- task_spawn_count += 1- if task_spawn_count > max_subtasks:+ # Block background task execution first — denied calls+ # should not consume a subtask slot.+ if tool_input.get("run_in_background"):+ logger.info(f"[SDK] Blocked background Task, user={user_id}")+ return cast(+ SyncHookJSONOutput,+ _deny(+ "Background task execution is not supported. "+ "Run tasks in the foreground instead "+ "(remove the run_in_background parameter)."+ ),+ )+ if task_spawn_count >= max_subtasks:logger.warning(f"[SDK] Task limit reached ({max_subtasks}), user={user_id}")@@ -200,6 +211,7 @@ def create_security_hooks("Please continue in the main conversation."),)+ task_spawn_count += 1# Strip MCP prefix for consistent validationis_copilot_tool = tool_name.startswith(MCP_TOOL_PREFIX)diff --git a/autogpt_platform/backend/backend/copilot/sdk/security_hooks_test.py b/autogpt_platform/backend/backend/copilot/sdk/security_hooks_test.pyindex e1891cf1bd..68a6925689 100644--- a/autogpt_platform/backend/backend/copilot/sdk/security_hooks_test.py+++ b/autogpt_platform/backend/backend/copilot/sdk/security_hooks_test.py@@ -7,11 +7,23 @@ tool access, and dangerous input patterns.import os+import pytest+from .security_hooks import _validate_tool_access, _validate_user_isolation+from .service import _is_tool_error_or_denialSDK_CWD = "/tmp/copilot-abc123"+def _sdk_available() -> bool:+ try:+ import claude_agent_sdk # noqa: F401++ return True+ except ImportError:+ return False++def _is_denied(result: dict) -> bool:hook = result.get("hookSpecificOutput", {})return hook.get("permissionDecision") == "deny"@@ -188,3 +200,135 @@ def test_bash_builtin_blocked_message_clarity():reason = _reason(_validate_tool_access("Bash", {"command": "echo hello"}))assert "[SECURITY]" in reasonassert "cannot be bypassed" in reason+++# -- Task sub-agent hooks (require SDK) --------------------------------------+++def _hooks():+ """Create security hooks and return the PreToolUse handler."""+ from .security_hooks import create_security_hooks++ hooks = create_security_hooks(user_id="u1", sdk_cwd=SDK_CWD, max_subtasks=2)+ pre = hooks["PreToolUse"][0].hooks[0]+ return pre++[email protected](not _sdk_available(), reason="claude_agent_sdk not installed")+async def test_task_background_blocked(_hooks):+ """Task with run_in_background=true must be denied."""+ result = await _hooks(+ {"tool_name": "Task", "tool_input": {"run_in_background": True, "prompt": "x"}},+ tool_use_id=None,+ context={},+ )+ assert _is_denied(result)+ assert "foreground" in _reason(result).lower()++[email protected](not _sdk_available(), reason="claude_agent_sdk not installed")+async def test_task_foreground_allowed(_hooks):+ """Task without run_in_background should be allowed."""+ result = await _hooks(+ {"tool_name": "Task", "tool_input": {"prompt": "do stuff"}},+ tool_use_id=None,+ context={},+ )+ assert not _is_denied(result)++[email protected](not _sdk_available(), reason="claude_agent_sdk not installed")+async def test_task_limit_enforced(_hooks):+ """Task spawns beyond max_subtasks should be denied."""+ # First two should pass+ for _ in range(2):+ result = await _hooks(+ {"tool_name": "Task", "tool_input": {"prompt": "ok"}},+ tool_use_id=None,+ context={},+ )+ assert not _is_denied(result)++ # Third should be denied (limit=2)+ result = await _hooks(+ {"tool_name": "Task", "tool_input": {"prompt": "over limit"}},+ tool_use_id=None,+ context={},+ )+ assert _is_denied(result)+ assert "Maximum" in _reason(result)+++# -- _is_tool_error_or_denial ------------------------------------------------+++class TestIsToolErrorOrDenial:+ def test_none_content(self):+ assert _is_tool_error_or_denial(None) is False++ def test_empty_content(self):+ assert _is_tool_error_or_denial("") is False++ def test_benign_output(self):+ assert _is_tool_error_or_denial("All good, no issues.") is False++ def test_security_marker(self):+ assert _is_tool_error_or_denial("[SECURITY] Tool access blocked") is True++ def test_cannot_be_bypassed(self):+ assert _is_tool_error_or_denial("This restriction cannot be bypassed.") is True++ def test_not_allowed(self):+ assert _is_tool_error_or_denial("Operation not allowed in sandbox") is True++ def test_background_task_denial(self):+ assert (+ _is_tool_error_or_denial(+ "Background task execution is not supported. "+ "Run tasks in the foreground instead."+ )+ is True+ )++ def test_subtask_limit_denial(self):+ assert (+ _is_tool_error_or_denial(+ "Maximum 2 sub-tasks per session. Please continue in the main conversation."+ )+ is True+ )++ def test_denied_marker(self):+ assert (+ _is_tool_error_or_denial("Access denied: insufficient privileges") is True+ )++ def test_blocked_marker(self):+ assert _is_tool_error_or_denial("Request blocked by security policy") is True++ def test_failed_marker(self):+ assert _is_tool_error_or_denial("Failed to execute tool: timeout") is True++ def test_mcp_iserror(self):+ assert _is_tool_error_or_denial('{"isError": true, "content": []}') is True++ def test_benign_error_in_value(self):+ """Content like '0 errors found' should not trigger — 'error' was removed."""+ assert _is_tool_error_or_denial("0 errors found") is False++ def test_benign_permission_field(self):+ """Schema descriptions mentioning 'permission' should not trigger."""+ assert (+ _is_tool_error_or_denial(+ '{"fields": [{"name": "permission_level", "type": "int"}]}'+ )+ is False+ )++ def test_benign_not_found_in_listing(self):+ """File listing containing 'not found' in filenames should not trigger."""+ assert _is_tool_error_or_denial("readme.md\nfile-not-found-handler.py") is Falsediff --git a/autogpt_platform/backend/backend/copilot/sdk/service.py b/autogpt_platform/backend/backend/copilot/sdk/service.pyindex 15777fc89e..ae2f41390f 100644--- a/autogpt_platform/backend/backend/copilot/sdk/service.py+++ b/autogpt_platform/backend/backend/copilot/sdk/service.py@@ -24,6 +24,7 @@ from ..response_model import (StreamBaseResponse,StreamError,StreamFinish,+ StreamHeartbeat,StreamStart,StreamTextDelta,StreamToolInputAvailable,@@ -76,6 +77,9 @@ class CapturedTranscript:_SDK_CWD_PREFIX = WORKSPACE_PREFIX+# Heartbeat interval — keep SSE alive through proxies/LBs during tool execution.+_HEARTBEAT_INTERVAL = 15.0 # seconds+# Appended to the system prompt to inform the agent about available tools.# The SDK built-in Bash is NOT available — use mcp__copilot__bash_exec instead,# which has kernel-level network isolation (unshare --net).@@ -96,6 +100,8 @@ _SDK_TOOL_SUPPLEMENT = """- Long-running tools (create_agent, edit_agent, etc.) are handledasynchronously. You will receive an immediate response; the actual resultis delivered to the user via a background stream.+- When using the Task tool, NEVER set `run_in_background` to true.+ All tasks must run in the foreground."""@@ -393,7 +399,12 @@ def _format_conversation_context(messages: list[ChatMessage]) -> str | None:lines.append(f"User: {msg.content}")elif msg.role == "assistant":lines.append(f"You responded: {msg.content}")- # Skip tool messages — they're internal details+ elif msg.role == "tool":+ # Include tool error/denial outcomes so the agent doesn't+ # hallucinate that blocked or failed operations succeeded.+ content = msg.content+ if _is_tool_error_or_denial(content):+ lines.append(f"Tool result: {content}")if not lines:return None@@ -401,6 +412,31 @@ def _format_conversation_context(messages: list[ChatMessage]) -> str | None:return "<conversation_history>\n" + "\n".join(lines) + "\n</conversation_history>"+def _is_tool_error_or_denial(content: str | None) -> bool:+ """Check if a tool message content indicates an error or denial.++ We include these in conversation context so the agent doesn't+ hallucinate success for operations that actually failed.+ """+ if not content:+ return False+ lower = content.lower()+ return any(+ marker in lower+ for marker in (+ "[security]",+ "cannot be bypassed",+ "not allowed",+ "not supported", # background-task denial+ "maximum", # subtask-limit denial+ "denied",+ "blocked",+ "failed", # internal tool execution failures+ '"iserror": true', # MCP protocol error flag+ )+ )++async def stream_chat_completion_sdk(session_id: str,message: str | None = None,@@ -622,7 +658,23 @@ async def stream_chat_completion_sdk(has_appended_assistant = Falsehas_tool_results = False- async for sdk_msg in client.receive_messages():+ # Use an explicit async iterator with timeout to send+ # heartbeats when the CLI is idle (e.g. executing tools).+ # This prevents proxies/LBs from closing the SSE connection.+ # asyncio.timeout() is preferred over asyncio.wait_for()+ # because wait_for wraps in a separate Task whose cancellation+ # can leave the async generator in a broken state.+ msg_iter = client.receive_messages().__aiter__()+ while not stream_completed:+ try:+ async with asyncio.timeout(_HEARTBEAT_INTERVAL):+ sdk_msg = await msg_iter.__anext__()+ except TimeoutError:+ yield StreamHeartbeat()+ continue+ except StopAsyncIteration:+ break+logger.debug(f"[SDK] Received: {type(sdk_msg).__name__} "f"{getattr(sdk_msg, 'subtype', '')}"@@ -631,6 +683,17 @@ async def stream_chat_completion_sdk(if isinstance(response, StreamStart):continue+ # Log tool events for debugging visibility issues+ if isinstance(+ response,+ (StreamToolInputAvailable, StreamToolOutputAvailable),+ ):+ logger.info(+ "[SDK] Tool event: %s, tool=%s",+ type(response).__name__,+ getattr(response, "toolName", "N/A"),+ )+yield responseif isinstance(response, StreamTextDelta):@@ -687,9 +750,6 @@ async def stream_chat_completion_sdk(elif isinstance(response, StreamFinish):stream_completed = True- if stream_completed:- break-if (assistant_response.content or assistant_response.tool_calls) and not has_appended_assistant:@@ -704,11 +764,24 @@ async def stream_chat_completion_sdk(# complete). Otherwise use the Stop hook path.if use_resume and resume_file:raw_transcript = read_transcript_file(resume_file)+ logger.debug("[SDK] Transcript source: resume file")elif captured_transcript.path:raw_transcript = read_transcript_file(captured_transcript.path)+ logger.debug(+ "[SDK] Transcript source: stop hook (%s), " "read result: %s",+ captured_transcript.path,+ f"{len(raw_transcript)}B" if raw_transcript else "None",+ )else:raw_transcript = None+ if not raw_transcript:+ logger.debug(+ "[SDK] No usable transcript — CLI file had no "+ "conversation entries (expected for first turn "+ "without --resume)"+ )+if raw_transcript:# Shield the upload from generator cancellation so a# client disconnect / page refresh doesn't lose thediff --git a/autogpt_platform/backend/backend/copilot/sdk/transcript.py b/autogpt_platform/backend/backend/copilot/sdk/transcript.pyindex adab96b98b..39a9b1352c 100644--- a/autogpt_platform/backend/backend/copilot/sdk/transcript.py+++ b/autogpt_platform/backend/backend/copilot/sdk/transcript.py@@ -131,17 +131,20 @@ def read_transcript_file(transcript_path: str) -> str | None:content = f.read()if not content.strip():+ logger.debug("[Transcript] File is empty: %s", transcript_path)return Nonelines = content.strip().split("\n")- if len(lines) < 3:- # Raw files with ≤2 lines are metadata-only- # (queue-operation + file-history-snapshot, no conversation).- return None- # Quick structural validation — parse first and last lines.- json.loads(lines[0])- json.loads(lines[-1])+ # Validate that the transcript has real conversation content+ # (not just metadata like queue-operation entries).+ if not validate_transcript(content):+ logger.debug(+ "[Transcript] No conversation content (%d lines) in %s",+ len(lines),+ transcript_path,+ )+ return Nonelogger.info(f"[Transcript] Read {len(lines)} lines, "diff --git a/autogpt_platform/backend/backend/copilot/service_test.py b/autogpt_platform/backend/backend/copilot/service_test.pyindex 347dd9e1a2..133a4d81a1 100644--- a/autogpt_platform/backend/backend/copilot/service_test.py+++ b/autogpt_platform/backend/backend/copilot/service_test.py@@ -132,17 +132,23 @@ async def test_sdk_resume_multi_turn(setup_test_user, test_user_id):assert not turn1_errors, f"Turn 1 errors: {turn1_errors}"assert turn1_text, "Turn 1 produced no text"- # Wait for background upload task to complete (retry up to 5s)+ # Wait for background upload task to complete (retry up to 5s).+ # The CLI may not produce a usable transcript for very short+ # conversations (only metadata entries) — this is environment-dependent+ # (CLI version, platform). When that happens, multi-turn still works+ # via conversation compression (non-resume path), but we can't test+ # the --resume round-trip.transcript = Nonefor _ in range(10):await asyncio.sleep(0.5)transcript = await download_transcript(test_user_id, session.session_id)if transcript:break- assert transcript, (- "Transcript was not uploaded to bucket after turn 1 — "- "Stop hook may not have fired or transcript was too small"- )+ if not transcript:+ return pytest.skip(+ "CLI did not produce a usable transcript — "+ "cannot test --resume round-trip in this environment"+ )logger.info(f"Turn 1 transcript uploaded: {len(transcript.content)} bytes")# Reload session for turn 2diff --git a/autogpt_platform/backend/backend/copilot/stream_registry.py b/autogpt_platform/backend/backend/copilot/stream_registry.pyindex 7579b54134..6510b45296 100644--- a/autogpt_platform/backend/backend/copilot/stream_registry.py+++ b/autogpt_platform/backend/backend/copilot/stream_registry.py@@ -227,7 +227,14 @@ async def publish_chunk(# Only log timing for significant chunks or slow operationsif (chunk_type- in ("StreamStart", "StreamFinish", "StreamTextStart", "StreamTextEnd")+ in (+ "StreamStart",+ "StreamFinish",+ "StreamTextStart",+ "StreamTextEnd",+ "StreamToolInputAvailable",+ "StreamToolOutputAvailable",+ )or total_time > 50):logger.info(diff --git a/autogpt_platform/backend/backend/copilot/tools/sandbox.py b/autogpt_platform/backend/backend/copilot/tools/sandbox.pyindex beb326f909..aa15637822 100644--- a/autogpt_platform/backend/backend/copilot/tools/sandbox.py+++ b/autogpt_platform/backend/backend/copilot/tools/sandbox.py@@ -13,6 +13,7 @@ import loggingimport osimport platformimport shutil+import signallogger = logging.getLogger(__name__)@@ -245,6 +246,7 @@ async def run_sandboxed(stderr=asyncio.subprocess.PIPE,cwd=cwd,env=safe_env,+ start_new_session=True, # Own process group for clean kill)try:@@ -255,7 +257,18 @@ async def run_sandboxed(stderr = stderr_bytes.decode("utf-8", errors="replace")return stdout, stderr, proc.returncode or 0, Falseexcept asyncio.TimeoutError:- proc.kill()+ # Kill entire process group (bwrap + all children).+ # proc.kill() alone only kills the bwrap parent, leaving+ # children running until they finish naturally.+ try:+ os.killpg(proc.pid, signal.SIGKILL)+ except ProcessLookupError:+ pass # Already exited+ except OSError as kill_err:+ logger.warning(+ "Failed to kill process group %d: %s", proc.pid, kill_err+ )+ # Always reap the subprocess regardless of killpg outcome.await proc.communicate()return "", f"Execution timed out after {timeout}s", -1, Truediff --git a/autogpt_platform/backend/backend/util/prompt.py b/autogpt_platform/backend/backend/util/prompt.pyindex 3ec25dd61b..94fd251a36 100644--- a/autogpt_platform/backend/backend/util/prompt.py+++ b/autogpt_platform/backend/backend/util/prompt.py@@ -535,14 +535,18 @@ async def _summarize_messages_llm({"role": "system","content": (- "Create a detailed summary of the conversation so far. "+ "Create a factual summary of the conversation so far. ""This summary will be used as context when continuing the conversation.\n\n"+ "CRITICAL: Only include information that is EXPLICITLY present in the "+ "conversation. Do NOT fabricate, infer, or invent any details. "+ "If a section has no relevant content in the conversation, skip it entirely.\n\n""Before writing the summary, analyze each message chronologically to identify:\n""- User requests and their explicit goals\n"- "- Your approach and key decisions made\n"+ "- Actions taken and key decisions made\n""- Technical specifics (file names, tool outputs, function signatures)\n""- Errors encountered and resolutions applied\n\n"- "You MUST include ALL of the following sections:\n\n"+ "Include ONLY the sections below that have relevant content "+ "(skip sections with nothing to report):\n\n""## 1. Primary Request and Intent\n""The user's explicit goals and what they are trying to accomplish.\n\n""## 2. Key Technical Concepts\n"@@ -550,19 +554,14 @@ async def _summarize_messages_llm("## 3. Files and Resources Involved\n""Specific files examined or modified, with relevant snippets and identifiers.\n\n""## 4. Errors and Fixes\n"- "Problems encountered, error messages, and their resolutions. "- "Include any user feedback on fixes.\n\n"- "## 5. Problem Solving\n"- "Issues that have been resolved and how they were addressed.\n\n"- "## 6. All User Messages\n"- "A complete list of all user inputs (excluding tool outputs) to preserve their exact requests.\n\n"- "## 7. Pending Tasks\n"+ "Problems encountered, error messages, and their resolutions.\n\n"+ "## 5. All User Messages\n"+ "A complete list of all user inputs (excluding tool outputs) "+ "to preserve their exact requests.\n\n"+ "## 6. Pending Tasks\n""Work items the user explicitly requested that have not yet been completed.\n\n"- "## 8. Current Work\n"- "Precise description of what was being worked on most recently, including relevant context.\n\n"- "## 9. Next Steps\n"- "What should happen next, aligned with the user's most recent requests. "- "Include verbatim quotes of recent instructions if relevant."+ "## 7. Current State\n"+ "What was happening most recently in the conversation."),},{"role": "user", "content": f"Summarize:\n\n{conversation_text}"},