Agent Zero AI framework
Fix MCP timeouts from hanging agent loop
Alessandro8fda0ee69c8b
diff --git a/helpers/mcp_handler.py b/helpers/mcp_handler.pyindex 6aadfc80c..2b4e0d8b9 100644--- a/helpers/mcp_handler.py+++ b/helpers/mcp_handler.py@@ -43,10 +43,13 @@ from pydantic import BaseModel, Field, Discriminator, Tag, PrivateAttrfrom helpers import dirty_json, media_artifactsfrom helpers.print_style import PrintStylefrom helpers.tool import Tool, Response+from helpers.defer import DeferredTaskMCP_MEDIA_TOKENS_ESTIMATE = 1500MAX_MCP_RESOURCE_TEXT_CHARS = 12_000+MCP_SESSION_CLEANUP_TIMEOUT_SECONDS = 5.0+MCP_OPERATION_TIMEOUT_GRACE_SECONDS = MCP_SESSION_CLEANUP_TIMEOUT_SECONDS + 2.0DEFAULT_MCP_SERVERS_CONFIG = '{\n "mcpServers": {}\n}'@@ -661,10 +664,10 @@ class MCPConfig(BaseModel):@classmethoddef get_instance(cls) -> "MCPConfig":- # with cls.__lock:- if cls.__instance is None:- cls.__instance = cls(servers_list=[], config_scope="global")- return cls.__instance+ with cls.__lock:+ if cls.__instance is None:+ cls.__instance = cls(servers_list=[], config_scope="global")+ return cls.__instance@classmethoddef clear_project_instances(cls):@@ -791,35 +794,20 @@ class MCPConfig(BaseModel):@classmethoddef update(cls, config_str: str) -> Any:+ servers_data = cls.parse_config_string(config_str)+ new_instance = cls(servers_list=servers_data, config_scope="global")with cls.__lock:- servers_data = cls.parse_config_string(config_str)-- # Initialize/update the singleton instance with the (potentially empty) list of server data- instance = cls.get_instance()- # Directly update the servers attribute of the existing instance or re-initialize carefully- # For simplicity and to ensure __init__ logic runs if needed for setup:- new_instance_data = {- "servers": servers_data- } # Prepare data for re-initialization or update-- # Option 1: Re-initialize the existing instance (if __init__ is idempotent for other fields)- instance.__init__(servers_list=servers_data, config_scope="global")+ # Build and initialize outside the class lock so a slow or wedged MCP+ # server cannot freeze status reads, prompts, or later tool calls.+ instance = cls.__instance+ if instance is None:+ instance = new_instance+ cls.__instance = instance+ else:+ instance.servers = new_instance.servers+ instance.disconnected_servers = new_instance.disconnected_servers+ instance.config_scope = new_instance.config_scopecls.__project_instances = {}-- # Option 2: Or, if __init__ has side effects we don't want to repeat,- # and 'servers' is the primary thing 'update' changes:- # instance.servers = [] # Clear existing servers first- # for server_item_data in servers_data:- # try:- # if server_item_data.get("url", None):- # instance.servers.append(MCPServerRemote(server_item_data))- # else:- # instance.servers.append(MCPServerLocal(server_item_data))- # except Exception as e_init:- # PrintStyle(background_color="grey", font_color="red", padding=True).print(- # f"MCPConfig.update: Failed to create MCPServer from item '{server_item_data.get('name', 'Unknown')}': {e_init}"- # )-cls.__initialized = Truereturn instance@@ -1146,11 +1134,15 @@ class MCPConfig(BaseModel):) -> CallToolResult:"""Call a tool with the given input data"""server_name_part, tool_name_part = _split_qualified_tool_name(tool_name)+ matched_server = Nonewith self.__lock:for server in self.servers:if server.name == server_name_part and server.has_tool(tool_name_part):- return await server.call_tool(tool_name_part, input_data)+ matched_server = server+ break+ if matched_server is None:raise ValueError(f"Tool {tool_name} not found")+ return await matched_server.call_tool(tool_name_part, input_data)T = TypeVar("T")@@ -1170,6 +1162,49 @@ class MCPClientBase(ABC):self.log: List[str] = []self.log_file: Optional[TextIO] = None+ def _operation_timeout_seconds(self, read_timeout_seconds: float) -> float:+ try:+ seconds = float(read_timeout_seconds)+ except (TypeError, ValueError):+ seconds = 60.0+ if seconds <= 0:+ seconds = 60.0+ return seconds + MCP_OPERATION_TIMEOUT_GRACE_SECONDS++ def _operation_thread_name(self, operation_name: str) -> str:+ server_name = normalize_name(str(getattr(self.server, "name", "") or "server"))+ return f"MCPClient-{server_name[:32] or 'server'}-{operation_name}-{uuid.uuid4().hex[:8]}"++ async def _run_isolated_operation(+ self,+ operation_name: str,+ operation: Callable[[], Awaitable[T]],+ timeout_seconds: float,+ ) -> T:+ worker = DeferredTask(thread_name=self._operation_thread_name(operation_name))+ timed_out = False+ try:+ return await asyncio.wait_for(+ worker.execute_inside(operation),+ timeout=timeout_seconds,+ )+ except asyncio.TimeoutError as exc:+ timed_out = True+ message = (+ f"MCPClientBase ({self.server.name} - {operation_name}): "+ f"operation did not finish within {timeout_seconds:.1f}s; "+ "abandoning the isolated worker so Agent Zero can continue."+ )+ PrintStyle.warning(message)+ with self.__lock:+ self.error = message+ raise TimeoutError(message) from exc+ finally:+ if timed_out:+ worker.kill(terminate_thread=False)+ else:+ worker.kill(terminate_thread=True)+# Protected method@abstractmethodasync def _create_stdio_transport(@@ -1192,54 +1227,56 @@ class MCPClientBase(ABC):"""operation_name = coro_func.__name__ # For logging# PrintStyle(font_color="cyan").print(f"MCPClientBase ({self.server.name}): Creating new session for operation '{operation_name}'...")- # Store the original exception outside the async blockoriginal_exception = None+ result: T | None = None+ has_result = False+ temp_stack = AsyncExitStack()try:- async with AsyncExitStack() as temp_stack:- try:-- stdio, write = await self._create_stdio_transport(temp_stack)- # PrintStyle(font_color="cyan").print(f"MCPClientBase ({self.server.name} - {operation_name}): Transport created. Initializing session...")- session = await temp_stack.enter_async_context(- ClientSession(- stdio, # type: ignore- write, # type: ignore- read_timeout_seconds=timedelta(- seconds=read_timeout_seconds- ),- )- )- await session.initialize()-- result = await coro_func(session)+ stdio, write = await self._create_stdio_transport(temp_stack)+ # PrintStyle(font_color="cyan").print(f"MCPClientBase ({self.server.name} - {operation_name}): Transport created. Initializing session...")+ session = await temp_stack.enter_async_context(+ ClientSession(+ stdio, # type: ignore+ write, # type: ignore+ read_timeout_seconds=timedelta(+ seconds=read_timeout_seconds+ ),+ )+ )+ await session.initialize()- return result- except Exception as e:- # Store the original exception and raise a dummy exception- excs = getattr(e, "exceptions", None) # Python 3.11+ ExceptionGroup- if excs:- original_exception = excs[0]- else:- original_exception = e- # Create a dummy exception to break out of the async block- raise RuntimeError("Dummy exception to break out of async block")+ result = await coro_func(session)+ has_result = Trueexcept Exception as e:- # Check if this is our dummy exception- if original_exception is not None:- e = original_exception- # We have the original exception stored+ excs = getattr(e, "exceptions", None) # Python 3.11+ ExceptionGroup+ if excs:+ original_exception = excs[0]+ else:+ original_exception = e+ try:+ await asyncio.wait_for(+ temp_stack.aclose(),+ timeout=MCP_SESSION_CLEANUP_TIMEOUT_SECONDS,+ )+ except asyncio.TimeoutError:+ PrintStyle.warning(+ f"MCPClientBase ({self.server.name} - {operation_name}): "+ f"session cleanup exceeded {MCP_SESSION_CLEANUP_TIMEOUT_SECONDS:.1f}s."+ )+ except Exception as cleanup_exception:+ PrintStyle.warning(+ f"MCPClientBase ({self.server.name} - {operation_name}): "+ f"session cleanup failed: {type(cleanup_exception).__name__}: {cleanup_exception}"+ )+ if original_exception is not None:PrintStyle(background_color="#AA4455", font_color="white", padding=False).print(- f"MCPClientBase ({self.server.name} - {operation_name}): Error during operation: {type(e).__name__}: {e}"+ f"MCPClientBase ({self.server.name} - {operation_name}): Error during operation: {type(original_exception).__name__}: {original_exception}")- raise e # Re-raise the original exception- # finally:- # PrintStyle(font_color="cyan").print(- # f"MCPClientBase ({self.server.name} - {operation_name}): Session and transport will be closed by AsyncExitStack."- # )- # This line should ideally be unreachable if the try/except/finally logic within the 'async with' is exhaustive.- # Adding it to satisfy linters that might not fully trace the raise/return paths through async context managers.+ raise original_exception+ if has_result:+ return cast(T, result)raise RuntimeError(f"MCPClientBase ({self.server.name} - {operation_name}): _execute_with_session exited 'async with' block unexpectedly.")@@ -1270,9 +1307,13 @@ class MCPClientBase(ABC):or current_settings.get("mcp_client_init_timeout", 10)or 10)- await self._execute_with_session(- list_tools_op,- read_timeout_seconds=init_timeout,+ await self._run_isolated_operation(+ "update_tools",+ lambda: self._execute_with_session(+ list_tools_op,+ read_timeout_seconds=init_timeout,+ ),+ timeout_seconds=self._operation_timeout_seconds(init_timeout),)except Exception as e:# e = eg.exceptions[0]@@ -1339,10 +1380,17 @@ class MCPClientBase(ABC):return responsetry:- return await self._execute_with_session(- call_tool_op,- read_timeout_seconds=tool_timeout,+ response = await self._run_isolated_operation(+ "call_tool",+ lambda: self._execute_with_session(+ call_tool_op,+ read_timeout_seconds=tool_timeout,+ ),+ timeout_seconds=self._operation_timeout_seconds(tool_timeout),)+ with self.__lock:+ self.error = ""+ return responseexcept Exception as e:# Error logged by _execute_with_session. Re-raise a specific error for the caller.PrintStyle(diff --git a/helpers/mcp_handler.py.dox.md b/helpers/mcp_handler.py.dox.mdindex d645a0b16..906a6f7b1 100644--- a/helpers/mcp_handler.py.dox.md+++ b/helpers/mcp_handler.py.dox.md@@ -67,7 +67,7 @@- `_split_qualified_tool_name(tool_name: str) -> tuple[str, str]`: Split `server.tool` names while preserving dots inside MCP tool names.- `_normalize_disabled_tools(value: Any) -> list[str]`: Normalize the optional per-server disabled tool list.- `initialize_mcp(mcp_servers_config: str)`-- Notable constants/configuration names: `DEFAULT_MCP_SERVERS_CONFIG`, `MCP_MEDIA_TOKENS_ESTIMATE`, `MAX_MCP_RESOURCE_TEXT_CHARS`, `T`.+- Notable constants/configuration names: `DEFAULT_MCP_SERVERS_CONFIG`, `MCP_MEDIA_TOKENS_ESTIMATE`, `MAX_MCP_RESOURCE_TEXT_CHARS`, `MCP_SESSION_CLEANUP_TIMEOUT_SECONDS`, `MCP_OPERATION_TIMEOUT_GRACE_SECONDS`, `T`.## Runtime Contracts@@ -81,13 +81,15 @@- MCP tool names are qualified as `server_name.tool_name`; server names are normalized without dots, and the tool portion may contain dots.- Servers may define `disabled_tools` as a list of MCP tool names. Disabled tools are omitted from agent-facing prompts, status counts, `has_tool`, and calls, while detail views can still retrieve them through `get_all_tools()` with a `disabled` flag so users can re-enable them.- Server-specific `init_timeout` and `tool_timeout` override global MCP client timeout settings for list-tools and call-tool operations.+- MCP config locks must not be held across awaited server initialization or tool-call operations. Slow or wedged MCP servers must not block status reads, prompt construction, unrelated MCP servers, or later tool calls through the shared config lock.+- MCP client session work runs inside disposable isolated `DeferredTask` workers with an outer timeout. Normal `AsyncExitStack` cleanup is also bounded; if cleanup or transport shutdown does not finish, the operation reports failure or warning while Agent Zero keeps control of the agent loop.- Server status marks initialized server objects with cached initialization errors as disconnected, even if the config object exists.- Observed side-effect areas: filesystem writes, network calls, WebSocket state, settings/state persistence, secret handling.-- Imported dependency areas include: `abc`, `anyio.streams.memory`, `asyncio`, `contextlib`, `datetime`, `helpers`, `helpers.log`, `helpers.print_style`, `helpers.tool`, `httpx`, `json`, `mcp`, `mcp.client.sse`, `mcp.client.stdio`, `mcp.client.streamable_http`, `mcp.shared.message`.+- Imported dependency areas include: `abc`, `anyio.streams.memory`, `asyncio`, `contextlib`, `datetime`, `helpers`, `helpers.defer`, `helpers.log`, `helpers.print_style`, `helpers.tool`, `httpx`, `json`, `mcp`, `mcp.client.sse`, `mcp.client.stdio`, `mcp.client.streamable_http`, `mcp.shared.message`.## Key Concepts-- Important called helpers/classes observed in the source: `TypeVar`, `name.strip.lower`, `re.sub`, `Field`, `PrivateAttr`, `threading.Lock`, `_split_qualified_tool_name`, `config_dict.lower`, `server_type.lower`, `MCPConfig.get_instance.is_initialized`, `MCPConfig.get_for_agent`, `projects.validate_project_name`, `projects.load_project_mcp_servers`, `settings.get_settings`, `self.agent.context.log.log`, `str.strip`, `media_artifacts.guess_extension`, `callable`, `self._content_item_dump`, `join`, `Response`, `self.get_log_object`, `self._raw_tool_response`, `additional.pop`, `self._coerce_media_token_estimate`.+- Important called helpers/classes observed in the source: `TypeVar`, `name.strip.lower`, `re.sub`, `Field`, `PrivateAttr`, `threading.Lock`, `DeferredTask`, `_split_qualified_tool_name`, `config_dict.lower`, `server_type.lower`, `MCPConfig.get_instance.is_initialized`, `MCPConfig.get_for_agent`, `projects.validate_project_name`, `projects.load_project_mcp_servers`, `settings.get_settings`, `self.agent.context.log.log`, `str.strip`, `media_artifacts.guess_extension`, `callable`, `self._content_item_dump`, `join`, `Response`, `self.get_log_object`, `self._raw_tool_response`, `additional.pop`, `self._coerce_media_token_estimate`.- Keep request/response, tool, or helper semantics documented here at the same time as source changes.## Work Guidance@@ -95,6 +97,7 @@- Preserve public helper APIs used by core code and plugins unless every caller is updated.- Keep path, auth, secret, persistence, network, and subprocess behavior explicit and bounded.- Prefer adding cohesive helper functions here only when behavior is reused across modules.+- Keep MCP timeout and cleanup changes covered by deterministic tests that do not require real MCP servers or network credentials.## Verificationdiff --git a/tests/test_mcp_handler_multimodal.py b/tests/test_mcp_handler_multimodal.pyindex 421fa4224..040955e0e 100644--- a/tests/test_mcp_handler_multimodal.py+++ b/tests/test_mcp_handler_multimodal.py@@ -54,6 +54,20 @@ class _FakeCallToolResult(SimpleNamespace):pass+class _TrackingLock:+ def __init__(self):+ self.held = False++ def __enter__(self):+ assert self.held is False+ self.held = True+ return self++ def __exit__(self, exc_type, exc, tb):+ self.held = False+ return False++@pytest.fixturedef mcp_handler_module(monkeypatch, tmp_path):monkeypatch.delitem(sys.modules, "helpers.mcp_handler", raising=False)@@ -118,6 +132,10 @@ def mcp_handler_module(monkeypatch, tmp_path):def stream(self, *args, **kwargs):return self+ @staticmethod+ def warning(*args, **kwargs):+ return None+def _fake_get_abs_path(*parts):return str(tmp_path.joinpath(*parts))@@ -194,6 +212,57 @@ def test_mcp_config_preserves_dotted_tool_names(mcp_handler_module):assert called == [("alpha.beta", {"value": 7})]+def test_mcp_config_call_tool_releases_config_lock_before_await(+ mcp_handler_module, monkeypatch+):+ module, _tmp_path = mcp_handler_module+ lock = _TrackingLock()+ observed_lock_state: list[bool] = []++ monkeypatch.setattr(module.MCPConfig, "_MCPConfig__lock", lock, raising=False)++ class _FakeServer:+ name = "server"+ description = "Fake MCP server"+ type = "stdio"+ scope = "global"++ def has_tool(self, tool_name):+ return tool_name == "run"++ async def call_tool(self, tool_name, input_data):+ observed_lock_state.append(lock.held)+ await asyncio.sleep(0)+ return _FakeCallToolResult(content=[], isError=False)++ config = module.MCPConfig(servers_list=[])+ config.servers = [_FakeServer()]++ asyncio.run(config.call_tool("server.run", {}))++ assert observed_lock_state == [False]+++def test_mcp_config_update_initializes_outside_config_lock(+ mcp_handler_module, monkeypatch+):+ module, _tmp_path = mcp_handler_module+ lock = _TrackingLock()+ observed_lock_state: list[bool] = []+ original_init = module.MCPConfig.__init__++ def tracking_init(self, *args, **kwargs):+ observed_lock_state.append(lock.held)+ original_init(self, *args, **kwargs)++ monkeypatch.setattr(module.MCPConfig, "_MCPConfig__lock", lock, raising=False)+ monkeypatch.setattr(module.MCPConfig, "__init__", tracking_init)++ module.MCPConfig.update('{"mcpServers": {}}')++ assert observed_lock_state[-1] is False++def test_mcp_status_marks_servers_with_errors_disconnected(mcp_handler_module):module, _tmp_path = mcp_handler_module@@ -310,6 +379,69 @@ def test_mcp_client_call_tool_uses_server_tool_timeout(mcp_handler_module, monkeassert call_timeouts[0].total_seconds() == 7+def test_mcp_session_cleanup_timeout_does_not_mask_success(+ mcp_handler_module, monkeypatch+):+ module, _tmp_path = mcp_handler_module+ monkeypatch.setattr(module, "MCP_SESSION_CLEANUP_TIMEOUT_SECONDS", 0.01)++ class _HangingTransport:+ async def __aenter__(self):+ return "stdio", "write"++ async def __aexit__(self, exc_type, exc, tb):+ await asyncio.sleep(60)++ class _FakeSession:+ def __init__(self, *args, **kwargs):+ pass++ async def __aenter__(self):+ return self++ async def __aexit__(self, exc_type, exc, tb):+ return False++ async def initialize(self):+ pass++ class _FakeClient(module.MCPClientBase):+ async def _create_stdio_transport(self, current_exit_stack):+ return await current_exit_stack.enter_async_context(_HangingTransport())++ async def operation(_session):+ return "ok"++ monkeypatch.setattr(module, "ClientSession", _FakeSession)+ client = _FakeClient(SimpleNamespace(name="server"))++ assert asyncio.run(client._execute_with_session(operation)) == "ok"+++def test_mcp_isolated_operation_timeout_returns_control(mcp_handler_module):+ module, _tmp_path = mcp_handler_module++ class _FakeClient(module.MCPClientBase):+ async def _create_stdio_transport(self, current_exit_stack):+ raise AssertionError("transport should not be used")++ async def never_finishes():+ await asyncio.sleep(60)++ client = _FakeClient(SimpleNamespace(name="server"))++ with pytest.raises(TimeoutError):+ asyncio.run(+ client._run_isolated_operation(+ "wedged",+ never_finishes,+ timeout_seconds=0.01,+ )+ )++ assert "operation did not finish" in client.error++def test_mcp_image_content_becomes_history_image_attachment(mcp_handler_module, monkeypatch):module, _tmp_path = mcp_handler_moduleagent, log, tool_results, messages, updates, warnings = _agent_recorder()