RAGFlow is an agent-native database: a Retrieval-Augmented Generation (RAG) engine that grew into context infrastructure.
Go agent-harnessagent-native-databaseagentic-aiagentic-retrievalagentic-searchaiai-agentscontext-enginecontext-engineeringcontext-managementharness-engineeringknowledge-compilationragretrieval-augmented-generationretrieval-harnesssearch-harness
fix(agent): await MCP transport cleanup before stopping event loop (#19093)
Lem0nTea2002bef34757b2b0
diff --git a/api/ragflow_server.py b/api/ragflow_server.pyindex 47409e1348..438a763cc9 100644--- a/api/ragflow_server.py+++ b/api/ragflow_server.py@@ -49,6 +49,7 @@ from rag.utils.redis_conn import RedisDistributedLockstop_event = threading.Event()chat_channel_thread = None+shutdown_requested = FalseRAGFLOW_DEBUGPY_LISTEN = int(os.environ.get("RAGFLOW_DEBUGPY_LISTEN", "0"))@@ -81,13 +82,15 @@ def stop_background_services():def signal_handler(sig, frame):- logging.info("Received interrupt signal, shutting down...")- shutdown_all_mcp_sessions()- stop_background_services()+ global shutdown_requested+ if shutdown_requested:+ os.kill(os.getpid(), signal.SIGKILL)+ return+ shutdown_requested = Truesys.exit(0)-if __name__ == "__main__":+def run_server():faulthandler.enable()init_root_logger("ragflow_server")logging.info(r"""@@ -136,9 +139,6 @@ if __name__ == "__main__":GlobalPluginManager.load_plugins()- signal.signal(signal.SIGINT, signal_handler)- signal.signal(signal.SIGTERM, signal_handler)-def delayed_start_update_progress():logging.info("Starting update_progress thread (delayed)")t = threading.Thread(target=update_progress, daemon=True)@@ -169,12 +169,31 @@ if __name__ == "__main__":start_chat_channels()# start http server+ logging.info(f"RAGFlow server is ready after {time.time() - start_ts}s initialization.")+ app.run(host=settings.HOST_IP, port=settings.HOST_PORT, use_reloader=RuntimeConfig.DEBUG, debug=False)+++def main():+ force_kill = Falsetry:- logging.info(f"RAGFlow server is ready after {time.time() - start_ts}s initialization.")- app.run(host=settings.HOST_IP, port=settings.HOST_PORT, use_reloader=RuntimeConfig.DEBUG, debug=False)+ signal.signal(signal.SIGINT, signal_handler)+ signal.signal(signal.SIGTERM, signal_handler)+ run_server()except Exception as e:+ force_kill = Truelogging.exception(f"Unhandled exception: {e}")- stop_background_services()- os.kill(os.getpid(), signal.SIGKILL)finally:- stop_background_services()+ if shutdown_requested:+ logging.info("Received interrupt signal, shutting down...")+ try:+ shutdown_all_mcp_sessions()+ finally:+ try:+ stop_background_services()+ finally:+ if force_kill:+ os.kill(os.getpid(), signal.SIGKILL)+++if __name__ == "__main__":+ main()diff --git a/common/mcp_tool_call_conn.py b/common/mcp_tool_call_conn.pyindex b73b1ce671..7925bea71b 100644--- a/common/mcp_tool_call_conn.py+++ b/common/mcp_tool_call_conn.py@@ -19,7 +19,8 @@ import loggingimport threadingimport timeimport weakref-from concurrent.futures import ThreadPoolExecutor+from concurrent.futures import CancelledError as FuturesCancelledError+from concurrent.futures import Future, ThreadPoolExecutorfrom concurrent.futures import TimeoutError as FuturesTimeoutErrorfrom dataclasses import dataclassfrom string import Template@@ -51,21 +52,131 @@ class MCPToolBinding:class MCPToolCallSession(ToolCallSession):_ALL_INSTANCES: weakref.WeakSet["MCPToolCallSession"] = weakref.WeakSet()+ _INSTANCES_LOCK = threading.Lock()+ _GLOBAL_SHUTDOWN_COUNT = 0def __init__(self, mcp_server: Any, server_variables: dict[str, Any] | None = None, custom_header=None) -> None:- self.__class__._ALL_INSTANCES.add(self)-self._custom_header = custom_headerself._mcp_server = mcp_serverself._server_variables = server_variables or {}self._queue: asyncio.Queue[MCPTask] = asyncio.Queue()self._close = False+ self._pending_calls: dict[asyncio.Task[Any], asyncio.Queue[Any]] = {}+ self._shutdown_lock = threading.Lock()+ self._shutdown_future: Future[None] | None = None+ self._shutdown_complete = threading.Event()+ self._server_task: asyncio.Task[None] | None = None+ self._server_task_started = asyncio.Event()+ self._owner_close_waiters = 0+ self._stop_handle: asyncio.TimerHandle | None = Noneself._event_loop = asyncio.new_event_loop()- self._thread_pool = ThreadPoolExecutor(max_workers=1)- self._thread_pool.submit(self._event_loop.run_forever)+ self._thread = threading.Thread(target=self._run_event_loop, name=f"mcp-session-{mcp_server.id}", daemon=True)+ coroutine = self._run_mcp_server_loop()+ thread_started = False+ coroutine_submitted = False+ try:+ with self.__class__._INSTANCES_LOCK:+ if self.__class__._GLOBAL_SHUTDOWN_COUNT:+ raise RuntimeError("Cannot create MCP session during global shutdown")+ self._thread.start()+ thread_started = True+ self._server_future = asyncio.run_coroutine_threadsafe(coroutine, self._event_loop)+ coroutine_submitted = True+ self.__class__._ALL_INSTANCES.add(self)+ except BaseException as startup_error: # noqa: BLE001+ startup_traceback = startup_error.__traceback__+ was_started = thread_started or self._thread.ident is not None+ if was_started:+ try:+ if not self._event_loop.is_closed():+ self._event_loop.call_soon_threadsafe(self._event_loop.stop)+ except BaseException:+ logger.exception("Failed to stop MCP session event loop during startup rollback")+ try:+ self._thread.join()+ except BaseException:+ logger.exception("Failed to join MCP session thread during startup rollback")+ else:+ try:+ self._event_loop.close()+ except BaseException:+ logger.exception("Failed to close MCP session event loop during startup rollback")+ finally:+ self._shutdown_complete.set()+ try:+ if not coroutine_submitted and coroutine.cr_frame is not None:+ coroutine.close()+ except BaseException:+ logger.exception("Failed to close MCP server coroutine during startup rollback")+ try:+ with self.__class__._INSTANCES_LOCK:+ self.__class__._ALL_INSTANCES.discard(self)+ except BaseException:+ logger.exception("Failed to unregister MCP session during startup rollback")+ raise startup_error.with_traceback(startup_traceback)++ @classmethod+ def _active_instances(cls) -> list["MCPToolCallSession"]:+ with cls._INSTANCES_LOCK:+ return list(cls._ALL_INSTANCES)++ @classmethod+ def _begin_global_shutdown(cls) -> list["MCPToolCallSession"]:+ with cls._INSTANCES_LOCK:+ cls._GLOBAL_SHUTDOWN_COUNT += 1+ try:+ return cls._active_instances()+ except BaseException:+ cls._end_global_shutdown()+ raise- asyncio.run_coroutine_threadsafe(self._mcp_server_loop(), self._event_loop)+ @classmethod+ def _end_global_shutdown(cls) -> None:+ with cls._INSTANCES_LOCK:+ cls._GLOBAL_SHUTDOWN_COUNT -= 1++ def _run_event_loop(self) -> None:+ asyncio.set_event_loop(self._event_loop)+ try:+ self._event_loop.run_forever()+ finally:+ try:+ try:+ self._drain_pending_tasks()+ except Exception:+ logger.exception("Failed to drain pending tasks for MCP server %s", self._mcp_server.id)+ try:+ self._event_loop.run_until_complete(self._event_loop.shutdown_asyncgens())+ except Exception:+ logger.exception("Failed to shut down async generators for MCP server %s", self._mcp_server.id)+ try:+ self._drain_pending_tasks()+ except Exception:+ logger.exception("Failed to drain late tasks for MCP server %s", self._mcp_server.id)+ try:+ self._event_loop.run_until_complete(self._event_loop.shutdown_default_executor())+ except Exception:+ logger.exception("Failed to shut down default executor for MCP server %s", self._mcp_server.id)+ finally:+ try:+ self._event_loop.close()+ finally:+ with self.__class__._INSTANCES_LOCK:+ self.__class__._ALL_INSTANCES.discard(self)+ self._shutdown_complete.set()++ def _drain_pending_tasks(self) -> None:+ pending_tasks = [task for task in asyncio.all_tasks(self._event_loop) if not task.done()]+ for task in pending_tasks:+ task.cancel()+ if pending_tasks:+ self._event_loop.run_until_complete(asyncio.gather(*pending_tasks, return_exceptions=True))++ async def _run_mcp_server_loop(self) -> None:+ self._server_task = asyncio.current_task()+ self._server_task_started.set()+ await self._mcp_server_loop()async def _mcp_server_loop(self) -> None:url = self._mcp_server.url.strip()@@ -101,6 +212,8 @@ class MCPToolCallSession(ToolCallSession):logging.warning(f"SSE transport MCP session cancelled for server {self._mcp_server.id}")returnexcept Exception:+ if self._close:+ raisemsg = "Connection failed (possibly due to auth error). Please check authentication settings first"await self._process_mcp_tasks(None, msg)@@ -121,6 +234,8 @@ class MCPToolCallSession(ToolCallSession):logging.warning(f"STREAMABLE_HTTP MCP session cancelled for server {self._mcp_server.id}")returnexcept Exception as e:+ if self._close:+ raiselogging.exception(e)msg = "Connection failed (possibly due to auth error). Please check authentication settings first"await self._process_mcp_tasks(None, msg)@@ -178,6 +293,10 @@ class MCPToolCallSession(ToolCallSession):raise ValueError("Session is closed")results = asyncio.Queue()+ caller_task = asyncio.current_task()+ if caller_task is None:+ raise RuntimeError("MCP calls require an asyncio task")+ self._pending_calls[caller_task] = resultsdeadline = deadline if deadline is not None else time.monotonic() + request_timeoutabandoned = asyncio.Event()await self._queue.put((task_type, kwargs, results, deadline, abandoned))@@ -193,6 +312,7 @@ class MCPToolCallSession(ToolCallSession):raisefinally:abandoned.set()+ self._pending_calls.pop(caller_task, None)async def _call_mcp_tool(self, name: str, arguments: dict[str, Any], request_timeout: float | int = 10, deadline: float | None = None) -> str:result: CallToolResult = await self._call_mcp_server("tool_call", name=name, arguments=arguments, request_timeout=request_timeout, deadline=deadline)@@ -216,11 +336,16 @@ class MCPToolCallSession(ToolCallSession):raisedef get_tools(self, timeout: float | int = 10) -> list[Tool]:- if self._close:- raise ValueError("Session is closed")-deadline = time.monotonic() + timeout- future = asyncio.run_coroutine_threadsafe(self._get_tools_from_mcp_server(request_timeout=timeout, deadline=deadline), self._event_loop)+ with self._shutdown_lock:+ if self._close:+ raise ValueError("Session is closed")+ coroutine = self._get_tools_from_mcp_server(request_timeout=timeout, deadline=deadline)+ try:+ future = asyncio.run_coroutine_threadsafe(coroutine, self._event_loop)+ except Exception:+ coroutine.close()+ raisetry:return future.result(timeout=max(0, deadline - time.monotonic()))except FuturesTimeoutError:@@ -234,14 +359,16 @@ class MCPToolCallSession(ToolCallSession):@overridedef tool_call(self, name: str, arguments: dict[str, Any], timeout: float | int = 10) -> str:- if self._close:- return "Error: Session is closed"-deadline = time.monotonic() + timeout- future = asyncio.run_coroutine_threadsafe(- self._call_mcp_tool(name, arguments, request_timeout=timeout, deadline=deadline),- self._event_loop,- )+ with self._shutdown_lock:+ if self._close:+ return "Error: Session is closed"+ coroutine = self._call_mcp_tool(name, arguments, request_timeout=timeout, deadline=deadline)+ try:+ future = asyncio.run_coroutine_threadsafe(coroutine, self._event_loop)+ except Exception:+ coroutine.close()+ raisetry:return future.result(timeout=max(0, deadline - time.monotonic()))except FuturesTimeoutError:@@ -252,90 +379,204 @@ class MCPToolCallSession(ToolCallSession):logging.exception(f"Error calling tool '{name}' on MCP server: {self._mcp_server.id}")return f"Error calling tool '{name}': {e}."- async def close(self) -> None:- if self._close:- return-- self._close = True+ async def _shutdown_on_event_loop(self) -> None:+ pending_calls = list(self._pending_calls.items())+ for _, result_queue in pending_calls:+ result_queue.put_nowait(ValueError("Session is closing"))while not self._queue.empty():+ self._queue.get_nowait()++ try:+ await self._server_task_started.wait()+ if self._server_task and not self._server_task.done():+ self._server_task.cancel()++ request_tasks = [task for task, _ in pending_calls]+ if request_tasks:+ await asyncio.gather(*request_tasks, return_exceptions=True)+ if self._server_task:+ try:+ await self._server_task+ except asyncio.CancelledError:+ pass+ finally:try:- _, _, result_queue, _, _ = self._queue.get_nowait()+ await self._event_loop.shutdown_asyncgens()+ finally:+ await self._event_loop.shutdown_default_executor()++ def _begin_shutdown(self) -> Future[None]:+ with self._shutdown_lock:+ if self._shutdown_future is None:+ self._close = True+ logger.info("Closing MCP session for server %s", self._mcp_server.id)+ coroutine = self._shutdown_on_event_loop()try:- await result_queue.put(asyncio.CancelledError("Session is closing"))+ self._shutdown_future = asyncio.run_coroutine_threadsafe(coroutine, self._event_loop)except Exception:- pass- except asyncio.QueueEmpty:- break- except Exception:- break+ coroutine.close()+ raise+ self._shutdown_future.add_done_callback(self._request_event_loop_stop)+ return self._shutdown_future- try:- self._event_loop.call_soon_threadsafe(self._event_loop.stop)- except Exception:- pass+ def _request_event_loop_stop(self, _: Future[None]) -> None:+ if self._event_loop.is_closed():+ return+ self._event_loop.call_soon_threadsafe(self._schedule_event_loop_stop)- try:- self._thread_pool.shutdown(wait=True)- except Exception:- pass+ def _schedule_event_loop_stop(self) -> None:+ if self._owner_close_waiters or self._event_loop.is_closed():+ return+ if self._stop_handle is None or self._stop_handle.cancelled():+ self._stop_handle = self._event_loop.call_later(0, self._event_loop.stop)- self.__class__._ALL_INSTANCES.discard(self)+ async def close(self, timeout: float = 5) -> None: # noqa: ASYNC109+ deadline = time.monotonic() + timeout+ shutdown_finished = False+ shutdown_error = None+ shutdown_traceback = None+ owner_loop = asyncio.get_running_loop() is self._event_loop+ if owner_loop:+ self._owner_close_waiters += 1+ if self._stop_handle is not None:+ self._stop_handle.cancel()+ self._stop_handle = None- def close_sync(self, timeout: float | int = 5) -> None:- if not self._event_loop.is_running():- logging.warning(f"Event loop already stopped for {self._mcp_server.id}")- return+ try:+ future = self._begin_shutdown()+ wrapped_future = asyncio.wrap_future(future)+ wrapped_future.add_done_callback(lambda completed: None if completed.cancelled() else completed.exception())+ done, _ = await asyncio.wait({wrapped_future}, timeout=max(0, deadline - time.monotonic()))+ if not done:+ logger.error("Timeout while closing session for server %s (timeout=%s)", self._mcp_server.id, timeout)+ else:+ shutdown_finished = True+ await wrapped_future+ except BaseException as error: # noqa: BLE001+ shutdown_error = error+ shutdown_traceback = error.__traceback__+ finally:+ if owner_loop:+ self._owner_close_waiters -= 1+ if self._owner_close_waiters == 0 and self._shutdown_future is not None and self._shutdown_future.done():+ self._schedule_event_loop_stop()++ join_error = None+ join_traceback = None+ if not owner_loop:+ remaining = max(0, deadline - time.monotonic())+ if self._thread.is_alive() and remaining:+ join_task = asyncio.create_task(asyncio.to_thread(self._thread.join, remaining))+ join_task.add_done_callback(lambda completed: None if completed.cancelled() else completed.exception())+ try:+ done, _ = await asyncio.wait({join_task}, timeout=remaining)+ if done:+ await join_task+ except BaseException as error: # noqa: BLE001+ join_error = error+ join_traceback = error.__traceback__+ if self._thread.is_alive() and shutdown_finished and join_error is None:+ logger.error("Timeout while joining MCP session thread for server %s (timeout=%s)", self._mcp_server.id, timeout)++ if shutdown_error is not None:+ if join_error is not None:+ logger.error(+ "Exception while joining MCP session thread for server %s",+ self._mcp_server.id,+ exc_info=(type(join_error), join_error, join_traceback),+ )+ raise shutdown_error.with_traceback(shutdown_traceback)+ if join_error is not None:+ raise join_error.with_traceback(join_traceback)+ def close_sync(self, timeout: float | int = 5) -> None:try:- future = asyncio.run_coroutine_threadsafe(self.close(), self._event_loop)- try:- future.result(timeout=timeout)- except FuturesTimeoutError:- logging.error(f"Timeout while closing session for server {self._mcp_server.id} (timeout={timeout})")- except Exception:- logging.exception(f"Unexpected error during close_sync for {self._mcp_server.id}")- except Exception:- logging.exception(f"Exception while scheduling close for server {self._mcp_server.id}")+ running_loop = asyncio.get_running_loop()+ except RuntimeError:+ running_loop = None+ if running_loop is self._event_loop:+ raise RuntimeError("close_sync() cannot block the MCP session event loop")+ deadline = time.monotonic() + timeout+ future = self._begin_shutdown()-def close_multiple_mcp_toolcall_sessions(sessions: list[MCPToolCallSession]) -> None:- logging.info(f"Want to clean up {len(sessions)} MCP sessions")+ try:+ shutdown_error = future.exception(timeout=max(0, deadline - time.monotonic()))+ except FuturesTimeoutError:+ logger.error("Timeout while closing session for server %s (timeout=%s)", self._mcp_server.id, timeout)+ return+ except FuturesCancelledError as error:+ shutdown_error = error- async def _gather_and_stop() -> None:+ join_error = None+ join_traceback = Nonetry:- await asyncio.gather(*[s.close() for s in sessions if s is not None], return_exceptions=True)- except Exception:- logging.exception("Exception during MCP session cleanup")- finally:+ self._thread.join(timeout=max(0, deadline - time.monotonic()))+ except Exception as error: # noqa: BLE001+ join_error = error+ join_traceback = error.__traceback__+ if self._thread.is_alive() and join_error is None:+ logger.error("Timeout while joining MCP session thread for server %s (timeout=%s)", self._mcp_server.id, timeout)+ if shutdown_error is not None:+ if join_error is not None:+ logger.error(+ "Exception while joining MCP session thread for server %s",+ self._mcp_server.id,+ exc_info=(type(join_error), join_error, join_traceback),+ )+ raise shutdown_error+ if join_error is not None:+ raise join_error.with_traceback(join_traceback)+++def close_multiple_mcp_toolcall_sessions(sessions: list[MCPToolCallSession]) -> int:+ unique_sessions = list({id(session): session for session in sessions if session is not None}.values())+ current_thread = threading.current_thread()+ if any(session._thread is current_thread for session in unique_sessions):+ raise RuntimeError("MCP sessions must be closed outside their event loop threads")++ logger.info("Want to clean up %s MCP sessions", len(unique_sessions))++ cleanup_errors = 0+ with ThreadPoolExecutor() as executor:+ futures = {executor.submit(session.close_sync): session for session in unique_sessions}+ for future, session in futures.items():try:- loop.call_soon_threadsafe(loop.stop)+ future.result()except Exception:- pass-- try:- loop = asyncio.new_event_loop()- thread = threading.Thread(target=loop.run_forever, daemon=True)- thread.start()-- asyncio.run_coroutine_threadsafe(_gather_and_stop(), loop).result()- thread.join()- except Exception:- logging.exception("Exception during MCP session cleanup thread management")+ cleanup_errors += 1+ logger.exception("Exception while closing MCP session for server %s", session._mcp_server.id)- logging.info(f"{len(sessions)} MCP sessions has been cleaned up. {len(list(MCPToolCallSession._ALL_INSTANCES))} in global context.")+ closed_count = sum(session._shutdown_complete.is_set() for session in unique_sessions)+ if closed_count != len(unique_sessions):+ logger.warning("%s of %s MCP sessions are still shutting down", len(unique_sessions) - closed_count, len(unique_sessions))+ if cleanup_errors:+ logger.warning("%s MCP sessions stopped; transport cleanup failure count: %s. %s in global context.", closed_count, cleanup_errors, len(MCPToolCallSession._active_instances()))+ else:+ logger.info("%s MCP sessions have been cleaned up. %s in global context.", closed_count, len(MCPToolCallSession._active_instances()))+ return cleanup_errorsdef shutdown_all_mcp_sessions():"""Gracefully shutdown all active MCPToolCallSession instances."""- sessions = list(MCPToolCallSession._ALL_INSTANCES)- if not sessions:- logging.info("No MCPToolCallSession instances to close.")- return-- logging.info(f"Shutting down {len(sessions)} MCPToolCallSession instances...")- close_multiple_mcp_toolcall_sessions(sessions)- logging.info("All MCPToolCallSession instances have been closed.")+ sessions = MCPToolCallSession._begin_global_shutdown()+ try:+ if not sessions:+ logging.info("No MCPToolCallSession instances to close.")+ return++ logging.info(f"Shutting down {len(sessions)} MCPToolCallSession instances...")+ cleanup_errors = close_multiple_mcp_toolcall_sessions(sessions)+ active_sessions = MCPToolCallSession._active_instances()+ if active_sessions:+ logger.warning("%s MCPToolCallSession instances are still shutting down.", len(active_sessions))+ elif cleanup_errors:+ logger.warning("All MCPToolCallSession event loops stopped; transport cleanup failure count: %s.", cleanup_errors)+ else:+ logger.info("All MCPToolCallSession instances have been closed.")+ finally:+ MCPToolCallSession._end_global_shutdown()def mcp_tool_metadata_to_openai_tool(mcp_tool: Tool | dict, function_name: str | None = None) -> dict[str, Any]:diff --git a/test/unit_test/api/test_ragflow_server_shutdown.py b/test/unit_test/api/test_ragflow_server_shutdown.pynew file mode 100644index 0000000000..28707f8f11--- /dev/null+++ b/test/unit_test/api/test_ragflow_server_shutdown.py@@ -0,0 +1,62 @@+import ast+from pathlib import Path+from types import SimpleNamespace+from unittest.mock import Mock++SERVER_PATH = Path(__file__).resolve().parents[3] / "api" / "ragflow_server.py"+++def test_signal_handler_forces_exit_on_second_signal():+ tree = ast.parse(SERVER_PATH.read_text(encoding="utf-8"), filename=str(SERVER_PATH))+ handler = next(node for node in tree.body if isinstance(node, ast.FunctionDef) and node.name == "signal_handler")+ namespace = {+ "shutdown_requested": False,+ "os": SimpleNamespace(getpid=Mock(return_value=1234), kill=Mock()),+ "signal": SimpleNamespace(SIGKILL=9),+ "sys": SimpleNamespace(exit=Mock(side_effect=SystemExit(0))),+ }+ exec(compile(ast.fix_missing_locations(ast.Module(body=[handler], type_ignores=[])), str(SERVER_PATH), "exec"), namespace)++ try:+ namespace["signal_handler"](2, None)+ except SystemExit as exc:+ assert exc.code == 0+ else:+ raise AssertionError("the first shutdown signal must request graceful process exit")++ assert namespace["shutdown_requested"] is True+ namespace["os"].kill.assert_not_called()++ namespace["signal_handler"](15, None)++ namespace["os"].getpid.assert_called_once_with()+ namespace["os"].kill.assert_called_once_with(1234, 9)+ namespace["sys"].exit.assert_called_once_with(0)+++def test_server_shutdown_wraps_full_startup_and_forces_cleanup_before_kill():+ tree = ast.parse(SERVER_PATH.read_text(encoding="utf-8"), filename=str(SERVER_PATH))+ main = next(node for node in tree.body if isinstance(node, ast.FunctionDef) and node.name == "main")+ main_try = next(node for node in main.body if isinstance(node, ast.Try))++ startup_calls = [ast.unparse(statement.value) for statement in main_try.body]+ assert startup_calls == [+ "signal.signal(signal.SIGINT, signal_handler)",+ "signal.signal(signal.SIGTERM, signal_handler)",+ "run_server()",+ ]++ exception_handler = main_try.handlers[0]+ assert ast.unparse(exception_handler.body[0]) == "force_kill = True"++ mcp_cleanup = next(node for node in main_try.finalbody if isinstance(node, ast.Try))+ assert ast.unparse(mcp_cleanup.body[0].value) == "shutdown_all_mcp_sessions()"++ background_cleanup = mcp_cleanup.finalbody[0]+ assert isinstance(background_cleanup, ast.Try)+ assert ast.unparse(background_cleanup.body[0].value) == "stop_background_services()"++ force_kill = background_cleanup.finalbody[0]+ assert isinstance(force_kill, ast.If)+ assert ast.unparse(force_kill.test) == "force_kill"+ assert ast.unparse(force_kill.body[0].value) == "os.kill(os.getpid(), signal.SIGKILL)"diff --git a/test/unit_test/common/test_mcp_tool_call_conn.py b/test/unit_test/common/test_mcp_tool_call_conn.pyindex 9cf56bcd69..9c6fa0072d 100644--- a/test/unit_test/common/test_mcp_tool_call_conn.py+++ b/test/unit_test/common/test_mcp_tool_call_conn.py@@ -1,12 +1,16 @@import asyncioimport threadingimport time+from concurrent.futures import CancelledError as FuturesCancelledErrorfrom concurrent.futures import ThreadPoolExecutor+from concurrent.futures import TimeoutError as FuturesTimeoutError+from contextlib import asynccontextmanagerfrom types import SimpleNamespaceimport pytestfrom common import mcp_tool_call_conn+from common.constants import MCPServerTypefrom common.mcp_tool_call_conn import MCPToolCallSession@@ -14,6 +18,8 @@ def _make_session() -> MCPToolCallSession:session = object.__new__(MCPToolCallSession)session._queue = asyncio.Queue()session._close = False+ session._pending_calls = {}+ session._shutdown_lock = threading.Lock()return session@@ -35,6 +41,1204 @@ async def _stop_tasks(session: MCPToolCallSession, *tasks: asyncio.Task) -> Noneassert all(task.done() for task in tasks)[email protected]("server_type", [MCPServerType.SSE, MCPServerType.STREAMABLE_HTTP])+def test_close_sync_waits_for_transport_context_cleanup(monkeypatch, server_type):+ initialized = threading.Event()+ client_closed = threading.Event()+ transport_closed = threading.Event()++ @asynccontextmanager+ async def fake_sse_client(url, headers):+ try:+ yield object(), object()+ finally:+ transport_closed.set()++ @asynccontextmanager+ async def fake_streamable_http_client(url, headers):+ try:+ yield object(), object(), None+ finally:+ transport_closed.set()++ class FakeClientSession:+ def __init__(self, *streams):+ pass++ async def __aenter__(self):+ return self++ async def __aexit__(self, exc_type, exc, traceback):+ client_closed.set()++ async def initialize(self):+ initialized.set()++ monkeypatch.setattr(mcp_tool_call_conn, "sse_client", fake_sse_client)+ monkeypatch.setattr(mcp_tool_call_conn, "streamablehttp_client", fake_streamable_http_client)+ monkeypatch.setattr(mcp_tool_call_conn, "ClientSession", FakeClientSession)++ server = SimpleNamespace(id="server-1", url="http://mcp.test", headers={}, server_type=server_type)+ session = MCPToolCallSession(server)++ try:+ assert initialized.wait(timeout=1)+ session.close_sync(timeout=1)++ assert client_closed.is_set()+ assert transport_closed.is_set()+ assert not session._event_loop.is_running()+ assert session._event_loop.is_closed()+ assert not asyncio.all_tasks(session._event_loop)+ assert session not in MCPToolCallSession._ALL_INSTANCES+ finally:+ session.close_sync(timeout=1)+++def test_close_sync_cancels_blocked_initialization(monkeypatch):+ initialize_started = threading.Event()+ initialize_cancelled = threading.Event()+ client_closed = threading.Event()+ transport_closed = threading.Event()++ @asynccontextmanager+ async def fake_sse_client(url, headers):+ try:+ yield object(), object()+ finally:+ transport_closed.set()++ class FakeClientSession:+ def __init__(self, *streams):+ pass++ async def __aenter__(self):+ return self++ async def __aexit__(self, exc_type, exc, traceback):+ client_closed.set()++ async def initialize(self):+ initialize_started.set()+ try:+ await asyncio.Future()+ except asyncio.CancelledError:+ initialize_cancelled.set()+ raise++ monkeypatch.setattr(mcp_tool_call_conn, "sse_client", fake_sse_client)+ monkeypatch.setattr(mcp_tool_call_conn, "ClientSession", FakeClientSession)++ server = SimpleNamespace(id="server-1", url="http://mcp.test", headers={}, server_type=MCPServerType.SSE)+ session = MCPToolCallSession(server)++ try:+ assert initialize_started.wait(timeout=1)+ session.close_sync(timeout=1)++ assert initialize_cancelled.is_set()+ assert client_closed.is_set()+ assert transport_closed.is_set()+ assert session._event_loop.is_closed()+ finally:+ session.close_sync(timeout=1)+++def test_async_close_waits_for_thread_exit(monkeypatch):+ server_started = threading.Event()+ finalized = threading.Event()++ async def fake_server_loop(self):+ server_started.set()+ try:+ await asyncio.Future()+ finally:+ finalized.set()++ monkeypatch.setattr(MCPToolCallSession, "_mcp_server_loop", fake_server_loop)++ session = MCPToolCallSession(SimpleNamespace(id="server-1"))++ try:+ assert server_started.wait(timeout=1)+ asyncio.run(session.close())++ assert finalized.is_set()+ assert not session._thread.is_alive()+ assert session._event_loop.is_closed()+ assert session not in MCPToolCallSession._ALL_INSTANCES+ finally:+ session.close_sync(timeout=1)+++def test_async_close_runs_on_session_event_loop(monkeypatch):+ server_started = threading.Event()+ finalized = threading.Event()++ async def fake_server_loop(self):+ server_started.set()+ try:+ await asyncio.Future()+ finally:+ finalized.set()++ monkeypatch.setattr(MCPToolCallSession, "_mcp_server_loop", fake_server_loop)++ session = MCPToolCallSession(SimpleNamespace(id="server-1"))++ try:+ assert server_started.wait(timeout=1)+ close_future = asyncio.run_coroutine_threadsafe(session.close(), session._event_loop)+ close_future.result(timeout=1)+ session._thread.join(timeout=1)++ assert finalized.is_set()+ assert not session._thread.is_alive()+ assert session._event_loop.is_closed()+ assert session not in MCPToolCallSession._ALL_INSTANCES+ finally:+ session.close_sync(timeout=1)+++def test_late_owner_close_is_cancelled_when_event_loop_stops(monkeypatch):+ server_started = threading.Event()+ stop_scheduled = threading.Event()+ release_stop_scheduler = threading.Event()++ async def fake_server_loop(self):+ server_started.set()+ await asyncio.Future()++ original_schedule_stop = MCPToolCallSession._schedule_event_loop_stop++ def blocking_schedule_stop(self):+ original_schedule_stop(self)+ stop_scheduled.set()+ release_stop_scheduler.wait(timeout=1)++ monkeypatch.setattr(MCPToolCallSession, "_mcp_server_loop", fake_server_loop)+ monkeypatch.setattr(MCPToolCallSession, "_schedule_event_loop_stop", blocking_schedule_stop)++ session = MCPToolCallSession(SimpleNamespace(id="server-1"))+ late_close = None++ try:+ assert server_started.wait(timeout=1)+ shutdown_future = session._begin_shutdown()+ shutdown_future.result(timeout=1)+ assert stop_scheduled.wait(timeout=1)++ late_close = asyncio.run_coroutine_threadsafe(session.close(), session._event_loop)+ release_stop_scheduler.set()++ with pytest.raises(FuturesCancelledError):+ late_close.result(timeout=1)+ session._thread.join(timeout=1)+ assert not session._thread.is_alive()+ assert session._event_loop.is_closed()+ assert not asyncio.all_tasks(session._event_loop)+ finally:+ release_stop_scheduler.set()+ if late_close is not None and not late_close.done():+ late_close.cancel()+ session.close_sync(timeout=1)+++def test_owner_close_queued_by_stop_callback_is_drained(monkeypatch):+ server_started = threading.Event()+ late_close_submitted = threading.Event()+ late_close_holder = {}++ async def fake_server_loop(self):+ server_started.set()+ await asyncio.Future()++ monkeypatch.setattr(MCPToolCallSession, "_mcp_server_loop", fake_server_loop)++ session = MCPToolCallSession(SimpleNamespace(id="server-1"))+ original_stop = session._event_loop.stop+ submitted = False++ def submit_close_then_stop():+ nonlocal submitted+ if not submitted:+ submitted = True+ late_close_holder["future"] = asyncio.run_coroutine_threadsafe(session.close(), session._event_loop)+ late_close_submitted.set()+ original_stop()++ monkeypatch.setattr(session._event_loop, "stop", submit_close_then_stop)++ try:+ assert server_started.wait(timeout=1)+ session.close_sync(timeout=1)+ assert late_close_submitted.is_set()++ late_close = late_close_holder["future"]+ with pytest.raises(FuturesCancelledError):+ late_close.result(timeout=1)+ assert not session._thread.is_alive()+ assert session._event_loop.is_closed()+ assert not asyncio.all_tasks(session._event_loop)+ finally:+ session.close_sync(timeout=1)+++def test_close_sync_propagates_transport_cleanup_error(monkeypatch, caplog):+ initialized = threading.Event()+ join_attempted = threading.Event()++ @asynccontextmanager+ async def fake_sse_client(url, headers):+ try:+ yield object(), object()+ finally:+ raise RuntimeError("transport cleanup failed")++ class FakeClientSession:+ def __init__(self, *streams):+ pass++ async def __aenter__(self):+ return self++ async def __aexit__(self, exc_type, exc, traceback):+ pass++ async def initialize(self):+ initialized.set()++ monkeypatch.setattr(mcp_tool_call_conn, "sse_client", fake_sse_client)+ monkeypatch.setattr(mcp_tool_call_conn, "ClientSession", FakeClientSession)++ server = SimpleNamespace(id="server-1", url="http://mcp.test", headers={}, server_type=MCPServerType.SSE)+ session = MCPToolCallSession(server)+ real_join = session._thread.join++ def failing_join(timeout=None):+ join_attempted.set()+ raise ValueError("join failed")++ monkeypatch.setattr(session._thread, "join", failing_join)++ try:+ assert initialized.wait(timeout=1)+ with caplog.at_level("ERROR"), pytest.raises(RuntimeError, match="transport cleanup failed"):+ session.close_sync(timeout=1)++ assert join_attempted.is_set()+ assert "Exception while joining MCP session thread for server server-1" in caplog.messages+ join_record = next(record for record in caplog.records if record.getMessage() == "Exception while joining MCP session thread for server server-1")+ assert join_record.exc_info is not None+ assert join_record.exc_info[0] is ValueError+ assert str(join_record.exc_info[1]) == "join failed"++ monkeypatch.setattr(session._thread, "join", real_join)+ assert session._shutdown_complete.wait(timeout=1)+ real_join(timeout=1)+ assert not session._thread.is_alive()+ assert session._event_loop.is_closed()+ assert session not in MCPToolCallSession._ALL_INSTANCES+ finally:+ monkeypatch.setattr(session._thread, "join", real_join)+ if session._thread.is_alive():+ try:+ session.close_sync(timeout=1)+ except RuntimeError as error:+ if str(error) != "transport cleanup failed":+ raise+++def test_close_sync_propagates_thread_join_error(monkeypatch):+ server_started = threading.Event()++ async def fake_server_loop(self):+ server_started.set()+ await asyncio.Future()++ monkeypatch.setattr(MCPToolCallSession, "_mcp_server_loop", fake_server_loop)++ session = MCPToolCallSession(SimpleNamespace(id="server-1"))+ real_join = session._thread.join++ def failing_join(timeout=None):+ raise ValueError("join failed")++ monkeypatch.setattr(session._thread, "join", failing_join)++ try:+ assert server_started.wait(timeout=1)+ with pytest.raises(ValueError, match="join failed"):+ session.close_sync(timeout=1)+ finally:+ monkeypatch.setattr(session._thread, "join", real_join)+ session.close_sync(timeout=1)+++def test_close_sync_cancels_in_flight_call_and_wakes_caller(monkeypatch):+ call_started = threading.Event()+ call_finalized = threading.Event()++ class FakeClientSession:+ async def call_tool(self, name, arguments):+ call_started.set()+ try:+ await asyncio.Future()+ finally:+ call_finalized.set()++ async def fake_server_loop(self):+ await self._process_mcp_tasks(FakeClientSession())++ monkeypatch.setattr(MCPToolCallSession, "_mcp_server_loop", fake_server_loop)++ server = SimpleNamespace(id="server-1")+ session = MCPToolCallSession(server)+ caller_pool = ThreadPoolExecutor(max_workers=1)+ caller = caller_pool.submit(session.tool_call, "slow", {}, 3)++ try:+ assert call_started.wait(timeout=1)+ session.close_sync(timeout=1)++ result = caller.result(timeout=1)+ assert "Session is closing" in result+ assert "Timeout" not in result+ assert call_finalized.is_set()+ finally:+ session.close_sync(timeout=1)+ caller_pool.shutdown(wait=True)+++def test_close_sync_wakes_in_flight_get_tools(monkeypatch):+ call_started = threading.Event()+ call_finalized = threading.Event()++ class FakeClientSession:+ async def list_tools(self):+ call_started.set()+ try:+ await asyncio.Future()+ finally:+ call_finalized.set()++ async def fake_server_loop(self):+ await self._process_mcp_tasks(FakeClientSession())++ monkeypatch.setattr(MCPToolCallSession, "_mcp_server_loop", fake_server_loop)++ session = MCPToolCallSession(SimpleNamespace(id="server-1"))+ caller_pool = ThreadPoolExecutor(max_workers=1)+ caller = caller_pool.submit(session.get_tools, 3)++ try:+ assert call_started.wait(timeout=1)+ session.close_sync(timeout=1)++ with pytest.raises(ValueError, match="Session is closing"):+ caller.result(timeout=1)+ assert call_finalized.is_set()+ finally:+ session.close_sync(timeout=1)+ caller_pool.shutdown(wait=True)+++def test_close_sync_wakes_queued_call_without_dispatching_it(monkeypatch):+ first_started = threading.Event()+ first_finalized = threading.Event()+ queued_enqueued = threading.Event()+ calls = []+ queue_type = asyncio.Queue++ class ObservedQueue(queue_type):+ async def put(self, item):+ await super().put(item)+ if isinstance(item, tuple) and len(item) > 1 and item[1].get("name") == "queued":+ queued_enqueued.set()++ class FakeClientSession:+ async def call_tool(self, name, arguments):+ calls.append(name)+ if name == "first":+ first_started.set()+ try:+ await asyncio.Future()+ finally:+ first_finalized.set()+ return SimpleNamespace(isError=False, content=[])++ async def fake_server_loop(self):+ await self._process_mcp_tasks(FakeClientSession())++ monkeypatch.setattr(mcp_tool_call_conn.asyncio, "Queue", ObservedQueue)+ monkeypatch.setattr(MCPToolCallSession, "_mcp_server_loop", fake_server_loop)++ session = MCPToolCallSession(SimpleNamespace(id="server-1"))+ caller_pool = ThreadPoolExecutor(max_workers=2)+ first = caller_pool.submit(session.tool_call, "first", {}, 3)+ queued = None++ try:+ assert first_started.wait(timeout=1)+ queued = caller_pool.submit(session.tool_call, "queued", {}, 3)+ assert queued_enqueued.wait(timeout=1)++ session.close_sync(timeout=1)++ assert "Session is closing" in first.result(timeout=1)+ assert "Session is closing" in queued.result(timeout=1)+ assert calls == ["first"]+ assert first_finalized.is_set()+ finally:+ session.close_sync(timeout=1)+ caller_pool.shutdown(wait=True)+++def test_concurrent_close_is_idempotent(monkeypatch, caplog):+ server_started = threading.Event()+ finalized = threading.Event()+ finalizer_count = 0+ count_lock = threading.Lock()++ async def fake_server_loop(self):+ nonlocal finalizer_count+ server_started.set()+ try:+ await asyncio.Future()+ finally:+ with count_lock:+ finalizer_count += 1+ finalized.set()++ monkeypatch.setattr(MCPToolCallSession, "_mcp_server_loop", fake_server_loop)++ session = MCPToolCallSession(SimpleNamespace(id="server-1"))+ barrier = threading.Barrier(4)++ def close_at_barrier():+ barrier.wait(timeout=1)+ session.close_sync(timeout=1)++ closer_pool = ThreadPoolExecutor(max_workers=3)+ closers = [closer_pool.submit(close_at_barrier) for _ in range(3)]++ try:+ assert server_started.wait(timeout=1)+ with caplog.at_level("INFO"):+ barrier.wait(timeout=1)+ for closer in closers:+ closer.result(timeout=2)++ session.close_sync(timeout=1)+ assert finalized.is_set()+ assert finalizer_count == 1+ assert caplog.messages.count("Closing MCP session for server server-1") == 1+ assert session.tool_call("after-close", {}, timeout=1) == "Error: Session is closed"+ assert not session._thread.is_alive()+ assert session._event_loop.is_closed()+ assert session not in MCPToolCallSession._ALL_INSTANCES+ finally:+ session.close_sync(timeout=1)+ closer_pool.shutdown(wait=True)++[email protected]("operation", ["tool_call", "get_tools"])+def test_close_waits_for_public_call_submission(monkeypatch, operation):+ server_started = threading.Event()+ submission_started = threading.Event()+ release_submission = threading.Event()+ close_started = threading.Event()++ async def fake_server_loop(self):+ server_started.set()+ await asyncio.Future()++ monkeypatch.setattr(MCPToolCallSession, "_mcp_server_loop", fake_server_loop)++ session = MCPToolCallSession(SimpleNamespace(id="server-1"))+ real_run_coroutine_threadsafe = asyncio.run_coroutine_threadsafe+ first_submission = True++ def blocking_first_submission(coroutine, event_loop):+ nonlocal first_submission+ if first_submission:+ first_submission = False+ submission_started.set()+ release_submission.wait(timeout=1)+ return real_run_coroutine_threadsafe(coroutine, event_loop)++ monkeypatch.setattr(mcp_tool_call_conn.asyncio, "run_coroutine_threadsafe", blocking_first_submission)++ caller_pool = ThreadPoolExecutor(max_workers=2)+ if operation == "tool_call":+ caller = caller_pool.submit(session.tool_call, "queued", {}, 3)+ else:+ caller = caller_pool.submit(session.get_tools, 3)++ def close_session():+ close_started.set()+ session.close_sync(timeout=1)++ closer = None+ try:+ assert server_started.wait(timeout=1)+ assert submission_started.wait(timeout=1)+ closer = caller_pool.submit(close_session)+ assert close_started.wait(timeout=1)+ assert not session._close++ release_submission.set()+ closer.result(timeout=2)++ if operation == "tool_call":+ assert "Session is clos" in caller.result(timeout=1)+ else:+ with pytest.raises(ValueError, match="Session is clos"):+ caller.result(timeout=1)+ assert session._event_loop.is_closed()+ finally:+ release_submission.set()+ if closer is not None:+ closer.result(timeout=2)+ session.close_sync(timeout=1)+ caller_pool.shutdown(wait=True)+++def test_close_timeout_keeps_shared_shutdown_running(monkeypatch):+ server_started = threading.Event()+ finalizer_started = threading.Event()+ finalizer_finished = threading.Event()+ release_finalizer = asyncio.Event()++ async def fake_server_loop(self):+ server_started.set()+ try:+ await asyncio.Future()+ finally:+ finalizer_started.set()+ await release_finalizer.wait()+ finalizer_finished.set()++ monkeypatch.setattr(MCPToolCallSession, "_mcp_server_loop", fake_server_loop)++ session = MCPToolCallSession(SimpleNamespace(id="server-1"))++ try:+ assert server_started.wait(timeout=1)+ session.close_sync(timeout=0.01)++ assert finalizer_started.wait(timeout=1)+ assert not session._shutdown_future.done()+ assert session._thread.is_alive()+ assert session in MCPToolCallSession._ALL_INSTANCES++ shutdown_future = session._shutdown_future+ session._event_loop.call_soon_threadsafe(release_finalizer.set)+ assert session._shutdown_complete.wait(timeout=1)+ session._thread.join(timeout=1)++ assert session._shutdown_future is shutdown_future+ assert finalizer_finished.is_set()+ assert not session._thread.is_alive()+ assert session._event_loop.is_closed()+ assert session not in MCPToolCallSession._ALL_INSTANCES+ finally:+ if not session._event_loop.is_closed():+ session._event_loop.call_soon_threadsafe(release_finalizer.set)+ session.close_sync(timeout=1)++[email protected]("owner_loop", [False, True], ids=["external-loop", "owner-loop"])+def test_async_close_timeout_keeps_shared_shutdown_running(monkeypatch, owner_loop):+ server_started = threading.Event()+ finalizer_started = threading.Event()+ finalizer_finished = threading.Event()+ release_finalizer = asyncio.Event()++ async def fake_server_loop(self):+ server_started.set()+ try:+ await asyncio.Future()+ finally:+ finalizer_started.set()+ await release_finalizer.wait()+ finalizer_finished.set()++ monkeypatch.setattr(MCPToolCallSession, "_mcp_server_loop", fake_server_loop)++ session = MCPToolCallSession(SimpleNamespace(id="server-1"))++ try:+ assert server_started.wait(timeout=1)+ if owner_loop:+ close_future = asyncio.run_coroutine_threadsafe(session.close(timeout=0.01), session._event_loop)+ close_future.result(timeout=1)+ else:+ asyncio.run(asyncio.wait_for(session.close(timeout=0.01), timeout=1))++ assert finalizer_started.wait(timeout=1)+ shutdown_future = session._shutdown_future+ assert shutdown_future is not None+ assert not shutdown_future.done()+ assert not shutdown_future.cancelled()+ assert session._owner_close_waiters == 0+ assert session._thread.is_alive()+ assert session in MCPToolCallSession._ALL_INSTANCES++ session._event_loop.call_soon_threadsafe(release_finalizer.set)+ assert session._shutdown_complete.wait(timeout=1)+ session._thread.join(timeout=1)++ assert session._shutdown_future is shutdown_future+ assert finalizer_finished.is_set()+ assert not session._thread.is_alive()+ assert session._event_loop.is_closed()+ assert session not in MCPToolCallSession._ALL_INSTANCES+ finally:+ if not session._event_loop.is_closed():+ session._event_loop.call_soon_threadsafe(release_finalizer.set)+ session.close_sync(timeout=1)+++def test_async_close_timeout_bounds_thread_join(monkeypatch):+ server_started = threading.Event()+ stop_scheduled = threading.Event()+ release_stop_scheduler = threading.Event()+ join_attempted = threading.Event()+ join_timeouts = []++ async def fake_server_loop(self):+ server_started.set()+ await asyncio.Future()++ original_schedule_stop = MCPToolCallSession._schedule_event_loop_stop++ def blocking_schedule_stop(self):+ stop_scheduled.set()+ release_stop_scheduler.wait(timeout=5)+ original_schedule_stop(self)++ monkeypatch.setattr(MCPToolCallSession, "_mcp_server_loop", fake_server_loop)+ monkeypatch.setattr(MCPToolCallSession, "_schedule_event_loop_stop", blocking_schedule_stop)++ session = MCPToolCallSession(SimpleNamespace(id="server-1"))+ real_join = session._thread.join++ def observed_join(timeout=None):+ join_timeouts.append(timeout)+ join_attempted.set()+ real_join(timeout)++ monkeypatch.setattr(session._thread, "join", observed_join)++ try:+ assert server_started.wait(timeout=1)+ shutdown_future = session._begin_shutdown()+ shutdown_future.result(timeout=1)+ assert stop_scheduled.wait(timeout=1)++ asyncio.run(asyncio.wait_for(session.close(timeout=0.5), timeout=1))++ assert join_attempted.is_set()+ assert 0 < join_timeouts[0] <= 0.5+ assert session._shutdown_future is shutdown_future+ assert session._thread.is_alive()++ release_stop_scheduler.set()+ assert session._shutdown_complete.wait(timeout=1)+ real_join(timeout=1)+ assert not session._thread.is_alive()+ assert session._event_loop.is_closed()+ assert session not in MCPToolCallSession._ALL_INSTANCES+ finally:+ monkeypatch.setattr(session._thread, "join", real_join)+ release_stop_scheduler.set()+ session.close_sync(timeout=1)+++def test_async_close_propagates_transport_cleanup_error(monkeypatch, caplog):+ server_started = threading.Event()+ release_stop_scheduler = threading.Event()+ join_attempted = threading.Event()++ async def fake_server_loop(self):+ server_started.set()+ try:+ await asyncio.Future()+ finally:+ raise RuntimeError("transport cleanup failed")++ monkeypatch.setattr(MCPToolCallSession, "_mcp_server_loop", fake_server_loop)++ session = MCPToolCallSession(SimpleNamespace(id="server-1"))+ real_join = session._thread.join++ def blocking_schedule_stop():+ release_stop_scheduler.wait(timeout=5)+ session._event_loop.stop()++ def failing_join(timeout=None):+ join_attempted.set()+ raise ValueError("join failed")++ monkeypatch.setattr(session, "_schedule_event_loop_stop", blocking_schedule_stop)+ monkeypatch.setattr(session._thread, "join", failing_join)++ try:+ assert server_started.wait(timeout=1)+ with caplog.at_level("ERROR"), pytest.raises(RuntimeError, match="transport cleanup failed"):+ asyncio.run(asyncio.wait_for(session.close(timeout=1), timeout=2))++ assert join_attempted.is_set()+ assert "Exception while joining MCP session thread for server server-1" in caplog.messages++ monkeypatch.setattr(session._thread, "join", real_join)+ release_stop_scheduler.set()+ assert session._shutdown_complete.wait(timeout=1)+ real_join(timeout=1)++ assert not session._thread.is_alive()+ assert session._event_loop.is_closed()+ assert session not in MCPToolCallSession._ALL_INSTANCES+ finally:+ monkeypatch.setattr(session._thread, "join", real_join)+ release_stop_scheduler.set()+ if session._thread.is_alive():+ try:+ session.close_sync(timeout=1)+ except RuntimeError as error:+ if str(error) != "transport cleanup failed":+ raise+++def test_async_close_propagates_thread_join_error(monkeypatch):+ server_started = threading.Event()+ release_stop_scheduler = threading.Event()++ async def fake_server_loop(self):+ server_started.set()+ await asyncio.Future()++ monkeypatch.setattr(MCPToolCallSession, "_mcp_server_loop", fake_server_loop)++ session = MCPToolCallSession(SimpleNamespace(id="server-1"))+ real_join = session._thread.join++ def blocking_schedule_stop():+ release_stop_scheduler.wait(timeout=5)+ session._event_loop.stop()++ def failing_join(timeout=None):+ raise ValueError("join failed")++ monkeypatch.setattr(session, "_schedule_event_loop_stop", blocking_schedule_stop)+ monkeypatch.setattr(session._thread, "join", failing_join)++ try:+ assert server_started.wait(timeout=1)+ with pytest.raises(ValueError, match="join failed"):+ asyncio.run(asyncio.wait_for(session.close(timeout=1), timeout=2))+ finally:+ monkeypatch.setattr(session._thread, "join", real_join)+ release_stop_scheduler.set()+ session.close_sync(timeout=1)+++def test_close_keeps_session_active_until_default_executor_finishes(monkeypatch):+ worker_started = threading.Event()+ release_worker = threading.Event()+ worker_finished = threading.Event()+ worker_holder = {}++ def blocking_worker():+ worker_holder["thread"] = threading.current_thread()+ worker_started.set()+ release_worker.wait(timeout=2)+ worker_finished.set()++ async def fake_server_loop(self):+ await asyncio.get_running_loop().run_in_executor(None, blocking_worker)++ monkeypatch.setattr(MCPToolCallSession, "_mcp_server_loop", fake_server_loop)++ session = MCPToolCallSession(SimpleNamespace(id="server-1"))++ try:+ assert worker_started.wait(timeout=1)+ session.close_sync(timeout=0.01)++ assert session._thread.is_alive()+ assert worker_holder["thread"].is_alive()+ assert not session._shutdown_complete.is_set()+ assert session in MCPToolCallSession._ALL_INSTANCES++ release_worker.set()+ session.close_sync(timeout=1)++ assert worker_finished.is_set()+ assert not worker_holder["thread"].is_alive()+ assert not session._thread.is_alive()+ assert session._event_loop.is_closed()+ assert session not in MCPToolCallSession._ALL_INSTANCES+ finally:+ release_worker.set()+ session.close_sync(timeout=1)+++def test_transport_cleanup_error_waits_for_default_executor(monkeypatch):+ server_started = threading.Event()+ worker_started = threading.Event()+ release_worker = threading.Event()+ worker_finished = threading.Event()+ worker_holder = {}++ def blocking_worker():+ worker_holder["thread"] = threading.current_thread()+ worker_started.set()+ release_worker.wait(timeout=2)+ worker_finished.set()++ async def fake_server_loop(self):+ asyncio.get_running_loop().run_in_executor(None, blocking_worker)+ server_started.set()+ try:+ await asyncio.Future()+ finally:+ raise RuntimeError("transport cleanup failed")++ monkeypatch.setattr(MCPToolCallSession, "_mcp_server_loop", fake_server_loop)++ session = MCPToolCallSession(SimpleNamespace(id="server-1"))++ try:+ assert server_started.wait(timeout=1)+ assert worker_started.wait(timeout=1)+ session.close_sync(timeout=0.01)++ assert session._thread.is_alive()+ assert worker_holder["thread"].is_alive()+ assert not session._shutdown_complete.is_set()+ assert session in MCPToolCallSession._ALL_INSTANCES++ release_worker.set()+ with pytest.raises(RuntimeError, match="transport cleanup failed"):+ session.close_sync(timeout=1)++ assert worker_finished.is_set()+ assert not worker_holder["thread"].is_alive()+ assert not session._thread.is_alive()+ assert session._event_loop.is_closed()+ assert session not in MCPToolCallSession._ALL_INSTANCES+ finally:+ release_worker.set()+ if session._thread.is_alive():+ try:+ session.close_sync(timeout=1)+ except RuntimeError as error:+ assert str(error) == "transport cleanup failed"+++def test_close_multiple_deduplicates_sessions_and_waits_for_cleanup(monkeypatch):+ started = {"server-1": threading.Event(), "server-2": threading.Event()}+ finalized = {"server-1": threading.Event(), "server-2": threading.Event()}+ finalizer_counts = {"server-1": 0, "server-2": 0}++ async def fake_server_loop(self):+ server_id = self._mcp_server.id+ started[server_id].set()+ try:+ await asyncio.Future()+ finally:+ finalizer_counts[server_id] += 1+ finalized[server_id].set()++ monkeypatch.setattr(MCPToolCallSession, "_mcp_server_loop", fake_server_loop)++ first = MCPToolCallSession(SimpleNamespace(id="server-1"))+ second = MCPToolCallSession(SimpleNamespace(id="server-2"))++ try:+ assert all(event.wait(timeout=1) for event in started.values())+ mcp_tool_call_conn.close_multiple_mcp_toolcall_sessions([first, None, first, second])++ assert all(event.is_set() for event in finalized.values())+ assert finalizer_counts == {"server-1": 1, "server-2": 1}+ assert all(session._event_loop.is_closed() for session in (first, second))+ assert all(session not in MCPToolCallSession._ALL_INSTANCES for session in (first, second))+ finally:+ first.close_sync(timeout=1)+ second.close_sync(timeout=1)+++def test_close_multiple_rejects_session_event_loop_thread(monkeypatch):+ server_started = threading.Event()++ async def fake_server_loop(self):+ server_started.set()+ await asyncio.Future()++ async def close_from_owner_loop(session):+ mcp_tool_call_conn.close_multiple_mcp_toolcall_sessions([session])++ monkeypatch.setattr(MCPToolCallSession, "_mcp_server_loop", fake_server_loop)++ session = MCPToolCallSession(SimpleNamespace(id="server-1"))++ try:+ assert server_started.wait(timeout=1)+ future = asyncio.run_coroutine_threadsafe(close_from_owner_loop(session), session._event_loop)++ with pytest.raises(RuntimeError, match="outside their event loop threads"):+ future.result(timeout=1)+ assert not session._close+ finally:+ session.close_sync(timeout=1)+++def test_constructor_preserves_error_when_thread_start_fails_after_start(monkeypatch):+ class StartupInterrupted(BaseException):+ pass++ interruption = StartupInterrupted("startup interrupted")+ loop_started = threading.Event()+ loop_holder = {}+ thread_holder = {}+ session_holder = {}+ instances_before = set(MCPToolCallSession._active_instances())+ real_new_event_loop = asyncio.new_event_loop+ real_thread_start = threading.Thread.start++ def observed_new_event_loop():+ event_loop = real_new_event_loop()+ loop_holder["event_loop"] = event_loop+ return event_loop++ def start_then_fail(thread):+ thread_holder["thread"] = thread+ session_holder["session"] = thread._target.__self__+ real_thread_start(thread)+ loop_holder["event_loop"].call_soon_threadsafe(loop_started.set)+ assert loop_started.wait(timeout=1)+ raise interruption++ monkeypatch.setattr(mcp_tool_call_conn.asyncio, "new_event_loop", observed_new_event_loop)+ monkeypatch.setattr(threading.Thread, "start", start_then_fail)++ with pytest.raises(StartupInterrupted) as raised:+ MCPToolCallSession(SimpleNamespace(id="startup-failure"))++ session = session_holder["session"]+ assert raised.value is interruption+ assert not thread_holder["thread"].is_alive()+ assert loop_holder["event_loop"].is_closed()+ assert session._shutdown_complete.is_set()+ assert session not in MCPToolCallSession._ALL_INSTANCES+ assert set(MCPToolCallSession._active_instances()) == instances_before+++def test_constructor_shutdowns_default_executor_when_registration_fails(monkeypatch):+ class StartupInterrupted(BaseException):+ pass++ interruption = StartupInterrupted("startup interrupted")+ worker_started = threading.Event()+ release_worker = threading.Event()+ worker_finished = threading.Event()+ session_holder = {}+ worker_holder = {}+ instances_before = set(MCPToolCallSession._active_instances())+ real_registry_add = MCPToolCallSession._ALL_INSTANCES.add++ def blocking_worker():+ worker_holder["thread"] = threading.current_thread()+ worker_started.set()+ release_worker.wait(timeout=5)+ worker_finished.set()++ async def fake_server_loop(self):+ session_holder["session"] = self+ await asyncio.get_running_loop().run_in_executor(None, blocking_worker)+ await asyncio.Future()++ def fail_registration(session):+ assert worker_started.wait(timeout=1)+ raise interruption++ monkeypatch.setattr(MCPToolCallSession, "_mcp_server_loop", fake_server_loop)+ monkeypatch.setattr(MCPToolCallSession._ALL_INSTANCES, "add", fail_registration)++ pool = ThreadPoolExecutor(max_workers=1)+ constructor = pool.submit(MCPToolCallSession, SimpleNamespace(id="startup-failure"))++ try:+ assert worker_started.wait(timeout=1)+ with pytest.raises(FuturesTimeoutError):+ constructor.result(timeout=0.05)++ release_worker.set()+ with pytest.raises(StartupInterrupted) as raised:+ constructor.result(timeout=2)++ session = session_holder["session"]+ assert raised.value is interruption+ assert worker_finished.is_set()+ assert not worker_holder["thread"].is_alive()+ assert not session._thread.is_alive()+ assert session._event_loop.is_closed()+ assert session._shutdown_complete.is_set()+ assert session not in MCPToolCallSession._ALL_INSTANCES+ assert set(MCPToolCallSession._active_instances()) == instances_before+ finally:+ release_worker.set()+ try:+ constructor.result(timeout=2)+ except StartupInterrupted:+ pass+ finally:+ pool.shutdown(wait=True)+ monkeypatch.setattr(MCPToolCallSession._ALL_INSTANCES, "add", real_registry_add)+++def test_shutdown_all_waits_for_session_registration(monkeypatch):+ server_started = threading.Event()+ submission_started = threading.Event()+ release_submission = threading.Event()+ shutdown_started = threading.Event()++ async def fake_server_loop(self):+ server_started.set()+ await asyncio.Future()++ real_run_coroutine_threadsafe = asyncio.run_coroutine_threadsafe++ def blocking_submission(coroutine, event_loop):+ future = real_run_coroutine_threadsafe(coroutine, event_loop)+ submission_started.set()+ release_submission.wait(timeout=5)+ return future++ real_begin_global_shutdown = MCPToolCallSession._begin_global_shutdown.__func__++ def observed_begin_global_shutdown(cls):+ shutdown_started.set()+ return real_begin_global_shutdown(cls)++ monkeypatch.setattr(MCPToolCallSession, "_mcp_server_loop", fake_server_loop)+ monkeypatch.setattr(mcp_tool_call_conn.asyncio, "run_coroutine_threadsafe", blocking_submission)+ monkeypatch.setattr(MCPToolCallSession, "_begin_global_shutdown", classmethod(observed_begin_global_shutdown))++ pool = ThreadPoolExecutor(max_workers=2)+ constructor = pool.submit(MCPToolCallSession, SimpleNamespace(id="server-1"))+ shutdown = None+ session = None++ try:+ assert submission_started.wait(timeout=1)+ shutdown = pool.submit(mcp_tool_call_conn.shutdown_all_mcp_sessions)+ assert shutdown_started.wait(timeout=1)+ with pytest.raises(FuturesTimeoutError):+ shutdown.result(timeout=0.05)++ release_submission.set()+ session = constructor.result(timeout=2)+ shutdown.result(timeout=2)++ assert server_started.is_set()+ assert session._shutdown_complete.is_set()+ assert not session._thread.is_alive()+ assert session._event_loop.is_closed()+ assert session not in MCPToolCallSession._ALL_INSTANCES+ assert MCPToolCallSession._GLOBAL_SHUTDOWN_COUNT == 0+ finally:+ release_submission.set()+ try:+ if session is None:+ session = constructor.result(timeout=2)+ if shutdown is not None:+ shutdown.result(timeout=2)+ finally:+ try:+ if session is not None and session._thread.is_alive():+ session.close_sync(timeout=1)+ finally:+ pool.shutdown(wait=True)+++def test_session_creation_is_rejected_during_global_shutdown(monkeypatch):+ shutdown_started = threading.Event()+ release_shutdown = threading.Event()+ active_instances = iter([[SimpleNamespace()], []])++ def blocking_close_multiple(sessions):+ shutdown_started.set()+ assert release_shutdown.wait(timeout=5)+ return 0++ monkeypatch.setattr(MCPToolCallSession, "_active_instances", classmethod(lambda cls: next(active_instances)))+ monkeypatch.setattr(mcp_tool_call_conn, "close_multiple_mcp_toolcall_sessions", blocking_close_multiple)++ pool = ThreadPoolExecutor(max_workers=1)+ shutdown = pool.submit(mcp_tool_call_conn.shutdown_all_mcp_sessions)++ try:+ assert shutdown_started.wait(timeout=1)+ with pytest.raises(RuntimeError, match="Cannot create MCP session during global shutdown"):+ MCPToolCallSession(SimpleNamespace(id="rejected-server"))+ finally:+ release_shutdown.set()+ try:+ shutdown.result(timeout=2)+ finally:+ pool.shutdown(wait=True)++ assert MCPToolCallSession._GLOBAL_SHUTDOWN_COUNT == 0+++def test_shutdown_all_reports_sessions_still_shutting_down(monkeypatch, caplog):+ session = SimpleNamespace()++ monkeypatch.setattr(MCPToolCallSession, "_active_instances", classmethod(lambda cls: [session]))+ monkeypatch.setattr(mcp_tool_call_conn, "close_multiple_mcp_toolcall_sessions", lambda sessions: 0)++ with caplog.at_level("INFO"):+ mcp_tool_call_conn.shutdown_all_mcp_sessions()++ assert "1 MCPToolCallSession instances are still shutting down." in caplog.messages+ assert "All MCPToolCallSession instances have been closed." not in caplog.messages+++def test_close_multiple_reports_transport_cleanup_error_once(caplog):+ class FailingSession:+ _thread = object()+ _mcp_server = SimpleNamespace(id="server-1")+ _shutdown_complete = threading.Event()++ def close_sync(self):+ self._shutdown_complete.set()+ raise RuntimeError("transport cleanup failed")++ with caplog.at_level("INFO"):+ cleanup_errors = mcp_tool_call_conn.close_multiple_mcp_toolcall_sessions([FailingSession()])++ assert cleanup_errors == 1+ assert caplog.messages.count("Exception while closing MCP session for server server-1") == 1+ assert any("1 MCP sessions stopped; transport cleanup failure count: 1" in message for message in caplog.messages)+ assert not any("MCP sessions have been cleaned up" in message for message in caplog.messages)+++def test_shutdown_all_reports_transport_cleanup_errors(monkeypatch, caplog):+ active_instances = iter([[SimpleNamespace()], []])++ monkeypatch.setattr(MCPToolCallSession, "_active_instances", classmethod(lambda cls: next(active_instances)))+ monkeypatch.setattr(mcp_tool_call_conn, "close_multiple_mcp_toolcall_sessions", lambda sessions: 1)++ with caplog.at_level("INFO"):+ mcp_tool_call_conn.shutdown_all_mcp_sessions()++ assert "All MCPToolCallSession event loops stopped; transport cleanup failure count: 1." in caplog.messages+ assert "All MCPToolCallSession instances have been closed." not in caplog.messages++@pytest.mark.parametrize("task_type", ["tool_call", "list_tools"])@pytest.mark.asyncioasync def test_in_flight_timeout_cancels_client_request_and_allows_next_call(task_type):@@ -137,6 +1341,7 @@ def timed_out_session(monkeypatch):session._close = Falsesession._event_loop = object()session._mcp_server = SimpleNamespace(id="server-1")+ session._shutdown_lock = threading.Lock()return session, timed_out_future@@ -168,6 +1373,8 @@ def test_public_tool_call_uses_one_absolute_deadline(monkeypatch):session = object.__new__(MCPToolCallSession)session._queue = asyncio.Queue()session._close = False+ session._pending_calls = {}+ session._shutdown_lock = threading.Lock()session._event_loop = loopsession._mcp_server = SimpleNamespace(id="server-1")