Skip to content

RAGFlow is an agent-native database: a Retrieval-Augmented Generation (RAG) engine that grew into context infrastructure.

fix(agent): await MCP transport cleanup before stopping event loop (#19093)

Lem0nTea2002bef34757b2b0
diff --git a/api/ragflow_server.py b/api/ragflow_server.py
index 47409e1348..438a763cc9 100644
--- a/api/ragflow_server.py
+++ b/api/ragflow_server.py
@@ -49,6 +49,7 @@ from rag.utils.redis_conn import RedisDistributedLock
stop_event = threading.Event()
chat_channel_thread = None
+shutdown_requested = False
RAGFLOW_DEBUGPY_LISTEN = int(os.environ.get("RAGFLOW_DEBUGPY_LISTEN", "0"))
@@ -81,13 +82,15 @@ def stop_background_services():
def signal_handler(sig, frame):
- logging.info("Received interrupt signal, shutting down...")
- shutdown_all_mcp_sessions()
- stop_background_services()
+ global shutdown_requested
+ if shutdown_requested:
+ os.kill(os.getpid(), signal.SIGKILL)
+ return
+ shutdown_requested = True
sys.exit(0)
-if __name__ == "__main__":
+def run_server():
faulthandler.enable()
init_root_logger("ragflow_server")
logging.info(r"""
@@ -136,9 +139,6 @@ if __name__ == "__main__":
GlobalPluginManager.load_plugins()
- signal.signal(signal.SIGINT, signal_handler)
- signal.signal(signal.SIGTERM, signal_handler)
-
def delayed_start_update_progress():
logging.info("Starting update_progress thread (delayed)")
t = threading.Thread(target=update_progress, daemon=True)
@@ -169,12 +169,31 @@ if __name__ == "__main__":
start_chat_channels()
# start http server
+ logging.info(f"RAGFlow server is ready after {time.time() - start_ts}s initialization.")
+ app.run(host=settings.HOST_IP, port=settings.HOST_PORT, use_reloader=RuntimeConfig.DEBUG, debug=False)
+
+
+def main():
+ force_kill = False
try:
- logging.info(f"RAGFlow server is ready after {time.time() - start_ts}s initialization.")
- app.run(host=settings.HOST_IP, port=settings.HOST_PORT, use_reloader=RuntimeConfig.DEBUG, debug=False)
+ signal.signal(signal.SIGINT, signal_handler)
+ signal.signal(signal.SIGTERM, signal_handler)
+ run_server()
except Exception as e:
+ force_kill = True
logging.exception(f"Unhandled exception: {e}")
- stop_background_services()
- os.kill(os.getpid(), signal.SIGKILL)
finally:
- stop_background_services()
+ if shutdown_requested:
+ logging.info("Received interrupt signal, shutting down...")
+ try:
+ shutdown_all_mcp_sessions()
+ finally:
+ try:
+ stop_background_services()
+ finally:
+ if force_kill:
+ os.kill(os.getpid(), signal.SIGKILL)
+
+
+if __name__ == "__main__":
+ main()
diff --git a/common/mcp_tool_call_conn.py b/common/mcp_tool_call_conn.py
index b73b1ce671..7925bea71b 100644
--- a/common/mcp_tool_call_conn.py
+++ b/common/mcp_tool_call_conn.py
@@ -19,7 +19,8 @@ import logging
import threading
import time
import weakref
-from concurrent.futures import ThreadPoolExecutor
+from concurrent.futures import CancelledError as FuturesCancelledError
+from concurrent.futures import Future, ThreadPoolExecutor
from concurrent.futures import TimeoutError as FuturesTimeoutError
from dataclasses import dataclass
from string import Template
@@ -51,21 +52,131 @@ class MCPToolBinding:
class MCPToolCallSession(ToolCallSession):
_ALL_INSTANCES: weakref.WeakSet["MCPToolCallSession"] = weakref.WeakSet()
+ _INSTANCES_LOCK = threading.Lock()
+ _GLOBAL_SHUTDOWN_COUNT = 0
def __init__(self, mcp_server: Any, server_variables: dict[str, Any] | None = None, custom_header=None) -> None:
- self.__class__._ALL_INSTANCES.add(self)
-
self._custom_header = custom_header
self._mcp_server = mcp_server
self._server_variables = server_variables or {}
self._queue: asyncio.Queue[MCPTask] = asyncio.Queue()
self._close = False
+ self._pending_calls: dict[asyncio.Task[Any], asyncio.Queue[Any]] = {}
+ self._shutdown_lock = threading.Lock()
+ self._shutdown_future: Future[None] | None = None
+ self._shutdown_complete = threading.Event()
+ self._server_task: asyncio.Task[None] | None = None
+ self._server_task_started = asyncio.Event()
+ self._owner_close_waiters = 0
+ self._stop_handle: asyncio.TimerHandle | None = None
self._event_loop = asyncio.new_event_loop()
- self._thread_pool = ThreadPoolExecutor(max_workers=1)
- self._thread_pool.submit(self._event_loop.run_forever)
+ self._thread = threading.Thread(target=self._run_event_loop, name=f"mcp-session-{mcp_server.id}", daemon=True)
+ coroutine = self._run_mcp_server_loop()
+ thread_started = False
+ coroutine_submitted = False
+ try:
+ with self.__class__._INSTANCES_LOCK:
+ if self.__class__._GLOBAL_SHUTDOWN_COUNT:
+ raise RuntimeError("Cannot create MCP session during global shutdown")
+ self._thread.start()
+ thread_started = True
+ self._server_future = asyncio.run_coroutine_threadsafe(coroutine, self._event_loop)
+ coroutine_submitted = True
+ self.__class__._ALL_INSTANCES.add(self)
+ except BaseException as startup_error: # noqa: BLE001
+ startup_traceback = startup_error.__traceback__
+ was_started = thread_started or self._thread.ident is not None
+ if was_started:
+ try:
+ if not self._event_loop.is_closed():
+ self._event_loop.call_soon_threadsafe(self._event_loop.stop)
+ except BaseException:
+ logger.exception("Failed to stop MCP session event loop during startup rollback")
+ try:
+ self._thread.join()
+ except BaseException:
+ logger.exception("Failed to join MCP session thread during startup rollback")
+ else:
+ try:
+ self._event_loop.close()
+ except BaseException:
+ logger.exception("Failed to close MCP session event loop during startup rollback")
+ finally:
+ self._shutdown_complete.set()
+ try:
+ if not coroutine_submitted and coroutine.cr_frame is not None:
+ coroutine.close()
+ except BaseException:
+ logger.exception("Failed to close MCP server coroutine during startup rollback")
+ try:
+ with self.__class__._INSTANCES_LOCK:
+ self.__class__._ALL_INSTANCES.discard(self)
+ except BaseException:
+ logger.exception("Failed to unregister MCP session during startup rollback")
+ raise startup_error.with_traceback(startup_traceback)
+
+ @classmethod
+ def _active_instances(cls) -> list["MCPToolCallSession"]:
+ with cls._INSTANCES_LOCK:
+ return list(cls._ALL_INSTANCES)
+
+ @classmethod
+ def _begin_global_shutdown(cls) -> list["MCPToolCallSession"]:
+ with cls._INSTANCES_LOCK:
+ cls._GLOBAL_SHUTDOWN_COUNT += 1
+ try:
+ return cls._active_instances()
+ except BaseException:
+ cls._end_global_shutdown()
+ raise
- asyncio.run_coroutine_threadsafe(self._mcp_server_loop(), self._event_loop)
+ @classmethod
+ def _end_global_shutdown(cls) -> None:
+ with cls._INSTANCES_LOCK:
+ cls._GLOBAL_SHUTDOWN_COUNT -= 1
+
+ def _run_event_loop(self) -> None:
+ asyncio.set_event_loop(self._event_loop)
+ try:
+ self._event_loop.run_forever()
+ finally:
+ try:
+ try:
+ self._drain_pending_tasks()
+ except Exception:
+ logger.exception("Failed to drain pending tasks for MCP server %s", self._mcp_server.id)
+ try:
+ self._event_loop.run_until_complete(self._event_loop.shutdown_asyncgens())
+ except Exception:
+ logger.exception("Failed to shut down async generators for MCP server %s", self._mcp_server.id)
+ try:
+ self._drain_pending_tasks()
+ except Exception:
+ logger.exception("Failed to drain late tasks for MCP server %s", self._mcp_server.id)
+ try:
+ self._event_loop.run_until_complete(self._event_loop.shutdown_default_executor())
+ except Exception:
+ logger.exception("Failed to shut down default executor for MCP server %s", self._mcp_server.id)
+ finally:
+ try:
+ self._event_loop.close()
+ finally:
+ with self.__class__._INSTANCES_LOCK:
+ self.__class__._ALL_INSTANCES.discard(self)
+ self._shutdown_complete.set()
+
+ def _drain_pending_tasks(self) -> None:
+ pending_tasks = [task for task in asyncio.all_tasks(self._event_loop) if not task.done()]
+ for task in pending_tasks:
+ task.cancel()
+ if pending_tasks:
+ self._event_loop.run_until_complete(asyncio.gather(*pending_tasks, return_exceptions=True))
+
+ async def _run_mcp_server_loop(self) -> None:
+ self._server_task = asyncio.current_task()
+ self._server_task_started.set()
+ await self._mcp_server_loop()
async def _mcp_server_loop(self) -> None:
url = self._mcp_server.url.strip()
@@ -101,6 +212,8 @@ class MCPToolCallSession(ToolCallSession):
logging.warning(f"SSE transport MCP session cancelled for server {self._mcp_server.id}")
return
except Exception:
+ if self._close:
+ raise
msg = "Connection failed (possibly due to auth error). Please check authentication settings first"
await self._process_mcp_tasks(None, msg)
@@ -121,6 +234,8 @@ class MCPToolCallSession(ToolCallSession):
logging.warning(f"STREAMABLE_HTTP MCP session cancelled for server {self._mcp_server.id}")
return
except Exception as e:
+ if self._close:
+ raise
logging.exception(e)
msg = "Connection failed (possibly due to auth error). Please check authentication settings first"
await self._process_mcp_tasks(None, msg)
@@ -178,6 +293,10 @@ class MCPToolCallSession(ToolCallSession):
raise ValueError("Session is closed")
results = asyncio.Queue()
+ caller_task = asyncio.current_task()
+ if caller_task is None:
+ raise RuntimeError("MCP calls require an asyncio task")
+ self._pending_calls[caller_task] = results
deadline = deadline if deadline is not None else time.monotonic() + request_timeout
abandoned = asyncio.Event()
await self._queue.put((task_type, kwargs, results, deadline, abandoned))
@@ -193,6 +312,7 @@ class MCPToolCallSession(ToolCallSession):
raise
finally:
abandoned.set()
+ self._pending_calls.pop(caller_task, None)
async def _call_mcp_tool(self, name: str, arguments: dict[str, Any], request_timeout: float | int = 10, deadline: float | None = None) -> str:
result: CallToolResult = await self._call_mcp_server("tool_call", name=name, arguments=arguments, request_timeout=request_timeout, deadline=deadline)
@@ -216,11 +336,16 @@ class MCPToolCallSession(ToolCallSession):
raise
def get_tools(self, timeout: float | int = 10) -> list[Tool]:
- if self._close:
- raise ValueError("Session is closed")
-
deadline = time.monotonic() + timeout
- future = asyncio.run_coroutine_threadsafe(self._get_tools_from_mcp_server(request_timeout=timeout, deadline=deadline), self._event_loop)
+ with self._shutdown_lock:
+ if self._close:
+ raise ValueError("Session is closed")
+ coroutine = self._get_tools_from_mcp_server(request_timeout=timeout, deadline=deadline)
+ try:
+ future = asyncio.run_coroutine_threadsafe(coroutine, self._event_loop)
+ except Exception:
+ coroutine.close()
+ raise
try:
return future.result(timeout=max(0, deadline - time.monotonic()))
except FuturesTimeoutError:
@@ -234,14 +359,16 @@ class MCPToolCallSession(ToolCallSession):
@override
def tool_call(self, name: str, arguments: dict[str, Any], timeout: float | int = 10) -> str:
- if self._close:
- return "Error: Session is closed"
-
deadline = time.monotonic() + timeout
- future = asyncio.run_coroutine_threadsafe(
- self._call_mcp_tool(name, arguments, request_timeout=timeout, deadline=deadline),
- self._event_loop,
- )
+ with self._shutdown_lock:
+ if self._close:
+ return "Error: Session is closed"
+ coroutine = self._call_mcp_tool(name, arguments, request_timeout=timeout, deadline=deadline)
+ try:
+ future = asyncio.run_coroutine_threadsafe(coroutine, self._event_loop)
+ except Exception:
+ coroutine.close()
+ raise
try:
return future.result(timeout=max(0, deadline - time.monotonic()))
except FuturesTimeoutError:
@@ -252,90 +379,204 @@ class MCPToolCallSession(ToolCallSession):
logging.exception(f"Error calling tool '{name}' on MCP server: {self._mcp_server.id}")
return f"Error calling tool '{name}': {e}."
- async def close(self) -> None:
- if self._close:
- return
-
- self._close = True
+ async def _shutdown_on_event_loop(self) -> None:
+ pending_calls = list(self._pending_calls.items())
+ for _, result_queue in pending_calls:
+ result_queue.put_nowait(ValueError("Session is closing"))
while not self._queue.empty():
+ self._queue.get_nowait()
+
+ try:
+ await self._server_task_started.wait()
+ if self._server_task and not self._server_task.done():
+ self._server_task.cancel()
+
+ request_tasks = [task for task, _ in pending_calls]
+ if request_tasks:
+ await asyncio.gather(*request_tasks, return_exceptions=True)
+ if self._server_task:
+ try:
+ await self._server_task
+ except asyncio.CancelledError:
+ pass
+ finally:
try:
- _, _, result_queue, _, _ = self._queue.get_nowait()
+ await self._event_loop.shutdown_asyncgens()
+ finally:
+ await self._event_loop.shutdown_default_executor()
+
+ def _begin_shutdown(self) -> Future[None]:
+ with self._shutdown_lock:
+ if self._shutdown_future is None:
+ self._close = True
+ logger.info("Closing MCP session for server %s", self._mcp_server.id)
+ coroutine = self._shutdown_on_event_loop()
try:
- await result_queue.put(asyncio.CancelledError("Session is closing"))
+ self._shutdown_future = asyncio.run_coroutine_threadsafe(coroutine, self._event_loop)
except Exception:
- pass
- except asyncio.QueueEmpty:
- break
- except Exception:
- break
+ coroutine.close()
+ raise
+ self._shutdown_future.add_done_callback(self._request_event_loop_stop)
+ return self._shutdown_future
- try:
- self._event_loop.call_soon_threadsafe(self._event_loop.stop)
- except Exception:
- pass
+ def _request_event_loop_stop(self, _: Future[None]) -> None:
+ if self._event_loop.is_closed():
+ return
+ self._event_loop.call_soon_threadsafe(self._schedule_event_loop_stop)
- try:
- self._thread_pool.shutdown(wait=True)
- except Exception:
- pass
+ def _schedule_event_loop_stop(self) -> None:
+ if self._owner_close_waiters or self._event_loop.is_closed():
+ return
+ if self._stop_handle is None or self._stop_handle.cancelled():
+ self._stop_handle = self._event_loop.call_later(0, self._event_loop.stop)
- self.__class__._ALL_INSTANCES.discard(self)
+ async def close(self, timeout: float = 5) -> None: # noqa: ASYNC109
+ deadline = time.monotonic() + timeout
+ shutdown_finished = False
+ shutdown_error = None
+ shutdown_traceback = None
+ owner_loop = asyncio.get_running_loop() is self._event_loop
+ if owner_loop:
+ self._owner_close_waiters += 1
+ if self._stop_handle is not None:
+ self._stop_handle.cancel()
+ self._stop_handle = None
- def close_sync(self, timeout: float | int = 5) -> None:
- if not self._event_loop.is_running():
- logging.warning(f"Event loop already stopped for {self._mcp_server.id}")
- return
+ try:
+ future = self._begin_shutdown()
+ wrapped_future = asyncio.wrap_future(future)
+ wrapped_future.add_done_callback(lambda completed: None if completed.cancelled() else completed.exception())
+ done, _ = await asyncio.wait({wrapped_future}, timeout=max(0, deadline - time.monotonic()))
+ if not done:
+ logger.error("Timeout while closing session for server %s (timeout=%s)", self._mcp_server.id, timeout)
+ else:
+ shutdown_finished = True
+ await wrapped_future
+ except BaseException as error: # noqa: BLE001
+ shutdown_error = error
+ shutdown_traceback = error.__traceback__
+ finally:
+ if owner_loop:
+ self._owner_close_waiters -= 1
+ if self._owner_close_waiters == 0 and self._shutdown_future is not None and self._shutdown_future.done():
+ self._schedule_event_loop_stop()
+
+ join_error = None
+ join_traceback = None
+ if not owner_loop:
+ remaining = max(0, deadline - time.monotonic())
+ if self._thread.is_alive() and remaining:
+ join_task = asyncio.create_task(asyncio.to_thread(self._thread.join, remaining))
+ join_task.add_done_callback(lambda completed: None if completed.cancelled() else completed.exception())
+ try:
+ done, _ = await asyncio.wait({join_task}, timeout=remaining)
+ if done:
+ await join_task
+ except BaseException as error: # noqa: BLE001
+ join_error = error
+ join_traceback = error.__traceback__
+ if self._thread.is_alive() and shutdown_finished and join_error is None:
+ logger.error("Timeout while joining MCP session thread for server %s (timeout=%s)", self._mcp_server.id, timeout)
+
+ if shutdown_error is not None:
+ if join_error is not None:
+ logger.error(
+ "Exception while joining MCP session thread for server %s",
+ self._mcp_server.id,
+ exc_info=(type(join_error), join_error, join_traceback),
+ )
+ raise shutdown_error.with_traceback(shutdown_traceback)
+ if join_error is not None:
+ raise join_error.with_traceback(join_traceback)
+ def close_sync(self, timeout: float | int = 5) -> None:
try:
- future = asyncio.run_coroutine_threadsafe(self.close(), self._event_loop)
- try:
- future.result(timeout=timeout)
- except FuturesTimeoutError:
- logging.error(f"Timeout while closing session for server {self._mcp_server.id} (timeout={timeout})")
- except Exception:
- logging.exception(f"Unexpected error during close_sync for {self._mcp_server.id}")
- except Exception:
- logging.exception(f"Exception while scheduling close for server {self._mcp_server.id}")
+ running_loop = asyncio.get_running_loop()
+ except RuntimeError:
+ running_loop = None
+ if running_loop is self._event_loop:
+ raise RuntimeError("close_sync() cannot block the MCP session event loop")
+ deadline = time.monotonic() + timeout
+ future = self._begin_shutdown()
-def close_multiple_mcp_toolcall_sessions(sessions: list[MCPToolCallSession]) -> None:
- logging.info(f"Want to clean up {len(sessions)} MCP sessions")
+ try:
+ shutdown_error = future.exception(timeout=max(0, deadline - time.monotonic()))
+ except FuturesTimeoutError:
+ logger.error("Timeout while closing session for server %s (timeout=%s)", self._mcp_server.id, timeout)
+ return
+ except FuturesCancelledError as error:
+ shutdown_error = error
- async def _gather_and_stop() -> None:
+ join_error = None
+ join_traceback = None
try:
- await asyncio.gather(*[s.close() for s in sessions if s is not None], return_exceptions=True)
- except Exception:
- logging.exception("Exception during MCP session cleanup")
- finally:
+ self._thread.join(timeout=max(0, deadline - time.monotonic()))
+ except Exception as error: # noqa: BLE001
+ join_error = error
+ join_traceback = error.__traceback__
+ if self._thread.is_alive() and join_error is None:
+ logger.error("Timeout while joining MCP session thread for server %s (timeout=%s)", self._mcp_server.id, timeout)
+ if shutdown_error is not None:
+ if join_error is not None:
+ logger.error(
+ "Exception while joining MCP session thread for server %s",
+ self._mcp_server.id,
+ exc_info=(type(join_error), join_error, join_traceback),
+ )
+ raise shutdown_error
+ if join_error is not None:
+ raise join_error.with_traceback(join_traceback)
+
+
+def close_multiple_mcp_toolcall_sessions(sessions: list[MCPToolCallSession]) -> int:
+ unique_sessions = list({id(session): session for session in sessions if session is not None}.values())
+ current_thread = threading.current_thread()
+ if any(session._thread is current_thread for session in unique_sessions):
+ raise RuntimeError("MCP sessions must be closed outside their event loop threads")
+
+ logger.info("Want to clean up %s MCP sessions", len(unique_sessions))
+
+ cleanup_errors = 0
+ with ThreadPoolExecutor() as executor:
+ futures = {executor.submit(session.close_sync): session for session in unique_sessions}
+ for future, session in futures.items():
try:
- loop.call_soon_threadsafe(loop.stop)
+ future.result()
except Exception:
- pass
-
- try:
- loop = asyncio.new_event_loop()
- thread = threading.Thread(target=loop.run_forever, daemon=True)
- thread.start()
-
- asyncio.run_coroutine_threadsafe(_gather_and_stop(), loop).result()
- thread.join()
- except Exception:
- logging.exception("Exception during MCP session cleanup thread management")
+ cleanup_errors += 1
+ logger.exception("Exception while closing MCP session for server %s", session._mcp_server.id)
- logging.info(f"{len(sessions)} MCP sessions has been cleaned up. {len(list(MCPToolCallSession._ALL_INSTANCES))} in global context.")
+ closed_count = sum(session._shutdown_complete.is_set() for session in unique_sessions)
+ if closed_count != len(unique_sessions):
+ logger.warning("%s of %s MCP sessions are still shutting down", len(unique_sessions) - closed_count, len(unique_sessions))
+ if cleanup_errors:
+ logger.warning("%s MCP sessions stopped; transport cleanup failure count: %s. %s in global context.", closed_count, cleanup_errors, len(MCPToolCallSession._active_instances()))
+ else:
+ logger.info("%s MCP sessions have been cleaned up. %s in global context.", closed_count, len(MCPToolCallSession._active_instances()))
+ return cleanup_errors
def shutdown_all_mcp_sessions():
"""Gracefully shutdown all active MCPToolCallSession instances."""
- sessions = list(MCPToolCallSession._ALL_INSTANCES)
- if not sessions:
- logging.info("No MCPToolCallSession instances to close.")
- return
-
- logging.info(f"Shutting down {len(sessions)} MCPToolCallSession instances...")
- close_multiple_mcp_toolcall_sessions(sessions)
- logging.info("All MCPToolCallSession instances have been closed.")
+ sessions = MCPToolCallSession._begin_global_shutdown()
+ try:
+ if not sessions:
+ logging.info("No MCPToolCallSession instances to close.")
+ return
+
+ logging.info(f"Shutting down {len(sessions)} MCPToolCallSession instances...")
+ cleanup_errors = close_multiple_mcp_toolcall_sessions(sessions)
+ active_sessions = MCPToolCallSession._active_instances()
+ if active_sessions:
+ logger.warning("%s MCPToolCallSession instances are still shutting down.", len(active_sessions))
+ elif cleanup_errors:
+ logger.warning("All MCPToolCallSession event loops stopped; transport cleanup failure count: %s.", cleanup_errors)
+ else:
+ logger.info("All MCPToolCallSession instances have been closed.")
+ finally:
+ MCPToolCallSession._end_global_shutdown()
def mcp_tool_metadata_to_openai_tool(mcp_tool: Tool | dict, function_name: str | None = None) -> dict[str, Any]:
diff --git a/test/unit_test/api/test_ragflow_server_shutdown.py b/test/unit_test/api/test_ragflow_server_shutdown.py
new file mode 100644
index 0000000000..28707f8f11
--- /dev/null
+++ b/test/unit_test/api/test_ragflow_server_shutdown.py
@@ -0,0 +1,62 @@
+import ast
+from pathlib import Path
+from types import SimpleNamespace
+from unittest.mock import Mock
+
+SERVER_PATH = Path(__file__).resolve().parents[3] / "api" / "ragflow_server.py"
+
+
+def test_signal_handler_forces_exit_on_second_signal():
+ tree = ast.parse(SERVER_PATH.read_text(encoding="utf-8"), filename=str(SERVER_PATH))
+ handler = next(node for node in tree.body if isinstance(node, ast.FunctionDef) and node.name == "signal_handler")
+ namespace = {
+ "shutdown_requested": False,
+ "os": SimpleNamespace(getpid=Mock(return_value=1234), kill=Mock()),
+ "signal": SimpleNamespace(SIGKILL=9),
+ "sys": SimpleNamespace(exit=Mock(side_effect=SystemExit(0))),
+ }
+ exec(compile(ast.fix_missing_locations(ast.Module(body=[handler], type_ignores=[])), str(SERVER_PATH), "exec"), namespace)
+
+ try:
+ namespace["signal_handler"](2, None)
+ except SystemExit as exc:
+ assert exc.code == 0
+ else:
+ raise AssertionError("the first shutdown signal must request graceful process exit")
+
+ assert namespace["shutdown_requested"] is True
+ namespace["os"].kill.assert_not_called()
+
+ namespace["signal_handler"](15, None)
+
+ namespace["os"].getpid.assert_called_once_with()
+ namespace["os"].kill.assert_called_once_with(1234, 9)
+ namespace["sys"].exit.assert_called_once_with(0)
+
+
+def test_server_shutdown_wraps_full_startup_and_forces_cleanup_before_kill():
+ tree = ast.parse(SERVER_PATH.read_text(encoding="utf-8"), filename=str(SERVER_PATH))
+ main = next(node for node in tree.body if isinstance(node, ast.FunctionDef) and node.name == "main")
+ main_try = next(node for node in main.body if isinstance(node, ast.Try))
+
+ startup_calls = [ast.unparse(statement.value) for statement in main_try.body]
+ assert startup_calls == [
+ "signal.signal(signal.SIGINT, signal_handler)",
+ "signal.signal(signal.SIGTERM, signal_handler)",
+ "run_server()",
+ ]
+
+ exception_handler = main_try.handlers[0]
+ assert ast.unparse(exception_handler.body[0]) == "force_kill = True"
+
+ mcp_cleanup = next(node for node in main_try.finalbody if isinstance(node, ast.Try))
+ assert ast.unparse(mcp_cleanup.body[0].value) == "shutdown_all_mcp_sessions()"
+
+ background_cleanup = mcp_cleanup.finalbody[0]
+ assert isinstance(background_cleanup, ast.Try)
+ assert ast.unparse(background_cleanup.body[0].value) == "stop_background_services()"
+
+ force_kill = background_cleanup.finalbody[0]
+ assert isinstance(force_kill, ast.If)
+ assert ast.unparse(force_kill.test) == "force_kill"
+ assert ast.unparse(force_kill.body[0].value) == "os.kill(os.getpid(), signal.SIGKILL)"
diff --git a/test/unit_test/common/test_mcp_tool_call_conn.py b/test/unit_test/common/test_mcp_tool_call_conn.py
index 9cf56bcd69..9c6fa0072d 100644
--- a/test/unit_test/common/test_mcp_tool_call_conn.py
+++ b/test/unit_test/common/test_mcp_tool_call_conn.py
@@ -1,12 +1,16 @@
import asyncio
import threading
import time
+from concurrent.futures import CancelledError as FuturesCancelledError
from concurrent.futures import ThreadPoolExecutor
+from concurrent.futures import TimeoutError as FuturesTimeoutError
+from contextlib import asynccontextmanager
from types import SimpleNamespace
import pytest
from common import mcp_tool_call_conn
+from common.constants import MCPServerType
from common.mcp_tool_call_conn import MCPToolCallSession
@@ -14,6 +18,8 @@ def _make_session() -> MCPToolCallSession:
session = object.__new__(MCPToolCallSession)
session._queue = asyncio.Queue()
session._close = False
+ session._pending_calls = {}
+ session._shutdown_lock = threading.Lock()
return session
@@ -35,6 +41,1204 @@ async def _stop_tasks(session: MCPToolCallSession, *tasks: asyncio.Task) -> None
assert all(task.done() for task in tasks)
[email protected]("server_type", [MCPServerType.SSE, MCPServerType.STREAMABLE_HTTP])
+def test_close_sync_waits_for_transport_context_cleanup(monkeypatch, server_type):
+ initialized = threading.Event()
+ client_closed = threading.Event()
+ transport_closed = threading.Event()
+
+ @asynccontextmanager
+ async def fake_sse_client(url, headers):
+ try:
+ yield object(), object()
+ finally:
+ transport_closed.set()
+
+ @asynccontextmanager
+ async def fake_streamable_http_client(url, headers):
+ try:
+ yield object(), object(), None
+ finally:
+ transport_closed.set()
+
+ class FakeClientSession:
+ def __init__(self, *streams):
+ pass
+
+ async def __aenter__(self):
+ return self
+
+ async def __aexit__(self, exc_type, exc, traceback):
+ client_closed.set()
+
+ async def initialize(self):
+ initialized.set()
+
+ monkeypatch.setattr(mcp_tool_call_conn, "sse_client", fake_sse_client)
+ monkeypatch.setattr(mcp_tool_call_conn, "streamablehttp_client", fake_streamable_http_client)
+ monkeypatch.setattr(mcp_tool_call_conn, "ClientSession", FakeClientSession)
+
+ server = SimpleNamespace(id="server-1", url="http://mcp.test", headers={}, server_type=server_type)
+ session = MCPToolCallSession(server)
+
+ try:
+ assert initialized.wait(timeout=1)
+ session.close_sync(timeout=1)
+
+ assert client_closed.is_set()
+ assert transport_closed.is_set()
+ assert not session._event_loop.is_running()
+ assert session._event_loop.is_closed()
+ assert not asyncio.all_tasks(session._event_loop)
+ assert session not in MCPToolCallSession._ALL_INSTANCES
+ finally:
+ session.close_sync(timeout=1)
+
+
+def test_close_sync_cancels_blocked_initialization(monkeypatch):
+ initialize_started = threading.Event()
+ initialize_cancelled = threading.Event()
+ client_closed = threading.Event()
+ transport_closed = threading.Event()
+
+ @asynccontextmanager
+ async def fake_sse_client(url, headers):
+ try:
+ yield object(), object()
+ finally:
+ transport_closed.set()
+
+ class FakeClientSession:
+ def __init__(self, *streams):
+ pass
+
+ async def __aenter__(self):
+ return self
+
+ async def __aexit__(self, exc_type, exc, traceback):
+ client_closed.set()
+
+ async def initialize(self):
+ initialize_started.set()
+ try:
+ await asyncio.Future()
+ except asyncio.CancelledError:
+ initialize_cancelled.set()
+ raise
+
+ monkeypatch.setattr(mcp_tool_call_conn, "sse_client", fake_sse_client)
+ monkeypatch.setattr(mcp_tool_call_conn, "ClientSession", FakeClientSession)
+
+ server = SimpleNamespace(id="server-1", url="http://mcp.test", headers={}, server_type=MCPServerType.SSE)
+ session = MCPToolCallSession(server)
+
+ try:
+ assert initialize_started.wait(timeout=1)
+ session.close_sync(timeout=1)
+
+ assert initialize_cancelled.is_set()
+ assert client_closed.is_set()
+ assert transport_closed.is_set()
+ assert session._event_loop.is_closed()
+ finally:
+ session.close_sync(timeout=1)
+
+
+def test_async_close_waits_for_thread_exit(monkeypatch):
+ server_started = threading.Event()
+ finalized = threading.Event()
+
+ async def fake_server_loop(self):
+ server_started.set()
+ try:
+ await asyncio.Future()
+ finally:
+ finalized.set()
+
+ monkeypatch.setattr(MCPToolCallSession, "_mcp_server_loop", fake_server_loop)
+
+ session = MCPToolCallSession(SimpleNamespace(id="server-1"))
+
+ try:
+ assert server_started.wait(timeout=1)
+ asyncio.run(session.close())
+
+ assert finalized.is_set()
+ assert not session._thread.is_alive()
+ assert session._event_loop.is_closed()
+ assert session not in MCPToolCallSession._ALL_INSTANCES
+ finally:
+ session.close_sync(timeout=1)
+
+
+def test_async_close_runs_on_session_event_loop(monkeypatch):
+ server_started = threading.Event()
+ finalized = threading.Event()
+
+ async def fake_server_loop(self):
+ server_started.set()
+ try:
+ await asyncio.Future()
+ finally:
+ finalized.set()
+
+ monkeypatch.setattr(MCPToolCallSession, "_mcp_server_loop", fake_server_loop)
+
+ session = MCPToolCallSession(SimpleNamespace(id="server-1"))
+
+ try:
+ assert server_started.wait(timeout=1)
+ close_future = asyncio.run_coroutine_threadsafe(session.close(), session._event_loop)
+ close_future.result(timeout=1)
+ session._thread.join(timeout=1)
+
+ assert finalized.is_set()
+ assert not session._thread.is_alive()
+ assert session._event_loop.is_closed()
+ assert session not in MCPToolCallSession._ALL_INSTANCES
+ finally:
+ session.close_sync(timeout=1)
+
+
+def test_late_owner_close_is_cancelled_when_event_loop_stops(monkeypatch):
+ server_started = threading.Event()
+ stop_scheduled = threading.Event()
+ release_stop_scheduler = threading.Event()
+
+ async def fake_server_loop(self):
+ server_started.set()
+ await asyncio.Future()
+
+ original_schedule_stop = MCPToolCallSession._schedule_event_loop_stop
+
+ def blocking_schedule_stop(self):
+ original_schedule_stop(self)
+ stop_scheduled.set()
+ release_stop_scheduler.wait(timeout=1)
+
+ monkeypatch.setattr(MCPToolCallSession, "_mcp_server_loop", fake_server_loop)
+ monkeypatch.setattr(MCPToolCallSession, "_schedule_event_loop_stop", blocking_schedule_stop)
+
+ session = MCPToolCallSession(SimpleNamespace(id="server-1"))
+ late_close = None
+
+ try:
+ assert server_started.wait(timeout=1)
+ shutdown_future = session._begin_shutdown()
+ shutdown_future.result(timeout=1)
+ assert stop_scheduled.wait(timeout=1)
+
+ late_close = asyncio.run_coroutine_threadsafe(session.close(), session._event_loop)
+ release_stop_scheduler.set()
+
+ with pytest.raises(FuturesCancelledError):
+ late_close.result(timeout=1)
+ session._thread.join(timeout=1)
+ assert not session._thread.is_alive()
+ assert session._event_loop.is_closed()
+ assert not asyncio.all_tasks(session._event_loop)
+ finally:
+ release_stop_scheduler.set()
+ if late_close is not None and not late_close.done():
+ late_close.cancel()
+ session.close_sync(timeout=1)
+
+
+def test_owner_close_queued_by_stop_callback_is_drained(monkeypatch):
+ server_started = threading.Event()
+ late_close_submitted = threading.Event()
+ late_close_holder = {}
+
+ async def fake_server_loop(self):
+ server_started.set()
+ await asyncio.Future()
+
+ monkeypatch.setattr(MCPToolCallSession, "_mcp_server_loop", fake_server_loop)
+
+ session = MCPToolCallSession(SimpleNamespace(id="server-1"))
+ original_stop = session._event_loop.stop
+ submitted = False
+
+ def submit_close_then_stop():
+ nonlocal submitted
+ if not submitted:
+ submitted = True
+ late_close_holder["future"] = asyncio.run_coroutine_threadsafe(session.close(), session._event_loop)
+ late_close_submitted.set()
+ original_stop()
+
+ monkeypatch.setattr(session._event_loop, "stop", submit_close_then_stop)
+
+ try:
+ assert server_started.wait(timeout=1)
+ session.close_sync(timeout=1)
+ assert late_close_submitted.is_set()
+
+ late_close = late_close_holder["future"]
+ with pytest.raises(FuturesCancelledError):
+ late_close.result(timeout=1)
+ assert not session._thread.is_alive()
+ assert session._event_loop.is_closed()
+ assert not asyncio.all_tasks(session._event_loop)
+ finally:
+ session.close_sync(timeout=1)
+
+
+def test_close_sync_propagates_transport_cleanup_error(monkeypatch, caplog):
+ initialized = threading.Event()
+ join_attempted = threading.Event()
+
+ @asynccontextmanager
+ async def fake_sse_client(url, headers):
+ try:
+ yield object(), object()
+ finally:
+ raise RuntimeError("transport cleanup failed")
+
+ class FakeClientSession:
+ def __init__(self, *streams):
+ pass
+
+ async def __aenter__(self):
+ return self
+
+ async def __aexit__(self, exc_type, exc, traceback):
+ pass
+
+ async def initialize(self):
+ initialized.set()
+
+ monkeypatch.setattr(mcp_tool_call_conn, "sse_client", fake_sse_client)
+ monkeypatch.setattr(mcp_tool_call_conn, "ClientSession", FakeClientSession)
+
+ server = SimpleNamespace(id="server-1", url="http://mcp.test", headers={}, server_type=MCPServerType.SSE)
+ session = MCPToolCallSession(server)
+ real_join = session._thread.join
+
+ def failing_join(timeout=None):
+ join_attempted.set()
+ raise ValueError("join failed")
+
+ monkeypatch.setattr(session._thread, "join", failing_join)
+
+ try:
+ assert initialized.wait(timeout=1)
+ with caplog.at_level("ERROR"), pytest.raises(RuntimeError, match="transport cleanup failed"):
+ session.close_sync(timeout=1)
+
+ assert join_attempted.is_set()
+ assert "Exception while joining MCP session thread for server server-1" in caplog.messages
+ join_record = next(record for record in caplog.records if record.getMessage() == "Exception while joining MCP session thread for server server-1")
+ assert join_record.exc_info is not None
+ assert join_record.exc_info[0] is ValueError
+ assert str(join_record.exc_info[1]) == "join failed"
+
+ monkeypatch.setattr(session._thread, "join", real_join)
+ assert session._shutdown_complete.wait(timeout=1)
+ real_join(timeout=1)
+ assert not session._thread.is_alive()
+ assert session._event_loop.is_closed()
+ assert session not in MCPToolCallSession._ALL_INSTANCES
+ finally:
+ monkeypatch.setattr(session._thread, "join", real_join)
+ if session._thread.is_alive():
+ try:
+ session.close_sync(timeout=1)
+ except RuntimeError as error:
+ if str(error) != "transport cleanup failed":
+ raise
+
+
+def test_close_sync_propagates_thread_join_error(monkeypatch):
+ server_started = threading.Event()
+
+ async def fake_server_loop(self):
+ server_started.set()
+ await asyncio.Future()
+
+ monkeypatch.setattr(MCPToolCallSession, "_mcp_server_loop", fake_server_loop)
+
+ session = MCPToolCallSession(SimpleNamespace(id="server-1"))
+ real_join = session._thread.join
+
+ def failing_join(timeout=None):
+ raise ValueError("join failed")
+
+ monkeypatch.setattr(session._thread, "join", failing_join)
+
+ try:
+ assert server_started.wait(timeout=1)
+ with pytest.raises(ValueError, match="join failed"):
+ session.close_sync(timeout=1)
+ finally:
+ monkeypatch.setattr(session._thread, "join", real_join)
+ session.close_sync(timeout=1)
+
+
+def test_close_sync_cancels_in_flight_call_and_wakes_caller(monkeypatch):
+ call_started = threading.Event()
+ call_finalized = threading.Event()
+
+ class FakeClientSession:
+ async def call_tool(self, name, arguments):
+ call_started.set()
+ try:
+ await asyncio.Future()
+ finally:
+ call_finalized.set()
+
+ async def fake_server_loop(self):
+ await self._process_mcp_tasks(FakeClientSession())
+
+ monkeypatch.setattr(MCPToolCallSession, "_mcp_server_loop", fake_server_loop)
+
+ server = SimpleNamespace(id="server-1")
+ session = MCPToolCallSession(server)
+ caller_pool = ThreadPoolExecutor(max_workers=1)
+ caller = caller_pool.submit(session.tool_call, "slow", {}, 3)
+
+ try:
+ assert call_started.wait(timeout=1)
+ session.close_sync(timeout=1)
+
+ result = caller.result(timeout=1)
+ assert "Session is closing" in result
+ assert "Timeout" not in result
+ assert call_finalized.is_set()
+ finally:
+ session.close_sync(timeout=1)
+ caller_pool.shutdown(wait=True)
+
+
+def test_close_sync_wakes_in_flight_get_tools(monkeypatch):
+ call_started = threading.Event()
+ call_finalized = threading.Event()
+
+ class FakeClientSession:
+ async def list_tools(self):
+ call_started.set()
+ try:
+ await asyncio.Future()
+ finally:
+ call_finalized.set()
+
+ async def fake_server_loop(self):
+ await self._process_mcp_tasks(FakeClientSession())
+
+ monkeypatch.setattr(MCPToolCallSession, "_mcp_server_loop", fake_server_loop)
+
+ session = MCPToolCallSession(SimpleNamespace(id="server-1"))
+ caller_pool = ThreadPoolExecutor(max_workers=1)
+ caller = caller_pool.submit(session.get_tools, 3)
+
+ try:
+ assert call_started.wait(timeout=1)
+ session.close_sync(timeout=1)
+
+ with pytest.raises(ValueError, match="Session is closing"):
+ caller.result(timeout=1)
+ assert call_finalized.is_set()
+ finally:
+ session.close_sync(timeout=1)
+ caller_pool.shutdown(wait=True)
+
+
+def test_close_sync_wakes_queued_call_without_dispatching_it(monkeypatch):
+ first_started = threading.Event()
+ first_finalized = threading.Event()
+ queued_enqueued = threading.Event()
+ calls = []
+ queue_type = asyncio.Queue
+
+ class ObservedQueue(queue_type):
+ async def put(self, item):
+ await super().put(item)
+ if isinstance(item, tuple) and len(item) > 1 and item[1].get("name") == "queued":
+ queued_enqueued.set()
+
+ class FakeClientSession:
+ async def call_tool(self, name, arguments):
+ calls.append(name)
+ if name == "first":
+ first_started.set()
+ try:
+ await asyncio.Future()
+ finally:
+ first_finalized.set()
+ return SimpleNamespace(isError=False, content=[])
+
+ async def fake_server_loop(self):
+ await self._process_mcp_tasks(FakeClientSession())
+
+ monkeypatch.setattr(mcp_tool_call_conn.asyncio, "Queue", ObservedQueue)
+ monkeypatch.setattr(MCPToolCallSession, "_mcp_server_loop", fake_server_loop)
+
+ session = MCPToolCallSession(SimpleNamespace(id="server-1"))
+ caller_pool = ThreadPoolExecutor(max_workers=2)
+ first = caller_pool.submit(session.tool_call, "first", {}, 3)
+ queued = None
+
+ try:
+ assert first_started.wait(timeout=1)
+ queued = caller_pool.submit(session.tool_call, "queued", {}, 3)
+ assert queued_enqueued.wait(timeout=1)
+
+ session.close_sync(timeout=1)
+
+ assert "Session is closing" in first.result(timeout=1)
+ assert "Session is closing" in queued.result(timeout=1)
+ assert calls == ["first"]
+ assert first_finalized.is_set()
+ finally:
+ session.close_sync(timeout=1)
+ caller_pool.shutdown(wait=True)
+
+
+def test_concurrent_close_is_idempotent(monkeypatch, caplog):
+ server_started = threading.Event()
+ finalized = threading.Event()
+ finalizer_count = 0
+ count_lock = threading.Lock()
+
+ async def fake_server_loop(self):
+ nonlocal finalizer_count
+ server_started.set()
+ try:
+ await asyncio.Future()
+ finally:
+ with count_lock:
+ finalizer_count += 1
+ finalized.set()
+
+ monkeypatch.setattr(MCPToolCallSession, "_mcp_server_loop", fake_server_loop)
+
+ session = MCPToolCallSession(SimpleNamespace(id="server-1"))
+ barrier = threading.Barrier(4)
+
+ def close_at_barrier():
+ barrier.wait(timeout=1)
+ session.close_sync(timeout=1)
+
+ closer_pool = ThreadPoolExecutor(max_workers=3)
+ closers = [closer_pool.submit(close_at_barrier) for _ in range(3)]
+
+ try:
+ assert server_started.wait(timeout=1)
+ with caplog.at_level("INFO"):
+ barrier.wait(timeout=1)
+ for closer in closers:
+ closer.result(timeout=2)
+
+ session.close_sync(timeout=1)
+ assert finalized.is_set()
+ assert finalizer_count == 1
+ assert caplog.messages.count("Closing MCP session for server server-1") == 1
+ assert session.tool_call("after-close", {}, timeout=1) == "Error: Session is closed"
+ assert not session._thread.is_alive()
+ assert session._event_loop.is_closed()
+ assert session not in MCPToolCallSession._ALL_INSTANCES
+ finally:
+ session.close_sync(timeout=1)
+ closer_pool.shutdown(wait=True)
+
+
[email protected]("operation", ["tool_call", "get_tools"])
+def test_close_waits_for_public_call_submission(monkeypatch, operation):
+ server_started = threading.Event()
+ submission_started = threading.Event()
+ release_submission = threading.Event()
+ close_started = threading.Event()
+
+ async def fake_server_loop(self):
+ server_started.set()
+ await asyncio.Future()
+
+ monkeypatch.setattr(MCPToolCallSession, "_mcp_server_loop", fake_server_loop)
+
+ session = MCPToolCallSession(SimpleNamespace(id="server-1"))
+ real_run_coroutine_threadsafe = asyncio.run_coroutine_threadsafe
+ first_submission = True
+
+ def blocking_first_submission(coroutine, event_loop):
+ nonlocal first_submission
+ if first_submission:
+ first_submission = False
+ submission_started.set()
+ release_submission.wait(timeout=1)
+ return real_run_coroutine_threadsafe(coroutine, event_loop)
+
+ monkeypatch.setattr(mcp_tool_call_conn.asyncio, "run_coroutine_threadsafe", blocking_first_submission)
+
+ caller_pool = ThreadPoolExecutor(max_workers=2)
+ if operation == "tool_call":
+ caller = caller_pool.submit(session.tool_call, "queued", {}, 3)
+ else:
+ caller = caller_pool.submit(session.get_tools, 3)
+
+ def close_session():
+ close_started.set()
+ session.close_sync(timeout=1)
+
+ closer = None
+ try:
+ assert server_started.wait(timeout=1)
+ assert submission_started.wait(timeout=1)
+ closer = caller_pool.submit(close_session)
+ assert close_started.wait(timeout=1)
+ assert not session._close
+
+ release_submission.set()
+ closer.result(timeout=2)
+
+ if operation == "tool_call":
+ assert "Session is clos" in caller.result(timeout=1)
+ else:
+ with pytest.raises(ValueError, match="Session is clos"):
+ caller.result(timeout=1)
+ assert session._event_loop.is_closed()
+ finally:
+ release_submission.set()
+ if closer is not None:
+ closer.result(timeout=2)
+ session.close_sync(timeout=1)
+ caller_pool.shutdown(wait=True)
+
+
+def test_close_timeout_keeps_shared_shutdown_running(monkeypatch):
+ server_started = threading.Event()
+ finalizer_started = threading.Event()
+ finalizer_finished = threading.Event()
+ release_finalizer = asyncio.Event()
+
+ async def fake_server_loop(self):
+ server_started.set()
+ try:
+ await asyncio.Future()
+ finally:
+ finalizer_started.set()
+ await release_finalizer.wait()
+ finalizer_finished.set()
+
+ monkeypatch.setattr(MCPToolCallSession, "_mcp_server_loop", fake_server_loop)
+
+ session = MCPToolCallSession(SimpleNamespace(id="server-1"))
+
+ try:
+ assert server_started.wait(timeout=1)
+ session.close_sync(timeout=0.01)
+
+ assert finalizer_started.wait(timeout=1)
+ assert not session._shutdown_future.done()
+ assert session._thread.is_alive()
+ assert session in MCPToolCallSession._ALL_INSTANCES
+
+ shutdown_future = session._shutdown_future
+ session._event_loop.call_soon_threadsafe(release_finalizer.set)
+ assert session._shutdown_complete.wait(timeout=1)
+ session._thread.join(timeout=1)
+
+ assert session._shutdown_future is shutdown_future
+ assert finalizer_finished.is_set()
+ assert not session._thread.is_alive()
+ assert session._event_loop.is_closed()
+ assert session not in MCPToolCallSession._ALL_INSTANCES
+ finally:
+ if not session._event_loop.is_closed():
+ session._event_loop.call_soon_threadsafe(release_finalizer.set)
+ session.close_sync(timeout=1)
+
+
[email protected]("owner_loop", [False, True], ids=["external-loop", "owner-loop"])
+def test_async_close_timeout_keeps_shared_shutdown_running(monkeypatch, owner_loop):
+ server_started = threading.Event()
+ finalizer_started = threading.Event()
+ finalizer_finished = threading.Event()
+ release_finalizer = asyncio.Event()
+
+ async def fake_server_loop(self):
+ server_started.set()
+ try:
+ await asyncio.Future()
+ finally:
+ finalizer_started.set()
+ await release_finalizer.wait()
+ finalizer_finished.set()
+
+ monkeypatch.setattr(MCPToolCallSession, "_mcp_server_loop", fake_server_loop)
+
+ session = MCPToolCallSession(SimpleNamespace(id="server-1"))
+
+ try:
+ assert server_started.wait(timeout=1)
+ if owner_loop:
+ close_future = asyncio.run_coroutine_threadsafe(session.close(timeout=0.01), session._event_loop)
+ close_future.result(timeout=1)
+ else:
+ asyncio.run(asyncio.wait_for(session.close(timeout=0.01), timeout=1))
+
+ assert finalizer_started.wait(timeout=1)
+ shutdown_future = session._shutdown_future
+ assert shutdown_future is not None
+ assert not shutdown_future.done()
+ assert not shutdown_future.cancelled()
+ assert session._owner_close_waiters == 0
+ assert session._thread.is_alive()
+ assert session in MCPToolCallSession._ALL_INSTANCES
+
+ session._event_loop.call_soon_threadsafe(release_finalizer.set)
+ assert session._shutdown_complete.wait(timeout=1)
+ session._thread.join(timeout=1)
+
+ assert session._shutdown_future is shutdown_future
+ assert finalizer_finished.is_set()
+ assert not session._thread.is_alive()
+ assert session._event_loop.is_closed()
+ assert session not in MCPToolCallSession._ALL_INSTANCES
+ finally:
+ if not session._event_loop.is_closed():
+ session._event_loop.call_soon_threadsafe(release_finalizer.set)
+ session.close_sync(timeout=1)
+
+
+def test_async_close_timeout_bounds_thread_join(monkeypatch):
+ server_started = threading.Event()
+ stop_scheduled = threading.Event()
+ release_stop_scheduler = threading.Event()
+ join_attempted = threading.Event()
+ join_timeouts = []
+
+ async def fake_server_loop(self):
+ server_started.set()
+ await asyncio.Future()
+
+ original_schedule_stop = MCPToolCallSession._schedule_event_loop_stop
+
+ def blocking_schedule_stop(self):
+ stop_scheduled.set()
+ release_stop_scheduler.wait(timeout=5)
+ original_schedule_stop(self)
+
+ monkeypatch.setattr(MCPToolCallSession, "_mcp_server_loop", fake_server_loop)
+ monkeypatch.setattr(MCPToolCallSession, "_schedule_event_loop_stop", blocking_schedule_stop)
+
+ session = MCPToolCallSession(SimpleNamespace(id="server-1"))
+ real_join = session._thread.join
+
+ def observed_join(timeout=None):
+ join_timeouts.append(timeout)
+ join_attempted.set()
+ real_join(timeout)
+
+ monkeypatch.setattr(session._thread, "join", observed_join)
+
+ try:
+ assert server_started.wait(timeout=1)
+ shutdown_future = session._begin_shutdown()
+ shutdown_future.result(timeout=1)
+ assert stop_scheduled.wait(timeout=1)
+
+ asyncio.run(asyncio.wait_for(session.close(timeout=0.5), timeout=1))
+
+ assert join_attempted.is_set()
+ assert 0 < join_timeouts[0] <= 0.5
+ assert session._shutdown_future is shutdown_future
+ assert session._thread.is_alive()
+
+ release_stop_scheduler.set()
+ assert session._shutdown_complete.wait(timeout=1)
+ real_join(timeout=1)
+ assert not session._thread.is_alive()
+ assert session._event_loop.is_closed()
+ assert session not in MCPToolCallSession._ALL_INSTANCES
+ finally:
+ monkeypatch.setattr(session._thread, "join", real_join)
+ release_stop_scheduler.set()
+ session.close_sync(timeout=1)
+
+
+def test_async_close_propagates_transport_cleanup_error(monkeypatch, caplog):
+ server_started = threading.Event()
+ release_stop_scheduler = threading.Event()
+ join_attempted = threading.Event()
+
+ async def fake_server_loop(self):
+ server_started.set()
+ try:
+ await asyncio.Future()
+ finally:
+ raise RuntimeError("transport cleanup failed")
+
+ monkeypatch.setattr(MCPToolCallSession, "_mcp_server_loop", fake_server_loop)
+
+ session = MCPToolCallSession(SimpleNamespace(id="server-1"))
+ real_join = session._thread.join
+
+ def blocking_schedule_stop():
+ release_stop_scheduler.wait(timeout=5)
+ session._event_loop.stop()
+
+ def failing_join(timeout=None):
+ join_attempted.set()
+ raise ValueError("join failed")
+
+ monkeypatch.setattr(session, "_schedule_event_loop_stop", blocking_schedule_stop)
+ monkeypatch.setattr(session._thread, "join", failing_join)
+
+ try:
+ assert server_started.wait(timeout=1)
+ with caplog.at_level("ERROR"), pytest.raises(RuntimeError, match="transport cleanup failed"):
+ asyncio.run(asyncio.wait_for(session.close(timeout=1), timeout=2))
+
+ assert join_attempted.is_set()
+ assert "Exception while joining MCP session thread for server server-1" in caplog.messages
+
+ monkeypatch.setattr(session._thread, "join", real_join)
+ release_stop_scheduler.set()
+ assert session._shutdown_complete.wait(timeout=1)
+ real_join(timeout=1)
+
+ assert not session._thread.is_alive()
+ assert session._event_loop.is_closed()
+ assert session not in MCPToolCallSession._ALL_INSTANCES
+ finally:
+ monkeypatch.setattr(session._thread, "join", real_join)
+ release_stop_scheduler.set()
+ if session._thread.is_alive():
+ try:
+ session.close_sync(timeout=1)
+ except RuntimeError as error:
+ if str(error) != "transport cleanup failed":
+ raise
+
+
+def test_async_close_propagates_thread_join_error(monkeypatch):
+ server_started = threading.Event()
+ release_stop_scheduler = threading.Event()
+
+ async def fake_server_loop(self):
+ server_started.set()
+ await asyncio.Future()
+
+ monkeypatch.setattr(MCPToolCallSession, "_mcp_server_loop", fake_server_loop)
+
+ session = MCPToolCallSession(SimpleNamespace(id="server-1"))
+ real_join = session._thread.join
+
+ def blocking_schedule_stop():
+ release_stop_scheduler.wait(timeout=5)
+ session._event_loop.stop()
+
+ def failing_join(timeout=None):
+ raise ValueError("join failed")
+
+ monkeypatch.setattr(session, "_schedule_event_loop_stop", blocking_schedule_stop)
+ monkeypatch.setattr(session._thread, "join", failing_join)
+
+ try:
+ assert server_started.wait(timeout=1)
+ with pytest.raises(ValueError, match="join failed"):
+ asyncio.run(asyncio.wait_for(session.close(timeout=1), timeout=2))
+ finally:
+ monkeypatch.setattr(session._thread, "join", real_join)
+ release_stop_scheduler.set()
+ session.close_sync(timeout=1)
+
+
+def test_close_keeps_session_active_until_default_executor_finishes(monkeypatch):
+ worker_started = threading.Event()
+ release_worker = threading.Event()
+ worker_finished = threading.Event()
+ worker_holder = {}
+
+ def blocking_worker():
+ worker_holder["thread"] = threading.current_thread()
+ worker_started.set()
+ release_worker.wait(timeout=2)
+ worker_finished.set()
+
+ async def fake_server_loop(self):
+ await asyncio.get_running_loop().run_in_executor(None, blocking_worker)
+
+ monkeypatch.setattr(MCPToolCallSession, "_mcp_server_loop", fake_server_loop)
+
+ session = MCPToolCallSession(SimpleNamespace(id="server-1"))
+
+ try:
+ assert worker_started.wait(timeout=1)
+ session.close_sync(timeout=0.01)
+
+ assert session._thread.is_alive()
+ assert worker_holder["thread"].is_alive()
+ assert not session._shutdown_complete.is_set()
+ assert session in MCPToolCallSession._ALL_INSTANCES
+
+ release_worker.set()
+ session.close_sync(timeout=1)
+
+ assert worker_finished.is_set()
+ assert not worker_holder["thread"].is_alive()
+ assert not session._thread.is_alive()
+ assert session._event_loop.is_closed()
+ assert session not in MCPToolCallSession._ALL_INSTANCES
+ finally:
+ release_worker.set()
+ session.close_sync(timeout=1)
+
+
+def test_transport_cleanup_error_waits_for_default_executor(monkeypatch):
+ server_started = threading.Event()
+ worker_started = threading.Event()
+ release_worker = threading.Event()
+ worker_finished = threading.Event()
+ worker_holder = {}
+
+ def blocking_worker():
+ worker_holder["thread"] = threading.current_thread()
+ worker_started.set()
+ release_worker.wait(timeout=2)
+ worker_finished.set()
+
+ async def fake_server_loop(self):
+ asyncio.get_running_loop().run_in_executor(None, blocking_worker)
+ server_started.set()
+ try:
+ await asyncio.Future()
+ finally:
+ raise RuntimeError("transport cleanup failed")
+
+ monkeypatch.setattr(MCPToolCallSession, "_mcp_server_loop", fake_server_loop)
+
+ session = MCPToolCallSession(SimpleNamespace(id="server-1"))
+
+ try:
+ assert server_started.wait(timeout=1)
+ assert worker_started.wait(timeout=1)
+ session.close_sync(timeout=0.01)
+
+ assert session._thread.is_alive()
+ assert worker_holder["thread"].is_alive()
+ assert not session._shutdown_complete.is_set()
+ assert session in MCPToolCallSession._ALL_INSTANCES
+
+ release_worker.set()
+ with pytest.raises(RuntimeError, match="transport cleanup failed"):
+ session.close_sync(timeout=1)
+
+ assert worker_finished.is_set()
+ assert not worker_holder["thread"].is_alive()
+ assert not session._thread.is_alive()
+ assert session._event_loop.is_closed()
+ assert session not in MCPToolCallSession._ALL_INSTANCES
+ finally:
+ release_worker.set()
+ if session._thread.is_alive():
+ try:
+ session.close_sync(timeout=1)
+ except RuntimeError as error:
+ assert str(error) == "transport cleanup failed"
+
+
+def test_close_multiple_deduplicates_sessions_and_waits_for_cleanup(monkeypatch):
+ started = {"server-1": threading.Event(), "server-2": threading.Event()}
+ finalized = {"server-1": threading.Event(), "server-2": threading.Event()}
+ finalizer_counts = {"server-1": 0, "server-2": 0}
+
+ async def fake_server_loop(self):
+ server_id = self._mcp_server.id
+ started[server_id].set()
+ try:
+ await asyncio.Future()
+ finally:
+ finalizer_counts[server_id] += 1
+ finalized[server_id].set()
+
+ monkeypatch.setattr(MCPToolCallSession, "_mcp_server_loop", fake_server_loop)
+
+ first = MCPToolCallSession(SimpleNamespace(id="server-1"))
+ second = MCPToolCallSession(SimpleNamespace(id="server-2"))
+
+ try:
+ assert all(event.wait(timeout=1) for event in started.values())
+ mcp_tool_call_conn.close_multiple_mcp_toolcall_sessions([first, None, first, second])
+
+ assert all(event.is_set() for event in finalized.values())
+ assert finalizer_counts == {"server-1": 1, "server-2": 1}
+ assert all(session._event_loop.is_closed() for session in (first, second))
+ assert all(session not in MCPToolCallSession._ALL_INSTANCES for session in (first, second))
+ finally:
+ first.close_sync(timeout=1)
+ second.close_sync(timeout=1)
+
+
+def test_close_multiple_rejects_session_event_loop_thread(monkeypatch):
+ server_started = threading.Event()
+
+ async def fake_server_loop(self):
+ server_started.set()
+ await asyncio.Future()
+
+ async def close_from_owner_loop(session):
+ mcp_tool_call_conn.close_multiple_mcp_toolcall_sessions([session])
+
+ monkeypatch.setattr(MCPToolCallSession, "_mcp_server_loop", fake_server_loop)
+
+ session = MCPToolCallSession(SimpleNamespace(id="server-1"))
+
+ try:
+ assert server_started.wait(timeout=1)
+ future = asyncio.run_coroutine_threadsafe(close_from_owner_loop(session), session._event_loop)
+
+ with pytest.raises(RuntimeError, match="outside their event loop threads"):
+ future.result(timeout=1)
+ assert not session._close
+ finally:
+ session.close_sync(timeout=1)
+
+
+def test_constructor_preserves_error_when_thread_start_fails_after_start(monkeypatch):
+ class StartupInterrupted(BaseException):
+ pass
+
+ interruption = StartupInterrupted("startup interrupted")
+ loop_started = threading.Event()
+ loop_holder = {}
+ thread_holder = {}
+ session_holder = {}
+ instances_before = set(MCPToolCallSession._active_instances())
+ real_new_event_loop = asyncio.new_event_loop
+ real_thread_start = threading.Thread.start
+
+ def observed_new_event_loop():
+ event_loop = real_new_event_loop()
+ loop_holder["event_loop"] = event_loop
+ return event_loop
+
+ def start_then_fail(thread):
+ thread_holder["thread"] = thread
+ session_holder["session"] = thread._target.__self__
+ real_thread_start(thread)
+ loop_holder["event_loop"].call_soon_threadsafe(loop_started.set)
+ assert loop_started.wait(timeout=1)
+ raise interruption
+
+ monkeypatch.setattr(mcp_tool_call_conn.asyncio, "new_event_loop", observed_new_event_loop)
+ monkeypatch.setattr(threading.Thread, "start", start_then_fail)
+
+ with pytest.raises(StartupInterrupted) as raised:
+ MCPToolCallSession(SimpleNamespace(id="startup-failure"))
+
+ session = session_holder["session"]
+ assert raised.value is interruption
+ assert not thread_holder["thread"].is_alive()
+ assert loop_holder["event_loop"].is_closed()
+ assert session._shutdown_complete.is_set()
+ assert session not in MCPToolCallSession._ALL_INSTANCES
+ assert set(MCPToolCallSession._active_instances()) == instances_before
+
+
+def test_constructor_shutdowns_default_executor_when_registration_fails(monkeypatch):
+ class StartupInterrupted(BaseException):
+ pass
+
+ interruption = StartupInterrupted("startup interrupted")
+ worker_started = threading.Event()
+ release_worker = threading.Event()
+ worker_finished = threading.Event()
+ session_holder = {}
+ worker_holder = {}
+ instances_before = set(MCPToolCallSession._active_instances())
+ real_registry_add = MCPToolCallSession._ALL_INSTANCES.add
+
+ def blocking_worker():
+ worker_holder["thread"] = threading.current_thread()
+ worker_started.set()
+ release_worker.wait(timeout=5)
+ worker_finished.set()
+
+ async def fake_server_loop(self):
+ session_holder["session"] = self
+ await asyncio.get_running_loop().run_in_executor(None, blocking_worker)
+ await asyncio.Future()
+
+ def fail_registration(session):
+ assert worker_started.wait(timeout=1)
+ raise interruption
+
+ monkeypatch.setattr(MCPToolCallSession, "_mcp_server_loop", fake_server_loop)
+ monkeypatch.setattr(MCPToolCallSession._ALL_INSTANCES, "add", fail_registration)
+
+ pool = ThreadPoolExecutor(max_workers=1)
+ constructor = pool.submit(MCPToolCallSession, SimpleNamespace(id="startup-failure"))
+
+ try:
+ assert worker_started.wait(timeout=1)
+ with pytest.raises(FuturesTimeoutError):
+ constructor.result(timeout=0.05)
+
+ release_worker.set()
+ with pytest.raises(StartupInterrupted) as raised:
+ constructor.result(timeout=2)
+
+ session = session_holder["session"]
+ assert raised.value is interruption
+ assert worker_finished.is_set()
+ assert not worker_holder["thread"].is_alive()
+ assert not session._thread.is_alive()
+ assert session._event_loop.is_closed()
+ assert session._shutdown_complete.is_set()
+ assert session not in MCPToolCallSession._ALL_INSTANCES
+ assert set(MCPToolCallSession._active_instances()) == instances_before
+ finally:
+ release_worker.set()
+ try:
+ constructor.result(timeout=2)
+ except StartupInterrupted:
+ pass
+ finally:
+ pool.shutdown(wait=True)
+ monkeypatch.setattr(MCPToolCallSession._ALL_INSTANCES, "add", real_registry_add)
+
+
+def test_shutdown_all_waits_for_session_registration(monkeypatch):
+ server_started = threading.Event()
+ submission_started = threading.Event()
+ release_submission = threading.Event()
+ shutdown_started = threading.Event()
+
+ async def fake_server_loop(self):
+ server_started.set()
+ await asyncio.Future()
+
+ real_run_coroutine_threadsafe = asyncio.run_coroutine_threadsafe
+
+ def blocking_submission(coroutine, event_loop):
+ future = real_run_coroutine_threadsafe(coroutine, event_loop)
+ submission_started.set()
+ release_submission.wait(timeout=5)
+ return future
+
+ real_begin_global_shutdown = MCPToolCallSession._begin_global_shutdown.__func__
+
+ def observed_begin_global_shutdown(cls):
+ shutdown_started.set()
+ return real_begin_global_shutdown(cls)
+
+ monkeypatch.setattr(MCPToolCallSession, "_mcp_server_loop", fake_server_loop)
+ monkeypatch.setattr(mcp_tool_call_conn.asyncio, "run_coroutine_threadsafe", blocking_submission)
+ monkeypatch.setattr(MCPToolCallSession, "_begin_global_shutdown", classmethod(observed_begin_global_shutdown))
+
+ pool = ThreadPoolExecutor(max_workers=2)
+ constructor = pool.submit(MCPToolCallSession, SimpleNamespace(id="server-1"))
+ shutdown = None
+ session = None
+
+ try:
+ assert submission_started.wait(timeout=1)
+ shutdown = pool.submit(mcp_tool_call_conn.shutdown_all_mcp_sessions)
+ assert shutdown_started.wait(timeout=1)
+ with pytest.raises(FuturesTimeoutError):
+ shutdown.result(timeout=0.05)
+
+ release_submission.set()
+ session = constructor.result(timeout=2)
+ shutdown.result(timeout=2)
+
+ assert server_started.is_set()
+ assert session._shutdown_complete.is_set()
+ assert not session._thread.is_alive()
+ assert session._event_loop.is_closed()
+ assert session not in MCPToolCallSession._ALL_INSTANCES
+ assert MCPToolCallSession._GLOBAL_SHUTDOWN_COUNT == 0
+ finally:
+ release_submission.set()
+ try:
+ if session is None:
+ session = constructor.result(timeout=2)
+ if shutdown is not None:
+ shutdown.result(timeout=2)
+ finally:
+ try:
+ if session is not None and session._thread.is_alive():
+ session.close_sync(timeout=1)
+ finally:
+ pool.shutdown(wait=True)
+
+
+def test_session_creation_is_rejected_during_global_shutdown(monkeypatch):
+ shutdown_started = threading.Event()
+ release_shutdown = threading.Event()
+ active_instances = iter([[SimpleNamespace()], []])
+
+ def blocking_close_multiple(sessions):
+ shutdown_started.set()
+ assert release_shutdown.wait(timeout=5)
+ return 0
+
+ monkeypatch.setattr(MCPToolCallSession, "_active_instances", classmethod(lambda cls: next(active_instances)))
+ monkeypatch.setattr(mcp_tool_call_conn, "close_multiple_mcp_toolcall_sessions", blocking_close_multiple)
+
+ pool = ThreadPoolExecutor(max_workers=1)
+ shutdown = pool.submit(mcp_tool_call_conn.shutdown_all_mcp_sessions)
+
+ try:
+ assert shutdown_started.wait(timeout=1)
+ with pytest.raises(RuntimeError, match="Cannot create MCP session during global shutdown"):
+ MCPToolCallSession(SimpleNamespace(id="rejected-server"))
+ finally:
+ release_shutdown.set()
+ try:
+ shutdown.result(timeout=2)
+ finally:
+ pool.shutdown(wait=True)
+
+ assert MCPToolCallSession._GLOBAL_SHUTDOWN_COUNT == 0
+
+
+def test_shutdown_all_reports_sessions_still_shutting_down(monkeypatch, caplog):
+ session = SimpleNamespace()
+
+ monkeypatch.setattr(MCPToolCallSession, "_active_instances", classmethod(lambda cls: [session]))
+ monkeypatch.setattr(mcp_tool_call_conn, "close_multiple_mcp_toolcall_sessions", lambda sessions: 0)
+
+ with caplog.at_level("INFO"):
+ mcp_tool_call_conn.shutdown_all_mcp_sessions()
+
+ assert "1 MCPToolCallSession instances are still shutting down." in caplog.messages
+ assert "All MCPToolCallSession instances have been closed." not in caplog.messages
+
+
+def test_close_multiple_reports_transport_cleanup_error_once(caplog):
+ class FailingSession:
+ _thread = object()
+ _mcp_server = SimpleNamespace(id="server-1")
+ _shutdown_complete = threading.Event()
+
+ def close_sync(self):
+ self._shutdown_complete.set()
+ raise RuntimeError("transport cleanup failed")
+
+ with caplog.at_level("INFO"):
+ cleanup_errors = mcp_tool_call_conn.close_multiple_mcp_toolcall_sessions([FailingSession()])
+
+ assert cleanup_errors == 1
+ assert caplog.messages.count("Exception while closing MCP session for server server-1") == 1
+ assert any("1 MCP sessions stopped; transport cleanup failure count: 1" in message for message in caplog.messages)
+ assert not any("MCP sessions have been cleaned up" in message for message in caplog.messages)
+
+
+def test_shutdown_all_reports_transport_cleanup_errors(monkeypatch, caplog):
+ active_instances = iter([[SimpleNamespace()], []])
+
+ monkeypatch.setattr(MCPToolCallSession, "_active_instances", classmethod(lambda cls: next(active_instances)))
+ monkeypatch.setattr(mcp_tool_call_conn, "close_multiple_mcp_toolcall_sessions", lambda sessions: 1)
+
+ with caplog.at_level("INFO"):
+ mcp_tool_call_conn.shutdown_all_mcp_sessions()
+
+ assert "All MCPToolCallSession event loops stopped; transport cleanup failure count: 1." in caplog.messages
+ assert "All MCPToolCallSession instances have been closed." not in caplog.messages
+
+
@pytest.mark.parametrize("task_type", ["tool_call", "list_tools"])
@pytest.mark.asyncio
async def test_in_flight_timeout_cancels_client_request_and_allows_next_call(task_type):
@@ -137,6 +1341,7 @@ def timed_out_session(monkeypatch):
session._close = False
session._event_loop = object()
session._mcp_server = SimpleNamespace(id="server-1")
+ session._shutdown_lock = threading.Lock()
return session, timed_out_future
@@ -168,6 +1373,8 @@ def test_public_tool_call_uses_one_absolute_deadline(monkeypatch):
session = object.__new__(MCPToolCallSession)
session._queue = asyncio.Queue()
session._close = False
+ session._pending_calls = {}
+ session._shutdown_lock = threading.Lock()
session._event_loop = loop
session._mcp_server = SimpleNamespace(id="server-1")