Skip to content

RAGFlow is an agent-native database: a Retrieval-Augmented Generation (RAG) engine that grew into context infrastructure.

fix(agent): enforce MCP task deadlines in worker (#18869)

Lem0nTea20021bf6fac75b31
diff --git a/common/mcp_tool_call_conn.py b/common/mcp_tool_call_conn.py
index 7cf54ed6c4..b73b1ce671 100644
--- a/common/mcp_tool_call_conn.py
+++ b/common/mcp_tool_call_conn.py
@@ -17,6 +17,7 @@
import asyncio
import logging
import threading
+import time
import weakref
from concurrent.futures import ThreadPoolExecutor
from concurrent.futures import TimeoutError as FuturesTimeoutError
@@ -32,8 +33,10 @@ from mcp.client.sse import sse_client
from mcp.client.streamable_http import streamablehttp_client
from mcp.types import CallToolResult, ListToolsResult, TextContent, Tool
+logger = logging.getLogger(__name__)
+
MCPTaskType = Literal["list_tools", "tool_call"]
-MCPTask = tuple[MCPTaskType, dict[str, Any], asyncio.Queue[Any]]
+MCPTask = tuple[MCPTaskType, dict[str, Any], asyncio.Queue[Any], float, asyncio.Event]
class ToolCallSession(Protocol):
@@ -55,7 +58,7 @@ class MCPToolCallSession(ToolCallSession):
self._custom_header = custom_header
self._mcp_server = mcp_server
self._server_variables = server_variables or {}
- self._queue = asyncio.Queue()
+ self._queue: asyncio.Queue[MCPTask] = asyncio.Queue()
self._close = False
self._event_loop = asyncio.new_event_loop()
@@ -128,7 +131,7 @@ class MCPToolCallSession(ToolCallSession):
async def _process_mcp_tasks(self, client_session: ClientSession | None, error_message: str | None = None) -> None:
while not self._close:
try:
- mcp_task, arguments, result_queue = await asyncio.wait_for(self._queue.get(), timeout=1)
+ mcp_task, arguments, result_queue, deadline, abandoned = await asyncio.wait_for(self._queue.get(), timeout=1)
except asyncio.TimeoutError:
continue
except asyncio.CancelledError:
@@ -136,6 +139,11 @@ class MCPToolCallSession(ToolCallSession):
logging.debug(f"Got MCP task {mcp_task} arguments {arguments}")
+ remaining = deadline - time.monotonic()
+ if abandoned.is_set() or remaining <= 0:
+ logger.debug(f"Skipping expired MCP task {mcp_task}")
+ continue
+
r: Any = None
if not client_session or error_message:
@@ -148,11 +156,13 @@ class MCPToolCallSession(ToolCallSession):
try:
if mcp_task == "list_tools":
- r = await client_session.list_tools()
+ r = await asyncio.wait_for(client_session.list_tools(), timeout=remaining)
elif mcp_task == "tool_call":
- r = await client_session.call_tool(**arguments)
+ r = await asyncio.wait_for(client_session.call_tool(**arguments), timeout=remaining)
else:
r = ValueError(f"Unknown MCP task {mcp_task}")
+ except TimeoutError:
+ r = TimeoutError(f"MCP task '{mcp_task}' timeout")
except Exception as e:
r = e
except asyncio.CancelledError:
@@ -163,15 +173,17 @@ class MCPToolCallSession(ToolCallSession):
except asyncio.CancelledError:
break
- async def _call_mcp_server(self, task_type: MCPTaskType, request_timeout: float | int = 8, **kwargs) -> Any:
+ async def _call_mcp_server(self, task_type: MCPTaskType, request_timeout: float | int = 8, deadline: float | None = None, **kwargs) -> Any:
if self._close:
raise ValueError("Session is closed")
results = asyncio.Queue()
- await self._queue.put((task_type, kwargs, results))
+ deadline = deadline if deadline is not None else time.monotonic() + request_timeout
+ abandoned = asyncio.Event()
+ await self._queue.put((task_type, kwargs, results, deadline, abandoned))
try:
- result: CallToolResult | Exception = await asyncio.wait_for(results.get(), timeout=request_timeout)
+ result: CallToolResult | Exception = await asyncio.wait_for(results.get(), timeout=deadline - time.monotonic())
if isinstance(result, Exception):
raise result
return result
@@ -179,9 +191,11 @@ class MCPToolCallSession(ToolCallSession):
raise asyncio.TimeoutError(f"MCP task '{task_type}' timeout after {request_timeout}s")
except Exception:
raise
+ finally:
+ abandoned.set()
- async def _call_mcp_tool(self, name: str, arguments: dict[str, Any], request_timeout: float | int = 10) -> str:
- result: CallToolResult = await self._call_mcp_server("tool_call", name=name, arguments=arguments, request_timeout=request_timeout)
+ async def _call_mcp_tool(self, name: str, arguments: dict[str, Any], request_timeout: float | int = 10, deadline: float | None = None) -> str:
+ result: CallToolResult = await self._call_mcp_server("tool_call", name=name, arguments=arguments, request_timeout=request_timeout, deadline=deadline)
if result.isError:
return f"MCP server error: {result.content}"
@@ -194,9 +208,9 @@ class MCPToolCallSession(ToolCallSession):
else:
return f"Unsupported content type {type(result.content)}"
- async def _get_tools_from_mcp_server(self, request_timeout: float | int = 8) -> list[Tool]:
+ async def _get_tools_from_mcp_server(self, request_timeout: float | int = 8, deadline: float | None = None) -> list[Tool]:
try:
- result: ListToolsResult = await self._call_mcp_server("list_tools", request_timeout=request_timeout)
+ result: ListToolsResult = await self._call_mcp_server("list_tools", request_timeout=request_timeout, deadline=deadline)
return result.tools
except Exception:
raise
@@ -205,10 +219,12 @@ class MCPToolCallSession(ToolCallSession):
if self._close:
raise ValueError("Session is closed")
- future = asyncio.run_coroutine_threadsafe(self._get_tools_from_mcp_server(request_timeout=timeout), self._event_loop)
+ deadline = time.monotonic() + timeout
+ future = asyncio.run_coroutine_threadsafe(self._get_tools_from_mcp_server(request_timeout=timeout, deadline=deadline), self._event_loop)
try:
- return future.result(timeout=timeout)
+ return future.result(timeout=max(0, deadline - time.monotonic()))
except FuturesTimeoutError:
+ future.cancel()
msg = f"Timeout when fetching tools from MCP server: {self._mcp_server.id} (timeout={timeout})"
logging.error(msg)
raise RuntimeError(msg)
@@ -221,13 +237,15 @@ class MCPToolCallSession(ToolCallSession):
if self._close:
return "Error: Session is closed"
+ deadline = time.monotonic() + timeout
future = asyncio.run_coroutine_threadsafe(
- self._call_mcp_tool(name, arguments, request_timeout=timeout),
+ self._call_mcp_tool(name, arguments, request_timeout=timeout, deadline=deadline),
self._event_loop,
)
try:
- return future.result(timeout=timeout)
+ return future.result(timeout=max(0, deadline - time.monotonic()))
except FuturesTimeoutError:
+ future.cancel()
logging.error(f"Timeout calling tool '{name}' on MCP server: {self._mcp_server.id} (timeout={timeout})")
return f"Timeout calling tool '{name}' (timeout={timeout})."
except Exception as e:
@@ -242,7 +260,7 @@ class MCPToolCallSession(ToolCallSession):
while not self._queue.empty():
try:
- _, _, result_queue = self._queue.get_nowait()
+ _, _, result_queue, _, _ = self._queue.get_nowait()
try:
await result_queue.put(asyncio.CancelledError("Session is closing"))
except Exception:
diff --git a/test/unit_test/common/test_mcp_tool_call_conn.py b/test/unit_test/common/test_mcp_tool_call_conn.py
new file mode 100644
index 0000000000..9cf56bcd69
--- /dev/null
+++ b/test/unit_test/common/test_mcp_tool_call_conn.py
@@ -0,0 +1,244 @@
+import asyncio
+import threading
+import time
+from concurrent.futures import ThreadPoolExecutor
+from types import SimpleNamespace
+
+import pytest
+
+from common import mcp_tool_call_conn
+from common.mcp_tool_call_conn import MCPToolCallSession
+
+
+def _make_session() -> MCPToolCallSession:
+ session = object.__new__(MCPToolCallSession)
+ session._queue = asyncio.Queue()
+ session._close = False
+ return session
+
+
+async def _call_tool(session: MCPToolCallSession, name: str, budget: float):
+ return await session._call_mcp_server(
+ "tool_call",
+ request_timeout=budget,
+ name=name,
+ arguments={},
+ )
+
+
+async def _stop_tasks(session: MCPToolCallSession, *tasks: asyncio.Task) -> None:
+ session._close = True
+ for task in tasks:
+ if not task.done():
+ task.cancel()
+ await asyncio.gather(*tasks, return_exceptions=True)
+ assert all(task.done() for task in tasks)
+
+
[email protected]("task_type", ["tool_call", "list_tools"])
+async def test_in_flight_timeout_cancels_client_request_and_allows_next_call(task_type):
+ class ClientSession:
+ def __init__(self):
+ self.started = asyncio.Event()
+ self.cancelled = asyncio.Event()
+
+ async def _slow_request(self):
+ self.started.set()
+ try:
+ await asyncio.Future()
+ except asyncio.CancelledError:
+ self.cancelled.set()
+ raise
+
+ async def call_tool(self, name, arguments):
+ if name == "slow":
+ return await self._slow_request()
+ return f"{name}-result"
+
+ async def list_tools(self):
+ return await self._slow_request()
+
+ session = _make_session()
+ client_session = ClientSession()
+ processor = asyncio.create_task(session._process_mcp_tasks(client_session))
+
+ try:
+ kwargs = {"name": "slow", "arguments": {}} if task_type == "tool_call" else {}
+ call = asyncio.create_task(session._call_mcp_server(task_type, request_timeout=0.05, **kwargs))
+ await asyncio.wait_for(client_session.started.wait(), timeout=1)
+
+ with pytest.raises(asyncio.TimeoutError, match=f"MCP task '{task_type}' timeout"):
+ await call
+
+ await asyncio.wait_for(client_session.cancelled.wait(), timeout=1)
+ result = await _call_tool(session, "next", budget=1)
+
+ assert result == "next-result"
+ finally:
+ await _stop_tasks(session, call, processor)
+
+
+async def test_queued_request_expiring_before_dispatch_is_never_called():
+ class ClientSession:
+ def __init__(self):
+ self.calls = []
+ self.first_started = asyncio.Event()
+ self.release_first = asyncio.Event()
+
+ async def call_tool(self, name, arguments):
+ self.calls.append(name)
+ if name == "first":
+ self.first_started.set()
+ await self.release_first.wait()
+ return f"{name}-result"
+
+ session = _make_session()
+ client_session = ClientSession()
+ processor = asyncio.create_task(session._process_mcp_tasks(client_session))
+
+ try:
+ first_call = asyncio.create_task(_call_tool(session, "first", budget=1))
+ await asyncio.wait_for(client_session.first_started.wait(), timeout=1)
+
+ with pytest.raises(asyncio.TimeoutError, match="MCP task 'tool_call' timeout"):
+ await _call_tool(session, "expired", budget=0.05)
+
+ client_session.release_first.set()
+ assert await first_call == "first-result"
+ assert await _call_tool(session, "last", budget=1) == "last-result"
+ assert client_session.calls == ["first", "last"]
+ finally:
+ await _stop_tasks(session, first_call, processor)
+
+
+def timed_out_session(monkeypatch):
+ class TimedOutFuture:
+ def __init__(self):
+ self.was_cancelled = False
+
+ def result(self, timeout):
+ raise mcp_tool_call_conn.FuturesTimeoutError
+
+ def cancel(self):
+ self.was_cancelled = True
+
+ timed_out_future = TimedOutFuture()
+
+ def run_coroutine_threadsafe(coroutine, event_loop):
+ coroutine.close()
+ return timed_out_future
+
+ monkeypatch.setattr(mcp_tool_call_conn.asyncio, "run_coroutine_threadsafe", run_coroutine_threadsafe)
+
+ session = object.__new__(MCPToolCallSession)
+ session._close = False
+ session._event_loop = object()
+ session._mcp_server = SimpleNamespace(id="server-1")
+ return session, timed_out_future
+
+
+def test_tool_call_timeout_preserves_message_and_cancels_coroutine(timed_out_session):
+ session, timed_out_future = timed_out_session
+
+ assert session.tool_call("send_email", {}, timeout=3) == "Timeout calling tool 'send_email' (timeout=3)."
+ assert timed_out_future.was_cancelled
+
+
+def test_get_tools_timeout_preserves_error_and_cancels_coroutine(timed_out_session):
+ session, timed_out_future = timed_out_session
+
+ with pytest.raises(RuntimeError, match="Timeout when fetching tools from MCP server: server-1"):
+ session.get_tools(timeout=3)
+ assert timed_out_future.was_cancelled
+
+
+def test_public_tool_call_uses_one_absolute_deadline(monkeypatch):
+ timeout = 0.6
+ loop_delay = 0.3
+ tool_delay = 0.45
+
+ loop = asyncio.new_event_loop()
+ loop_thread = threading.Thread(target=loop.run_forever, daemon=True)
+ loop_thread.start()
+ real_run_coroutine_threadsafe = asyncio.run_coroutine_threadsafe
+
+ session = object.__new__(MCPToolCallSession)
+ session._queue = asyncio.Queue()
+ session._close = False
+ session._event_loop = loop
+ session._mcp_server = SimpleNamespace(id="server-1")
+
+ tool_started = threading.Event()
+ tool_cancelled = threading.Event()
+ side_effect = threading.Event()
+
+ class ClientSession:
+ async def call_tool(self, name, arguments):
+ tool_started.set()
+ try:
+ await asyncio.sleep(tool_delay)
+ side_effect.set()
+ return SimpleNamespace(isError=False, content=[])
+ except asyncio.CancelledError:
+ tool_cancelled.set()
+ raise
+
+ processor = real_run_coroutine_threadsafe(session._process_mcp_tasks(ClientSession()), loop)
+ loop_blocked = threading.Event()
+ release_loop = threading.Event()
+
+ def block_event_loop():
+ loop_blocked.set()
+ release_loop.wait(timeout=2)
+
+ loop.call_soon_threadsafe(block_event_loop)
+ assert loop_blocked.wait(timeout=1)
+
+ public_call_scheduled = threading.Event()
+
+ def observed_run_coroutine_threadsafe(coroutine, target_loop):
+ future = real_run_coroutine_threadsafe(coroutine, target_loop)
+ public_call_scheduled.set()
+ return future
+
+ monkeypatch.setattr(mcp_tool_call_conn.asyncio, "run_coroutine_threadsafe", observed_run_coroutine_threadsafe)
+
+ caller_pool = ThreadPoolExecutor(max_workers=1)
+ caller = caller_pool.submit(session.tool_call, "side_effect", {}, timeout)
+ result = None
+ cancelled_seen = False
+
+ try:
+ assert public_call_scheduled.wait(timeout=1)
+ time.sleep(loop_delay)
+ release_loop.set()
+
+ result = caller.result(timeout=2)
+ cancelled_seen = tool_cancelled.wait(timeout=1)
+ finally:
+ release_loop.set()
+ caller_pool.shutdown(wait=True)
+
+ async def cancel_pending_tasks():
+ session._close = True
+ current = asyncio.current_task()
+ tasks = [task for task in asyncio.all_tasks() if task is not current]
+ for task in tasks:
+ task.cancel()
+ await asyncio.gather(*tasks, return_exceptions=True)
+
+ cleanup = real_run_coroutine_threadsafe(cancel_pending_tasks(), loop)
+ cleanup.result(timeout=2)
+ loop.call_soon_threadsafe(loop.stop)
+ loop_thread.join(timeout=2)
+ assert not loop_thread.is_alive()
+ loop.close()
+
+ assert result == "Timeout calling tool 'side_effect' (timeout=0.6)."
+ assert tool_started.is_set()
+ assert cancelled_seen
+ assert not side_effect.is_set()
+ assert processor.done()