RAGFlow is an agent-native database: a Retrieval-Augmented Generation (RAG) engine that grew into context infrastructure.
Go agent-harnessagent-native-databaseagentic-aiagentic-retrievalagentic-searchaiai-agentscontext-enginecontext-engineeringcontext-managementharness-engineeringknowledge-compilationragretrieval-augmented-generationretrieval-harnesssearch-harness
fix(agent): enforce MCP task deadlines in worker (#18869)
Lem0nTea20021bf6fac75b31
diff --git a/common/mcp_tool_call_conn.py b/common/mcp_tool_call_conn.pyindex 7cf54ed6c4..b73b1ce671 100644--- a/common/mcp_tool_call_conn.py+++ b/common/mcp_tool_call_conn.py@@ -17,6 +17,7 @@import asyncioimport loggingimport threading+import timeimport weakreffrom concurrent.futures import ThreadPoolExecutorfrom concurrent.futures import TimeoutError as FuturesTimeoutError@@ -32,8 +33,10 @@ from mcp.client.sse import sse_clientfrom mcp.client.streamable_http import streamablehttp_clientfrom mcp.types import CallToolResult, ListToolsResult, TextContent, Tool+logger = logging.getLogger(__name__)+MCPTaskType = Literal["list_tools", "tool_call"]-MCPTask = tuple[MCPTaskType, dict[str, Any], asyncio.Queue[Any]]+MCPTask = tuple[MCPTaskType, dict[str, Any], asyncio.Queue[Any], float, asyncio.Event]class ToolCallSession(Protocol):@@ -55,7 +58,7 @@ class MCPToolCallSession(ToolCallSession):self._custom_header = custom_headerself._mcp_server = mcp_serverself._server_variables = server_variables or {}- self._queue = asyncio.Queue()+ self._queue: asyncio.Queue[MCPTask] = asyncio.Queue()self._close = Falseself._event_loop = asyncio.new_event_loop()@@ -128,7 +131,7 @@ class MCPToolCallSession(ToolCallSession):async def _process_mcp_tasks(self, client_session: ClientSession | None, error_message: str | None = None) -> None:while not self._close:try:- mcp_task, arguments, result_queue = await asyncio.wait_for(self._queue.get(), timeout=1)+ mcp_task, arguments, result_queue, deadline, abandoned = await asyncio.wait_for(self._queue.get(), timeout=1)except asyncio.TimeoutError:continueexcept asyncio.CancelledError:@@ -136,6 +139,11 @@ class MCPToolCallSession(ToolCallSession):logging.debug(f"Got MCP task {mcp_task} arguments {arguments}")+ remaining = deadline - time.monotonic()+ if abandoned.is_set() or remaining <= 0:+ logger.debug(f"Skipping expired MCP task {mcp_task}")+ continue+r: Any = Noneif not client_session or error_message:@@ -148,11 +156,13 @@ class MCPToolCallSession(ToolCallSession):try:if mcp_task == "list_tools":- r = await client_session.list_tools()+ r = await asyncio.wait_for(client_session.list_tools(), timeout=remaining)elif mcp_task == "tool_call":- r = await client_session.call_tool(**arguments)+ r = await asyncio.wait_for(client_session.call_tool(**arguments), timeout=remaining)else:r = ValueError(f"Unknown MCP task {mcp_task}")+ except TimeoutError:+ r = TimeoutError(f"MCP task '{mcp_task}' timeout")except Exception as e:r = eexcept asyncio.CancelledError:@@ -163,15 +173,17 @@ class MCPToolCallSession(ToolCallSession):except asyncio.CancelledError:break- async def _call_mcp_server(self, task_type: MCPTaskType, request_timeout: float | int = 8, **kwargs) -> Any:+ async def _call_mcp_server(self, task_type: MCPTaskType, request_timeout: float | int = 8, deadline: float | None = None, **kwargs) -> Any:if self._close:raise ValueError("Session is closed")results = asyncio.Queue()- await self._queue.put((task_type, kwargs, results))+ deadline = deadline if deadline is not None else time.monotonic() + request_timeout+ abandoned = asyncio.Event()+ await self._queue.put((task_type, kwargs, results, deadline, abandoned))try:- result: CallToolResult | Exception = await asyncio.wait_for(results.get(), timeout=request_timeout)+ result: CallToolResult | Exception = await asyncio.wait_for(results.get(), timeout=deadline - time.monotonic())if isinstance(result, Exception):raise resultreturn result@@ -179,9 +191,11 @@ class MCPToolCallSession(ToolCallSession):raise asyncio.TimeoutError(f"MCP task '{task_type}' timeout after {request_timeout}s")except Exception:raise+ finally:+ abandoned.set()- async def _call_mcp_tool(self, name: str, arguments: dict[str, Any], request_timeout: float | int = 10) -> str:- result: CallToolResult = await self._call_mcp_server("tool_call", name=name, arguments=arguments, request_timeout=request_timeout)+ async def _call_mcp_tool(self, name: str, arguments: dict[str, Any], request_timeout: float | int = 10, deadline: float | None = None) -> str:+ result: CallToolResult = await self._call_mcp_server("tool_call", name=name, arguments=arguments, request_timeout=request_timeout, deadline=deadline)if result.isError:return f"MCP server error: {result.content}"@@ -194,9 +208,9 @@ class MCPToolCallSession(ToolCallSession):else:return f"Unsupported content type {type(result.content)}"- async def _get_tools_from_mcp_server(self, request_timeout: float | int = 8) -> list[Tool]:+ async def _get_tools_from_mcp_server(self, request_timeout: float | int = 8, deadline: float | None = None) -> list[Tool]:try:- result: ListToolsResult = await self._call_mcp_server("list_tools", request_timeout=request_timeout)+ result: ListToolsResult = await self._call_mcp_server("list_tools", request_timeout=request_timeout, deadline=deadline)return result.toolsexcept Exception:raise@@ -205,10 +219,12 @@ class MCPToolCallSession(ToolCallSession):if self._close:raise ValueError("Session is closed")- future = asyncio.run_coroutine_threadsafe(self._get_tools_from_mcp_server(request_timeout=timeout), self._event_loop)+ deadline = time.monotonic() + timeout+ future = asyncio.run_coroutine_threadsafe(self._get_tools_from_mcp_server(request_timeout=timeout, deadline=deadline), self._event_loop)try:- return future.result(timeout=timeout)+ return future.result(timeout=max(0, deadline - time.monotonic()))except FuturesTimeoutError:+ future.cancel()msg = f"Timeout when fetching tools from MCP server: {self._mcp_server.id} (timeout={timeout})"logging.error(msg)raise RuntimeError(msg)@@ -221,13 +237,15 @@ class MCPToolCallSession(ToolCallSession):if self._close:return "Error: Session is closed"+ deadline = time.monotonic() + timeoutfuture = asyncio.run_coroutine_threadsafe(- self._call_mcp_tool(name, arguments, request_timeout=timeout),+ self._call_mcp_tool(name, arguments, request_timeout=timeout, deadline=deadline),self._event_loop,)try:- return future.result(timeout=timeout)+ return future.result(timeout=max(0, deadline - time.monotonic()))except FuturesTimeoutError:+ future.cancel()logging.error(f"Timeout calling tool '{name}' on MCP server: {self._mcp_server.id} (timeout={timeout})")return f"Timeout calling tool '{name}' (timeout={timeout})."except Exception as e:@@ -242,7 +260,7 @@ class MCPToolCallSession(ToolCallSession):while not self._queue.empty():try:- _, _, result_queue = self._queue.get_nowait()+ _, _, result_queue, _, _ = self._queue.get_nowait()try:await result_queue.put(asyncio.CancelledError("Session is closing"))except Exception:diff --git a/test/unit_test/common/test_mcp_tool_call_conn.py b/test/unit_test/common/test_mcp_tool_call_conn.pynew file mode 100644index 0000000000..9cf56bcd69--- /dev/null+++ b/test/unit_test/common/test_mcp_tool_call_conn.py@@ -0,0 +1,244 @@+import asyncio+import threading+import time+from concurrent.futures import ThreadPoolExecutor+from types import SimpleNamespace++import pytest++from common import mcp_tool_call_conn+from common.mcp_tool_call_conn import MCPToolCallSession+++def _make_session() -> MCPToolCallSession:+ session = object.__new__(MCPToolCallSession)+ session._queue = asyncio.Queue()+ session._close = False+ return session+++async def _call_tool(session: MCPToolCallSession, name: str, budget: float):+ return await session._call_mcp_server(+ "tool_call",+ request_timeout=budget,+ name=name,+ arguments={},+ )+++async def _stop_tasks(session: MCPToolCallSession, *tasks: asyncio.Task) -> None:+ session._close = True+ for task in tasks:+ if not task.done():+ task.cancel()+ await asyncio.gather(*tasks, return_exceptions=True)+ assert all(task.done() for task in tasks)++[email protected]("task_type", ["tool_call", "list_tools"])+async def test_in_flight_timeout_cancels_client_request_and_allows_next_call(task_type):+ class ClientSession:+ def __init__(self):+ self.started = asyncio.Event()+ self.cancelled = asyncio.Event()++ async def _slow_request(self):+ self.started.set()+ try:+ await asyncio.Future()+ except asyncio.CancelledError:+ self.cancelled.set()+ raise++ async def call_tool(self, name, arguments):+ if name == "slow":+ return await self._slow_request()+ return f"{name}-result"++ async def list_tools(self):+ return await self._slow_request()++ session = _make_session()+ client_session = ClientSession()+ processor = asyncio.create_task(session._process_mcp_tasks(client_session))++ try:+ kwargs = {"name": "slow", "arguments": {}} if task_type == "tool_call" else {}+ call = asyncio.create_task(session._call_mcp_server(task_type, request_timeout=0.05, **kwargs))+ await asyncio.wait_for(client_session.started.wait(), timeout=1)++ with pytest.raises(asyncio.TimeoutError, match=f"MCP task '{task_type}' timeout"):+ await call++ await asyncio.wait_for(client_session.cancelled.wait(), timeout=1)+ result = await _call_tool(session, "next", budget=1)++ assert result == "next-result"+ finally:+ await _stop_tasks(session, call, processor)+++async def test_queued_request_expiring_before_dispatch_is_never_called():+ class ClientSession:+ def __init__(self):+ self.calls = []+ self.first_started = asyncio.Event()+ self.release_first = asyncio.Event()++ async def call_tool(self, name, arguments):+ self.calls.append(name)+ if name == "first":+ self.first_started.set()+ await self.release_first.wait()+ return f"{name}-result"++ session = _make_session()+ client_session = ClientSession()+ processor = asyncio.create_task(session._process_mcp_tasks(client_session))++ try:+ first_call = asyncio.create_task(_call_tool(session, "first", budget=1))+ await asyncio.wait_for(client_session.first_started.wait(), timeout=1)++ with pytest.raises(asyncio.TimeoutError, match="MCP task 'tool_call' timeout"):+ await _call_tool(session, "expired", budget=0.05)++ client_session.release_first.set()+ assert await first_call == "first-result"+ assert await _call_tool(session, "last", budget=1) == "last-result"+ assert client_session.calls == ["first", "last"]+ finally:+ await _stop_tasks(session, first_call, processor)+++def timed_out_session(monkeypatch):+ class TimedOutFuture:+ def __init__(self):+ self.was_cancelled = False++ def result(self, timeout):+ raise mcp_tool_call_conn.FuturesTimeoutError++ def cancel(self):+ self.was_cancelled = True++ timed_out_future = TimedOutFuture()++ def run_coroutine_threadsafe(coroutine, event_loop):+ coroutine.close()+ return timed_out_future++ monkeypatch.setattr(mcp_tool_call_conn.asyncio, "run_coroutine_threadsafe", run_coroutine_threadsafe)++ session = object.__new__(MCPToolCallSession)+ session._close = False+ session._event_loop = object()+ session._mcp_server = SimpleNamespace(id="server-1")+ return session, timed_out_future+++def test_tool_call_timeout_preserves_message_and_cancels_coroutine(timed_out_session):+ session, timed_out_future = timed_out_session++ assert session.tool_call("send_email", {}, timeout=3) == "Timeout calling tool 'send_email' (timeout=3)."+ assert timed_out_future.was_cancelled+++def test_get_tools_timeout_preserves_error_and_cancels_coroutine(timed_out_session):+ session, timed_out_future = timed_out_session++ with pytest.raises(RuntimeError, match="Timeout when fetching tools from MCP server: server-1"):+ session.get_tools(timeout=3)+ assert timed_out_future.was_cancelled+++def test_public_tool_call_uses_one_absolute_deadline(monkeypatch):+ timeout = 0.6+ loop_delay = 0.3+ tool_delay = 0.45++ loop = asyncio.new_event_loop()+ loop_thread = threading.Thread(target=loop.run_forever, daemon=True)+ loop_thread.start()+ real_run_coroutine_threadsafe = asyncio.run_coroutine_threadsafe++ session = object.__new__(MCPToolCallSession)+ session._queue = asyncio.Queue()+ session._close = False+ session._event_loop = loop+ session._mcp_server = SimpleNamespace(id="server-1")++ tool_started = threading.Event()+ tool_cancelled = threading.Event()+ side_effect = threading.Event()++ class ClientSession:+ async def call_tool(self, name, arguments):+ tool_started.set()+ try:+ await asyncio.sleep(tool_delay)+ side_effect.set()+ return SimpleNamespace(isError=False, content=[])+ except asyncio.CancelledError:+ tool_cancelled.set()+ raise++ processor = real_run_coroutine_threadsafe(session._process_mcp_tasks(ClientSession()), loop)+ loop_blocked = threading.Event()+ release_loop = threading.Event()++ def block_event_loop():+ loop_blocked.set()+ release_loop.wait(timeout=2)++ loop.call_soon_threadsafe(block_event_loop)+ assert loop_blocked.wait(timeout=1)++ public_call_scheduled = threading.Event()++ def observed_run_coroutine_threadsafe(coroutine, target_loop):+ future = real_run_coroutine_threadsafe(coroutine, target_loop)+ public_call_scheduled.set()+ return future++ monkeypatch.setattr(mcp_tool_call_conn.asyncio, "run_coroutine_threadsafe", observed_run_coroutine_threadsafe)++ caller_pool = ThreadPoolExecutor(max_workers=1)+ caller = caller_pool.submit(session.tool_call, "side_effect", {}, timeout)+ result = None+ cancelled_seen = False++ try:+ assert public_call_scheduled.wait(timeout=1)+ time.sleep(loop_delay)+ release_loop.set()++ result = caller.result(timeout=2)+ cancelled_seen = tool_cancelled.wait(timeout=1)+ finally:+ release_loop.set()+ caller_pool.shutdown(wait=True)++ async def cancel_pending_tasks():+ session._close = True+ current = asyncio.current_task()+ tasks = [task for task in asyncio.all_tasks() if task is not current]+ for task in tasks:+ task.cancel()+ await asyncio.gather(*tasks, return_exceptions=True)++ cleanup = real_run_coroutine_threadsafe(cancel_pending_tasks(), loop)+ cleanup.result(timeout=2)+ loop.call_soon_threadsafe(loop.stop)+ loop_thread.join(timeout=2)+ assert not loop_thread.is_alive()+ loop.close()++ assert result == "Timeout calling tool 'side_effect' (timeout=0.6)."+ assert tool_started.is_set()+ assert cancelled_seen+ assert not side_effect.is_set()+ assert processor.done()