Enterprise agent platform for building, deploying, and operating AI teammates.
fix(v1): cap MCP tool listings per key holder and bound the slot wait (#3792)
Alex Liu7aeab677b6fb
diff --git a/src/xagent/web/api/v1/errors.py b/src/xagent/web/api/v1/errors.pyindex be9eebff9..4f78a24da 100644--- a/src/xagent/web/api/v1/errors.py+++ b/src/xagent/web/api/v1/errors.py@@ -134,7 +134,8 @@ class V1ErrorCode(str, Enum):INVALID_INPUT = "invalid_input"# Emitted when a caller exceeds the v1 SSE stream concurrency caps- # (per-task or per-principal), or when no MCP tool-listing slot frees up.+ # (per-task or per-principal), or the MCP tool-listing caps (per key holder,+ # or no shared slot frees up in time).RATE_LIMITED = "rate_limited"# Monthly execution quota exhausted. 402 Payment Required. Two distinctdiff --git a/src/xagent/web/api/v1/mcp_servers.py b/src/xagent/web/api/v1/mcp_servers.pyindex a3b927e1d..5cc66fb4c 100644--- a/src/xagent/web/api/v1/mcp_servers.py+++ b/src/xagent/web/api/v1/mcp_servers.py@@ -582,15 +582,25 @@ async def set_mcp_server_enabled(# ===== listing a connector's tools =====_V1_TOOL_LISTING_MAX_INFLIGHT = 4+_V1_TOOL_LISTING_MAX_PER_HOLDER = 2+_TOOL_LISTING_SLOT_WAIT_SECONDS = 5_TOOL_LISTING_BUSY_MESSAGE = ("Too many connector tool listings are running; retry shortly.")+_TOOL_LISTING_HOLDER_BUSY_MESSAGE = (+ f"You already have {_V1_TOOL_LISTING_MAX_PER_HOLDER} connector tool listings "+ "in progress, including any whose request ended early; retry shortly."+)# Keyed by loop, weakly, like the web connection-test gate: a semaphore binds# to the loop it first waits on, and a finished loop must not pin it. This gate# is independent of the web one, so the two never queue behind each other._v1_tool_listing_gates: "weakref.WeakKeyDictionary[asyncio.AbstractEventLoop, asyncio.Semaphore]" = weakref.WeakKeyDictionary()+# Listings each key holder has in progress (waiting for a slot or running),+# per loop like the gate, so the two never disagree about which loop they count.+_v1_tool_listing_holder_counts: "weakref.WeakKeyDictionary[asyncio.AbstractEventLoop, dict[int, int]]" = weakref.WeakKeyDictionary()+# Listings that hold a slot. The done-callback gives the slot back; this set# keeps the task referenced until then._v1_tool_listing_loads: "set[asyncio.Task[Any]]" = set()@@ -605,6 +615,29 @@ def _v1_tool_listing_gate() -> asyncio.Semaphore:return gate+def _v1_tool_listing_holders() -> dict[int, int]:+ return _v1_tool_listing_holder_counts.setdefault(asyncio.get_running_loop(), {})+++def _release_holder(holders: dict[int, int], user_id: int) -> None:+ left = holders.get(user_id, 0) - 1+ if left > 0:+ holders[user_id] = left+ else:+ holders.pop(user_id, None)+++def _busy(message: str) -> V1ApiError:+ return V1ApiError(+ V1ErrorCode.RATE_LIMITED,+ 429,+ message,+ headers={+ "Retry-After": str(max(1, math.ceil(_TOOL_LISTING_SLOT_WAIT_SECONDS)))+ },+ )++async def _list_saved_connector_tools(server_id: int, user_id: int) -> ConnectorToolsResult:@@ -639,44 +672,63 @@ def _consume_abandoned_listing(task: "asyncio.Task[Any]") -> None:async def _run_tool_listing(server_id: int, user_id: int) -> ConnectorToolsResult:- """List a saved connector's tools under this module's own concurrency cap.+ """List a saved connector's tools under this module's own concurrency caps.* At most ``_V1_TOOL_LISTING_MAX_INFLIGHT`` listings run per event loop,shared by every key holder in the process.- * Waiting for a slot is bounded by the MCP initialization timeout; a- listing that cannot get one answers 429 ``rate_limited`` with a- ``Retry-After`` of that timeout in whole seconds.- * With a timeout above 0 a slot is given back when the listing returns,- never earlier: a cancelled request leaves the listing running. Starting- the connector is bounded by the initialization timeout; reading its+ * A key holder has at most ``_V1_TOOL_LISTING_MAX_PER_HOLDER`` listings in+ progress per event loop, counting those waiting for a slot and those+ whose request was cancelled while the listing still runs. A request+ beyond that answers 429 ``rate_limited`` at once.+ * Waiting for a slot takes at most ``_TOOL_LISTING_SLOT_WAIT_SECONDS``,+ whatever the initialization timeout; a request that gets none answers+ 429 ``rate_limited``. Both 429s carry a ``Retry-After`` of that wait in+ whole seconds, at least 1.+ * Once a slot is taken, both counts are given back when the listing ends,+ never earlier. With a timeout above 0 the listing runs in the+ background, so a cancelled request leaves it running; starting the+ connector is bounded by the initialization timeout, and reading itssettings before that, which can refresh an OAuth token, is not.- * A timeout of 0 means no deadline: wait for a slot as long as it takes- and await the listing directly. A client disconnect does not cancel the- request, so a listing that never returns keeps its slot until the+ * With a timeout of 0 the listing runs in the request and has no deadline;+ cancellation reaches it. A client disconnect does not cancel the+ request, so a listing that never returns keeps both counts until theprocess stops."""- gate = _v1_tool_listing_gate()+ # Read before anything is counted, so that once a slot is taken nothing+ # can raise before the finally or the done-callback below gives it back.timeout = get_mcp_tool_init_timeout_seconds()+ gate = _v1_tool_listing_gate()+ holders = _v1_tool_listing_holders()+ # Check and count with no await in between, so two requests on this loop+ # cannot both pass the check.+ if holders.get(user_id, 0) >= _V1_TOOL_LISTING_MAX_PER_HOLDER:+ raise _busy(_TOOL_LISTING_HOLDER_BUSY_MESSAGE)+ holders[user_id] = holders.get(user_id, 0) + 1+ try:+ await asyncio.wait_for(gate.acquire(), _TOOL_LISTING_SLOT_WAIT_SECONDS)+ except BaseException as exc:+ _release_holder(holders, user_id)+ if isinstance(exc, TimeoutError):+ raise _busy(_TOOL_LISTING_BUSY_MESSAGE) from None+ raise++ def _give_back() -> None:+ gate.release()+ _release_holder(holders, user_id)+if timeout <= 0:- async with gate:+ try:return await _list_saved_connector_tools(server_id, user_id)- try:- await asyncio.wait_for(gate.acquire(), timeout)- except TimeoutError:- raise V1ApiError(- V1ErrorCode.RATE_LIMITED,- 429,- _TOOL_LISTING_BUSY_MESSAGE,- headers={"Retry-After": str(max(1, math.ceil(timeout)))},- )+ finally:+ _give_back()load = asyncio.ensure_future(_list_saved_connector_tools(server_id, user_id))_v1_tool_listing_loads.add(load)- def _give_back(task: "asyncio.Task[Any]") -> None:+ def _done(task: "asyncio.Task[Any]") -> None:_v1_tool_listing_loads.discard(task)- gate.release()+ _give_back()- load.add_done_callback(_give_back)+ load.add_done_callback(_done)try:await asyncio.wait({load})except asyncio.CancelledError:@@ -776,15 +828,18 @@ async def list_mcp_server_tools() -> V1McpServerTools:"""Start one of the holder's remote connectors and list its tools.- The connector is started exactly as a task would start it. The request- first waits for a free slot (at most one MCP initialization timeout) and- then lists the tools. Starting the connector takes at most one more- initialization timeout, plus the time to read its settings. A connector- the holder has switched off, one shared by the team, or one whose- transport is not ``streamable_http`` or ``sse`` (catalog connectors- included) is refused before anything starts. Authorization is done in- the web app: an unauthorized OAuth connector answers 409- ``mcp_oauth_authorization_failed``.+ The connector is started exactly as a task would start it. Each server+ process lets an account have at most two listings in progress at once,+ across all of its keys and connectors and including ones whose request+ ended early; another request answers 429 ``rate_limited`` at once.+ Otherwise the request waits up to five seconds for a free slot, then+ answers 429 ``rate_limited``; both carry ``Retry-After``. Starting the+ connector takes at most one initialization timeout, plus the time to+ read its settings. A connector the holder has switched off, one shared+ by the team, or one whose transport is not ``streamable_http`` or+ ``sse`` (catalog connectors included) is refused before anything+ starts. Authorization is done in the web app: an unauthorized OAuth+ connector answers 409 ``mcp_oauth_authorization_failed``."""user, _key = autheduser_id = int(user.id)diff --git a/src/xagent/web/schemas/v1.py b/src/xagent/web/schemas/v1.pyindex 34a9fc95e..4fd6e40c2 100644--- a/src/xagent/web/schemas/v1.py+++ b/src/xagent/web/schemas/v1.py@@ -616,8 +616,8 @@ class V1McpServerTools(BaseModel):"""Response of ``GET /v1/mcp-servers/{server_id}/tools``.The connector is started the way a task would start it. Waiting for a- free slot takes at most one MCP initialization timeout; starting the- connector takes at most one more, plus the time to read its settings.+ free slot takes at most five seconds; starting the connector takes at most+ one MCP initialization timeout, plus the time to read its settings.``failures`` is non-empty when some loads failed but at least one toolcame back."""diff --git a/tests/web/api/v1/test_mcp_tool_listing.py b/tests/web/api/v1/test_mcp_tool_listing.pyindex 67d59c99c..9317b3e49 100644--- a/tests/web/api/v1/test_mcp_tool_listing.py+++ b/tests/web/api/v1/test_mcp_tool_listing.py@@ -25,7 +25,8 @@ from xagent.web.services.mcp_connector_session import (ConnectorToolsResult,)-from ..conftest import client+from ..conftest import _login, client+from .test_management import _bearerfrom .test_mcp_servers import (CONNECTOR_URL,Holder,@@ -527,6 +528,8 @@ def test_a_session_listing_that_finds_no_connector_is_not_found(bob, monkeypatchdef test_a_session_listing_that_raises_is_a_plain_500(bob, monkeypatch, caplog):server_id = _seed("records", owner=bob)+ holders = _recording_holders(monkeypatch)+ gates = _recording_gates(monkeypatch)async def raising(db, **kwargs):raise RuntimeError(SECRET)@@ -537,6 +540,8 @@ def test_a_session_listing_that_raises_is_a_plain_500(bob, monkeypatch, caplog):resp = _tools(bob, server_id)_error(resp, 500, "internal_error")+ assert holders == [{}]+ assert [g._value for g in gates] == [4]assert SECRET not in resp.textassert "RuntimeError" in caplog.textassert SECRET not in caplog.text@@ -581,6 +586,18 @@ def _use_timeout(monkeypatch, seconds: float) -> None:monkeypatch.setattr(routes, "get_mcp_tool_init_timeout_seconds", lambda: seconds)+def _recording_holders(monkeypatch) -> list[dict[int, int]]:+ real = routes._v1_tool_listing_holders+ seen: list[dict[int, int]] = []++ def holders():+ seen.append(real())+ return seen[-1]++ monkeypatch.setattr(routes, "_v1_tool_listing_holders", holders)+ return seen++def _recording_gates(monkeypatch) -> list[asyncio.Semaphore]:real = routes._v1_tool_listing_gateseen: list[asyncio.Semaphore] = []@@ -607,7 +624,9 @@ def _gate_cell_web_gate_full(bob, monkeypatch):def _gate_cell_own_gate_full(bob, monkeypatch):monkeypatch.setattr(routes, "_v1_tool_listing_gate", lambda: asyncio.Semaphore(0))+ monkeypatch.setattr(routes, "_TOOL_LISTING_SLOT_WAIT_SECONDS", 0.05)_use_timeout(monkeypatch, 1.2)+ holders = _recording_holders(monkeypatch)calls = _stub_loader(monkeypatch, _loaded(_tool()))server_id = _seed("records", owner=bob)@@ -615,21 +634,33 @@ def _gate_cell_own_gate_full(bob, monkeypatch):error = _error(resp, 429, "rate_limited")assert error["message"] == BUSY- assert resp.headers["Retry-After"] == "2"+ assert resp.headers["Retry-After"] == "1"+ assert holders == [{}]assert calls == []-def _gate_cell_normal_call(bob, monkeypatch):+def _normal_call(bob, monkeypatch, timeout: float) -> None:+ _use_timeout(monkeypatch, timeout)gates = _recording_gates(monkeypatch)+ holders = _recording_holders(monkeypatch)_stub_loader(monkeypatch, _loaded(_tool()))server_id = _seed("records", owner=bob)assert _tools(bob, server_id).status_code == 200assert [g._value for g in gates] == [4]+ assert holders == [{}]assert routes._v1_tool_listing_loads == set()+def _gate_cell_normal_call_with_a_deadline(bob, monkeypatch):+ _normal_call(bob, monkeypatch, 60)+++def _gate_cell_normal_call_without_a_deadline(bob, monkeypatch):+ _normal_call(bob, monkeypatch, 0)++def _gate_cell_session_listing_not_found(bob, monkeypatch):gates = _recording_gates(monkeypatch)server_id = _seed("records", owner=bob)@@ -662,19 +693,23 @@ async def _cancel_a_listing(monkeypatch, timeout: float) -> dict[str, Any]:request = asyncio.create_task(routes._run_tool_listing(1, 1))await asyncio.wait_for(entered.wait(), 5)gate = routes._v1_tool_listing_gate()+ holders = routes._v1_tool_listing_holders()state["value_while_running"] = gate._value+ state["holders_while_running"] = dict(holders)request.cancel()with pytest.raises(asyncio.CancelledError):await requestawait asyncio.sleep(0)state["gate"] = gatestate["value_after_cancel"] = gate._value+ state["holders_after_cancel"] = dict(holders)state["loads_after_cancel"] = set(routes._v1_tool_listing_loads)release.set()if state["loads_after_cancel"]:await asyncio.wait_for(asyncio.wait(state["loads_after_cancel"]), 5)await asyncio.sleep(0)state["value_at_end"] = gate._value+ state["holders_at_end"] = dict(holders)state["loads_at_end"] = set(routes._v1_tool_listing_loads)return state@@ -683,9 +718,11 @@ def _gate_cell_cancelled_with_a_deadline(bob, monkeypatch):state = asyncio.run(_cancel_a_listing(monkeypatch, 5))assert state["value_after_cancel"] == 3+ assert state["holders_after_cancel"] == {1: 1}assert state["load_cancelled"] is Falseassert len(state["loads_after_cancel"]) == 1assert state["value_at_end"] == 4+ assert state["holders_at_end"] == {}assert state["loads_at_end"] == set()@@ -693,8 +730,10 @@ def _gate_cell_cancelled_without_a_deadline(bob, monkeypatch):state = asyncio.run(_cancel_a_listing(monkeypatch, 0))assert state["value_while_running"] == 3+ assert state["holders_while_running"] == {1: 1}assert state["load_cancelled"] is Trueassert state["value_after_cancel"] == 4+ assert state["holders_after_cancel"] == {}assert state["loads_at_end"] == set()@@ -716,6 +755,7 @@ async def _abandon_a_failing_listing(monkeypatch) -> dict[str, Any]:request = asyncio.create_task(routes._run_tool_listing(1, 1))await asyncio.wait_for(entered.wait(), 5)gate = routes._v1_tool_listing_gate()+ holders = routes._v1_tool_listing_holders()request.cancel()with pytest.raises(asyncio.CancelledError):await request@@ -726,7 +766,11 @@ async def _abandon_a_failing_listing(monkeypatch) -> dict[str, Any]:del loads, requestgc.collect()await asyncio.sleep(0)- return {"unretrieved": unretrieved, "gate_value": gate._value}+ return {+ "unretrieved": unretrieved,+ "gate_value": gate._value,+ "holders": dict(holders),+ }def test_an_abandoned_listing_that_fails_has_its_error_retrieved(monkeypatch):@@ -734,6 +778,7 @@ def test_an_abandoned_listing_that_fails_has_its_error_retrieved(monkeypatch):assert state["unretrieved"] == []assert state["gate_value"] == 4+ assert state["holders"] == {}assert routes._v1_tool_listing_loads == set()@@ -746,6 +791,216 @@ def _gate_cell_one_gate_per_loop(bob, monkeypatch):assert first is not asyncio.run(gates())[0]+def _gate_cell_one_holder_map_per_loop(bob, monkeypatch):+ async def maps():+ return routes._v1_tool_listing_holders(), routes._v1_tool_listing_holders()++ first, again = asyncio.run(maps())+ assert first is again+ assert first is not asyncio.run(maps())[0]+++async def _until(condition) -> None:+ for _ in range(1000):+ if condition():+ return+ await asyncio.sleep(0)+ raise AssertionError("the condition never became true")+++async def _hang_listings(monkeypatch) -> asyncio.Event:+ """Make every listing wait for the returned event; timeout above 0."""+ _use_timeout(monkeypatch, 5)+ release = asyncio.Event()++ async def hanging(server_id, user_id):+ await release.wait()+ return _ok_listing()++ monkeypatch.setattr(routes, "_list_saved_connector_tools", hanging)+ return release+++def _assert_busy(error: BaseException, message: str) -> None:+ assert isinstance(error, routes.V1ApiError)+ assert (error.code, error.http_status) == (routes.V1ErrorCode.RATE_LIMITED, 429)+ assert error.message == message+ assert error.headers == {"Retry-After": "5"}+++def _gate_cell_no_slot_in_time_suggests_the_real_wait(bob, monkeypatch):+ class NoSlot:+ async def acquire(self):+ # What waiting for a slot raises once the wait has run out.+ raise TimeoutError++ monkeypatch.setattr(routes, "_v1_tool_listing_gate", NoSlot)++ async def run():+ holders = routes._v1_tool_listing_holders()+ with pytest.raises(routes.V1ApiError) as raised:+ await routes._run_tool_listing(1, 7)+ _assert_busy(raised.value, BUSY)+ assert holders == {}++ asyncio.run(run())+++def _gate_cell_holder_full(bob, monkeypatch):+ monkeypatch.setattr(routes, "_v1_tool_listing_holders", lambda: {bob.user_id: 2})+ calls = _stub_loader(monkeypatch, _loaded(_tool()))+ server_id = _seed("records", owner=bob)++ resp = _tools(bob, server_id)++ error = _error(resp, 429, "rate_limited")+ assert error["message"] == (+ "You already have 2 connector tool listings in progress, including "+ "any whose request ended early; retry shortly."+ )+ assert "details" not in error+ assert resp.headers["Retry-After"] == "5"+ assert calls == []+++def _gate_cell_two_listings_per_holder_whatever_the_connector(bob, monkeypatch):+ async def run():+ release = await _hang_listings(monkeypatch)+ holders = routes._v1_tool_listing_holders()+ # All three start before any of them yields, so a yield between the+ # holder check and the count would let the third one in too.+ requests = [+ asyncio.create_task(routes._run_tool_listing(i, 7)) for i in (1, 2, 3)+ ]+ await _until(requests[2].done)+ with pytest.raises(routes.V1ApiError) as raised:+ requests[2].result()+ _assert_busy(+ raised.value,+ "You already have 2 connector tool listings in progress, including "+ "any whose request ended early; retry shortly.",+ )+ assert holders == {7: 2}+ release.set()+ await asyncio.wait_for(asyncio.gather(*requests[:2]), 5)+ await asyncio.sleep(0)+ assert holders == {}++ asyncio.run(run())+++def _gate_cell_another_holder_is_not_affected(bob, monkeypatch):+ async def run():+ release = await _hang_listings(monkeypatch)+ holders = routes._v1_tool_listing_holders()+ gate = routes._v1_tool_listing_gate()+ mine = [asyncio.create_task(routes._run_tool_listing(i, 7)) for i in (1, 2)]+ await _until(lambda: holders == {7: 2})+ other = asyncio.create_task(routes._run_tool_listing(1, 8))+ await _until(lambda: holders == {7: 2, 8: 1})+ assert gate._value == 1+ release.set()+ await asyncio.wait_for(asyncio.gather(*mine, other), 5)+ await asyncio.sleep(0)+ assert holders == {}++ asyncio.run(run())+++def _gate_cell_a_cancelled_request_keeps_its_holder_count(bob, monkeypatch):+ async def run():+ release = await _hang_listings(monkeypatch)+ holders = routes._v1_tool_listing_holders()+ requests = [asyncio.create_task(routes._run_tool_listing(i, 7)) for i in (1, 2)]+ await _until(lambda: holders == {7: 2})+ for request in requests:+ request.cancel()+ for request in requests:+ with pytest.raises(asyncio.CancelledError):+ await request+ await asyncio.sleep(0)+ assert holders == {7: 2}+ with pytest.raises(routes.V1ApiError) as raised:+ await asyncio.wait_for(routes._run_tool_listing(3, 7), 1)+ assert raised.value.http_status == 429+ loads = set(routes._v1_tool_listing_loads)+ release.set()+ await asyncio.wait_for(asyncio.wait(loads), 5)+ await asyncio.sleep(0)+ assert holders == {}+ assert (await routes._run_tool_listing(3, 7)) is not None++ asyncio.run(run())+++def _gate_cell_a_request_cancelled_while_waiting_gives_back_its_count(bob, monkeypatch):+ async def run():+ _use_timeout(monkeypatch, 5)+ full = asyncio.Semaphore(0)+ monkeypatch.setattr(routes, "_v1_tool_listing_gate", lambda: full)+ holders = routes._v1_tool_listing_holders()+ request = asyncio.create_task(routes._run_tool_listing(1, 7))+ await _until(lambda: holders == {7: 1} and bool(full._waiters))+ request.cancel()+ with pytest.raises(asyncio.CancelledError):+ await request+ assert holders == {}+ assert full._value == 0++ asyncio.run(run())+++def _gate_cell_a_timeout_of_zero_still_waits_a_bounded_time(bob, monkeypatch):+ async def run():+ _use_timeout(monkeypatch, 0)+ monkeypatch.setattr(routes, "_TOOL_LISTING_SLOT_WAIT_SECONDS", 0.05)+ full = asyncio.Semaphore(0)+ monkeypatch.setattr(routes, "_v1_tool_listing_gate", lambda: full)+ holders = routes._v1_tool_listing_holders()+ with pytest.raises(routes.V1ApiError) as raised:+ await asyncio.wait_for(routes._run_tool_listing(1, 7), 5)+ assert raised.value.http_status == 429+ assert raised.value.message == BUSY+ assert holders == {}++ asyncio.run(run())+++def _gate_cell_a_full_holder_does_not_queue_for_a_slot(bob, monkeypatch):+ async def run():+ _use_timeout(monkeypatch, 5)+ monkeypatch.setattr(routes, "_TOOL_LISTING_SLOT_WAIT_SECONDS", 2)+ full = asyncio.Semaphore(0)+ monkeypatch.setattr(routes, "_v1_tool_listing_gate", lambda: full)+ holders = {7: 2}+ monkeypatch.setattr(routes, "_v1_tool_listing_holders", lambda: holders)+ with pytest.raises(routes.V1ApiError) as raised:+ await asyncio.wait_for(routes._run_tool_listing(1, 7), 0.5)+ assert "already have" in raised.value.message+ assert not full._waiters+ assert holders == {7: 2}++ asyncio.run(run())+++def _gate_cell_the_cap_counts_the_account_not_the_key(bob, monkeypatch):+ second = client.post("/api/me/personal-keys", headers=_login("bob", "bobpass1"))+ assert second.status_code == 200, second.text+ other_key = _bearer(second.json()["full_key"])+ assert other_key != bob.headers+ holders = {bob.user_id: 2}+ monkeypatch.setattr(routes, "_v1_tool_listing_holders", lambda: holders)+ calls = _stub_loader(monkeypatch, _loaded(_tool()))+ server_id = _seed("records", owner=bob)++ resp = client.get(f"/v1/mcp-servers/{server_id}/tools", headers=other_key)++ error = _error(resp, 429, "rate_limited")+ assert "already have" in error["message"]+ assert calls == []+ assert holders == {bob.user_id: 2}++_GATE_CELLS = {name.removeprefix("_gate_cell_"): fnfor name, fn in sorted(globals().items())@@ -755,7 +1010,7 @@ _GATE_CELLS = {@pytest.mark.parametrize("cell", list(_GATE_CELLS))def test_tool_listing_has_its_own_gate(bob, monkeypatch, cell):- assert len(_GATE_CELLS) == 7+ assert len(_GATE_CELLS) == 18_GATE_CELLS[cell](bob, monkeypatch)