Open-source AI teammates in Slack, Teams, Discord, Telegram and Mattermost. Self-host it, with any agent harness and any model.
Python agent-orchestrationaiai-agentschatopsdiscordlangchainmcpmicrosoft-teamsmulti-agentopen-sourceopenaiself-hostedslack
Stop a silent Mattermost from freezing all of switch-core (#660)
Abel Dantasee6f56fa09d8
diff --git a/core/switch_core/bridges/collaboration/mattermost/adapter.py b/core/switch_core/bridges/collaboration/mattermost/adapter.pyindex 14651f316..980aed0a1 100644--- a/core/switch_core/bridges/collaboration/mattermost/adapter.py+++ b/core/switch_core/bridges/collaboration/mattermost/adapter.py@@ -95,6 +95,11 @@ logger = logging.getLogger(__name__)# stream unbounded data into memory._MAX_BOT_ICON_BYTES = 5 * 1024 * 1024+# Every request to Mattermost gives up after this long. The driver's default is+# to wait for ever, so a server that accepts the connection and never answers+# would hold the call, and anything waiting on it, indefinitely.+_MM_REQUEST_TIMEOUT_SECONDS = 10.0+# Mattermost channel types a bot can be *added* to: open and private channels.# Membership of a DM ("D") or group DM ("G") is a property of the conversation# existing rather than of anyone joining, so Mattermost raises no `user_added`@@ -374,6 +379,7 @@ class MattermostAdapter(PlatformAdapter):self._agent_bots: dict[str, dict[str, str]] = {}self._bot_drivers: dict[str, Driver] = {}+ self._identity_locks: dict[str, asyncio.Lock] = {}self._bot_id_to_username: dict[str, str] = {}self._bridge_bot_ids: set[str] = set()@@ -2027,6 +2033,14 @@ class MattermostAdapter(PlatformAdapter):async def create_agent_identity(self, agent_name: str, agent_description: str+ ) -> None:+ # The Mattermost calls inside yield, so two overlapping registrations+ # of one agent would each mint a token and open a second socket.+ async with self._identity_locks.setdefault(agent_name, asyncio.Lock()):+ await self._create_agent_identity(agent_name, agent_description)++ async def _create_agent_identity(+ self, agent_name: str, agent_description: str) -> None:if not self._admin_driver or not self._main_loop:raise RuntimeError(@@ -2042,14 +2056,16 @@ class MattermostAdapter(PlatformAdapter):# identifier. Only the bot's own display name carries the label.label = (await self.agent_rendering(agent_name)).field_labelif label != agent_name:- self._warn_once_if_display_names_are_hidden()+ await self._warn_once_if_display_names_are_hidden()existing = await self._find_existing_bot(agent_name)if existing:bot_id: str = str(existing["user_id"])if existing.get("display_name") != label:try:- self._mm_api("put", f"/bots/{bot_id}", {"display_name": label})+ await self._mm_api(+ "put", f"/bots/{bot_id}", {"display_name": label}+ )except Exception as e:logger.exception("Failed to update the display name of Mattermost bot %s: %s",@@ -2058,7 +2074,7 @@ class MattermostAdapter(PlatformAdapter):)else:try:- bot = self._mm_api(+ bot = await self._mm_api("post","/bots",{@@ -2075,7 +2091,7 @@ class MattermostAdapter(PlatformAdapter):returntry:- token_resp = self._mm_api(+ token_resp = await self._mm_api("post",f"/users/{bot_id}/tokens",{"description": f"Switch bridge token for {agent_name}"},@@ -2210,7 +2226,7 @@ class MattermostAdapter(PlatformAdapter):bot_id = str(existing["user_id"])else:try:- bot = self._mm_api(+ bot = await self._mm_api("post","/bots",{@@ -2224,7 +2240,7 @@ class MattermostAdapter(PlatformAdapter):logger.exception("Failed to create Switch Admin bot")returntry:- token_resp = self._mm_api(+ token_resp = await self._mm_api("post",f"/users/{bot_id}/tokens",{"description": "Switch admin bot token"},@@ -2293,7 +2309,7 @@ class MattermostAdapter(PlatformAdapter):agent_names.append(bot_name)return agent_names- def _read_name_display_setting(self) -> str | None:+ async def _read_name_display_setting(self) -> str | None:"""`TeamSettings.TeammateNameDisplay`, read at most once per run.None when the read could not answer — the server config is readable@@ -2303,7 +2319,7 @@ class MattermostAdapter(PlatformAdapter):return self._name_display_settingself._name_display_read = Truetry:- config = self._mm_api("get", "/config")+ config = await self._mm_api("get", "/config")setting = (config.get("TeamSettings") or {}).get("TeammateNameDisplay")except Exception as e:logger.warning(@@ -2322,14 +2338,14 @@ class MattermostAdapter(PlatformAdapter):self._name_display_setting = settingreturn setting- def _warn_once_if_display_names_are_hidden(self) -> None:+ async def _warn_once_if_display_names_are_hidden(self) -> None:"""Say so when this server will store an agent's display name and shownobody. Mattermost renders a bot under its username unless the serveris told otherwise, so a display name Switch sets can be invisible withnothing about the write itself failing."""if self._name_display_warned:return- setting = self._read_name_display_setting()+ setting = await self._read_name_display_setting()if setting is None or setting in _NAME_DISPLAY_SHOWS_LABEL:returnself._name_display_warned = True@@ -2399,7 +2415,7 @@ class MattermostAdapter(PlatformAdapter):headers={"Authorization": f"Bearer {token}"},files={"image": ("icon.png", data, "image/png")},allow_redirects=False,- timeout=30,+ timeout=_MM_REQUEST_TIMEOUT_SECONDS,)if not r.ok:logger.error(@@ -2754,6 +2770,7 @@ class MattermostAdapter(PlatformAdapter):"scheme": scheme,"port": port,"verify": self._config.verify_tls,+ "request_timeout": _MM_REQUEST_TIMEOUT_SECONDS,}if token:opts["token"] = token@@ -2763,7 +2780,14 @@ class MattermostAdapter(PlatformAdapter):return Driver(opts, client_cls=NoRedirectClient)- def _mm_api(+ async def _mm_api(+ self, method: str, endpoint: str, data: dict[str, Any] | None = None+ ) -> dict[str, Any]:+ # On a thread: the driver is blocking, and on the event loop one slow+ # Mattermost response would stall every agent and bridge in Switch.+ return await asyncio.to_thread(self._mm_api_blocking, method, endpoint, data)++ def _mm_api_blocking(self, method: str, endpoint: str, data: dict[str, Any] | None = None) -> dict[str, Any]:if self._admin_driver is None:@@ -2785,7 +2809,7 @@ class MattermostAdapter(PlatformAdapter):page = 0per_page = 200while True:- result = self._mm_api(+ result = await self._mm_api("get",f"/bots?include_deleted=true&page={page}&per_page={per_page}",)@@ -2793,7 +2817,7 @@ class MattermostAdapter(PlatformAdapter):for bot in bots:if bot.get("username") == username:if bot.get("delete_at", 0) > 0:- self._mm_api("post", f"/bots/{bot['user_id']}/enable")+ await self._mm_api("post", f"/bots/{bot['user_id']}/enable")return botif len(bots) < per_page:breakdiff --git a/core/tests/switch_core/bridges/collaboration/test_bridge_agent_display_names.py b/core/tests/switch_core/bridges/collaboration/test_bridge_agent_display_names.pyindex bce192a92..e3e524407 100644--- a/core/tests/switch_core/bridges/collaboration/test_bridge_agent_display_names.py+++ b/core/tests/switch_core/bridges/collaboration/test_bridge_agent_display_names.py@@ -1645,6 +1645,26 @@ def test_mattermost_creates_the_bot_under_the_display_name() -> None:]+def test_overlapping_mattermost_registrations_of_one_agent_create_one_bot() -> None:+ """Registration and the background provisioner can reach the same agent at+ once, and every Mattermost call yields, so without a lock both would see no+ bot and each mint a token and open a socket."""+ bridge = _bridge(_agent("switchdev", "Switch Dev"))+ adapter, admin = _mattermost_identity_adapter(bridge, [], "full_name")++ async def _go() -> None:+ adapter._main_loop = asyncio.get_running_loop()+ await asyncio.gather(+ adapter.create_agent_identity("switchdev", "a Switch agent"),+ adapter.create_agent_identity("switchdev", "a Switch agent"),+ )++ _run(_go())++ assert len(admin.bodies("post", "/bots")) == 1+ assert len(admin.bodies("post", "/users/bot-switchdev/tokens")) == 1++def test_a_mattermost_bot_username_is_never_the_display_name() -> None:"""The hazard the whole split exists for: a username is the routing handleand is also constrained to lowercase alphanumerics, so a label like thisdiff --git a/core/tests/switch_core/bridges/collaboration/test_mattermost_unresponsive.py b/core/tests/switch_core/bridges/collaboration/test_mattermost_unresponsive.pynew file mode 100644index 000000000..6bf05318b--- /dev/null+++ b/core/tests/switch_core/bridges/collaboration/test_mattermost_unresponsive.py@@ -0,0 +1,100 @@+"""A Mattermost that accepts connections and never answers.++The driver is blocking and, by default, waits for ever. Called on the event+loop, one such request froze all of Switch: every agent connection lapsed,+every bridge stopped and /health stopped answering, for as long as Mattermost+stayed silent. Requests now give up after a timeout, and run off the loop.+"""++from __future__ import annotations++import asyncio+import socket+import threading+import time+from collections.abc import Iterator++import pytest++from switch_core.bridges.collaboration.mattermost import adapter as adapter_module+from switch_core.bridges.collaboration.mattermost.adapter import (+ MattermostAdapter,+ MattermostConnectionConfig,+)++# Past this the server hangs up, so a regressed client fails instead of hanging.+_HANG_UP_AFTER_SECONDS = 5.0+++def silent_server() -> Iterator[int]:+ """A TCP server that accepts every connection and never sends a byte."""+ server = socket.socket()+ server.bind(("127.0.0.1", 0))+ server.listen()+ held: list[socket.socket] = []+ stop = threading.Event()+ deadline = time.monotonic() + _HANG_UP_AFTER_SECONDS++ def accept() -> None:+ server.settimeout(0.1)+ while not stop.is_set() and time.monotonic() < deadline:+ try:+ conn, _ = server.accept()+ except TimeoutError:+ continue+ held.append(conn)+ for conn in held:+ conn.close()+ server.close()++ thread = threading.Thread(target=accept, daemon=True)+ thread.start()+ yield server.getsockname()[1]+ stop.set()+ thread.join()+ server.close()+++def adapter(silent_server: int, monkeypatch: pytest.MonkeyPatch) -> MattermostAdapter:+ monkeypatch.setattr(adapter_module, "_MM_REQUEST_TIMEOUT_SECONDS", 0.5)+ mm = MattermostAdapter(+ config=MattermostConnectionConfig(+ url=f"http://127.0.0.1:{silent_server}",+ admin_user="admin",+ admin_password="pw",+ team_name="team",+ )+ )+ mm._admin_driver = mm._create_driver(token="placeholder")+ return mm+++async def test_a_request_mattermost_never_answers_gives_up(+ adapter: MattermostAdapter,+) -> None:+ started = time.monotonic()++ with pytest.raises(Exception): # noqa: B017, the driver's own timeout error+ await adapter._mm_api("get", "/config")++ assert time.monotonic() - started < _HANG_UP_AFTER_SECONDS+++async def test_the_event_loop_keeps_running_while_mattermost_is_silent(+ adapter: MattermostAdapter,+) -> None:+ request = asyncio.ensure_future(adapter._mm_api("get", "/config"))+ gaps: list[float] = []+ last = time.monotonic()+ while not request.done():+ await asyncio.sleep(0.02)+ now = time.monotonic()+ gaps.append(now - last)+ last = now++ with pytest.raises(Exception): # noqa: B017+ request.result()+ assert len(gaps) > 5+ assert max(gaps) < 0.25