Skip to content

Open-source AI teammates in Slack, Teams, Discord, Telegram and Mattermost. Self-host it, with any agent harness and any model.

Stop a silent Mattermost from freezing all of switch-core (#660)

Abel Dantasee6f56fa09d8
diff --git a/core/switch_core/bridges/collaboration/mattermost/adapter.py b/core/switch_core/bridges/collaboration/mattermost/adapter.py
index 14651f316..980aed0a1 100644
--- a/core/switch_core/bridges/collaboration/mattermost/adapter.py
+++ b/core/switch_core/bridges/collaboration/mattermost/adapter.py
@@ -95,6 +95,11 @@ logger = logging.getLogger(__name__)
# stream unbounded data into memory.
_MAX_BOT_ICON_BYTES = 5 * 1024 * 1024
+# Every request to Mattermost gives up after this long. The driver's default is
+# to wait for ever, so a server that accepts the connection and never answers
+# would hold the call, and anything waiting on it, indefinitely.
+_MM_REQUEST_TIMEOUT_SECONDS = 10.0
+
# Mattermost channel types a bot can be *added* to: open and private channels.
# Membership of a DM ("D") or group DM ("G") is a property of the conversation
# existing rather than of anyone joining, so Mattermost raises no `user_added`
@@ -374,6 +379,7 @@ class MattermostAdapter(PlatformAdapter):
self._agent_bots: dict[str, dict[str, str]] = {}
self._bot_drivers: dict[str, Driver] = {}
+ self._identity_locks: dict[str, asyncio.Lock] = {}
self._bot_id_to_username: dict[str, str] = {}
self._bridge_bot_ids: set[str] = set()
@@ -2027,6 +2033,14 @@ class MattermostAdapter(PlatformAdapter):
async def create_agent_identity(
self, agent_name: str, agent_description: str
+ ) -> None:
+ # The Mattermost calls inside yield, so two overlapping registrations
+ # of one agent would each mint a token and open a second socket.
+ async with self._identity_locks.setdefault(agent_name, asyncio.Lock()):
+ await self._create_agent_identity(agent_name, agent_description)
+
+ async def _create_agent_identity(
+ self, agent_name: str, agent_description: str
) -> None:
if not self._admin_driver or not self._main_loop:
raise RuntimeError(
@@ -2042,14 +2056,16 @@ class MattermostAdapter(PlatformAdapter):
# identifier. Only the bot's own display name carries the label.
label = (await self.agent_rendering(agent_name)).field_label
if label != agent_name:
- self._warn_once_if_display_names_are_hidden()
+ await self._warn_once_if_display_names_are_hidden()
existing = await self._find_existing_bot(agent_name)
if existing:
bot_id: str = str(existing["user_id"])
if existing.get("display_name") != label:
try:
- self._mm_api("put", f"/bots/{bot_id}", {"display_name": label})
+ await self._mm_api(
+ "put", f"/bots/{bot_id}", {"display_name": label}
+ )
except Exception as e:
logger.exception(
"Failed to update the display name of Mattermost bot %s: %s",
@@ -2058,7 +2074,7 @@ class MattermostAdapter(PlatformAdapter):
)
else:
try:
- bot = self._mm_api(
+ bot = await self._mm_api(
"post",
"/bots",
{
@@ -2075,7 +2091,7 @@ class MattermostAdapter(PlatformAdapter):
return
try:
- token_resp = self._mm_api(
+ token_resp = await self._mm_api(
"post",
f"/users/{bot_id}/tokens",
{"description": f"Switch bridge token for {agent_name}"},
@@ -2210,7 +2226,7 @@ class MattermostAdapter(PlatformAdapter):
bot_id = str(existing["user_id"])
else:
try:
- bot = self._mm_api(
+ bot = await self._mm_api(
"post",
"/bots",
{
@@ -2224,7 +2240,7 @@ class MattermostAdapter(PlatformAdapter):
logger.exception("Failed to create Switch Admin bot")
return
try:
- token_resp = self._mm_api(
+ token_resp = await self._mm_api(
"post",
f"/users/{bot_id}/tokens",
{"description": "Switch admin bot token"},
@@ -2293,7 +2309,7 @@ class MattermostAdapter(PlatformAdapter):
agent_names.append(bot_name)
return agent_names
- def _read_name_display_setting(self) -> str | None:
+ async def _read_name_display_setting(self) -> str | None:
"""`TeamSettings.TeammateNameDisplay`, read at most once per run.
None when the read could not answer — the server config is readable
@@ -2303,7 +2319,7 @@ class MattermostAdapter(PlatformAdapter):
return self._name_display_setting
self._name_display_read = True
try:
- config = self._mm_api("get", "/config")
+ config = await self._mm_api("get", "/config")
setting = (config.get("TeamSettings") or {}).get("TeammateNameDisplay")
except Exception as e:
logger.warning(
@@ -2322,14 +2338,14 @@ class MattermostAdapter(PlatformAdapter):
self._name_display_setting = setting
return setting
- def _warn_once_if_display_names_are_hidden(self) -> None:
+ async def _warn_once_if_display_names_are_hidden(self) -> None:
"""Say so when this server will store an agent's display name and show
nobody. Mattermost renders a bot under its username unless the server
is told otherwise, so a display name Switch sets can be invisible with
nothing about the write itself failing."""
if self._name_display_warned:
return
- setting = self._read_name_display_setting()
+ setting = await self._read_name_display_setting()
if setting is None or setting in _NAME_DISPLAY_SHOWS_LABEL:
return
self._name_display_warned = True
@@ -2399,7 +2415,7 @@ class MattermostAdapter(PlatformAdapter):
headers={"Authorization": f"Bearer {token}"},
files={"image": ("icon.png", data, "image/png")},
allow_redirects=False,
- timeout=30,
+ timeout=_MM_REQUEST_TIMEOUT_SECONDS,
)
if not r.ok:
logger.error(
@@ -2754,6 +2770,7 @@ class MattermostAdapter(PlatformAdapter):
"scheme": scheme,
"port": port,
"verify": self._config.verify_tls,
+ "request_timeout": _MM_REQUEST_TIMEOUT_SECONDS,
}
if token:
opts["token"] = token
@@ -2763,7 +2780,14 @@ class MattermostAdapter(PlatformAdapter):
return Driver(opts, client_cls=NoRedirectClient)
- def _mm_api(
+ async def _mm_api(
+ self, method: str, endpoint: str, data: dict[str, Any] | None = None
+ ) -> dict[str, Any]:
+ # On a thread: the driver is blocking, and on the event loop one slow
+ # Mattermost response would stall every agent and bridge in Switch.
+ return await asyncio.to_thread(self._mm_api_blocking, method, endpoint, data)
+
+ def _mm_api_blocking(
self, method: str, endpoint: str, data: dict[str, Any] | None = None
) -> dict[str, Any]:
if self._admin_driver is None:
@@ -2785,7 +2809,7 @@ class MattermostAdapter(PlatformAdapter):
page = 0
per_page = 200
while True:
- result = self._mm_api(
+ result = await self._mm_api(
"get",
f"/bots?include_deleted=true&page={page}&per_page={per_page}",
)
@@ -2793,7 +2817,7 @@ class MattermostAdapter(PlatformAdapter):
for bot in bots:
if bot.get("username") == username:
if bot.get("delete_at", 0) > 0:
- self._mm_api("post", f"/bots/{bot['user_id']}/enable")
+ await self._mm_api("post", f"/bots/{bot['user_id']}/enable")
return bot
if len(bots) < per_page:
break
diff --git a/core/tests/switch_core/bridges/collaboration/test_bridge_agent_display_names.py b/core/tests/switch_core/bridges/collaboration/test_bridge_agent_display_names.py
index bce192a92..e3e524407 100644
--- a/core/tests/switch_core/bridges/collaboration/test_bridge_agent_display_names.py
+++ b/core/tests/switch_core/bridges/collaboration/test_bridge_agent_display_names.py
@@ -1645,6 +1645,26 @@ def test_mattermost_creates_the_bot_under_the_display_name() -> None:
]
+def test_overlapping_mattermost_registrations_of_one_agent_create_one_bot() -> None:
+ """Registration and the background provisioner can reach the same agent at
+ once, and every Mattermost call yields, so without a lock both would see no
+ bot and each mint a token and open a socket."""
+ bridge = _bridge(_agent("switchdev", "Switch Dev"))
+ adapter, admin = _mattermost_identity_adapter(bridge, [], "full_name")
+
+ async def _go() -> None:
+ adapter._main_loop = asyncio.get_running_loop()
+ await asyncio.gather(
+ adapter.create_agent_identity("switchdev", "a Switch agent"),
+ adapter.create_agent_identity("switchdev", "a Switch agent"),
+ )
+
+ _run(_go())
+
+ assert len(admin.bodies("post", "/bots")) == 1
+ assert len(admin.bodies("post", "/users/bot-switchdev/tokens")) == 1
+
+
def test_a_mattermost_bot_username_is_never_the_display_name() -> None:
"""The hazard the whole split exists for: a username is the routing handle
and is also constrained to lowercase alphanumerics, so a label like this
diff --git a/core/tests/switch_core/bridges/collaboration/test_mattermost_unresponsive.py b/core/tests/switch_core/bridges/collaboration/test_mattermost_unresponsive.py
new file mode 100644
index 000000000..6bf05318b
--- /dev/null
+++ b/core/tests/switch_core/bridges/collaboration/test_mattermost_unresponsive.py
@@ -0,0 +1,100 @@
+"""A Mattermost that accepts connections and never answers.
+
+The driver is blocking and, by default, waits for ever. Called on the event
+loop, one such request froze all of Switch: every agent connection lapsed,
+every bridge stopped and /health stopped answering, for as long as Mattermost
+stayed silent. Requests now give up after a timeout, and run off the loop.
+"""
+
+from __future__ import annotations
+
+import asyncio
+import socket
+import threading
+import time
+from collections.abc import Iterator
+
+import pytest
+
+from switch_core.bridges.collaboration.mattermost import adapter as adapter_module
+from switch_core.bridges.collaboration.mattermost.adapter import (
+ MattermostAdapter,
+ MattermostConnectionConfig,
+)
+
+# Past this the server hangs up, so a regressed client fails instead of hanging.
+_HANG_UP_AFTER_SECONDS = 5.0
+
+
+def silent_server() -> Iterator[int]:
+ """A TCP server that accepts every connection and never sends a byte."""
+ server = socket.socket()
+ server.bind(("127.0.0.1", 0))
+ server.listen()
+ held: list[socket.socket] = []
+ stop = threading.Event()
+ deadline = time.monotonic() + _HANG_UP_AFTER_SECONDS
+
+ def accept() -> None:
+ server.settimeout(0.1)
+ while not stop.is_set() and time.monotonic() < deadline:
+ try:
+ conn, _ = server.accept()
+ except TimeoutError:
+ continue
+ held.append(conn)
+ for conn in held:
+ conn.close()
+ server.close()
+
+ thread = threading.Thread(target=accept, daemon=True)
+ thread.start()
+ yield server.getsockname()[1]
+ stop.set()
+ thread.join()
+ server.close()
+
+
+def adapter(silent_server: int, monkeypatch: pytest.MonkeyPatch) -> MattermostAdapter:
+ monkeypatch.setattr(adapter_module, "_MM_REQUEST_TIMEOUT_SECONDS", 0.5)
+ mm = MattermostAdapter(
+ config=MattermostConnectionConfig(
+ url=f"http://127.0.0.1:{silent_server}",
+ admin_user="admin",
+ admin_password="pw",
+ team_name="team",
+ )
+ )
+ mm._admin_driver = mm._create_driver(token="placeholder")
+ return mm
+
+
+async def test_a_request_mattermost_never_answers_gives_up(
+ adapter: MattermostAdapter,
+) -> None:
+ started = time.monotonic()
+
+ with pytest.raises(Exception): # noqa: B017, the driver's own timeout error
+ await adapter._mm_api("get", "/config")
+
+ assert time.monotonic() - started < _HANG_UP_AFTER_SECONDS
+
+
+async def test_the_event_loop_keeps_running_while_mattermost_is_silent(
+ adapter: MattermostAdapter,
+) -> None:
+ request = asyncio.ensure_future(adapter._mm_api("get", "/config"))
+ gaps: list[float] = []
+ last = time.monotonic()
+ while not request.done():
+ await asyncio.sleep(0.02)
+ now = time.monotonic()
+ gaps.append(now - last)
+ last = now
+
+ with pytest.raises(Exception): # noqa: B017
+ request.result()
+ assert len(gaps) > 5
+ assert max(gaps) < 0.25