Skip to content

Open-source AI teammates in Slack, Teams, Discord, Telegram and Mattermost. Self-host it, with any agent harness and any model.

Close each Mattermost bot socket's event loop, cancelling what it left running

Petr Bauch048b80142ea6
diff --git a/core/switch_core/bridges/collaboration/mattermost/adapter.py b/core/switch_core/bridges/collaboration/mattermost/adapter.py
index 97b302113..dea330783 100644
--- a/core/switch_core/bridges/collaboration/mattermost/adapter.py
+++ b/core/switch_core/bridges/collaboration/mattermost/adapter.py
@@ -298,6 +298,24 @@ def _ephemeral(text: str) -> dict[str, Any]:
return {"ephemeral_text": text, "skip_slack_parsing": True}
+def _close_loop(loop: asyncio.AbstractEventLoop) -> None:
+ """Cancel what is still running on `loop`, then close it.
+
+ A socket's `websockets` connection runs a keepalive task beside it that
+ outlives the socket's own coroutine. A loop dropped with that task still
+ pending is reported as "Task was destroyed but it is pending!" whenever it
+ is collected, at ERROR, in the middle of whatever the server is doing.
+ """
+ pending = asyncio.all_tasks(loop)
+ for task in pending:
+ task.cancel()
+ if pending:
+ loop.run_until_complete(asyncio.gather(*pending, return_exceptions=True))
+ loop.run_until_complete(loop.shutdown_asyncgens())
+ loop.close()
+ asyncio.set_event_loop(None)
+
+
class MattermostAdapter(PlatformAdapter):
display_name: ClassVar[str] = "Mattermost"
docs_slug: ClassVar[str | None] = "mattermost"
@@ -2211,23 +2229,32 @@ class MattermostAdapter(PlatformAdapter):
return
backoff = 2
while not stopped.is_set():
+ # `init_websocket` is synchronous: it runs the thread's current
+ # loop until the socket closes, so each connection gets a loop
+ # of its own, closed with it.
+ ws_loop = asyncio.new_event_loop()
+ asyncio.set_event_loop(ws_loop)
try:
- ws_loop = asyncio.new_event_loop()
- asyncio.set_event_loop(ws_loop)
- ws_loop.run_until_complete(driver.init_websocket(_handler))
+ driver.init_websocket(_handler)
except Exception as e:
- if stopped.is_set():
- break
- logger.debug(
- "WebSocket for %s disconnected: %s, retrying in %ds",
- name,
- e,
- backoff,
- )
- stopped.wait(backoff)
- backoff = min(backoff * 2, 30)
+ reason = str(e)
+ backoff_next = min(backoff * 2, 30)
else:
- backoff = 2
+ # It returns only once an established connection closed.
+ reason = "connection closed"
+ backoff, backoff_next = 2, 2
+ finally:
+ _close_loop(ws_loop)
+ if stopped.is_set():
+ break
+ logger.debug(
+ "WebSocket for %s disconnected: %s, retrying in %ds",
+ name,
+ reason,
+ backoff,
+ )
+ stopped.wait(backoff)
+ backoff = backoff_next
ws_thread = threading.Thread(
target=_run_ws, daemon=True, name=f"mm-ws-{agent_name}"
diff --git a/core/tests/switch_core/bridges/collaboration/test_bridge_agent_display_names.py b/core/tests/switch_core/bridges/collaboration/test_bridge_agent_display_names.py
index 15747e84b..a5d929c01 100644
--- a/core/tests/switch_core/bridges/collaboration/test_bridge_agent_display_names.py
+++ b/core/tests/switch_core/bridges/collaboration/test_bridge_agent_display_names.py
@@ -1617,8 +1617,10 @@ class _MattermostBot:
def login(self) -> None:
return None
- async def init_websocket(self, handler: Any) -> None:
- await asyncio.Event().wait()
+ def init_websocket(self, handler: Any) -> asyncio.AbstractEventLoop:
+ loop = asyncio.get_event_loop()
+ loop.run_until_complete(asyncio.Event().wait())
+ return loop
def _mattermost_identity_adapter(
diff --git a/core/tests/switch_core/bridges/collaboration/test_mattermost_bot_sockets.py b/core/tests/switch_core/bridges/collaboration/test_mattermost_bot_sockets.py
index 6f7ece58c..00b7c5209 100644
--- a/core/tests/switch_core/bridges/collaboration/test_mattermost_bot_sockets.py
+++ b/core/tests/switch_core/bridges/collaboration/test_mattermost_bot_sockets.py
@@ -33,8 +33,15 @@ class _Bot:
def login(self) -> None:
return None
- async def init_websocket(self, handler: Any) -> None:
+ def init_websocket(self, handler: Any) -> asyncio.AbstractEventLoop:
+ """As `mattermostdriver.Driver.init_websocket`: synchronous, running
+ the thread's current event loop until the socket closes."""
self.websocket = object()
+ loop = asyncio.get_event_loop()
+ loop.run_until_complete(self._connect(handler))
+ return loop
+
+ async def _connect(self, handler: Any) -> None:
self.connects += 1
while not self.disconnected.is_set():
await asyncio.sleep(0.01)
@@ -101,3 +108,44 @@ def test_removing_an_agent_closes_its_socket_for_good() -> None:
assert bot.disconnected.is_set()
assert not alive
assert bot.connects == 1
+
+
+class _BotWithKeepalive(_Bot):
+ """A bot driver whose socket, like the `websockets` connection under
+ `mattermostdriver`, runs a keepalive task beside it that outlives the
+ socket's own coroutine unless something cancels it."""
+
+ def __init__(self) -> None:
+ super().__init__()
+ self.keepalive: asyncio.Task[None] | None = None
+ self.loop: asyncio.AbstractEventLoop | None = None
+
+ async def _connect(self, handler: Any) -> None:
+ self.loop = asyncio.get_running_loop()
+ self.keepalive = asyncio.create_task(asyncio.sleep(3600))
+ await super()._connect(handler)
+
+
+def test_a_closed_socket_leaves_no_task_pending_on_its_loop() -> None:
+ """A loop dropped with a task still pending on it is reported by asyncio
+ as `Task was destroyed but it is pending!` whenever the loop is collected,
+ an error logged in the middle of whatever the server is doing then."""
+ bridge = _bridge(_agent("switchdev", "Switch Dev"))
+ adapter, _ = _mattermost_identity_adapter(bridge, [], "full_name")
+ bot = _BotWithKeepalive()
+ adapter._create_driver = lambda **_kwargs: bot # type: ignore[assignment,method-assign,return-value]
+
+ async def go() -> None:
+ adapter._main_loop = asyncio.get_running_loop()
+ before = set(threading.enumerate())
+ await adapter.create_agent_identity("switchdev", "a Switch agent")
+ thread = _socket_thread("switchdev", before)
+ await _until(lambda: bot.connects == 1)
+ await adapter.remove_agent_identity("switchdev")
+ await asyncio.to_thread(thread.join, 5)
+ assert not thread.is_alive()
+
+ _run(go())
+
+ assert bot.keepalive is not None and bot.keepalive.cancelled()
+ assert bot.loop is not None and bot.loop.is_closed()