Open-source AI teammates in Slack, Teams, Discord, Telegram and Mattermost. Self-host it, with any agent harness and any model.
Python agent-orchestrationaiai-agentschatopsdiscordlangchainmcpmicrosoft-teamsmulti-agentopen-sourceopenaiself-hostedslack
fix(bridges): serialise runtime-indicator refresh and repositioning (CHOO-1104)
Louis Amaudruzc363260a36d3
diff --git a/core/switch_core/bridges/collaboration/adapter.py b/core/switch_core/bridges/collaboration/adapter.pyindex e48d00345..769bace0b 100644--- a/core/switch_core/bridges/collaboration/adapter.py+++ b/core/switch_core/bridges/collaboration/adapter.py@@ -1,5 +1,6 @@from __future__ import annotations+import asyncioimport loggingfrom abc import ABC, abstractmethodfrom collections.abc import Awaitable, Callable@@ -53,6 +54,9 @@ class CollaborationAdapter(ABC):# typing-indicator default leaves them empty.self._working_msg: dict[tuple[str, str], LiveRuntimeIndicator] = {}self._input_pings: dict[tuple[str, str], list[str]] = {}+ # One lock per (channel_id, agent_name). Every mutation of the entries+ # above happens under it — see _runtime_lock.+ self._runtime_locks: dict[tuple[str, str], asyncio.Lock] = {}def set_max_attachment_bytes(self, max_bytes: int) -> None:self._max_attachment_bytes = max_bytes@@ -178,6 +182,20 @@ class CollaborationAdapter(ABC):self, channel_id: str, sender_name: str, is_typing: bool) -> None: ...+ def _runtime_lock(self, channel_id: str, agent_name: str) -> asyncio.Lock:+ """The lock serialising runtime-indicator work for one agent in one+ channel.++ The indicator is mutated from two independent places — the periodic+ activity refresh and a reposition triggered by new traffic — and each+ reads the tracked message, awaits a platform call, then writes it back.+ Left to interleave, the later write restores a superseded message ref:+ the entry then names a message that has just been deleted while the one+ actually on screen is referenced by nothing, so the end-of-turn clear+ cannot remove it and it stays in the channel for good.+ """+ return self._runtime_locks.setdefault((channel_id, agent_name), asyncio.Lock())+async def apply_runtime_state(self,channel_id: str,@@ -188,6 +206,39 @@ class CollaborationAdapter(ABC):thread_root_id: str | None,deeplink_url: str | None = None,detail: str | None = None,+ ) -> None:+ """Serialise against any other runtime-indicator work for this agent,+ then apply the state. Adapters override ``_apply_runtime_state``."""+ async with self._runtime_lock(channel_id, agent_name):+ await self._apply_runtime_state(+ channel_id,+ agent_name,+ state,+ notify_user=notify_user,+ thread_root_id=thread_root_id,+ deeplink_url=deeplink_url,+ detail=detail,+ )++ async def reposition_runtime_state(+ self, channel_id: str, agent_name: str, thread_root_id: str | None+ ) -> None:+ """Serialise against any other runtime-indicator work for this agent,+ then move the indicator. Adapters override+ ``_reposition_runtime_state``."""+ async with self._runtime_lock(channel_id, agent_name):+ await self._reposition_runtime_state(channel_id, agent_name, thread_root_id)++ async def _apply_runtime_state(+ self,+ channel_id: str,+ agent_name: str,+ state: str,+ *,+ notify_user: str | None,+ thread_root_id: str | None,+ deeplink_url: str | None = None,+ detail: str | None = None,) -> None:"""Surface a switchdash-managed agent's runtime state on the channel.@@ -229,7 +280,7 @@ class CollaborationAdapter(ABC):if posted_channel == channel_id]- async def reposition_runtime_state(+ async def _reposition_runtime_state(self, channel_id: str, agent_name: str, thread_root_id: str | None) -> None:"""Move the agent's live runtime indicator to follow the latest message.@@ -245,6 +296,9 @@ class CollaborationAdapter(ABC):out to the channel root) rather than being stranded in whichever threadthe turn happened to start in.+ Runs under the agent's runtime lock, so the tracked indicator cannot be+ cleared or refreshed part-way through.+Adapters that render runtime state as a typing indicator have nothingpositional to move, so the default does nothing."""@@ -263,14 +317,6 @@ class CollaborationAdapter(ABC):)return- if self._working_msg.get(key) is not live:- # The turn ended (or another move landed) while this post was in- # flight, so whoever cleared it has already removed the message this- # one replaces. Nothing will ever clear the replacement, so take it- # back down rather than strand it in the channel.- await self._remove_runtime_indicator(channel_id, ref)- return-self._working_msg[key] = replace(live, message_ref=ref, thread_root_id=thread_root_id)diff --git a/core/switch_core/bridges/collaboration/bridge_core.py b/core/switch_core/bridges/collaboration/bridge_core.pyindex ac38417cf..2f71453d1 100644--- a/core/switch_core/bridges/collaboration/bridge_core.py+++ b/core/switch_core/bridges/collaboration/bridge_core.py@@ -1418,7 +1418,12 @@ class BridgeCore:)async def _run_indicator_move(self, key: tuple[str, str]) -> None:- self._indicator_move_timers.pop(key, None)+ # An anchor-driven move runs immediately rather than through the timer,+ # so a coalescing window opened by outbound traffic may still be+ # pending; it would otherwise fire a second, redundant move.+ timer = self._indicator_move_timers.pop(key, None)+ if timer is not None:+ timer.cancel()thread_root_id = self._indicator_move_targets.pop(key, None)channel_id, agent_name = keytry:diff --git a/core/switch_core/bridges/collaboration/discord/adapter.py b/core/switch_core/bridges/collaboration/discord/adapter.pyindex 8db0ea3e3..cddf6ed97 100644--- a/core/switch_core/bridges/collaboration/discord/adapter.py+++ b/core/switch_core/bridges/collaboration/discord/adapter.py@@ -442,7 +442,7 @@ class DiscordAdapter(CollaborationAdapter):# ── Runtime state ────────────────────────────────────────────────────────- async def apply_runtime_state(+ async def _apply_runtime_state(self,channel_id: str,agent_name: str,diff --git a/core/switch_core/bridges/collaboration/mattermost/adapter.py b/core/switch_core/bridges/collaboration/mattermost/adapter.pyindex 90ee66022..6fc82dbcd 100644--- a/core/switch_core/bridges/collaboration/mattermost/adapter.py+++ b/core/switch_core/bridges/collaboration/mattermost/adapter.py@@ -440,7 +440,7 @@ class MattermostAdapter(CollaborationAdapter):# ── Runtime state ──────────────────────────────────────────────────────────- async def apply_runtime_state(+ async def _apply_runtime_state(self,channel_id: str,agent_name: str,@@ -503,7 +503,7 @@ class MattermostAdapter(CollaborationAdapter):agent_name, post_id, self.translate_outbound("✓ input received"))- async def reposition_runtime_state(+ async def _reposition_runtime_state(self, channel_id: str, agent_name: str, thread_root_id: str | None) -> None:"""Move the indicator only when Mattermost will truly delete the old one.@@ -535,12 +535,6 @@ class MattermostAdapter(CollaborationAdapter):)return- if self._working_msg.get(key) is not live:- # The turn ended while the delete was in flight. The old post is- # already gone and the entry with it, so there is nothing left to- # move — reposting now would strand an indicator no one will clear.- return-ref = await self.send_message(channel_id, agent_name, live.body, thread_root_id)if ref is None:# The old post is already gone, so there is nothing to fall back to.@@ -553,12 +547,6 @@ class MattermostAdapter(CollaborationAdapter):)return- if self._working_msg.get(key) is not live:- # Same race, one step later: the replacement has no owner now, so- # take it back down rather than leave it in the channel.- await self._permanent_delete(ref)- return-self._working_msg[key] = replace(live, message_ref=ref, thread_root_id=thread_root_id)diff --git a/core/switch_core/bridges/collaboration/slack/adapter.py b/core/switch_core/bridges/collaboration/slack/adapter.pyindex c7f5ca59c..bfa6def8d 100644--- a/core/switch_core/bridges/collaboration/slack/adapter.py+++ b/core/switch_core/bridges/collaboration/slack/adapter.py@@ -444,7 +444,7 @@ class SlackAdapter(CollaborationAdapter):# ── Runtime state ──────────────────────────────────────────────────────────- async def apply_runtime_state(+ async def _apply_runtime_state(self,channel_id: str,agent_name: str,diff --git a/core/switch_core/bridges/collaboration/teams/adapter.py b/core/switch_core/bridges/collaboration/teams/adapter.pyindex 19f281a6a..9ec868e89 100644--- a/core/switch_core/bridges/collaboration/teams/adapter.py+++ b/core/switch_core/bridges/collaboration/teams/adapter.py@@ -531,7 +531,7 @@ class TeamsAdapter(CollaborationAdapter):# ── Runtime state ──────────────────────────────────────────────────────────- async def apply_runtime_state(+ async def _apply_runtime_state(self,channel_id: str,agent_name: str,diff --git a/core/tests/switch_core/bridges/collaboration/test_mattermost_runtime_state.py b/core/tests/switch_core/bridges/collaboration/test_mattermost_runtime_state.pyindex 1ae4eb5aa..31dd35ab4 100644--- a/core/tests/switch_core/bridges/collaboration/test_mattermost_runtime_state.py+++ b/core/tests/switch_core/bridges/collaboration/test_mattermost_runtime_state.py@@ -37,6 +37,9 @@ class _Recorder:self.hard_deletes: list[str] = []self.patches: list[tuple[str, str]] = []self.sends: list[tuple[str, str, str, str | None]] = []+ # Post ids actually handed back, so a test can assert that everything+ # posted was also removed.+ self.sent_ids: list[str] = []self._next_id = iter(["post-2", "post-3", "post-4"])def install(self, adapter: MattermostAdapter) -> None:@@ -54,7 +57,9 @@ class _Recorder:thread_root_id: str | None = None,) -> str | None:self.sends.append((channel_id, sender_name, content, thread_root_id))- return next(self._next_id)+ ref = next(self._next_id)+ self.sent_ids.append(ref)+ return refadapter._permanent_delete = permanent_delete # type: ignore[method-assign]adapter._patch_post_as = patch_post_as # type: ignore[method-assign]@@ -155,47 +160,58 @@ def test_failed_repost_after_delete_drops_the_stale_ref() -> None:assert ("chan-1", "worker") not in adapter._working_msg-def test_a_turn_ending_mid_delete_abandons_the_move() -> None:- # Mattermost deletes first, so a turn ending during that delete leaves- # nothing to move — reposting anyway would strand an indicator forever.+def test_a_turn_ending_during_a_move_leaves_nothing_behind() -> None:+ # The move and the end-of-turn clear share the agent's runtime lock, so+ # they cannot interleave: whichever runs second sees the other's result.+ # Either way the channel ends up with no indicator and none is tracked.adapter = _adapter()recorder = _Recorder(hard_delete_works=True)recorder.install(adapter)_seed_indicator(adapter)- real_delete = adapter._permanent_delete-- async def delete_then_go_idle(post_id: str) -> bool:- ok = await real_delete(post_id)- adapter._working_msg.pop(("chan-1", "worker"), None)- return ok+ async def scenario() -> None:+ await asyncio.gather(+ adapter.reposition_runtime_state("chan-1", "worker", None),+ adapter.apply_runtime_state(+ "chan-1", "worker", "idle", notify_user=None, thread_root_id=None+ ),+ )- adapter._permanent_delete = delete_then_go_idle # type: ignore[method-assign]- _run(adapter.reposition_runtime_state("chan-1", "worker", None))+ _run(scenario())- assert recorder.sends == []+ posted = {"post-1", *(ref for ref in recorder.sent_ids)}+ assert posted <= set(recorder.hard_deletes)assert ("chan-1", "worker") not in adapter._working_msg-def test_a_turn_ending_mid_repost_takes_the_replacement_back_down() -> None:+def test_a_refresh_during_a_move_does_not_strand_the_replacement() -> None:+ # The reported leftover: the periodic activity refresh and the move both+ # read-modify-write the tracked indicator. Serialised, the refresh edits+ # whichever message is actually posted rather than restoring a dead ref.adapter = _adapter()recorder = _Recorder(hard_delete_works=True)recorder.install(adapter)_seed_indicator(adapter)- real_send = adapter.send_message-- async def send_then_go_idle(*args: Any, **kwargs: Any) -> str | None:- ref = await real_send(*args, **kwargs)- adapter._working_msg.pop(("chan-1", "worker"), None)- return ref+ async def scenario() -> None:+ await asyncio.gather(+ adapter.reposition_runtime_state("chan-1", "worker", None),+ adapter.apply_runtime_state(+ "chan-1",+ "worker",+ "working",+ notify_user=None,+ thread_root_id=None,+ detail="Ran tool post_message",+ ),+ )- adapter.send_message = send_then_go_idle # type: ignore[method-assign]- _run(adapter.reposition_runtime_state("chan-1", "worker", None))+ _run(scenario())- # post-1 was the move's own delete; post-2 is the orphan being cleaned up.- assert recorder.hard_deletes == ["post-1", "post-2"]- assert ("chan-1", "worker") not in adapter._working_msg+ live = adapter._working_msg[("chan-1", "worker")]+ posted = {"post-1", *recorder.sent_ids}+ still_up = posted - set(recorder.hard_deletes)+ assert still_up == {live.message_ref}def test_idle_still_falls_back_to_a_terminal_marker() -> None:diff --git a/core/tests/switch_core/bridges/collaboration/test_runtime_indicator_race.py b/core/tests/switch_core/bridges/collaboration/test_runtime_indicator_race.pynew file mode 100644index 000000000..24836805f--- /dev/null+++ b/core/tests/switch_core/bridges/collaboration/test_runtime_indicator_race.py@@ -0,0 +1,186 @@+"""Moving the runtime indicator and refreshing it must not interleave.++Two independent callers mutate ``_working_msg`` for the same agent: the+periodic activity refresh (``apply_runtime_state`` with ``working``, every few+seconds) and a reposition triggered by new traffic. Both read the entry, await+a platform call, then write it back.++Interleaved, the refresh's write lands after the move's and restores the+superseded message ref. The entry then points at a message the move has just+deleted, and the message the move posted is referenced by nothing — so the+end-of-turn clear cannot remove it and it stays in the channel forever.++The invariant each test asserts is the same: whatever is still posted on the+platform is exactly what the adapter thinks is posted.+"""++from __future__ import annotations++import asyncio+from typing import Any++from switch_core.bridges.collaboration.adapter import LiveRuntimeIndicator+from switch_core.bridges.collaboration.slack.adapter import (+ SlackAdapter,+ SlackConnectionConfig,+)++CHANNEL = "chan-1"+AGENT = "worker"+KEY = (CHANNEL, AGENT)+++class _Platform:+ """Records posts, edits and deletes, yielding once per call.++ The single yield is what lets the two coroutines interleave at all — it+ stands in for the network round trip each of these calls really makes.+ """++ def __init__(self, adapter: SlackAdapter, seeded_ref: str) -> None:+ self.live: set[str] = {seeded_ref}+ self.edits: list[tuple[str, str]] = []+ self._next = iter(f"msg-{n}" for n in range(2, 20))++ async def send_message(+ channel_id: str,+ sender_name: str,+ content: str,+ thread_root_id: str | None = None,+ ) -> str | None:+ await asyncio.sleep(0)+ ref = next(self._next)+ self.live.add(ref)+ return ref++ async def update_message(+ channel_id: str, message_ref: str, new_content: str+ ) -> None:+ await asyncio.sleep(0)+ self.edits.append((message_ref, new_content))++ async def delete_message(channel_id: str, message_ref: str) -> None:+ await asyncio.sleep(0)+ self.live.discard(message_ref)++ adapter.send_message = send_message # type: ignore[method-assign]+ adapter.update_message = update_message # type: ignore[method-assign]+ adapter.delete_message = delete_message # type: ignore[method-assign]+++def _adapter() -> tuple[SlackAdapter, _Platform]:+ adapter = SlackAdapter(+ config=SlackConnectionConfig(+ bot_token="xoxb-test", app_token="xapp-test", workspace_id="T-test"+ )+ )+ adapter._working_msg[KEY] = LiveRuntimeIndicator(+ message_ref="msg-1",+ body="⚙️ _Working on it…_",+ thread_root_id=None,+ )+ return adapter, _Platform(adapter, "msg-1")+++def _refresh(adapter: SlackAdapter, detail: str) -> Any:+ return adapter.apply_runtime_state(+ CHANNEL,+ AGENT,+ "working",+ notify_user=None,+ thread_root_id=None,+ detail=detail,+ )+++def _assert_consistent(adapter: SlackAdapter, platform: _Platform) -> None:+ live = adapter._working_msg.get(KEY)+ tracked = {live.message_ref} if live is not None else set()+ assert platform.live == tracked, (+ f"platform still shows {sorted(platform.live)} but the adapter tracks "+ f"{sorted(tracked)} — the difference is stranded and nothing will remove it"+ )+++async def test_a_refresh_landing_during_a_move_does_not_strand_the_new_message() -> (+ None+):+ # The reported bug: the refresh read the entry before the move rewrote it,+ # so its write restored the old ref and orphaned the message the move+ # posted. The turn's clear then removed the already-deleted one.+ adapter, platform = _adapter()++ await asyncio.gather(+ adapter.reposition_runtime_state(CHANNEL, AGENT, None),+ _refresh(adapter, "Ran tool post_message"),+ )++ _assert_consistent(adapter, platform)+++async def test_a_move_still_moves_when_a_refresh_races_it() -> None:+ # The same interleaving in the other order silently abandons the move: the+ # indicator stays where it was, which is the whole defect this feature+ # exists to fix.+ adapter, platform = _adapter()++ await asyncio.gather(+ _refresh(adapter, "Ran tool post_message"),+ adapter.reposition_runtime_state(CHANNEL, AGENT, None),+ )++ live = adapter._working_msg[KEY]+ assert live.message_ref != "msg-1", (+ "the indicator was never repositioned — a concurrent activity refresh "+ "cancelled the move"+ )+ _assert_consistent(adapter, platform)+++async def test_the_refresh_body_survives_a_concurrent_move() -> None:+ # Whichever order they run in, the latest activity line must end up on the+ # message that is actually still posted — not on a deleted one.+ adapter, platform = _adapter()++ await asyncio.gather(+ adapter.reposition_runtime_state(CHANNEL, AGENT, None),+ _refresh(adapter, "Editing foo.py"),+ )++ live = adapter._working_msg[KEY]+ assert "Editing foo.py" in live.body+ assert live.message_ref in platform.live+++async def test_a_burst_of_moves_and_refreshes_leaves_exactly_one_message() -> None:+ # The steady state under load: several repositions and refreshes overlapping+ # must still converge on a single tracked, still-posted indicator.+ adapter, platform = _adapter()++ await asyncio.gather(+ *(adapter.reposition_runtime_state(CHANNEL, AGENT, None) for _ in range(4)),+ *(_refresh(adapter, f"step {n}") for n in range(4)),+ )++ assert len(platform.live) == 1+ _assert_consistent(adapter, platform)+++async def test_the_turn_end_clear_removes_everything() -> None:+ # After a contended turn, going idle must leave the channel clean.+ adapter, platform = _adapter()++ await asyncio.gather(+ adapter.reposition_runtime_state(CHANNEL, AGENT, None),+ _refresh(adapter, "Ran tool post_message"),+ )+ await adapter.apply_runtime_state(+ CHANNEL,+ AGENT,+ "idle",+ notify_user=None,+ thread_root_id=None,+ )++ assert platform.live == set()+ assert KEY not in adapter._working_msgdiff --git a/core/tests/switch_core/bridges/collaboration/test_slack_adapter.py b/core/tests/switch_core/bridges/collaboration/test_slack_adapter.pyindex eb096f2c2..1e2c189b8 100644--- a/core/tests/switch_core/bridges/collaboration/test_slack_adapter.py+++ b/core/tests/switch_core/bridges/collaboration/test_slack_adapter.py@@ -50,6 +50,9 @@ class _FakeWebClient:self.calls: list[dict[str, Any]] = []self.deletes: list[dict[str, Any]] = []self.updates: list[dict[str, Any]] = []+ # ts values actually handed back, in order — lets a test assert that+ # everything posted was also removed.+ self.issued: list[str] = []self._next_ts = iter(["999.9", "888.8", "777.7"])async def chat_postMessage(self, **kwargs: Any) -> dict[str, str]:@@ -58,6 +61,7 @@ class _FakeWebClient:ts = next(self._next_ts)except StopIteration:ts = "000.0"+ self.issued.append(ts)return {"ts": ts}async def chat_delete(self, **kwargs: Any) -> dict[str, bool]:@@ -771,35 +775,30 @@ def test_reposition_is_a_noop_without_a_live_indicator() -> None:assert fake.deletes == []-def test_a_turn_ending_mid_move_does_not_strand_the_replacement() -> None:- # The turn can go idle while the replacement post is still in flight. The- # clear removes the message this move was replacing, so the replacement- # would be left with nothing to ever clear it — it must be taken back down.+def test_a_turn_ending_during_a_move_clears_what_the_move_left() -> None:+ # A move and the end-of-turn clear cannot interleave: both run under the+ # agent's runtime lock, so the clear sees the moved indicator rather than+ # the message the move already deleted. Nothing is left in the channel.adapter = _adapter()fake = _FakeWebClient()adapter._web_client = fake # type: ignore[assignment]- _run(- adapter.apply_runtime_state(+ async def scenario() -> None:+ await adapter.apply_runtime_state("C123", "agent-bot", "working", notify_user=None, thread_root_id=None)- )-- real_send = adapter.send_message-- async def send_then_go_idle(*args: Any, **kwargs: Any) -> str | None:- ref = await real_send(*args, **kwargs)- await adapter.apply_runtime_state(- "C123", "agent-bot", "idle", notify_user=None, thread_root_id=None+ await asyncio.gather(+ adapter.reposition_runtime_state("C123", "agent-bot", None),+ adapter.apply_runtime_state(+ "C123", "agent-bot", "idle", notify_user=None, thread_root_id=None+ ),)- return ref- adapter.send_message = send_then_go_idle # type: ignore[method-assign]- _run(adapter.reposition_runtime_state("C123", "agent-bot", None))+ _run(scenario())- # Both the original and the orphaned replacement are gone, and nothing is- # left tracked for a turn that has ended.- assert {d["ts"] for d in fake.deletes} == {"999.9", "888.8"}+ # Every message the adapter posted was deleted again — whichever order the+ # two ran in, the channel ends up empty and nothing is left tracked.+ assert set(fake.issued) == {d["ts"] for d in fake.deletes}assert ("C123", "agent-bot") not in adapter._working_msgdiff --git a/core/uv.lock b/core/uv.lockindex 62611169c..5d720864b 100644--- a/core/uv.lock+++ b/core/uv.lock@@ -2433,7 +2433,7 @@ wheels = [[[package]]name = "switch-core"-version = "0.12.0"+version = "0.12.1"source = { editable = "." }dependencies = [{ name = "aiohttp" },